mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-27 01:21:18 +00:00
2a0c97c178
* fix: add platform-aware semantic fallback Make VECTOR an optional capability so Windows analysis remains stable while semantic embeddings can fall back to exact scan when native vector indexing is unavailable. Made-with: Cursor * fix: remove stale vector pool import Keep the merge with main lint-clean after VECTOR loading moved out of the read pool.
483 lines
17 KiB
TypeScript
483 lines
17 KiB
TypeScript
/**
|
|
* Analyze Command
|
|
*
|
|
* Indexes a repository and stores the knowledge graph in .gitnexus/
|
|
*
|
|
* Delegates core analysis to the shared runFullAnalysis orchestrator.
|
|
* This CLI wrapper handles: heap management, progress bar, SIGINT,
|
|
* skill generation (--skills), summary output, and process.exit().
|
|
*/
|
|
|
|
import path from 'path';
|
|
import { execFileSync } from 'child_process';
|
|
import v8 from 'v8';
|
|
import cliProgress from 'cli-progress';
|
|
import { closeLbug } from '../core/lbug/lbug-adapter.js';
|
|
import {
|
|
getStoragePaths,
|
|
getGlobalRegistryPath,
|
|
RegistryNameCollisionError,
|
|
} from '../storage/repo-manager.js';
|
|
import { getGitRoot, hasGitDir } from '../storage/git.js';
|
|
import { runFullAnalysis } from '../core/run-analyze.js';
|
|
import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js';
|
|
import fs from 'fs/promises';
|
|
|
|
const HEAP_MB = 8192;
|
|
const HEAP_FLAG = `--max-old-space-size=${HEAP_MB}`;
|
|
/** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */
|
|
const STACK_KB = 4096;
|
|
const STACK_FLAG = `--stack-size=${STACK_KB}`;
|
|
|
|
/** Re-exec the process with an 8GB heap and larger stack if we're currently below that. */
|
|
function ensureHeap(): boolean {
|
|
const nodeOpts = process.env.NODE_OPTIONS || '';
|
|
if (nodeOpts.includes('--max-old-space-size')) return false;
|
|
|
|
const v8Heap = v8.getHeapStatistics().heap_size_limit;
|
|
if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false;
|
|
|
|
// --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+,
|
|
// so pass it only as a direct CLI argument, not via the environment.
|
|
const cliFlags = [HEAP_FLAG];
|
|
if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG);
|
|
|
|
try {
|
|
execFileSync(process.execPath, [...cliFlags, ...process.argv.slice(1)], {
|
|
stdio: 'inherit',
|
|
env: { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim() },
|
|
});
|
|
} catch (e: any) {
|
|
process.exitCode = e.status ?? 1;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
export interface AnalyzeOptions {
|
|
force?: boolean;
|
|
embeddings?: boolean;
|
|
/**
|
|
* Explicitly drop existing embeddings on rebuild instead of preserving
|
|
* them. Without this flag, a routine `analyze` keeps any embeddings
|
|
* already present in the index even when `--embeddings` is omitted.
|
|
*/
|
|
dropEmbeddings?: boolean;
|
|
skills?: boolean;
|
|
verbose?: boolean;
|
|
/** Skip AGENTS.md and CLAUDE.md gitnexus block updates. */
|
|
skipAgentsMd?: boolean;
|
|
/** Omit volatile symbol/relationship counts from AGENTS.md and CLAUDE.md. */
|
|
noStats?: boolean;
|
|
/** Index the folder even when no .git directory is present. */
|
|
skipGit?: boolean;
|
|
/**
|
|
* Override the default basename-derived registry `name` with a
|
|
* user-supplied alias (#829). Disambiguates repos whose paths share a
|
|
* basename. Persisted — subsequent re-analyses of the same path without
|
|
* `--name` preserve the alias.
|
|
*/
|
|
name?: string;
|
|
/**
|
|
* Allow registration even when another path already uses the same
|
|
* `--name` alias (#829). Intentionally a distinct flag from `--force`
|
|
* because the user may want to coexist under the same name WITHOUT
|
|
* paying the cost of a pipeline re-index. Maps to registerRepo's
|
|
* `allowDuplicateName` option end-to-end.
|
|
*/
|
|
allowDuplicateName?: boolean;
|
|
/**
|
|
* Override the walker's large-file skip threshold (#991). Value in KB;
|
|
* clamped downstream to the tree-sitter 32 MB ceiling. Sets
|
|
* `GITNEXUS_MAX_FILE_SIZE` for the rest of the pipeline.
|
|
*/
|
|
maxFileSize?: string;
|
|
/** Override worker sub-batch idle timeout in seconds. */
|
|
workerTimeout?: string;
|
|
embeddingThreads?: string;
|
|
embeddingBatchSize?: string;
|
|
embeddingSubBatchSize?: string;
|
|
embeddingDevice?: string;
|
|
}
|
|
|
|
export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOptions) => {
|
|
if (ensureHeap()) return;
|
|
|
|
if (options?.verbose) {
|
|
process.env.GITNEXUS_VERBOSE = '1';
|
|
}
|
|
|
|
if (options?.maxFileSize) {
|
|
process.env.GITNEXUS_MAX_FILE_SIZE = options.maxFileSize;
|
|
}
|
|
|
|
if (options?.workerTimeout) {
|
|
const workerTimeoutSeconds = Number(options.workerTimeout);
|
|
if (!Number.isFinite(workerTimeoutSeconds) || workerTimeoutSeconds < 1) {
|
|
console.error(' --worker-timeout must be at least 1 second.\n');
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
process.env.GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS = String(
|
|
Math.round(workerTimeoutSeconds * 1000),
|
|
);
|
|
}
|
|
|
|
const setPositiveEnv = (
|
|
optionName: string,
|
|
envName: string,
|
|
value: string | undefined,
|
|
): boolean => {
|
|
if (value === undefined) return true;
|
|
const parsed = Number(value);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
console.error(` ${optionName} must be a positive integer.\n`);
|
|
process.exitCode = 1;
|
|
return false;
|
|
}
|
|
process.env[envName] = String(parsed);
|
|
return true;
|
|
};
|
|
|
|
if (
|
|
!setPositiveEnv(
|
|
'--embedding-threads',
|
|
'GITNEXUS_EMBEDDING_THREADS',
|
|
options?.embeddingThreads,
|
|
) ||
|
|
!setPositiveEnv(
|
|
'--embedding-batch-size',
|
|
'GITNEXUS_EMBEDDING_BATCH_SIZE',
|
|
options?.embeddingBatchSize,
|
|
) ||
|
|
!setPositiveEnv(
|
|
'--embedding-sub-batch-size',
|
|
'GITNEXUS_EMBEDDING_SUB_BATCH_SIZE',
|
|
options?.embeddingSubBatchSize,
|
|
)
|
|
) {
|
|
return;
|
|
}
|
|
|
|
if (options?.embeddingDevice) {
|
|
const allowed = new Set(['auto', 'cpu', 'dml', 'cuda', 'wasm']);
|
|
if (!allowed.has(options.embeddingDevice)) {
|
|
console.error(' --embedding-device must be one of: auto, cpu, dml, cuda, wasm.\n');
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
process.env.GITNEXUS_EMBEDDING_DEVICE = options.embeddingDevice;
|
|
}
|
|
|
|
console.log('\n GitNexus Analyzer\n');
|
|
|
|
let repoPath: string;
|
|
if (inputPath) {
|
|
repoPath = path.resolve(inputPath);
|
|
} else {
|
|
const gitRoot = getGitRoot(process.cwd());
|
|
if (!gitRoot) {
|
|
if (!options?.skipGit) {
|
|
console.log(
|
|
' Not inside a git repository.\n Tip: pass --skip-git to index any folder without a .git directory.\n',
|
|
);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
// --skip-git: fall back to cwd as the root
|
|
repoPath = path.resolve(process.cwd());
|
|
} else {
|
|
repoPath = gitRoot;
|
|
}
|
|
}
|
|
|
|
const repoHasGit = hasGitDir(repoPath);
|
|
if (!repoHasGit && !options?.skipGit) {
|
|
console.log(
|
|
' Not a git repository.\n Tip: pass --skip-git to index any folder without a .git directory.\n',
|
|
);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
if (!repoHasGit) {
|
|
console.log(
|
|
' Warning: no .git directory found \u2014 commit-tracking and incremental updates disabled.\n',
|
|
);
|
|
}
|
|
|
|
// KuzuDB migration cleanup is handled by runFullAnalysis internally.
|
|
// Note: --skills is handled after runFullAnalysis using the returned pipelineResult.
|
|
|
|
if (process.env.GITNEXUS_NO_GITIGNORE) {
|
|
console.log(
|
|
' GITNEXUS_NO_GITIGNORE is set — skipping .gitignore (still reading .gitnexusignore)\n',
|
|
);
|
|
}
|
|
|
|
const maxFileSizeBanner = getMaxFileSizeBannerMessage();
|
|
if (maxFileSizeBanner) {
|
|
console.log(`${maxFileSizeBanner}\n`);
|
|
}
|
|
|
|
// ── CLI progress bar setup ─────────────────────────────────────────
|
|
const bar = new cliProgress.SingleBar(
|
|
{
|
|
format: ' {bar} {percentage}% | {phase}',
|
|
barCompleteChar: '\u2588',
|
|
barIncompleteChar: '\u2591',
|
|
hideCursor: true,
|
|
barGlue: '',
|
|
autopadding: true,
|
|
clearOnComplete: false,
|
|
stopOnComplete: false,
|
|
},
|
|
cliProgress.Presets.shades_grey,
|
|
);
|
|
|
|
bar.start(100, 0, { phase: 'Initializing...' });
|
|
|
|
// Graceful SIGINT handling
|
|
let aborted = false;
|
|
const sigintHandler = () => {
|
|
if (aborted) process.exit(1);
|
|
aborted = true;
|
|
bar.stop();
|
|
console.log('\n Interrupted — cleaning up...');
|
|
closeLbug()
|
|
.catch(() => {})
|
|
.finally(() => process.exit(130));
|
|
};
|
|
process.on('SIGINT', sigintHandler);
|
|
|
|
// Route console output through bar.log() to prevent progress bar corruption
|
|
const origLog = console.log.bind(console);
|
|
const origWarn = console.warn.bind(console);
|
|
const origError = console.error.bind(console);
|
|
let barCurrentValue = 0;
|
|
const barLog = (...args: any[]) => {
|
|
process.stdout.write('\x1b[2K\r');
|
|
origLog(args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
|
|
bar.update(barCurrentValue);
|
|
};
|
|
console.log = barLog;
|
|
console.warn = barLog;
|
|
console.error = barLog;
|
|
|
|
// Track elapsed time per phase
|
|
let lastPhaseLabel = 'Initializing...';
|
|
let phaseStart = Date.now();
|
|
|
|
const updateBar = (value: number, phaseLabel: string) => {
|
|
barCurrentValue = value;
|
|
if (phaseLabel !== lastPhaseLabel) {
|
|
lastPhaseLabel = phaseLabel;
|
|
phaseStart = Date.now();
|
|
}
|
|
const elapsed = Math.round((Date.now() - phaseStart) / 1000);
|
|
const display = elapsed >= 3 ? `${phaseLabel} (${elapsed}s)` : phaseLabel;
|
|
bar.update(value, { phase: display });
|
|
};
|
|
|
|
const elapsedTimer = setInterval(() => {
|
|
const elapsed = Math.round((Date.now() - phaseStart) / 1000);
|
|
if (elapsed >= 3) {
|
|
bar.update({ phase: `${lastPhaseLabel} (${elapsed}s)` });
|
|
}
|
|
}, 1000);
|
|
|
|
const t0 = Date.now();
|
|
|
|
// ── Run shared analysis orchestrator ───────────────────────────────
|
|
try {
|
|
const result = await runFullAnalysis(
|
|
repoPath,
|
|
{
|
|
// Pipeline re-index — OR'd with --skills because skill generation
|
|
// needs a fresh pipelineResult. Has no bearing on the registry
|
|
// collision guard (see allowDuplicateName below).
|
|
force: options?.force || options?.skills,
|
|
embeddings: options?.embeddings,
|
|
dropEmbeddings: options?.dropEmbeddings,
|
|
skipGit: options?.skipGit,
|
|
skipAgentsMd: options?.skipAgentsMd,
|
|
noStats: options?.noStats,
|
|
registryName: options?.name,
|
|
// Registry-collision bypass — its own CLI flag, intentionally NOT
|
|
// overloading --force. A user who hits the collision guard should
|
|
// be able to accept the duplicate name without also paying the
|
|
// cost of a full pipeline re-index. See #829 review round 2.
|
|
allowDuplicateName: options?.allowDuplicateName,
|
|
},
|
|
{
|
|
onProgress: (_phase, percent, message) => {
|
|
updateBar(percent, message);
|
|
},
|
|
onLog: barLog,
|
|
},
|
|
);
|
|
|
|
if (result.alreadyUpToDate) {
|
|
clearInterval(elapsedTimer);
|
|
process.removeListener('SIGINT', sigintHandler);
|
|
console.log = origLog;
|
|
console.warn = origWarn;
|
|
console.error = origError;
|
|
bar.stop();
|
|
console.log(' Already up to date\n');
|
|
// Safe to return without process.exit(0) — the early-return path in
|
|
// runFullAnalysis never opens LadybugDB, so no native handles prevent exit.
|
|
return;
|
|
}
|
|
|
|
// Skill generation (CLI-only, uses pipeline result from analysis)
|
|
if (options?.skills && result.pipelineResult) {
|
|
updateBar(99, 'Generating skill files...');
|
|
try {
|
|
const { generateSkillFiles } = await import('./skill-gen.js');
|
|
const { generateAIContextFiles } = await import('./ai-context.js');
|
|
const skillResult = await generateSkillFiles(
|
|
repoPath,
|
|
result.repoName,
|
|
result.pipelineResult,
|
|
);
|
|
if (skillResult.skills.length > 0) {
|
|
barLog(` Generated ${skillResult.skills.length} skill files`);
|
|
// Re-generate AI context files now that we have skill info
|
|
const s = result.stats;
|
|
const communityResult = result.pipelineResult?.communityResult;
|
|
let aggregatedClusterCount = 0;
|
|
if (communityResult?.communities) {
|
|
const groups = new Map<string, number>();
|
|
for (const c of communityResult.communities) {
|
|
const label = c.heuristicLabel || c.label || 'Unknown';
|
|
groups.set(label, (groups.get(label) || 0) + c.symbolCount);
|
|
}
|
|
aggregatedClusterCount = Array.from(groups.values()).filter(
|
|
(count: number) => count >= 5,
|
|
).length;
|
|
}
|
|
const { storagePath: sp } = getStoragePaths(repoPath);
|
|
await generateAIContextFiles(
|
|
repoPath,
|
|
sp,
|
|
result.repoName,
|
|
{
|
|
files: s.files ?? 0,
|
|
nodes: s.nodes ?? 0,
|
|
edges: s.edges ?? 0,
|
|
communities: s.communities,
|
|
clusters: aggregatedClusterCount,
|
|
processes: s.processes,
|
|
},
|
|
skillResult.skills,
|
|
{ skipAgentsMd: options?.skipAgentsMd, noStats: options?.noStats },
|
|
);
|
|
}
|
|
} catch {
|
|
/* best-effort */
|
|
}
|
|
}
|
|
|
|
const totalTime = ((Date.now() - t0) / 1000).toFixed(1);
|
|
|
|
clearInterval(elapsedTimer);
|
|
process.removeListener('SIGINT', sigintHandler);
|
|
|
|
console.log = origLog;
|
|
console.warn = origWarn;
|
|
console.error = origError;
|
|
|
|
bar.update(100, { phase: 'Done' });
|
|
bar.stop();
|
|
|
|
// ── Summary ────────────────────────────────────────────────────
|
|
const s = result.stats;
|
|
console.log(`\n Repository indexed successfully (${totalTime}s)\n`);
|
|
console.log(
|
|
` ${(s.nodes ?? 0).toLocaleString()} nodes | ${(s.edges ?? 0).toLocaleString()} edges | ${s.communities ?? 0} clusters | ${s.processes ?? 0} flows`,
|
|
);
|
|
console.log(` ${repoPath}`);
|
|
|
|
try {
|
|
await fs.access(getGlobalRegistryPath());
|
|
} catch {
|
|
console.log('\n Tip: Run `gitnexus setup` to configure MCP for your editor.');
|
|
}
|
|
|
|
console.log('');
|
|
} catch (err: any) {
|
|
clearInterval(elapsedTimer);
|
|
process.removeListener('SIGINT', sigintHandler);
|
|
console.log = origLog;
|
|
console.warn = origWarn;
|
|
console.error = origError;
|
|
bar.stop();
|
|
|
|
const msg = err.message || String(err);
|
|
|
|
// Registry name-collision from --name (#829) — surface as an
|
|
// actionable error rather than a generic stack-trace.
|
|
if (err instanceof RegistryNameCollisionError) {
|
|
console.error(`\n Registry name collision:\n`);
|
|
console.error(` "${err.registryName}" is already used by "${err.existingPath}".\n`);
|
|
console.error(` Options:`);
|
|
console.error(` • Pick a different alias: gitnexus analyze --name <alias>`);
|
|
console.error(
|
|
` • Allow the duplicate: gitnexus analyze --allow-duplicate-name (leaves "-r ${err.registryName}" ambiguous)`,
|
|
);
|
|
console.error('');
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
console.error(`\n Analysis failed: ${msg}\n`);
|
|
|
|
// Provide helpful guidance for known failure modes
|
|
if (
|
|
msg.includes('Maximum call stack size exceeded') ||
|
|
msg.includes('call stack') ||
|
|
msg.includes('Map maximum size') ||
|
|
msg.includes('Invalid array length') ||
|
|
msg.includes('Invalid string length') ||
|
|
msg.includes('allocation failed') ||
|
|
msg.includes('heap out of memory') ||
|
|
msg.includes('JavaScript heap')
|
|
) {
|
|
console.error(' This error typically occurs on very large repositories.');
|
|
console.error(' Suggestions:');
|
|
console.error(' 1. Add large vendored/generated directories to .gitnexusignore');
|
|
console.error(' 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"');
|
|
console.error(' 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"');
|
|
console.error('');
|
|
} else if (msg.includes('ERESOLVE') || msg.includes('Could not resolve dependency')) {
|
|
// Note: the original arborist "Cannot destructure property 'package' of
|
|
// 'node.target'" crash happens inside npm *before* gitnexus code runs,
|
|
// so it can't be caught here. This branch handles dependency-resolution
|
|
// errors that surface at runtime (e.g. dynamic require failures).
|
|
console.error(' This looks like an npm dependency resolution issue.');
|
|
console.error(' Suggestions:');
|
|
console.error(' 1. Clear the npm cache: npm cache clean --force');
|
|
console.error(' 2. Update npm: npm install -g npm@latest');
|
|
console.error(' 3. Reinstall gitnexus: npm install -g gitnexus@latest');
|
|
console.error(' 4. Or try npx directly: npx gitnexus@latest analyze');
|
|
console.error('');
|
|
} else if (
|
|
msg.includes('MODULE_NOT_FOUND') ||
|
|
msg.includes('Cannot find module') ||
|
|
msg.includes('ERR_MODULE_NOT_FOUND')
|
|
) {
|
|
console.error(' A required module could not be loaded. The installation may be corrupt.');
|
|
console.error(' Suggestions:');
|
|
console.error(' 1. Reinstall: npm install -g gitnexus@latest');
|
|
console.error(' 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze');
|
|
console.error('');
|
|
}
|
|
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
// LadybugDB's native module holds open handles that prevent Node from exiting.
|
|
// ONNX Runtime also registers native atexit hooks that segfault on some
|
|
// platforms (#38, #40). Force-exit to ensure clean termination.
|
|
process.exit(0);
|
|
};
|