diff --git a/gitnexus/src/core/incremental/file-hash.ts b/gitnexus/src/core/incremental/file-hash.ts new file mode 100644 index 000000000..673e655af --- /dev/null +++ b/gitnexus/src/core/incremental/file-hash.ts @@ -0,0 +1,61 @@ +/** + * File-content hashing — v1 "surface signature" for incremental indexing. + * + * v1 trade-off: we use SHA-256 of file content as the surface signature. + * This is conservative — a body-only edit (which doesn't actually change + * any other file's resolution) still triggers 1-hop closure expansion + * because the content hash differs. + * + * v2 will switch to a real surface-only signature (extracted from the + * post-parse graph via `extractSurfaceSignature` in `surface.ts`) so that + * body-only edits stay at closure size 1. The plumbing for that is in + * place — `surface.ts` and the closure module are signature-agnostic — + * but reusing the parse-worker output for one-off surface extraction + * requires more pipeline integration than is needed for v1. + */ + +import { createHash } from 'crypto'; +import fs from 'fs/promises'; +import path from 'path'; + +/** + * Compute SHA-256 hex digest of a single file. Returns null when the + * file can't be read (deleted between scan and hash, permission error, + * etc.) — caller should treat null as "no signature available, assume + * changed". + */ +export async function computeFileHash(absPath: string): Promise { + try { + const buf = await fs.readFile(absPath); + return createHash('sha256').update(buf).digest('hex'); + } catch { + return null; + } +} + +/** + * Compute SHA-256 hashes for a list of files in `repoPath` (paths are + * repo-relative). Parallel batched I/O bounded at 100 concurrent reads + * to avoid fd exhaustion on huge repos. + * + * Returns a Map. Files that fail to read are omitted + * from the result (caller treats them as "no signature"). + */ +export async function computeFileHashes( + repoPath: string, + relPaths: readonly string[], +): Promise> { + const out = new Map(); + const BATCH = 100; + for (let i = 0; i < relPaths.length; i += BATCH) { + const batch = relPaths.slice(i, i + BATCH); + const results = await Promise.all( + batch.map(async (rel) => { + const hash = await computeFileHash(path.join(repoPath, rel)); + return hash ? ([rel, hash] as const) : null; + }), + ); + for (const r of results) if (r) out.set(r[0], r[1]); + } + return out; +} diff --git a/gitnexus/src/core/incremental/orchestrator.ts b/gitnexus/src/core/incremental/orchestrator.ts new file mode 100644 index 000000000..69324da3a --- /dev/null +++ b/gitnexus/src/core/incremental/orchestrator.ts @@ -0,0 +1,267 @@ +/** + * Incremental-indexing orchestrator helpers. + * + * High-level helpers used by `run-analyze.ts` to keep the incremental path + * out of the main full-rebuild orchestrator function. Each helper has one + * responsibility: + * + * isIncrementalEligible(...) — should this run go incremental? + * computeIncrementalClosure(...) — git diff → content-hash → closure + * extractIncrementalSubgraph(...) — ctx.graph → "nodes/edges to write" + * commitIncrementalProgress(...) — set the dirty flag in meta.json + * + * See docs/superpowers/specs/2026-05-10-incremental-indexing-design.md. + */ + +import { execFileSync } from 'child_process'; +import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; +import { createKnowledgeGraph } from '../graph/graph.js'; +import type { KnowledgeGraph } from '../graph/types.js'; +import { + type RepoMeta, + INCREMENTAL_SCHEMA_VERSION, + saveMeta, +} from '../../storage/repo-manager.js'; +import { hasGitDir } from '../../storage/git.js'; +import { + getChangedFilesSinceCommit, + type ChangedFiles, +} from './git-diff.js'; +import { computeImporterClosure } from './closure.js'; +import { computeFileHashes } from './file-hash.js'; +import { queryImporters } from '../lbug/lbug-adapter.js'; + +/** Decision returned by isIncrementalEligible. */ +export interface IncrementalEligibility { + /** True iff this run should attempt the incremental path. */ + eligible: boolean; + /** When `eligible === false`, a short reason for logging. */ + reason?: string; + /** The previous lastCommit, when eligible. */ + lastCommit?: string; +} + +/** + * Decide whether a run is eligible for the incremental path. + * + * All conditions must hold: + * - --force not passed + * - existing meta.json present and previously a full rebuild has populated + * surfaceSignatures + schemaVersion (matching CURRENT) + * - repo has .git (non-git repos always do full rebuild) + * - meta.lastCommit still resolvable (not rebased away) + * - no incrementalInProgress flag (would force full rebuild for safety) + */ +export function isIncrementalEligible( + repoPath: string, + existingMeta: RepoMeta | null | undefined, + optionsForce: boolean | undefined, +): IncrementalEligibility { + if (optionsForce) { + return { eligible: false, reason: '--force passed' }; + } + if (!existingMeta) { + return { eligible: false, reason: 'no existing index' }; + } + if (existingMeta.incrementalInProgress) { + return { + eligible: false, + reason: 'previous incremental run did not complete cleanly', + }; + } + if ( + existingMeta.schemaVersion === undefined || + existingMeta.schemaVersion !== INCREMENTAL_SCHEMA_VERSION + ) { + return { + eligible: false, + reason: `schemaVersion mismatch (have ${existingMeta.schemaVersion}, want ${INCREMENTAL_SCHEMA_VERSION})`, + }; + } + if (!existingMeta.surfaceSignatures || Object.keys(existingMeta.surfaceSignatures).length === 0) { + return { + eligible: false, + reason: 'no prior surfaceSignatures in meta.json', + }; + } + if (!hasGitDir(repoPath)) { + return { eligible: false, reason: 'non-git repo' }; + } + if (!existingMeta.lastCommit) { + return { eligible: false, reason: 'no lastCommit recorded' }; + } + if (!commitExists(repoPath, existingMeta.lastCommit)) { + return { + eligible: false, + reason: `lastCommit ${existingMeta.lastCommit.slice(0, 7)} not in repo`, + }; + } + return { eligible: true, lastCommit: existingMeta.lastCommit }; +} + +function commitExists(repoPath: string, commit: string): boolean { + try { + execFileSync('git', ['cat-file', '-e', `${commit}^{commit}`], { + cwd: repoPath, + stdio: ['ignore', 'ignore', 'ignore'], + }); + return true; + } catch { + return false; + } +} + +export interface IncrementalSetupResult { + /** Files that must be re-parsed in this run. */ + closure: Set; + /** Files deleted on disk since lastCommit (rows must be removed from DB). */ + deletedFiles: string[]; + /** Per-file content hashes computed during closure expansion. */ + newFileHashes: Map; + /** ChangedFiles result for diagnostics. */ + changes: ChangedFiles; +} + +/** + * Compute the closure of files to re-parse for an incremental run. + * + * Algorithm: + * 1. git diff lastCommit HEAD ∪ git status → ChangedFiles + * 2. closure ← modified ∪ added + * 3. For each f in closure: hash(f); if hash differs from previous, query + * DB importers and add them to closure. Iterate to fixpoint. + * + * Throws if `lastCommit` is gone (caller should fall back to full rebuild). + */ +export async function computeIncrementalClosure( + repoPath: string, + lastCommit: string, + prevSurfaces: Record, +): Promise { + const changes = getChangedFilesSinceCommit(repoPath, lastCommit); + + const initialChangedFiles = new Set([...changes.modified, ...changes.added]); + + // Closure module wants generic parseFile + surfaceFor. v1 uses + // file-content hashes (cheap, conservative). The "ParseResult" type + // is just the content hash itself. + const { closure, newSurfaces } = await computeImporterClosure({ + initialChangedFiles, + prevSurfaces, + parseFile: async (filePath) => { + const hashes = await computeFileHashes(repoPath, [filePath]); + // Missing files (race with delete) → empty hash; counts as "changed". + return hashes.get(filePath) ?? ''; + }, + surfaceFor: (_filePath, parsed) => parsed, + queryImporters: async (filePath) => queryImporters(filePath), + }); + + return { + closure, + deletedFiles: changes.deleted, + newFileHashes: newSurfaces, + changes, + }; +} + +/** + * Persist the `incrementalInProgress` dirty flag to meta.json BEFORE any + * destructive DB mutation. The flag is cleared on success by overwriting + * meta.json with the final state. If the run crashes between, the next + * run sees the flag and forces a full rebuild. + */ +export async function commitIncrementalProgress( + storagePath: string, + existingMeta: RepoMeta, + closure: Set, +): Promise { + const meta: RepoMeta = { + ...existingMeta, + incrementalInProgress: { + closure: [...closure], + startedAt: Date.now(), + }, + }; + await saveMeta(storagePath, meta); +} + +/** + * Build a subgraph of `ctx.graph` containing ONLY the nodes/edges that + * need to be written to LadybugDB in incremental mode: + * + * - All nodes whose filePath is in `closure` (newly parsed in this run). + * - All graph-wide nodes (Community, Process) — they're regenerated by + * the communities/processes phases on every run. + * - All edges where AT LEAST ONE endpoint is in this set. Edges entirely + * between hydrated unchanged-file nodes are NOT included — they're + * already in the DB and re-inserting them would PK-conflict. + * + * This lets us call the existing `loadGraphToLbug` against the filtered + * subgraph: the COPY semantics will write only what's missing, while the + * unchanged-unchanged DB rows we never deleted stay intact. + */ +export function extractIncrementalSubgraph( + fullGraph: KnowledgeGraph, + closure: ReadonlySet, +): KnowledgeGraph { + const sub = createKnowledgeGraph(); + + const isGraphWide = (label: string): boolean => label === 'Community' || label === 'Process'; + + // Phase 1: nodes + const writableNodeIds = new Set(); + fullGraph.forEachNode((n: GraphNode) => { + const filePath = n.properties?.filePath as string | undefined; + const inClosure = filePath ? closure.has(filePath) : false; + if (inClosure || isGraphWide(n.label)) { + sub.addNode(n); + writableNodeIds.add(n.id); + } + }); + + // Phase 2: edges where AT LEAST ONE endpoint is in the writable set. + // Edges entirely between hydrated nodes are skipped (already in DB). + fullGraph.forEachRelationship((r: GraphRelationship) => { + if (writableNodeIds.has(r.sourceId) || writableNodeIds.has(r.targetId)) { + sub.addRelationship(r); + } + }); + + return sub; +} + +/** + * Compute the surface signatures for every file in the repo from a graph. + * In v1 this is just the per-file content hash (cheap, conservative). v2 + * will switch to true surface-only signatures derived via `surface.ts`. + * + * Used after a full rebuild to populate `meta.json.surfaceSignatures` so + * the next run can run incrementally. + */ +export async function computeAllFileSignatures( + repoPath: string, + filePaths: readonly string[], +): Promise> { + const map = await computeFileHashes(repoPath, filePaths); + const out: Record = {}; + for (const [k, v] of map) out[k] = v; + return out; +} + +/** + * Merge an updated set of file hashes into a previous snapshot. Used after + * an incremental run: `prevSurfaces` is the set in meta.json, `newSurfaces` + * is what we computed for closure files this run, and `deletedFiles` are + * files that no longer exist on disk. + */ +export function mergeSurfaceSignatures( + prevSurfaces: Record, + newSurfaces: Map, + deletedFiles: readonly string[], +): Record { + const merged: Record = { ...prevSurfaces }; + for (const f of deletedFiles) delete merged[f]; + for (const [f, h] of newSurfaces) merged[f] = h; + return merged; +} diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 7227fd945..b15394019 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -11,6 +11,7 @@ import path from 'path'; import fs from 'fs/promises'; +import { execFileSync } from 'child_process'; import { runPipelineFromRepo } from './ingestion/pipeline.js'; import { initLbug, @@ -20,6 +21,8 @@ import { executeWithReusedStatement, closeLbug, loadCachedEmbeddings, + deleteNodesForFile, + deleteAllCommunitiesAndProcesses, } from './lbug/lbug-adapter.js'; import { createSearchFTSIndexes } from './search/fts-indexes.js'; import { @@ -29,6 +32,7 @@ import { ensureGitNexusIgnored, registerRepo, cleanupOldKuzuFiles, + INCREMENTAL_SCHEMA_VERSION, } from '../storage/repo-manager.js'; import { getCurrentCommit, @@ -41,6 +45,14 @@ import type { CachedEmbedding } from './embeddings/types.js'; import { generateAIContextFiles } from '../cli/ai-context.js'; import { EMBEDDING_TABLE_NAME } from './lbug/schema.js'; import { STALE_HASH_SENTINEL } from './lbug/schema.js'; +import { + isIncrementalEligible, + computeIncrementalClosure, + commitIncrementalProgress, + extractIncrementalSubgraph, + mergeSurfaceSignatures, + computeAllFileSignatures, +} from './incremental/orchestrator.js'; // --------------------------------------------------------------------------- // Public types @@ -180,22 +192,67 @@ export async function runFullAnalysis( if (existingMeta && !options.force && existingMeta.lastCommit === currentCommit) { // Non-git folders have currentCommit = '' — always rebuild since we can't detect changes if (currentCommit !== '') { - await ensureGitNexusIgnored(repoPath); - return { - // `resolveRepoIdentityRoot` collapses worktree roots to the - // canonical repo basename (#1259) but leaves arbitrary subdirs - // and `--skip-git` paths unchanged (#1232/#1233 intent preserved). - repoName: - options.registryName ?? - getInferredRepoName(repoPath) ?? - path.basename(resolveRepoIdentityRoot(repoPath)), - repoPath, - stats: existingMeta.stats ?? {}, - alreadyUpToDate: true, - }; + // For git repos, even if HEAD matches lastCommit, the working tree + // may have uncommitted changes. Only short-circuit when the working + // tree is also clean. + const dirty = hasDirtyTree(repoPath); + if (!dirty) { + await ensureGitNexusIgnored(repoPath); + return { + // `resolveRepoIdentityRoot` collapses worktree roots to the + // canonical repo basename (#1259) but leaves arbitrary subdirs + // and `--skip-git` paths unchanged (#1232/#1233 intent preserved). + repoName: + options.registryName ?? + getInferredRepoName(repoPath) ?? + path.basename(resolveRepoIdentityRoot(repoPath)), + repoPath, + stats: existingMeta.stats ?? {}, + alreadyUpToDate: true, + }; + } } } + // ── Incremental branch ───────────────────────────────────────────── + // Try the incremental path first. Falls through to full rebuild on: + // - --force + // - missing/old meta.json + // - lastCommit gone (rebased) + // - schemaVersion mismatch + // - non-git repo + // - any error during incremental setup + const eligibility = isIncrementalEligible(repoPath, existingMeta, options.force); + if (eligibility.eligible && existingMeta) { + try { + const result = await runIncrementalBranch( + repoPath, + storagePath, + lbugPath, + existingMeta, + currentCommit, + options, + callbacks, + ); + if (result) return result; + // result === null → falls through to full rebuild (e.g. setup failed) + } catch (err) { + log( + `Incremental run failed (${(err as Error).message}); ` + + `next run will full-rebuild via dirty-flag.`, + ); + // Re-throw — the dirty flag is set; next run forces full rebuild. + try { + await closeLbug(); + } catch { + /* swallow */ + } + throw err; + } + } else if (existingMeta) { + log(`Incremental skipped: ${eligibility.reason ?? 'unknown reason'}`); + } + // ── Cache embeddings from existing index before rebuild ──────────── // Four modes: // --embeddings -> load cache, restore, then generate any new ones @@ -454,6 +511,32 @@ export async function runFullAnalysis( const effectiveSemanticMode = semanticMode ?? (runtimeCapabilities.semanticMode === 'vector-index' ? 'vector-index' : 'exact-scan'); + + // Compute per-file surface signatures so the next run can be incremental. + // v1 uses content-hash (cheap, conservative). v2 will switch to a + // surface-only signature so body-only edits don't expand the closure. + let surfaceSignatures: Record | undefined; + if (hasGitDir(repoPath)) { + try { + const allFilePaths: string[] = []; + pipelineResult.graph.forEachNode((n) => { + const fp = n.properties?.filePath as string | undefined; + if (fp && (n.label === 'File' || n.label === 'Folder')) { + // File nodes carry their own path; we want every distinct repo-relative + // file path that participated in indexing. Use File nodes as the + // authoritative source. + if (n.label === 'File') allFilePaths.push(fp); + } + }); + if (allFilePaths.length > 0) { + surfaceSignatures = await computeAllFileSignatures(repoPath, allFilePaths); + } + } catch { + /* surface signatures are best-effort; their absence just means the + * next run will fall back to full rebuild. */ + } + } + const meta = { repoPath, lastCommit: currentCommit, @@ -465,6 +548,14 @@ export async function runFullAnalysis( // origin remote, which is fine: paths-only repos behave as // before. remoteUrl: hasGitDir(repoPath) ? getRemoteUrl(repoPath) : undefined, + // Incremental-indexing fields — populated for git repos only. The + // next analyze run reads these to decide whether to take the + // incremental path. See the Incremental branch above. + schemaVersion: surfaceSignatures ? INCREMENTAL_SCHEMA_VERSION : undefined, + surfaceSignatures, + incrementalInProgress: undefined as + | { closure: string[]; startedAt: number } + | undefined, stats: { files: pipelineResult.totalFileCount, nodes: stats.nodes, @@ -554,3 +645,242 @@ export async function runFullAnalysis( throw err; } } + +// =========================================================================== +// Incremental analysis branch — invoked from runFullAnalysis when eligible. +// See docs/superpowers/specs/2026-05-10-incremental-indexing-design.md. +// =========================================================================== + +/** + * Cheap check: does the working tree have uncommitted changes? Used to + * decide whether the "lastCommit == HEAD" early-exit is safe to take. + */ +function hasDirtyTree(repoPath: string): boolean { + try { + const out = execFileSync('git', ['status', '--porcelain'], { + cwd: repoPath, + stdio: ['ignore', 'pipe', 'ignore'], + encoding: 'utf8', + }); + return out.trim().length > 0; + } catch { + // If git status fails for any reason, conservatively assume dirty so + // we don't accidentally short-circuit a real change. + return true; + } +} + +/** + * Execute the incremental branch of `runFullAnalysis`. Returns null when + * setup determines a full rebuild is needed instead (caller falls through). + * + * Throws if the run fails after the dirty flag is set — caller is + * responsible for surfacing the error; the next run will detect the dirty + * flag and force a full rebuild. + */ +async function runIncrementalBranch( + repoPath: string, + storagePath: string, + lbugPath: string, + existingMeta: import('../storage/repo-manager.js').RepoMeta, + currentCommit: string, + options: AnalyzeOptions, + callbacks: AnalyzeCallbacks, +): Promise { + const log = (msg: string) => callbacks.onLog?.(msg); + const progress = (phase: string, percent: number, message: string) => + callbacks.onProgress(phase, percent, message); + + log( + `Incremental: probing for changes since ${existingMeta.lastCommit.slice(0, 7)}...`, + ); + + // 1. Compute closure (parses each file's content hash, walks DB importers). + let setup: Awaited>; + try { + // Open the existing DB so closure expansion can query the IMPORTS edges. + await initLbug(lbugPath); + setup = await computeIncrementalClosure( + repoPath, + existingMeta.lastCommit, + existingMeta.surfaceSignatures ?? {}, + ); + } catch (e) { + try { + await closeLbug(); + } catch { + /* swallow */ + } + log( + `Incremental setup failed: ${(e as Error).message}. Falling back to full rebuild.`, + ); + return null; + } + + // 2. No changes? Update lastCommit and return early. + if (setup.closure.size === 0 && setup.deletedFiles.length === 0) { + log('Incremental: no file changes detected — refreshing meta only.'); + try { + await closeLbug(); + } catch { + /* swallow */ + } + const meta: import('../storage/repo-manager.js').RepoMeta = { + ...existingMeta, + lastCommit: currentCommit, + indexedAt: new Date().toISOString(), + }; + await saveMeta(storagePath, meta); + await ensureGitNexusIgnored(repoPath); + return { + repoName: + options.registryName ?? + getInferredRepoName(repoPath) ?? + path.basename(resolveRepoIdentityRoot(repoPath)), + repoPath, + stats: existingMeta.stats ?? {}, + alreadyUpToDate: true, + }; + } + + log( + `Incremental: closure=${setup.closure.size} (changed=${setup.changes.modified.length} ` + + `+ added=${setup.changes.added.length} + importers=${setup.closure.size - setup.changes.modified.length - setup.changes.added.length}), ` + + `deleted=${setup.deletedFiles.length}`, + ); + + // 3. Mark dirty BEFORE any DB mutation. Closes lbug temporarily to + // release the connection while saveMeta writes. + await commitIncrementalProgress(storagePath, existingMeta, setup.closure); + + // 4. Delete stale rows from the DB. + progress('lbug', 5, 'Removing stale rows for changed files...'); + for (const file of setup.closure) { + try { + await deleteNodesForFile(file); + } catch { + /* file may not have been indexed yet — fine */ + } + } + for (const file of setup.deletedFiles) { + try { + await deleteNodesForFile(file); + } catch { + /* fine */ + } + } + // Always wipe Community / Process — they're regenerated by downstream + // pipeline phases and must come from the merged graph for correctness + // (Leiden runs on the FULL hydrated + parsed graph). + await deleteAllCommunitiesAndProcesses(); + + // 5. Run the pipeline with filesToParse set so: + // - hydrate phase fills ctx.graph with unchanged-file nodes from DB + // - parse phase only parses closure files + // - downstream phases (mro, communities, processes) see the full graph + const pipelineResult = await runPipelineFromRepo( + repoPath, + (p) => { + const phaseLabel = PHASE_LABELS[p.phase] || p.phase; + const scaled = 10 + Math.round(p.percent * 0.55); // 10–65% + const message = p.detail ? `${p.message || phaseLabel} (${p.detail})` : p.message || phaseLabel; + progress(p.phase, scaled, message); + }, + { filesToParse: setup.closure }, + ); + + // 6. Extract the subgraph that needs to be written: closure-file nodes + // + graph-wide nodes + edges incident to them. Hydrated unchanged-file + // nodes are NOT in this subgraph — their rows are still in DB. + progress('lbug', 70, 'Writing incremental updates to LadybugDB...'); + const subgraph = extractIncrementalSubgraph(pipelineResult.graph, setup.closure); + await loadGraphToLbug(subgraph, repoPath, storagePath, (msg) => { + progress('lbug', 80, msg); + }); + + // 7. Recreate FTS indexes (cheap; full rebuild over the merged DB state). + progress('fts', 90, 'Refreshing search indexes...'); + try { + await createSearchFTSIndexes(); + } catch { + /* FTS is best-effort; log only */ + } + + // 8. Compute final stats from the live DB state. + const stats = await getLbugStats(); + + // 9. Compute new meta (merge surface signatures, clear dirty flag). + const mergedSurfaces = mergeSurfaceSignatures( + existingMeta.surfaceSignatures ?? {}, + setup.newFileHashes, + setup.deletedFiles, + ); + + const newMeta: import('../storage/repo-manager.js').RepoMeta = { + ...existingMeta, + lastCommit: currentCommit, + indexedAt: new Date().toISOString(), + remoteUrl: getRemoteUrl(repoPath) ?? existingMeta.remoteUrl, + schemaVersion: INCREMENTAL_SCHEMA_VERSION, + surfaceSignatures: mergedSurfaces, + incrementalInProgress: undefined, // explicit clear + stats: { + ...(existingMeta.stats ?? {}), + files: pipelineResult.totalFileCount, + nodes: stats.nodes, + edges: stats.edges, + communities: pipelineResult.communityResult?.stats.totalCommunities, + processes: pipelineResult.processResult?.stats.totalProcesses, + }, + }; + + // 10. Persist meta + register repo. + await saveMeta(storagePath, newMeta); + const projectName = await registerRepo(repoPath, newMeta, { + name: options.registryName, + allowDuplicateName: options.allowDuplicateName, + }); + + // 11. Best-effort AI-context regeneration so AGENTS.md/CLAUDE.md stay + // in sync with the post-incremental graph state. + let aggregatedClusterCount = 0; + if (pipelineResult.communityResult?.communities) { + const groups = new Map(); + for (const c of pipelineResult.communityResult.communities) { + const label = c.heuristicLabel || c.label || 'Unknown'; + groups.set(label, (groups.get(label) || 0) + c.symbolCount); + } + aggregatedClusterCount = Array.from(groups.values()).filter((cnt) => cnt >= 5).length; + } + try { + await generateAIContextFiles( + repoPath, + storagePath, + projectName, + { + files: pipelineResult.totalFileCount, + nodes: stats.nodes, + edges: stats.edges, + communities: pipelineResult.communityResult?.stats.totalCommunities, + clusters: aggregatedClusterCount, + processes: pipelineResult.processResult?.stats.totalProcesses, + }, + undefined, + { skipAgentsMd: options.skipAgentsMd, noStats: options.noStats }, + ); + } catch { + /* best-effort */ + } + + await ensureGitNexusIgnored(repoPath); + await closeLbug(); + + progress('done', 100, 'Incremental complete'); + + return { + repoName: projectName, + repoPath, + stats: newMeta.stats ?? {}, + pipelineResult, + }; +} diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 8c0bda95f..088f3262b 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -71,8 +71,48 @@ export interface RepoMeta { processes?: number; embeddings?: number; }; + /** + * Bumped whenever incremental-indexing invariants change in an + * incompatible way (schema bump, closure-algorithm fix, etc.). + * Mismatch → run-analyze forces a full rebuild. + * See docs/superpowers/specs/2026-05-10-incremental-indexing-design.md. + */ + schemaVersion?: number; + /** + * Per-file content/surface signatures used to drive incremental closure + * expansion. v1: SHA-256 of file content (cheap, conservative — body-only + * edits trigger 1-hop closure expansion). v2 will switch to a real + * surface-only signature so body-only edits stay at closure size 1. + * + * Map keys are repo-relative file paths; values are hex digests. Files + * not in this map are treated as "no previous signature" → first-time + * processing. + */ + surfaceSignatures?: Record; + /** + * Crash-recovery dirty flag. Written by run-analyze BEFORE any DB + * mutation in an incremental run; cleared by overwrite on success. Its + * presence at the start of the next run forces a full rebuild — the + * cheapest path back to a known-good index after a crashed/cancelled + * incremental run. Consumed by run-analyze.ts. + */ + incrementalInProgress?: { + closure: string[]; + startedAt: number; + }; } +/** + * Bumped whenever incremental-indexing invariants change in an + * incompatible way. v1 is `1`. Increment when: + * - The closure-expansion algorithm changes in a way that prior + * surfaceSignatures cannot be trusted. + * - The hydrate phase's DB schema assumptions change. + * - The surface-signature derivation changes (so prior hashes are not + * comparable to current ones). + */ +export const INCREMENTAL_SCHEMA_VERSION = 1; + export interface IndexedRepo { repoPath: string; storagePath: string;