diff --git a/gitnexus-shared/src/pipeline.ts b/gitnexus-shared/src/pipeline.ts index 5f7e61c57..70b8645da 100644 --- a/gitnexus-shared/src/pipeline.ts +++ b/gitnexus-shared/src/pipeline.ts @@ -6,6 +6,7 @@ export type PipelinePhase = | 'idle' | 'extracting' | 'structure' + | 'hydrate' | 'parsing' | 'imports' | 'calls' diff --git a/gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts b/gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts new file mode 100644 index 000000000..a0c00c2ff --- /dev/null +++ b/gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts @@ -0,0 +1,91 @@ +/** + * Phase: hydrate + * + * In incremental-indexing mode, populates `ctx.graph` with all nodes and + * relationships belonging to files OUTSIDE the current closure (i.e., files + * that didn't change). This way the parse phase only needs to re-emit nodes + * and edges for the closure files, while downstream phases (mro, communities, + * processes) see a fully-populated graph and produce results equivalent to a + * full rebuild. + * + * No-op in full-rebuild mode: when `ctx.options.filesToParse` is unset, + * the pipeline starts with an empty graph and parses every file (existing + * behavior). + * + * @deps structure + * @reads allPaths (from structure) + * @writes graph (every node/edge belonging to unchanged files) + */ + +import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; +import { getPhaseOutput } from './types.js'; +import type { StructureOutput } from './structure.js'; +import { loadGraphFromLbug } from '../../lbug/lbug-adapter.js'; +import { isDev } from '../utils/env.js'; +import { logger } from '../../logger.js'; + +export interface HydrateOutput { + /** True when this run actually loaded prior state (incremental). */ + readonly hydrated: boolean; + /** Number of nodes loaded from DB (0 in full-rebuild mode). */ + readonly nodesLoaded: number; + /** Number of relationships loaded from DB (0 in full-rebuild mode). */ + readonly edgesLoaded: number; +} + +export const hydratePhase: PipelinePhase = { + name: 'hydrate', + deps: ['structure'], + + async execute( + ctx: PipelineContext, + deps: ReadonlyMap>, + ): Promise { + const filesToParse = ctx.options?.filesToParse; + + // Full-rebuild mode: nothing to hydrate. + if (!filesToParse) { + return { hydrated: false, nodesLoaded: 0, edgesLoaded: 0 }; + } + + const { allPaths, totalFiles } = getPhaseOutput(deps, 'structure'); + + // Compute the unchanged complement: every scanned path NOT in the closure. + const unchanged = new Set(); + for (const p of allPaths) { + if (!filesToParse.has(p)) unchanged.add(p); + } + + ctx.onProgress({ + phase: 'hydrate', + percent: 22, + message: `Hydrating ${unchanged.size} unchanged files from index...`, + stats: { filesProcessed: 0, totalFiles, nodesCreated: ctx.graph.nodeCount }, + }); + + const result = await loadGraphFromLbug(ctx.graph, unchanged); + + if (isDev) { + logger.info( + `💧 Hydrate: ${result.nodesLoaded} nodes, ${result.edgesLoaded} edges loaded for ${unchanged.size} unchanged files (closure: ${filesToParse.size})`, + ); + } + + ctx.onProgress({ + phase: 'hydrate', + percent: 25, + message: `Hydrated ${result.nodesLoaded} nodes from previous index`, + stats: { + filesProcessed: unchanged.size, + totalFiles, + nodesCreated: ctx.graph.nodeCount, + }, + }); + + return { + hydrated: true, + nodesLoaded: result.nodesLoaded, + edgesLoaded: result.edgesLoaded, + }; + }, +}; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/index.ts b/gitnexus/src/core/ingestion/pipeline-phases/index.ts index b1dcf9082..908d6c0a7 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/index.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/index.ts @@ -9,6 +9,7 @@ export { scanPhase, type ScanOutput } from './scan.js'; export { structurePhase, type StructureOutput } from './structure.js'; +export { hydratePhase, type HydrateOutput } from './hydrate.js'; export { markdownPhase, type MarkdownOutput } from './markdown.js'; export { cobolPhase, type CobolOutput } from './cobol.js'; export { parsePhase, type ParseOutput } from './parse.js'; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index a20d1e4b0..d0217d033 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -96,9 +96,18 @@ export const parsePhase: PipelinePhase = { 'structure', ); + // Incremental-indexing filter: when `filesToParse` is set, only the + // closure files get parsed in this run. Files outside the closure had + // their nodes/edges pre-loaded by the `hydrate` phase. See + // docs/superpowers/specs/2026-05-10-incremental-indexing-design.md. + const filesToParse = ctx.options?.filesToParse; + const targetScanned = filesToParse + ? scannedFiles.filter((f) => filesToParse.has(f.path)) + : scannedFiles; + const result = await runChunkedParseAndResolve( ctx.graph, - scannedFiles, + targetScanned, allPaths, totalFiles, ctx.repoPath, diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index c220ea224..e93444352 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -23,6 +23,7 @@ import { getPhaseOutput, scanPhase, structurePhase, + hydratePhase, markdownPhase, cobolPhase, parsePhase, @@ -55,6 +56,18 @@ export interface PipelineOptions { minFiles?: number; minBytes?: number; }; + /** + * Incremental-indexing mode: when set, the parse phase only re-parses + * files in this set, and the new `hydrate` phase pre-loads node/edge + * state for everything else from the existing LadybugDB index. + * + * Unset (the default) → full-rebuild mode: parse phase processes every + * scanned file and hydrate is a no-op. Set by `runFullAnalysis` when it + * detects an eligible incremental run; never set by callers directly. + * + * See `docs/superpowers/specs/2026-05-10-incremental-indexing-design.md`. + */ + filesToParse?: ReadonlySet; } // ── Phase registry ───────────────────────────────────────────────────────── @@ -74,6 +87,7 @@ function buildPhaseList(options?: PipelineOptions): PipelinePhase[] { const phases: PipelinePhase[] = [ scanPhase, structurePhase, + hydratePhase, markdownPhase, cobolPhase, parsePhase,