feat(pipeline): hydrate phase + parse-filter for incremental indexing

Wires the incremental-indexing infrastructure into the phase-based
pipeline. Three coordinated changes:

* New hydratePhase (deps: structure) — loads node/edge state for files
  OUTSIDE ctx.options.filesToParse from the existing LadybugDB index.
  Runs before parse so the parse phase can produce a partial graph
  while downstream phases (mro, communities, processes) still see the
  full graph. No-op in full-rebuild mode (filesToParse unset).

* PipelineOptions.filesToParse: optional ReadonlySet<string>. When
  set, parse phase filters scanned files to this set; hydrate fills
  the complement. Set by runFullAnalysis when it detects an eligible
  incremental run; never set by callers directly.

* gitnexus-shared PipelinePhase enum: 'hydrate' added so progress
  callbacks can report the new phase distinctly from 'structure'.

Phase order: scan → structure → hydrate → markdown,cobol → parse
→ routes,tools,orm → crossFile → scopeResolution → mro → communities
→ processes. Communities (Leiden) still runs on the full graph,
satisfying the correctness invariant.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
abhigyanpatwari 2026-05-10 04:29:25 +05:30
parent 98bb893d00
commit bc03968640
5 changed files with 117 additions and 1 deletions

View file

@ -6,6 +6,7 @@ export type PipelinePhase =
| 'idle'
| 'extracting'
| 'structure'
| 'hydrate'
| 'parsing'
| 'imports'
| 'calls'

View file

@ -0,0 +1,91 @@
/**
* Phase: hydrate
*
* In incremental-indexing mode, populates `ctx.graph` with all nodes and
* relationships belonging to files OUTSIDE the current closure (i.e., files
* that didn't change). This way the parse phase only needs to re-emit nodes
* and edges for the closure files, while downstream phases (mro, communities,
* processes) see a fully-populated graph and produce results equivalent to a
* full rebuild.
*
* No-op in full-rebuild mode: when `ctx.options.filesToParse` is unset,
* the pipeline starts with an empty graph and parses every file (existing
* behavior).
*
* @deps structure
* @reads allPaths (from structure)
* @writes graph (every node/edge belonging to unchanged files)
*/
import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
import { getPhaseOutput } from './types.js';
import type { StructureOutput } from './structure.js';
import { loadGraphFromLbug } from '../../lbug/lbug-adapter.js';
import { isDev } from '../utils/env.js';
import { logger } from '../../logger.js';
export interface HydrateOutput {
/** True when this run actually loaded prior state (incremental). */
readonly hydrated: boolean;
/** Number of nodes loaded from DB (0 in full-rebuild mode). */
readonly nodesLoaded: number;
/** Number of relationships loaded from DB (0 in full-rebuild mode). */
readonly edgesLoaded: number;
}
export const hydratePhase: PipelinePhase<HydrateOutput> = {
name: 'hydrate',
deps: ['structure'],
async execute(
ctx: PipelineContext,
deps: ReadonlyMap<string, PhaseResult<unknown>>,
): Promise<HydrateOutput> {
const filesToParse = ctx.options?.filesToParse;
// Full-rebuild mode: nothing to hydrate.
if (!filesToParse) {
return { hydrated: false, nodesLoaded: 0, edgesLoaded: 0 };
}
const { allPaths, totalFiles } = getPhaseOutput<StructureOutput>(deps, 'structure');
// Compute the unchanged complement: every scanned path NOT in the closure.
const unchanged = new Set<string>();
for (const p of allPaths) {
if (!filesToParse.has(p)) unchanged.add(p);
}
ctx.onProgress({
phase: 'hydrate',
percent: 22,
message: `Hydrating ${unchanged.size} unchanged files from index...`,
stats: { filesProcessed: 0, totalFiles, nodesCreated: ctx.graph.nodeCount },
});
const result = await loadGraphFromLbug(ctx.graph, unchanged);
if (isDev) {
logger.info(
`💧 Hydrate: ${result.nodesLoaded} nodes, ${result.edgesLoaded} edges loaded for ${unchanged.size} unchanged files (closure: ${filesToParse.size})`,
);
}
ctx.onProgress({
phase: 'hydrate',
percent: 25,
message: `Hydrated ${result.nodesLoaded} nodes from previous index`,
stats: {
filesProcessed: unchanged.size,
totalFiles,
nodesCreated: ctx.graph.nodeCount,
},
});
return {
hydrated: true,
nodesLoaded: result.nodesLoaded,
edgesLoaded: result.edgesLoaded,
};
},
};

View file

@ -9,6 +9,7 @@
export { scanPhase, type ScanOutput } from './scan.js';
export { structurePhase, type StructureOutput } from './structure.js';
export { hydratePhase, type HydrateOutput } from './hydrate.js';
export { markdownPhase, type MarkdownOutput } from './markdown.js';
export { cobolPhase, type CobolOutput } from './cobol.js';
export { parsePhase, type ParseOutput } from './parse.js';

View file

@ -96,9 +96,18 @@ export const parsePhase: PipelinePhase<ParseOutput> = {
'structure',
);
// Incremental-indexing filter: when `filesToParse` is set, only the
// closure files get parsed in this run. Files outside the closure had
// their nodes/edges pre-loaded by the `hydrate` phase. See
// docs/superpowers/specs/2026-05-10-incremental-indexing-design.md.
const filesToParse = ctx.options?.filesToParse;
const targetScanned = filesToParse
? scannedFiles.filter((f) => filesToParse.has(f.path))
: scannedFiles;
const result = await runChunkedParseAndResolve(
ctx.graph,
scannedFiles,
targetScanned,
allPaths,
totalFiles,
ctx.repoPath,

View file

@ -23,6 +23,7 @@ import {
getPhaseOutput,
scanPhase,
structurePhase,
hydratePhase,
markdownPhase,
cobolPhase,
parsePhase,
@ -55,6 +56,18 @@ export interface PipelineOptions {
minFiles?: number;
minBytes?: number;
};
/**
* Incremental-indexing mode: when set, the parse phase only re-parses
* files in this set, and the new `hydrate` phase pre-loads node/edge
* state for everything else from the existing LadybugDB index.
*
* Unset (the default) full-rebuild mode: parse phase processes every
* scanned file and hydrate is a no-op. Set by `runFullAnalysis` when it
* detects an eligible incremental run; never set by callers directly.
*
* See `docs/superpowers/specs/2026-05-10-incremental-indexing-design.md`.
*/
filesToParse?: ReadonlySet<string>;
}
// ── Phase registry ─────────────────────────────────────────────────────────
@ -74,6 +87,7 @@ function buildPhaseList(options?: PipelineOptions): PipelinePhase[] {
const phases: PipelinePhase[] = [
scanPhase,
structurePhase,
hydratePhase,
markdownPhase,
cobolPhase,
parsePhase,