mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-08-28 05:25:25 +00:00
feat(pipeline): hydrate phase + parse-filter for incremental indexing
Wires the incremental-indexing infrastructure into the phase-based pipeline. Three coordinated changes: * New hydratePhase (deps: structure) — loads node/edge state for files OUTSIDE ctx.options.filesToParse from the existing LadybugDB index. Runs before parse so the parse phase can produce a partial graph while downstream phases (mro, communities, processes) still see the full graph. No-op in full-rebuild mode (filesToParse unset). * PipelineOptions.filesToParse: optional ReadonlySet<string>. When set, parse phase filters scanned files to this set; hydrate fills the complement. Set by runFullAnalysis when it detects an eligible incremental run; never set by callers directly. * gitnexus-shared PipelinePhase enum: 'hydrate' added so progress callbacks can report the new phase distinctly from 'structure'. Phase order: scan → structure → hydrate → markdown,cobol → parse → routes,tools,orm → crossFile → scopeResolution → mro → communities → processes. Communities (Leiden) still runs on the full graph, satisfying the correctness invariant. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
98bb893d00
commit
bc03968640
5 changed files with 117 additions and 1 deletions
|
|
@ -6,6 +6,7 @@ export type PipelinePhase =
|
|||
| 'idle'
|
||||
| 'extracting'
|
||||
| 'structure'
|
||||
| 'hydrate'
|
||||
| 'parsing'
|
||||
| 'imports'
|
||||
| 'calls'
|
||||
|
|
|
|||
91
gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts
Normal file
91
gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
/**
|
||||
* Phase: hydrate
|
||||
*
|
||||
* In incremental-indexing mode, populates `ctx.graph` with all nodes and
|
||||
* relationships belonging to files OUTSIDE the current closure (i.e., files
|
||||
* that didn't change). This way the parse phase only needs to re-emit nodes
|
||||
* and edges for the closure files, while downstream phases (mro, communities,
|
||||
* processes) see a fully-populated graph and produce results equivalent to a
|
||||
* full rebuild.
|
||||
*
|
||||
* No-op in full-rebuild mode: when `ctx.options.filesToParse` is unset,
|
||||
* the pipeline starts with an empty graph and parses every file (existing
|
||||
* behavior).
|
||||
*
|
||||
* @deps structure
|
||||
* @reads allPaths (from structure)
|
||||
* @writes graph (every node/edge belonging to unchanged files)
|
||||
*/
|
||||
|
||||
import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
|
||||
import { getPhaseOutput } from './types.js';
|
||||
import type { StructureOutput } from './structure.js';
|
||||
import { loadGraphFromLbug } from '../../lbug/lbug-adapter.js';
|
||||
import { isDev } from '../utils/env.js';
|
||||
import { logger } from '../../logger.js';
|
||||
|
||||
export interface HydrateOutput {
|
||||
/** True when this run actually loaded prior state (incremental). */
|
||||
readonly hydrated: boolean;
|
||||
/** Number of nodes loaded from DB (0 in full-rebuild mode). */
|
||||
readonly nodesLoaded: number;
|
||||
/** Number of relationships loaded from DB (0 in full-rebuild mode). */
|
||||
readonly edgesLoaded: number;
|
||||
}
|
||||
|
||||
export const hydratePhase: PipelinePhase<HydrateOutput> = {
|
||||
name: 'hydrate',
|
||||
deps: ['structure'],
|
||||
|
||||
async execute(
|
||||
ctx: PipelineContext,
|
||||
deps: ReadonlyMap<string, PhaseResult<unknown>>,
|
||||
): Promise<HydrateOutput> {
|
||||
const filesToParse = ctx.options?.filesToParse;
|
||||
|
||||
// Full-rebuild mode: nothing to hydrate.
|
||||
if (!filesToParse) {
|
||||
return { hydrated: false, nodesLoaded: 0, edgesLoaded: 0 };
|
||||
}
|
||||
|
||||
const { allPaths, totalFiles } = getPhaseOutput<StructureOutput>(deps, 'structure');
|
||||
|
||||
// Compute the unchanged complement: every scanned path NOT in the closure.
|
||||
const unchanged = new Set<string>();
|
||||
for (const p of allPaths) {
|
||||
if (!filesToParse.has(p)) unchanged.add(p);
|
||||
}
|
||||
|
||||
ctx.onProgress({
|
||||
phase: 'hydrate',
|
||||
percent: 22,
|
||||
message: `Hydrating ${unchanged.size} unchanged files from index...`,
|
||||
stats: { filesProcessed: 0, totalFiles, nodesCreated: ctx.graph.nodeCount },
|
||||
});
|
||||
|
||||
const result = await loadGraphFromLbug(ctx.graph, unchanged);
|
||||
|
||||
if (isDev) {
|
||||
logger.info(
|
||||
`💧 Hydrate: ${result.nodesLoaded} nodes, ${result.edgesLoaded} edges loaded for ${unchanged.size} unchanged files (closure: ${filesToParse.size})`,
|
||||
);
|
||||
}
|
||||
|
||||
ctx.onProgress({
|
||||
phase: 'hydrate',
|
||||
percent: 25,
|
||||
message: `Hydrated ${result.nodesLoaded} nodes from previous index`,
|
||||
stats: {
|
||||
filesProcessed: unchanged.size,
|
||||
totalFiles,
|
||||
nodesCreated: ctx.graph.nodeCount,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
hydrated: true,
|
||||
nodesLoaded: result.nodesLoaded,
|
||||
edgesLoaded: result.edgesLoaded,
|
||||
};
|
||||
},
|
||||
};
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
|
||||
export { scanPhase, type ScanOutput } from './scan.js';
|
||||
export { structurePhase, type StructureOutput } from './structure.js';
|
||||
export { hydratePhase, type HydrateOutput } from './hydrate.js';
|
||||
export { markdownPhase, type MarkdownOutput } from './markdown.js';
|
||||
export { cobolPhase, type CobolOutput } from './cobol.js';
|
||||
export { parsePhase, type ParseOutput } from './parse.js';
|
||||
|
|
|
|||
|
|
@ -96,9 +96,18 @@ export const parsePhase: PipelinePhase<ParseOutput> = {
|
|||
'structure',
|
||||
);
|
||||
|
||||
// Incremental-indexing filter: when `filesToParse` is set, only the
|
||||
// closure files get parsed in this run. Files outside the closure had
|
||||
// their nodes/edges pre-loaded by the `hydrate` phase. See
|
||||
// docs/superpowers/specs/2026-05-10-incremental-indexing-design.md.
|
||||
const filesToParse = ctx.options?.filesToParse;
|
||||
const targetScanned = filesToParse
|
||||
? scannedFiles.filter((f) => filesToParse.has(f.path))
|
||||
: scannedFiles;
|
||||
|
||||
const result = await runChunkedParseAndResolve(
|
||||
ctx.graph,
|
||||
scannedFiles,
|
||||
targetScanned,
|
||||
allPaths,
|
||||
totalFiles,
|
||||
ctx.repoPath,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import {
|
|||
getPhaseOutput,
|
||||
scanPhase,
|
||||
structurePhase,
|
||||
hydratePhase,
|
||||
markdownPhase,
|
||||
cobolPhase,
|
||||
parsePhase,
|
||||
|
|
@ -55,6 +56,18 @@ export interface PipelineOptions {
|
|||
minFiles?: number;
|
||||
minBytes?: number;
|
||||
};
|
||||
/**
|
||||
* Incremental-indexing mode: when set, the parse phase only re-parses
|
||||
* files in this set, and the new `hydrate` phase pre-loads node/edge
|
||||
* state for everything else from the existing LadybugDB index.
|
||||
*
|
||||
* Unset (the default) → full-rebuild mode: parse phase processes every
|
||||
* scanned file and hydrate is a no-op. Set by `runFullAnalysis` when it
|
||||
* detects an eligible incremental run; never set by callers directly.
|
||||
*
|
||||
* See `docs/superpowers/specs/2026-05-10-incremental-indexing-design.md`.
|
||||
*/
|
||||
filesToParse?: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
// ── Phase registry ─────────────────────────────────────────────────────────
|
||||
|
|
@ -74,6 +87,7 @@ function buildPhaseList(options?: PipelineOptions): PipelinePhase[] {
|
|||
const phases: PipelinePhase[] = [
|
||||
scanPhase,
|
||||
structurePhase,
|
||||
hydratePhase,
|
||||
markdownPhase,
|
||||
cobolPhase,
|
||||
parsePhase,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue