mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-09 03:17:54 +00:00
feat(pipeline): hydrate phase + parse-filter for incremental indexing
Wires the incremental-indexing infrastructure into the phase-based pipeline. Three coordinated changes: * New hydratePhase (deps: structure) — loads node/edge state for files OUTSIDE ctx.options.filesToParse from the existing LadybugDB index. Runs before parse so the parse phase can produce a partial graph while downstream phases (mro, communities, processes) still see the full graph. No-op in full-rebuild mode (filesToParse unset). * PipelineOptions.filesToParse: optional ReadonlySet<string>. When set, parse phase filters scanned files to this set; hydrate fills the complement. Set by runFullAnalysis when it detects an eligible incremental run; never set by callers directly. * gitnexus-shared PipelinePhase enum: 'hydrate' added so progress callbacks can report the new phase distinctly from 'structure'. Phase order: scan → structure → hydrate → markdown,cobol → parse → routes,tools,orm → crossFile → scopeResolution → mro → communities → processes. Communities (Leiden) still runs on the full graph, satisfying the correctness invariant. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
98bb893d00
commit
bc03968640
5 changed files with 117 additions and 1 deletions
|
|
@ -6,6 +6,7 @@ export type PipelinePhase =
|
||||||
| 'idle'
|
| 'idle'
|
||||||
| 'extracting'
|
| 'extracting'
|
||||||
| 'structure'
|
| 'structure'
|
||||||
|
| 'hydrate'
|
||||||
| 'parsing'
|
| 'parsing'
|
||||||
| 'imports'
|
| 'imports'
|
||||||
| 'calls'
|
| 'calls'
|
||||||
|
|
|
||||||
91
gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts
Normal file
91
gitnexus/src/core/ingestion/pipeline-phases/hydrate.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
||||||
|
/**
|
||||||
|
* Phase: hydrate
|
||||||
|
*
|
||||||
|
* In incremental-indexing mode, populates `ctx.graph` with all nodes and
|
||||||
|
* relationships belonging to files OUTSIDE the current closure (i.e., files
|
||||||
|
* that didn't change). This way the parse phase only needs to re-emit nodes
|
||||||
|
* and edges for the closure files, while downstream phases (mro, communities,
|
||||||
|
* processes) see a fully-populated graph and produce results equivalent to a
|
||||||
|
* full rebuild.
|
||||||
|
*
|
||||||
|
* No-op in full-rebuild mode: when `ctx.options.filesToParse` is unset,
|
||||||
|
* the pipeline starts with an empty graph and parses every file (existing
|
||||||
|
* behavior).
|
||||||
|
*
|
||||||
|
* @deps structure
|
||||||
|
* @reads allPaths (from structure)
|
||||||
|
* @writes graph (every node/edge belonging to unchanged files)
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
|
||||||
|
import { getPhaseOutput } from './types.js';
|
||||||
|
import type { StructureOutput } from './structure.js';
|
||||||
|
import { loadGraphFromLbug } from '../../lbug/lbug-adapter.js';
|
||||||
|
import { isDev } from '../utils/env.js';
|
||||||
|
import { logger } from '../../logger.js';
|
||||||
|
|
||||||
|
export interface HydrateOutput {
|
||||||
|
/** True when this run actually loaded prior state (incremental). */
|
||||||
|
readonly hydrated: boolean;
|
||||||
|
/** Number of nodes loaded from DB (0 in full-rebuild mode). */
|
||||||
|
readonly nodesLoaded: number;
|
||||||
|
/** Number of relationships loaded from DB (0 in full-rebuild mode). */
|
||||||
|
readonly edgesLoaded: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const hydratePhase: PipelinePhase<HydrateOutput> = {
|
||||||
|
name: 'hydrate',
|
||||||
|
deps: ['structure'],
|
||||||
|
|
||||||
|
async execute(
|
||||||
|
ctx: PipelineContext,
|
||||||
|
deps: ReadonlyMap<string, PhaseResult<unknown>>,
|
||||||
|
): Promise<HydrateOutput> {
|
||||||
|
const filesToParse = ctx.options?.filesToParse;
|
||||||
|
|
||||||
|
// Full-rebuild mode: nothing to hydrate.
|
||||||
|
if (!filesToParse) {
|
||||||
|
return { hydrated: false, nodesLoaded: 0, edgesLoaded: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const { allPaths, totalFiles } = getPhaseOutput<StructureOutput>(deps, 'structure');
|
||||||
|
|
||||||
|
// Compute the unchanged complement: every scanned path NOT in the closure.
|
||||||
|
const unchanged = new Set<string>();
|
||||||
|
for (const p of allPaths) {
|
||||||
|
if (!filesToParse.has(p)) unchanged.add(p);
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.onProgress({
|
||||||
|
phase: 'hydrate',
|
||||||
|
percent: 22,
|
||||||
|
message: `Hydrating ${unchanged.size} unchanged files from index...`,
|
||||||
|
stats: { filesProcessed: 0, totalFiles, nodesCreated: ctx.graph.nodeCount },
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await loadGraphFromLbug(ctx.graph, unchanged);
|
||||||
|
|
||||||
|
if (isDev) {
|
||||||
|
logger.info(
|
||||||
|
`💧 Hydrate: ${result.nodesLoaded} nodes, ${result.edgesLoaded} edges loaded for ${unchanged.size} unchanged files (closure: ${filesToParse.size})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx.onProgress({
|
||||||
|
phase: 'hydrate',
|
||||||
|
percent: 25,
|
||||||
|
message: `Hydrated ${result.nodesLoaded} nodes from previous index`,
|
||||||
|
stats: {
|
||||||
|
filesProcessed: unchanged.size,
|
||||||
|
totalFiles,
|
||||||
|
nodesCreated: ctx.graph.nodeCount,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
return {
|
||||||
|
hydrated: true,
|
||||||
|
nodesLoaded: result.nodesLoaded,
|
||||||
|
edgesLoaded: result.edgesLoaded,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
@ -9,6 +9,7 @@
|
||||||
|
|
||||||
export { scanPhase, type ScanOutput } from './scan.js';
|
export { scanPhase, type ScanOutput } from './scan.js';
|
||||||
export { structurePhase, type StructureOutput } from './structure.js';
|
export { structurePhase, type StructureOutput } from './structure.js';
|
||||||
|
export { hydratePhase, type HydrateOutput } from './hydrate.js';
|
||||||
export { markdownPhase, type MarkdownOutput } from './markdown.js';
|
export { markdownPhase, type MarkdownOutput } from './markdown.js';
|
||||||
export { cobolPhase, type CobolOutput } from './cobol.js';
|
export { cobolPhase, type CobolOutput } from './cobol.js';
|
||||||
export { parsePhase, type ParseOutput } from './parse.js';
|
export { parsePhase, type ParseOutput } from './parse.js';
|
||||||
|
|
|
||||||
|
|
@ -96,9 +96,18 @@ export const parsePhase: PipelinePhase<ParseOutput> = {
|
||||||
'structure',
|
'structure',
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Incremental-indexing filter: when `filesToParse` is set, only the
|
||||||
|
// closure files get parsed in this run. Files outside the closure had
|
||||||
|
// their nodes/edges pre-loaded by the `hydrate` phase. See
|
||||||
|
// docs/superpowers/specs/2026-05-10-incremental-indexing-design.md.
|
||||||
|
const filesToParse = ctx.options?.filesToParse;
|
||||||
|
const targetScanned = filesToParse
|
||||||
|
? scannedFiles.filter((f) => filesToParse.has(f.path))
|
||||||
|
: scannedFiles;
|
||||||
|
|
||||||
const result = await runChunkedParseAndResolve(
|
const result = await runChunkedParseAndResolve(
|
||||||
ctx.graph,
|
ctx.graph,
|
||||||
scannedFiles,
|
targetScanned,
|
||||||
allPaths,
|
allPaths,
|
||||||
totalFiles,
|
totalFiles,
|
||||||
ctx.repoPath,
|
ctx.repoPath,
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ import {
|
||||||
getPhaseOutput,
|
getPhaseOutput,
|
||||||
scanPhase,
|
scanPhase,
|
||||||
structurePhase,
|
structurePhase,
|
||||||
|
hydratePhase,
|
||||||
markdownPhase,
|
markdownPhase,
|
||||||
cobolPhase,
|
cobolPhase,
|
||||||
parsePhase,
|
parsePhase,
|
||||||
|
|
@ -55,6 +56,18 @@ export interface PipelineOptions {
|
||||||
minFiles?: number;
|
minFiles?: number;
|
||||||
minBytes?: number;
|
minBytes?: number;
|
||||||
};
|
};
|
||||||
|
/**
|
||||||
|
* Incremental-indexing mode: when set, the parse phase only re-parses
|
||||||
|
* files in this set, and the new `hydrate` phase pre-loads node/edge
|
||||||
|
* state for everything else from the existing LadybugDB index.
|
||||||
|
*
|
||||||
|
* Unset (the default) → full-rebuild mode: parse phase processes every
|
||||||
|
* scanned file and hydrate is a no-op. Set by `runFullAnalysis` when it
|
||||||
|
* detects an eligible incremental run; never set by callers directly.
|
||||||
|
*
|
||||||
|
* See `docs/superpowers/specs/2026-05-10-incremental-indexing-design.md`.
|
||||||
|
*/
|
||||||
|
filesToParse?: ReadonlySet<string>;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Phase registry ─────────────────────────────────────────────────────────
|
// ── Phase registry ─────────────────────────────────────────────────────────
|
||||||
|
|
@ -74,6 +87,7 @@ function buildPhaseList(options?: PipelineOptions): PipelinePhase[] {
|
||||||
const phases: PipelinePhase[] = [
|
const phases: PipelinePhase[] = [
|
||||||
scanPhase,
|
scanPhase,
|
||||||
structurePhase,
|
structurePhase,
|
||||||
|
hydratePhase,
|
||||||
markdownPhase,
|
markdownPhase,
|
||||||
cobolPhase,
|
cobolPhase,
|
||||||
parsePhase,
|
parsePhase,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue