diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 257a04bc8..b557625eb 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -33,6 +33,7 @@ import { persistParsedFileChunk, getDurableParsedFileDir, loadDurableParsedFileIndex, + prepareDurableParsedFileChunk, restoreDurableParsedFileShard, } from '../../../storage/parsedfile-store.js'; import type { ParseWorkerResult } from '../workers/parse-worker.js'; @@ -984,6 +985,9 @@ export async function runChunkedParseAndResolve( // Cache miss: dispatch to workers, capture the raw results, store // them under the chunk hash for the next run. chunkCacheMisses++; + if (durableParsedFileDir !== undefined && chunkHash !== null) { + await prepareDurableParsedFileChunk(durableParsedFileDir, chunkHash); + } const progressForChunk = (current: number, _total: number, filePath: string) => { const globalCurrent = filesParsedSoFar + current; // Parse phase covers 20-70 (M2). Deferred extraction handles 70-95. diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index 1353f6324..b11b8c27f 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -55,7 +55,7 @@ import type { ParseWorkerResult } from '../core/ingestion/workers/parse-worker.j // the main thread (the #1983 OOM). Because the two stores share this version, // any future change to the `ParsedFile` serialization shape MUST bump // SCHEMA_BUMP so both invalidate in lockstep. -const SCHEMA_BUMP = 12; // #2391 follow-up: extractPythonModuleConstants changed what it EMITS for the same source (binding mutual-exclusivity clears stale imports; RHS refs are snapshotted; `$imp$N` aliases). `moduleConstants` is cached verbatim, so a warm shard built pre-fix would replay stale/WRONG folds and the correctness fixes would silently no-op on upgrade — bump to force re-extraction. (11 = #2391: ExtractedDecoratorRoute gained `routePathExpr`/`routePathOperands` + ParseWorkerResult gained per-file `moduleConstants`. 10 = PR #2200: Property nodes gained `rawDeclaredType` + `annotations` for Spring DI) +const SCHEMA_BUMP = 13; // Durable ParsedFile chunk directories now replace one complete generation instead of accumulating worker shards across cache-miss analyses. Invalidate once so existing unbounded stores are rebuilt under the bounded contract. (12 = #2391 follow-up: Python module constant extraction semantics changed.) const GITNEXUS_PKG_VERSION = (() => { try { // package.json sits at gitnexus/package.json — two levels up from diff --git a/gitnexus/src/storage/parsedfile-store.ts b/gitnexus/src/storage/parsedfile-store.ts index 842be5c2e..302202c5d 100644 --- a/gitnexus/src/storage/parsedfile-store.ts +++ b/gitnexus/src/storage/parsedfile-store.ts @@ -292,6 +292,22 @@ export const getDurableParsedFileDir = (storagePath: string): string => const durableChunkDir = (durableDir: string, chunkHash: string): string => path.join(durableDir, chunkHash); +/** + * Start a fresh durable generation for one content-addressed parse chunk. + * The main thread calls this once before dispatching a cache miss, before any + * worker can write that chunk. Recreating the directory immediately keeps the + * worker-side mkdir memoization valid while preventing old worker shard names + * from accumulating across analyses. + */ +export const prepareDurableParsedFileChunk = async ( + durableDir: string, + chunkHash: string, +): Promise => { + const dir = durableChunkDir(durableDir, chunkHash); + await fs.rm(dir, { recursive: true, force: true }); + await fs.mkdir(dir, { recursive: true }); +}; + // Per-process set of durable chunk subdirs already `mkdir`ed (mirrors // `createdStoreDirs`) so the worker doesn't `mkdirSync` on every shard. const createdDurableDirs = new Set(); diff --git a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts index e98083737..dabca8001 100644 --- a/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts +++ b/gitnexus/test/unit/parse-impl-warm-cache-parsedfile-coverage.test.ts @@ -41,6 +41,7 @@ import { } from '../../src/storage/parse-cache.js'; import { getDurableParsedFileDir, + prepareDurableParsedFileChunk, persistDurableParsedFileShardSync, restoreDurableParsedFileShard, loadParsedFilesForPaths, @@ -100,6 +101,29 @@ describe('durable ParsedFile store — content-addressed warm-cache coverage', ( expect(restored).toBe(0); }); + it('prepares a fresh durable generation without retaining old worker shards', async () => { + const durableDir = getDurableParsedFileDir(tempDir); + const chunkHash = 'f'.repeat(64); + const chunkDir = path.join(durableDir, chunkHash); + + persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile('old.ts')]); + await prepareDurableParsedFileChunk(durableDir, chunkHash); + persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile('new-a.ts')]); + persistDurableParsedFileShardSync(durableDir, chunkHash, 2, 0, [mkParsedFile('new-b.ts')]); + + const shards = fs + .readdirSync(chunkDir) + .filter((name) => name.endsWith('.json')) + .sort(); + expect(shards).toEqual([`${chunkHash}-w1-0.json`, `${chunkHash}-w2-0.json`]); + await restoreDurableParsedFileShard(durableDir, tempDir, chunkHash); + const files = await loadParsedFilesForPaths( + tempDir, + new Set(['old.ts', 'new-a.ts', 'new-b.ts']), + ); + expect([...files.keys()].sort()).toEqual(['new-a.ts', 'new-b.ts']); + }); + it('index load is version-gated (PARSE_CACHE_VERSION mismatch ⇒ empty)', async () => { const durableDir = getDurableParsedFileDir(tempDir); const chunkHash = 'c'.repeat(64); @@ -291,6 +315,27 @@ describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { expect(cache.usedKeys.has(chunkHash)).toBe(true); }); + it('a repeated cache miss replaces the durable chunk generation', async () => { + const f = writeFile('src/repeated.ts', 'export function repeated() { return 1; }\n'); + const chunkHash = computeChunkHash([ + { + filePath: f.path, + contentHash: fileContentHash(fs.readFileSync(path.join(repoDir, f.path), 'utf-8')), + }, + ]); + + await run(newCache(), [f]); + await run(newCache(), [f]); + + const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); + const shards = fs.readdirSync(chunkDir).filter((name) => name.endsWith('.json')); + expect(shards).toHaveLength(1); + const parsed = JSON.parse(fs.readFileSync(path.join(chunkDir, shards[0]!), 'utf-8')) as Array<{ + filePath: string; + }>; + expect(parsed.map((item) => item.filePath)).toEqual(['src/repeated.ts']); + }); + it('run #2 (all hits) spawns NO worker — the warm path is served from caches', async () => { const f = writeFile('src/cached.ts', 'export function cached() { return 1; }\n'); const cache = newCache();