/** * Regression coverage for the warm-cache ParsedFile gap (#2038, abhigyanpatwari * review on parse-cache.ts). * * #1983 made the parse worker the sole parser: it serializes ParsedFiles to a * disk store that scope-resolution streams back, so the main thread NEVER * re-parses (the unbounded tree-sitter 0.21.1 native leak → OOM on huge repos). * The gap: on a WARM re-analyze where every chunk is a parse-cache HIT, no * worker runs, the run-scoped store is cleared at parse start, and the cached * `ParseWorkerResult` carries no ParsedFiles — so scope-resolution would find an * empty store and fall back to main-thread `extractParsedFile`, re-opening the * OOM. * * The fix: workers ALSO write a durable, content-addressed ParsedFile store * keyed by chunk hash (`parsedfile-cache/`); a warm hit LOADS those shards * in place (no copy into the run-scoped store) so scope-resolution streams * them exactly as on a cold run — zero re-parse, byte-identical. * * Two layers of coverage: * (1) Store-level — the durable persist → `loadParsedFilesForPaths` * round-trip at the EXACT seam scope-resolution consumes (phase.ts:255), * plus the index version gate and the prune-coherence rule. Build-free. * (2) Integration — a two-run `runChunkedParseAndResolve`: run #1 (all miss) * populates the durable store; run #2 (all hits) spawns NO worker and * loads full coverage from durable shards; the coherence gate re-dispatches * when durable shards are absent; and a mixed-mode run (one file changed) * hits the unchanged chunk while re-parsing the changed one. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; // Partial mock: lets one test make prepareDurableParsedFileChunk fail without // touching the worker-side persist path (which shares the same directory). // Receives the chunk hash so a test can fail the reset for ONE chunk while its // siblings reset normally — the shape a correlated-vs-isolated failure needs. const prepareOverride = vi.hoisted(() => ({ impl: undefined as undefined | ((durableDir: string, chunkHash: string) => Promise), })); const persistOverride = vi.hoisted(() => ({ impl: undefined as undefined | (() => Promise), })); vi.mock('../../src/storage/parsedfile-store.js', async (importOriginal) => { const real = await importOriginal(); return { ...real, prepareDurableParsedFileChunk: (durableDir: string, chunkHash: string) => prepareOverride.impl ? prepareOverride.impl(durableDir, chunkHash) : real.prepareDurableParsedFileChunk(durableDir, chunkHash), persistParsedFileChunk: ( storagePath: string, shardId: string, parsedFiles: readonly ParsedFile[], ) => persistOverride.impl ? persistOverride.impl() : real.persistParsedFileChunk(storagePath, shardId, parsedFiles), }; }); import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; import { computeChunkHash, fileContentHash, PARSE_CACHE_VERSION, } from '../../src/storage/parse-cache.js'; import { getDurableParsedFileDir, getParsedFileStoreDir, prepareDurableParsedFileChunk, persistDurableParsedFileShardSync, durableChunkHasShards, loadParsedFilesForPaths, loadDurableParsedFileIndex, pruneAndSaveDurableParsedFileStore, clearParsedFileStore, } from '../../src/storage/parsedfile-store.js'; import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; import type { ParseCache } from '../../src/storage/parse-cache.js'; import type { ParsedFile } from 'gitnexus-shared'; // A structurally-minimal ParsedFile. `loadParsedFilesForPaths` keys on // `filePath`; the rest are empty so a restored shard is byte-stable and // scope-resolution has nothing to resolve (no edges) but no malformed input. const mkParsedFile = (filePath: string): ParsedFile => ({ filePath, moduleScope: '', scopes: [], parsedImports: [], localDefs: [], referenceSites: [], }) as unknown as ParsedFile; // ─── Layer 1: durable store mechanics (build-free) ────────────────────────── describe('durable ParsedFile store — content-addressed warm-cache coverage', () => { let tempDir = ''; beforeEach(() => { tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'durable-parsedfile-store-')); }); afterEach(() => { if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); }); it('persist → loadParsedFilesForPaths gives full coverage (the warm seam)', async () => { const durableDir = getDurableParsedFileDir(tempDir); const chunkHash = 'a'.repeat(64); const files = ['src/a.ts', 'src/b.ts']; // A worker would write this at flush on a cache MISS. persistDurableParsedFileShardSync(durableDir, chunkHash, 7, 0, files.map(mkParsedFile)); await clearParsedFileStore(tempDir); const wanted = new Set(files); expect(await durableChunkHasShards(tempDir, chunkHash, wanted)).toBe(true); const loaded = await loadParsedFilesForPaths(tempDir, wanted); expect([...loaded.keys()].sort()).toEqual([...files].sort()); }); it('durableChunkHasShards is false when the chunk has no durable shards', async () => { expect(await durableChunkHasShards(tempDir, 'b'.repeat(64), new Set(['missing.ts']))).toBe( false, ); }); it('prepares a fresh durable generation without retaining old worker shards', async () => { const durableDir = getDurableParsedFileDir(tempDir); const chunkHash = 'f'.repeat(64); const chunkDir = path.join(durableDir, chunkHash); persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile('old.ts')]); await prepareDurableParsedFileChunk(durableDir, chunkHash); persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile('new-a.ts')]); persistDurableParsedFileShardSync(durableDir, chunkHash, 2, 0, [mkParsedFile('new-b.ts')]); const shards = fs .readdirSync(chunkDir) .filter((name) => name.endsWith('.v8')) .sort(); expect(shards).toEqual([`${chunkHash}-w1-0.v8`, `${chunkHash}-w2-0.v8`]); const wanted = new Set(['old.ts', 'new-a.ts', 'new-b.ts']); expect(await durableChunkHasShards(tempDir, chunkHash, new Set(['new-a.ts', 'new-b.ts']))).toBe( true, ); const files = await loadParsedFilesForPaths(tempDir, wanted); expect([...files.keys()].sort()).toEqual(['new-a.ts', 'new-b.ts']); }); it('index load is version-gated (PARSE_CACHE_VERSION mismatch ⇒ empty)', async () => { const durableDir = getDurableParsedFileDir(tempDir); const chunkHash = 'c'.repeat(64); persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile('x.ts')]); await pruneAndSaveDurableParsedFileStore(durableDir, PARSE_CACHE_VERSION, new Set([chunkHash])); expect(await loadDurableParsedFileIndex(durableDir, PARSE_CACHE_VERSION)).toEqual( new Map([[chunkHash, new Set(['x.ts'])]]), ); // A schema bump (different version) invalidates the whole durable store. expect(await loadDurableParsedFileIndex(durableDir, '999+9.9.9')).toEqual(new Map()); }); it('prune keeps only keepKeys subdirs with ≥1 shard, drops the rest, and re-indexes', async () => { const durableDir = getDurableParsedFileDir(tempDir); const keep = 'd'.repeat(64); const drop = 'e'.repeat(64); // present on disk but NOT in keepKeys (e.g. quarantined / stale) persistDurableParsedFileShardSync(durableDir, keep, 1, 0, [mkParsedFile('keep.ts')]); persistDurableParsedFileShardSync(durableDir, drop, 1, 0, [mkParsedFile('drop.ts')]); await pruneAndSaveDurableParsedFileStore(durableDir, PARSE_CACHE_VERSION, new Set([keep])); expect(fs.existsSync(path.join(durableDir, keep))).toBe(true); expect(fs.existsSync(path.join(durableDir, drop))).toBe(false); expect(await loadDurableParsedFileIndex(durableDir, PARSE_CACHE_VERSION)).toEqual( new Map([[keep, new Set(['keep.ts'])]]), ); }); }); // ─── Layer 2: parse-impl integration (injected worker, build-free) ─────────── // A test worker that mirrors the production flush contract: it writes a // run-scoped V8 shard AND a durable, content-addressed V8 shard (when the // flush carries a chunk hash) using the SAME directory layout as the real worker. const writeStoreWorker = (workerPath: string, markerPath: string): void => { fs.writeFileSync( workerPath, ` const fs = require('node:fs'); const path = require('node:path'); const v8 = require('node:v8'); const { createHash } = require('node:crypto'); const { parentPort, threadId, workerData } = require('node:worker_threads'); const storePath = workerData && workerData.parsedFileStoreStoragePath; const durablePath = workerData && workerData.durableParsedFileStoragePath; let shardSeq = 0; fs.writeFileSync(${JSON.stringify(markerPath)}, 'spawned'); parentPort.postMessage({ type: 'ready' }); const writeV8 = (filePath, graph, paths) => { const payload = v8.serialize(graph); const listing = paths.some((p) => /[\\r\\n\\0]/.test(p)) ? Buffer.alloc(0) : Buffer.from(paths.length + '\\n' + paths.join('\\n') + '\\n', 'utf8'); const MAGIC = Buffer.from('GNXV8CF1'); const v8ver = Buffer.from(process.versions.v8, 'utf8'); const nodeMajor = Number.parseInt(process.versions.node.split('.')[0], 10); const header = Buffer.allocUnsafe(16 + v8ver.length + 12); MAGIC.copy(header, 0); header.writeUInt32LE(5, 8); header.writeUInt16LE(nodeMajor, 12); header.writeUInt16LE(v8ver.length, 14); v8ver.copy(header, 16); let off = 16 + v8ver.length; header.writeUInt32LE(listing.length === 0 ? 0 : paths.length, off); header.writeUInt32LE(listing.length, off + 4); header.writeUInt32LE(payload.length, off + 8); fs.mkdirSync(path.dirname(filePath), { recursive: true }); const payloadHash = createHash('sha256').update(listing).update(payload).digest(); fs.writeFileSync(filePath, Buffer.concat([header, listing, payload, payloadHash])); }; const reset = () => ({ nodes: [], relationships: [], symbols: [], imports: [], calls: [], assignments: [], heritage: [], routes: [], fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], routerIncludes: [], routerImports: [], toolDefs: [], ormQueries: [], constructorBindings: [], fileScopeBindings: [], parsedFiles: [], skippedLanguages: {}, fileCount: 0, scopeExtractionFailures: [], }); let accumulated = reset(); parentPort.on('message', (msg) => { if (msg && msg.type === 'sub-batch') { for (const file of msg.files) { const filePath = file.path; const name = filePath.split('/').pop().replace(/\\.ts$/, ''); accumulated.nodes.push({ id: 'Function:' + filePath + ':' + name, label: 'Function', properties: { name, filePath, startLine: 1, endLine: 1, language: 'typescript' }, }); accumulated.parsedFiles.push({ filePath, moduleScope: '', scopes: [], parsedImports: [], localDefs: [], referenceSites: [], }); if (filePath.includes('broken')) accumulated.scopeExtractionFailures.push(filePath); accumulated.fileCount++; } parentPort.postMessage({ type: 'progress', filesProcessed: accumulated.fileCount }); parentPort.postMessage({ type: 'sub-batch-done' }); return; } if (msg && msg.type === 'flush') { if ((storePath || durablePath) && accumulated.parsedFiles.length > 0) { const seq = shardSeq++; const paths = accumulated.parsedFiles.map((pf) => pf.filePath); let wroteStore = false; if (durablePath && typeof msg.chunkHash === 'string') { writeV8( path.join(durablePath, msg.chunkHash, msg.chunkHash + '-w' + threadId + '-' + seq + '.v8'), accumulated.parsedFiles, paths, ); } if (storePath) { writeV8( path.join(storePath, 'parsedfile-store', 'w' + threadId + '-' + seq + '.v8'), accumulated.parsedFiles, paths, ); wroteStore = true; } const keepForMain = accumulated.parsedFiles.some((pf) => pf.filePath.includes('persist-fallback') ); if (wroteStore && !keepForMain) accumulated.parsedFiles = []; } parentPort.postMessage({ type: 'result', data: accumulated }); accumulated = reset(); } }); `, ); }; describe('parse-impl warm-cache ParsedFile coverage (#2038)', () => { let tempDir = ''; let repoDir = ''; let storageDir = ''; let workerPath = ''; let markerPath = ''; beforeEach(() => { tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'warm-cache-coverage-')); repoDir = path.join(tempDir, 'repo'); storageDir = path.join(tempDir, 'storage'); fs.mkdirSync(repoDir, { recursive: true }); fs.mkdirSync(storageDir, { recursive: true }); workerPath = path.join(tempDir, 'store-worker.js'); markerPath = path.join(tempDir, 'worker-spawned.marker'); writeStoreWorker(workerPath, markerPath); }); afterEach(() => { if (tempDir) fs.rmSync(tempDir, { recursive: true, force: true }); prepareOverride.impl = undefined; persistOverride.impl = undefined; }); const writeFile = (rel: string, content: string): { path: string; size: number } => { const full = path.join(repoDir, rel); fs.mkdirSync(path.dirname(full), { recursive: true }); fs.writeFileSync(full, content); return { path: rel, size: fs.statSync(full).size }; }; const newCache = (): ParseCache => ({ version: PARSE_CACHE_VERSION, entries: new Map(), usedKeys: new Set(), storagePath: storageDir, onDiskKeys: new Set(), }); // The post-run orchestrator step (run-analyze) — persist parse cache + prune // the durable store to the surviving keys. Mirrored here so run #2 sees an // index, exactly like a real second invocation. const persistCaches = async (cache: ReturnType): Promise => { const { saveParseCache, pruneCache } = await import('../../src/storage/parse-cache.js'); pruneCache(cache, cache.usedKeys); const saved = await saveParseCache(storageDir, cache); await pruneAndSaveDurableParsedFileStore( getDurableParsedFileDir(storageDir), PARSE_CACHE_VERSION, new Set(saved), ); }; const run = async ( cache: ReturnType, files: { path: string; size: number }[], chunkByteBudget?: number, ): Promise>> => { const rels = files.map((f) => f.path); return runChunkedParseAndResolve( createKnowledgeGraph(), files, rels, files.length, repoDir, Date.now(), () => {}, { workerUrlForTest: pathToFileURL(workerPath), workerPoolSize: 1, parseCache: cache, ...(chunkByteBudget !== undefined ? { chunkByteBudget } : {}), }, ); }; it('run #1 (miss) populates the durable store, keyed by chunk hash', async () => { const f = writeFile('src/cached.ts', 'export function cached() { return 1; }\n'); const chunkHash = computeChunkHash([ { filePath: f.path, contentHash: fileContentHash(fs.readFileSync(path.join(repoDir, f.path), 'utf-8')), }, ]); const cache = newCache(); await run(cache, [f]); expect(fs.existsSync(markerPath)).toBe(true); // worker ran (miss) const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); expect(fs.existsSync(chunkDir)).toBe(true); expect(fs.readdirSync(chunkDir).filter((n) => n.endsWith('.v8')).length).toBeGreaterThan(0); expect(cache.usedKeys.has(chunkHash)).toBe(true); }); it('a failing durable-generation reset degrades instead of failing the analyze', async () => { const f = writeFile('src/degrade.ts', 'export function degrade() { return 1; }\n'); prepareOverride.impl = () => Promise.reject(new Error('EACCES: simulated cache failure')); try { await expect(run(newCache(), [f])).resolves.toBeDefined(); } finally { prepareOverride.impl = undefined; } }); it('does not cache a chunk whose durable generation could not be reset', async () => { // The reset failed, so the previous generation's shards are still on disk. // Caching this chunk would let a future warm hit union those stale shards // with the new ones. Same posture as a quarantined chunk: leave it uncached // so the next run re-dispatches into a directory it can actually clear. const f = writeFile('src/stale-generation.ts', 'export function stale() { return 1; }\n'); const cache = newCache(); prepareOverride.impl = () => Promise.reject(new Error('EACCES: simulated cache failure')); try { await expect(run(cache, [f])).resolves.toBeDefined(); } finally { prepareOverride.impl = undefined; } // Nothing was written under any key -- neither on disk nor in memory. expect(cache.onDiskKeys.size + cache.entries.size).toBe(0); }); // #3204: the chunk above had no prior generation. When one EXISTS, skipping // the write is not enough — `saveParseCache` copies the pre-existing `.v8` // forward from `usedKeys`, and the durable prune then keeps the mixed // directory because it prunes to exactly those saved keys. const seedThenFailReset = async ( rel: string, source: string, ): Promise<{ file: { path: string; size: number }; chunkHash: string; warm: ReturnType; }> => { const file = writeFile(rel, source); const chunkHash = computeChunkHash([ { filePath: file.path, contentHash: fileContentHash(source) }, ]); const cold = newCache(); await run(cold, [file]); // miss → populates the parse cache + durable shards await persistCaches(cold); // Corrupt (do not delete) one durable shard: the coherence gate then // re-dispatches while the previous generation stays on disk, which is the // only way a chunk is both a live `.v8` entry and a miss in one run. const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); const shard = fs.readdirSync(chunkDir).find((name) => name.endsWith('.v8')); if (!shard) throw new Error('expected a durable shard to corrupt'); fs.writeFileSync(path.join(chunkDir, shard), Buffer.from([0, 1, 2])); const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = (await loadParseCache(storageDir)) as ReturnType; expect(warm.onDiskKeys.has(chunkHash)).toBe(true); // the old generation is live fs.rmSync(markerPath, { force: true }); prepareOverride.impl = () => Promise.reject(new Error('EACCES: simulated cache failure')); try { await run(warm, [file]); } finally { prepareOverride.impl = undefined; } expect(fs.existsSync(markerPath)).toBe(true); // the gate did fall through return { file, chunkHash, warm }; }; const readSavedIndexKeys = (): string[] => { const raw = fs.readFileSync(path.join(storageDir, 'parse-cache', 'index.json'), 'utf-8'); return (JSON.parse(raw) as { keys: string[] }).keys; }; it('drops a pre-existing cache entry when the durable generation could not be reset', async () => { const { chunkHash, warm } = await seedThenFailReset( 'src/stale-carryforward.ts', 'export function carried() { return 1; }\n', ); const { saveParseCache, pruneCache } = await import('../../src/storage/parse-cache.js'); pruneCache(warm, warm.usedKeys); const saved = await saveParseCache(storageDir, warm); expect(saved).not.toContain(chunkHash); expect(readSavedIndexKeys()).not.toContain(chunkHash); expect(fs.existsSync(path.join(storageDir, 'parse-cache', `${chunkHash}.v8`))).toBe(false); await pruneAndSaveDurableParsedFileStore( getDurableParsedFileDir(storageDir), PARSE_CACHE_VERSION, new Set(saved), ); const { loadDurableParsedFileIndex } = await import('../../src/storage/parsedfile-store.js'); const durable = await loadDurableParsedFileIndex( getDurableParsedFileDir(storageDir), PARSE_CACHE_VERSION, ); expect(durable.has(chunkHash)).toBe(false); }); it('keeps the chunk excluded when a post-parse merge re-adds its key', async () => { // run-analyze folds sibling-branch keys into usedKeys AFTER the parse phase // (#2106), and retains every loaded key when a sibling meta is unreadable. // Invalidation has to outlive both, which is why it is filtered at save. const { chunkHash, warm } = await seedThenFailReset( 'src/stale-readd.ts', 'export function readded() { return 1; }\n', ); // Model both merges: the sibling fold re-adds the key, and the // unreadable-meta fallback unions `entries.keys()` back into `usedKeys`. // Either one would resurrect the chunk if invalidation were a deletion // from `usedKeys` instead of a filter at save time. warm.usedKeys.add(chunkHash); warm.entries.set(chunkHash, [] as unknown as ParseWorkerResult[]); warm.onDiskKeys?.add(chunkHash); for (const key of warm.entries.keys()) warm.usedKeys.add(key); const { saveParseCache } = await import('../../src/storage/parse-cache.js'); const saved = await saveParseCache(storageDir, warm); expect(saved).not.toContain(chunkHash); expect(readSavedIndexKeys()).not.toContain(chunkHash); }); it('retires the chunk when the failed reset left the old generation on disk', async () => { const { chunkHash, warm } = await seedThenFailReset( 'src/stale-marking-site.ts', 'export function marked() { return 1; }\n', ); expect(warm.staleKeys?.has(chunkHash)).toBe(true); // `onDiskKeys` carried this hash from the loaded index, so clearing it is // observable; `entries` is empty on the sharded path, which is why the // in-memory half is proved by the legacy-cache case below instead. expect(warm.onDiskKeys?.has(chunkHash)).toBe(false); }); it('clears an in-memory entry for a retired chunk', async () => { // `markParseCacheChunkStale` also drops `entries`, which only matters for a // legacy (non-sharded) cache whose payloads live in memory. Drive the // helper directly — the sharded pipeline never populates `entries`, so the // pipeline test above cannot observe this half. const { markParseCacheChunkStale, saveParseCache } = await import('../../src/storage/parse-cache.js'); const cache = newCache(); const chunkHash = 'a'.repeat(64); cache.entries.set(chunkHash, [] as unknown as ParseWorkerResult[]); cache.onDiskKeys?.add(chunkHash); cache.usedKeys.add(chunkHash); markParseCacheChunkStale(cache, chunkHash); expect(cache.entries.has(chunkHash)).toBe(false); expect(cache.onDiskKeys?.has(chunkHash)).toBe(false); expect(await saveParseCache(storageDir, cache)).not.toContain(chunkHash); }); it('does NOT retire a chunk when the failed reset left no old generation', async () => { // `prepareDurableParsedFileChunk` is rm-then-mkdir. When the rm succeeded // and the mkdir failed there is nothing stale to protect against — the // workers recreate the directory and write a clean generation — so // retiring would throw away a good cache entry. Only an rm failure, which // leaves shards behind, justifies retirement. const source = 'export function mkdirOnly() { return 1; }\n'; const file = writeFile('src/mkdir-only.ts', source); const chunkHash = computeChunkHash([ { filePath: file.path, contentHash: fileContentHash(source) }, ]); const cold = newCache(); await run(cold, [file]); await persistCaches(cold); // Corrupt a shard so the coherence gate re-dispatches, then model the // rm-succeeded/mkdir-failed shape: the directory is gone when prepare throws. const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); const shard = fs.readdirSync(chunkDir).find((name) => name.endsWith('.v8')); if (!shard) throw new Error('expected a durable shard to corrupt'); fs.writeFileSync(path.join(chunkDir, shard), Buffer.from([0, 1, 2])); const { loadParseCache, saveParseCache } = await import('../../src/storage/parse-cache.js'); const warm = (await loadParseCache(storageDir)) as ParseCache; prepareOverride.impl = async (durableDir, hash) => { fs.rmSync(path.join(durableDir, hash), { recursive: true, force: true }); throw new Error('EMFILE: simulated mkdir failure after a clean rm'); }; try { await run(warm, [file]); } finally { prepareOverride.impl = undefined; } expect(warm.staleKeys?.has(chunkHash) ?? false).toBe(false); expect(await saveParseCache(storageDir, warm)).toContain(chunkHash); }); it('still saves a chunk whose durable generation reset succeeded', async () => { const f = writeFile('src/healthy.ts', 'export function healthy() { return 1; }\n'); const chunkHash = computeChunkHash([ { filePath: f.path, contentHash: fileContentHash('export function healthy() { return 1; }\n'), }, ]); const cache = newCache(); await run(cache, [f]); const { saveParseCache } = await import('../../src/storage/parse-cache.js'); const saved = await saveParseCache(storageDir, cache); expect(saved).toContain(chunkHash); expect(cache.staleKeys?.has(chunkHash) ?? false).toBe(false); expect(fs.existsSync(path.join(storageDir, 'parse-cache', `${chunkHash}.v8`))).toBe(true); const { loadDurableParsedFileIndex: loadIdx } = await import('../../src/storage/parsedfile-store.js'); await pruneAndSaveDurableParsedFileStore( getDurableParsedFileDir(storageDir), PARSE_CACHE_VERSION, new Set(saved), ); expect( (await loadIdx(getDurableParsedFileDir(storageDir), PARSE_CACHE_VERSION)).has(chunkHash), ).toBe(true); }); it('re-dispatches on the run after a failed reset instead of taking a warm hit', async () => { const { file, chunkHash, warm } = await seedThenFailReset( 'src/stale-nextrun.ts', 'export function nextRun() { return 1; }\n', ); await persistCaches(warm); const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const third = (await loadParseCache(storageDir)) as ReturnType; expect(third.onDiskKeys.has(chunkHash)).toBe(false); fs.rmSync(markerPath, { force: true }); await run(third, [file]); expect(fs.existsSync(markerPath)).toBe(true); }); it('retires only the failing chunk — a sibling chunk stays warm through the next run', async () => { // The single-chunk tests cannot tell "retires the failing chunk" from // "retires everything": one chunk plus a global spawn marker look the same // either way. Two chunks, one failure, and a per-chunk assertion can. const aSrc = 'export function a() { return 1; }\n'; const bSrc = 'export function b() { return 2; }\n'; const a = writeFile('src/a.ts', aSrc); const b = writeFile('src/b.ts', bSrc); const hashOf = (rel: string, src: string): string => computeChunkHash([{ filePath: rel, contentHash: fileContentHash(src) }]); const aHash = hashOf(a.path, aSrc); const bHash = hashOf(b.path, bSrc); // chunkByteBudget 1 forces one file per chunk, so a and b hash distinctly. const cold = newCache(); await run(cold, [a, b], 1); await persistCaches(cold); // Corrupt only a's durable shard: a re-dispatches, b still hits warm. const aDir = path.join(getDurableParsedFileDir(storageDir), aHash); const aShard = fs.readdirSync(aDir).find((name) => name.endsWith('.v8')); if (!aShard) throw new Error('expected a durable shard for a'); fs.writeFileSync(path.join(aDir, aShard), Buffer.from([0, 1, 2])); const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = (await loadParseCache(storageDir)) as ParseCache; prepareOverride.impl = (_durableDir, hash) => hash === aHash ? Promise.reject(new Error('EACCES: simulated cache failure')) : Promise.resolve(); try { await run(warm, [a, b], 1); } finally { prepareOverride.impl = undefined; } await persistCaches(warm); // b survived the save; only a was retired. expect(warm.staleKeys?.has(aHash)).toBe(true); expect(warm.staleKeys?.has(bHash) ?? false).toBe(false); // Actually perform the next run. An index entry alone does not exercise the // warm-hit/coherence path, so the claim in the title has to be paid for. // Run each chunk on its own so the single global spawn marker is // unambiguous about WHICH chunk re-dispatched. const bOnly = (await loadParseCache(storageDir)) as ParseCache; fs.rmSync(markerPath, { force: true }); await run(bOnly, [b], 1); expect(fs.existsSync(markerPath)).toBe(false); // sibling served warm const aOnly = (await loadParseCache(storageDir)) as ParseCache; fs.rmSync(markerPath, { force: true }); await run(aOnly, [a], 1); expect(fs.existsSync(markerPath)).toBe(true); // retired chunk re-parsed }); it('retains worker ParsedFiles when the main-thread run-store write fails', async () => { const f = writeFile( 'src/persist-fallback.ts', 'export function persistFallback() { return 1; }\n', ); persistOverride.impl = () => Promise.resolve(false); const result = await run(newCache(), [f]); expect(result.parsedFiles.map((parsed) => parsed.filePath)).toContain(f.path); }); it('does not snapshot durable shards when the parse-cache payload is missing', async () => { const f = writeFile('src/orphan.ts', 'export function orphan() { return 1; }\n'); const chunkHash = computeChunkHash([ { filePath: f.path, contentHash: fileContentHash(fs.readFileSync(path.join(repoDir, f.path), 'utf-8')), }, ]); const durableDir = getDurableParsedFileDir(storageDir); persistDurableParsedFileShardSync(durableDir, chunkHash, 1, 0, [mkParsedFile(f.path)]); await pruneAndSaveDurableParsedFileStore(durableDir, PARSE_CACHE_VERSION, new Set([chunkHash])); await run(newCache(), [f]); const runShards = fs .readdirSync(getParsedFileStoreDir(storageDir)) .filter((name) => name.endsWith('.v8')); expect(runShards.length).toBeGreaterThan(0); expect(runShards.every((name) => !name.startsWith(chunkHash))).toBe(true); }); it('a repeated cache miss replaces the durable chunk generation', async () => { const f = writeFile('src/repeated.ts', 'export function repeated() { return 1; }\n'); const chunkHash = computeChunkHash([ { filePath: f.path, contentHash: fileContentHash(fs.readFileSync(path.join(repoDir, f.path), 'utf-8')), }, ]); await run(newCache(), [f]); await run(newCache(), [f]); const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); const shards = fs.readdirSync(chunkDir).filter((name) => name.endsWith('.v8')); expect(shards).toHaveLength(1); const shard = shards[0]; if (!shard) throw new Error('expected one durable V8 shard'); const { tryLoadV8Cache } = await import('../../src/storage/v8-sidecar.js'); const hit = await tryLoadV8Cache(path.join(chunkDir, shard)); expect(hit?.kind).toBe('hit'); if (hit?.kind !== 'hit') return; const parsed = hit.value as Array<{ filePath: string }>; expect(parsed.map((item) => item.filePath)).toEqual(['src/repeated.ts']); }); it('run #2 (all hits) spawns NO worker — the warm path is served from caches', async () => { const f = writeFile('src/cached.ts', 'export function cached() { return 1; }\n'); const cache = newCache(); await run(cache, [f]); // miss → populates await persistCaches(cache); // Reload caches from disk for the warm run, like a fresh invocation. const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = await loadParseCache(storageDir); fs.rmSync(markerPath, { force: true }); // reset the spawn marker await run(warm as ReturnType, [f]); expect(fs.existsSync(markerPath)).toBe(false); // NO worker spawned on the warm hit }); it('replays scope-extraction failures from a warm parse-cache hit', async () => { const f = writeFile('src/broken.ts', 'export function broken() { return 1; }\n'); const cache = newCache(); const cold = await run(cache, [f]); expect(cold.scopeExtractionFailures).toEqual([f.path]); await persistCaches(cache); const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = await loadParseCache(storageDir); fs.rmSync(markerPath, { force: true }); const replayed = await run(warm as ReturnType, [f]); expect(fs.existsSync(markerPath)).toBe(false); expect(replayed.scopeExtractionFailures).toEqual([f.path]); }); it('coherence gate: a parse-cache hit with NO durable shards re-dispatches the worker', async () => { const f = writeFile('src/cached.ts', 'export function cached() { return 1; }\n'); const cache = newCache(); await run(cache, [f]); // miss → populates parse cache + durable await persistCaches(cache); // Wipe ONLY the durable store, leaving the parse cache intact — simulates a // first run after the durable store was introduced, or a pruned shard. fs.rmSync(getDurableParsedFileDir(storageDir), { recursive: true, force: true }); const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = await loadParseCache(storageDir); fs.rmSync(markerPath, { force: true }); await run(warm as ReturnType, [f]); // The gate must NOT silently skip — it falls through to a worker re-dispatch // (which repopulates the durable store), never a main-thread re-extract. expect(fs.existsSync(markerPath)).toBe(true); }); it('coherence gate: a parse-cache hit with a corrupt durable shard re-dispatches', async () => { const f = writeFile('src/corrupt.ts', 'export function corrupt() { return 1; }\n'); const cache = newCache(); await run(cache, [f]); await persistCaches(cache); const chunkHash = computeChunkHash([ { filePath: f.path, contentHash: fileContentHash('export function corrupt() { return 1; }\n'), }, ]); const chunkDir = path.join(getDurableParsedFileDir(storageDir), chunkHash); const shard = fs.readdirSync(chunkDir).find((name) => name.endsWith('.v8')); expect(shard).toBeDefined(); if (!shard) return; fs.writeFileSync(path.join(chunkDir, shard), Buffer.from([0, 1, 2])); const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = await loadParseCache(storageDir); fs.rmSync(markerPath, { force: true }); await run(warm as ReturnType, [f]); expect(fs.existsSync(markerPath)).toBe(true); }); it('mixed-mode: changing one file re-parses its chunk while the unchanged chunk restores', async () => { // Force one file per chunk (chunkByteBudget: 1) so a and b hash to DISTINCT // chunks — the true mixed-mode the pr-2038 mixed-mode gap warns about. const a = writeFile('src/a.ts', 'export function a() { return 1; }\n'); const b = writeFile('src/b.ts', 'export function b() { return 2; }\n'); const cache = newCache(); await run(cache, [a, b], 1); // both miss → both durable subdirs populated await persistCaches(cache); const aHash = computeChunkHash([ { filePath: a.path, contentHash: fileContentHash('export function a() { return 1; }\n') }, ]); // run #1 populated a's durable subdir (the chunk that will HIT on run #2). expect(fs.existsSync(path.join(getDurableParsedFileDir(storageDir), aHash))).toBe(true); // Change b's content → b's chunk hash changes → b misses, a still hits. fs.writeFileSync(path.join(repoDir, b.path), 'export function b() { return 999; }\n'); const b2 = { path: b.path, size: fs.statSync(path.join(repoDir, b.path)).size }; const { loadParseCache } = await import('../../src/storage/parse-cache.js'); const warm = await loadParseCache(storageDir); fs.rmSync(markerPath, { force: true }); await run(warm as ReturnType, [a, b2], 1); // The worker spawned (for the changed file b); a was loaded from durable. expect(fs.existsSync(markerPath)).toBe(true); // a's UNCHANGED chunk is still a hit served from the durable store. expect((warm as ReturnType).usedKeys.has(aHash)).toBe(true); }); });