/** * U20 — Integration regression test for chunk-cache corruption on * worker quarantine. * * Pins the fix for the Codex adversarial review finding on PR #1693 * (`docs/plans/2026-05-20-002-fix-chunk-cache-corruption-on-worker-quarantine-plan.md`): * * The chunk hash is computed from every file in the chunk, but the * worker pool's Layer 3 quarantine filters quarantined files out of * dispatch. Before the fix, the chunk-loop would cache the partial * worker results under the full-coverage chunk hash, locking in * silent corruption that the next analyze would replay. * * Runs with REAL `worker_threads` + `createWorkerPool`. Injects a * custom worker script via `workerUrlForTest` that: * 1. Implements the U17/U19 IPC protocol (decode Buffer or hybrid * envelope/contents shape, decode header + JSON payload). * 2. Emits a `{type:'ready'}` handshake so the pool's * `waitForWorkerReady` resolves promptly. * 3. On a sub-batch containing `poison.ts`, emits a starting-file * and exits with code 134 — deterministic worker death the pool * attributes to `poison.ts` via the in-flight signal, then * adds to its session-scoped quarantine. * 4. On a sub-batch without poison, synthesizes a minimal valid * ParseWorkerResult with a Function node per file (no * tree-sitter dependency in the test worker — the synthesized * nodes give the merge step deterministic content to add to the * graph). * * U20 design pivot — no sequential fallback. The U1 sequential * reparse for quarantined chunk files was removed: relying on the * worker pool's resilience layers (respawn budget, circuit breaker, * quarantine, slot-attribution, cumulative timeout) as the SOLE * contract avoids re-triggering tree-sitter native crashes on the * main thread and gives operators a clear hard signal when workers * exhaust. Quarantined files are missing from this run's graph; * they're surfaced in the per-chunk warn log; U2's cache-skip keeps * the chunk uncached so the next analyze with a fresh pool retries. * * Assertions exercised here: * - Worker-path runs and produces results for surviving files * (good_a, good_c) via the synthesized worker output. * - The quarantined file (poison.ts) is NOT in the graph — no * sequential reparse fired. * - U2 (cache-write suppression): `parseCache.entries` does NOT * contain the chunk hash after the run. `parseCache.usedKeys` * DOES contain it (chunk was processed; the cache write was * specifically skipped). A cross-run scenario verifies that a * subsequent dispatch with a fresh pool re-attempts the chunk * (cache miss) and the cache stays empty for that chunk. * * Why integration over unit: * - The fix lives at the boundary between processParsing * (`parsing-processor.ts`) and the chunk-loop * (`pipeline-phases/parse-impl.ts`) under a real * workerPool. Unit-mocking the worker-pool import bypasses the * structured-clone boundary, the dispatch lifecycle, and the * actual quarantine flow — it verifies the test setup rather * than the contract. The real worker thread executing the test * script through the U17/U19 IPC protocol IS the load-bearing * surface; this test exercises it end-to-end. * - The `writeReadyWorker` pattern from `worker-pool.test.ts` is * reused inline here (the READY_PREAMBLE + test-worker script * composition). * * Wall-clock budget: well under 5 s under normal CI conditions. */ import { describe, it, expect, afterEach, beforeEach } from 'vitest'; import { tmpdir } from 'node:os'; import { mkdtempSync, writeFileSync, mkdirSync, rmSync, statSync } from 'node:fs'; import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import { runChunkedParseAndResolve } from '../../src/core/ingestion/pipeline-phases/parse-impl.js'; import { computeChunkHash, fileContentHash, packParseCacheChunks, } from '../../src/storage/parse-cache.js'; import type { ParseCache } from '../../src/storage/parse-cache.js'; import type { ParseWorkerResult } from '../../src/core/ingestion/workers/parse-worker.js'; /** * Inline READY preamble + IPC decode wrapper (mirrors * `test/integration/worker-pool.test.ts`'s READY_PREAMBLE). Lets the * test worker script below speak the production U17/U19 IPC protocol * without importing dist/protocol.js (the script runs as a standalone * CJS file at a temp path, so it can't resolve dist/ via relative * paths reliably). */ const READY_PREAMBLE = ` const { parentPort: __pp } = require('node:worker_threads'); const __decoder = new TextDecoder('utf-8'); const __decodeFrame = (raw) => { if ( raw && typeof raw === 'object' && raw.type === 'sub-batch' && Array.isArray(raw.files) ) { return { type: 'sub-batch', files: raw.files.map((f) => ({ path: f.path, content: typeof f.content === 'string' ? f.content : __decoder.decode(f.content), })), }; } return raw; }; const __origOn = __pp.on.bind(__pp); __pp.on = (event, handler) => { if (event !== 'message') return __origOn(event, handler); return __origOn(event, (raw) => handler(__decodeFrame(raw))); }; __pp.postMessage({ type: 'ready' }); `; /** * Test worker script. Synthesizes minimal ParseWorkerResult entries for * non-poison files; deterministically crashes on poison.ts via * `process.exit(134)`. Accumulates across sub-batches; emits the * accumulated result on `flush`. */ const TEST_WORKER_SCRIPT = ` const { parentPort } = require('node:worker_threads'); const accumulated = { nodes: [], relationships: [], symbols: [], imports: [], calls: [], assignments: [], heritage: [], routes: [], fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], routerIncludes: [], routerImports: [], toolDefs: [], ormQueries: [], constructorBindings: [], fileScopeBindings: [], parsedFiles: [], skippedLanguages: {}, fileCount: 0, }; parentPort.on('message', (msg) => { if (msg && msg.type === 'sub-batch') { const poison = msg.files.find((f) => f.path.endsWith('poison.ts')); if (poison) { parentPort.postMessage({ type: 'starting-file', path: poison.path }); process.exit(134); } for (const file of msg.files) { const baseName = file.path.split('/').pop().replace(/\\.ts$/, ''); accumulated.nodes.push({ id: 'func:' + file.path, label: 'Function', properties: { name: baseName, filePath: file.path, startLine: 1, endLine: 1, language: 'typescript', isExported: true, }, }); accumulated.fileCount++; } parentPort.postMessage({ type: 'progress', filesProcessed: accumulated.fileCount }); parentPort.postMessage({ type: 'sub-batch-done' }); return; } if (msg && msg.type === 'flush') { parentPort.postMessage({ type: 'result', data: accumulated }); } }); `; const FIXTURE_FILES = { 'src/good_a.ts': 'export function good_a() { return 1; }\n', 'src/poison.ts': 'export function poison() { return 2; }\n', 'src/good_c.ts': 'export function good_c() { return 3; }\n', }; const POISON_PATH = 'src/poison.ts'; const DEFAULT_TEST_CHUNK_BUDGET = 2 * 1024 * 1024; const resolveTestChunkByteBudget = (): number => { const env = Number(process.env.GITNEXUS_CHUNK_BYTE_BUDGET); if (Number.isFinite(env) && env > 0) return env; return DEFAULT_TEST_CHUNK_BUDGET; }; const hashPacks = ( scanned: { path: string; size: number }[], ): { poison: string; others: string[] } => { const packs = packParseCacheChunks( scanned.map((file) => ({ path: file.path, size: file.size, language: 'typescript', })), resolveTestChunkByteBudget(), ); const hashOf = (pack: string[]) => computeChunkHash( pack.map((p) => ({ filePath: p, contentHash: fileContentHash(FIXTURE_FILES[p as keyof typeof FIXTURE_FILES]), })), ); const poisonPack = packs.find((paths) => paths.includes(POISON_PATH)); if (!poisonPack) throw new Error('poison.ts was not packed'); return { poison: hashOf(poisonPack), others: packs.filter((paths) => !paths.includes(POISON_PATH)).map(hashOf), }; }; describe('U20: parse-impl quarantine + chunk-cache integration (PR #1693 Codex finding)', () => { let tempDir: string; let repoDir: string; let workerPath: string; beforeEach(() => { tempDir = mkdtempSync(path.join(tmpdir(), 'parse-impl-quarantine-cache-skip-')); repoDir = path.join(tempDir, 'repo'); mkdirSync(repoDir, { recursive: true }); // Write the fixture files to repoDir so filesystem-walker / chunk // loop pick them up by relative path. for (const [rel, content] of Object.entries(FIXTURE_FILES)) { const full = path.join(repoDir, rel); mkdirSync(path.dirname(full), { recursive: true }); writeFileSync(full, content); } // Write the test worker script to the same tempDir so it doesn't // collide with anything else. The READY preamble + test script // share one .js file the pool spawns via `new Worker(URL)`. workerPath = path.join(tempDir, 'test-quarantine-worker.js'); writeFileSync(workerPath, READY_PREAMBLE + TEST_WORKER_SCRIPT); }); afterEach(() => { rmSync(tempDir, { recursive: true, force: true }); }); it('worker quarantine leaves poison.ts out of the graph AND suppresses chunk-cache write', async () => { const filePaths = Object.keys(FIXTURE_FILES); const scanned = filePaths.map((rel) => ({ path: rel, size: statSync(path.join(repoDir, rel)).size, })); const expectedChunkHash = hashPacks(scanned).poison; const parseCache: ParseCache = { version: 'test', entries: new Map(), usedKeys: new Set(), }; const graph = createKnowledgeGraph(); await runChunkedParseAndResolve( graph, scanned, filePaths, filePaths.length, repoDir, Date.now(), () => {}, { skipWorkers: false, // Force the worker-pool gate to open on the 3-file fixture. // Inject the custom worker script — the pool will spawn it // instead of the production parse-worker.js. workerUrlForTest: pathToFileURL(workerPath) as URL, // Test-only worker pool size — keep at 1 so the poison-file // sub-batch deterministically lands on the only slot (no // chance of poison + good landing in different slots). workerPoolSize: 1, parseCache, }, ); const nodes = Array.from(graph.nodes.values()); // Quarantine contract: poison.ts is genuinely missing from the // graph for this run. The custom worker crashed on it; no // sequential reparse rescued it; the operator sees the per-chunk // quarantine warn log. A future analyze with a fresh pool gets // another chance via U2's cache-skip below. expect( nodes.some( (n) => n.label === 'Function' && (n.properties as { name?: string }).name === 'poison', ), ).toBe(false); // Surviving files' symbols come from the custom worker's // synthesized output via the normal worker-path merge. Pinning // them here catches a regression that would drop worker results // entirely when quarantine fires. expect( nodes.some( (n) => n.label === 'Function' && (n.properties as { name?: string }).name === 'good_a', ), ).toBe(true); expect( nodes.some( (n) => n.label === 'Function' && (n.properties as { name?: string }).name === 'good_c', ), ).toBe(true); // U2 assertion: chunk-cache write was suppressed. The chunk hash // is in usedKeys (chunk WAS processed) but absent from entries // (cache write skipped because of the quarantine intersection). // This is the load-bearing cross-run protection: a future analyze // with unchanged content will re-derive the same chunkHash, miss // the cache, and re-dispatch — giving the file another chance // against a fresh-quarantine pool. expect(parseCache.entries.has(expectedChunkHash)).toBe(false); expect(parseCache.usedKeys.has(expectedChunkHash)).toBe(true); // #3204: `usedKeys` alone would let `saveParseCache` copy a pre-existing // shard forward, so the skipped chunk is also retired from the save. expect(parseCache.staleKeys?.has(expectedChunkHash)).toBe(true); for (const hash of hashPacks(scanned).others) { expect(parseCache.entries.has(hash)).toBe(true); } }); it('cross-run: unchanged fixture re-dispatches the poison pack because that pack was not cached', async () => { // First pass: same setup as the previous test. The poison pack is not // cached; other packs may be. const filePaths = Object.keys(FIXTURE_FILES); const scanned = filePaths.map((rel) => ({ path: rel, size: statSync(path.join(repoDir, rel)).size, })); const expectedChunkHash = hashPacks(scanned).poison; const parseCache: ParseCache = { version: 'test', entries: new Map(), usedKeys: new Set(), }; // FIRST PASS. { const graph = createKnowledgeGraph(); await runChunkedParseAndResolve( graph, scanned, filePaths, filePaths.length, repoDir, Date.now(), () => {}, { skipWorkers: false, workerUrlForTest: pathToFileURL(workerPath) as URL, workerPoolSize: 1, parseCache, }, ); // Confirm the precondition for the second-pass test: cache is // empty for this chunk hash. expect(parseCache.entries.has(expectedChunkHash)).toBe(false); } // SECOND PASS — same content, same parseCache, fresh worker pool // (createWorkerPool is called per `runChunkedParseAndResolve`, so // every invocation gets a clean quarantine slate). With the cache // empty for this chunk, the second pass MUST dispatch the chunk // again rather than replaying a cache entry. The custom worker // crashes again on poison.ts → quarantine again → cache still // skipped. Symptom: cache state unchanged, graph still complete. { const graph2 = createKnowledgeGraph(); await runChunkedParseAndResolve( graph2, scanned, filePaths, filePaths.length, repoDir, Date.now(), () => {}, { skipWorkers: false, workerUrlForTest: pathToFileURL(workerPath) as URL, workerPoolSize: 1, parseCache, }, ); // Cache stayed empty (still no entry for this chunk hash) — the // load-bearing cross-run protection. expect(parseCache.entries.has(expectedChunkHash)).toBe(false); expect(parseCache.usedKeys.has(expectedChunkHash)).toBe(true); for (const hash of hashPacks(scanned).others) { expect(parseCache.entries.has(hash)).toBe(true); } // Worker path ran again; surviving files in the graph; poison // still absent per the U20 contract (workers are the sole // resilience layer, no sequential reparse). const nodes2 = Array.from(graph2.nodes.values()); expect( nodes2.some( (n) => n.label === 'Function' && (n.properties as { name?: string }).name === 'good_a', ), ).toBe(true); expect( nodes2.some( (n) => n.label === 'Function' && (n.properties as { name?: string }).name === 'poison', ), ).toBe(false); } }); });