From 263ca353a6b60d6e9bf1b8c695f184603247a486 Mon Sep 17 00:00:00 2001 From: BlackOvOoo <168417937+BlackOvOoo@users.noreply.github.com> Date: Sat, 16 May 2026 14:19:06 +0800 Subject: [PATCH] fix: shard parse cache persistence on large repos (#1580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: shard parse cache persistence on large repos * fix(parse-cache): validate shard keys, docs, and sharded-cache tests - Reject non-sha256-hex keys from index.json before path.join (path traversal). - saveParseCache: skip invalid keys defensively; try/catch per-shard JSON.stringify. - Clarify save comment (tmp dir + rename vs atomic). - Tests: hex keys throughout, traversal keys, multi-shard, version-mismatch+legacy, second save, legacy removal. - AGENTS.md / GUARDRAILS.md: document .gitnexus/parse-cache/ vs legacy parse-cache.json. Co-authored-by: Cursor * chore(autofix): apply prettier + eslint fixes via /autofix command --------- Co-authored-by: Gergő Magyar Co-authored-by: Cursor Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- AGENTS.md | 4 +- GUARDRAILS.md | 2 +- gitnexus/src/storage/parse-cache.ts | 124 +++++++++-- .../test/unit/incremental-parse-cache.test.ts | 204 +++++++++++++++++- 4 files changed, 310 insertions(+), 24 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b9b9138b1..99fc68bd0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,9 +155,9 @@ npx gitnexus analyze --embeddings # also generate embeddings for new/changed npx gitnexus analyze --drop-embeddings # explicit opt-in to wipe existing embeddings ``` -`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** at `.gitnexus/parse-cache.json` for chunks whose file contents haven't changed since the last run. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). +`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** under `.gitnexus/parse-cache/` (per-chunk JSON shards plus `index.json`) for chunks whose file contents haven't changed since the last run. Older installs may still have a legacy single file `.gitnexus/parse-cache.json`, which is read for backward compatibility but no longer written. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). -The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete `.gitnexus/parse-cache.json` at any time — it'll be rebuilt on the next analyze. +The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete the whole `.gitnexus/parse-cache/` directory (and remove any legacy `.gitnexus/parse-cache.json` if present) at any time — it'll be rebuilt on the next analyze. Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). A plain `analyze` no longer drops existing vectors — pass `--drop-embeddings` to wipe. diff --git a/GUARDRAILS.md b/GUARDRAILS.md index c09f0319a..2e2b9db41 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -36,7 +36,7 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Index seems corrupt or "incremental" is misbehaving - **Trigger:** `analyze` produces unexpected results, or `meta.json.incrementalInProgress` is set, or the index is in a half-state after a crash. -- **Do:** `npx gitnexus analyze --force` to rebuild from scratch. The dirty-flag check forces this automatically when a previous incremental run didn't complete cleanly, but `--force` is the manual escape hatch. Safe to delete `.gitnexus/parse-cache.json` at any time — content-addressed, will be regenerated. +- **Do:** `npx gitnexus analyze --force` to rebuild from scratch. The dirty-flag check forces this automatically when a previous incremental run didn't complete cleanly, but `--force` is the manual escape hatch. Safe to delete the `.gitnexus/parse-cache/` directory (and any legacy `.gitnexus/parse-cache.json`) at any time — content-addressed, will be regenerated. - **Why:** Incremental writeback is selective DB row replacement; if the on-disk state is inconsistent for any reason, a full rebuild is the cheapest path back to a known-good index. ### Embeddings vanished after analyze diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index a1abf76fa..680ca1ab2 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -70,15 +70,28 @@ const GITNEXUS_PKG_VERSION = (() => { })(); export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`; -const CACHE_FILENAME = 'parse-cache.json'; +const LEGACY_CACHE_FILENAME = 'parse-cache.json'; +const CACHE_DIRNAME = 'parse-cache'; +const CACHE_INDEX_FILENAME = 'index.json'; -/** On-disk shape. */ +/** Keys on disk always come from `computeChunkHash` — 64-char lowercase hex. */ +const CHUNK_CACHE_KEY_HEX_RE = /^[a-f0-9]{64}$/; + +const isValidChunkCacheKey = (chunkHash: string): boolean => CHUNK_CACHE_KEY_HEX_RE.test(chunkHash); + +/** On-disk shape for the legacy single-file format. */ interface ParseCacheFile { version: string; /** key = chunk hash (hex) → cached chunk result list. */ entries: Record; } +/** On-disk shape for the sharded directory format. */ +interface ShardedParseCacheIndex { + version: string; + keys: string[]; +} + /** Runtime view: keyed Map for fast lookup; mutated in place during a run. */ export interface ParseCache { version: string; @@ -144,12 +157,19 @@ const mapReviver = (_key: string, value: unknown): unknown => { return value; }; -/** - * Load the parse cache. Returns an empty cache on any failure (missing - * file, corrupt JSON, version mismatch). Never throws on a normal load. - */ -export const loadParseCache = async (storagePath: string): Promise => { - const cachePath = path.join(storagePath, CACHE_FILENAME); +const getLegacyCachePath = (storagePath: string): string => + path.join(storagePath, LEGACY_CACHE_FILENAME); + +const getCacheDirPath = (storagePath: string): string => path.join(storagePath, CACHE_DIRNAME); + +const getCacheIndexPath = (storagePath: string): string => + path.join(getCacheDirPath(storagePath), CACHE_INDEX_FILENAME); + +const getCacheChunkPath = (storagePath: string, chunkHash: string): string => + path.join(getCacheDirPath(storagePath), `${chunkHash}.json`); + +const loadLegacyParseCache = async (storagePath: string): Promise => { + const cachePath = getLegacyCachePath(storagePath); try { const raw = await fs.readFile(cachePath, 'utf-8'); const data = JSON.parse(raw, mapReviver) as ParseCacheFile; @@ -172,22 +192,90 @@ export const loadParseCache = async (storagePath: string): Promise = } }; +const loadShardedParseCache = async (storagePath: string): Promise => { + const indexPath = getCacheIndexPath(storagePath); + try { + const raw = await fs.readFile(indexPath, 'utf-8'); + const data = JSON.parse(raw) as ShardedParseCacheIndex; + if ( + typeof data !== 'object' || + data === null || + data.version !== PARSE_CACHE_VERSION || + !Array.isArray(data.keys) + ) { + return emptyCache(); + } + + const entries = new Map(); + for (const chunkHash of data.keys) { + if (typeof chunkHash !== 'string' || !isValidChunkCacheKey(chunkHash)) continue; + try { + const chunkRaw = await fs.readFile(getCacheChunkPath(storagePath, chunkHash), 'utf-8'); + const chunkData = JSON.parse(chunkRaw, mapReviver) as ParseWorkerResult[]; + if (Array.isArray(chunkData)) entries.set(chunkHash, chunkData); + } catch { + /* skip corrupt or missing shard */ + } + } + + return { version: PARSE_CACHE_VERSION, entries, usedKeys: new Set() }; + } catch { + return null; + } +}; + /** - * Persist the cache to disk atomically (write-and-rename) so a crash - * mid-write doesn't leave a corrupt file. + * Load the parse cache. Returns an empty cache on any failure (missing + * file, corrupt JSON, version mismatch). Never throws on a normal load. + */ +export const loadParseCache = async (storagePath: string): Promise => { + const sharded = await loadShardedParseCache(storagePath); + if (sharded) return sharded; + return loadLegacyParseCache(storagePath); +}; + +/** + * Persist the cache to disk using a temp directory + rename. + * + * Writes shards under `${cacheDir}.tmp`, then removes the old `cacheDir` and + * renames the temp directory into place. There is a crash window after + * `rm(cacheDir)` and before `rename(tmpDir, cacheDir)` where no cache exists; + * that is acceptable — `loadParseCache` yields empty and the next run + * reparses. This is not a single atomic swap of the whole tree, but avoids + * leaving a half-written shard set visible to readers. */ export const saveParseCache = async (storagePath: string, cache: ParseCache): Promise => { await fs.mkdir(storagePath, { recursive: true }); - const cachePath = path.join(storagePath, CACHE_FILENAME); - const tmpPath = `${cachePath}.tmp`; - const out: ParseCacheFile = { + const cacheDir = getCacheDirPath(storagePath); + const tmpDir = `${cacheDir}.tmp`; + await fs.rm(tmpDir, { recursive: true, force: true }); + await fs.mkdir(tmpDir, { recursive: true }); + + const keys: string[] = []; + for (const [chunkHash, chunkResults] of cache.entries) { + if (!isValidChunkCacheKey(chunkHash)) continue; + let payload: string; + try { + payload = JSON.stringify(chunkResults, mapReplacer); + } catch { + // Extremely dense chunks could theoretically exceed string limits; skip + // rather than failing the entire save (orchestrator catches save errors). + continue; + } + keys.push(chunkHash); + const chunkPath = path.join(tmpDir, `${chunkHash}.json`); + await fs.writeFile(chunkPath, payload, 'utf-8'); + } + + const index: ShardedParseCacheIndex = { version: cache.version, - entries: Object.fromEntries(cache.entries), + keys, }; - // Compact JSON; this file can be tens of MB on a large repo and pretty- - // printing roughly doubles size for no value. - await fs.writeFile(tmpPath, JSON.stringify(out, mapReplacer), 'utf-8'); - await fs.rename(tmpPath, cachePath); + await fs.writeFile(path.join(tmpDir, CACHE_INDEX_FILENAME), JSON.stringify(index), 'utf-8'); + + await fs.rm(cacheDir, { recursive: true, force: true }); + await fs.rename(tmpDir, cacheDir); + await fs.rm(getLegacyCachePath(storagePath), { force: true }); }; /** diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index 757b9cf3a..17ec9c2e1 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -135,12 +135,15 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { it('round-trips an empty cache', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { + const fs = await import('fs/promises'); const cache: ParseCache = { version: PARSE_CACHE_VERSION, entries: new Map(), usedKeys: new Set(), }; await saveParseCache(dir, cache); + await expect(fs.access(path.join(dir, 'parse-cache', 'index.json'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(dir, 'parse-cache.json'))).rejects.toThrow(); const loaded = await loadParseCache(dir); expect(loaded.version).toBe(PARSE_CACHE_VERSION); expect(loaded.entries.size).toBe(0); @@ -189,6 +192,60 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { } }); + it('loads a legacy single-file cache for backwards compatibility', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { + legacyChunk: [minimalResult({ fileCount: 7 })], + }, + }), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get('legacyChunk')?.[0]?.fileCount).toBe(7); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('skips corrupt or missing shards while loading the sharded cache', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + const goodKey = 'a'.repeat(64); + const missingKey = 'b'.repeat(64); + const badKey = 'c'.repeat(64); + await fs.mkdir(cacheDir, { recursive: true }); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + keys: [goodKey, missingKey, badKey], + }), + 'utf-8', + ); + await fs.writeFile( + path.join(cacheDir, `${goodKey}.json`), + JSON.stringify([minimalResult({ fileCount: 3 })]), + 'utf-8', + ); + await fs.writeFile(path.join(cacheDir, `${badKey}.json`), '{not-json', 'utf-8'); + + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(goodKey)?.[0]?.fileCount).toBe(3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('round-trips Map and Set values through the JSON replacer/reviver', async () => { // ParsedFile.scopes[*].typeBindings is a ReadonlyMap. // Without the replacer/reviver pair, JSON.stringify collapses Maps to @@ -196,6 +253,7 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { // with "is not iterable". This test pins the round-trip behaviour. const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { + const fs = await import('fs/promises'); const innerMap = new Map([ ['k1', 'v1'], ['k2', 'v2'], @@ -218,14 +276,18 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { ], }); + const chunkKey = 'd'.repeat(64); const cache: ParseCache = { version: PARSE_CACHE_VERSION, - entries: new Map([['chunk-h', [fake]]]), - usedKeys: new Set(['chunk-h']), + entries: new Map([[chunkKey, [fake]]]), + usedKeys: new Set([chunkKey]), }; await saveParseCache(dir, cache); + const persisted = await fs.readdir(path.join(dir, 'parse-cache')); + expect(persisted).toContain('index.json'); + expect(persisted).toContain(`${chunkKey}.json`); const loaded = await loadParseCache(dir); - const reloaded = loaded.entries.get('chunk-h')?.[0]; + const reloaded = loaded.entries.get(chunkKey)?.[0]; expect(reloaded).toBeDefined(); const scope = (reloaded as ParseWorkerResult).parsedFiles[0]?.scopes[0] as unknown as { typeBindings?: unknown; @@ -240,4 +302,140 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { await rm(dir, { recursive: true, force: true }); } }); + + it('ignores traversal-like and non-hex keys in sharded index.json', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + await fs.mkdir(cacheDir, { recursive: true }); + const safeKey = 'e'.repeat(64); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + keys: ['../evil', '/absolute', 'G'.repeat(64), safeKey], + }), + 'utf-8', + ); + await fs.writeFile( + path.join(cacheDir, `${safeKey}.json`), + JSON.stringify([minimalResult({ fileCount: 9 })]), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(safeKey)?.[0]?.fileCount).toBe(9); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('writes one shard file per cache entry (three distinct keys)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const k1 = '1'.repeat(64); + const k2 = '2'.repeat(64); + const k3 = '3'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map([ + [k1, [minimalResult({ fileCount: 1 })]], + [k2, [minimalResult({ fileCount: 2 })]], + [k3, [minimalResult({ fileCount: 3 })]], + ]), + usedKeys: new Set([k1, k2, k3]), + }; + await saveParseCache(dir, cache); + const cacheDir = path.join(dir, 'parse-cache'); + const names = await fs.readdir(cacheDir); + expect(names).toContain('index.json'); + expect(names.filter((n) => n.endsWith('.json') && n !== 'index.json').length).toBe(3); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('returns empty when sharded index version mismatches even if legacy parse-cache.json is valid', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + await fs.mkdir(cacheDir, { recursive: true }); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ version: 'foreign-sharded-1', keys: [] }), + 'utf-8', + ); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { legacyChunk: [minimalResult({ fileCount: 42 })] }, + }), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('second saveParseCache replaces the first sharded cache', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const k1 = '4'.repeat(64); + const k2 = '5'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k1, [minimalResult()]]]), + usedKeys: new Set([k1]), + }); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k2, [minimalResult({ fileCount: 99 })]]]), + usedKeys: new Set([k2]), + }); + const names = await fs.readdir(path.join(dir, 'parse-cache')); + expect(names).not.toContain(`${k1}.json`); + expect(names).toContain(`${k2}.json`); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(k2)?.[0]?.fileCount).toBe(99); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('removes legacy parse-cache.json after a successful sharded save', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { oldLegacy: [minimalResult({ fileCount: 5 })] }, + }), + 'utf-8', + ); + const k = '6'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k, [minimalResult({ fileCount: 6 })]]]), + usedKeys: new Set([k]), + }); + await expect(fs.access(path.join(dir, 'parse-cache.json'))).rejects.toThrow(); + const loaded = await loadParseCache(dir); + expect(loaded.entries.get(k)?.[0]?.fileCount).toBe(6); + expect(loaded.entries.has('oldLegacy')).toBe(false); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); });