From 22ad5e312dfaf62c92b772f9aa6f3bc124a13645 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 24 Sep 2026 07:43:38 +0000 Subject: [PATCH] feat(analyze): keep one parse cache per shared store (#3352) Linked worktrees read and write the parse cache and durable ParsedFile store under the store's caches/ directory. Before pruning, a run folds in the chunk keys recorded by every member slot and commit graph, and the fold, prune and save run under a store-wide cache lock so one member never evicts another's live chunks. Co-Authored-By: Claude Opus 5.5 (1M context) --- gitnexus/src/core/run-analyze.ts | 36 ++++- gitnexus/src/core/shared-store-analyze.ts | 32 +++- .../integration/shared-store-cache.test.ts | 138 ++++++++++++++++++ 3 files changed, 194 insertions(+), 12 deletions(-) create mode 100644 gitnexus/test/integration/shared-store-cache.test.ts diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index e12262419..3e4a88d83 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -169,8 +169,10 @@ import { resolveSharedStore, type SharedStoreLayout } from '../storage/shared-st import { LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { ensurePrivateSharedGraph, + listStoreMetaRoots, publishSharedGraph, seedSharedSlot, + withStoreLock, } from './shared-store-analyze.js'; import { DEFAULT_PDG_MAX_FUNCTION_LINES } from './ingestion/cfg/collect.js'; import { @@ -1324,6 +1326,9 @@ async function runFullAnalysisInner( // does not own the flat slot. See resolveWriteTarget for the full contract. const { storagePath, repoHasGit, currentCommit, branchLabel, placement, lbugPath, metaDir } = writeTarget; + // Content-addressed caches live once per shared store (#3352), else in the + // flat slot shared by its branch slots (#2106 KTD7). + const cacheRoot = writeTarget.sharedStore?.cachesDir ?? storagePath; let storageWritable: Promise | undefined; const ensureWritableStorage = (): Promise => { storageWritable ??= ensureStoragePathWritable(storagePath); @@ -2425,13 +2430,13 @@ async function runFullAnalysisInner( // after success. Unique because index locks are per branch slot while this // cache root is shared across branches. if (options.useParseCache === false) { - coldParseRebuildDir = await createColdParseRebuildDir(storagePath); + coldParseRebuildDir = await createColdParseRebuildDir(cacheRoot); forgetCreatedParseCacheDir(coldParseRebuildDir); } const parseCache = options.useParseCache === false ? emptyParseCache(coldParseRebuildDir) - : await loadParseCache(storagePath); + : await loadParseCache(cacheRoot); // Streamed structural emit (#2680). Resolved ONCE, so the pipeline flag and // the CSV-dir resolution below cannot disagree — and resolved HERE, not at @@ -4839,15 +4844,25 @@ async function runFullAnalysisInner( // so the cache file size stays bounded across runs (chunks whose // composition no longer matches anything in the current scan are dead // weight; the parse phase populates `usedKeys` as it processes chunks). - try { + const saveCaches = async (): Promise => { // #2106 R6: the parse cache + durable store are shared across branches. // Before pruning to this run's keys, fold in the OTHER branches' recorded // chunk keys so a branch switch doesn't evict their still-live shards. // Adding to usedKeys makes them survive pruneCache AND land in the saved // index (saveParseCache builds the index from usedKeys). Excludes this // run's own meta dir, so a single-branch repo folds in nothing → prune - // set byte-identical to today. - const { keys: siblingKeys, complete } = await collectBranchCacheKeys(storagePath, metaDir); + // set byte-identical to today. A shared store (#3352) folds in every + // member checkout and commit graph the same way. + const keyRoots = writeTarget.sharedStore + ? await listStoreMetaRoots(writeTarget.sharedStore) + : [storagePath]; + const siblingKeys = new Set(); + let complete = true; + for (const root of keyRoots) { + const folded = await collectBranchCacheKeys(root, metaDir); + for (const k of folded.keys) siblingKeys.add(k); + if (!folded.complete) complete = false; + } if (complete) { for (const k of siblingKeys) parseCache.usedKeys.add(k); } else { @@ -4860,7 +4875,7 @@ async function runFullAnalysisInner( if (pruned > 0) { log(`Parse cache: pruned ${pruned} stale chunk entries`); } - const savedKeys = await saveParseCache(storagePath, parseCache); + const savedKeys = await saveParseCache(cacheRoot, parseCache); // Prune the durable ParsedFile store to EXACTLY the parse cache's // surviving keys (#2038 warm-cache coverage), so the two content-addressed // stores stay coherent: a chunk is "cached" iff both its parse-cache shard @@ -4871,11 +4886,16 @@ async function runFullAnalysisInner( // durable-store write must never // break an otherwise successful run (next run treats it as a miss). await mergeStagedDurableParsedFileStore( - storagePath, - parseCache.storagePath ?? storagePath, + cacheRoot, + parseCache.storagePath ?? cacheRoot, PARSE_CACHE_VERSION, new Set(savedKeys), ); + }; + try { + await (writeTarget.sharedStore + ? withStoreLock(writeTarget.sharedStore, 'cache', saveCaches) + : saveCaches()); } catch (e) { log(`Warning: could not save parse cache (${(e as Error).message}); continuing.`); } diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index d273a44a6..e6167caaa 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -250,18 +250,42 @@ export const ensurePrivateSharedGraph = async (slot: string, log: Log): Promise< return true; }; -const withPublishLock = async (layout: SharedStoreLayout, fn: () => Promise): Promise => { - const lockDir = path.join(layout.root, 'locks', 'publish'); +/** + * Serialize one kind of store-wide write (`publish`, `cache`) across + * checkouts. Each checkout's own slot is already covered by its index lock. + */ +export const withStoreLock = async ( + layout: SharedStoreLayout, + name: 'publish' | 'cache', + fn: () => Promise, +): Promise => { + const lockDir = path.join(layout.root, 'locks', name); await fs.mkdir(lockDir, { recursive: true }); const lock = await acquireIndexLock(lockDir); try { - requireExclusiveIndexLock(lock, `Cannot acquire the shared-store publish lock at ${lockDir}.`); + requireExclusiveIndexLock(lock, `Cannot acquire the shared-store ${name} lock at ${lockDir}.`); return await fn(); } finally { lock.release(); } }; +/** + * Every directory in the store whose metadata may record parse-cache keys: + * each checkout slot (its branch slots are read by the caller's per-root + * fold) and each commit graph. + */ +export const listStoreMetaRoots = async (layout: SharedStoreLayout): Promise => { + const roots: string[] = []; + for (const dir of [layout.checkoutsDir, layout.commitsDir]) { + const names = await fs.readdir(dir).catch(() => [] as string[]); + for (const name of names) { + if (!name.startsWith('.')) roots.push(path.join(dir, name)); + } + } + return roots; +}; + /** * After a successful analyze of the flat slot: publish or reuse the commit * graph when the checkout is clean at HEAD, and make sure the registry points @@ -286,7 +310,7 @@ export const publishSharedGraph = async ( if (shareable) { const target = commitGraphDir(layout, currentCommit, featureKeyOf(meta)); const targetGraph = path.join(target, LBUG_DIRECTORY); - const published = await withPublishLock(layout, async () => { + const published = await withStoreLock(layout, 'publish', async () => { if (await exists(targetGraph)) { await wipeLbugDbFiles(own); return true; diff --git a/gitnexus/test/integration/shared-store-cache.test.ts b/gitnexus/test/integration/shared-store-cache.test.ts new file mode 100644 index 000000000..cc1cda64f --- /dev/null +++ b/gitnexus/test/integration/shared-store-cache.test.ts @@ -0,0 +1,138 @@ +import { execFileSync } from 'child_process'; +import { existsSync } from 'fs'; +import fs from 'fs/promises'; +import path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { withStoreLock } from '../../src/core/shared-store-analyze.js'; +import { loadMeta } from '../../src/storage/repo-manager.js'; +import { + resolveSharedStore, + sharedStoreLayout, + type SharedStoreLayout, +} from '../../src/storage/shared-store.js'; +import { createTempDir } from '../helpers/test-db.js'; + +/** + * #3352 U5 — linked worktrees keep one parse cache and ParsedFile store per + * shared store, and one member's prune never evicts chunks another member + * still records. + */ +const git = (cwd: string, ...args: string[]): string => + execFileSync('git', args, { cwd, stdio: 'pipe', encoding: 'utf-8' }).trim(); + +const commitAll = (cwd: string, message: string): void => { + git(cwd, 'add', '-A'); + git(cwd, '-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '-m', message); +}; + +const layoutOf = (checkout: string): SharedStoreLayout => { + const layout = resolveSharedStore(checkout); + expect(layout).not.toBeNull(); + return layout as SharedStoreLayout; +}; + +const indexedCacheKeys = async (layout: SharedStoreLayout): Promise => { + const raw = await fs.readFile(path.join(layout.cachesDir, 'parse-cache', 'index.json'), 'utf-8'); + return (JSON.parse(raw) as { keys: string[] }).keys; +}; + +describe('shared store caches (#3352)', () => { + let tmpHome: Awaited>; + let tmpRepo: Awaited>; + let savedHome: string | undefined; + let wtA: string; + let wtB: string; + + beforeEach(async () => { + tmpHome = await createTempDir('gitnexus-cache-home-'); + tmpRepo = await createTempDir('gitnexus-cache-repo-'); + savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + const root = await fs.realpath(tmpRepo.dbPath); + const main = path.join(root, 'main'); + await fs.mkdir(main); + git(main, 'init', '-q', '-b', 'main'); + await fs.writeFile(path.join(main, 'a.ts'), 'export function alpha() { return 1; }\n'); + await fs.writeFile(path.join(main, 'b.ts'), 'export function beta() { return 2; }\n'); + commitAll(main, 'init'); + wtA = path.join(root, 'wt-a'); + wtB = path.join(root, 'wt-b'); + git(main, 'worktree', 'add', '-q', '-b', 'wt-a', wtA); + git(main, 'worktree', 'add', '-q', '-b', 'wt-b', wtB); + }); + + afterEach(async () => { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpRepo.cleanup(); + await tmpHome.cleanup(); + }); + + it('keeps one cache tree in the store and none in any checkout', async () => { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis(wtA, {}, { onProgress: () => {} }); + await fs.writeFile(path.join(wtB, 'c.ts'), 'export function gamma() { return 3; }\n'); + await runFullAnalysis(wtB, {}, { onProgress: () => {} }); + + const layout = layoutOf(wtA); + expect(existsSync(path.join(layout.cachesDir, 'parse-cache'))).toBe(true); + expect(existsSync(path.join(layout.cachesDir, 'parsedfile-cache'))).toBe(true); + for (const checkout of [wtA, wtB]) { + for (const dir of [layoutOf(checkout).checkoutSlot, path.join(checkout, '.gitnexus')]) { + expect(existsSync(path.join(dir, 'parse-cache'))).toBe(false); + expect(existsSync(path.join(dir, 'parsedfile-cache'))).toBe(false); + } + } + }, 240_000); + + it("keeps chunks another member records when one member's file set changes", async () => { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis(wtA, {}, { onProgress: () => {} }); + const layout = layoutOf(wtA); + const commitDir = (await fs.readdir(layout.commitsDir)).find((n) => !n.startsWith('.')); + const keysA = (await loadMeta(path.join(layout.commitsDir, commitDir as string)))?.cacheKeys; + expect(keysA?.length).toBeGreaterThan(0); + + await fs.rm(path.join(wtB, 'b.ts')); + await runFullAnalysis(wtB, {}, { onProgress: () => {} }); + const keysB = (await loadMeta(layoutOf(wtB).checkoutSlot))?.cacheKeys; + expect(keysB?.length).toBeGreaterThan(0); + expect(keysB).not.toEqual(keysA); + + const indexed = await indexedCacheKeys(layout); + for (const key of [...(keysA ?? []), ...(keysB ?? [])]) expect(indexed).toContain(key); + }, 240_000); +}); + +describe('withStoreLock', () => { + let tmpHome: Awaited>; + let savedHome: string | undefined; + + beforeEach(async () => { + tmpHome = await createTempDir('gitnexus-store-lock-home-'); + savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + }); + + afterEach(async () => { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + }); + + it('runs same-named sections one at a time', async () => { + const layout = sharedStoreLayout('repo-0123456789ab', '/tmp/checkout'); + const events: string[] = []; + const section = (name: string) => async () => { + events.push(`${name}:start`); + await new Promise((r) => setTimeout(r, 50)); + events.push(`${name}:end`); + }; + await Promise.all([ + withStoreLock(layout, 'cache', section('one')), + withStoreLock(layout, 'cache', section('two')), + ]); + expect(events[1]).toBe(`${events[0].split(':')[0]}:end`); + expect(events[3]).toBe(`${events[2].split(':')[0]}:end`); + }); +});