diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 902872b73..e12262419 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1360,9 +1360,11 @@ async function runFullAnalysisInner( // first graph open: here for the paths that open it before the up-to-date // check, and below once that check falls through. const ensurePrivateGraph = async (): Promise => { - if (writeTarget.sharedStore && !placement.branch) { - await ensurePrivateSharedGraph(metaDir, log); - } + if (!writeTarget.sharedStore || placement.branch) return; + if (!(await ensurePrivateSharedGraph(metaDir, log))) options = { ...options, force: true }; + // Later dirty-flag writes spread the in-memory metadata; keep them from + // re-recording the pointer this slot just left. + delete loadedMeta?.graphPath; }; const loadedMeta = await loadMeta(metaDir); if (loadedMeta?.incrementalInProgress || options.repairFts) await ensurePrivateGraph(); @@ -2320,6 +2322,7 @@ async function runFullAnalysisInner( await ensureWritableStorage(); await ensurePrivateGraph(); + delete existingMeta?.graphPath; // ── Cache embeddings from existing index before rebuild ──────────── // Four modes: diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 0d39868b2..d273a44a6 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -28,7 +28,7 @@ import { resolveGraphPath, type SharedStoreLayout, } from '../storage/shared-store.js'; -import { INDEX_METADATA_FILE, LBUG_DIRECTORY } from '../storage/storage-constants.js'; +import { GITNEXUS_DIR, INDEX_METADATA_FILE, LBUG_DIRECTORY } from '../storage/storage-constants.js'; import { wipeLbugDbFiles } from './lbug/lbug-adapter.js'; import { inspectLbugSidecars } from './lbug/sidecar-recovery.js'; @@ -134,9 +134,60 @@ const pickSeed = (repoPath: string, graphs: CommitGraph[]): CommitGraph | null = }; /** - * Give a slot with no metadata a pointer to the best commit graph, so the - * run that follows is up to date (same commit, clean) or incremental from - * that graph's file hashes. Caller holds the slot's index lock. + * Copy a repository-local index (`/.gitnexus`) into an empty slot + * as its private graph. Used before the store has any commit graph: the main + * checkout was indexed before its first worktree existed, or a worktree still + * has its pre-store index. The source is left untouched (R12). Returns false + * when the source is missing, not an ancestor of HEAD, busy, or not + * consolidated. + */ +const seedFromLocalIndex = async ( + slot: string, + repoPath: string, + source: string, + log: Log, +): Promise => { + const sourceGraph = path.join(source, LBUG_DIRECTORY); + const meta = await loadMeta(source); + if (!meta || meta.incrementalInProgress || !meta.lastCommit) return false; + if (!(await exists(sourceGraph))) return false; + if (commitDistanceToHead(repoPath, meta.lastCommit) === null) return false; + let lock; + try { + lock = await acquireIndexLock(source, { timeoutMs: 2_000 }); + } catch { + return false; // another analyze is writing it; seed from scratch instead + } + try { + if (lock.lockFree || (await inspectLbugSidecars(sourceGraph)).kind !== 'clean') return false; + await fs.mkdir(slot, { recursive: true }); + const own = path.join(slot, LBUG_DIRECTORY); + const tmp = `${own}.new.${randomUUID()}`; + try { + await fs.copyFile(sourceGraph, tmp, fsConstants.COPYFILE_FICLONE); + await fs.rename(tmp, own); + } catch (err) { + await fs.rm(tmp, { force: true }).catch(() => {}); + log(`Shared store: could not copy ${sourceGraph} (${(err as Error).message}).`); + return false; + } + } finally { + lock.release(); + } + const seeded: RepoMeta = { ...meta, repoPath, storagePath: slot }; + delete seeded.graphPath; + await saveMeta(slot, seeded); + log(`Shared store: seeded from the local index at ${source}.`); + return true; +}; + +/** + * Seed a slot that has no metadata so the run that follows is up to date or + * incremental instead of a full build. Preference order: + * 1. a pointer to the store's commit graph nearest to HEAD; + * 2. a copy of this checkout's own repository-local index; + * 3. a copy of the main checkout's repository-local index. + * Caller holds the slot's index lock. */ export const seedSharedSlot = async ( layout: SharedStoreLayout, @@ -145,40 +196,58 @@ export const seedSharedSlot = async ( ): Promise => { if (await loadMeta(layout.checkoutSlot)) return; const seed = pickSeed(repoPath, await listCommitGraphs(layout)); - if (!seed) return; - await fs.mkdir(layout.checkoutSlot, { recursive: true }); - const meta: RepoMeta = { - ...seed.meta, - repoPath, - storagePath: layout.checkoutSlot, - graphPath: path.join(seed.dir, LBUG_DIRECTORY), - }; - delete meta.incrementalInProgress; - await saveMeta(layout.checkoutSlot, meta); - log(`Shared store: seeded from commit graph ${seed.commit.slice(0, 12)}.`); + if (seed) { + await fs.mkdir(layout.checkoutSlot, { recursive: true }); + const meta: RepoMeta = { + ...seed.meta, + repoPath, + storagePath: layout.checkoutSlot, + graphPath: path.join(seed.dir, LBUG_DIRECTORY), + }; + delete meta.incrementalInProgress; + await saveMeta(layout.checkoutSlot, meta); + log(`Shared store: seeded from commit graph ${seed.commit.slice(0, 12)}.`); + return; + } + const locals = [repoPath, layout.canonicalCheckout] + .filter((p): p is string => p !== null) + .map((p) => path.join(p, GITNEXUS_DIR)); + for (const source of new Set(locals)) { + if (await seedFromLocalIndex(layout.checkoutSlot, repoPath, source, log)) return; + } }; /** * Turn a pointer slot into a private one before analyze opens or writes the - * graph. No-op for a slot that already owns its graph or has no graph at all. - * Caller holds the slot's index lock. + * graph. Returns false when the pointed-at shared graph cannot be copied + * (garbage-collected or unreadable): the slot's file hashes then describe a + * graph that is not there, and the caller must do a full build. Caller holds + * the slot's index lock. */ -export const ensurePrivateSharedGraph = async (slot: string, log: Log): Promise => { +export const ensurePrivateSharedGraph = async (slot: string, log: Log): Promise => { const own = path.join(slot, LBUG_DIRECTORY); const pointed = resolveGraphPath(slot); - if (pointed === own) return; + if (pointed === own) return true; const meta = await loadMeta(slot); - if (!meta) return; + if (!meta) return true; if (!(await exists(own))) { // `lbug.new.` is swept by the slot lock if this process dies mid-copy. const tmp = `${own}.new.${randomUUID()}`; const started = Date.now(); - await fs.copyFile(pointed, tmp, fsConstants.COPYFILE_FICLONE); - await fs.rename(tmp, own); + try { + await fs.copyFile(pointed, tmp, fsConstants.COPYFILE_FICLONE); + await fs.rename(tmp, own); + } catch (err) { + await fs.rm(tmp, { force: true }).catch(() => {}); + const reason = (err as NodeJS.ErrnoException).code ?? (err as Error).message; + log(`Shared store: shared graph unavailable (${reason}); doing a full build.`); + return false; + } log(`Shared store: copied the shared graph for local changes in ${Date.now() - started}ms.`); } delete meta.graphPath; await saveMeta(slot, meta); + return true; }; const withPublishLock = async (layout: SharedStoreLayout, fn: () => Promise): Promise => { diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index 8e778df53..0391c4c15 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -55,6 +55,8 @@ export interface SharedStoreLayout { checkoutsDir: string; /** This checkout's slot — the registry `storagePath` for a shared checkout. */ checkoutSlot: string; + /** The main checkout (parent of the git common dir); null for a bare repository. */ + canonicalCheckout: string | null; } /** Sharing is off globally, or an explicit storage env override takes precedence. */ @@ -111,21 +113,30 @@ const readCommonDir = (checkoutPath: string): string | null => { * Store key for a checkout, or null when the checkout does not share. * Main checkout and every linked worktree of one repository get the same key. */ -export const resolveSharedStoreKey = ( +const resolveIdentity = ( checkoutPath: string, - env: NodeJS.ProcessEnv = process.env, -): string | null => { + env: NodeJS.ProcessEnv, +): { key: string; canonicalCheckout: string | null } | null => { if (isSharedStoreDisabled(env)) return null; const commonDir = readCommonDir(path.resolve(checkoutPath)); if (!commonDir || !hasLinkedWorktrees(commonDir)) return null; // `/.git` keys on `` for a readable name; a bare common dir // (`repo.git`) keys on itself. Both hash the canonical absolute path. - const identity = path.basename(commonDir) === '.git' ? path.dirname(commonDir) : commonDir; - return slotName(identity); + const canonicalCheckout = path.basename(commonDir) === '.git' ? path.dirname(commonDir) : null; + return { key: slotName(canonicalCheckout ?? commonDir), canonicalCheckout }; }; +export const resolveSharedStoreKey = ( + checkoutPath: string, + env: NodeJS.ProcessEnv = process.env, +): string | null => resolveIdentity(checkoutPath, env)?.key ?? null; + /** Name every store path for `checkoutPath` under store `key`. */ -export const sharedStoreLayout = (key: string, checkoutPath: string): SharedStoreLayout => { +export const sharedStoreLayout = ( + key: string, + checkoutPath: string, + canonicalCheckout: string | null = null, +): SharedStoreLayout => { const storesRoot = path.join(getGlobalDir(), STORES_DIR); const root = path.resolve(storesRoot, key); if (path.dirname(root) !== path.resolve(storesRoot)) { @@ -139,6 +150,7 @@ export const sharedStoreLayout = (key: string, checkoutPath: string): SharedStor commitsDir: path.join(root, 'commits'), checkoutsDir, checkoutSlot: path.join(checkoutsDir, slotName(checkoutPath)), + canonicalCheckout, }; }; @@ -147,8 +159,10 @@ export const resolveSharedStore = ( checkoutPath: string, env: NodeJS.ProcessEnv = process.env, ): SharedStoreLayout | null => { - const key = resolveSharedStoreKey(checkoutPath, env); - return key ? sharedStoreLayout(key, checkoutPath) : null; + const identity = resolveIdentity(checkoutPath, env); + return identity + ? sharedStoreLayout(identity.key, checkoutPath, identity.canonicalCheckout) + : null; }; /** Directory of the immutable graph for one commit and feature key. */ diff --git a/gitnexus/test/integration/shared-store-seed.test.ts b/gitnexus/test/integration/shared-store-seed.test.ts new file mode 100644 index 000000000..7f712bd89 --- /dev/null +++ b/gitnexus/test/integration/shared-store-seed.test.ts @@ -0,0 +1,252 @@ +import { execFileSync } from 'child_process'; +import { existsSync } from 'fs'; +import fs from 'fs/promises'; +import path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ensurePrivateSharedGraph } from '../../src/core/shared-store-analyze.js'; +import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { resolveSharedStore, type SharedStoreLayout } from '../../src/storage/shared-store.js'; +import { createTempDir } from '../helpers/test-db.js'; + +/** + * #3352 U4 — a checkout that needs its own graph is seeded from the nearest + * commit graph and updated incrementally instead of rebuilt from scratch. + */ +const git = (cwd: string, ...args: string[]): string => + execFileSync('git', args, { cwd, stdio: 'pipe', encoding: 'utf-8' }).trim(); + +const commitAll = (cwd: string, message: string): void => { + git(cwd, 'add', '-A'); + git(cwd, '-c', 'user.name=t', '-c', 'user.email=t@t', 'commit', '-q', '-m', message); +}; + +const layoutOf = (checkout: string): SharedStoreLayout => { + const layout = resolveSharedStore(checkout); + expect(layout).not.toBeNull(); + return layout as SharedStoreLayout; +}; + +const queryNames = async (graph: string): Promise => { + const lbug = (await import('@ladybugdb/core')).default; + const db = new lbug.Database(graph, 0, true, true); + const conn = new lbug.Connection(db); + try { + const rows = (await ( + await conn.query('MATCH (f:Function) RETURN f.name AS n ORDER BY n') + ).getAll()) as { n: string }[]; + return rows.map((r) => r.n); + } finally { + await conn.close(); + await db.close(); + } +}; + +const graphOf = (checkout: string): string => + getStoragePaths(checkout, undefined, layoutOf(checkout).checkoutSlot).lbugPath; + +describe('shared store seeding (#3352)', () => { + let tmpHome: Awaited>; + let tmpRepo: Awaited>; + let savedHome: string | undefined; + let root: string; + let main: string; + + const analyze = async (checkout: string, logs?: string[]) => { + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + return runFullAnalysis(checkout, {}, { onProgress: () => {}, onLog: (m) => logs?.push(m) }); + }; + + const addWorktree = (name: string, base = 'main'): string => { + const wt = path.join(root, name); + git(main, 'worktree', 'add', '-q', '-b', name, wt, base); + return wt; + }; + + beforeEach(async () => { + tmpHome = await createTempDir('gitnexus-seed-home-'); + tmpRepo = await createTempDir('gitnexus-seed-repo-'); + savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + root = await fs.realpath(tmpRepo.dbPath); + main = path.join(root, 'main'); + await fs.mkdir(main); + git(main, 'init', '-q', '-b', 'main'); + await fs.writeFile(path.join(main, 'a.ts'), 'export function alpha() { return 1; }\n'); + await fs.writeFile(path.join(main, 'b.ts'), 'export function beta() { return 2; }\n'); + commitAll(main, 'init'); + }); + + afterEach(async () => { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpRepo.cleanup(); + await tmpHome.cleanup(); + }); + + it('Covers AE2: an edited worktree gets a private incremental graph; siblings are unchanged', async () => { + const wtA = addWorktree('wt-a'); + const wtB = addWorktree('wt-b'); + await analyze(main); + await analyze(wtA); + await analyze(wtB); + const shared = graphOf(wtB); + expect(graphOf(wtA)).toBe(shared); + + await fs.writeFile(path.join(wtA, 'a.ts'), 'export function alphaEdited() { return 1; }\n'); + const logs: string[] = []; + await analyze(wtA, logs); + + expect(graphOf(wtA)).toBe(path.join(layoutOf(wtA).checkoutSlot, 'lbug')); + expect(logs.some((m) => m.startsWith('Incremental:'))).toBe(true); + expect(await queryNames(graphOf(wtA))).toEqual(['alphaEdited', 'beta']); + expect(graphOf(wtB)).toBe(shared); + expect(await queryNames(shared)).toEqual(['alpha', 'beta']); + }, 240_000); + + it('Covers AE3: committing the edits publishes a commit graph and drops the private one', async () => { + const wtA = addWorktree('wt-a'); + await analyze(wtA); + await fs.writeFile(path.join(wtA, 'c.ts'), 'export function gamma() { return 3; }\n'); + await analyze(wtA); + const slot = layoutOf(wtA).checkoutSlot; + expect(existsSync(path.join(slot, 'lbug'))).toBe(true); + + commitAll(wtA, 'gamma'); + await analyze(wtA); + + expect(existsSync(path.join(slot, 'lbug'))).toBe(false); + expect(path.dirname(path.dirname(graphOf(wtA)))).toBe(layoutOf(wtA).commitsDir); + expect(await queryNames(graphOf(wtA))).toEqual(['alpha', 'beta', 'gamma']); + }, 240_000); + + it('seeds a new worktree at a descendant commit from its indexed ancestor', async () => { + await analyze(main); + git(main, 'checkout', '-q', '-b', 'next'); + await fs.writeFile(path.join(main, 'c.ts'), 'export function gamma() { return 3; }\n'); + commitAll(main, 'gamma'); + git(main, 'checkout', '-q', 'main'); + const wt = addWorktree('wt-next', 'next'); + + const logs: string[] = []; + await analyze(wt, logs); + + // `main` was indexed before any worktree existed, so its index is + // repository-local; the first worktree seeds from that copy. + expect(logs).toContain( + `Shared store: seeded from the local index at ${path.join(main, '.gitnexus')}.`, + ); + expect(logs.some((m) => m.startsWith('Incremental:'))).toBe(true); + expect(await queryNames(graphOf(wt))).toEqual(['alpha', 'beta', 'gamma']); + // The source index is left in place. + expect(existsSync(path.join(main, '.gitnexus', 'lbug'))).toBe(true); + }, 240_000); + + it('seeds a descendant worktree from the store commit graph once one exists', async () => { + const wtA = addWorktree('wt-a'); + await analyze(wtA); + git(main, 'checkout', '-q', '-b', 'next'); + await fs.writeFile(path.join(main, 'c.ts'), 'export function gamma() { return 3; }\n'); + commitAll(main, 'gamma'); + git(main, 'checkout', '-q', 'main'); + const wt = addWorktree('wt-next', 'next'); + + const logs: string[] = []; + await analyze(wt, logs); + + const base = git(main, 'rev-parse', 'main'); + expect(logs).toContain(`Shared store: seeded from commit graph ${base.slice(0, 12)}.`); + expect(logs.some((m) => m.startsWith('Incremental:'))).toBe(true); + expect(await queryNames(graphOf(wt))).toEqual(['alpha', 'beta', 'gamma']); + }, 240_000); + + it('runs a full build for a worktree with no indexed ancestor', async () => { + await analyze(main); + const orphan = path.join(root, 'orphan'); + git(main, 'worktree', 'add', '-q', '--detach', orphan); + git(orphan, 'checkout', '-q', '--orphan', 'unrelated'); + git(orphan, 'rm', '-q', '-rf', '.'); + await fs.writeFile(path.join(orphan, 'z.ts'), 'export function zeta() { return 0; }\n'); + commitAll(orphan, 'unrelated root'); + + const logs: string[] = []; + await analyze(orphan, logs); + + expect(logs.some((m) => m.startsWith('Shared store: seeded'))).toBe(false); + expect(logs.some((m) => m.startsWith('Incremental:'))).toBe(false); + expect(await queryNames(graphOf(orphan))).toEqual(['zeta']); + }, 240_000); + + it('matches a from-scratch build after seeding and updating (R9)', async () => { + const wt = addWorktree('wt-a'); + await analyze(main); + await fs.writeFile(path.join(wt, 'b.ts'), 'export function betaTwo() { return 22; }\n'); + await fs.writeFile(path.join(wt, 'c.ts'), 'export function gamma() { return 3; }\n'); + await analyze(wt); + const seeded = await queryNames(graphOf(wt)); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis(wt, { force: true }, { onProgress: () => {} }); + expect(await queryNames(graphOf(wt))).toEqual(seeded); + expect(seeded).toEqual(['alpha', 'betaTwo', 'gamma']); + }, 240_000); +}); + +describe('ensurePrivateSharedGraph', () => { + let tmpHome: Awaited>; + let savedHome: string | undefined; + + beforeEach(async () => { + tmpHome = await createTempDir('gitnexus-private-home-'); + savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + }); + + afterEach(async () => { + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + }); + + const pointerSlot = async (): Promise<{ slot: string; graph: string }> => { + const storeRoot = path.join(tmpHome.dbPath, 'stores', 'repo-0123456789ab'); + const slot = path.join(storeRoot, 'checkouts', 'wt-0123456789ab'); + const graph = path.join(storeRoot, 'commits', 'abc1234-deadbeefdeadbeef', 'lbug'); + await fs.mkdir(path.dirname(graph), { recursive: true }); + await fs.mkdir(slot, { recursive: true }); + await saveMeta(slot, { + repoPath: '/tmp/wt', + storagePath: slot, + lastCommit: 'abc1234', + indexedAt: new Date().toISOString(), + graphPath: graph, + }); + return { slot, graph }; + }; + + it('copies the shared graph and clears the pointer', async () => { + const { slot, graph } = await pointerSlot(); + await fs.writeFile(graph, 'shared graph bytes'); + expect(await ensurePrivateSharedGraph(slot, () => {})).toBe(true); + expect(await fs.readFile(path.join(slot, 'lbug'), 'utf-8')).toBe('shared graph bytes'); + expect((await loadMeta(slot))?.graphPath).toBeUndefined(); + expect(await fs.readFile(graph, 'utf-8')).toBe('shared graph bytes'); + }); + + it('reports an unusable baseline when the shared graph is gone', async () => { + const { slot } = await pointerSlot(); + const logs: string[] = []; + expect(await ensurePrivateSharedGraph(slot, (m) => logs.push(m))).toBe(false); + expect(existsSync(path.join(slot, 'lbug'))).toBe(false); + expect(logs.join('\n')).toMatch(/shared graph unavailable \(ENOENT\)/); + expect((await fs.readdir(slot)).filter((n) => n.startsWith('lbug'))).toEqual([]); + }); + + it('is a no-op for a slot that already owns its graph', async () => { + const { slot } = await pointerSlot(); + const meta = await loadMeta(slot); + delete meta?.graphPath; + await saveMeta(slot, meta as NonNullable); + expect(await ensurePrivateSharedGraph(slot, () => {})).toBe(true); + expect(existsSync(path.join(slot, 'lbug'))).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/storage/shared-store.test.ts b/gitnexus/test/unit/storage/shared-store.test.ts index 360f3226f..ec3565e67 100644 --- a/gitnexus/test/unit/storage/shared-store.test.ts +++ b/gitnexus/test/unit/storage/shared-store.test.ts @@ -155,6 +155,7 @@ describe('sharedStoreLayout', () => { commitsDir: path.join(root, 'commits'), checkoutsDir: path.join(root, 'checkouts'), checkoutSlot: path.join(root, 'checkouts', storageSlotName(wts[0])), + canonicalCheckout: main, }); });