diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index bf8a5e381..8a13abd94 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -491,9 +491,22 @@ CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) └── meta.json # legacy mirror of gitnexus.json, kept in sync (see MIGRATION.md) ~/.gitnexus/ - └── registry.json # Global repo registry (MCP discovery) + ├── registry.json # Global repo registry (MCP discovery) + └── stores// # Shared sibling index store (see below) + ├── caches/ # parse cache + durable ParsedFile store + ├── commits/-/ # one immutable graph per commit + settings + └── checkouts// # one checkout's metadata, membership, and + # private graph when it has local edits ``` +The flat `/.gitnexus/` layout applies to a standalone repository and +whenever `GITNEXUS_STORAGE_PATH` / `GITNEXUS_STORAGE_ROOT` is set. A repository +with linked worktrees, and clones with the same `origin` URL, share one +`stores//` automatically (a clone opts out with `analyze --no-share`; +`GITNEXUS_SHARED_STORE=off` turns sharing off entirely). Each sharing checkout +keeps only a `.gitnexus/store.json` pointer to its store. Path resolution lives +in `shared-store.ts`. + Read-only opens self-heal an interrupted checkpoint: the refusal is classified and cleared by one writable open (probe + `CHECKPOINT`) before the read-only open is retried — see `sidecar-recovery.ts` diff --git a/README.md b/README.md index a4422c949..bd426b15f 100644 --- a/README.md +++ b/README.md @@ -628,7 +628,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_FTS_STEMMER` | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` for matching repository comments. Re-run `gitnexus analyze --repair-fts` after changing it. | Keyword search quality is poor for non-English comments or identifiers under English stemming. | | `GITNEXUS_STORAGE_PATH` | unset (`/.gitnexus/`) | Complete external index directory. This preserves the existing configuration semantics and takes precedence over `GITNEXUS_STORAGE_ROOT` when both are set. | You already keep one repository index outside its checkout or need one explicit index location. | | `GITNEXUS_STORAGE_ROOT` | unset | Absolute root directory for external indexes. GitNexus creates an isolated `-/` slot beneath it for each repository, then registers the resolved slot so `status`, MCP, and `serve` can reopen it later. | You want to manage multiple repository indexes centrally or keep generated data outside source checkouts. | -| `GITNEXUS_SHARED_STORE` | unset (on) | Set to `off` to stop linked git worktrees from sharing one index store; every checkout then indexes into its own `.gitnexus/`. Sharing is also off whenever `GITNEXUS_STORAGE_PATH` or `GITNEXUS_STORAGE_ROOT` is set. | Disk or memory is not a concern, or you want each worktree's index fully independent. | +| `GITNEXUS_SHARED_STORE` | unset (on) | Set to `off` (or `0`, `false`, `no`) to turn off shared index stores for both linked git worktrees and sibling clones; every checkout then indexes into its own `.gitnexus/`. Sharing is also off whenever `GITNEXUS_STORAGE_PATH` or `GITNEXUS_STORAGE_ROOT` is set. | Disk or memory is not a concern, or you want each worktree's index fully independent. | | `GITNEXUS_CONTENT_RETENTION` | `full` | Source-text retention profile: `full` keeps file and symbol text, `symbol` keeps symbol snippets without full file content, and `none` keeps the structural graph without source body text. | You need to reduce persisted source text while preserving graph structure. | | `GITNEXUS_SKIP_FTS` | unset | When exactly `1`, skips FTS extension loading and keyword index creation during analyze. Equivalent to `--skip-fts`; a later analyze without either option restores FTS. | Graph-only consumers with their own retrieval, or short-lived indexes that do not need keyword search. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold in bytes. Equivalent to `--wal-checkpoint-threshold `. `-1` keeps LadybugDB's stock threshold (~16 MiB). Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | You need a larger or smaller WAL auto-checkpoint threshold for your analyze workload. | @@ -710,7 +710,7 @@ GitNexus uses a **global registry** so one MCP server can serve multiple indexed Each `gitnexus analyze` stores the index in `.gitnexus/` inside the repo by default (portable, gitignored). `GITNEXUS_STORAGE_PATH` selects one complete external index directory and preserves the established configuration behavior. To manage multiple repositories under one external directory, set `GITNEXUS_STORAGE_ROOT`; GitNexus derives an isolated `-/` slot beneath it for each repository. If both variables are set, `GITNEXUS_STORAGE_PATH` takes precedence. GitNexus registers the resolved slot in `~/.gitnexus/registry.json`, allowing later `status`, MCP, and `serve` commands to reopen the index without repeating the environment variable. LadybugDB connections are opened lazily on first query and evicted after 5 minutes of inactivity (max 5 concurrent). Read-only tools can omit `repo` when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout. Outside those paths—and for mutating tools with multiple indexed repos and no MCP default—pass `repo` explicitly. -**Worktrees share one index store.** When a repository has linked worktrees (`git worktree add`), the main checkout and every worktree index into one store at `~/.gitnexus/stores//` instead of each keeping a full `.gitnexus/`. Checkouts at the same commit with no local changes read one shared, read-only graph: one copy on disk and one open database in MCP. A checkout with uncommitted changes gets its own graph, copied from the nearest shared graph and updated incrementally rather than rebuilt. Parse caches are shared too. Each worktree keeps a small `.gitnexus/store.json` pointer, and an index it had before sharing is left in place; `gitnexus status` reports it and `gitnexus clean --local-index --force` removes it. `gitnexus clean` in one worktree removes only that worktree's slot and any shared graph no other checkout uses; `gitnexus clean --gc` also drops slots whose worktree was deleted. Independent clones of one repository share too: when another registered clone has the same `origin` URL, `gitnexus analyze` in a clone joins that clone's store (or starts one the other clone joins on its next analyze). A lone clone keeps its own `.gitnexus/`. `gitnexus analyze --share-with ` joins a specific checkout's store after checking the `origin` URLs match, and `--no-share` moves a clone back to its own `.gitnexus/` and keeps it out until `--share-with`. On filesystems with copy-on-write clones (APFS, btrfs, XFS) a checkout's private graph shares its unchanged pages with the shared graph on disk; elsewhere it is a full copy, and `gitnexus status` says which. Queries cannot combine two graphs, because LadybugDB reads one database per query, so a checkout with edits always has a complete graph of its own. Set `GITNEXUS_SHARED_STORE=off` to turn sharing off. +**Worktrees share one index store.** When a repository has linked worktrees (`git worktree add`), the main checkout and every worktree index into one store at `~/.gitnexus/stores//` instead of each keeping a full `.gitnexus/`. Checkouts at the same commit with no local changes read one shared, read-only graph: one copy on disk and one open database in MCP. A checkout with uncommitted changes gets its own graph, copied from the nearest shared graph and updated incrementally rather than rebuilt. Parse caches are shared too. Each worktree keeps a small `.gitnexus/store.json` pointer, and an index it had before sharing is left in place; `gitnexus status` reports it and `gitnexus clean --local-index --force` removes it. `gitnexus clean` in one worktree removes only that worktree's slot and any shared graph no other checkout uses; `gitnexus clean --gc` also drops slots whose worktree was deleted. Independent clones of one repository share too: when another registered clone has the same `origin` URL, `gitnexus analyze` in a clone joins that clone's store (or starts one the other clone joins on its next analyze). A lone clone keeps its own `.gitnexus/`. `gitnexus analyze --share-with ` joins a specific checkout's store after checking the `origin` URLs match, and `--no-share` moves a clone back to its own `.gitnexus/` and keeps it out until `--share-with`. On filesystems with copy-on-write clones (APFS, btrfs, XFS) a checkout's private graph shares its unchanged pages with the shared graph on disk; elsewhere it is a full copy, and `gitnexus status` says which. Queries cannot combine two graphs, because LadybugDB reads one database per query, so a checkout with edits always has a complete graph of its own. Set `GITNEXUS_SHARED_STORE=off` (or `0`, `false`, `no`) to turn sharing off for worktrees and clones alike.
Architecture diagram diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index d1801011d..ddec290ce 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -71,7 +71,7 @@ export const en = { 'clean.shared.reclaimed': 'Shared store: removed {{count}} commit graph(s) no checkout references.', 'clean.shared.kept': - 'Shared store: kept {{count}} unreferenced commit graph(s) that are still open; run `gitnexus clean --gc` later.', + 'Shared store: kept {{count}} unreferenced commit graph(s) that could not be removed (in use or not writable); run `gitnexus clean --gc` later.', 'clean.shared.storeRemoved': 'Shared store: removed {{path}} (no checkouts remain).', 'clean.gc.none': 'No shared stores to collect.', 'clean.gc.keptMembers': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 05eec8322..50f1b8eb1 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -67,7 +67,7 @@ export const zhCN = { 'clean.notFoundHere': '当前目录未找到已索引仓库。', 'clean.shared.reclaimed': '共享存储：已删除 {{count}} 个不再被任何检出引用的提交图。', 'clean.shared.kept': - '共享存储：保留了 {{count}} 个仍处于打开状态的未引用提交图；请稍后运行 `gitnexus clean --gc`。', + '共享存储：保留了 {{count}} 个无法删除的未引用提交图（正在使用或不可写）；请稍后运行 `gitnexus clean --gc`。', 'clean.shared.storeRemoved': '共享存储：已删除 {{path}}（没有剩余检出）。', 'clean.gc.none': '没有可回收的共享存储。', 'clean.gc.keptMembers': diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 1126a3f65..99c7c3da5 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -4927,12 +4927,14 @@ async function runFullAnalysisInner( // run's own meta dir, so a single-branch repo folds in nothing → prune // set byte-identical to today. A shared store (#3352) folds in every // member checkout and commit graph the same way. - const keyRoots = writeTarget.sharedStore + // An unlistable store directory starts the fold incomplete, so the + // retention branch below keeps other slots' chunks. + const listing = writeTarget.sharedStore ? await listStoreMetaRoots(writeTarget.sharedStore) - : [storagePath]; + : { roots: [storagePath], complete: true }; const siblingKeys = new Set(); - let complete = true; - for (const root of keyRoots) { + let complete = listing.complete; + for (const root of listing.roots) { const folded = await collectBranchCacheKeys(root, metaDir); for (const k of folded.keys) siblingKeys.add(k); if (!folded.complete) complete = false; diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index 98d7e49ee..fefa62afa 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -36,7 +36,7 @@ import { saveMeta, type RegistryEntry, } from '../storage/repo-manager.js'; -import { loadMeta, type RepoMeta } from '../storage/repo-meta.js'; +import { isMissingFilesystemError, loadMeta, type RepoMeta } from '../storage/repo-meta.js'; import { cloneStoreKey, commitGraphDir, @@ -337,17 +337,29 @@ export const ensurePrivateSharedGraph = async ( /** * Every directory in the store whose metadata may record parse-cache keys: * each checkout slot (its branch slots are read by the caller's per-root - * fold) and each commit graph. + * fold) and each commit graph. `complete` is false when a store directory + * exists but could not be listed: the caller must then keep every cached + * chunk instead of pruning to a partial key set. A missing directory just + * has no members, so the listing stays complete. */ -export const listStoreMetaRoots = async (layout: SharedStoreLayout): Promise => { +export const listStoreMetaRoots = async ( + layout: SharedStoreLayout, +): Promise<{ roots: string[]; complete: boolean }> => { const roots: string[] = []; + let complete = true; for (const dir of [layout.checkoutsDir, layout.commitsDir]) { - const names = await fs.readdir(dir).catch(() => [] as string[]); + let names: string[]; + try { + names = await fs.readdir(dir); + } catch (err) { + if (!isMissingFilesystemError(err)) complete = false; + continue; + } for (const name of names) { if (!name.startsWith('.')) roots.push(path.join(dir, name)); } } - return roots; + return { roots, complete }; }; /** diff --git a/gitnexus/src/storage/index-lock.ts b/gitnexus/src/storage/index-lock.ts index 19997379a..9ba7cd898 100644 --- a/gitnexus/src/storage/index-lock.ts +++ b/gitnexus/src/storage/index-lock.ts @@ -152,6 +152,11 @@ export interface AcquireOptions { pollMs?: number; /** Called once when we start waiting on a live holder. */ onWaitStart?: (holder: LockRecord) => void; + /** + * Sweep orphaned staging files once the lock is held. Default true; pass + * false for a read-only caller, such as a dry run, that must delete nothing. + */ + sweep?: boolean; } export class IndexLockTimeoutError extends Error { @@ -907,7 +912,7 @@ export const acquireIndexLock = async ( } // Without ownership, a staging file may belong to an active writer. try { - if (!handle.lockFree) sweepStagingArtifacts(lockDir, opts.log); + if (!handle.lockFree && opts.sweep !== false) sweepStagingArtifacts(lockDir, opts.log); } catch { /* best-effort */ } diff --git a/gitnexus/src/storage/shared-store-lifecycle.ts b/gitnexus/src/storage/shared-store-lifecycle.ts index 605647bd7..081441cc7 100644 --- a/gitnexus/src/storage/shared-store-lifecycle.ts +++ b/gitnexus/src/storage/shared-store-lifecycle.ts @@ -168,7 +168,8 @@ export const reclaimSharedStoreLocked = async ( // whose lock is free, so the preview matches what --force would do. let lock: IndexLockHandle; try { - lock = await acquireIndexLock(slot, { timeoutMs: 1 }); + // A preview deletes nothing, not even the staging files the lock sweeps. + lock = await acquireIndexLock(slot, { timeoutMs: 1, sweep: !opts.dryRun }); } catch { orphans.delete(slot); continue; @@ -439,7 +440,18 @@ export const removeCheckoutStorage = async ( requireExclusiveIndexLock(lock, `Cannot acquire the index lock at ${storagePath}.`); await unregister(); if (checkoutPath) await removeSharedStorePointer(checkoutPath); - await fs.rm(storagePath, { recursive: true, force: true }); + try { + await fs.rm(storagePath, { recursive: true, force: true }); + } catch (err) { + // The checkout is already unregistered, so a plain `clean` can no longer + // find this slot. It is now an orphan member, which `clean --gc` removes. + const reason = err instanceof Error ? err.message : String(err); + throw new Error( + `The checkout was unregistered, but its index storage at ${storagePath} could not be deleted (${reason}). ` + + 'Run `gitnexus clean --gc --force` to remove it.', + { cause: err }, + ); + } } finally { lock.release(); } diff --git a/gitnexus/test/integration/shared-store-analyze.test.ts b/gitnexus/test/integration/shared-store-analyze.test.ts index 22d144ac2..b42c5dd6d 100644 --- a/gitnexus/test/integration/shared-store-analyze.test.ts +++ b/gitnexus/test/integration/shared-store-analyze.test.ts @@ -11,6 +11,7 @@ import { SCHEMA_FINGERPRINT } from '../../src/core/lbug/schema.js'; import { ensurePrivateSharedGraph, featureKeyOf, + listStoreMetaRoots, publishSharedGraph, seedSharedSlot, } from '../../src/core/shared-store-analyze.js'; @@ -182,11 +183,15 @@ describe('shared sibling store analyze (#3352)', () => { const graph = getStoragePaths(wtB, undefined, layoutOf(wtB).checkoutSlot).lbugPath; const db = new lbug.Database(graph, 0, true, true); const conn = new lbug.Connection(db); - const rows = (await ( - await conn.query('MATCH (f:File) RETURN f.filePath AS p ORDER BY p') - ).getAll()) as { p: string }[]; - await conn.close(); - await db.close(); + let rows: { p: string }[]; + try { + rows = (await ( + await conn.query('MATCH (f:File) RETURN f.filePath AS p ORDER BY p') + ).getAll()) as { p: string }[]; + } finally { + await conn.close(); + await db.close(); + } expect(rows.map((r) => r.p)).toEqual(['a.ts']); }, 180_000); @@ -584,3 +589,57 @@ describe('up-to-date fast path over a missing shared graph (#3374)', () => { expect(existsSync(resolveGraphPath(slot))).toBe(true); }, 120_000); }); + +describe('listStoreMetaRoots', () => { + let tmp: Awaited>; + let layout: SharedStoreLayout; + + beforeEach(async () => { + tmp = await createTempDir('gitnexus-test-store-roots-'); + const root = path.join(tmp.dbPath, 'store'); + layout = { + key: 'repo-0000', + root, + cachesDir: path.join(root, 'caches'), + commitsDir: path.join(root, 'commits'), + checkoutsDir: path.join(root, 'checkouts'), + checkoutSlot: path.join(root, 'checkouts', 'slot-a'), + canonicalCheckout: null, + }; + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await tmp.cleanup(); + }); + + it('stays complete when the store directories do not exist yet', async () => { + expect(await listStoreMetaRoots(layout)).toEqual({ roots: [], complete: true }); + }); + + it('lists checkout slots and commit graphs, skipping dot entries', async () => { + await fs.mkdir(path.join(layout.checkoutsDir, 'slot-a'), { recursive: true }); + await fs.mkdir(path.join(layout.checkoutsDir, '.lock'), { recursive: true }); + await fs.mkdir(path.join(layout.commitsDir, 'abc'), { recursive: true }); + expect(await listStoreMetaRoots(layout)).toEqual({ + roots: [path.join(layout.checkoutsDir, 'slot-a'), path.join(layout.commitsDir, 'abc')], + complete: true, + }); + }); + + it('reports an incomplete listing when a store directory cannot be read', async () => { + await fs.mkdir(path.join(layout.commitsDir, 'abc'), { recursive: true }); + await fs.mkdir(layout.checkoutsDir, { recursive: true }); + const realReaddir = fs.readdir; + vi.spyOn(fs, 'readdir').mockImplementation((async (dir: string) => { + if (dir === layout.checkoutsDir) { + throw Object.assign(new Error('permission denied'), { code: 'EACCES' }); + } + return realReaddir(dir); + }) as unknown as typeof fs.readdir); + expect(await listStoreMetaRoots(layout)).toEqual({ + roots: [path.join(layout.commitsDir, 'abc')], + complete: false, + }); + }); +}); diff --git a/gitnexus/test/integration/shared-store-clean.test.ts b/gitnexus/test/integration/shared-store-clean.test.ts index 2872c069a..aa3444311 100644 --- a/gitnexus/test/integration/shared-store-clean.test.ts +++ b/gitnexus/test/integration/shared-store-clean.test.ts @@ -3,7 +3,14 @@ import { constants as fsConstants, existsSync, readFileSync } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; -import { getStoragePaths, loadMeta, saveMeta } from '../../src/storage/repo-manager.js'; +import { + getStoragePaths, + loadMeta, + readRegistry, + registerRepo, + saveMeta, + unregisterRepo, +} from '../../src/storage/repo-manager.js'; import { resolveSharedStore, sharedStoreLayout, @@ -14,6 +21,7 @@ import { reclaimAfterSlotRemoval, readGraphCloneKind, reclaimSharedStore, + removeCheckoutStorage, removeLegacyLocalIndex, removeSharedStorePointer, writeSharedStorePointer, @@ -352,6 +360,48 @@ describe('reclaimSharedStore', () => { expect(existsSync(slot)).toBe(true); }); + it("clean --gc preview leaves an orphan slot's staging files in place", async () => { + const slot = await member('gone-000000000000', { repoPath: '/nonexistent/checkout' }); + const staging = path.join(slot, 'lbug.staging.x'); + await fs.writeFile(staging, 'partial'); + const preview = await reclaimSharedStore(layout().root, { gc: true, dryRun: true }); + expect(preview.droppedMembers).toEqual([slot]); + expect(existsSync(staging)).toBe(true); + }); + + it('names the leftover slot and clean --gc when the final slot deletion fails', async () => { + const checkout = path.join(tmpHome.dbPath, 'checkout'); + await fs.mkdir(checkout); + const slot = await member('wt-000000000000', { repoPath: checkout }); + await registerRepo( + checkout, + { repoPath: checkout, storagePath: slot, lastCommit: '', indexedAt: '' }, + { storagePath: slot }, + ); + const realRm = fs.rm; + const rm = vi.spyOn(fs, 'rm').mockImplementation((async ( + target: string, + ...rest: unknown[] + ) => { + if (String(target) === slot) throw Object.assign(new Error('busy'), { code: 'EBUSY' }); + return (realRm as (...a: unknown[]) => Promise)(target, ...rest); + }) as typeof fs.rm); + try { + await expect( + removeCheckoutStorage(slot, () => unregisterRepo(checkout), checkout), + ).rejects.toThrow(/was unregistered.*gitnexus clean --gc --force/s); + } finally { + rm.mockRestore(); + } + expect(await readRegistry()).toEqual([]); + expect(existsSync(slot)).toBe(true); + + // The leftover is now an orphan member, which `clean --gc` collects. + const result = await reclaimSharedStore(layout().root, { gc: true }); + expect(result.droppedMembers).toEqual([slot]); + expect(existsSync(slot)).toBe(false); + }); + it('counts references correctly when GITNEXUS_HOME is relative', async () => { const absoluteHome = process.env.GITNEXUS_HOME as string; process.env.GITNEXUS_HOME = path.relative(process.cwd(), absoluteHome); diff --git a/gitnexus/test/unit/git-utils.test.ts b/gitnexus/test/unit/git-utils.test.ts index ccbb1d06d..ac7286d18 100644 --- a/gitnexus/test/unit/git-utils.test.ts +++ b/gitnexus/test/unit/git-utils.test.ts @@ -1142,7 +1142,7 @@ describe('listWorkingTreeDirtyPaths', () => { // that hides committed content (sparse checkout, index bits, an uninitialized // submodule) must never become the commit graph other checkouts reuse. -/** A repo with one committed file, `a.ts`. */ +/** A repo with two committed files, `a.ts` and `lib/b.ts`. */ function makeCommittedRepo(): string { const repo = makeIsolatedGitRepo(); fs.writeFileSync(path.join(repo, 'a.ts'), 'export const a = 1;');