From 6ad8d7d38929fb493d6393a14f0fcf977b6cf333 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Thu, 24 Sep 2026 21:43:48 +0000 Subject: [PATCH] feat(storage): share the index store between clones automatically (#3352) Clones of one repository now share like linked worktrees. A clone whose normalized origin URL matches another registered, still-present clone joins that clone's store, or founds one (keyed on its own path) that the sibling joins on its next analyze. A lone clone keeps its own .gitnexus. Graphs stay keyed by commit and feature key, so clones only ever share a graph built from the same commit with the same settings. `analyze --no-share` now records a lasting opt-out (`shareOptOut` on the registry entry, preserved across re-registration); `--share-with` clears it. The analyze worker reports the storage it wrote over IPC so the server settles a clone's first shared slot. A query-time base-plus-overlay graph stays out: LadybugDB reads one database per query. Instead, private graph copies record whether the filesystem cloned them copy-on-write (sharing unchanged pages on disk) or made a full copy, and `status` reports it. Co-Authored-By: Claude Opus 5.5 (1M context) --- AGENTS.md | 5 +- README.md | 2 +- gitnexus/src/cli/i18n/en.ts | 8 +- gitnexus/src/cli/i18n/zh-CN.ts | 7 +- gitnexus/src/cli/index.ts | 10 ++- gitnexus/src/cli/status.ts | 23 ++++- gitnexus/src/core/run-analyze.ts | 4 + gitnexus/src/core/shared-store-analyze.ts | 83 ++++++++++++++++--- gitnexus/src/server/analyze-launch.ts | 14 ++-- gitnexus/src/server/analyze-worker-ipc.ts | 4 + gitnexus/src/storage/repo-manager.ts | 20 +++++ .../src/storage/shared-store-lifecycle.ts | 12 +++ gitnexus/src/storage/shared-store.ts | 6 ++ .../integration/shared-store-adoption.test.ts | 1 + .../integration/shared-store-clean.test.ts | 73 +++++++++++++++- .../shared-store-clone-optin.test.ts | 59 +++++++++++-- gitnexus/test/unit/analyze-worker-ipc.test.ts | 1 + 17 files changed, 292 insertions(+), 40 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index baa3e3ecb..2172825f5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ - + Last reviewed: 2026-09-24 @@ -91,6 +91,7 @@ mirror. `gitnexus/test/unit/shipped-skills-sync.test.ts` guards the copies. Toke | Date | Version | Change | |------|---------|--------| +| 2026-09-24 | 1.17.0 | Clones with the same `origin` URL now share a store automatically; `--no-share` records a lasting opt-out (#3352). | | 2026-09-24 | 1.16.0 | Documented the shared worktree index store (`/stores/`, `analyze --share-with`, `GITNEXUS_SHARED_STORE=off`) in the storage notes (#3352). | | 2026-09-07 | 1.15.0 | Added the Objective-C provider guide as the required reference before changing Objective-C parsing or resolution. | | 2026-07-20 | 1.14.0 | `gitnexus-review` gains a coordinated swarm: six `ci-personas/` lanes the CI review agent dispatches as subagents (via the `Agent` tool), with a bounded critic gate and sidechain-excluded evidence. | @@ -199,4 +200,4 @@ npx gitnexus serve # HTTP API on port 4747 (from any ind - `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (`build-tree-sitter-grammars.cjs` activates committed prebuilds in place under `vendor/`, and only source-builds when none matches). A C/C++ toolchain (`python3`, `make`, `g++`) is needed only for that source-build fallback. - The vendored grammars `tree-sitter-{c,dart,proto,swift,kotlin,zig}` are handled uniformly: c is required; dart/proto/swift/kotlin/zig are optional and skippable via `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`. Install warnings appear only when no prebuild matches the platform-arch and no toolchain is present, and are non-fatal — only that language's parsing is unavailable. - ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`. -- Index storage defaults to `/.gitnexus/`. `GITNEXUS_STORAGE_PATH` selects one complete external index directory and wins over `GITNEXUS_STORAGE_ROOT`, which creates an isolated `-<12-hex>/` slot per repository. Linked worktrees share one store under `/stores//` (one immutable graph per commit, private graphs for checkouts with local changes, shared parse caches); clones join only via `analyze --share-with`, and `GITNEXUS_SHARED_STORE=off` or either storage env var disables sharing (#3352). `GITNEXUS_CONTENT_RETENTION` is `full` (default), `symbol`, or `none`. MCP `list_repos`, `gitnexus://repo/{name}/context`, and HTTP `GET /api/repos` / `GET /api/repo` expose `storagePath`, `contentRetention`, and `sourceAvailable`. HTTP `/api/file` and `/api/grep` return 410 unless retention is `full`; MCP `include_content` may still return symbol spans at `symbol`. +- Index storage defaults to `/.gitnexus/`. `GITNEXUS_STORAGE_PATH` selects one complete external index directory and wins over `GITNEXUS_STORAGE_ROOT`, which creates an isolated `-<12-hex>/` slot per repository. Linked worktrees share one store under `/stores//` (one immutable graph per commit, private graphs for checkouts with local changes, shared parse caches); clones with the same `origin` URL join a registered sibling's store automatically (`analyze --share-with` names one, `--no-share` opts out and is remembered), and `GITNEXUS_SHARED_STORE=off` or either storage env var disables sharing (#3352). `GITNEXUS_CONTENT_RETENTION` is `full` (default), `symbol`, or `none`. MCP `list_repos`, `gitnexus://repo/{name}/context`, and HTTP `GET /api/repos` / `GET /api/repo` expose `storagePath`, `contentRetention`, and `sourceAvailable`. HTTP `/api/file` and `/api/grep` return 410 unless retention is `full`; MCP `include_content` may still return symbol spans at `symbol`. diff --git a/README.md b/README.md index 9d919aa1c..a4422c949 100644 --- a/README.md +++ b/README.md @@ -710,7 +710,7 @@ GitNexus uses a **global registry** so one MCP server can serve multiple indexed Each `gitnexus analyze` stores the index in `.gitnexus/` inside the repo by default (portable, gitignored). `GITNEXUS_STORAGE_PATH` selects one complete external index directory and preserves the established configuration behavior. To manage multiple repositories under one external directory, set `GITNEXUS_STORAGE_ROOT`; GitNexus derives an isolated `-/` slot beneath it for each repository. If both variables are set, `GITNEXUS_STORAGE_PATH` takes precedence. GitNexus registers the resolved slot in `~/.gitnexus/registry.json`, allowing later `status`, MCP, and `serve` commands to reopen the index without repeating the environment variable. LadybugDB connections are opened lazily on first query and evicted after 5 minutes of inactivity (max 5 concurrent). Read-only tools can omit `repo` when only one repo is indexed, an MCP default is configured, or the GitNexus process cwd is inside a registered path without crossing into an unindexed nested Git checkout. Outside those paths—and for mutating tools with multiple indexed repos and no MCP default—pass `repo` explicitly. -**Worktrees share one index store.** When a repository has linked worktrees (`git worktree add`), the main checkout and every worktree index into one store at `~/.gitnexus/stores//` instead of each keeping a full `.gitnexus/`. Checkouts at the same commit with no local changes read one shared, read-only graph: one copy on disk and one open database in MCP. A checkout with uncommitted changes gets its own graph, copied from the nearest shared graph and updated incrementally rather than rebuilt. Parse caches are shared too. Each worktree keeps a small `.gitnexus/store.json` pointer, and an index it had before sharing is left in place; `gitnexus status` reports it and `gitnexus clean --local-index --force` removes it. `gitnexus clean` in one worktree removes only that worktree's slot and any shared graph no other checkout uses; `gitnexus clean --gc` also drops slots whose worktree was deleted. An independent clone (not a worktree) joins only on request: `gitnexus analyze --share-with ` checks that both have the same `origin` URL, and `--no-share` moves the clone back to its own `.gitnexus/`. Set `GITNEXUS_SHARED_STORE=off` to turn sharing off. +**Worktrees share one index store.** When a repository has linked worktrees (`git worktree add`), the main checkout and every worktree index into one store at `~/.gitnexus/stores//` instead of each keeping a full `.gitnexus/`. Checkouts at the same commit with no local changes read one shared, read-only graph: one copy on disk and one open database in MCP. A checkout with uncommitted changes gets its own graph, copied from the nearest shared graph and updated incrementally rather than rebuilt. Parse caches are shared too. Each worktree keeps a small `.gitnexus/store.json` pointer, and an index it had before sharing is left in place; `gitnexus status` reports it and `gitnexus clean --local-index --force` removes it. `gitnexus clean` in one worktree removes only that worktree's slot and any shared graph no other checkout uses; `gitnexus clean --gc` also drops slots whose worktree was deleted. Independent clones of one repository share too: when another registered clone has the same `origin` URL, `gitnexus analyze` in a clone joins that clone's store (or starts one the other clone joins on its next analyze). A lone clone keeps its own `.gitnexus/`. `gitnexus analyze --share-with ` joins a specific checkout's store after checking the `origin` URLs match, and `--no-share` moves a clone back to its own `.gitnexus/` and keeps it out until `--share-with`. On filesystems with copy-on-write clones (APFS, btrfs, XFS) a checkout's private graph shares its unchanged pages with the shared graph on disk; elsewhere it is a full copy, and `gitnexus status` says which. Queries cannot combine two graphs, because LadybugDB reads one database per query, so a checkout with edits always has a complete graph of its own. Set `GITNEXUS_SHARED_STORE=off` to turn sharing off.
Architecture diagram diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 9a666ebfb..e661a5eb3 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -31,6 +31,10 @@ export const en = { 'status.sharedStoreShared': 'Shared index: store {{key}}, shared graph for commit {{commit}}', 'status.sharedStorePrivate': 'Shared index: store {{key}}, private graph (local changes or a pinned branch index)', + 'status.sharedStoreCloneCow': + ' Copied copy-on-write: unchanged pages are shared with the commit graph on disk', + 'status.sharedStoreCloneCopy': + ' Full copy: this filesystem cannot clone copy-on-write (APFS, btrfs and XFS can)', 'status.legacyLocalIndex': 'Leftover local index: {{path}} ({{size}}); remove it with `gitnexus clean --local-index --force`', 'status.notGitRepo': 'Not a git repository.', @@ -291,9 +295,9 @@ export const en = { 'help.option.analyze.allowDuplicateName': 'Register this repo even if another path already uses the same --name alias. Leaves `-r ` ambiguous for the two paths; use -r to disambiguate.', 'help.option.analyze.shareWith': - 'Join the shared index store of a registered worktree of the same repository (name or path); the remote URL must match. Remembered for later runs.', + 'Join the shared index store of a registered checkout of the same repository (name or path); the remote URL must match. Clones join a sibling clone’s store automatically; this names one explicitly and clears a --no-share opt-out.', 'help.option.analyze.noShare': - 'Opted-in clones only: leave the shared index store and index into /.gitnexus again (linked worktrees always share; set GITNEXUS_SHARED_STORE=off instead)', + 'Clones only: leave the shared index store, index into /.gitnexus again, and stop joining sibling clones automatically until --share-with (linked worktrees always share; set GITNEXUS_SHARED_STORE=off instead)', 'help.option.verbose': 'Enable verbose output', 'help.option.analyze.maxFileSize': 'Skip files larger than this (KB). Default: 512. Hard cap: 32768 (tree-sitter limit).', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 3b53b151f..5eb0baf22 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -31,6 +31,9 @@ export const zhCN = { 'list.unknown': 'unknown', 'status.sharedStoreShared': '共享索引：存储 {{key}}，提交 {{commit}} 的共享图', 'status.sharedStorePrivate': '共享索引：存储 {{key}}，私有图（有本地更改或固定分支索引）', + 'status.sharedStoreCloneCow': ' 写时复制副本：未更改的页面在磁盘上与提交图共享', + 'status.sharedStoreCloneCopy': + ' 完整副本：此文件系统不支持写时复制克隆（APFS、btrfs 和 XFS 支持）', 'status.legacyLocalIndex': '残留的本地索引：{{path}}（{{size}}）；使用 `gitnexus clean --local-index --force` 删除', 'status.notGitRepo': '当前目录不是 git 仓库。', @@ -266,9 +269,9 @@ export const zhCN = { 'help.option.analyze.allowDuplicateName': '即使已有其他路径使用相同 --name 别名，也注册该仓库。会使两个路径的 `-r ` 产生歧义；请用 -r 消除歧义。', 'help.option.analyze.shareWith': - '加入同一仓库已注册工作树的共享索引存储（名称或路径）；远程 URL 必须一致。之后的运行会记住此选择。', + '加入同一仓库已注册检出的共享索引存储（名称或路径）；远程 URL 必须一致。克隆会自动加入同源克隆的存储；此选项显式指定存储，并清除 --no-share 的退出设置。', 'help.option.analyze.noShare': - '仅限已加入的克隆：离开共享索引存储，重新索引到 /.gitnexus（链接工作树始终共享；请改用 GITNEXUS_SHARED_STORE=off）', + '仅限克隆：离开共享索引存储，重新索引到 /.gitnexus，并在使用 --share-with 之前不再自动加入同源克隆（链接工作树始终共享；请改用 GITNEXUS_SHARED_STORE=off）', 'help.option.verbose': '启用详细输出', 'help.option.analyze.maxFileSize': '跳过大于该值的文件（KB）。默认：512。硬上限：32768（tree-sitter 限制）。', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index a8eab181c..0563f0c2f 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -147,13 +147,15 @@ program ) .option( '--share-with ', - 'Join the shared index store of a registered worktree of the same repository ' + - '(name or path); the remote URL must match. Remembered for later runs.', + 'Join the shared index store of a registered checkout of the same repository ' + + '(name or path); the remote URL must match. Clones join a sibling clone’s store ' + + 'automatically; this names one explicitly and clears a --no-share opt-out.', ) .option( '--no-share', - 'Opted-in clones only: leave the shared index store and index into /.gitnexus again ' + - '(linked worktrees always share; set GITNEXUS_SHARED_STORE=off instead)', + 'Clones only: leave the shared index store, index into /.gitnexus again, and stop ' + + 'joining sibling clones automatically until --share-with (linked worktrees always share; ' + + 'set GITNEXUS_SHARED_STORE=off instead)', ) .option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)') .option( diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index f33ed4603..26dd780b4 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -5,7 +5,11 @@ */ import { resolveGraphPath, storeRootOfCheckoutSlot } from '../storage/shared-store.js'; -import { describeSharedGraph, findLegacyLocalIndex } from '../storage/shared-store-lifecycle.js'; +import { + describeSharedGraph, + findLegacyLocalIndex, + readGraphCloneKind, +} from '../storage/shared-store-lifecycle.js'; import { formatSlotSize } from './stale-branch-format.js'; import path from 'path'; import { @@ -367,6 +371,12 @@ export const statusCommand = async (options: StatusOptions = {}) => { commit: activeMeta.lastCommit, } : null; + // A private flat graph copied from a shared one: say whether the filesystem + // shared its unchanged pages (copy-on-write) or it is a full copy. + const privateClone = + sharedStore?.graph === 'private' && activeMeta === repo.meta + ? await readGraphCloneKind(repo.storagePath) + : null; const legacyLocalIndex = await findLegacyLocalIndex(repo.repoPath, repo.storagePath); if (options.json) { console.log( @@ -388,7 +398,7 @@ export const statusCommand = async (options: StatusOptions = {}) => { runnerIdentity: currentRunnerIdentity, }, contentDrift: describeContentDrift(contentDrift), - sharedStore, + sharedStore: sharedStore ? { ...sharedStore, privateClone } : null, legacyLocalIndex: legacyLocalIndex ? { path: legacyLocalIndex.dir, bytes: legacyLocalIndex.bytes } : null, @@ -414,6 +424,15 @@ export const statusCommand = async (options: StatusOptions = {}) => { }) : t('status.sharedStorePrivate', { key: sharedStore.key }), ); + if (privateClone) { + console.log( + t( + privateClone === 'copy-on-write' + ? 'status.sharedStoreCloneCow' + : 'status.sharedStoreCloneCopy', + ), + ); + } } if (legacyLocalIndex) { console.log( diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index f72476d85..51e1a86bd 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -146,6 +146,7 @@ import { loadMeta, ensureGitNexusIgnored, registerRepo, + setShareOptOut, adoptFlatBranchLabel, isReadOnlyFilesystemError, isRepoRegistered, @@ -1340,6 +1341,9 @@ export async function runFullAnalysis( await registerLeftStore(repoPath, writeTarget.storagePath); } } + // A clone that left stays out of sibling stores until `--share-with`. + if (options.noShare) await setShareOptOut(repoPath, true); + else if (options.shareWith) await setShareOptOut(repoPath, false); return result; } finally { discardScopedEmbeddingSpills(); diff --git a/gitnexus/src/core/shared-store-analyze.ts b/gitnexus/src/core/shared-store-analyze.ts index ddaa663bc..2f2b91fa0 100644 --- a/gitnexus/src/core/shared-store-analyze.ts +++ b/gitnexus/src/core/shared-store-analyze.ts @@ -16,7 +16,7 @@ */ import { createHash, randomUUID } from 'crypto'; -import { constants as fsConstants } from 'fs'; +import { existsSync, constants as fsConstants } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { acquireIndexLock, requireExclusiveIndexLock } from '../storage/index-lock.js'; @@ -29,9 +29,11 @@ import { registryPathEquals, resolveRegistryEntry, saveMeta, + type RegistryEntry, } from '../storage/repo-manager.js'; import { loadMeta, type RepoMeta } from '../storage/repo-meta.js'; import { + cloneStoreKey, commitGraphDir, resolveGraphPath, resolveSharedStore, @@ -40,6 +42,8 @@ import { type SharedStoreLayout, } from '../storage/shared-store.js'; import { + GRAPH_CLONE_MARKER, + type GraphCloneKind, reclaimAfterSlotRemoval, reclaimSharedStoreLocked, removeSharedStorePointer, @@ -108,13 +112,24 @@ const exists = (p: string): Promise => /** * Copy a graph file to `dest` via a unique `lbug.new.` temp (swept by the * slot lock if this process dies mid-copy), cloning copy-on-write where the - * filesystem supports it. On failure the temp is removed and the error thrown. + * filesystem supports it (APFS, btrfs, XFS with reflink). A clone shares every + * unchanged page with the source, so a private graph costs only what the + * checkout's edits rewrite; elsewhere it is a full copy. Which one happened is + * recorded next to `dest` for `status`. On failure the temp is removed and + * the error thrown. */ const cloneGraphFile = async (source: string, dest: string): Promise => { const tmp = `${dest}.new.${randomUUID()}`; try { - await fs.copyFile(source, tmp, fsConstants.COPYFILE_FICLONE); + let kind: GraphCloneKind = 'copy-on-write'; + try { + await fs.copyFile(source, tmp, fsConstants.COPYFILE_FICLONE_FORCE); + } catch { + kind = 'copy'; + await fs.copyFile(source, tmp); + } await fs.rename(tmp, dest); + await fs.writeFile(path.join(path.dirname(dest), GRAPH_CLONE_MARKER), kind); } catch (err) { await fs.rm(tmp, { force: true }).catch(() => {}); throw err; @@ -287,7 +302,9 @@ export const ensurePrivateSharedGraph = async ( if (!meta) return true; // `copy: false` — the caller rebuilds from scratch and reads nothing from // the old graph, so only the pointer is dropped. - if (opts.copy !== false && !(await exists(own))) { + if (opts.copy === false) { + await fs.rm(path.join(slot, GRAPH_CLONE_MARKER), { force: true }); + } else if (!(await exists(own))) { const started = Date.now(); try { await cloneGraphFile(pointed, own); @@ -414,11 +431,52 @@ export const publishSharedGraph = async ( await writeSharedStorePointer(repoPath, layout); }; +/** The store a checkout's registry entry already points into, if any. */ +const registeredStore = ( + entries: readonly RegistryEntry[], + repoPath: string, +): SharedStoreLayout | undefined => { + const own = findRegistryEntryByRepoPath(entries, repoPath); + const root = own ? storeRootOfCheckoutSlot(own.storagePath) : null; + return root ? sharedStoreLayout(path.basename(root), repoPath) : undefined; +}; + +/** + * Store for a clone with no store of its own: the store a registered sibling + * clone (same normalized `origin` URL, checkout still present) already uses, + * or, when siblings exist but none shares yet, a new store keyed on this + * clone. Graphs are keyed by commit and feature key, so clones only ever share + * a graph built from the same commit with the same settings. A lone clone + * keeps its repository-local index. + */ +const siblingCloneStore = ( + entries: readonly RegistryEntry[], + repoPath: string, +): SharedStoreLayout | undefined => { + const remote = getRemoteUrl(repoPath); + if (!remote) return undefined; + const self = canonicalizePath(repoPath); + const siblings = entries.filter( + (e) => + e.remoteUrl === remote && + !registryPathEquals(canonicalizePath(e.path), self) && + existsSync(e.path), + ); + if (siblings.length === 0) return undefined; + const keys = siblings + .map((e) => storeRootOfCheckoutSlot(e.storagePath)) + .filter((root): root is string => root !== null) + .map((root) => path.basename(root)) + .sort(); + return sharedStoreLayout(keys[0] ?? cloneStoreKey(repoPath), repoPath); +}; + /** * Store for a checkout that is not a linked worktree: the store named by - * `--share-with`, or the one its registry entry already points into. Clones - * only join by explicit opt-in (#3352 R2), and only when their normalized - * remote URL matches the member they name (R3). + * `--share-with`, the one its registry entry already points into, or a + * sibling clone's store (#3352). `--share-with` requires the normalized remote + * URL to match the member it names (R3); `analyze --no-share` records an + * opt-out that stops automatic joining. */ export const resolveOptedInStore = async ( repoPath: string, @@ -449,15 +507,16 @@ export const resolveOptedInStore = async ( } return sharedStoreLayout(path.basename(root), repoPath); } - const own = findRegistryEntryByRepoPath(entries, repoPath); - const root = own ? storeRootOfCheckoutSlot(own.storagePath) : null; - return root ? sharedStoreLayout(path.basename(root), repoPath) : undefined; + const registered = registeredStore(entries, repoPath); + if (registered) return registered; + if (findRegistryEntryByRepoPath(entries, repoPath)?.shareOptOut) return undefined; + return siblingCloneStore(entries, repoPath); }; /** * The store slot a checkout is registered at, for `--no-share`. Linked * worktrees always share (turn sharing off with GITNEXUS_SHARED_STORE=off), - * so only an opted-in clone can leave. + * so only a clone can leave. */ export const optedInSlotToLeave = async (repoPath: string): Promise => { if (resolveSharedStore(repoPath)) { @@ -466,7 +525,7 @@ export const optedInSlotToLeave = async (repoPath: string): Promise; /** @@ -84,5 +85,8 @@ export function projectAnalyzeResultForIpc(result: AnalyzeResult): AnalyzeResult // `branches//` sub-slot — so its finalization gate watches the files // this job actually rewrote (#3199 review). isPrimaryBranch: result.isPrimaryBranch, + // The storage the run wrote. A checkout's first analyze may create a + // shared-store slot (#3352) the launcher's pre-run lookup could not see. + storagePath: result.storagePath, }; } diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 7cbcf7203..c6a362bff 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -191,6 +191,11 @@ export interface RegistryEntry { * legacy registry shape. */ branches?: BranchSummary[]; + /** + * The checkout left sharing with `analyze --no-share` (#3352), so it does + * not join a sibling clone's store automatically. Cleared by `--share-with`. + */ + shareOptOut?: true; } /** Path-only registry lookup. Canonicalizes `repoPath` once. Does not throw. */ @@ -1176,6 +1181,7 @@ const registerRepoUnlocked = async ( merged = { ...entry }; if (freshExisting?.branches && !opts?.dropBranches) merged.branches = freshExisting.branches; else delete merged.branches; + if (freshExisting?.shareOptOut) merged.shareOptOut = true; } if (freshIdx >= 0) { fresh[freshIdx] = merged; @@ -1234,6 +1240,20 @@ const unregisterRepoUnlocked = async (repoPath: string): Promise => { export const unregisterRepo = async (repoPath: string): Promise => withRegistryLock(() => unregisterRepoUnlocked(repoPath)); +/** + * Record (or clear) a checkout's opt-out from automatic clone sharing + * (#3352). A no-op when the checkout is not registered. + */ +export const setShareOptOut = async (repoPath: string, optOut: boolean): Promise => + withRegistryLock(async () => { + const entries = await readRegistryStrict(); + const entry = findRegistryEntryByRepoPath(entries, repoPath); + if (!entry || !!entry.shareOptOut === optOut) return; + if (optOut) entry.shareOptOut = true; + else delete entry.shareOptOut; + await writeRegistry(entries); + }); + /** * Remove a single non-primary branch's summary from a repo's registry entry * (#2106 R7). Called by `gitnexus clean --branch`. Returns `true` when a diff --git a/gitnexus/src/storage/shared-store-lifecycle.ts b/gitnexus/src/storage/shared-store-lifecycle.ts index e30173efe..3da0e49b4 100644 --- a/gitnexus/src/storage/shared-store-lifecycle.ts +++ b/gitnexus/src/storage/shared-store-lifecycle.ts @@ -225,6 +225,18 @@ export const reclaimAfterSlotRemoval = async ( } }; +/** Records in a checkout slot how its private graph was copied from a shared one. */ +export const GRAPH_CLONE_MARKER = 'graph-clone'; +export type GraphCloneKind = 'copy-on-write' | 'copy'; + +/** How the slot's private graph was copied, or null when it was built, not copied. */ +export const readGraphCloneKind = async (storagePath: string): Promise => { + const text = await fs + .readFile(path.join(storagePath, GRAPH_CLONE_MARKER), 'utf-8') + .catch(() => null); + return text === 'copy-on-write' || text === 'copy' ? text : null; +}; + /** Whether a checkout slot reads a shared commit graph or its own private graph. */ export const describeSharedGraph = ( graphPath: string, diff --git a/gitnexus/src/storage/shared-store.ts b/gitnexus/src/storage/shared-store.ts index 2d7893824..a8da36260 100644 --- a/gitnexus/src/storage/shared-store.ts +++ b/gitnexus/src/storage/shared-store.ts @@ -145,6 +145,12 @@ const resolveIdentity = ( return { key: slotName(canonicalCheckout ?? commonDir), canonicalCheckout }; }; +/** + * Key for a store founded by a standalone clone (#3352): the clone's own + * checkout path, in the same canonical form a main checkout keys on. + */ +export const cloneStoreKey = (checkoutPath: string): string => slotName(checkoutPath); + export const resolveSharedStoreKey = ( checkoutPath: string, env: NodeJS.ProcessEnv = process.env, diff --git a/gitnexus/test/integration/shared-store-adoption.test.ts b/gitnexus/test/integration/shared-store-adoption.test.ts index c1390eeb5..9c7485f5e 100644 --- a/gitnexus/test/integration/shared-store-adoption.test.ts +++ b/gitnexus/test/integration/shared-store-adoption.test.ts @@ -125,6 +125,7 @@ describe('shared store adoption and reporting (#3352)', () => { key: layoutOf(wt).key, graph: 'shared', commit: git(wt, 'rev-parse', 'HEAD'), + privateClone: null, }); expect(json.legacyLocalIndex).toMatchObject({ path: path.join(wt, '.gitnexus') }); diff --git a/gitnexus/test/integration/shared-store-clean.test.ts b/gitnexus/test/integration/shared-store-clean.test.ts index f7d34fa9b..ab5e8c682 100644 --- a/gitnexus/test/integration/shared-store-clean.test.ts +++ b/gitnexus/test/integration/shared-store-clean.test.ts @@ -1,5 +1,5 @@ import { execFileSync } from 'child_process'; -import { existsSync, readFileSync } from 'fs'; +import { constants as fsConstants, existsSync, readFileSync } from 'fs'; import fs from 'fs/promises'; import path from 'path'; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; @@ -11,6 +11,7 @@ import { } from '../../src/storage/shared-store.js'; import { reclaimAfterSlotRemoval, + readGraphCloneKind, reclaimSharedStore, removeSharedStorePointer, } from '../../src/storage/shared-store-lifecycle.js'; @@ -439,3 +440,73 @@ describe('reclaimSharedStore', () => { } }); }); + +describe('private graph copies (#3352)', () => { + let tmpHome: Awaited>; + let savedHome: string | undefined; + + beforeEach(async () => { + tmpHome = await createTempDir('gitnexus-test-clone-kind-home-'); + savedHome = process.env.GITNEXUS_HOME; + process.env.GITNEXUS_HOME = tmpHome.dbPath; + }); + + afterEach(async () => { + vi.restoreAllMocks(); + if (savedHome === undefined) delete process.env.GITNEXUS_HOME; + else process.env.GITNEXUS_HOME = savedHome; + await tmpHome.cleanup(); + }); + + const pointerSlot = async (): Promise => { + const layout = sharedStoreLayout('repo-0123456789ab', '/tmp/wt'); + const graph = path.join(layout.commitsDir, 'aaaaaaa-1111111111111111'); + await fs.mkdir(graph, { recursive: true }); + await fs.writeFile(path.join(graph, 'lbug'), 'graph'); + await fs.mkdir(layout.checkoutSlot, { recursive: true }); + await saveMeta(layout.checkoutSlot, { + lastCommit: 'aaaaaaa', + indexedAt: '', + repoPath: '/tmp/wt', + graphPath: path.join(graph, 'lbug'), + }); + return layout.checkoutSlot; + }; + + // Stand in for the filesystem: FICLONE_FORCE succeeds only when `cow` is set. + const fakeCopyFile = (cow: boolean) => { + const realCopyFile = fs.copyFile; + vi.spyOn(fs, 'copyFile').mockImplementation(async (src, dest, mode) => { + if (mode === fsConstants.COPYFILE_FICLONE_FORCE && !cow) { + throw Object.assign(new Error('not supported'), { code: 'ENOTSUP' }); + } + return realCopyFile(src, dest); + }); + }; + + it('records a copy-on-write clone', async () => { + const slot = await pointerSlot(); + fakeCopyFile(true); + const { ensurePrivateSharedGraph } = await import('../../src/core/shared-store-analyze.js'); + expect(await ensurePrivateSharedGraph(slot, () => {})).toBe(true); + expect(await readGraphCloneKind(slot)).toBe('copy-on-write'); + expect(await fs.readFile(path.join(slot, 'lbug'), 'utf-8')).toBe('graph'); + }); + + it('falls back to a full copy and records it', async () => { + const slot = await pointerSlot(); + fakeCopyFile(false); + const { ensurePrivateSharedGraph } = await import('../../src/core/shared-store-analyze.js'); + expect(await ensurePrivateSharedGraph(slot, () => {})).toBe(true); + expect(await readGraphCloneKind(slot)).toBe('copy'); + expect(await fs.readFile(path.join(slot, 'lbug'), 'utf-8')).toBe('graph'); + }); + + it('forgets the record when the private graph is rebuilt instead of copied', async () => { + const slot = await pointerSlot(); + await fs.writeFile(path.join(slot, 'graph-clone'), 'copy'); + const { ensurePrivateSharedGraph } = await import('../../src/core/shared-store-analyze.js'); + await ensurePrivateSharedGraph(slot, () => {}, { copy: false }); + expect(await readGraphCloneKind(slot)).toBeNull(); + }); +}); diff --git a/gitnexus/test/integration/shared-store-clone-optin.test.ts b/gitnexus/test/integration/shared-store-clone-optin.test.ts index 07ac6ba78..e175d418b 100644 --- a/gitnexus/test/integration/shared-store-clone-optin.test.ts +++ b/gitnexus/test/integration/shared-store-clone-optin.test.ts @@ -18,15 +18,16 @@ afterAll(() => { }); /** - * #3352 U7 — an independent clone joins a shared store only by explicit - * opt-in, and only when its remote matches the member it names. + * #3352 U7 — an independent clone joins the store of a registered sibling + * clone (same normalized origin URL) automatically, or the member named by + * `--share-with`; `--no-share` leaves and stays out. */ const git = (cwd: string, ...args: string[]): string => execFileSync('git', args, { cwd, stdio: 'pipe', encoding: 'utf-8' }).trim(); const REMOTE = 'https://example.com/acme/widgets'; -describe('shared store clone opt-in (#3352)', () => { +describe('shared store clone sharing (#3352)', () => { let tmpHome: Awaited>; let tmpRepo: Awaited>; let savedHome: string | undefined; @@ -77,13 +78,50 @@ describe('shared store clone opt-in (#3352)', () => { await tmpHome.cleanup(); }); - it('Covers AE7: a clone that has not opted in keeps its own .gitnexus', async () => { + it('a clone joins the store of a registered sibling and reuses the commit graph', async () => { const clone = cloneWithRemote('clone', REMOTE); + const result = await analyze(clone); + + const slot = await registeredStorage(clone); + expect(path.dirname(slot as string)).toBe(storeLayout.checkoutsDir); + expect(result.alreadyUpToDate).toBe(true); + expect(getStoragePaths(clone, undefined, slot).lbugPath).toBe( + getStoragePaths(wt, undefined, storeLayout.checkoutSlot).lbugPath, + ); + expect(existsSync(path.join(clone, '.gitnexus', 'lbug'))).toBe(false); + }, 240_000); + + it('Covers AE7: a clone with no registered sibling keeps its own .gitnexus', async () => { + const clone = cloneWithRemote('clone', 'https://example.com/acme/gadgets'); await analyze(clone); expect(await registeredStorage(clone)).toBe(path.join(clone, '.gitnexus')); expect(existsSync(path.join(clone, '.gitnexus', 'lbug'))).toBe(true); }, 240_000); + it('two standalone clones of one repository found a store and share its graph', async () => { + const solo = 'https://example.com/acme/solo'; + const first = cloneWithRemote('first', solo); + await analyze(first); + expect(await registeredStorage(first)).toBe(path.join(first, '.gitnexus')); + + const second = cloneWithRemote('second', solo); + await analyze(second); + const secondSlot = (await registeredStorage(second)) as string; + const checkouts = path.dirname(secondSlot); + expect(path.basename(checkouts)).toBe('checkouts'); + expect(checkouts).not.toBe(storeLayout.checkoutsDir); + + // The first clone joins on its next analyze and reads the same graph. + await analyze(first); + const firstSlot = (await registeredStorage(first)) as string; + expect(path.dirname(firstSlot)).toBe(checkouts); + expect(getStoragePaths(first, undefined, firstSlot).lbugPath).toBe( + getStoragePaths(second, undefined, secondSlot).lbugPath, + ); + // Adoption leaves the old repository-local index in place. + expect(existsSync(path.join(first, '.gitnexus', 'lbug'))).toBe(true); + }, 240_000); + it('joins the store with --share-with and reuses the commit graph at its HEAD', async () => { const clone = cloneWithRemote('clone', REMOTE); const result = await analyze(clone, { shareWith: wt }); @@ -124,7 +162,7 @@ describe('shared store clone opt-in (#3352)', () => { it('refuses a --share-with target that is not in a shared store', async () => { const plain = cloneWithRemote('plain', REMOTE); - await analyze(plain); + await analyze(plain, { noShare: true }); const clone = cloneWithRemote('clone', REMOTE); await expect(analyze(clone, { shareWith: plain })).rejects.toThrow( /does not use a shared index store/, @@ -143,11 +181,20 @@ describe('shared store clone opt-in (#3352)', () => { expect(existsSync(getStoragePaths(wt, undefined, storeLayout.checkoutSlot).lbugPath)).toBe( true, ); + + // The opt-out sticks: a plain analyze does not rejoin the sibling store. + await analyze(clone); + expect(await registeredStorage(clone)).toBe(path.join(clone, '.gitnexus')); + + // --share-with clears it: the clone is back in, and stays in. + await analyze(clone, { shareWith: wt }); + await analyze(clone); + expect(path.dirname((await registeredStorage(clone)) as string)).toBe(storeLayout.checkoutsDir); }, 240_000); it('--no-share on an up-to-date local index still re-registers there', async () => { const clone = cloneWithRemote('clone', REMOTE); - await analyze(clone); + await analyze(clone, { noShare: true }); await analyze(clone, { shareWith: wt }); const slot = (await registeredStorage(clone)) as string; diff --git a/gitnexus/test/unit/analyze-worker-ipc.test.ts b/gitnexus/test/unit/analyze-worker-ipc.test.ts index b99a857b4..d6dde3fcc 100644 --- a/gitnexus/test/unit/analyze-worker-ipc.test.ts +++ b/gitnexus/test/unit/analyze-worker-ipc.test.ts @@ -49,6 +49,7 @@ describe('#2112: analyze-worker IPC projection', () => { alreadyUpToDate: false, ftsRepairedOnly: undefined, ftsSkipped: true, + storagePath: '/repos/demo/.gitnexus', }); expect('pipelineResult' in projected).toBe(false); });