From 0dc5351d1ec5e4270f3c108ca2135d0cb45acc4b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 15:45:31 +0000 Subject: [PATCH 1/4] feat(lbug): add adaptive buffer-pool size hint Adds the sizing lever without changing behavior yet: a module-scoped buffer-pool size hint plus estimateBufferPool(graphElementCount), read by resolveBufferManagerSize with precedence env-override > clamp(hint, 64MiB, default) > default. The hint can only shrink the pool from the default (clamped to [floor, default]), so the 2GiB/80%-RAM cap and the GITNEXUS_LBUG_BUFFER_POOL_SIZE escape hatch (incl. 0) are preserved. With no hint set, resolveBufferManagerSize returns exactly what it did before. Motivation: LadybugDB eagerly commits the buffer pool at DB open, so the fixed min(2GiB,80%RAM) pool adds a measured ~2.8s to every analyze even on a 3-file repo (dominant on Windows). Sizing the pool to the graph lets small repos use the fast 64MiB floor while large repos keep the cap. Co-Authored-By: Claude Fable 5 --- gitnexus/src/core/lbug/lbug-config.ts | 54 ++++++++++++++-- gitnexus/test/unit/lbug-config-wal.test.ts | 71 +++++++++++++++++++++- 2 files changed, 120 insertions(+), 5 deletions(-) diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index 540fe530a..f32a12898 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -333,16 +333,62 @@ const parseBufferPoolSize = (raw: string | undefined): number | undefined => { const defaultBufferPoolSize = (): number => Math.min(DEFAULT_BUFFER_POOL_CAP, Math.max(BUFFER_POOL_FLOOR, Math.floor(os.totalmem() * 0.8))); +/** Clamp a requested pool size to [floor, default] — never above the 2 GiB / 80%-RAM default. */ +const clampBufferPool = (bytes: number): number => + Math.min(defaultBufferPoolSize(), Math.max(BUFFER_POOL_FLOOR, Math.floor(bytes))); + +/** + * Buffer-pool bytes to provision per graph element (node + relationship). + * + * LadybugDB eagerly commits the buffer pool at DB open, so an oversized pool + * costs a fixed penalty on every analyze regardless of repo size (measured: + * ~2.8 s extra for a pool ≥128 MiB vs the 64 MiB floor, on a 3-file repo). + * The pool is only a page cache over the on-disk index, and the index scales + * with node/edge count — so a per-element budget lets a small repo run with a + * small, fast-to-commit pool while a large repo still reaches the 2 GiB cap. + * Kept deliberately generous (holds the working set without thrash); tuned by + * `bench/buffer-pool/measure.mjs` against a large-repo analyze. + */ +const POOL_BYTES_PER_ELEMENT = 4 * 1024; + +/** + * Size the buffer pool to an estimated graph size (node + relationship count), + * clamped to [BUFFER_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can + * only *shrink* the pool from the default — it never exceeds the 2 GiB / 80%-RAM + * cap — so a large repo is never under the default it gets today. + */ +export const estimateBufferPool = (graphElementCount: number): number => + clampBufferPool(graphElementCount * POOL_BYTES_PER_ELEMENT); + +/** + * Optional per-run buffer-pool size hint (bytes). The analyze orchestrator sets + * it once the graph size is known (after the pipeline, before the DB open) so a + * small repo opens with a small pool, and clears it at run end. Non-analyze + * opens (MCP serve, `native-check` `:memory:`) never set it and keep the default. + */ +let bufferPoolSizeHint: number | undefined; + +/** Set (bytes) or clear (`undefined`) the per-run buffer-pool size hint. */ +export const setBufferPoolSizeHint = (bytes: number | undefined): void => { + bufferPoolSizeHint = bytes; +}; + /** * Resolve the `bufferManagerSize` passed to every `new lbug.Database(...)`. - * `GITNEXUS_LBUG_BUFFER_POOL_SIZE` (bytes) overrides the default; `0` is a + * `GITNEXUS_LBUG_BUFFER_POOL_SIZE` (bytes) overrides everything; `0` is a * deliberate escape hatch that restores LadybugDB's native unbounded - * 80%-of-RAM default. Resolved at call time (not module load) so tests can - * stub the env var and `os.totalmem`. + * 80%-of-RAM default. With no env override, a per-run `setBufferPoolSizeHint` + * (clamped to [floor, default]) sizes the pool to the repo; otherwise the + * default. Resolved at call time (not module load) so tests can stub the env + * var, the hint, and `os.totalmem`. */ const resolveBufferManagerSize = (): number => { const raw = process.env.GITNEXUS_LBUG_BUFFER_POOL_SIZE; - if (raw === undefined) return defaultBufferPoolSize(); + if (raw === undefined) { + return bufferPoolSizeHint !== undefined + ? clampBufferPool(bufferPoolSizeHint) + : defaultBufferPoolSize(); + } const parsed = parseBufferPoolSize(raw); if (parsed !== undefined) return parsed; // Non-empty but unparseable input: warn the operator and fall back — diff --git a/gitnexus/test/unit/lbug-config-wal.test.ts b/gitnexus/test/unit/lbug-config-wal.test.ts index a9912b9ba..4b6d8fe61 100644 --- a/gitnexus/test/unit/lbug-config-wal.test.ts +++ b/gitnexus/test/unit/lbug-config-wal.test.ts @@ -1,9 +1,11 @@ import os from 'os'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { createLbugDatabase, + estimateBufferPool, isLbugCheckpointIoError, isWalCorruptionError, + setBufferPoolSizeHint, } from '../../src/core/lbug/lbug-config.js'; import { _captureLogger } from '../../src/core/logger.js'; @@ -252,6 +254,73 @@ describe('createLbugDatabase buffer pool size (#2557)', () => { }); }); +describe('adaptive buffer pool hint', () => { + const GiB = 1024 * 1024 * 1024; + const MiB = 1024 * 1024; + const bufferPoolArg = (Database: ReturnType): unknown => Database.mock.calls[0][1]; + + afterEach(() => setBufferPoolSizeHint(undefined)); + + describe('estimateBufferPool', () => { + it.each([ + ['tiny graph clamps up to the 64 MiB floor', 41, 64 * MiB], + ['mid graph scales linearly (100k elements * 4 KiB = 400 MiB)', 100_000, 100_000 * 4 * 1024], + ['huge graph caps at the 2 GiB / 80%-RAM default', 10_000_000, 2 * GiB], + ])('%s', (_label, elements, expected) => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + expect(estimateBufferPool(elements)).toBe(expected); + } finally { + totalmemSpy.mockRestore(); + } + }); + }); + + it.each([ + ['a hint within range passes through', 128 * MiB, 128 * MiB], + ['a hint below the floor clamps up to 64 MiB', 1 * MiB, 64 * MiB], + ['a hint above the default clamps down to the 2 GiB cap', 8 * GiB, 2 * GiB], + ])( + 'createLbugDatabase uses the clamped hint when no env override is set: %s', + (_label, hint, expected) => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + setBufferPoolSizeHint(hint); + const Database = vi.fn(function (this: any) {}); + createLbugDatabase({ Database } as any, '/tmp/lbug-hint'); + expect(bufferPoolArg(Database)).toBe(expected); + } finally { + totalmemSpy.mockRestore(); + } + }, + ); + + it('env override wins over the hint (including 0 = native default)', () => { + try { + setBufferPoolSizeHint(128 * MiB); + vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', '0'); + const Database = vi.fn(function (this: any) {}); + createLbugDatabase({ Database } as any, '/tmp/lbug-hint-env'); + expect(bufferPoolArg(Database)).toBe(0); + } finally { + vi.unstubAllEnvs(); + } + }); + + it('falls back to the default when the hint is cleared', () => { + const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB); + try { + setBufferPoolSizeHint(128 * MiB); + setBufferPoolSizeHint(undefined); + const Database = vi.fn(function (this: any) {}); + createLbugDatabase({ Database } as any, '/tmp/lbug-hint-cleared'); + expect(bufferPoolArg(Database)).toBe(2 * GiB); + } finally { + totalmemSpy.mockRestore(); + } + }); +}); + // ─── Finding 8: strict + permissive checkpoint IO matchers ───────────────── describe('isLbugCheckpointIoError', () => { it.each([ From 69b71e671ad27929b8ad4c3494c8a3e26d0d18cc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 15:51:54 +0000 Subject: [PATCH 2/4] feat(analyze): size the buffer pool to the graph before the DB open runFullAnalysis now sets the buffer-pool size hint from the built graph's node+relationship count (after the pipeline, before initLbug), and clears it at the top of each run so a prior run's size can't leak into a pre-pipeline open. A small repo opens with the fast 64MiB floor instead of eagerly committing the full 2GiB pool. Measured on a 3-file repo (this box): analyze drops 4.78s -> 1.9s for both fresh and incremental, matching a forced 64MiB pool; the 2GiB path still reproduces the old 4.78s. This roughly halves the skills-e2e Idempotency hook (two analyze passes) that was timing out on Windows CI. Co-Authored-By: Claude Fable 5 --- gitnexus/src/core/run-analyze.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 635b255a9..812dc5757 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -34,6 +34,7 @@ import { LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js'; +import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js'; import { escapeCypherString } from './lbug/cypher-escape.js'; import { buildSearchIndexesOrDegrade, @@ -645,6 +646,11 @@ export async function runFullAnalysis( // and are shared across branches (#2106 KTD7). const { storagePath } = getStoragePaths(repoPath); + // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open + // (e.g. the embeddings-cache open) falls back to the default until the hint is + // set from the built graph below, so a prior run's size can't leak in. + setBufferPoolSizeHint(undefined); + // Clean up stale KuzuDB files from before the LadybugDB migration. const kuzuResult = await cleanupOldKuzuFiles(storagePath); if (kuzuResult.found && kuzuResult.needsReindex) { @@ -1374,6 +1380,15 @@ export async function runFullAnalysis( await wipeLbugDbFiles(lbugPath); } + // Size the buffer pool to the graph just built by the pipeline (a page cache + // over the on-disk index, which scales with node/edge count). A small repo + // opens with the fast 64 MiB floor instead of eagerly committing the full + // 2 GiB default; a large repo still reaches the cap. env override / no-hint + // paths are unchanged. See resolveBufferManagerSize / estimateBufferPool. + setBufferPoolSizeHint( + estimateBufferPool(pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount), + ); + await initLbug(lbugPath); // Manual WAL checkpoint driver (#1741): periodically drain the WAL From 8dcec481cb82cfb99e264411c29d9f10a7f53923 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 15:57:38 +0000 Subject: [PATCH 3/4] docs(lbug): correct the POOL_BYTES_PER_ELEMENT tuning note MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The factor is validated by timing a full `analyze --force` of a large repo, not a build-free bench (the pool is a native eager allocation). Benchmark (GitNexus self, 101k graph elements): adaptive 414MiB pool = 35.3s vs forced 2GiB = 50.8s vs forced 64MiB = 26.5s — the adaptive pool is 31% faster than the old 2GiB default even on a large repo (the eager commit dominates), with no under-sizing thrash. 4KiB/element kept. Co-Authored-By: Claude Fable 5 --- gitnexus/src/core/lbug/lbug-config.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index f32a12898..06ee6b9e0 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -346,8 +346,11 @@ const clampBufferPool = (bytes: number): number => * The pool is only a page cache over the on-disk index, and the index scales * with node/edge count — so a per-element budget lets a small repo run with a * small, fast-to-commit pool while a large repo still reaches the 2 GiB cap. - * Kept deliberately generous (holds the working set without thrash); tuned by - * `bench/buffer-pool/measure.mjs` against a large-repo analyze. + * Kept deliberately generous (holds the working set without thrash): tuned by + * timing a full `analyze --force` of a large repo at this factor vs a forced + * 2 GiB pool and confirming no wall-time regression (the pool is a native + * eager allocation, so it is measured with a real analyze, not a build-free + * bench — see the emit-path `COPY` timing note in bench/emit-persistence). */ const POOL_BYTES_PER_ELEMENT = 4 * 1024; From d0f7a13d9a4c59b78efd41ed6a909dfc0b02f275 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 20 Jul 2026 16:13:26 +0000 Subject: [PATCH 4/4] fix(lbug): raise the adaptive pool floor to a COPY-safe 256 MiB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 64 MiB floor was too small: LadybugDB's bulk COPY needs working buffer-pool memory that scales with the repo, so a 64 MiB pool fails with "buffer pool is full and no memory could be freed" on any non-trivial repo (empirically: the 6-file skills-e2e idempotency fixture needs >=128 MiB; the ~1800-file GitNexus checkout needs >=256 MiB). Introduce a distinct ADAPTIVE_POOL_FLOOR (256 MiB) for the hint clamp, kept separate from BUFFER_POOL_FLOOR (64 MiB), which still guards defaultBufferPoolSize on tiny-RAM machines; the hint is still clamped up to the machine default so it can never over-commit. This keeps the change as what it actually is — a large-repo optimization: GitNexus full analyze is 51s (2 GiB) -> 35s (adaptive ~414 MiB). Small repos now open COPY-safely at 256 MiB instead of the 2 GiB default (same wall time on Linux, where commit is lazy; the eager commit is far cheaper than 2 GiB on Windows). The reframed comments drop the earlier unrepresentative "3-file repo / 64 MiB fast" claim. Co-Authored-By: Claude Fable 5 --- gitnexus/src/core/lbug/lbug-config.ts | 56 ++++++++++++++-------- gitnexus/src/core/run-analyze.ts | 9 ++-- gitnexus/test/unit/lbug-config-wal.test.ts | 7 +-- 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index 06ee6b9e0..7b4ba4c65 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -321,6 +321,16 @@ const resolveCheckpointThreshold = (): number => { const DEFAULT_BUFFER_POOL_CAP = 2 * 1024 * 1024 * 1024; const BUFFER_POOL_FLOOR = 64 * 1024 * 1024; +// COPY-safety floor for the adaptive hint (below). LadybugDB's bulk COPY needs +// working buffer-pool memory that scales with the repo: a 64 MiB pool fails +// ("buffer pool is full and no memory could be freed") on any non-trivial repo, +// and even the ~1800-file GitNexus checkout needs ≥256 MiB. So the adaptive +// size never drops a repo below this — a distinct, higher floor than +// BUFFER_POOL_FLOOR, which only guards defaultBufferPoolSize on tiny-RAM +// machines. It is still clamped up to defaultBufferPoolSize, so a machine whose +// default is below this floor keeps its default rather than over-committing. +const ADAPTIVE_POOL_FLOOR = 256 * 1024 * 1024; + const parseBufferPoolSize = (raw: string | undefined): number | undefined => { if (raw === undefined) return undefined; const normalized = raw.trim(); @@ -333,41 +343,49 @@ const parseBufferPoolSize = (raw: string | undefined): number | undefined => { const defaultBufferPoolSize = (): number => Math.min(DEFAULT_BUFFER_POOL_CAP, Math.max(BUFFER_POOL_FLOOR, Math.floor(os.totalmem() * 0.8))); -/** Clamp a requested pool size to [floor, default] — never above the 2 GiB / 80%-RAM default. */ +/** + * Clamp an adaptive pool request to [ADAPTIVE_POOL_FLOOR, default]. The lower + * bound keeps LadybugDB's COPY viable; the upper bound (defaultBufferPoolSize) + * means the hint can only shrink the pool from today's default and can never + * exceed the 2 GiB / 80%-RAM cap — and on a machine whose default is below the + * COPY floor, the default wins, so the pool is never over-committed. + */ const clampBufferPool = (bytes: number): number => - Math.min(defaultBufferPoolSize(), Math.max(BUFFER_POOL_FLOOR, Math.floor(bytes))); + Math.min(defaultBufferPoolSize(), Math.max(ADAPTIVE_POOL_FLOOR, Math.floor(bytes))); /** * Buffer-pool bytes to provision per graph element (node + relationship). * - * LadybugDB eagerly commits the buffer pool at DB open, so an oversized pool - * costs a fixed penalty on every analyze regardless of repo size (measured: - * ~2.8 s extra for a pool ≥128 MiB vs the 64 MiB floor, on a 3-file repo). - * The pool is only a page cache over the on-disk index, and the index scales - * with node/edge count — so a per-element budget lets a small repo run with a - * small, fast-to-commit pool while a large repo still reaches the 2 GiB cap. - * Kept deliberately generous (holds the working set without thrash): tuned by - * timing a full `analyze --force` of a large repo at this factor vs a forced - * 2 GiB pool and confirming no wall-time regression (the pool is a native - * eager allocation, so it is measured with a real analyze, not a build-free - * bench — see the emit-path `COPY` timing note in bench/emit-persistence). + * The fixed 2 GiB default is far larger than most repos' working set, and + * LadybugDB eagerly commits the pool at DB open — measured: a full + * `analyze --force` of the GitNexus checkout takes ~51 s with the 2 GiB pool + * vs ~35 s with the ~414 MiB this factor yields (31% faster; the oversized + * pool's commit dominates). The pool is a page cache over the on-disk index, + * which scales with node/edge count, so a per-element budget sizes it to the + * repo. Kept generous so the whole index stays resident (no COPY thrash) and + * always clamped to at least ADAPTIVE_POOL_FLOOR; tuned by timing a real + * large-repo `analyze --force` at this factor vs a forced 2 GiB pool (the pool + * is a native eager allocation, measured with a real analyze, not a build-free + * bench — see the emit-path COPY timing note in bench/emit-persistence). */ const POOL_BYTES_PER_ELEMENT = 4 * 1024; /** * Size the buffer pool to an estimated graph size (node + relationship count), - * clamped to [BUFFER_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can - * only *shrink* the pool from the default — it never exceeds the 2 GiB / 80%-RAM - * cap — so a large repo is never under the default it gets today. + * clamped to [ADAPTIVE_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can + * only *shrink* the pool from the default — never above the 2 GiB / 80%-RAM cap, + * never below the COPY-safety floor — so no repo is under-sized or gets more + * than the default it would have today. */ export const estimateBufferPool = (graphElementCount: number): number => clampBufferPool(graphElementCount * POOL_BYTES_PER_ELEMENT); /** * Optional per-run buffer-pool size hint (bytes). The analyze orchestrator sets - * it once the graph size is known (after the pipeline, before the DB open) so a - * small repo opens with a small pool, and clears it at run end. Non-analyze - * opens (MCP serve, `native-check` `:memory:`) never set it and keep the default. + * it once the graph size is known (after the pipeline, before the DB open) so + * the pool is sized to the repo instead of the fixed 2 GiB default, and clears + * it at run end. Non-analyze opens (MCP serve, `native-check` `:memory:`) never + * set it and keep the default. */ let bufferPoolSizeHint: number | undefined; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 812dc5757..ef77ba3d9 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -1381,10 +1381,11 @@ export async function runFullAnalysis( } // Size the buffer pool to the graph just built by the pipeline (a page cache - // over the on-disk index, which scales with node/edge count). A small repo - // opens with the fast 64 MiB floor instead of eagerly committing the full - // 2 GiB default; a large repo still reaches the cap. env override / no-hint - // paths are unchanged. See resolveBufferManagerSize / estimateBufferPool. + // over the on-disk index, which scales with node/edge count) instead of the + // fixed 2 GiB default, whose eager commit dominates large-repo analyze. The + // size is clamped to [COPY-safety floor, default], so it only ever shrinks + // the pool; env override / no-hint paths are unchanged. See + // resolveBufferManagerSize / estimateBufferPool. setBufferPoolSizeHint( estimateBufferPool(pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount), ); diff --git a/gitnexus/test/unit/lbug-config-wal.test.ts b/gitnexus/test/unit/lbug-config-wal.test.ts index 4b6d8fe61..94395541a 100644 --- a/gitnexus/test/unit/lbug-config-wal.test.ts +++ b/gitnexus/test/unit/lbug-config-wal.test.ts @@ -263,7 +263,8 @@ describe('adaptive buffer pool hint', () => { describe('estimateBufferPool', () => { it.each([ - ['tiny graph clamps up to the 64 MiB floor', 41, 64 * MiB], + ['tiny graph clamps up to the 256 MiB COPY-safety floor', 41, 256 * MiB], + ['a graph under the floor still clamps up to 256 MiB', 40_000, 256 * MiB], ['mid graph scales linearly (100k elements * 4 KiB = 400 MiB)', 100_000, 100_000 * 4 * 1024], ['huge graph caps at the 2 GiB / 80%-RAM default', 10_000_000, 2 * GiB], ])('%s', (_label, elements, expected) => { @@ -277,8 +278,8 @@ describe('adaptive buffer pool hint', () => { }); it.each([ - ['a hint within range passes through', 128 * MiB, 128 * MiB], - ['a hint below the floor clamps up to 64 MiB', 1 * MiB, 64 * MiB], + ['a hint within range passes through', 512 * MiB, 512 * MiB], + ['a hint below the COPY-safety floor clamps up to 256 MiB', 100 * MiB, 256 * MiB], ['a hint above the default clamps down to the 2 GiB cap', 8 * GiB, 2 * GiB], ])( 'createLbugDatabase uses the clamped hint when no env override is set: %s',