Merge pull request #2582 from magyargergo/fix/buffer-pool-adaptive-sizing

perf(lbug): size the buffer pool to the repo (adaptive, COPY-safe)

gitnexus/src/core/lbug/lbug-config.ts

@@ -321,6 +321,16 @@ const resolveCheckpointThreshold = (): number => {
 const DEFAULT_BUFFER_POOL_CAP = 2 * 1024 * 1024 * 1024;
 const BUFFER_POOL_FLOOR = 64 * 1024 * 1024;
 
+// COPY-safety floor for the adaptive hint (below). LadybugDB's bulk COPY needs
+// working buffer-pool memory that scales with the repo: a 64 MiB pool fails
+// ("buffer pool is full and no memory could be freed") on any non-trivial repo,
+// and even the ~1800-file GitNexus checkout needs ≥256 MiB. So the adaptive
+// size never drops a repo below this — a distinct, higher floor than
+// BUFFER_POOL_FLOOR, which only guards defaultBufferPoolSize on tiny-RAM
+// machines. It is still clamped up to defaultBufferPoolSize, so a machine whose
+// default is below this floor keeps its default rather than over-committing.
+const ADAPTIVE_POOL_FLOOR = 256 * 1024 * 1024;
+
 const parseBufferPoolSize = (raw: string | undefined): number | undefined => {
   if (raw === undefined) return undefined;
   const normalized = raw.trim();
@@ -333,16 +343,73 @@ const parseBufferPoolSize = (raw: string | undefined): number | undefined => {
 const defaultBufferPoolSize = (): number =>
   Math.min(DEFAULT_BUFFER_POOL_CAP, Math.max(BUFFER_POOL_FLOOR, Math.floor(os.totalmem() * 0.8)));
 
+/**
+ * Clamp an adaptive pool request to [ADAPTIVE_POOL_FLOOR, default]. The lower
+ * bound keeps LadybugDB's COPY viable; the upper bound (defaultBufferPoolSize)
+ * means the hint can only shrink the pool from today's default and can never
+ * exceed the 2 GiB / 80%-RAM cap — and on a machine whose default is below the
+ * COPY floor, the default wins, so the pool is never over-committed.
+ */
+const clampBufferPool = (bytes: number): number =>
+  Math.min(defaultBufferPoolSize(), Math.max(ADAPTIVE_POOL_FLOOR, Math.floor(bytes)));
+
+/**
+ * Buffer-pool bytes to provision per graph element (node + relationship).
+ *
+ * The fixed 2 GiB default is far larger than most repos' working set, and
+ * LadybugDB eagerly commits the pool at DB open — measured: a full
+ * `analyze --force` of the GitNexus checkout takes ~51 s with the 2 GiB pool
+ * vs ~35 s with the ~414 MiB this factor yields (31% faster; the oversized
+ * pool's commit dominates). The pool is a page cache over the on-disk index,
+ * which scales with node/edge count, so a per-element budget sizes it to the
+ * repo. Kept generous so the whole index stays resident (no COPY thrash) and
+ * always clamped to at least ADAPTIVE_POOL_FLOOR; tuned by timing a real
+ * large-repo `analyze --force` at this factor vs a forced 2 GiB pool (the pool
+ * is a native eager allocation, measured with a real analyze, not a build-free
+ * bench — see the emit-path COPY timing note in bench/emit-persistence).
+ */
+const POOL_BYTES_PER_ELEMENT = 4 * 1024;
+
+/**
+ * Size the buffer pool to an estimated graph size (node + relationship count),
+ * clamped to [ADAPTIVE_POOL_FLOOR, defaultBufferPoolSize()]. The estimate can
+ * only *shrink* the pool from the default — never above the 2 GiB / 80%-RAM cap,
+ * never below the COPY-safety floor — so no repo is under-sized or gets more
+ * than the default it would have today.
+ */
+export const estimateBufferPool = (graphElementCount: number): number =>
+  clampBufferPool(graphElementCount * POOL_BYTES_PER_ELEMENT);
+
+/**
+ * Optional per-run buffer-pool size hint (bytes). The analyze orchestrator sets
+ * it once the graph size is known (after the pipeline, before the DB open) so
+ * the pool is sized to the repo instead of the fixed 2 GiB default, and clears
+ * it at run end. Non-analyze opens (MCP serve, `native-check` `:memory:`) never
+ * set it and keep the default.
+ */
+let bufferPoolSizeHint: number | undefined;
+
+/** Set (bytes) or clear (`undefined`) the per-run buffer-pool size hint. */
+export const setBufferPoolSizeHint = (bytes: number | undefined): void => {
+  bufferPoolSizeHint = bytes;
+};
+
 /**
  * Resolve the `bufferManagerSize` passed to every `new lbug.Database(...)`.
- * `GITNEXUS_LBUG_BUFFER_POOL_SIZE` (bytes) overrides the default; `0` is a
+ * `GITNEXUS_LBUG_BUFFER_POOL_SIZE` (bytes) overrides everything; `0` is a
  * deliberate escape hatch that restores LadybugDB's native unbounded
- * 80%-of-RAM default. Resolved at call time (not module load) so tests can
- * stub the env var and `os.totalmem`.
+ * 80%-of-RAM default. With no env override, a per-run `setBufferPoolSizeHint`
+ * (clamped to [floor, default]) sizes the pool to the repo; otherwise the
+ * default. Resolved at call time (not module load) so tests can stub the env
+ * var, the hint, and `os.totalmem`.
  */
 const resolveBufferManagerSize = (): number => {
   const raw = process.env.GITNEXUS_LBUG_BUFFER_POOL_SIZE;
-  if (raw === undefined) return defaultBufferPoolSize();
+  if (raw === undefined) {
+    return bufferPoolSizeHint !== undefined
+      ? clampBufferPool(bufferPoolSizeHint)
+      : defaultBufferPoolSize();
+  }
   const parsed = parseBufferPoolSize(raw);
   if (parsed !== undefined) return parsed;
   // Non-empty but unparseable input: warn the operator and fall back —

gitnexus/src/core/run-analyze.ts

@@ -34,6 +34,7 @@ import {
   LbugWipeError,
   DELETE_FILES_CHUNK_SIZE,
 } from './lbug/lbug-adapter.js';
+import { estimateBufferPool, setBufferPoolSizeHint } from './lbug/lbug-config.js';
 import { escapeCypherString } from './lbug/cypher-escape.js';
 import {
   buildSearchIndexesOrDegrade,
@@ -645,6 +646,11 @@ export async function runFullAnalysis(
   // and are shared across branches (#2106 KTD7).
   const { storagePath } = getStoragePaths(repoPath);
 
+  // Start each analyze with a clean buffer-pool hint: any pre-pipeline DB open
+  // (e.g. the embeddings-cache open) falls back to the default until the hint is
+  // set from the built graph below, so a prior run's size can't leak in.
+  setBufferPoolSizeHint(undefined);
+
   // Clean up stale KuzuDB files from before the LadybugDB migration.
   const kuzuResult = await cleanupOldKuzuFiles(storagePath);
   if (kuzuResult.found && kuzuResult.needsReindex) {
@@ -1374,6 +1380,16 @@ export async function runFullAnalysis(
     await wipeLbugDbFiles(lbugPath);
   }
 
+  // Size the buffer pool to the graph just built by the pipeline (a page cache
+  // over the on-disk index, which scales with node/edge count) instead of the
+  // fixed 2 GiB default, whose eager commit dominates large-repo analyze. The
+  // size is clamped to [COPY-safety floor, default], so it only ever shrinks
+  // the pool; env override / no-hint paths are unchanged. See
+  // resolveBufferManagerSize / estimateBufferPool.
+  setBufferPoolSizeHint(
+    estimateBufferPool(pipelineResult.graph.nodeCount + pipelineResult.graph.relationshipCount),
+  );
+
   await initLbug(lbugPath);
 
   // Manual WAL checkpoint driver (#1741): periodically drain the WAL

gitnexus/test/unit/lbug-config-wal.test.ts

@@ -1,9 +1,11 @@
 import os from 'os';
-import { describe, expect, it, vi } from 'vitest';
+import { afterEach, describe, expect, it, vi } from 'vitest';
 import {
   createLbugDatabase,
+  estimateBufferPool,
   isLbugCheckpointIoError,
   isWalCorruptionError,
+  setBufferPoolSizeHint,
 } from '../../src/core/lbug/lbug-config.js';
 import { _captureLogger } from '../../src/core/logger.js';
 
@@ -252,6 +254,74 @@ describe('createLbugDatabase buffer pool size (#2557)', () => {
   });
 });
 
+describe('adaptive buffer pool hint', () => {
+  const GiB = 1024 * 1024 * 1024;
+  const MiB = 1024 * 1024;
+  const bufferPoolArg = (Database: ReturnType<typeof vi.fn>): unknown => Database.mock.calls[0][1];
+
+  afterEach(() => setBufferPoolSizeHint(undefined));
+
+  describe('estimateBufferPool', () => {
+    it.each([
+      ['tiny graph clamps up to the 256 MiB COPY-safety floor', 41, 256 * MiB],
+      ['a graph under the floor still clamps up to 256 MiB', 40_000, 256 * MiB],
+      ['mid graph scales linearly (100k elements * 4 KiB = 400 MiB)', 100_000, 100_000 * 4 * 1024],
+      ['huge graph caps at the 2 GiB / 80%-RAM default', 10_000_000, 2 * GiB],
+    ])('%s', (_label, elements, expected) => {
+      const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB);
+      try {
+        expect(estimateBufferPool(elements)).toBe(expected);
+      } finally {
+        totalmemSpy.mockRestore();
+      }
+    });
+  });
+
+  it.each([
+    ['a hint within range passes through', 512 * MiB, 512 * MiB],
+    ['a hint below the COPY-safety floor clamps up to 256 MiB', 100 * MiB, 256 * MiB],
+    ['a hint above the default clamps down to the 2 GiB cap', 8 * GiB, 2 * GiB],
+  ])(
+    'createLbugDatabase uses the clamped hint when no env override is set: %s',
+    (_label, hint, expected) => {
+      const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB);
+      try {
+        setBufferPoolSizeHint(hint);
+        const Database = vi.fn(function (this: any) {});
+        createLbugDatabase({ Database } as any, '/tmp/lbug-hint');
+        expect(bufferPoolArg(Database)).toBe(expected);
+      } finally {
+        totalmemSpy.mockRestore();
+      }
+    },
+  );
+
+  it('env override wins over the hint (including 0 = native default)', () => {
+    try {
+      setBufferPoolSizeHint(128 * MiB);
+      vi.stubEnv('GITNEXUS_LBUG_BUFFER_POOL_SIZE', '0');
+      const Database = vi.fn(function (this: any) {});
+      createLbugDatabase({ Database } as any, '/tmp/lbug-hint-env');
+      expect(bufferPoolArg(Database)).toBe(0);
+    } finally {
+      vi.unstubAllEnvs();
+    }
+  });
+
+  it('falls back to the default when the hint is cleared', () => {
+    const totalmemSpy = vi.spyOn(os, 'totalmem').mockReturnValue(32 * GiB);
+    try {
+      setBufferPoolSizeHint(128 * MiB);
+      setBufferPoolSizeHint(undefined);
+      const Database = vi.fn(function (this: any) {});
+      createLbugDatabase({ Database } as any, '/tmp/lbug-hint-cleared');
+      expect(bufferPoolArg(Database)).toBe(2 * GiB);
+    } finally {
+      totalmemSpy.mockRestore();
+    }
+  });
+});
+
 // ─── Finding 8: strict + permissive checkpoint IO matchers ─────────────────
 describe('isLbugCheckpointIoError', () => {
   it.each([