scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-13 03:58:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(lbug): fail-fast storage-version mismatch and unlock pool lock-retry

Browse source 
Incremental analyze was warning through a version mismatch, and lock-retry
sleep held the pool mutex so one analyze-locked repo blocked every other
init. Fail immediately with the rebuild hint on both adapters, and sleep
outside withPoolLock so other repos can open during backoff.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Gergo Magyar 2026-09-12 13:10:36 +00:00
parent ef070baf3b
commit c50541bd63
14 changed files with 498 additions and 95 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
gitnexus/src/core/lbug/lbug-adapter.ts
View file

@ -51,12 +51,12 @@ import {
isOpenRetryExhausted,
isStorageVersionMismatchError,
isWalCorruptionError,
throwIfStorageVersionMismatch,
bufferPoolExhaustionRemedy,
openLbugConnection,
sleep,
toNativeSafePath,
resolveNativeSafeStorageDir,
STORAGE_VERSION_MISMATCH_SUGGESTION,
WAL_RECOVERY_SUGGESTION,
waitForWindowsHandleRelease,
type LbugConnectionHandle,
@ -702,6 +702,11 @@ const runSchemaCreationQueries = async (dbPath: string): Promise<unknown | null>
` Original error: ${msg.slice(0, 200)}`,
);
}
if (isStorageVersionMismatchError(err)) {
await safeClose();
resetOpenConnectionState();
throwIfStorageVersionMismatch(err);
}
if (!msg.includes('already exists') && !isDbBusyError(err) && !isReadOnlyDbError(err)) {
logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`);
}
@ -832,10 +837,7 @@ const doInitLbug = async (
// downgrading the dependency). Mirrors the pool-adapter.ts check for
// the same error, on the separate open path /api/graph and /api/query
// actually use (withLbugDb, not the pool).
if (isStorageVersionMismatchError(err)) {
const msg = err instanceof Error ? err.message : String(err);
throw new Error(`${STORAGE_VERSION_MISMATCH_SUGGESTION} (${msg})`);
}
throwIfStorageVersionMismatch(err);
throw err;
}
db = usable.db;
@ -946,10 +948,18 @@ const doInitLbug = async (
allowQuarantine: true,
});
const opened = await openLbugConnection(lbug, dbPath);
db = opened.db;
conn = opened.conn;
currentDbReadOnly = false;
try {
const opened = await openLbugConnection(lbug, dbPath);
db = opened.db;
conn = opened.conn;
currentDbReadOnly = false;
} catch (err) {
// Incremental analyze can hit a storage-version mismatch on construct
// or (more often) on the first schema query below. Fail immediately
// with the rebuild hint instead of warn-and-continue.
throwIfStorageVersionMismatch(err);
throw err;
}
} finally {
await releaseInitLock();
}

7
gitnexus/src/core/lbug/lbug-config.ts
View file

@ -579,6 +579,13 @@ export function isStorageVersionMismatchError(err: unknown): boolean {
return STORAGE_VERSION_MISMATCH_RE.test(msg);
}
/** Throws the rebuild-hint Error when `err` is a storage-version mismatch. */
export function throwIfStorageVersionMismatch(err: unknown): void {
if (!isStorageVersionMismatchError(err)) return;
const msg = err instanceof Error ? err.message : String(err);
throw new Error(`${STORAGE_VERSION_MISMATCH_SUGGESTION} (${msg})`);
}
// ─── Ladybug WAL checkpoint IO error matchers ───────────────────────────────
//
// Matched against LadybugDB v0.18.0 (see `gitnexus/package.json`

178
gitnexus/src/core/lbug/pool-adapter.ts
View file

@ -22,9 +22,9 @@ import { closeQueryResults } from './query-result-utils.js';
import { warnIfQueryTextUnbounded } from './query-batch.js';
import {
createLbugDatabase,
isStorageVersionMismatchError,
isWalCorruptionError,
STORAGE_VERSION_MISMATCH_SUGGESTION,
sleep,
throwIfStorageVersionMismatch,
toNativeSafePath,
WAL_RECOVERY_SUGGESTION,
} from './lbug-config.js';
@ -697,29 +697,10 @@ async function tryQuarantineAndReopen(dbPath: string, repoId: string): Promise<l
return await openReadOnlyDatabase(dbPath);
}
/**
* Initialize (or reuse) a Database + connection pool for a specific repo.
* Retries on lock errors (e.g., when `gitnexus analyze` is running).
*
* Concurrent calls (for the same repoId or different ones) serialize on
* poolLock below, so a second caller for a repo already being initialized
* simply waits its turn and then hits the "existing" fast path once its turn
* comes — no separate per-repoId dedup needed on top of that.
*/
/**
* Returns `true` when this call (re)opened a fresh handle onto the current
* on-disk file, `false` when it reused/served the existing handle (unchanged,
* or changed-but-a-query-is-in-flight). Callers that gate their own freshness
* bookkeeping on "did the pool actually roll over" (LocalBackend) use the
* return value; callers that only need the pool ready can ignore it.
*/
// Serializes the whole initLbug body across concurrent callers. Awaiting
// closeOne/evictLRU (above) closes the race within a single call, but two
// concurrent initLbug() calls for two different repoIds can still each evict
// their own LRU victim and open their own new connection at the same time,
// racing each other's checkpoint on whatever global lock the native engine
// holds. This mutex makes pool mutations (evict + reopen, for any repo)
// mutually exclusive across all callers, closing that cross-request race too.
// Serializes pool mutations (evict / close / native open / register) across
// concurrent callers. Awaiting closeOne/evictLRU closes the race within a
// single call; this mutex makes those mutations mutually exclusive across
// repos so two inits cannot race each other's checkpoint.
let poolLock: Promise<unknown> = Promise.resolve();
function withPoolLock<T>(fn: () => Promise<T>): Promise<T> {
const run = poolLock.then(fn, fn);
@ -730,10 +711,45 @@ function withPoolLock<T>(fn: () => Promise<T>): Promise<T> {
return run;
}
export const initLbug = (repoId: string, dbPath: string): Promise<boolean> =>
withPoolLock(() => initLbugInner(repoId, dbPath));
type InitLbugAttempt = { status: 'done'; reopened: boolean } | { status: 'retry'; error: Error };
const initLbugInner = async (repoId: string, dbPath: string): Promise<boolean> => {
function ladybugUnavailableError(repoId: string, err: Error | undefined): Error {
return new Error(
`LadybugDB unavailable for ${repoId}. Another process may be rebuilding the index. ` +
`Retry later. (${err?.message || 'unknown error'})`,
);
}
/**
* Initialize (or reuse) a Database + connection pool for a specific repo.
* Retries on lock errors (e.g., when `gitnexus analyze` is running).
*
* Concurrent calls (for the same repoId or different ones) serialize on
* poolLock below for evict / close / native open / register, so a second
* caller for a repo already being initialized waits its turn and then hits
* the "existing" fast path — no separate per-repoId dedup needed. Lock-retry
* *sleeps* run outside the mutex so one analyze-locked repo does not block
* every other pool init for LOCK_RETRY_DELAY_MS * attempt.
*
* Returns `true` when this call (re)opened a fresh handle onto the current
* on-disk file, `false` when it reused/served the existing handle (unchanged,
* or changed-but-a-query-is-in-flight). Callers that gate their own freshness
* bookkeeping on "did the pool actually roll over" (LocalBackend) use the
* return value; callers that only need the pool ready can ignore it.
*/
export const initLbug = async (repoId: string, dbPath: string): Promise<boolean> => {
let lastError: Error | undefined;
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
const result = await withPoolLock(() => initLbugInner(repoId, dbPath));
if (result.status === 'done') return result.reopened;
lastError = result.error;
if (attempt === LOCK_RETRY_ATTEMPTS) break;
await sleep(LOCK_RETRY_DELAY_MS * attempt);
}
throw ladybugUnavailableError(repoId, lastError);
};
const initLbugInner = async (repoId: string, dbPath: string): Promise<InitLbugAttempt> => {
const existing = pool.get(repoId);
if (existing) {
existing.lastUsed = Date.now();
@ -742,7 +758,9 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise<boolean> =
// unlinked-but-open) inode until LRU/idle eviction — a stale-read window
// of up to IDLE_TIMEOUT_MS after analyze finishes.
const current = await statDbIdentity(dbPath);
if (!dbIdentityChanged(existing.dbIdentity, current)) return false; // unchanged → reuse
if (!dbIdentityChanged(existing.dbIdentity, current)) {
return { status: 'done', reopened: false }; // unchanged → reuse
}
// A query is in flight on this entry; closing its connection (and the
// shared Database at refCount 0) mid-use is a native use-after-free. Serve
// the current handle for this dispatch — the next initLbug that finds the
@ -753,13 +771,12 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise<boolean> =
// (a complete older snapshot), just not the newest. Callers that route
// freshness THROUGH initLbug (rather than calling closeLbug directly) get
// this guard for free; that is why LocalBackend delegates here (#2614).
if (existing.checkedOut > 0) return false;
if (existing.checkedOut > 0) return { status: 'done', reopened: false };
// Awaited: see the rationale on the evictLRU call site in doInitLbug below.
await closeOne(repoId); // idle & changed → evict, then fall through to reopen the new file
}
await doInitLbug(repoId, dbPath);
return true;
return doInitLbug(repoId, dbPath);
};
/**
@ -767,7 +784,7 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise<boolean> =
* Pool entry is registered LAST so concurrent executeQuery calls see either
* "not initialized" (and throw) or a fully ready pool — never a half-built one.
*/
async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
async function doInitLbug(repoId: string, dbPath: string): Promise<InitLbugAttempt> {
// Check if database exists
try {
await fs.stat(dbPath);
@ -777,9 +794,12 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
// Awaited: without this, the connection opened just below could race the
// checkpoint from the LRU victim's still-in-flight close (see evictLRU /
// closeOne). initLbug's outer poolLock keeps this whole function exclusive
// across concurrent repos too, so this await only needs to cover this call's
// own evict-then-reopen — not other callers.
// closeOne). The caller holds withPoolLock for this attempt, so this await
// only covers this call's own evict-then-reopen — not other callers.
// Lock-retry re-enters this function after sleeping *outside* the mutex.
// evictLRU is a no-op unless the pool is full again (another repo may have
// taken the slot we freed on a prior attempt). Skipping it on retry would
// let a 6th native open race a still-resident victim's checkpoint.
await evictLRU();
// Reuse an existing native Database if another repoId already opened this path.
@ -805,47 +825,38 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
if (!shared) {
// Open in read-only mode — MCP server never writes to the database.
// This allows multiple MCP server instances to read concurrently, and
// avoids lock conflicts when `gitnexus analyze` is writing.
let lastError: Error | null = null;
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
try {
const db = await openReadOnlyDatabase(dbPath);
shared = { db, refCount: 0, ftsLoaded: false, dbIdentity: await statDbIdentity(dbPath) };
dbCache.set(dbPath, shared);
break;
} catch (err: any) {
lastError = err instanceof Error ? err : new Error(String(err));
// avoids lock conflicts when `gitnexus analyze` is writing. This attempt
// is one native open; lock-retry backoff lives in initLbug.
try {
const db = await openReadOnlyDatabase(dbPath);
shared = { db, refCount: 0, ftsLoaded: false, dbIdentity: await statDbIdentity(dbPath) };
dbCache.set(dbPath, shared);
} catch (err: unknown) {
const lastError = err instanceof Error ? err : new Error(String(err));
// Not retryable: the on-disk file's storage version doesn't change
// on its own, so looping LOCK_RETRY_ATTEMPTS times here would just
// repeat the same native exception. Fail immediately with an
// actionable message instead of GitNexus's generic "unavailable,
// retry later" (review finding on PR #3189 — this became reachable
// once the pinned engine version can trail behind whatever version
// last wrote an index, e.g. after downgrading the dependency).
if (isStorageVersionMismatchError(lastError)) {
throw new Error(`${STORAGE_VERSION_MISMATCH_SUGGESTION} (${lastError.message})`);
}
if (isWalCorruptionError(lastError)) {
try {
const db = await tryQuarantineAndReopen(dbPath, repoId);
shared = {
db,
refCount: 0,
ftsLoaded: false,
dbIdentity: await statDbIdentity(dbPath),
};
dbCache.set(dbPath, shared);
break;
} catch (retryErr) {
throw new Error(
`LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` +
`(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`,
);
}
// Not retryable: the on-disk file's storage version doesn't change
// on its own. Fail immediately with an actionable message.
throwIfStorageVersionMismatch(lastError);
if (isWalCorruptionError(lastError)) {
try {
const db = await tryQuarantineAndReopen(dbPath, repoId);
shared = {
db,
refCount: 0,
ftsLoaded: false,
dbIdentity: await statDbIdentity(dbPath),
};
dbCache.set(dbPath, shared);
} catch (retryErr) {
throw new Error(
`LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` +
`(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`,
);
}
}
if (!shared) {
if (
lastError.message.startsWith('LadybugDB checkpoint sidecar is missing') ||
lastError.message.startsWith('LadybugDB checkpoint sidecar is present but unreachable') ||
@ -854,21 +865,15 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
) {
throw lastError;
}
const isLockError =
if (
lastError.message.includes('Could not set lock') ||
/\block(\b|ed|ing)/i.test(lastError.message);
if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break;
await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_DELAY_MS * attempt));
/\block(\b|ed|ing)/i.test(lastError.message)
) {
return { status: 'retry', error: lastError };
}
throw ladybugUnavailableError(repoId, lastError);
}
}
if (!shared) {
throw new Error(
`LadybugDB unavailable for ${repoId}. Another process may be rebuilding the index. ` +
`Retry later. (${lastError?.message || 'unknown error'})`,
);
}
}
shared.refCount++;
@ -914,6 +919,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
});
ensureIdleTimer();
traceRss('init', repoId);
return { status: 'done', reopened: true };
}
/**

1
gitnexus/test/unit/basicblock-callee-ids-schema.test.ts
View file

@ -178,6 +178,7 @@ const makeConfigMock = () => {
WAL_RECOVERY_SUGGESTION: 'run analyze --force',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
};
return { mock, queries };

1
gitnexus/test/unit/convex-metadata-persistence-contract.test.ts
View file

@ -33,6 +33,7 @@ function makeConfigMock() {
WAL_RECOVERY_SUGGESTION: 'run analyze --force',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
},
};

2
gitnexus/test/unit/group/sync-windowed-resolution.test.ts
View file

@ -142,6 +142,8 @@ vi.mock('../../../src/core/lbug/lbug-config.js', () => ({
isWalCorruptionError: vi.fn(() => false),
WAL_RECOVERY_SUGGESTION: '',
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
sleep: vi.fn(async () => {}),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));

159
gitnexus/test/unit/lbug-adapter-wal-schema.test.ts
View file

@ -13,6 +13,7 @@
import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
import fs from 'node:fs/promises';
import path from 'node:path';
import { STORAGE_VERSION_MISMATCH_SUGGESTION } from '../../src/core/lbug/lbug-config.js';
// ─── Helpers ─────────────────────────────────────────────────────────────────
@ -37,6 +38,20 @@ const schemaMockFactory = async () => ({
...SCHEMA_MOCK,
});
async function mockLbugConfigForStorageVersion(overrides: Record<string, unknown>) {
const actual = await vi.importActual<typeof import('../../src/core/lbug/lbug-config.js')>(
'../../src/core/lbug/lbug-config.js',
);
vi.doMock('../../src/core/lbug/lbug-config.js', () => ({
...actual,
isDbBusyError: vi.fn(() => false),
isOpenRetryExhausted: vi.fn(() => false),
isWalCorruptionError: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
...overrides,
}));
}
function makeFsMock(dbPath: string) {
const ENOENT = Object.assign(new Error(`ENOENT: ${dbPath}`), { code: 'ENOENT' });
return {
@ -104,6 +119,17 @@ describe('doInitLbug WAL corruption guard — structural', () => {
expect(warnIdx).toBeGreaterThan(-1);
expect(walGuardIdx).toBeLessThan(warnIdx);
});
it('imports throwIfStorageVersionMismatch and uses it in the schema catch', () => {
expect(adapterSource).toMatch(/throwIfStorageVersionMismatch/);
expect(schemaLoopBody).toMatch(/isStorageVersionMismatchError\(err\)/);
expect(schemaLoopBody).toMatch(/throwIfStorageVersionMismatch\(err\)/);
const mismatchIdx = schemaLoopBody.indexOf('isStorageVersionMismatchError(err)');
const warnIdx = schemaLoopBody.indexOf('Schema creation warning');
expect(mismatchIdx).toBeGreaterThan(-1);
expect(mismatchIdx).toBeLessThan(warnIdx);
});
});
// ─── Behavioural tests ────────────────────────────────────────────────────────
@ -148,6 +174,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -202,6 +229,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -265,6 +293,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -319,6 +348,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -398,6 +428,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -481,6 +512,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -537,6 +569,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => {
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -661,6 +694,7 @@ describe('Symmetric WAL-size gate during missing-shadow recovery (PR #1747 D2)',
'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.',
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -837,3 +871,128 @@ describe('Symmetric WAL-size gate during missing-shadow recovery (PR #1747 D2)',
);
});
});
const NATIVE_STORAGE_VERSION_MISMATCH =
'Runtime exception: Trying to read a database file with a different version. Database file version: 43, Current build storage version: 42';
describe('doInitLbug storage-version fail-fast — behavioural', () => {
afterEach(() => {
vi.doUnmock('fs/promises');
vi.doUnmock('../../src/core/lbug/schema.js');
vi.doUnmock('../../src/core/lbug/lbug-config.js');
vi.doUnmock('../../src/core/lbug/extension-loader.js');
vi.doUnmock('../../src/core/logger.js');
vi.resetModules();
vi.clearAllMocks();
});
it('throws the rebuild hint when a writable schema query raises a storage-version mismatch', async () => {
vi.resetModules();
const dbPath = '/tmp/gitnexus-lbug-storage-version-schema/lbug';
const versionError = new Error(NATIVE_STORAGE_VERSION_MISMATCH);
const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() };
const conn = {
query: vi.fn().mockRejectedValueOnce(versionError).mockResolvedValue(queryResult),
close: vi.fn(async () => {}),
};
const db = { close: vi.fn(async () => {}) };
const warnMock = vi.fn();
vi.doMock('fs/promises', () => makeFsMock(dbPath));
vi.doMock('../../src/core/lbug/schema.js', schemaMockFactory);
await mockLbugConfigForStorageVersion({
openLbugConnection: vi.fn(async () => ({ db, conn })),
closeLbugConnection: vi.fn(async () => {}),
});
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
extensionManager: {
ensure: vi.fn(async () => true),
getCapabilities: vi.fn(() => []),
reset: vi.fn(),
},
}));
vi.doMock('../../src/core/logger.js', () => ({
logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
const err = await adapter.initLbug(dbPath).catch((e: unknown) => e);
expect(err).toBeInstanceOf(Error);
expect((err as Error).message).toContain(STORAGE_VERSION_MISMATCH_SUGGESTION);
expect((err as Error).message).toMatch(/database file with a different version/i);
expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining('Schema creation warning'));
expect(db.close).toHaveBeenCalled();
// Schema DDL is not retried. A second query is the CHECKPOINT inside safeClose.
expect(conn.query).toHaveBeenNthCalledWith(1, SCHEMA_MOCK.SCHEMA_QUERIES[0]);
expect(
conn.query.mock.calls.filter((call) => call[0] === SCHEMA_MOCK.SCHEMA_QUERIES[0]),
).toHaveLength(1);
});
it('throws the rebuild hint when writable openLbugConnection raises a storage-version mismatch', async () => {
vi.resetModules();
const dbPath = '/tmp/gitnexus-lbug-storage-version-writable-open/lbug';
const versionError = new Error(NATIVE_STORAGE_VERSION_MISMATCH);
const openLbugConnection = vi.fn(async () => {
throw versionError;
});
const warnMock = vi.fn();
vi.doMock('fs/promises', () => makeFsMock(dbPath));
vi.doMock('../../src/core/lbug/schema.js', schemaMockFactory);
await mockLbugConfigForStorageVersion({
openLbugConnection,
closeLbugConnection: vi.fn(async () => {}),
});
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
extensionManager: {
ensure: vi.fn(async () => true),
getCapabilities: vi.fn(() => []),
reset: vi.fn(),
},
}));
vi.doMock('../../src/core/logger.js', () => ({
logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
await expect(adapter.initLbug(dbPath)).rejects.toThrow(STORAGE_VERSION_MISMATCH_SUGGESTION);
expect(openLbugConnection).toHaveBeenCalledTimes(1);
expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining('Schema creation warning'));
});
it('throws the rebuild hint on a read-only open storage-version mismatch without lock-retrying', async () => {
vi.resetModules();
const dbPath = '/tmp/gitnexus-lbug-storage-version-readonly/lbug';
const versionError = new Error(NATIVE_STORAGE_VERSION_MISMATCH);
const openLbugConnection = vi.fn(async () => {
throw versionError;
});
vi.doMock('fs/promises', () => makeFsMock(dbPath));
vi.doMock('../../src/core/lbug/schema.js', schemaMockFactory);
await mockLbugConfigForStorageVersion({
openLbugConnection,
closeLbugConnection: vi.fn(async () => {}),
});
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
extensionManager: {
ensure: vi.fn(async () => true),
getCapabilities: vi.fn(() => []),
reset: vi.fn(),
},
}));
vi.doMock('../../src/core/logger.js', () => ({
logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() },
}));
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
await expect(
adapter.withLbugDb(dbPath, async () => 'unreached', { readOnly: true }),
).rejects.toThrow(STORAGE_VERSION_MISMATCH_SUGGESTION);
expect(openLbugConnection).toHaveBeenCalledTimes(1);
});
});

12
gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts
View file

@ -96,6 +96,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -170,6 +171,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -237,6 +239,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -307,6 +310,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -368,6 +372,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -442,6 +447,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -519,6 +525,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -580,6 +587,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -634,6 +642,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -699,6 +708,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -779,6 +789,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({
@ -859,6 +870,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => {
isOpenRetryExhausted: vi.fn(() => false),
waitForWindowsHandleRelease: vi.fn(async () => true),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));
vi.doMock('../../src/core/lbug/extension-loader.js', () => ({

50
gitnexus/test/unit/lbug-config-wal.test.ts
View file

@ -4,8 +4,11 @@ import {
createLbugDatabase,
estimateBufferPool,
isLbugCheckpointIoError,
isStorageVersionMismatchError,
isWalCorruptionError,
setBufferPoolSizeHint,
STORAGE_VERSION_MISMATCH_SUGGESTION,
throwIfStorageVersionMismatch,
_setOsPageSizeForTests,
bufferPoolExhaustionRemedy,
} from '../../src/core/lbug/lbug-config.js';
@ -46,6 +49,53 @@ describe('isWalCorruptionError', () => {
});
});
describe('isStorageVersionMismatchError / throwIfStorageVersionMismatch', () => {
const NATIVE =
'Runtime exception: Trying to read a database file with a different version. Database file version: 43, Current build storage version: 42';
it.each([
['documented native message', NATIVE],
['bare phrase', 'Trying to read a database file with a different version'],
])('matches storage-version mismatch: %s', (_label, msg) => {
expect(isStorageVersionMismatchError(msg)).toBe(true);
expect(isStorageVersionMismatchError(new Error(msg))).toBe(true);
});
it.each([
['lock error', 'Could not set lock on file : /path/to/db'],
['WAL corruption', 'Runtime exception: Corrupted wal file. Read out invalid WAL record type.'],
['generic', 'Query failed'],
['schema version wording', 'schema version mismatch in WAL'],
])('does not match non-version error: %s', (_label, msg) => {
expect(isStorageVersionMismatchError(msg)).toBe(false);
expect(isStorageVersionMismatchError(new Error(msg))).toBe(false);
});
it('handles non-string input', () => {
expect(isStorageVersionMismatchError(undefined)).toBe(false);
expect(isStorageVersionMismatchError(null)).toBe(false);
expect(isStorageVersionMismatchError(42)).toBe(false);
expect(isStorageVersionMismatchError(new Error('ok'))).toBe(false);
});
it('throwIfStorageVersionMismatch wraps the native message with the rebuild hint', () => {
expect(() => throwIfStorageVersionMismatch(new Error(NATIVE))).toThrow(
new RegExp(
`${STORAGE_VERSION_MISMATCH_SUGGESTION.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}.*different version`,
),
);
});
it('throwIfStorageVersionMismatch is a no-op for lock and WAL errors', () => {
expect(() =>
throwIfStorageVersionMismatch(new Error('Could not set lock on file : /path')),
).not.toThrow();
expect(() =>
throwIfStorageVersionMismatch(new Error('Runtime exception: Corrupted wal file.')),
).not.toThrow();
});
});
describe('createLbugDatabase WAL replay option', () => {
it('enables auto-checkpoint by default and uses default threshold (64 MiB)', () => {
const Database = vi.fn(function (this: any) {});

2
gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts
View file

@ -52,6 +52,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({
isWalCorruptionError: vi.fn(() => false),
WAL_RECOVERY_SUGGESTION: '',
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
sleep: vi.fn(async () => {}),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));

2
gitnexus/test/unit/lbug-pool-fts-load.test.ts
View file

@ -42,6 +42,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({
isWalCorruptionError: vi.fn(() => false),
WAL_RECOVERY_SUGGESTION: '',
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
sleep: vi.fn(async () => {}),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));

2
gitnexus/test/unit/lbug-pool-pinning.test.ts
View file

@ -51,6 +51,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({
isWalCorruptionError: vi.fn(() => false),
WAL_RECOVERY_SUGGESTION: '',
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
sleep: vi.fn(async () => {}),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));

147
gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts Normal file
View file

@ -0,0 +1,147 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import * as os from 'node:os';
import * as path from 'node:path';
import { mkdtempSync, writeFileSync, rmSync } from 'node:fs';
const { loadFTSExtensionMock, loadVectorExtensionMock, createLbugDatabaseMock } = vi.hoisted(
() => ({
loadFTSExtensionMock: vi.fn(),
loadVectorExtensionMock: vi.fn().mockResolvedValue(false),
createLbugDatabaseMock: vi.fn(),
}),
);
vi.mock('@ladybugdb/core', () => ({
default: {
Database: vi.fn(),
Connection: vi.fn(function (this: any) {
this.query = vi.fn().mockResolvedValue({
getAll: vi.fn().mockResolvedValue([]),
close: vi.fn(),
});
this.close = vi.fn().mockResolvedValue(undefined);
}),
},
}));
vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({
isReadOnlyDbError: vi.fn(() => false),
loadFTSExtension: loadFTSExtensionMock,
loadVectorExtension: loadVectorExtensionMock,
}));
vi.mock('../../src/core/lbug/lbug-config.js', async (importOriginal) => {
const actual = await importOriginal<typeof import('../../src/core/lbug/lbug-config.js')>();
return {
...actual,
createLbugDatabase: createLbugDatabaseMock,
toNativeSafePath: vi.fn((p: string) => p),
isWalCorruptionError: vi.fn(() => false),
WAL_RECOVERY_SUGGESTION: '',
};
});
vi.mock('../../src/core/lbug/sidecar-recovery.js', () => ({
preflightLbugSidecars: vi.fn().mockResolvedValue(undefined),
guardWalQuarantine: vi.fn().mockResolvedValue(undefined),
isMissingFsError: vi.fn(() => false),
isMissingShadowSidecarError: vi.fn(() => false),
isReadOnlyShadowReplayError: vi.fn(() => false),
quarantineWalForMissingShadow: vi.fn().mockResolvedValue(''),
quarantineSidecarsForDirtyRecovery: vi
.fn()
.mockResolvedValue({ moved: [], removed: [], failed: [] }),
renameFailureMessage: vi.fn((p: string) => `rename failed for ${p}`),
statIfExists: vi.fn().mockResolvedValue(null),
}));
const { initLbug, closeLbug, isLbugReady, unpinRepo } =
await import('../../src/core/lbug/pool-adapter.js');
const { STORAGE_VERSION_MISMATCH_SUGGESTION } =
await import('../../src/core/lbug/lbug-config.js');
const NATIVE_STORAGE_VERSION_MISMATCH =
'Runtime exception: Trying to read a database file with a different version. Database file version: 43, Current build storage version: 42';
function goodDb() {
return {
init: vi.fn().mockResolvedValue(undefined),
close: vi.fn().mockResolvedValue(undefined),
};
}
describe('pool-adapter storage-version fail-fast and lock-retry backoff', () => {
let tmpDir: string;
const touched = new Set<string>();
const dbPathFor = (repoId: string): string => {
const p = path.join(tmpDir, `${repoId}.lbug`);
writeFileSync(p, '');
return p;
};
beforeEach(() => {
tmpDir = mkdtempSync(path.join(os.tmpdir(), 'gn-pool-sv-lock-'));
loadFTSExtensionMock.mockResolvedValue(true);
createLbugDatabaseMock.mockReset();
createLbugDatabaseMock.mockImplementation(() => goodDb());
});
afterEach(async () => {
vi.useRealTimers();
await closeLbug().catch(() => {});
for (const id of touched) unpinRepo(id);
touched.clear();
loadFTSExtensionMock.mockReset();
rmSync(tmpDir, { recursive: true, force: true });
});
it('fail-fasts a storage-version mismatch with the rebuild hint and does not lock-retry', async () => {
const repoId = 'version-mismatch';
touched.add(repoId);
createLbugDatabaseMock.mockImplementation(() => {
throw new Error(NATIVE_STORAGE_VERSION_MISMATCH);
});
await expect(initLbug(repoId, dbPathFor(repoId))).rejects.toThrow(
STORAGE_VERSION_MISMATCH_SUGGESTION,
);
expect(createLbugDatabaseMock).toHaveBeenCalledTimes(1);
});
it('does not hold withPoolLock across lock-retry sleep — another repo can init during backoff', async () => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
const lockedId = 'locked';
const otherId = 'other';
touched.add(lockedId);
touched.add(otherId);
const lockedPath = dbPathFor(lockedId);
const otherPath = dbPathFor(otherId);
createLbugDatabaseMock.mockImplementation((_mod: unknown, dbPath: string) => {
if (String(dbPath).includes(`${path.sep}locked.lbug`) || String(dbPath).endsWith('locked.lbug')) {
throw new Error('Could not set lock on file : /tmp/locked.lbug');
}
return goodDb();
});
const lockedInit = initLbug(lockedId, lockedPath);
await vi.waitFor(() => {
expect(createLbugDatabaseMock).toHaveBeenCalledTimes(1);
});
const otherInit = initLbug(otherId, otherPath);
await expect(otherInit).resolves.toBe(true);
expect(isLbugReady(otherId)).toBe(true);
expect(createLbugDatabaseMock).toHaveBeenCalledTimes(2);
await vi.advanceTimersByTimeAsync(2000);
await vi.waitFor(() => {
expect(createLbugDatabaseMock).toHaveBeenCalledTimes(3);
});
await vi.advanceTimersByTimeAsync(4000);
await expect(lockedInit).rejects.toThrow(/LadybugDB unavailable for locked/);
expect(createLbugDatabaseMock).toHaveBeenCalledTimes(4);
});
});

2
gitnexus/test/unit/pool-wal-recovery.test.ts
View file

@ -45,6 +45,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({
return /corrupt(ed)?\s+wal|invalid\s+wal\s+record/i.test(msg);
}),
isStorageVersionMismatchError: vi.fn(() => false),
throwIfStorageVersionMismatch: vi.fn(),
sleep: vi.fn(async () => {}),
STORAGE_VERSION_MISMATCH_SUGGESTION: '',
}));