diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index b38a5f4d6..71c3dfe69 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -51,12 +51,12 @@ import { isOpenRetryExhausted, isStorageVersionMismatchError, isWalCorruptionError, + throwIfStorageVersionMismatch, bufferPoolExhaustionRemedy, openLbugConnection, sleep, toNativeSafePath, resolveNativeSafeStorageDir, - STORAGE_VERSION_MISMATCH_SUGGESTION, WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, type LbugConnectionHandle, @@ -702,6 +702,11 @@ const runSchemaCreationQueries = async (dbPath: string): Promise ` Original error: ${msg.slice(0, 200)}`, ); } + if (isStorageVersionMismatchError(err)) { + await safeClose(); + resetOpenConnectionState(); + throwIfStorageVersionMismatch(err); + } if (!msg.includes('already exists') && !isDbBusyError(err) && !isReadOnlyDbError(err)) { logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`); } @@ -832,10 +837,7 @@ const doInitLbug = async ( // downgrading the dependency). Mirrors the pool-adapter.ts check for // the same error, on the separate open path /api/graph and /api/query // actually use (withLbugDb, not the pool). - if (isStorageVersionMismatchError(err)) { - const msg = err instanceof Error ? err.message : String(err); - throw new Error(`${STORAGE_VERSION_MISMATCH_SUGGESTION} (${msg})`); - } + throwIfStorageVersionMismatch(err); throw err; } db = usable.db; @@ -946,10 +948,18 @@ const doInitLbug = async ( allowQuarantine: true, }); - const opened = await openLbugConnection(lbug, dbPath); - db = opened.db; - conn = opened.conn; - currentDbReadOnly = false; + try { + const opened = await openLbugConnection(lbug, dbPath); + db = opened.db; + conn = opened.conn; + currentDbReadOnly = false; + } catch (err) { + // Incremental analyze can hit a storage-version mismatch on construct + // or (more often) on the first schema query below. Fail immediately + // with the rebuild hint instead of warn-and-continue. + throwIfStorageVersionMismatch(err); + throw err; + } } finally { await releaseInitLock(); } diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index df799140a..3fe32b1aa 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -579,6 +579,13 @@ export function isStorageVersionMismatchError(err: unknown): boolean { return STORAGE_VERSION_MISMATCH_RE.test(msg); } +/** Throws the rebuild-hint Error when `err` is a storage-version mismatch. */ +export function throwIfStorageVersionMismatch(err: unknown): void { + if (!isStorageVersionMismatchError(err)) return; + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`${STORAGE_VERSION_MISMATCH_SUGGESTION} (${msg})`); +} + // ─── Ladybug WAL checkpoint IO error matchers ─────────────────────────────── // // Matched against LadybugDB v0.18.0 (see `gitnexus/package.json` diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index 34514c416..8e9291d08 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -22,9 +22,9 @@ import { closeQueryResults } from './query-result-utils.js'; import { warnIfQueryTextUnbounded } from './query-batch.js'; import { createLbugDatabase, - isStorageVersionMismatchError, isWalCorruptionError, - STORAGE_VERSION_MISMATCH_SUGGESTION, + sleep, + throwIfStorageVersionMismatch, toNativeSafePath, WAL_RECOVERY_SUGGESTION, } from './lbug-config.js'; @@ -697,29 +697,10 @@ async function tryQuarantineAndReopen(dbPath: string, repoId: string): Promise = Promise.resolve(); function withPoolLock(fn: () => Promise): Promise { const run = poolLock.then(fn, fn); @@ -730,10 +711,45 @@ function withPoolLock(fn: () => Promise): Promise { return run; } -export const initLbug = (repoId: string, dbPath: string): Promise => - withPoolLock(() => initLbugInner(repoId, dbPath)); +type InitLbugAttempt = { status: 'done'; reopened: boolean } | { status: 'retry'; error: Error }; -const initLbugInner = async (repoId: string, dbPath: string): Promise => { +function ladybugUnavailableError(repoId: string, err: Error | undefined): Error { + return new Error( + `LadybugDB unavailable for ${repoId}. Another process may be rebuilding the index. ` + + `Retry later. (${err?.message || 'unknown error'})`, + ); +} + +/** + * Initialize (or reuse) a Database + connection pool for a specific repo. + * Retries on lock errors (e.g., when `gitnexus analyze` is running). + * + * Concurrent calls (for the same repoId or different ones) serialize on + * poolLock below for evict / close / native open / register, so a second + * caller for a repo already being initialized waits its turn and then hits + * the "existing" fast path — no separate per-repoId dedup needed. Lock-retry + * *sleeps* run outside the mutex so one analyze-locked repo does not block + * every other pool init for LOCK_RETRY_DELAY_MS * attempt. + * + * Returns `true` when this call (re)opened a fresh handle onto the current + * on-disk file, `false` when it reused/served the existing handle (unchanged, + * or changed-but-a-query-is-in-flight). Callers that gate their own freshness + * bookkeeping on "did the pool actually roll over" (LocalBackend) use the + * return value; callers that only need the pool ready can ignore it. + */ +export const initLbug = async (repoId: string, dbPath: string): Promise => { + let lastError: Error | undefined; + for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) { + const result = await withPoolLock(() => initLbugInner(repoId, dbPath)); + if (result.status === 'done') return result.reopened; + lastError = result.error; + if (attempt === LOCK_RETRY_ATTEMPTS) break; + await sleep(LOCK_RETRY_DELAY_MS * attempt); + } + throw ladybugUnavailableError(repoId, lastError); +}; + +const initLbugInner = async (repoId: string, dbPath: string): Promise => { const existing = pool.get(repoId); if (existing) { existing.lastUsed = Date.now(); @@ -742,7 +758,9 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise = // unlinked-but-open) inode until LRU/idle eviction — a stale-read window // of up to IDLE_TIMEOUT_MS after analyze finishes. const current = await statDbIdentity(dbPath); - if (!dbIdentityChanged(existing.dbIdentity, current)) return false; // unchanged → reuse + if (!dbIdentityChanged(existing.dbIdentity, current)) { + return { status: 'done', reopened: false }; // unchanged → reuse + } // A query is in flight on this entry; closing its connection (and the // shared Database at refCount 0) mid-use is a native use-after-free. Serve // the current handle for this dispatch — the next initLbug that finds the @@ -753,13 +771,12 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise = // (a complete older snapshot), just not the newest. Callers that route // freshness THROUGH initLbug (rather than calling closeLbug directly) get // this guard for free; that is why LocalBackend delegates here (#2614). - if (existing.checkedOut > 0) return false; + if (existing.checkedOut > 0) return { status: 'done', reopened: false }; // Awaited: see the rationale on the evictLRU call site in doInitLbug below. await closeOne(repoId); // idle & changed → evict, then fall through to reopen the new file } - await doInitLbug(repoId, dbPath); - return true; + return doInitLbug(repoId, dbPath); }; /** @@ -767,7 +784,7 @@ const initLbugInner = async (repoId: string, dbPath: string): Promise = * Pool entry is registered LAST so concurrent executeQuery calls see either * "not initialized" (and throw) or a fully ready pool — never a half-built one. */ -async function doInitLbug(repoId: string, dbPath: string): Promise { +async function doInitLbug(repoId: string, dbPath: string): Promise { // Check if database exists try { await fs.stat(dbPath); @@ -777,9 +794,12 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { // Awaited: without this, the connection opened just below could race the // checkpoint from the LRU victim's still-in-flight close (see evictLRU / - // closeOne). initLbug's outer poolLock keeps this whole function exclusive - // across concurrent repos too, so this await only needs to cover this call's - // own evict-then-reopen — not other callers. + // closeOne). The caller holds withPoolLock for this attempt, so this await + // only covers this call's own evict-then-reopen — not other callers. + // Lock-retry re-enters this function after sleeping *outside* the mutex. + // evictLRU is a no-op unless the pool is full again (another repo may have + // taken the slot we freed on a prior attempt). Skipping it on retry would + // let a 6th native open race a still-resident victim's checkpoint. await evictLRU(); // Reuse an existing native Database if another repoId already opened this path. @@ -805,47 +825,38 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { if (!shared) { // Open in read-only mode — MCP server never writes to the database. // This allows multiple MCP server instances to read concurrently, and - // avoids lock conflicts when `gitnexus analyze` is writing. - let lastError: Error | null = null; - for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) { - try { - const db = await openReadOnlyDatabase(dbPath); - shared = { db, refCount: 0, ftsLoaded: false, dbIdentity: await statDbIdentity(dbPath) }; - dbCache.set(dbPath, shared); - break; - } catch (err: any) { - lastError = err instanceof Error ? err : new Error(String(err)); + // avoids lock conflicts when `gitnexus analyze` is writing. This attempt + // is one native open; lock-retry backoff lives in initLbug. + try { + const db = await openReadOnlyDatabase(dbPath); + shared = { db, refCount: 0, ftsLoaded: false, dbIdentity: await statDbIdentity(dbPath) }; + dbCache.set(dbPath, shared); + } catch (err: unknown) { + const lastError = err instanceof Error ? err : new Error(String(err)); - // Not retryable: the on-disk file's storage version doesn't change - // on its own, so looping LOCK_RETRY_ATTEMPTS times here would just - // repeat the same native exception. Fail immediately with an - // actionable message instead of GitNexus's generic "unavailable, - // retry later" (review finding on PR #3189 — this became reachable - // once the pinned engine version can trail behind whatever version - // last wrote an index, e.g. after downgrading the dependency). - if (isStorageVersionMismatchError(lastError)) { - throw new Error(`${STORAGE_VERSION_MISMATCH_SUGGESTION} (${lastError.message})`); - } - - if (isWalCorruptionError(lastError)) { - try { - const db = await tryQuarantineAndReopen(dbPath, repoId); - shared = { - db, - refCount: 0, - ftsLoaded: false, - dbIdentity: await statDbIdentity(dbPath), - }; - dbCache.set(dbPath, shared); - break; - } catch (retryErr) { - throw new Error( - `LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` + - `(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`, - ); - } + // Not retryable: the on-disk file's storage version doesn't change + // on its own. Fail immediately with an actionable message. + throwIfStorageVersionMismatch(lastError); + + if (isWalCorruptionError(lastError)) { + try { + const db = await tryQuarantineAndReopen(dbPath, repoId); + shared = { + db, + refCount: 0, + ftsLoaded: false, + dbIdentity: await statDbIdentity(dbPath), + }; + dbCache.set(dbPath, shared); + } catch (retryErr) { + throw new Error( + `LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` + + `(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`, + ); } + } + if (!shared) { if ( lastError.message.startsWith('LadybugDB checkpoint sidecar is missing') || lastError.message.startsWith('LadybugDB checkpoint sidecar is present but unreachable') || @@ -854,21 +865,15 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { ) { throw lastError; } - - const isLockError = + if ( lastError.message.includes('Could not set lock') || - /\block(\b|ed|ing)/i.test(lastError.message); - if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break; - await new Promise((resolve) => setTimeout(resolve, LOCK_RETRY_DELAY_MS * attempt)); + /\block(\b|ed|ing)/i.test(lastError.message) + ) { + return { status: 'retry', error: lastError }; + } + throw ladybugUnavailableError(repoId, lastError); } } - - if (!shared) { - throw new Error( - `LadybugDB unavailable for ${repoId}. Another process may be rebuilding the index. ` + - `Retry later. (${lastError?.message || 'unknown error'})`, - ); - } } shared.refCount++; @@ -914,6 +919,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { }); ensureIdleTimer(); traceRss('init', repoId); + return { status: 'done', reopened: true }; } /** diff --git a/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts b/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts index 22b170958..a9e7ab1ad 100644 --- a/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts +++ b/gitnexus/test/unit/basicblock-callee-ids-schema.test.ts @@ -178,6 +178,7 @@ const makeConfigMock = () => { WAL_RECOVERY_SUGGESTION: 'run analyze --force', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', }; return { mock, queries }; diff --git a/gitnexus/test/unit/convex-metadata-persistence-contract.test.ts b/gitnexus/test/unit/convex-metadata-persistence-contract.test.ts index ed215eedb..b5e55a1cd 100644 --- a/gitnexus/test/unit/convex-metadata-persistence-contract.test.ts +++ b/gitnexus/test/unit/convex-metadata-persistence-contract.test.ts @@ -33,6 +33,7 @@ function makeConfigMock() { WAL_RECOVERY_SUGGESTION: 'run analyze --force', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', }, }; diff --git a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts index 90dfe39bd..ec42adc3d 100644 --- a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts +++ b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts @@ -142,6 +142,8 @@ vi.mock('../../../src/core/lbug/lbug-config.js', () => ({ isWalCorruptionError: vi.fn(() => false), WAL_RECOVERY_SUGGESTION: '', isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), + sleep: vi.fn(async () => {}), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); diff --git a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts index dde36271f..f5eded56a 100644 --- a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts +++ b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts @@ -13,6 +13,7 @@ import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; import fs from 'node:fs/promises'; import path from 'node:path'; +import { STORAGE_VERSION_MISMATCH_SUGGESTION } from '../../src/core/lbug/lbug-config.js'; // ─── Helpers ───────────────────────────────────────────────────────────────── @@ -37,6 +38,20 @@ const schemaMockFactory = async () => ({ ...SCHEMA_MOCK, }); +async function mockLbugConfigForStorageVersion(overrides: Record) { + const actual = await vi.importActual( + '../../src/core/lbug/lbug-config.js', + ); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + ...actual, + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn(() => false), + waitForWindowsHandleRelease: vi.fn(async () => true), + ...overrides, + })); +} + function makeFsMock(dbPath: string) { const ENOENT = Object.assign(new Error(`ENOENT: ${dbPath}`), { code: 'ENOENT' }); return { @@ -104,6 +119,17 @@ describe('doInitLbug WAL corruption guard — structural', () => { expect(warnIdx).toBeGreaterThan(-1); expect(walGuardIdx).toBeLessThan(warnIdx); }); + + it('imports throwIfStorageVersionMismatch and uses it in the schema catch', () => { + expect(adapterSource).toMatch(/throwIfStorageVersionMismatch/); + expect(schemaLoopBody).toMatch(/isStorageVersionMismatchError\(err\)/); + expect(schemaLoopBody).toMatch(/throwIfStorageVersionMismatch\(err\)/); + const mismatchIdx = schemaLoopBody.indexOf('isStorageVersionMismatchError(err)'); + const warnIdx = schemaLoopBody.indexOf('Schema creation warning'); + expect(mismatchIdx).toBeGreaterThan(-1); + expect(mismatchIdx).toBeLessThan(warnIdx); + }); + }); // ─── Behavioural tests ──────────────────────────────────────────────────────── @@ -148,6 +174,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -202,6 +229,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -265,6 +293,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -319,6 +348,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -398,6 +428,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -481,6 +512,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -537,6 +569,7 @@ describe('doInitLbug WAL corruption guard — behavioural', () => { 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -661,6 +694,7 @@ describe('Symmetric WAL-size gate during missing-shadow recovery (PR #1747 D2)', 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -837,3 +871,128 @@ describe('Symmetric WAL-size gate during missing-shadow recovery (PR #1747 D2)', ); }); }); + +const NATIVE_STORAGE_VERSION_MISMATCH = + 'Runtime exception: Trying to read a database file with a different version. Database file version: 43, Current build storage version: 42'; + +describe('doInitLbug storage-version fail-fast — behavioural', () => { + afterEach(() => { + vi.doUnmock('fs/promises'); + vi.doUnmock('../../src/core/lbug/schema.js'); + vi.doUnmock('../../src/core/lbug/lbug-config.js'); + vi.doUnmock('../../src/core/lbug/extension-loader.js'); + vi.doUnmock('../../src/core/logger.js'); + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('throws the rebuild hint when a writable schema query raises a storage-version mismatch', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-storage-version-schema/lbug'; + const versionError = new Error(NATIVE_STORAGE_VERSION_MISMATCH); + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const conn = { + query: vi.fn().mockRejectedValueOnce(versionError).mockResolvedValue(queryResult), + close: vi.fn(async () => {}), + }; + const db = { close: vi.fn(async () => {}) }; + const warnMock = vi.fn(); + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', schemaMockFactory); + await mockLbugConfigForStorageVersion({ + openLbugConnection: vi.fn(async () => ({ db, conn })), + closeLbugConnection: vi.fn(async () => {}), + }); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + const err = await adapter.initLbug(dbPath).catch((e: unknown) => e); + expect(err).toBeInstanceOf(Error); + expect((err as Error).message).toContain(STORAGE_VERSION_MISMATCH_SUGGESTION); + expect((err as Error).message).toMatch(/database file with a different version/i); + expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining('Schema creation warning')); + expect(db.close).toHaveBeenCalled(); + // Schema DDL is not retried. A second query is the CHECKPOINT inside safeClose. + expect(conn.query).toHaveBeenNthCalledWith(1, SCHEMA_MOCK.SCHEMA_QUERIES[0]); + expect( + conn.query.mock.calls.filter((call) => call[0] === SCHEMA_MOCK.SCHEMA_QUERIES[0]), + ).toHaveLength(1); + }); + + it('throws the rebuild hint when writable openLbugConnection raises a storage-version mismatch', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-storage-version-writable-open/lbug'; + const versionError = new Error(NATIVE_STORAGE_VERSION_MISMATCH); + const openLbugConnection = vi.fn(async () => { + throw versionError; + }); + const warnMock = vi.fn(); + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', schemaMockFactory); + await mockLbugConfigForStorageVersion({ + openLbugConnection, + closeLbugConnection: vi.fn(async () => {}), + }); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await expect(adapter.initLbug(dbPath)).rejects.toThrow(STORAGE_VERSION_MISMATCH_SUGGESTION); + expect(openLbugConnection).toHaveBeenCalledTimes(1); + expect(warnMock).not.toHaveBeenCalledWith(expect.stringContaining('Schema creation warning')); + }); + + it('throws the rebuild hint on a read-only open storage-version mismatch without lock-retrying', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-storage-version-readonly/lbug'; + const versionError = new Error(NATIVE_STORAGE_VERSION_MISMATCH); + const openLbugConnection = vi.fn(async () => { + throw versionError; + }); + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', schemaMockFactory); + await mockLbugConfigForStorageVersion({ + openLbugConnection, + closeLbugConnection: vi.fn(async () => {}), + }); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await expect( + adapter.withLbugDb(dbPath, async () => 'unreached', { readOnly: true }), + ).rejects.toThrow(STORAGE_VERSION_MISMATCH_SUGGESTION); + expect(openLbugConnection).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts b/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts index 63acae912..8592546dc 100644 --- a/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts +++ b/gitnexus/test/unit/lbug-checkpoint-lifecycle.test.ts @@ -96,6 +96,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -170,6 +171,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -237,6 +239,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -307,6 +310,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -368,6 +372,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -442,6 +447,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -519,6 +525,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -580,6 +587,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -634,6 +642,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -699,6 +708,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -779,6 +789,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ @@ -859,6 +870,7 @@ describe('lbug adapter CHECKPOINT lifecycle', () => { isOpenRetryExhausted: vi.fn(() => false), waitForWindowsHandleRelease: vi.fn(async () => true), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ diff --git a/gitnexus/test/unit/lbug-config-wal.test.ts b/gitnexus/test/unit/lbug-config-wal.test.ts index 1150a0da7..61a291da7 100644 --- a/gitnexus/test/unit/lbug-config-wal.test.ts +++ b/gitnexus/test/unit/lbug-config-wal.test.ts @@ -4,8 +4,11 @@ import { createLbugDatabase, estimateBufferPool, isLbugCheckpointIoError, + isStorageVersionMismatchError, isWalCorruptionError, setBufferPoolSizeHint, + STORAGE_VERSION_MISMATCH_SUGGESTION, + throwIfStorageVersionMismatch, _setOsPageSizeForTests, bufferPoolExhaustionRemedy, } from '../../src/core/lbug/lbug-config.js'; @@ -46,6 +49,53 @@ describe('isWalCorruptionError', () => { }); }); +describe('isStorageVersionMismatchError / throwIfStorageVersionMismatch', () => { + const NATIVE = + 'Runtime exception: Trying to read a database file with a different version. Database file version: 43, Current build storage version: 42'; + + it.each([ + ['documented native message', NATIVE], + ['bare phrase', 'Trying to read a database file with a different version'], + ])('matches storage-version mismatch: %s', (_label, msg) => { + expect(isStorageVersionMismatchError(msg)).toBe(true); + expect(isStorageVersionMismatchError(new Error(msg))).toBe(true); + }); + + it.each([ + ['lock error', 'Could not set lock on file : /path/to/db'], + ['WAL corruption', 'Runtime exception: Corrupted wal file. Read out invalid WAL record type.'], + ['generic', 'Query failed'], + ['schema version wording', 'schema version mismatch in WAL'], + ])('does not match non-version error: %s', (_label, msg) => { + expect(isStorageVersionMismatchError(msg)).toBe(false); + expect(isStorageVersionMismatchError(new Error(msg))).toBe(false); + }); + + it('handles non-string input', () => { + expect(isStorageVersionMismatchError(undefined)).toBe(false); + expect(isStorageVersionMismatchError(null)).toBe(false); + expect(isStorageVersionMismatchError(42)).toBe(false); + expect(isStorageVersionMismatchError(new Error('ok'))).toBe(false); + }); + + it('throwIfStorageVersionMismatch wraps the native message with the rebuild hint', () => { + expect(() => throwIfStorageVersionMismatch(new Error(NATIVE))).toThrow( + new RegExp( + `${STORAGE_VERSION_MISMATCH_SUGGESTION.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}.*different version`, + ), + ); + }); + + it('throwIfStorageVersionMismatch is a no-op for lock and WAL errors', () => { + expect(() => + throwIfStorageVersionMismatch(new Error('Could not set lock on file : /path')), + ).not.toThrow(); + expect(() => + throwIfStorageVersionMismatch(new Error('Runtime exception: Corrupted wal file.')), + ).not.toThrow(); + }); +}); + describe('createLbugDatabase WAL replay option', () => { it('enables auto-checkpoint by default and uses default threshold (64 MiB)', () => { const Database = vi.fn(function (this: any) {}); diff --git a/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts b/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts index 27db187d9..5cc2fc32f 100644 --- a/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts +++ b/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts @@ -52,6 +52,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({ isWalCorruptionError: vi.fn(() => false), WAL_RECOVERY_SUGGESTION: '', isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), + sleep: vi.fn(async () => {}), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); diff --git a/gitnexus/test/unit/lbug-pool-fts-load.test.ts b/gitnexus/test/unit/lbug-pool-fts-load.test.ts index 921df150e..0bf9cfb83 100644 --- a/gitnexus/test/unit/lbug-pool-fts-load.test.ts +++ b/gitnexus/test/unit/lbug-pool-fts-load.test.ts @@ -42,6 +42,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({ isWalCorruptionError: vi.fn(() => false), WAL_RECOVERY_SUGGESTION: '', isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), + sleep: vi.fn(async () => {}), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); diff --git a/gitnexus/test/unit/lbug-pool-pinning.test.ts b/gitnexus/test/unit/lbug-pool-pinning.test.ts index ab13ff1fd..106456cb6 100644 --- a/gitnexus/test/unit/lbug-pool-pinning.test.ts +++ b/gitnexus/test/unit/lbug-pool-pinning.test.ts @@ -51,6 +51,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({ isWalCorruptionError: vi.fn(() => false), WAL_RECOVERY_SUGGESTION: '', isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), + sleep: vi.fn(async () => {}), STORAGE_VERSION_MISMATCH_SUGGESTION: '', })); diff --git a/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts b/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts new file mode 100644 index 000000000..8a70a177e --- /dev/null +++ b/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts @@ -0,0 +1,147 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; + +const { loadFTSExtensionMock, loadVectorExtensionMock, createLbugDatabaseMock } = vi.hoisted( + () => ({ + loadFTSExtensionMock: vi.fn(), + loadVectorExtensionMock: vi.fn().mockResolvedValue(false), + createLbugDatabaseMock: vi.fn(), + }), +); + +vi.mock('@ladybugdb/core', () => ({ + default: { + Database: vi.fn(), + Connection: vi.fn(function (this: any) { + this.query = vi.fn().mockResolvedValue({ + getAll: vi.fn().mockResolvedValue([]), + close: vi.fn(), + }); + this.close = vi.fn().mockResolvedValue(undefined); + }), + }, +})); + +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + isReadOnlyDbError: vi.fn(() => false), + loadFTSExtension: loadFTSExtensionMock, + loadVectorExtension: loadVectorExtensionMock, +})); + +vi.mock('../../src/core/lbug/lbug-config.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createLbugDatabase: createLbugDatabaseMock, + toNativeSafePath: vi.fn((p: string) => p), + isWalCorruptionError: vi.fn(() => false), + WAL_RECOVERY_SUGGESTION: '', + }; +}); + +vi.mock('../../src/core/lbug/sidecar-recovery.js', () => ({ + preflightLbugSidecars: vi.fn().mockResolvedValue(undefined), + guardWalQuarantine: vi.fn().mockResolvedValue(undefined), + isMissingFsError: vi.fn(() => false), + isMissingShadowSidecarError: vi.fn(() => false), + isReadOnlyShadowReplayError: vi.fn(() => false), + quarantineWalForMissingShadow: vi.fn().mockResolvedValue(''), + quarantineSidecarsForDirtyRecovery: vi + .fn() + .mockResolvedValue({ moved: [], removed: [], failed: [] }), + renameFailureMessage: vi.fn((p: string) => `rename failed for ${p}`), + statIfExists: vi.fn().mockResolvedValue(null), +})); + +const { initLbug, closeLbug, isLbugReady, unpinRepo } = + await import('../../src/core/lbug/pool-adapter.js'); +const { STORAGE_VERSION_MISMATCH_SUGGESTION } = + await import('../../src/core/lbug/lbug-config.js'); + +const NATIVE_STORAGE_VERSION_MISMATCH = + 'Runtime exception: Trying to read a database file with a different version. Database file version: 43, Current build storage version: 42'; + +function goodDb() { + return { + init: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + }; +} + +describe('pool-adapter storage-version fail-fast and lock-retry backoff', () => { + let tmpDir: string; + const touched = new Set(); + + const dbPathFor = (repoId: string): string => { + const p = path.join(tmpDir, `${repoId}.lbug`); + writeFileSync(p, ''); + return p; + }; + + beforeEach(() => { + tmpDir = mkdtempSync(path.join(os.tmpdir(), 'gn-pool-sv-lock-')); + loadFTSExtensionMock.mockResolvedValue(true); + createLbugDatabaseMock.mockReset(); + createLbugDatabaseMock.mockImplementation(() => goodDb()); + }); + + afterEach(async () => { + vi.useRealTimers(); + await closeLbug().catch(() => {}); + for (const id of touched) unpinRepo(id); + touched.clear(); + loadFTSExtensionMock.mockReset(); + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('fail-fasts a storage-version mismatch with the rebuild hint and does not lock-retry', async () => { + const repoId = 'version-mismatch'; + touched.add(repoId); + createLbugDatabaseMock.mockImplementation(() => { + throw new Error(NATIVE_STORAGE_VERSION_MISMATCH); + }); + + await expect(initLbug(repoId, dbPathFor(repoId))).rejects.toThrow( + STORAGE_VERSION_MISMATCH_SUGGESTION, + ); + expect(createLbugDatabaseMock).toHaveBeenCalledTimes(1); + }); + + it('does not hold withPoolLock across lock-retry sleep — another repo can init during backoff', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + + const lockedId = 'locked'; + const otherId = 'other'; + touched.add(lockedId); + touched.add(otherId); + const lockedPath = dbPathFor(lockedId); + const otherPath = dbPathFor(otherId); + + createLbugDatabaseMock.mockImplementation((_mod: unknown, dbPath: string) => { + if (String(dbPath).includes(`${path.sep}locked.lbug`) || String(dbPath).endsWith('locked.lbug')) { + throw new Error('Could not set lock on file : /tmp/locked.lbug'); + } + return goodDb(); + }); + + const lockedInit = initLbug(lockedId, lockedPath); + await vi.waitFor(() => { + expect(createLbugDatabaseMock).toHaveBeenCalledTimes(1); + }); + + const otherInit = initLbug(otherId, otherPath); + await expect(otherInit).resolves.toBe(true); + expect(isLbugReady(otherId)).toBe(true); + expect(createLbugDatabaseMock).toHaveBeenCalledTimes(2); + + await vi.advanceTimersByTimeAsync(2000); + await vi.waitFor(() => { + expect(createLbugDatabaseMock).toHaveBeenCalledTimes(3); + }); + await vi.advanceTimersByTimeAsync(4000); + await expect(lockedInit).rejects.toThrow(/LadybugDB unavailable for locked/); + expect(createLbugDatabaseMock).toHaveBeenCalledTimes(4); + }); +}); diff --git a/gitnexus/test/unit/pool-wal-recovery.test.ts b/gitnexus/test/unit/pool-wal-recovery.test.ts index dbbd7f9de..6ae7cfe06 100644 --- a/gitnexus/test/unit/pool-wal-recovery.test.ts +++ b/gitnexus/test/unit/pool-wal-recovery.test.ts @@ -45,6 +45,8 @@ vi.mock('../../src/core/lbug/lbug-config.js', () => ({ return /corrupt(ed)?\s+wal|invalid\s+wal\s+record/i.test(msg); }), isStorageVersionMismatchError: vi.fn(() => false), + throwIfStorageVersionMismatch: vi.fn(), + sleep: vi.fn(async () => {}), STORAGE_VERSION_MISMATCH_SUGGESTION: '', }));