diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index 5534f7d8b..a3e90051f 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -42,10 +42,23 @@ export const LBUG_MAX_DB_SIZE: number = (() => { return 16 * 1024 * 1024 * 1024; })(); +/** Matches WAL corruption errors from the LadybugDB engine. */ +const WAL_CORRUPTION_RE = /corrupt(ed)?\s+wal|invalid\s+wal\s+record|wal.*corrupt|checksum.*wal/i; + +export const WAL_RECOVERY_SUGGESTION = + 'WAL corruption detected. Run `gitnexus analyze` to rebuild the index.'; + +export function isWalCorruptionError(err: unknown): boolean { + if (!err) return false; + const msg = err instanceof Error ? err.message : String(err); + return WAL_CORRUPTION_RE.test(msg); +} + type LbugModule = typeof lbug; export interface LbugDatabaseOptions { readOnly?: boolean; + throwOnWalReplayFailure?: boolean; } export interface LbugConnectionHandle { @@ -58,13 +71,18 @@ export function createLbugDatabase( databasePath: string, options: LbugDatabaseOptions = {}, ): lbug.Database { - return new lbugModule.Database( + // .d.ts declares fewer args than the native constructor accepts. + return new (lbugModule.Database as any)( databasePath, - 0, - false, + 0, // bufferManagerSize + false, // enableCompression (pinned for v0.16.0) options.readOnly ?? false, LBUG_MAX_DB_SIZE, - ); + true, // autoCheckpoint + -1, // checkpointThreshold + options.throwOnWalReplayFailure ?? true, + true, // enableChecksums + ) as lbug.Database; } export async function openLbugConnection( diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index ca1c45611..ed999907e 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -18,7 +18,7 @@ import fs from 'fs/promises'; import lbug from '@ladybugdb/core'; import { loadFTSExtension } from './lbug-adapter.js'; -import { createLbugDatabase } from './lbug-config.js'; +import { createLbugDatabase, isWalCorruptionError } from './lbug-config.js'; /** Per-repo pool: one Database, many Connections */ interface PoolEntry { @@ -97,7 +97,7 @@ let idleTimer: ReturnType | null = null; // @ladybugdb/core), corrupting stdout in the pre-sentinel window. Routing // through the leaf breaks that chain. export { realStdoutWrite, realStderrWrite, setActiveStdoutWrite } from '../../mcp/stdio-capture.js'; -import { getActiveStdoutWrite } from '../../mcp/stdio-capture.js'; +import { getActiveStdoutWrite, realStderrWrite } from '../../mcp/stdio-capture.js'; let stdoutSilenceCount = 0; /** True while pre-warming connections — prevents watchdog from prematurely restoring stdout */ @@ -263,6 +263,46 @@ const WAITER_TIMEOUT_MS = 15_000; const LOCK_RETRY_ATTEMPTS = 3; const LOCK_RETRY_DELAY_MS = 2000; +async function openReadOnlyDatabase(dbPath: string): Promise { + let db: lbug.Database | undefined; + silenceStdout(); + try { + db = createLbugDatabase(lbug, dbPath, { + readOnly: true, + throwOnWalReplayFailure: false, + }); + await db.init(); + return db; + } catch (err) { + if (db) await db.close().catch(() => {}); + throw err; + } finally { + restoreStdout(); + } +} + +/** + * Quarantine the .wal file and retry opening the database. + * Used when the initial open fails with a WAL corruption error. + */ +async function tryQuarantineAndReopen(dbPath: string, repoId: string): Promise { + const walPath = dbPath + '.wal'; + const quarantineName = `${walPath}.corrupt.${Date.now()}-${Math.random().toString(36).slice(2)}`; + try { + await fs.rename(walPath, quarantineName); + } catch { + throw new Error( + `LadybugDB WAL corruption detected for ${repoId}. ` + + `Run \`gitnexus analyze\` to rebuild the index. (quarantine failed)`, + ); + } + realStderrWrite( + `GitNexus: LadybugDB WAL quarantined for ${repoId}; graph may be stale. ` + + `Run \`gitnexus analyze\` to rebuild the index.\n`, + ); + return await openReadOnlyDatabase(dbPath); +} + /** Deduplicates concurrent initLbug calls for the same repoId */ const initPromises = new Map>(); @@ -319,16 +359,29 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { // avoids lock conflicts when `gitnexus analyze` is writing. let lastError: Error | null = null; for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) { - silenceStdout(); try { - const db = createLbugDatabase(lbug, dbPath, { readOnly: true }); - restoreStdout(); + const db = await openReadOnlyDatabase(dbPath); shared = { db, refCount: 0, ftsLoaded: false }; dbCache.set(dbPath, shared); break; } catch (err: any) { - restoreStdout(); lastError = err instanceof Error ? err : new Error(String(err)); + + if (isWalCorruptionError(lastError)) { + try { + const db = await tryQuarantineAndReopen(dbPath, repoId); + shared = { db, refCount: 0, ftsLoaded: false }; + dbCache.set(dbPath, shared); + break; + } catch (retryErr) { + throw new Error( + `LadybugDB WAL corruption detected for ${repoId}. ` + + `Run \`gitnexus analyze\` to rebuild the index. ` + + `(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`, + ); + } + } + const isLockError = lastError.message.includes('Could not set lock') || lastError.message.includes('lock'); if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break; diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 34049ab31..2f16c1fb2 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -16,6 +16,7 @@ import { isLbugReady, isWriteQuery, } from '../../core/lbug/pool-adapter.js'; +import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../../core/lbug/lbug-config.js'; export { isWriteQuery }; // Embedding imports are lazy (dynamic import) to avoid loading onnxruntime-node // at MCP server startup — crashes on unsupported Node ABI versions (#89) @@ -1225,7 +1226,14 @@ export class LocalBackend { const result = await executeQuery(repo.id, params.query); return result; } catch (err: any) { - return { error: err.message || 'Query failed' }; + const msg = err.message || 'Query failed'; + if (isWalCorruptionError(err)) { + return { + error: msg, + recoverySuggestion: WAL_RECOVERY_SUGGESTION, + }; + } + return { error: msg }; } } @@ -1679,6 +1687,30 @@ export class LocalBackend { kind?: string; include_content?: boolean; }, + ): Promise { + try { + return await this._contextImpl(repo, params); + } catch (err: any) { + const msg = (err instanceof Error ? err.message : String(err)) || 'Context query failed'; + if (isWalCorruptionError(err)) { + return { + error: msg, + recoverySuggestion: WAL_RECOVERY_SUGGESTION, + }; + } + throw err; + } + } + + private async _contextImpl( + repo: RepoHandle, + params: { + name?: string; + uid?: string; + file_path?: string; + kind?: string; + include_content?: boolean; + }, ): Promise { await this.ensureInitialized(repo.id); @@ -2440,6 +2472,7 @@ export class LocalBackend { impactedCount: 0, risk: 'UNKNOWN', suggestion: 'The graph query failed — try gitnexus context as a fallback', + ...(isWalCorruptionError(err) ? { recoverySuggestion: WAL_RECOVERY_SUGGESTION } : {}), }; } } diff --git a/gitnexus/test/unit/lbug-config-wal.test.ts b/gitnexus/test/unit/lbug-config-wal.test.ts new file mode 100644 index 000000000..6baea3621 --- /dev/null +++ b/gitnexus/test/unit/lbug-config-wal.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it, vi } from 'vitest'; +import { createLbugDatabase, isWalCorruptionError } from '../../src/core/lbug/lbug-config.js'; + +describe('isWalCorruptionError', () => { + it.each([ + [ + 'Corrupted wal file', + 'Runtime exception: Corrupted wal file. Read out invalid WAL record type.', + ], + ['invalid WAL record', 'Error: invalid WAL record type'], + ['WAL checksum', 'Checksum verification failed, the WAL file is corrupted.'], + ['WAL + corrupt', 'the WAL file is corrupted'], + ])('matches WAL corruption: %s', (_label, msg) => { + expect(isWalCorruptionError(msg)).toBe(true); + expect(isWalCorruptionError(new Error(msg))).toBe(true); + }); + + it.each([ + ['lock error', 'Could not set lock on file : /path/to/db'], + ['generic', 'Query failed'], + ['not found', 'LadybugDB not found at /path'], + ['checksum without WAL', 'Checksum verification failed for parquet file'], + ['permission path with WAL', "EACCES: permission denied '/path/to/wal'"], + ['schema mismatch WAL', 'schema version mismatch in WAL'], + ])('does not match non-WAL error: %s', (_label, msg) => { + expect(isWalCorruptionError(msg)).toBe(false); + }); + + it('handles non-string input', () => { + expect(isWalCorruptionError(undefined)).toBe(false); + expect(isWalCorruptionError(null)).toBe(false); + expect(isWalCorruptionError(42)).toBe(false); + expect(isWalCorruptionError(new Error('ok'))).toBe(false); + }); +}); + +describe('createLbugDatabase WAL replay option', () => { + it('passes throwOnWalReplayFailure and checksum constructor args explicitly', () => { + const Database = vi.fn(function (this: any) {}); + const lbugModule = { Database } as any; + + createLbugDatabase(lbugModule, '/tmp/lbug', { + readOnly: true, + throwOnWalReplayFailure: false, + }); + + expect(Database).toHaveBeenCalledWith( + '/tmp/lbug', + 0, + false, + true, + expect.any(Number), + true, + -1, + false, + true, + ); + }); +}); diff --git a/gitnexus/test/unit/mcp-wal-feedback.test.ts b/gitnexus/test/unit/mcp-wal-feedback.test.ts new file mode 100644 index 000000000..e387e0b97 --- /dev/null +++ b/gitnexus/test/unit/mcp-wal-feedback.test.ts @@ -0,0 +1,160 @@ +/** + * Tests for WAL corruption feedback in MCP error responses (#1402). + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { lbugMocks, platformMocks, repoMocks } = vi.hoisted(() => ({ + lbugMocks: { + initLbug: vi.fn().mockResolvedValue(undefined), + executeQuery: vi.fn(), + executeParameterized: vi.fn(), + closeLbug: vi.fn().mockResolvedValue(undefined), + isLbugReady: vi.fn().mockReturnValue(true), + isWriteQuery: vi.fn().mockReturnValue(false), + }, + platformMocks: { + isVectorExtensionSupportedByPlatform: vi.fn().mockReturnValue(true), + }, + repoMocks: { + listRegisteredRepos: vi.fn(), + }, +})); + +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, ...lbugMocks }; +}); + +vi.mock('../../src/mcp/core/lbug-adapter.js', async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, ...lbugMocks }; +}); + +vi.mock('../../src/storage/repo-manager.js', () => ({ + listRegisteredRepos: repoMocks.listRegisteredRepos, + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), +})); + +vi.mock('../../src/core/git-staleness.js', () => ({ + checkStaleness: vi.fn().mockReturnValue({ isStale: false, commitsBehind: 0 }), + checkCwdMatch: vi.fn().mockResolvedValue({ match: 'none' }), +})); + +vi.mock('../../src/core/platform/capabilities.js', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + isVectorExtensionSupportedByPlatform: platformMocks.isVectorExtensionSupportedByPlatform, + }; +}); + +vi.mock('../../src/core/search/bm25-index.js', () => ({ + searchFTSFromLbug: vi.fn().mockResolvedValue([]), +})); + +vi.mock('../../src/mcp/core/embedder.js', () => ({ + embedQuery: vi.fn().mockResolvedValue([]), + getEmbeddingDims: vi.fn().mockReturnValue(384), +})); + +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +const MOCK_REPO_ENTRY = { + name: 'test-repo', + path: '/tmp/test', + storagePath: '/tmp/test/.gitnexus', + indexedAt: '2026-05-01T00:00:00Z', + lastCommit: 'abc1234', +}; + +async function makeBackend(): Promise { + const backend = new LocalBackend(); + await backend.init(); + return backend; +} + +describe('WAL corruption feedback in MCP responses (#1402)', () => { + beforeEach(() => { + vi.clearAllMocks(); + lbugMocks.initLbug.mockResolvedValue(undefined); + lbugMocks.executeQuery.mockResolvedValue([]); + lbugMocks.executeParameterized.mockResolvedValue([]); + lbugMocks.isLbugReady.mockReturnValue(true); + lbugMocks.isWriteQuery.mockReturnValue(false); + repoMocks.listRegisteredRepos.mockResolvedValue([MOCK_REPO_ENTRY]); + }); + + it('impact returns WAL suggestion on corrupted WAL error', async () => { + const backend = await makeBackend(); + lbugMocks.executeParameterized.mockRejectedValueOnce( + new Error('Runtime exception: Corrupted wal file. Read out invalid WAL record type.'), + ); + + const result = await backend.callTool('impact', { + repo: 'test-repo', + target: 'MyClass', + direction: 'upstream', + }); + + expect(result.error).toBeDefined(); + expect(result.suggestion).toBe( + 'The graph query failed — try gitnexus context as a fallback', + ); + expect(result.recoverySuggestion).toBeDefined(); + }); + + it('cypher returns WAL recoverySuggestion on corrupted WAL error', async () => { + const backend = await makeBackend(); + lbugMocks.executeQuery.mockRejectedValueOnce(new Error('Corrupted wal file')); + + const result = await backend.callTool('cypher', { + repo: 'test-repo', + query: 'MATCH (n) RETURN n LIMIT 1', + }); + + expect(result.error).toBe('Corrupted wal file'); + expect(result.recoverySuggestion).toBeDefined(); + }); + + it('context returns WAL recoverySuggestion on corrupted WAL error', async () => { + const backend = await makeBackend(); + lbugMocks.executeParameterized.mockRejectedValueOnce(new Error('Corrupted wal file')); + + const result = await backend.callTool('context', { + repo: 'test-repo', + name: 'MyClass', + }); + + expect(result.error).toBe('Corrupted wal file'); + expect(result.recoverySuggestion).toBeDefined(); + }); + + it('non-WAL errors do not include WAL suggestion', async () => { + const backend = await makeBackend(); + lbugMocks.executeParameterized.mockRejectedValueOnce(new Error('Some other error')); + + const result = await backend.callTool('impact', { + repo: 'test-repo', + target: 'MyClass', + direction: 'upstream', + }); + + expect(result.error).toBeDefined(); + expect(result.suggestion).toBe( + 'The graph query failed — try gitnexus context as a fallback', + ); + }); + + it('context preserves non-WAL throw behavior', async () => { + const backend = await makeBackend(); + lbugMocks.executeParameterized.mockRejectedValueOnce(new Error('Some other error')); + + await expect( + backend.callTool('context', { + repo: 'test-repo', + name: 'MyClass', + }), + ).rejects.toThrow('Some other error'); + }); +}); diff --git a/gitnexus/test/unit/pool-wal-recovery.test.ts b/gitnexus/test/unit/pool-wal-recovery.test.ts new file mode 100644 index 000000000..19b24c583 --- /dev/null +++ b/gitnexus/test/unit/pool-wal-recovery.test.ts @@ -0,0 +1,177 @@ +/** + * Tests for WAL corruption recovery in the connection pool (#1402). + * + * Mocks createLbugDatabase and fs to verify quarantine + retry behavior + * without needing a real LadybugDB instance or corrupted WAL file. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { stderrWriteMock } = vi.hoisted(() => ({ + stderrWriteMock: vi.fn(), +})); + +vi.mock('fs/promises', () => ({ + default: { + stat: vi.fn().mockResolvedValue({}), + unlink: vi.fn().mockResolvedValue(undefined), + rename: vi.fn().mockResolvedValue(undefined), + }, +})); + +vi.mock('@ladybugdb/core', () => ({ + default: { + Database: vi.fn(), + Connection: vi.fn(function (this: any) { + this.close = vi.fn().mockResolvedValue(undefined); + }), + }, +})); + +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + loadFTSExtension: vi.fn().mockResolvedValue(true), +})); + +vi.mock('../../src/core/lbug/lbug-config.js', () => ({ + createLbugDatabase: vi.fn(), + LBUG_MAX_DB_SIZE: 1024, + isWalCorruptionError: vi.fn((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err ?? ''); + return /corrupt(ed)?\s+wal|invalid\s+wal\s+record/i.test(msg); + }), +})); + +vi.mock('../../src/mcp/stdio-capture.js', () => ({ + realStdoutWrite: vi.fn(), + realStderrWrite: stderrWriteMock, + setActiveStdoutWrite: vi.fn(), + getActiveStdoutWrite: vi.fn(() => vi.fn()), +})); + +import fs from 'fs/promises'; +import { createLbugDatabase } from '../../src/core/lbug/lbug-config.js'; + +const { closeLbug } = await import('../../src/core/lbug/pool-adapter.js'); + +const mockInit = vi.fn().mockResolvedValue(undefined); +const mockClose = vi.fn().mockResolvedValue(undefined); + +function makeMockDb() { + return { init: mockInit, close: mockClose, _isClosed: false } as any; +} + +describe('WAL corruption recovery in doInitLbug (#1402)', () => { + beforeEach(() => { + (createLbugDatabase as any).mockReset(); + (fs.stat as any).mockReset(); + (fs.rename as any).mockReset(); + mockInit.mockReset(); + mockClose.mockReset(); + mockInit.mockResolvedValue(undefined); + mockClose.mockResolvedValue(undefined); + (fs.stat as any).mockResolvedValue({}); + (fs.rename as any).mockResolvedValue(undefined); + }); + + afterEach(async () => { + vi.useRealTimers(); + await closeLbug().catch(() => {}); + vi.clearAllMocks(); + }); + + it('retries with WAL quarantine on corrupted WAL init error', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const dbPath = '/tmp/test-wal-recovery/lbug'; + + const badDb = makeMockDb(); + const goodDb = makeMockDb(); + badDb.init = vi.fn().mockRejectedValueOnce(new Error('Corrupted wal file')); + (createLbugDatabase as any).mockReturnValueOnce(badDb).mockReturnValueOnce(goodDb); + + await initLbug('test-repo-init', dbPath); + + expect(badDb.init).toHaveBeenCalledTimes(1); + expect(createLbugDatabase).toHaveBeenCalledTimes(2); + expect(createLbugDatabase).toHaveBeenCalledWith( + expect.anything(), + dbPath, + expect.objectContaining({ + readOnly: true, + throwOnWalReplayFailure: false, + }), + ); + expect(fs.rename).toHaveBeenCalledWith( + dbPath + '.wal', + expect.stringContaining('.wal.corrupt.'), + ); + expect(stderrWriteMock).toHaveBeenCalledWith( + expect.stringContaining('WAL quarantined for test-repo-init'), + ); + }); + + it('does not quarantine on lock error (preserves existing lock retry)', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const setTimeoutSpy = vi.spyOn(global, 'setTimeout').mockImplementation((callback: any) => { + callback(); + return 0 as any; + }); + const dbPath = '/tmp/test-wal-recovery/lbug'; + + (createLbugDatabase as any).mockImplementation(() => { + throw new Error('Could not set lock on file'); + }); + + try { + await expect(initLbug('test-repo-lock', dbPath)).rejects.toThrow(); + } finally { + setTimeoutSpy.mockRestore(); + } + + expect(fs.rename).not.toHaveBeenCalled(); + }); + + it('throws with analyze suggestion after retry also fails', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const dbPath = '/tmp/test-wal-recovery/lbug'; + + (createLbugDatabase as any) + .mockImplementationOnce(() => { + throw new Error('Corrupted wal file'); + }) + .mockImplementationOnce(() => { + throw new Error('Still broken'); + }); + + await expect(initLbug('test-repo-fail', dbPath)).rejects.toThrow(/gitnexus analyze/); + expect(createLbugDatabase).toHaveBeenCalledTimes(2); + }); + + it('does not reuse poisoned state after WAL failure', async () => { + const { initLbug, isLbugReady: ready } = await import('../../src/core/lbug/pool-adapter.js'); + const dbPath = '/tmp/test-wal-recovery/lbug'; + + (createLbugDatabase as any) + .mockImplementationOnce(() => { + throw new Error('Corrupted wal file'); + }) + .mockImplementationOnce(() => { + throw new Error('Still broken'); + }); + + await expect(initLbug('test-repo-nocache', dbPath)).rejects.toThrow(); + + expect(ready('test-repo-nocache')).toBe(false); + }); + + it('handles quarantine gracefully when .wal file does not exist', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const dbPath = '/tmp/test-wal-recovery/lbug'; + + (fs.rename as any).mockRejectedValueOnce(new Error('ENOENT: no such file')); + + (createLbugDatabase as any).mockImplementationOnce(() => { + throw new Error('Corrupted wal file'); + }); + + await expect(initLbug('test-repo-enoent', dbPath)).rejects.toThrow(/gitnexus analyze/); + }); +});