diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index a20503bc4..ec3636afc 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -13,6 +13,7 @@ import { execFileSync } from 'child_process'; import v8 from 'v8'; import cliProgress from 'cli-progress'; import { closeLbug } from '../core/lbug/lbug-adapter.js'; +import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../core/lbug/lbug-config.js'; import { getStoragePaths, getGlobalRegistryPath, @@ -638,6 +639,20 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption return; } + // WAL corruption — the index file is unreadable. Give a clear recovery + // path without a confusing stack trace (the native error message alone + // is enough signal). + if (isWalCorruptionError(err) || msg.includes('LadybugDB WAL corruption')) { + cliError( + ` The GitNexus index has a corrupted WAL file.\n` + + ` This usually happens when a previous analysis was interrupted mid-write.\n` + + ` ${WAL_RECOVERY_SUGGESTION}\n`, + { recoveryHint: 'wal-corruption' }, + ); + process.exitCode = 1; + return; + } + // HF download failure — show clean guidance without the raw stack trace. // Checked before writeFatalToStderr so the user sees one focused message // rather than a stack-trace dump followed by a second remediation block. diff --git a/gitnexus/src/cli/serve.ts b/gitnexus/src/cli/serve.ts index 9356b5bab..003e2ce69 100644 --- a/gitnexus/src/cli/serve.ts +++ b/gitnexus/src/cli/serve.ts @@ -1,6 +1,7 @@ import { createServer } from '../server/api.js'; import { logger, flushLoggerSync } from '../core/logger.js'; import { cliError } from './cli-message.js'; +import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../core/lbug/lbug-config.js'; // Catch anything that would cause a silent exit. Pino v10's default // destination is `sync: false` (SonicBoom buffered) — call @@ -34,7 +35,13 @@ export const serveCommand = async (options?: { port?: string; host?: string }) = try { await createServer(port, host); } catch (err: any) { - if (err.code === 'EADDRINUSE') { + if (isWalCorruptionError(err)) { + cliError( + `\nGitNexus server could not start: the index has a corrupted WAL file.\n` + + ` ${WAL_RECOVERY_SUGGESTION}\n`, + { recoveryHint: 'wal-corruption' }, + ); + } else if (err.code === 'EADDRINUSE') { cliError( `\nFailed to start GitNexus server:\n` + ` ${err.message || err}\n\n` + diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 54d98b667..c01966d0c 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -21,7 +21,9 @@ import { closeLbugConnection, isDbBusyError, isOpenRetryExhausted, + isWalCorruptionError, openLbugConnection, + WAL_RECOVERY_SUGGESTION, waitForWindowsHandleRelease, type LbugConnectionHandle, } from './lbug-config.js'; @@ -594,6 +596,24 @@ const doInitLbug = async (dbPath: string) => { // anyway and any genuine cross-process lock contention surfaces // on the next operation via withLbugDb's retry. Logging it here // would just be noise in CI. + // + // WAL corruption: the first DDL write after DB open triggers WAL + // replay — if the WAL file was left in a corrupt state by an + // interrupted previous run, the native engine throws here. Rather + // than logging a WARN and continuing in a broken state, close the + // DB cleanly and surface an actionable error so the caller (serve, + // MCP, analyze) can exit with a clear recovery message. + if (isWalCorruptionError(err)) { + await safeClose(); + currentDbPath = null; + ftsLoaded = false; + vectorExtensionLoaded = false; + ensuredFTSIndexes.clear(); + throw new Error( + `LadybugDB WAL corruption detected at ${dbPath}. ${WAL_RECOVERY_SUGGESTION}\n` + + ` Original error: ${msg.slice(0, 200)}`, + ); + } if (!msg.includes('already exists') && !isDbBusyError(err)) { logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`); } diff --git a/gitnexus/src/core/lbug/lbug-config.ts b/gitnexus/src/core/lbug/lbug-config.ts index ceb445693..22f2d3b18 100644 --- a/gitnexus/src/core/lbug/lbug-config.ts +++ b/gitnexus/src/core/lbug/lbug-config.ts @@ -49,7 +49,7 @@ export const LBUG_MAX_DB_SIZE: number = (() => { const WAL_CORRUPTION_RE = /corrupt(ed)?\s+wal|invalid\s+wal\s+record|wal.*corrupt|checksum.*wal/i; export const WAL_RECOVERY_SUGGESTION = - 'WAL corruption detected. Run `gitnexus analyze` to rebuild the index.'; + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.'; export function isWalCorruptionError(err: unknown): boolean { if (!err) return false; diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index f18d7fcc3..2b432cba3 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -18,7 +18,11 @@ import fs from 'fs/promises'; import lbug from '@ladybugdb/core'; import { loadFTSExtension } from './lbug-adapter.js'; -import { createLbugDatabase, isWalCorruptionError } from './lbug-config.js'; +import { + createLbugDatabase, + isWalCorruptionError, + WAL_RECOVERY_SUGGESTION, +} from './lbug-config.js'; /** Per-repo pool: one Database, many Connections */ interface PoolEntry { @@ -375,8 +379,7 @@ async function doInitLbug(repoId: string, dbPath: string): Promise { break; } catch (retryErr) { throw new Error( - `LadybugDB WAL corruption detected for ${repoId}. ` + - `Run \`gitnexus analyze\` to rebuild the index. ` + + `LadybugDB WAL corruption detected for ${repoId}. ${WAL_RECOVERY_SUGGESTION} ` + `(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`, ); } diff --git a/gitnexus/test/unit/analyze-wal-error.test.ts b/gitnexus/test/unit/analyze-wal-error.test.ts new file mode 100644 index 000000000..1b4ed5101 --- /dev/null +++ b/gitnexus/test/unit/analyze-wal-error.test.ts @@ -0,0 +1,137 @@ +/** + * Tests for WAL corruption error handling in the `analyzeCommand` CLI. + * + * Before this fix, a WAL corruption error surfaced as a raw stack-trace dump. + * After the fix, it is caught before the generic error path and rendered as + * a clean, actionable message telling the user to run `gitnexus analyze --force`. + * + * Mirrors the test shape of analyze-worker-timeout.test.ts: + * - vi.mock the heavy dependencies so no real DB / git is touched + * - drive `analyzeCommand` with a mocked `runFullAnalysis` that throws + * - assert on process.exitCode and the logged output + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const runFullAnalysisMock = vi.fn(); + +vi.mock('../../src/core/run-analyze.js', () => ({ + runFullAnalysis: runFullAnalysisMock, +})); + +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + closeLbug: vi.fn(async () => undefined), +})); + +vi.mock('../../src/storage/repo-manager.js', () => ({ + getStoragePaths: vi.fn(() => ({ storagePath: '.gitnexus', lbugPath: '.gitnexus/lbug' })), + getGlobalRegistryPath: vi.fn(() => 'registry.json'), + RegistryNameCollisionError: class RegistryNameCollisionError extends Error {}, + AnalysisNotFinalizedError: class AnalysisNotFinalizedError extends Error {}, + assertAnalysisFinalized: vi.fn(async () => undefined), +})); + +vi.mock('../../src/storage/git.js', () => ({ + getGitRoot: vi.fn(() => '/repo'), + hasGitDir: vi.fn(() => true), +})); + +vi.mock('../../src/core/ingestion/utils/max-file-size.js', () => ({ + getMaxFileSizeBannerMessage: vi.fn(() => null), +})); + +// analyze.ts imports isHfDownloadFailure from hf-env.js, which in turn imports +// from gitnexus-shared (not linked in dev). Mock the module to break the chain. +vi.mock('../../src/core/embeddings/hf-env.js', () => ({ + isHfDownloadFailure: vi.fn(() => false), +})); + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +describe('analyzeCommand WAL corruption error handling', () => { + beforeEach(() => { + vi.resetModules(); + runFullAnalysisMock.mockReset(); + process.exitCode = undefined; + // Ensure ensureHeap() short-circuits (heap already at target size) + process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=8192`.trim(); + }); + + it('surfaces a clean recovery message on a re-wrapped WAL corruption error', async () => { + // This error shape is what lbug-adapter throws after detecting WAL corruption + // in doInitLbug and re-wrapping it with the recovery suggestion. + const walError = new Error( + 'LadybugDB WAL corruption detected at /repo/.gitnexus/lbug. ' + + 'Run `gitnexus analyze` to rebuild the index.\n' + + ' Original error: Runtime exception: Corrupted wal file.', + ); + runFullAnalysisMock.mockRejectedValue(walError); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(1); + + const records = cap.records(); + const walRecord = records.find( + (r) => typeof r.msg === 'string' && r.msg.includes('gitnexus analyze --force'), + ); + expect(walRecord).toBeDefined(); + + // Raw stack trace must NOT appear via cliError + const stackRecord = records.find( + (r) => typeof r.msg === 'string' && r.msg.includes('at analyzeCommand'), + ); + expect(stackRecord).toBeUndefined(); + + cap.restore(); + }); + + it('surfaces a clean recovery message when the native WAL error fires directly', async () => { + // isWalCorruptionError fires on the native engine message before re-wrapping. + const nativeWalError = new Error( + 'Runtime exception: Corrupted wal file. Read out invalid WAL record type.', + ); + runFullAnalysisMock.mockRejectedValue(nativeWalError); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(1); + + const records = cap.records(); + const walRecord = records.find( + (r) => typeof r.msg === 'string' && r.msg.includes('gitnexus analyze --force'), + ); + expect(walRecord).toBeDefined(); + + cap.restore(); + }); + + it('does NOT route non-WAL errors through the WAL handler', async () => { + const genericError = new Error('Some unexpected failure unrelated to WAL'); + runFullAnalysisMock.mockRejectedValue(genericError); + + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + + await analyzeCommand(undefined, {}); + + expect(process.exitCode).toBe(1); + + // The WAL recovery message must NOT appear for unrelated errors + const records = cap.records(); + const walRecord = records.find( + (r) => typeof r.msg === 'string' && r.msg.includes('gitnexus analyze --force'), + ); + expect(walRecord).toBeUndefined(); + + cap.restore(); + }); +}); diff --git a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts new file mode 100644 index 000000000..c5f4b6773 --- /dev/null +++ b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts @@ -0,0 +1,259 @@ +/** + * Tests for WAL corruption detection in the doInitLbug schema creation loop. + * + * Before this fix, a corrupt WAL that threw during schema DDL was silently + * logged as WARN. After the fix, `isWalCorruptionError` is checked first: + * the DB is closed cleanly and an Error with `WAL_RECOVERY_SUGGESTION` is + * thrown so the caller (serve / MCP / analyze) can exit with a clear message. + * + * Two test layers (same pattern as lbug-checkpoint-lifecycle.test.ts): + * 1. Structural — grep the adapter source to verify the guard is wired in. + * 2. Behavioural — vi.doMock + vi.resetModules to exercise the runtime path. + */ +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import path from 'node:path'; + +// ─── Helpers ───────────────────────────────────────────────────────────────── + +const makeOpenMock = () => + vi.fn(async () => ({ + writeFile: vi.fn(async () => {}), + close: vi.fn(async () => {}), + })); + +const SCHEMA_MOCK = { + NODE_TABLES: ['File', 'Function', 'Class'], + REL_TABLE_NAME: 'CodeRelation', + EMBEDDING_TABLE_NAME: 'Embedding', + STALE_HASH_SENTINEL: '__stale__', + SCHEMA_QUERIES: ['CREATE NODE TABLE IF NOT EXISTS File (id STRING, PRIMARY KEY(id))'], +}; + +function makeFsMock(dbPath: string) { + const ENOENT = Object.assign(new Error(`ENOENT: ${dbPath}`), { code: 'ENOENT' }); + return { + default: { + lstat: vi.fn(async () => { + throw ENOENT; + }), + access: vi.fn(async () => { + throw ENOENT; + }), + unlink: vi.fn(async () => {}), + mkdir: vi.fn(async () => {}), + open: makeOpenMock(), + }, + }; +} + +// ─── Structural tests ───────────────────────────────────────────────────────── + +describe('doInitLbug WAL corruption guard — structural', () => { + let adapterSource: string; + let schemaLoopBody: string; + + beforeAll(async () => { + adapterSource = await fs.readFile( + path.join(__dirname, '..', '..', 'src', 'core', 'lbug', 'lbug-adapter.ts'), + 'utf-8', + ); + // 3000-char window from the SCHEMA_QUERIES loop comfortably covers the + // full catch block including the throw with WAL_RECOVERY_SUGGESTION. + const loopIdx = adapterSource.indexOf('for (const schemaQuery of SCHEMA_QUERIES)'); + schemaLoopBody = adapterSource.slice(loopIdx, loopIdx + 3000); + }); + + it('imports isWalCorruptionError and WAL_RECOVERY_SUGGESTION from lbug-config', () => { + expect(adapterSource).toMatch(/isWalCorruptionError/); + expect(adapterSource).toMatch(/WAL_RECOVERY_SUGGESTION/); + expect(adapterSource).toMatch(/from '\.\/lbug-config\.js'/); + }); + + it('calls isWalCorruptionError inside the schema creation loop catch block', () => { + expect(schemaLoopBody).toMatch(/isWalCorruptionError\(err\)/); + }); + + it('WAL guard calls safeClose() to avoid leaving an open handle', () => { + expect(schemaLoopBody).toMatch(/await safeClose\(\)/); + }); + + it('WAL guard resets currentDbPath to null', () => { + expect(schemaLoopBody).toMatch(/currentDbPath = null/); + }); + + it('WAL guard throws with WAL_RECOVERY_SUGGESTION in the message', () => { + expect(schemaLoopBody).toMatch(/WAL_RECOVERY_SUGGESTION/); + expect(schemaLoopBody).toMatch(/throw new Error/); + }); + + it('WAL guard appears BEFORE the generic schema-warning logger.warn', () => { + const walGuardIdx = schemaLoopBody.indexOf('isWalCorruptionError(err)'); + // Avoid multi-byte emoji — search for the text portion only + const warnIdx = schemaLoopBody.indexOf('Schema creation warning'); + expect(walGuardIdx).toBeGreaterThan(-1); + expect(warnIdx).toBeGreaterThan(-1); + expect(walGuardIdx).toBeLessThan(warnIdx); + }); +}); + +// ─── Behavioural tests ──────────────────────────────────────────────────────── + +describe('doInitLbug WAL corruption guard — behavioural', () => { + afterEach(() => { + vi.doUnmock('fs/promises'); + vi.doUnmock('../../src/core/lbug/schema.js'); + vi.doUnmock('../../src/core/lbug/lbug-config.js'); + vi.doUnmock('../../src/core/lbug/extension-loader.js'); + vi.doUnmock('../../src/core/logger.js'); + vi.resetModules(); + vi.clearAllMocks(); + }); + + it('throws with WAL recovery message when a schema query raises a WAL corruption error', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-wal-schema-throw/lbug'; + const walError = new Error( + 'Runtime exception: Corrupted wal file. Read out invalid WAL record type.', + ); + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const conn = { + query: vi.fn().mockRejectedValueOnce(walError).mockResolvedValue(queryResult), + close: vi.fn(async () => {}), + }; + const db = { close: vi.fn(async () => {}) }; + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: vi.fn(async () => ({ db, conn })), + closeLbugConnection: vi.fn(async () => {}), + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + return /corrupt.*wal|invalid.*wal.*record/i.test(msg); + }), + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + // Catch the error once and assert both patterns in the message. + // (mockRejectedValueOnce is consumed on the first call, so a second + // initLbug call would succeed — test both patterns in one shot.) + const err = await adapter.initLbug(dbPath).catch((e: unknown) => e); + expect(err).toBeInstanceOf(Error); + expect((err as Error).message).toMatch(/LadybugDB WAL corruption detected/); + expect((err as Error).message).toMatch(/gitnexus analyze/); + }); + + it('does NOT throw for unrecognised schema errors — logs warn and continues', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-wal-schema-nonwal/lbug'; + const genericError = new Error('some unrelated schema warning'); + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + let callCount = 0; + const conn = { + query: vi.fn(async () => { + callCount++; + if (callCount === 1) throw genericError; + return queryResult; + }), + close: vi.fn(async () => {}), + }; + const db = { close: vi.fn(async () => {}) }; + const warnMock = vi.fn(); + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: vi.fn(async () => ({ db, conn })), + closeLbugConnection: vi.fn(async () => {}), + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn(() => false), // always false → generic warn path + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: warnMock, info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + // Must resolve without throwing — non-WAL schema errors are swallowed (logged as WARN) + await expect(adapter.initLbug(dbPath)).resolves.toBeDefined(); + expect(warnMock).toHaveBeenCalledWith(expect.stringContaining('Schema creation warning')); + + await adapter.closeLbug(); + }); + + it('calls safeClose() (db.close) when WAL corruption is detected mid-schema', async () => { + vi.resetModules(); + + const dbPath = '/tmp/gitnexus-lbug-wal-schema-state/lbug'; + const walError = new Error('Corrupted wal file. Read out invalid WAL record type.'); + const queryResult = { getAll: vi.fn(async () => []), close: vi.fn() }; + const conn = { + query: vi.fn().mockRejectedValueOnce(walError).mockResolvedValue(queryResult), + close: vi.fn(async () => {}), + }; + const db = { close: vi.fn(async () => {}) }; + + vi.doMock('fs/promises', () => makeFsMock(dbPath)); + vi.doMock('../../src/core/lbug/schema.js', () => SCHEMA_MOCK); + vi.doMock('../../src/core/lbug/lbug-config.js', () => ({ + openLbugConnection: vi.fn(async () => ({ db, conn })), + closeLbugConnection: vi.fn(async () => {}), + isDbBusyError: vi.fn(() => false), + isOpenRetryExhausted: vi.fn(() => false), + isWalCorruptionError: vi.fn((err: unknown) => { + const msg = err instanceof Error ? err.message : String(err); + return /corrupt.*wal|invalid.*wal.*record/i.test(msg); + }), + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', + waitForWindowsHandleRelease: vi.fn(async () => true), + })); + vi.doMock('../../src/core/lbug/extension-loader.js', () => ({ + extensionManager: { + ensure: vi.fn(async () => true), + getCapabilities: vi.fn(() => []), + reset: vi.fn(), + }, + })); + vi.doMock('../../src/core/logger.js', () => ({ + logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn(), debug: vi.fn() }, + })); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + + await expect(adapter.initLbug(dbPath)).rejects.toThrow(/LadybugDB WAL corruption/); + + // safeClose was called — db.close is its final step + expect(db.close).toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/pool-wal-recovery.test.ts b/gitnexus/test/unit/pool-wal-recovery.test.ts index 19b24c583..cda42806a 100644 --- a/gitnexus/test/unit/pool-wal-recovery.test.ts +++ b/gitnexus/test/unit/pool-wal-recovery.test.ts @@ -34,6 +34,8 @@ vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ vi.mock('../../src/core/lbug/lbug-config.js', () => ({ createLbugDatabase: vi.fn(), LBUG_MAX_DB_SIZE: 1024, + WAL_RECOVERY_SUGGESTION: + 'WAL corruption detected. Run `gitnexus analyze --force` to rebuild the index.', isWalCorruptionError: vi.fn((err: unknown) => { const msg = err instanceof Error ? err.message : String(err ?? ''); return /corrupt(ed)?\s+wal|invalid\s+wal\s+record/i.test(msg);