diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1c6a99c36..bf8a5e381 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -485,6 +485,7 @@ CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) ├── lbug.wal # Write-ahead log ├── lbug.shadow # Shadow sidecar (checkpoint staging) ├── lbug.lock # Single-writer lock + ├── lbug.wal.checkpoint, lbug.checkpoint.{intent,apply}.lock # checkpoint-in-flight artifacts; left behind only by an interrupted checkpoint, consumed by the next writable open ├── lbug.{wal,shadow}.dirty-recovery # parked sidecars from a crashed run; safe to delete ├── gitnexus.json # lastCommit, indexedAt, stats (primary metadata file) └── meta.json # legacy mirror of gitnexus.json, kept in sync (see MIGRATION.md) @@ -493,6 +494,12 @@ CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) └── registry.json # Global repo registry (MCP discovery) ``` +Read-only opens self-heal an interrupted checkpoint: the refusal is +classified and cleared by one writable open (probe + `CHECKPOINT`) before +the read-only open is retried — see `sidecar-recovery.ts` +(`isReadOnlyCheckpointInProgressError`) and the +`lbug-interrupted-checkpoint-recovery` integration test. + Managed by `repo-manager.ts`. ## LadybugDB schema diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 89e5842f7..de688cf5a 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -165,6 +165,7 @@ const LBUG_NATIVE = [ 'test/integration/lbug-open-retry.test.ts', 'test/integration/lbug-close-handle-release.test.ts', 'test/integration/lbug-orphan-sidecar-recovery.test.ts', + 'test/integration/lbug-interrupted-checkpoint-recovery.test.ts', 'test/integration/lbug-readonly-init.test.ts', 'test/integration/lbug-non-ascii-path.test.ts', // Cross-repo trace e2e: builds two real lbug indexes + a real bridge and diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index 5f5a076d4..7df9d530d 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -80,10 +80,13 @@ import { guardWalQuarantine, type WalCrashEvidence, isMissingShadowSidecarError, + isReadOnlyCheckpointInProgressError, + isReadOnlyRecoveryFailure, isReadOnlyShadowReplayError, lbugLockRemediation, preflightLbugSidecars, quarantineWalForMissingShadow, + readOnlyRecoveryFailureMessage, renameFailureMessage, shadowSidecarRecoveryMessage, sidecarPreflightDisabled, @@ -547,6 +550,11 @@ const queryAndDrain = async (targetConn: lbug.Connection, cypher: string): Promi // whether the read-only shadow replay throws, so no row identity is read. const READ_ONLY_SHADOW_REPLAY_PROBE = 'MATCH (n) RETURN n LIMIT 1'; +// The durability half of writable recovery: replayed pages only persist when +// an explicit CHECKPOINT applies them to the main file (see +// recoverReadOnlyViaWritableOpen). +const RECOVERY_CHECKPOINT_QUERY = 'CHECKPOINT'; + /** * Serve-side entry to the shared WAL-quarantine safety gate. Refuses (throws) * when the `.shadow` is present on disk or the orphan WAL is too large to @@ -650,7 +658,7 @@ const ensureReadOnlyConnectionUsable = async ( await closeLbugConnection(handle); return await reopenReadOnlyAfterMissingShadow(dbPath, err); } - if (!isReadOnlyShadowReplayError(err)) { + if (!isReadOnlyShadowReplayError(err) && !isReadOnlyCheckpointInProgressError(err)) { await closeLbugConnection(handle); throw err; } @@ -658,7 +666,27 @@ const ensureReadOnlyConnectionUsable = async ( } await closeLbugConnection(handle); + return await recoverReadOnlyViaWritableOpen(dbPath, shadowReplayErr); +}; +/** + * Clear an interrupted-checkpoint / pending-shadow-replay state by opening the + * database WRITABLE once — probe (forces the WAL replay) then an explicit + * CHECKPOINT (persists it; see the comment at the call site) — and reopening + * read-only. Recovery for every read-only refusal an interrupted checkpoint + * produces — `isReadOnlyShadowReplayError` and + * `isReadOnlyCheckpointInProgressError` — shared by the probe path + * (`ensureReadOnlyConnectionUsable`) and the open path (`doInitLbug`'s + * read-only branch, where the native open itself refuses before any probe can + * run). Homelab repro 2026-09-19: a wiki pod killed mid-CHECKPOINT left the + * checkpoint sidecars behind, and every read-only open thereafter failed until + * a writable open (any `gitnexus analyze`) recovered it — this makes the read + * path self-heal instead. + */ +const recoverReadOnlyViaWritableOpen = async ( + dbPath: string, + triggeringErr: unknown, +): Promise => { let writable: LbugConnectionHandle; try { writable = await openLbugConnection(lbug, dbPath); @@ -666,18 +694,28 @@ const ensureReadOnlyConnectionUsable = async ( const code = extractErrnoCode(openErr); if (code === 'EROFS' || code === 'EACCES' || code === 'EPERM') { throw new Error( - shadowSidecarRecoveryMessage(dbPath, shadowReplayErr) + - '\n The workspace appears to be read-only — mount it read-write to perform shadow replay recovery,' + + readOnlyRecoveryFailureMessage(dbPath, triggeringErr) + + '\n The workspace appears to be read-only — mount it read-write to perform WAL recovery,' + ' or re-run `gitnexus analyze` on a writable filesystem to rebuild the index.', ); } throw openErr; } let missingShadowError: unknown; + let probeSucceeded = false; try { await queryAndDrain(writable.conn, READ_ONLY_SHADOW_REPLAY_PROBE); + probeSucceeded = true; + // Load-bearing durability step (engine 0.19.1 matrix, homelab repro + // 2026-09-19): the probe replays the WAL in MEMORY only. Without an + // explicit CHECKPOINT the engine drops those pages at close and the + // follow-up read-only open silently serves the pre-checkpoint state. + // CHECKPOINT applies the replay to the main file, consuming the + // `lbug.wal.checkpoint` / `lbug.shadow` sidecars and clearing the + // checkpoint locks the interrupted checkpoint left behind. + await queryAndDrain(writable.conn, RECOVERY_CHECKPOINT_QUERY); } catch (err) { - if (isMissingShadowSidecarError(err)) { + if (!probeSucceeded && isMissingShadowSidecarError(err)) { missingShadowError = err; } else { throw err; @@ -695,8 +733,12 @@ const ensureReadOnlyConnectionUsable = async ( return reopened; } catch (err) { await closeLbugConnection(reopened); - if (isMissingShadowSidecarError(err)) { - throw new Error(shadowSidecarRecoveryMessage(dbPath, err)); + if ( + isMissingShadowSidecarError(err) || + isReadOnlyShadowReplayError(err) || + isReadOnlyCheckpointInProgressError(err) + ) { + throw new Error(readOnlyRecoveryFailureMessage(dbPath, err)); } throw err; } @@ -871,17 +913,26 @@ const doInitLbug = async ( // mismatched file. Wrap both. usable = await ensureReadOnlyConnectionUsable(dbPath, opened); } catch (err) { - // Not retryable: the on-disk file's storage version doesn't change on - // its own, so withLbugDb's retry loop (which only handles - // isDbBusyError) would just repeat the same native exception. Fail - // immediately with an actionable message instead (review finding on - // PR #3189 — this became reachable once the pinned engine version can - // trail behind whatever version last wrote an index, e.g. after - // downgrading the dependency). Mirrors the pool-adapter.ts check for - // the same error, on the separate open path /api/graph and /api/query - // actually use (withLbugDb, not the pool). - throwIfStorageVersionMismatch(err); - throw err; + // An interrupted checkpoint can make the OPEN itself refuse read-only + // ("Cannot open database in read-only mode while checkpoint is in + // progress") before any probe runs. Clear it with one writable open, + // then reopen read-only — the same self-heal the probe path applies. + // Skip already-wrapped failures so we do not re-enter writable recovery. + if (isReadOnlyCheckpointInProgressError(err) && !isReadOnlyRecoveryFailure(err)) { + usable = await recoverReadOnlyViaWritableOpen(dbPath, err); + } else { + // Not retryable: the on-disk file's storage version doesn't change on + // its own, so withLbugDb's retry loop (which only handles + // isDbBusyError) would just repeat the same native exception. Fail + // immediately with an actionable message instead (review finding on + // PR #3189 — this became reachable once the pinned engine version can + // trail behind whatever version last wrote an index, e.g. after + // downgrading the dependency). Mirrors the pool-adapter.ts check for + // the same error, on the separate open path /api/graph and /api/query + // actually use (withLbugDb, not the pool). + throwIfStorageVersionMismatch(err); + throw err; + } } db = usable.db; conn = usable.conn; diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index 09d28abf9..8f220e227 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -34,6 +34,7 @@ import { guardWalQuarantine, isMissingFsError, isMissingShadowSidecarError, + isReadOnlyCheckpointInProgressError, isReadOnlyShadowReplayError, preflightLbugSidecars, quarantineWalForMissingShadow, @@ -608,12 +609,33 @@ async function probeDatabaseForShadowReplay(db: lbug.Database): Promise { async function replayShadowPagesWithWritableOpen(dbPath: string): Promise { let db: lbug.Database | undefined; + // Mirrors the direct adapter's `probeSucceeded` guard: once the probe has + // replayed, a MISSING-SHADOW error can only come from the CHECKPOINT itself + // — quarantining the WAL then would park a live sidecar on a db whose main + // file just changed underneath it. Fail closed instead (review finding: + // policy drift vs the serve path). + let replaySucceeded = false; try { db = createLbugDatabase(lbug, toNativeSafePath(dbPath), { throwOnWalReplayFailure: false }); await db.init(); await probeDatabaseForShadowReplay(db); + replaySucceeded = true; + // Load-bearing durability step (engine 0.19.1 matrix, homelab repro + // 2026-09-19): the probe replays the WAL in MEMORY only. Without an + // explicit CHECKPOINT the engine drops those pages at close and the + // follow-up read-only open silently serves the pre-checkpoint state. + const conn = createConnection(db); + try { + const checkpointResult = await conn.query('CHECKPOINT'); + const result = Array.isArray(checkpointResult) ? checkpointResult[0] : checkpointResult; + await result.getAll(); + // Shared best-effort closer (awaits + swallows) — never roll this loop. + await closeQueryResults(result); + } finally { + await conn.close().catch(() => {}); + } } catch (err) { - if (isMissingShadowSidecarError(err)) { + if (isMissingShadowSidecarError(err) && !replaySucceeded) { await tryQuarantineForMissingShadow(dbPath, { reason: 'pool writable replay recovery', err, @@ -640,8 +662,14 @@ async function openReadOnlyDatabase(dbPath: string): Promise { readOnly: true, throwOnWalReplayFailure: false, }); - await db.init(); + // init() is inside the try: an interrupted checkpoint (pod killed + // mid-CHECKPOINT leaves `lbug.wal` + `lbug.shadow`) can make the read-only + // OPEN itself refuse — "Cannot open database in read-only mode while + // checkpoint is in progress" — before any probe runs (homelab repro + // 2026-09-19). Both refusal classes recover identically below: one + // writable open replays the WAL/completes the checkpoint. try { + await db.init(); await probeDatabaseForShadowReplay(db); } catch (err) { if (isMissingShadowSidecarError(err)) { @@ -664,7 +692,7 @@ async function openReadOnlyDatabase(dbPath: string): Promise { await probeDatabaseForShadowReplay(db); return db; } - if (!isReadOnlyShadowReplayError(err)) { + if (!isReadOnlyShadowReplayError(err) && !isReadOnlyCheckpointInProgressError(err)) { throw err; } await db.close().catch(() => {}); diff --git a/gitnexus/src/core/lbug/sidecar-recovery.ts b/gitnexus/src/core/lbug/sidecar-recovery.ts index 9272ffd3e..5398093c0 100644 --- a/gitnexus/src/core/lbug/sidecar-recovery.ts +++ b/gitnexus/src/core/lbug/sidecar-recovery.ts @@ -233,6 +233,61 @@ export const isReadOnlyShadowReplayError = (err: unknown): boolean => { return /replay shadow pages under read-only mode/i.test(msg); }; +// LADYBUGDB-CONTRACT: native error text. When bumping LadybugDB, re-validate +// this regex against the new error format — `git grep "LADYBUGDB-CONTRACT"` +// enumerates every version-coupled spot. +// Version honesty (review finding on the interrupted-checkpoint PR): this +// string is FIRST OBSERVED on @ladybugdb/core 0.19.1 — live-reproduced by +// SIGKILLing a writer mid-CHECKPOINT (homelab 2026-09-19, GitNexus image +// 1.6.10-20260917, built on 0.19.x). The 0.18.3 binary does NOT contain it +// (checked via `strings`), and 0.18.3 tolerates the same on-disk state, so on +// 0.18.x this classifier simply never fires. If the engine pin ever moves +// back to ^0.19, the read-path self-heal is already in place. +export const isReadOnlyCheckpointInProgressError = (err: unknown): boolean => { + const msg = err instanceof Error ? err.message : String(err); + return /cannot open database in read-only mode while checkpoint is in progress/i.test(msg); +}; + +/** Prefix of the interrupted-checkpoint wrap — must not rematch the native classifier. */ +export const INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX = + 'LadybugDB could not finish an interrupted checkpoint'; + +/** Prefix of the pending-shadow-replay wrap — must not rematch the native classifier. */ +export const PENDING_SHADOW_REPLAY_RECOVERY_PREFIX = + 'LadybugDB could not finish a pending shadow replay'; + +const READ_ONLY_RECOVERY_REMOUNT = + 'Mount the workspace read-write or re-run `gitnexus analyze` to complete recovery.'; + +/** True when `err` is already a classifier-aware recovery wrap (not a native refusal). */ +export const isReadOnlyRecoveryFailure = (err: unknown): boolean => { + const msg = err instanceof Error ? err.message : String(err); + return ( + msg.startsWith(INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX) || + msg.startsWith(PENDING_SHADOW_REPLAY_RECOVERY_PREFIX) + ); +}; + +/** + * Operator-facing wrap for a failed read-only self-heal. Checkpoint-in-progress + * and pending-shadow-replay must not reuse `shadowSidecarRecoveryMessage` — + * that copy claims the sidecar is missing and appends the native text, which + * rematches the classifiers and can re-enter writable CHECKPOINT. + */ +export const readOnlyRecoveryFailureMessage = (dbPath: string, err: unknown): string => { + if (isReadOnlyCheckpointInProgressError(err)) { + return ( + `${INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX} for ${dbPath}. ` + + `${READ_ONLY_RECOVERY_REMOUNT} ` + + 'Retry later if another writer is still checkpointing.' + ); + } + if (isReadOnlyShadowReplayError(err)) { + return `${PENDING_SHADOW_REPLAY_RECOVERY_PREFIX} for ${dbPath}. ${READ_ONLY_RECOVERY_REMOUNT}`; + } + return shadowSidecarRecoveryMessage(dbPath, err); +}; + export const shadowSidecarRecoveryMessage = (dbPath: string, err: unknown): string => { const msg = err instanceof Error ? err.message : String(err); return ( diff --git a/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts b/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts new file mode 100644 index 000000000..d2dea28cf --- /dev/null +++ b/gitnexus/test/integration/lbug-interrupted-checkpoint-recovery.test.ts @@ -0,0 +1,221 @@ +/** + * Interrupted-checkpoint self-heal — end-to-end against the REAL pool adapter. + * + * Homelab repro 2026-09-19 ("LadybugDB unavailable for __wiki__ ... Cannot + * open database in read-only mode while checkpoint is in progress"): a wiki + * pod killed mid-CHECKPOINT left the engine's checkpoint artifacts on disk, + * and every later read-only open refused — permanently — until a writable + * open (any `gitnexus analyze`) recovered it. The read path now self-heals: + * the refusal is classified (`isReadOnlyCheckpointInProgressError`) and + * cleared by one writable open + probe + CHECKPOINT, then the read-only open + * is retried. + * + * The killed-checkpoint SIGNATURE is planted deterministically — no process + * killing, no race: a checkpointed db plus a `lbug.wal.checkpoint` sidecar, an + * empty `lbug.shadow`, and the zero-byte checkpoint intent/apply lock files + * the engine leaves mid-checkpoint. Verified against @ladybugdb/core 0.19.1, + * where this exact state refuses with the exact production message. The suite + * version-gates itself: on engines that tolerate the planted state (< 0.19, + * e.g. the committed 0.18.3 pin) it skips — a refusal cannot be forced there, + * and the behavioral contract is held on every pin by the mocked + * forced-refusal suites instead. + */ +import { afterAll, describe, expect, it } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { closeLbug, executeQuery, initLbug } from '../../src/core/lbug/pool-adapter.js'; +import lbug from '@ladybugdb/core'; + +const REPO = 'test-interrupted-checkpoint'; +const ROWS = 300; + +/** + * Windows: the native close() resolves before the kernel releases the file's + * handles and byte-range locks — the next open then dies with Win32 Error 33 + * ("another process has locked a portion of the file"), which is exactly how + * this fixture failed its first hosted run. Probe-read both the db and its + * residual WAL until the engine's locks are gone. Bounded, so a real handle + * leak fails loudly instead of hanging; a pass-through on POSIX (first probe + * always succeeds). + */ +async function waitForFixtureRelease(dbPath: string): Promise { + for (const target of [dbPath, `${dbPath}.wal`]) { + for (let attempt = 0; ; attempt++) { + try { + const fh = await fs.open(target, 'r'); + try { + await fh.read(Buffer.alloc(1), 0, 1, 0); + } finally { + await fh.close(); + } + break; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') break; // nothing planted there + if (attempt >= 40) throw err; // ~6s of retries: report the leak + await new Promise((resolve) => setTimeout(resolve, 150)); + } + } + } +} + +/** + * Deterministic interrupted-checkpoint signature: build rows on a writable + * session with AUTO-CHECKPOINT DISABLED and close WITHOUT checkpointing, so — + * exactly like a CHECKPOINT killed mid-flight — the main file is stale and + * every row lives only in the WAL. Then rename that WAL to the + * `lbug.wal.checkpoint` name the engine gives it during checkpoint, and plant + * the shadow + intent/apply lock files it leaves behind. + */ +async function plantInterruptedCheckpoint(dbPath: string): Promise { + // Raw constructor (positional args mirror createLbugDatabase) because the + // autoCheckpoint toggle is not exposed through the config helpers — and + // auto-checkpoint-on-close is precisely what must NOT happen here. + const db = new lbug.Database( + dbPath, + 128 * 1024 * 1024, // bufferManagerSize + false, // enableCompression + false, // readOnly + 16 * 1024 * 1024 * 1024, // maxDBSize + false, // autoCheckpoint — the whole point + 64 * 1024 * 1024, // checkpointThreshold + false, // throwOnWalReplayFailure + true, // enableChecksums + ); + await db.init(); + const conn = new lbug.Connection(db); + try { + await conn.query('CREATE NODE TABLE Person (name STRING, PRIMARY KEY(name))'); + for (let i = 0; i < ROWS; i += 100) { + const batch = Array.from({ length: 100 }, (_, j) => `{name: 'p${i + j}'}`).join(', '); + await conn.query(`UNWIND [${batch}] AS r CREATE (:Person {name: r.name})`); + } + const walBuffer = await fs.readFile(`${dbPath}.wal`); + // Honesty check: the rows must actually LIVE in the WAL — on an engine + // that tolerates the planted state this is the only proof the plant is + // not an empty shell (review finding: unused walBuffer). + expect(walBuffer.byteLength).toBeGreaterThan(0); + // Close WITHOUT checkpoint: rows stay WAL-only, main file stays stale. + // Explicitly awaited release BEFORE the rename/reopen — on Windows the + // kernel releases the engine's handles/locks asynchronously and the WAL + // rename + pooled reopen race them (Win32 Error 33, seen in CI). + await conn.close().catch(() => {}); + await db.close().catch(() => {}); + await waitForFixtureRelease(dbPath); + // Re-plant the captured WAL bytes rather than renaming the original: a + // close-time auto-checkpoint can consume the live .wal file out from + // under the rename (ENOENT — the fixture's other CI flake), while the + // captured buffer is what a killed checkpoint would have left behind. + await fs.writeFile(`${dbPath}.wal.checkpoint`, walBuffer); + await fs.writeFile(`${dbPath}.wal`, ''); + await fs.writeFile(`${dbPath}.shadow`, ''); + await fs.writeFile(`${dbPath}.checkpoint.intent.lock`, ''); + await fs.writeFile(`${dbPath}.checkpoint.apply.lock`, ''); + } catch (err) { + await conn.close().catch(() => {}); + await db.close().catch(() => {}); + throw err; + } +} + +describe('interrupted-checkpoint recovery (pooled read path self-heal)', () => { + let dbPath: string; + let tmpDir: string; + + afterAll(async () => { + await closeLbug(REPO).catch(() => {}); + if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it('opens read-only through the pool refusal and answers queries', async (ctx) => { + // Engine-version honesty: only 0.19+ treats the planted signature as an + // interrupted checkpoint ("Cannot open database in read-only mode while + // checkpoint is in progress"). On the 0.18.x pin the engine TOLERATES the + // plant — and worse, its staging-replay of the synthetic sidecars is + // nondeterministic (double-apply → "Person already exists in catalog"; + // observed as a hosted-CI flake), so running the plant there buys a + // vacuous smoke test at flake prices. The behavioral coverage on ANY pin + // lives in the forced-refusal units (lbug-pool-forced-refusal-heal, + // lbug-direct-forced-refusal-heal); this native plant runs where the bug + // actually exists — 0.19+ — and is registered in lbug-db / LBUG_NATIVE so + // Windows and macOS exercise it the moment the pin moves off 0.18.3. + const engineVersion = JSON.parse( + await fs.readFile( + path.join( + path.dirname(fileURLToPath(import.meta.url)), + '../../node_modules/@ladybugdb/core/package.json', + ), + 'utf-8', + ), + ).version as string; + const [major, minor] = engineVersion.split('.').map((part) => Number(part)); + // Skip only 0.x below 0.19. A 1.0.0 pin is newer than 0.19 and must run. + if (Number.isFinite(major) && Number.isFinite(minor) && major === 0 && minor < 19) { + ctx.skip(); + } + + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-lbug-interrupted-cp-')); + dbPath = path.join(tmpDir, 'lbug'); + await plantInterruptedCheckpoint(dbPath); + + // Prove the planted state is the real one before the pool heals it. + await expect( + (async () => { + const probe = new lbug.Database( + dbPath, + 128 * 1024 * 1024, + false, + true, + 16 * 1024 * 1024 * 1024, + true, + 64 * 1024 * 1024, + false, + true, + ); + try { + await probe.init(); + } finally { + await probe.close().catch(() => {}); + } + })(), + ).rejects.toThrow(/checkpoint is in progress/i); + // The probe's native close is best-effort; its handles must be gone + // before the pooled open below (Windows Error 33 otherwise). + await waitForFixtureRelease(dbPath); + + // The wiki path: pooled READ-ONLY open. Before the fix this refused with + // "Cannot open database in read-only mode while checkpoint is in + // progress" on 0.19.x engines and never recovered on its own. + await initLbug(REPO, dbPath); + + const rows = await executeQuery(REPO, 'MATCH (n:Person) RETURN count(n) AS c'); + expect(rows.length).toBe(1); + expect(Number((rows[0] as Record)['c'])).toBe(ROWS); + + // Normalize to the state a healthy engine leaves after recovery: on + // 0.19.x the recovery CHECKPOINT consumes the staged wal.checkpoint and + // the checkpoint locks, but on 0.18.3 (which never staged them) they + // survive the recovery — and a second open would replay the stale + // staging WAL onto a main file that already has the rows ("Person + // already exists in catalog", the fixture's other CI flake). The + // post-recovery contract is "sidecars consumed"; pin that before + // reopening. + for (const artifact of [ + 'lbug.wal.checkpoint', + 'lbug.shadow', + 'lbug.checkpoint.intent.lock', + 'lbug.checkpoint.apply.lock', + ]) { + await fs.rm(path.join(tmpDir, artifact), { force: true }); + } + + // A second open must answer without needing recovery again. + await closeLbug(REPO); + await waitForFixtureRelease(dbPath); + await initLbug(REPO, dbPath); + const again = await executeQuery(REPO, 'MATCH (n:Person) RETURN count(n) AS c'); + expect(again.length).toBe(1); + expect(Number((again[0] as Record)['c'])).toBe(ROWS); + }); +}); diff --git a/gitnexus/test/unit/lbug-direct-forced-refusal-heal.test.ts b/gitnexus/test/unit/lbug-direct-forced-refusal-heal.test.ts new file mode 100644 index 000000000..85042b2ed --- /dev/null +++ b/gitnexus/test/unit/lbug-direct-forced-refusal-heal.test.ts @@ -0,0 +1,157 @@ +/** + * Behavioral recovery test — DIRECT adapter, refusal FORCED on any engine pin. + * + * Complements `lbug-pool-forced-refusal-heal.test.ts`: the direct adapter is + * LAZY (the native Database constructor defers the file open to the first + * query — `ensureReadOnlyConnectionUsable`'s probe), so its refusal surfaces + * at probe time and must route through the same writable-recovery: probe → + * writable open → probe → CHECKPOINT → read-only reopen. Pinned here behind a + * mocked native layer so CI enforces it on the 0.18.3 pin too (where no real + * engine emits the refusal). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const REFUSAL = + 'Connection exception: Cannot open database in read-only mode while checkpoint is in progress. Please retry later.'; + +const { native } = vi.hoisted(() => { + const native = { + calls: { + constructions: [] as Array<'ro' | 'rw'>, + checkpoints: 0, + refusalProbes: 0, + }, + /** Script the FIRST read-only Database (index 0) as the checkpoint victim. */ + refuseFirst: true, + /** Script constructor-time refusal (direct adapter has no init() on open). */ + refuseOnOpen: false, + reset(options: { refuseFirst?: boolean; refuseOnOpen?: boolean } = {}) { + native.seq = 0; + native.calls.constructions = []; + native.calls.checkpoints = 0; + native.calls.refusalProbes = 0; + native.refuseFirst = options.refuseFirst ?? true; + native.refuseOnOpen = options.refuseOnOpen ?? false; + }, + seq: 0, + }; + return { native }; +}); + +// The DIRECT adapter is lazy — its index-0 read-only Database never gets +// init(); the open happens at the FIRST probe query, which refuses. The +// writable recovery open and the read-only retry (index 1 and 2) never +// refuse, or the recovery would kill itself. +vi.mock('@ladybugdb/core', () => { + class Database { + role: 'ro' | 'rw'; + index: number; + constructor( + _path: unknown, + _bufferManagerSize: unknown, + _compression: unknown, + readOnly = false, + ) { + this.role = readOnly ? 'ro' : 'rw'; + this.index = native.seq++; + native.calls.constructions.push(this.role); + // Open-time path: `createLbugDatabase` / `openLbugConnection` never + // call init(); a 0.19 constructor refusal must surface here. + if (this.index === 0 && this.role === 'ro' && native.refuseOnOpen) { + throw new Error(REFUSAL); + } + } + async init(): Promise {} + async close(): Promise {} + } + const refuseIfVictim = (db: Database, text: string): void => { + const t = text.trim().toUpperCase(); + if (db.index === 0 && db.role === 'ro' && native.refuseFirst && t.startsWith('MATCH')) { + native.calls.refusalProbes++; + throw new Error(REFUSAL); + } + if (t === 'CHECKPOINT') native.calls.checkpoints++; + }; + const emptyResult = { + getAll: async () => [], + close: async () => {}, + isSuccess: () => true, + getErrorMessage: async () => '', + }; + class Connection { + constructor(private db: Database) {} + async query(text: string) { + refuseIfVictim(this.db, text); + return emptyResult; + } + async prepare(cypher: string) { + refuseIfVictim(this.db, cypher); + return { isSuccess: () => true, getErrorMessage: async () => '' }; + } + async execute() { + return emptyResult; + } + async close(): Promise {} + } + const mod = { Database, Connection }; + return { ...mod, default: mod, lbug: mod }; +}); + +import { closeLbug, withLbugDb } from '../../src/core/lbug/lbug-adapter.js'; + +describe('direct adapter self-heals a refused read-only probe (forced refusal)', () => { + let dbPath: string; + let tmpDir: string; + + beforeEach(async () => { + native.reset(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-direct-forced-cp-')); + dbPath = path.join(tmpDir, 'lbug'); + await fs.writeFile(dbPath, ''); + }); + + afterEach(async () => { + await closeLbug().catch(() => {}); + if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it('recovers via writable CHECKPOINT and answers inside the same withLbugDb call', async () => { + const result = await withLbugDb( + dbPath, + async () => { + // Running on the recovered connection: any read-only query lands on + // the healed reopen (constructions[2]) and must not refuse. + return 'served'; + }, + { readOnly: true }, + ); + + expect(result).toBe('served'); + expect(native.calls.constructions).toEqual(['ro', 'rw', 'ro']); + expect(native.calls.checkpoints).toBe(1); + expect(native.calls.refusalProbes).toBe(1); + }); + + it('does not open writable when the read-only probe succeeds outright', async () => { + // No refusal scripted: the singleton serves the whole call read-only. + native.reset({ refuseFirst: false }); + const result = await withLbugDb(dbPath, async () => 'served', { readOnly: true }); + + expect(result).toBe('served'); + expect(native.calls.constructions).toEqual(['ro']); + expect(native.calls.checkpoints).toBe(0); + }); + + it('recovers when the read-only constructor refuses before any probe', async () => { + native.reset({ refuseFirst: false, refuseOnOpen: true }); + const result = await withLbugDb(dbPath, async () => 'served', { readOnly: true }); + + expect(result).toBe('served'); + expect(native.calls.constructions).toEqual(['ro', 'rw', 'ro']); + expect(native.calls.checkpoints).toBe(1); + expect(native.calls.refusalProbes).toBe(0); + }); +}); diff --git a/gitnexus/test/unit/lbug-pool-forced-refusal-heal.test.ts b/gitnexus/test/unit/lbug-pool-forced-refusal-heal.test.ts new file mode 100644 index 000000000..d5aafa868 --- /dev/null +++ b/gitnexus/test/unit/lbug-pool-forced-refusal-heal.test.ts @@ -0,0 +1,151 @@ +/** + * Behavioral recovery test — POOL adapter, refusal FORCED on any engine pin. + * + * The integration plant (`lbug-interrupted-checkpoint-recovery.test.ts`) only + * exercises the refusal on engines that emit it (0.19+); on the committed + * 0.18.3 pin CI can stay green with the new classifier and recovery CHECKPOINT + * deleted (review finding: "0.18.3 CI never forces the refusal"). This suite + * pins the BEHAVIOR behind a mocked native layer instead: the read-only open + * throws the canonical 0.19 refusal, and the pool must run the full + * self-heal — writable open, probe, CHECKPOINT, read-only retry — regardless + * of which engine binary is installed. + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +const REFUSAL = + 'Connection exception: Cannot open database in read-only mode while checkpoint is in progress. Please retry later.'; + +const { native } = vi.hoisted(() => { + const native = { + calls: { + /** Every Database construction in order — the self-heal shape. */ + constructions: [] as Array<'ro' | 'rw'>, + checkpoints: 0, + /** Every MATCH — victim, writable replay probe, and post-heal retry. */ + probes: 0, + }, + /** Script the FIRST read-only Database (index 0) as the checkpoint victim. */ + refuseFirst: true, + reset(options: { refuseFirst?: boolean } = {}) { + native.seq = 0; + native.calls.constructions = []; + native.calls.checkpoints = 0; + native.calls.probes = 0; + native.refuseFirst = options.refuseFirst ?? true; + }, + seq: 0, + }; + return { native }; +}); + +// The Database at index 0 is the interrupted-checkpoint victim: its init() +// throws the canonical refusal (the pool calls init explicitly) and any query +// on it refuses. Every LATER Database is healthy — in particular the WRITABLE +// recovery open and the read-only retry must never refuse, or the recovery +// would kill itself. +vi.mock('@ladybugdb/core', () => { + class Database { + role: 'ro' | 'rw'; + index: number; + constructor( + _path: unknown, + _bufferManagerSize: unknown, + _compression: unknown, + readOnly = false, + ) { + this.role = readOnly ? 'ro' : 'rw'; + this.index = native.seq++; + native.calls.constructions.push(this.role); + } + async init(): Promise { + if (this.index === 0 && this.role === 'ro' && native.refuseFirst) { + throw new Error(REFUSAL); + } + } + async close(): Promise {} + } + const refuseIfVictim = (db: Database, text: string): void => { + const t = text.trim().toUpperCase(); + if (t.startsWith('MATCH')) native.calls.probes++; + if (db.index === 0 && db.role === 'ro' && native.refuseFirst && t.startsWith('MATCH')) { + throw new Error(REFUSAL); + } + if (t === 'CHECKPOINT') native.calls.checkpoints++; + }; + const emptyResult = { + getAll: async () => [], + close: async () => {}, + isSuccess: () => true, + getErrorMessage: async () => '', + }; + class Connection { + constructor(private db: Database) {} + async query(text: string) { + refuseIfVictim(this.db, text); + return emptyResult; + } + async prepare(cypher: string) { + refuseIfVictim(this.db, cypher); + return { isSuccess: () => true, getErrorMessage: async () => '' }; + } + async execute() { + return emptyResult; + } + async close(): Promise {} + } + const mod = { Database, Connection }; + return { ...mod, default: mod, lbug: mod }; +}); + +import { closeLbug, executeQuery, initLbug } from '../../src/core/lbug/pool-adapter.js'; + +describe('pool adapter self-heals a refused read-only open (forced refusal)', () => { + let dbPath: string; + let tmpDir: string; + const REPO = 'test-pool-forced-refusal'; + + beforeEach(async () => { + native.reset(); + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-pool-forced-cp-')); + dbPath = path.join(tmpDir, 'lbug'); + // The native layer is mocked; the file only has to EXIST (doInitLbug + // stats it before opening). + await fs.writeFile(dbPath, ''); + }); + + afterEach(async () => { + await closeLbug(REPO).catch(() => {}); + if (tmpDir) await fs.rm(tmpDir, { recursive: true, force: true }); + }); + + it('runs writable open + CHECKPOINT + read-only retry, then serves queries', async () => { + await initLbug(REPO, dbPath); + + // Self-heal shape: refused read-only open → writable recovery open → + // read-only retry. Exactly one CHECKPOINT, issued by the recovery. + expect(native.calls.constructions).toEqual(['ro', 'rw', 'ro']); + expect(native.calls.checkpoints).toBe(1); + // Writable replay MATCH + post-heal read-only MATCH. A CHECKPOINT without + // its required replay probe would leave this at 1. + expect(native.calls.probes).toBe(2); + + const rows = await executeQuery(REPO, 'MATCH (n:Person) RETURN count(n) AS c'); + expect(rows).toEqual([]); + }); + + it('does not issue a CHECKPOINT when the read-only open succeeds outright', async () => { + // A healthy db: the first read-only construction is the only one, and the + // recovery machinery must stay out of the way (no writable open, no + // CHECKPOINT on the read path). + native.reset({ refuseFirst: false }); + await initLbug(REPO, dbPath); + await executeQuery(REPO, 'MATCH (n:Person) RETURN count(n) AS c'); + + expect(native.calls.constructions).toEqual(['ro']); + expect(native.calls.checkpoints).toBe(0); + expect(native.calls.probes).toBe(2); + }); +}); diff --git a/gitnexus/test/unit/sidecar-recovery.test.ts b/gitnexus/test/unit/sidecar-recovery.test.ts index b1634cdf2..3853ffaf0 100644 --- a/gitnexus/test/unit/sidecar-recovery.test.ts +++ b/gitnexus/test/unit/sidecar-recovery.test.ts @@ -15,7 +15,12 @@ import { inspectLbugSidecars, isMissingShadowSidecarError, isPermissionRenameError, + INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX, + isReadOnlyCheckpointInProgressError, + isReadOnlyRecoveryFailure, isReadOnlyShadowReplayError, + PENDING_SHADOW_REPLAY_RECOVERY_PREFIX, + readOnlyRecoveryFailureMessage, listParkedDirtyRecoverySidecars, listParkedLbugSidecars, listQuarantinedMissingShadowWals, @@ -228,13 +233,125 @@ describe('LadybugDB sidecar recovery', () => { expect(source).not.toMatch(/replay shadow pages under read-only mode/); }); - it('structural: sidecar-recovery.ts carries exactly two LADYBUGDB-CONTRACT markers (one per shadow predicate)', () => { + it('structural: sidecar-recovery.ts carries exactly three LADYBUGDB-CONTRACT markers (one per native-error predicate)', () => { const source = readFileSync( path.join(__dirname, '..', '..', 'src', 'core', 'lbug', 'sidecar-recovery.ts'), 'utf-8', ); const markers = source.match(/\/\/ LADYBUGDB-CONTRACT:/g) ?? []; - expect(markers.length).toBe(2); + expect(markers.length).toBe(3); + }); + }); + + describe('isReadOnlyCheckpointInProgressError (interrupted checkpoint, homelab repro 2026-09-19)', () => { + // Byte text from the live occurrence: a wiki pod killed mid-CHECKPOINT + // left lbug.wal + lbug.shadow, and every later read-only open refused with + // exactly this message (wrapped by the pool as "LadybugDB unavailable for + // __wiki__ ..."). + const CANONICAL = + 'Connection exception: Cannot open database in read-only mode while checkpoint is in progress.'; + + it('matches the canonical native message', () => { + expect(isReadOnlyCheckpointInProgressError(new Error(CANONICAL))).toBe(true); + }); + + it('matches prefix-wrapped and case-variant forms', () => { + expect( + isReadOnlyCheckpointInProgressError( + new Error(`LadybugDB unavailable for __wiki__. Retry later. (${CANONICAL})`), + ), + ).toBe(true); + expect( + isReadOnlyCheckpointInProgressError( + new Error('cannot Open Database in Read-Only Mode while checkpoint is in Progress'), + ), + ).toBe(true); + }); + + it('accepts non-Error values and rejects unrelated read-only errors', () => { + expect(isReadOnlyCheckpointInProgressError(CANONICAL)).toBe(true); + expect(isReadOnlyCheckpointInProgressError(null)).toBe(false); + expect(isReadOnlyCheckpointInProgressError(undefined)).toBe(false); + // The sibling refusals must NOT match — each has its own classifier and + // its own recovery nuance. + expect( + isReadOnlyCheckpointInProgressError( + new Error("Couldn't replay shadow pages under read-only mode."), + ), + ).toBe(false); + expect( + isReadOnlyCheckpointInProgressError( + new Error('Cannot execute write operations in a read-only database!'), + ), + ).toBe(false); + expect( + isReadOnlyCheckpointInProgressError(new Error('Cannot open file x.shadow: No such file')), + ).toBe(false); + }); + + it('structural: both adapters route the new classifier through the writable-recovery path', () => { + for (const file of ['lbug-adapter.ts', 'pool-adapter.ts']) { + const source = readFileSync( + path.join(__dirname, '..', '..', 'src', 'core', 'lbug', file), + 'utf-8', + ); + expect(source, file).toContain('isReadOnlyCheckpointInProgressError'); + // The refusal rides the SAME recovery as the shadow-replay error — + // neither adapter may quarantine or rebuild for this state. The + // leading `!` matters: this must pin the THROW-THROUGH guard + // (`!shadowReplay && !checkpoint`), and a substring match without it + // would also accept the inverted predicate that recovers ONLY the + // shadow-replay class — the exact regression this guards against. + expect(source, file).toMatch( + /!isReadOnlyShadowReplayError\(err\) &&\s*!isReadOnlyCheckpointInProgressError\(err\)/, + ); + } + // The classifier regex itself lives only in sidecar-recovery.ts. + const adapter = readFileSync( + path.join(__dirname, '..', '..', 'src', 'core', 'lbug', 'lbug-adapter.ts'), + 'utf-8', + ); + expect(adapter).not.toMatch(/checkpoint is in progress\/i/); + expect(adapter).toContain('readOnlyRecoveryFailureMessage'); + expect(adapter).toContain('isReadOnlyRecoveryFailure'); + const pool = readFileSync( + path.join(__dirname, '..', '..', 'src', 'core', 'lbug', 'pool-adapter.ts'), + 'utf-8', + ); + expect(pool).not.toMatch(/checkpoint is in progress\/i/); + }); + }); + + describe('readOnlyRecoveryFailureMessage does not rematch native classifiers', () => { + const CANONICAL = + 'Connection exception: Cannot open database in read-only mode while checkpoint is in progress.'; + const SHADOW = + "Runtime exception: Couldn't replay shadow pages under read-only mode. Please re-open the database with read-write mode to replay shadow pages."; + + it('wraps checkpoint refusal without rematching the native classifier', () => { + const wrapped = readOnlyRecoveryFailureMessage(dbPath, new Error(CANONICAL)); + expect(wrapped.startsWith(INTERRUPTED_CHECKPOINT_RECOVERY_PREFIX)).toBe(true); + expect(wrapped).toMatch(/gitnexus analyze/); + expect(wrapped).not.toMatch(/sidecar is missing/i); + expect(wrapped).not.toMatch(/--force/); + expect(wrapped).not.toContain(CANONICAL); + expect(isReadOnlyCheckpointInProgressError(new Error(wrapped))).toBe(false); + expect(isReadOnlyRecoveryFailure(new Error(wrapped))).toBe(true); + }); + + it('wraps shadow-replay refusal without rematching the native classifier', () => { + const wrapped = readOnlyRecoveryFailureMessage(dbPath, new Error(SHADOW)); + expect(wrapped.startsWith(PENDING_SHADOW_REPLAY_RECOVERY_PREFIX)).toBe(true); + expect(wrapped).not.toContain(SHADOW); + expect(isReadOnlyShadowReplayError(new Error(wrapped))).toBe(false); + expect(isReadOnlyRecoveryFailure(new Error(wrapped))).toBe(true); + }); + + it('delegates missing-shadow to shadowSidecarRecoveryMessage', () => { + const err = new Error('Cannot open file /tmp/lbug.shadow: No such file or directory'); + expect(readOnlyRecoveryFailureMessage('/tmp/lbug', err)).toBe( + shadowSidecarRecoveryMessage('/tmp/lbug', err), + ); }); }); diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 1239862a3..519b51f09 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -96,6 +96,7 @@ export default defineConfig({ 'test/integration/java-class-impact.test.ts', 'test/integration/class-impact-all-languages.test.ts', 'test/integration/lbug-orphan-sidecar-recovery.test.ts', + 'test/integration/lbug-interrupted-checkpoint-recovery.test.ts', 'test/integration/lbug-readonly-init.test.ts', 'test/integration/analyze-wal-checkpoint-failure.test.ts', 'test/integration/lbug-non-ascii-path.test.ts', @@ -176,6 +177,7 @@ export default defineConfig({ 'test/integration/java-class-impact.test.ts', 'test/integration/class-impact-all-languages.test.ts', 'test/integration/lbug-orphan-sidecar-recovery.test.ts', + 'test/integration/lbug-interrupted-checkpoint-recovery.test.ts', 'test/integration/lbug-readonly-init.test.ts', 'test/integration/analyze-wal-checkpoint-failure.test.ts', 'test/integration/lbug-non-ascii-path.test.ts',