diff --git a/gitnexus/README.md b/gitnexus/README.md index bd2e59110..67e9d39a8 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -444,14 +444,14 @@ The prefix defaults to `~/.gitnexus/embedding-runtime`; set `GITNEXUS_EMBEDDING_ ### Analyze warns about unavailable FTS or VECTOR extensions -GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnexus serve` and MCP read paths only ever try to `LOAD` the extensions — they never block on a network install. The `analyze` command, by default, attempts one bounded out-of-process `INSTALL` if `LOAD` fails and proceeds even when that install times out, so the index is always written to disk; BM25/vector search degrade gracefully until the extensions become available. +GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnexus serve` and MCP read paths only ever try to `LOAD` the extensions — they never block on a network install. The `analyze` command, by default, attempts one bounded out-of-process install if `LOAD` fails (a plain `INSTALL` to download a missing extension, escalating to `FORCE INSTALL` only when the `LOAD` error shows the existing file is broken or truncated, so a permanent non-file failure does not re-download on every run) and proceeds even when that install times out, so the index is always written to disk; BM25/vector search degrade gracefully until the extensions become available. Configure the behavior with these environment variables: | Variable | Values | Default | Effect | | -------------------------------------------- | ---------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded INSTALL if LOAD fails. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | -| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process `INSTALL` child before it is killed. | +| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | +| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | | `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | | `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | | `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 14a20bfec..342b214b4 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -100,6 +100,10 @@ const SPAWN_CLI = [ 'test/integration/cli-limit-e2e.test.ts', 'test/integration/hooks-e2e.test.ts', 'test/integration/skills-e2e.test.ts', + // Spawns the real CLI across hermetic HOME/USERPROFILE homes to exercise the + // FTS extension lifecycle — the #2374 bug was Windows-reported, so this must + // run on the Windows/macOS matrix, not just the Ubuntu full suite. + 'test/integration/fts-extension-e2e.test.ts', 'test/integration/server-http-startup.test.ts', 'test/integration/mcp/server-startup.test.ts', 'test/integration/analyze-heap-oom-e2e.test.ts', diff --git a/gitnexus/scripts/install-duckdb-extension.mjs b/gitnexus/scripts/install-duckdb-extension.mjs index 7492e084f..b3b8bebc2 100644 --- a/gitnexus/scripts/install-duckdb-extension.mjs +++ b/gitnexus/scripts/install-duckdb-extension.mjs @@ -3,9 +3,39 @@ import fs from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/; +// Positive on-disk-corruption signatures. `FORCE INSTALL` re-downloads even when +// a file is already present; we only want that when the LOAD error proves the +// existing file is bad (truncated/wrong-platform, #2374). For everything else — +// a missing file (plain INSTALL downloads it), or a permanent non-file failure a +// re-download can never fix (missing runtime dep: "cannot open shared object") — +// plain INSTALL avoids re-downloading ~2 MB on every analyze run forever. +const FILE_CORRUPTION_SIGNATURES = [ + /invalid elf/i, + /file too short/i, + /not a valid/i, + /bad magic/i, + /wrong architecture/i, + /mach-o/i, + /truncat/i, +]; + +/** + * Decide the install verb from the LOAD error that triggered this install. + * `FORCE INSTALL` only when the error positively indicates file-level breakage; + * otherwise plain `INSTALL` (missing file, missing-dependency dlopen failure, + * or unknown/absent error). + */ +export function chooseInstallVerb(loadError) { + if (loadError && FILE_CORRUPTION_SIGNATURES.some((re) => re.test(loadError))) { + return 'FORCE INSTALL'; + } + return 'INSTALL'; +} + function parseLbugMaxDbSize(raw) { const parsed = raw ? Number(raw) : NaN; if (!Number.isFinite(parsed) || parsed <= 0) { @@ -14,28 +44,54 @@ function parseLbugMaxDbSize(raw) { return Math.floor(parsed); } -async function installDuckDbExtension(extensionName, verifyOnly = false) { - if (!extensionName || !EXTENSION_NAME_PATTERN.test(extensionName)) { - throw new Error(`Invalid DuckDB extension name: ${extensionName ?? ''}`); - } - - const require = createRequire(import.meta.url); - const lbugModule = require('@ladybugdb/core'); - const lbug = lbugModule.default ?? lbugModule; +function resolveMaxDbSize() { // argv[3] is the optional positional size; ignore it when it is actually a // flag token (e.g. `--verify-only`) and fall back to the env default. const sizeArg = process.argv[3] && !process.argv[3].startsWith('--') ? process.argv[3] : undefined; - const lbugMaxDbSize = parseLbugMaxDbSize(sizeArg ?? process.env.GITNEXUS_LBUG_MAX_DB_SIZE); + return parseLbugMaxDbSize(sizeArg ?? process.env.GITNEXUS_LBUG_MAX_DB_SIZE); +} + +/** Open a scratch LadybugDB and return its connection plus a disposer. */ +async function defaultConnect(lbugMaxDbSize) { + const require = createRequire(import.meta.url); + const lbugModule = require('@ladybugdb/core'); + const lbug = lbugModule.default ?? lbugModule; const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-ext-install-')); const dbPath = path.join(tmpDir, 'install.lbug'); - let db; - let conn; + const db = new lbug.Database(dbPath, 0, false, false, lbugMaxDbSize); + const conn = new lbug.Connection(db); + return { + conn, + dispose: async () => { + await conn.close().catch(() => {}); + await db.close().catch(() => {}); + await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + }, + }; +} + +/** + * Install (or verify) an optional LadybugDB extension in this short-lived process. + * + * @param {string} extensionName + * @param {object} [options] + * @param {boolean} [options.verifyOnly] LOAD-only Docker build gate — no install. + * @param {string} [options.loadError] The parent's LOAD failure; selects the verb. + * @param {(size: number) => Promise<{conn: {query: (sql: string) => Promise}, dispose: () => Promise}>} [options.connect] + * Connection factory; injectable for offline unit tests. + */ +export async function installDuckDbExtension(extensionName, options = {}) { + const { verifyOnly = false, loadError, connect } = options; + if (!extensionName || !EXTENSION_NAME_PATTERN.test(extensionName)) { + throw new Error(`Invalid DuckDB extension name: ${extensionName ?? ''}`); + } + + const makeConnection = connect ?? (() => defaultConnect(resolveMaxDbSize())); + const { conn, dispose } = await makeConnection(); try { - db = new lbug.Database(dbPath, 0, false, false, lbugMaxDbSize); - conn = new lbug.Connection(db); if (verifyOnly) { // Prove a previously-baked extension is resolvable by a FRESH process // under the current HOME (the runtime `LOAD EXTENSION` path) — no INSTALL, @@ -46,19 +102,22 @@ async function installDuckDbExtension(extensionName, verifyOnly = false) { `[install-ext] LOAD-only verify OK for '${extensionName}' (HOME=${process.env.HOME})`, ); } else { - await conn.query(`INSTALL ${extensionName}`); + // Plain INSTALL is a no-op when the file already exists; escalate to FORCE + // only when the LOAD error proves the on-disk file is broken (#2374). + await conn.query(`${chooseInstallVerb(loadError)} ${extensionName}`); } } finally { - if (conn) await conn.close().catch(() => {}); - if (db) await db.close().catch(() => {}); - await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => {}); + await dispose(); } } -installDuckDbExtension( - process.argv[2] ?? process.env.GITNEXUS_LBUG_EXTENSION_NAME, - process.argv.includes('--verify-only'), -).catch((err) => { - console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); - process.exitCode = 1; -}); +// Only run when executed directly — imported (e.g. by unit tests) it stays inert. +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + installDuckDbExtension(process.argv[2] ?? process.env.GITNEXUS_LBUG_EXTENSION_NAME, { + verifyOnly: process.argv.includes('--verify-only'), + loadError: process.env.GITNEXUS_LBUG_EXTENSION_LOAD_ERROR, + }).catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); + process.exitCode = 1; + }); +} diff --git a/gitnexus/src/cli/doctor.ts b/gitnexus/src/cli/doctor.ts index dd5af7f48..f16b6c44d 100644 --- a/gitnexus/src/cli/doctor.ts +++ b/gitnexus/src/cli/doctor.ts @@ -12,7 +12,7 @@ import { type EmbeddingRuntimeResolution, } from '../core/embeddings/runtime-install.js'; import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js'; -import { checkLbugNative } from '../core/lbug/native-check.js'; +import { checkLbugNative, probeFtsExtensionLoad } from '../core/lbug/native-check.js'; import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js'; import { t } from './i18n/index.js'; @@ -133,7 +133,17 @@ export const doctorCommand = async () => { console.log(''); console.log(t('doctor.capabilities')); console.log(` ${label('doctor.labels.graphStore', 18)}${capabilities.graph}`); - console.log(` ${label('doctor.labels.fullTextSearch', 18)}${capabilities.fts}`); + // Live LOAD probe, not the static platform capability — the static value + // said "available" while analyze failed to load the extension (#2374). + const ftsProbe = nativeCheck.ok + ? await probeFtsExtensionLoad() + : { loaded: false, reason: 'LadybugDB native module (lbugjs.node) failed to load' }; + console.log( + ` ${label('doctor.labels.fullTextSearch', 18)}${ftsProbe.loaded ? 'available' : 'unavailable'}`, + ); + if (!ftsProbe.loaded && ftsProbe.reason) { + console.log(` ${padDisplayEnd('', 18)}${ftsProbe.reason}`); + } console.log(` ${label('doctor.labels.vectorIndex', 18)}${capabilities.vector}`); console.log(` ${label('doctor.labels.semanticMode', 18)}${capabilities.semanticMode}`); // Surface the optional-extension install policy so offline users can see diff --git a/gitnexus/src/core/lbug/extension-loader.ts b/gitnexus/src/core/lbug/extension-loader.ts index e01ed0ea5..4fe1e220a 100644 --- a/gitnexus/src/core/lbug/extension-loader.ts +++ b/gitnexus/src/core/lbug/extension-loader.ts @@ -41,7 +41,11 @@ export interface ExtensionEnsureOptions { export interface ExtensionManagerOptions { policy?: ExtensionInstallPolicy; installTimeoutMs?: number; - installExtension?: (extensionName: string, timeoutMs: number) => Promise; + installExtension?: ( + extensionName: string, + timeoutMs: number, + loadError?: string, + ) => Promise; warn?: (message: string) => void; } @@ -50,6 +54,9 @@ const alreadyAvailable = (message: string): boolean => message.includes('already installed') || message.includes('already exists'); +/** LadybugDB errors are multi-line; collapse for single-line warn/reason strings. */ +const oneLine = (value: string): string => value.replace(/\s+/g, ' ').trim(); + const resolvePolicyFromEnv = (): ExtensionInstallPolicy => { const raw = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL; if (raw === 'load-only' || raw === 'never' || raw === 'auto') return raw; @@ -103,6 +110,7 @@ export const getExtensionInstallChildProcessArgs = ( export const installDuckDbExtensionOutOfProcess = async ( extensionName: string, timeoutMs: number = getExtensionInstallTimeoutMs(), + loadError?: string, ): Promise => { if (!EXTENSION_NAME_PATTERN.test(extensionName)) { throw new Error(`Invalid DuckDB extension name: ${extensionName}`); @@ -113,6 +121,9 @@ export const installDuckDbExtensionOutOfProcess = async ( env: { ...process.env, GITNEXUS_LBUG_EXTENSION_NAME: extensionName, + // The child picks INSTALL vs FORCE INSTALL from this LOAD error so it + // only re-downloads when the on-disk extension file is actually broken. + ...(loadError ? { GITNEXUS_LBUG_EXTENSION_LOAD_ERROR: loadError } : {}), }, stdio: ['ignore', 'ignore', 'pipe'], windowsHide: true, @@ -132,7 +143,7 @@ export const installDuckDbExtensionOutOfProcess = async ( resolve({ success: false, timedOut: true, - message: `INSTALL ${extensionName} timed out after ${timeoutMs}ms`, + message: `extension install for ${extensionName} timed out after ${timeoutMs}ms`, }); }, timeoutMs); @@ -152,8 +163,8 @@ export const installDuckDbExtensionOutOfProcess = async ( timedOut: false, message: code === 0 - ? `INSTALL ${extensionName} completed` - : `INSTALL ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`, + ? `extension install for ${extensionName} completed` + : `extension install for ${extensionName} failed with ${signal ?? `exit code ${code}`}${stderr ? `: ${stderr.trim()}` : ''}`, }); }); }); @@ -218,44 +229,74 @@ export class ExtensionManager { return false; } - if (await this.tryLoad(query, name)) { + const loadError = await this.tryLoad(query, name); + if (loadError === null) { this.markLoaded(name); return true; } if (policy === 'load-only') { - this.markUnavailable(name, label, 'load-only policy: extension not pre-installed', warn); + this.markUnavailable( + name, + label, + `load-only policy (no install attempted); LOAD ${name} failed: ${loadError}`, + warn, + ); return false; } let install = this.installAttempted.get(name); if (!install) { const installFn = this.options.installExtension ?? installDuckDbExtensionOutOfProcess; - install = await installFn(name, timeoutMs); + // Hand the child the LOAD error so it re-downloads (FORCE) only when the + // present extension file is provably broken, not on every LOAD failure. + install = await installFn(name, timeoutMs, loadError); this.installAttempted.set(name, install); } if (!install.success) { - this.markUnavailable(name, label, install.message, warn); + this.markUnavailable( + name, + label, + `${install.message}; LOAD ${name} had failed: ${loadError}`, + warn, + ); return false; } - if (await this.tryLoad(query, name)) { + const retryError = await this.tryLoad(query, name); + if (retryError === null) { this.markLoaded(name); return true; } - this.markUnavailable(name, label, `LOAD ${name} failed after successful INSTALL`, warn); + this.markUnavailable( + name, + label, + `LOAD ${name} failed after successful INSTALL: ${retryError}`, + warn, + ); return false; } - private async tryLoad(query: (sql: string) => Promise, name: string): Promise { + /** + * Attempt `LOAD EXTENSION `; returns `null` on success and the + * collapsed error message on failure. The message is the load-side ground + * truth — LadybugDB distinguishes a missing extension file from a present + * but unloadable one (wrong platform, truncated download, version mismatch), + * and discarding it left users staring at "not pre-installed" when the file + * existed all along (#2374). + */ + private async tryLoad( + query: (sql: string) => Promise, + name: string, + ): Promise { try { await query(`LOAD EXTENSION ${name}`); - return true; + return null; } catch (err) { const msg = err instanceof Error ? err.message : String(err); - return alreadyAvailable(msg); + return alreadyAvailable(msg) ? null : oneLine(msg); } } diff --git a/gitnexus/src/core/lbug/native-check.ts b/gitnexus/src/core/lbug/native-check.ts index 8bbca7bc7..accf43a52 100644 --- a/gitnexus/src/core/lbug/native-check.ts +++ b/gitnexus/src/core/lbug/native-check.ts @@ -89,3 +89,87 @@ export function checkLbugNative(overridePkgDir?: string): NativeCheckResult { return { ok: true, binaryPath }; } + +export interface FtsProbeResult { + loaded: boolean; + /** Collapsed LadybugDB error when `loaded` is false. */ + reason?: string; +} + +const DEFAULT_FTS_PROBE_TIMEOUT_MS = 10_000; + +/** A LadybugDB query result exposes a synchronous `close()`. */ +interface CloseableResult { + close(): void; +} + +/** Close each result, swallowing close-time errors so a successful LOAD is not + * misreported as a failure (native-check keeps no static lbug dependency, so it + * cannot reuse the adapter's closeQueryResults — that would eagerly load the + * module and defeat the dynamic import below). */ +const closeProbeResults = (result: unknown): void => { + for (const r of Array.isArray(result) ? result : [result]) { + try { + (r as CloseableResult)?.close?.(); + } catch { + // ignore — a close failure must not flip a successful LOAD to failed + } + } +}; + +/** + * Live-probe `LOAD EXTENSION fts` on a throwaway in-memory database. + * + * `doctor` used to print the static platform capability, which contradicted + * analyze whenever the extension file was missing or unloadable (#2374). + * LOAD never touches the network, so the probe is safe offline, and it + * surfaces LadybugDB's real error — which distinguishes a missing extension + * file from a present-but-broken one (wrong platform, truncated download). + * Dynamic import so doctor still runs when the native module itself is broken. + * + * Bounded by `timeoutMs`: an unresponsive extension file (e.g. on a hung + * network home dir) must never freeze `doctor` — the tool the degradation + * warnings send users to. `Promise.race` lets doctor report and move on; it + * cannot cancel an in-flight native call, so a future thread-blocking case + * would need an out-of-process probe. + */ +export async function probeFtsExtensionLoad( + timeoutMs: number = DEFAULT_FTS_PROBE_TIMEOUT_MS, +): Promise { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout( + () => + resolve({ + loaded: false, + reason: 'probe timed out — extension file or filesystem unresponsive', + }), + timeoutMs, + ); + }); + + const probe = (async (): Promise => { + try { + const { default: lbug } = await import('@ladybugdb/core'); + const db = new lbug.Database(':memory:'); + // Nested finallys so `db` is closed even if the Connection ctor throws. + try { + const conn = new lbug.Connection(db); + try { + const result = await conn.query('LOAD EXTENSION fts'); + closeProbeResults(result); + return { loaded: true }; + } finally { + await conn.close().catch(() => {}); + } + } finally { + await db.close().catch(() => {}); + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { loaded: false, reason: message.replace(/\s+/g, ' ').trim() }; + } + })(); + + return await Promise.race([probe, timeout]).finally(() => clearTimeout(timer)); +} diff --git a/gitnexus/src/core/platform/capabilities.ts b/gitnexus/src/core/platform/capabilities.ts index 6ced56d5b..6bfe459e4 100644 --- a/gitnexus/src/core/platform/capabilities.ts +++ b/gitnexus/src/core/platform/capabilities.ts @@ -1,4 +1,6 @@ +import fs from 'fs'; import os from 'os'; +import path from 'path'; import { createRequire } from 'module'; const require = createRequire(import.meta.url); @@ -28,7 +30,31 @@ const packageVersion = (name: string): string | undefined => { try { return require(`${name}/package.json`).version; } catch { - return undefined; + // Packages whose `exports` map omits ./package.json (e.g. @ladybugdb/core) + // reject the direct require with ERR_PACKAGE_PATH_NOT_EXPORTED, which made + // doctor print "LadybugDB: unknown" on every platform (#2374). Resolve the + // entry point instead and walk up to the package's own package.json. + try { + let dir = path.dirname(require.resolve(name)); + // Entry points sit at the package root or a shallow dist/ dir; a few + // hops always reach the package's own package.json. + for (let hops = 0; hops < 5; hops++) { + const candidate = path.join(dir, 'package.json'); + if (fs.existsSync(candidate)) { + const pkg = JSON.parse(fs.readFileSync(candidate, 'utf8')) as { + name?: string; + version?: string; + }; + if (pkg.name === name) return pkg.version; + } + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + return undefined; + } catch { + return undefined; + } } }; diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index 0300c0416..787268b82 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -41,7 +41,7 @@ import { getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js'; -import { resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js'; +import { getExtensionCapabilities, resolveAnalyzeInstallPolicy } from './lbug/extension-loader.js'; import { startWalCheckpointDriver, type WalCheckpointDriver, @@ -679,10 +679,17 @@ export async function runFullAnalysis( policy: resolveAnalyzeInstallPolicy(), }); if (!repairFtsAvailable) { + // Surface the load-side reason (#2374): "not pre-installed" was wrong + // and doctor never installed anything, so the old message trapped + // users in a query → repair-fts → doctor loop with no way out. + const ftsReason = getExtensionCapabilities() + .find((c) => c.name === 'fts') + ?.reason?.replace(/\.$/, ''); throw new Error( - 'Cannot repair FTS indexes: the LadybugDB FTS extension is unavailable ' + - '(not pre-installed and could not be installed on this machine). ' + - 'Run `gitnexus doctor` to install it, then retry `--repair-fts`.', + 'Cannot repair FTS indexes: the LadybugDB FTS extension failed to load' + + (ftsReason ? ` — ${ftsReason}` : '') + + '. Retry with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to install it, ' + + 'or pre-install the extension file; run `gitnexus doctor` for live FTS status.', ); } progress('fts', 85, 'Repairing search indexes...'); diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index b53990620..2eac01b62 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -1,6 +1,38 @@ import { createFTSIndex, dropFTSIndex, DEFAULT_FTS_STEMMER } from '../lbug/lbug-adapter.js'; +import { getExtensionCapabilities } from '../lbug/extension-loader.js'; import { FTS_INDEXES } from './fts-schema.js'; +/** + * Strip filesystem paths from a LadybugDB error before it reaches the HTTP + * `/api/search` and MCP query surfaces (#2374, PR #2375): the raw LOAD error + * embeds the absolute extension path (username, home dir) which must not leak to + * a network client. The error class words ("Failed to load library", "invalid + * ELF header", "has not been installed") have no leading path separator and + * survive. CLI/doctor/log surfaces keep the full path (they read the reason + * directly, not through this function). + */ +const redactPaths = (reason: string): string => + reason.replace(/(?:[A-Za-z]:\\|\/)[^\s'"]+/g, ''); + +/** + * Warning attached to search responses when BM25/FTS is degraded. Prefers the + * live extension-load failure (with LadybugDB's real reason, #2374) over the + * generic indexes-missing message, so "indexes exist but the extension broke" + * is not misreported as missing indexes. + */ +export const ftsDegradedWarning = (): string => { + const fts = getExtensionCapabilities().find((c) => c.name === 'fts'); + if (fts && !fts.loaded) { + const reason = fts.reason ? redactPaths(fts.reason).replace(/\.$/, '') : undefined; + return ( + 'FTS extension failed to load — keyword search degraded' + + (reason ? ` (${reason})` : '') + + '. Run `gitnexus doctor` for details, then `gitnexus analyze --repair-fts` with network access to reinstall.' + ); + } + return 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.'; +}; + // Stemmers shipped by the LadybugDB FTS extension. Mirrors the lowercase token // set in the extension bundled with @ladybugdb/core 0.18.x (see package.json). // Keep in sync on a LadybugDB minor bump — a value here that the installed diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 1065d75b9..fe32bc960 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -62,6 +62,7 @@ import { isVectorExtensionSupportedByPlatform, } from '../../core/platform/capabilities.js'; import { PhaseTimer } from '../../core/search/phase-timer.js'; +import { ftsDegradedWarning } from '../../core/search/fts-indexes.js'; import { cjkSegmentationModeMismatch, containsSegmentableCjkRun, @@ -2084,9 +2085,7 @@ export class LocalBackend { // path, leaving the success-path response shape byte-identical. const warnings: string[] = []; if (!ftsUsed) { - warnings.push( - 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.', - ); + warnings.push(ftsDegradedWarning()); } // #2331: a CJK query against a server process resolving // GITNEXUS_FTS_CJK_SEGMENTATION to 'none' silently misses sub-phrase diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 383bed5ad..68fa6f872 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -28,6 +28,7 @@ import { isValidQueryParams } from '../core/lbug/query-params.js'; import { NODE_TABLES, type GraphNode, type GraphRelationship } from 'gitnexus-shared'; import { searchFTSFromLbug } from '../core/search/bm25-index.js'; import { hybridSearch } from '../core/search/hybrid-search.js'; +import { ftsDegradedWarning } from '../core/search/fts-indexes.js'; import { LocalBackend } from '../mcp/local/local-backend.js'; import { mountMCPEndpoints } from './mcp-http.js'; import { fileURLToPath } from 'url'; @@ -1318,8 +1319,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => ); const response: any = { results: results.searchResults ?? results }; if (results.ftsAvailable === false) { - response.warning = - 'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --repair-fts (or gitnexus analyze --force) to rebuild indexes.'; + response.warning = ftsDegradedWarning(); } res.json(response); } catch (err: any) { diff --git a/gitnexus/test/helpers/fts-availability.ts b/gitnexus/test/helpers/fts-availability.ts index fdb93314f..963b0a962 100644 --- a/gitnexus/test/helpers/fts-availability.ts +++ b/gitnexus/test/helpers/fts-availability.ts @@ -1,5 +1,5 @@ export const FTS_UNAVAILABLE_NOTE = - 'FTS extension unavailable (load-only policy; not pre-installed on this machine)'; + 'FTS extension unavailable (load-only policy; LOAD failed on this machine)'; /** * Dynamically skip an FTS-primitive test when the extension cannot load. diff --git a/gitnexus/test/helpers/test-indexed-db.ts b/gitnexus/test/helpers/test-indexed-db.ts index 1d7db5217..cf3765503 100644 --- a/gitnexus/test/helpers/test-indexed-db.ts +++ b/gitnexus/test/helpers/test-indexed-db.ts @@ -226,7 +226,7 @@ export function withTestLbugDB( ftsSkipWarned = true; console.warn( `[withTestLbugDB(${prefix})] Skipping FTS-dependent tests — the LadybugDB ` + - `FTS extension is unavailable (not pre-installed and could not be installed).`, + `FTS extension is unavailable (LOAD failed and it could not be installed).`, ); } ctx.skip(); diff --git a/gitnexus/test/integration/fts-extension-e2e.test.ts b/gitnexus/test/integration/fts-extension-e2e.test.ts new file mode 100644 index 000000000..bae89942a --- /dev/null +++ b/gitnexus/test/integration/fts-extension-e2e.test.ts @@ -0,0 +1,338 @@ +/** + * P1 Integration Tests: FTS extension lifecycle end-to-end (#2374) + * + * Everything real, nothing mocked: each test spawns the actual CLI entry as a + * child process, LadybugDB loads the actual extension shared library from + * disk, and the real out-of-process installer downloads the real extension in + * the network-gated cases. + * + * Isolation: LadybugDB resolves its extension directory from the process HOME + * (USERPROFILE on Windows), so every scenario owns a hermetic fake home with + * its own `.lbdb/extension///fts/` state — the machine's + * real ~/.lbdb is never read or written. GITNEXUS_HOME additionally isolates + * the registry (#829), following cli-e2e.test.ts conventions. + * + * Scenario matrix (the #2374 report, codified): + * - happy: valid extension pre-installed, offline (load-only) + * - unhappy: extension file present but broken — the reporter's exact state + * - unhappy: extension file missing entirely (distinguishable reason) + * - heal: FORCE INSTALL replaces a broken file over the network (auto) + */ +import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; +import { spawnSync } from 'child_process'; +import path from 'path'; +import fs from 'fs'; +import os from 'os'; +import { fileURLToPath, pathToFileURL } from 'url'; +import { createRequire } from 'module'; + +import lbug from '@ladybugdb/core'; +import { getExtensionInstallChildProcessArgs } from '../../src/core/lbug/extension-loader.js'; +import { cleanupTempDirSync } from '../helpers/test-db.js'; + +const testDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(testDir, '../..'); +const cliEntry = path.join(repoRoot, 'src/cli/index.ts'); + +const _require = createRequire(import.meta.url); +const tsxPkgDir = path.dirname(_require.resolve('tsx/package.json')); +const tsxImportUrl = pathToFileURL(path.join(tsxPkgDir, 'dist', 'loader.mjs')).href; + +/** `.lbdb/extension///fts/libfts.lbug_extension`, discovered not hardcoded. */ +let extensionRelPath: string; +/** Canonical valid extension bytes (path to a known-good file). */ +let seedExtensionFile: string | null = null; +/** Real reachability of the extension repo — gates the auto-install cases. */ +let networkAvailable = false; + +const REQUIRE_FTS = process.env.GITNEXUS_REQUIRE_FTS === '1'; +const tmpDirs: string[] = []; + +const makeTmpDir = (label: string): string => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `gn-fts-e2e-${label}-`)); + tmpDirs.push(dir); + return dir; +}; + +/** + * Locate a known-good extension file for the running LadybugDB version. + * Prefers a copy already installed under the machine's real home (pure file + * read, offline); falls back to one real out-of-process install into a probe + * home — the production installer script, not a reimplementation. + */ +const resolveSeedExtension = (): void => { + const relBase = path.join('.lbdb', 'extension', lbug.VERSION); + const realVersionDir = path.join(os.homedir(), relBase); + const platformDirs = fs.existsSync(realVersionDir) ? fs.readdirSync(realVersionDir) : []; + for (const platform of platformDirs) { + const candidate = path.join(realVersionDir, platform, 'fts', 'libfts.lbug_extension'); + if (fs.existsSync(candidate) && fs.statSync(candidate).size > 1024 * 1024) { + extensionRelPath = path.join(relBase, platform, 'fts', 'libfts.lbug_extension'); + seedExtensionFile = candidate; + return; + } + } + // No local copy — run the real installer against a hermetic probe home. + const probeHome = makeTmpDir('seed-home'); + const install = spawnSync(process.execPath, getExtensionInstallChildProcessArgs('fts'), { + encoding: 'utf8', + timeout: 120_000, + env: { ...process.env, HOME: probeHome, USERPROFILE: probeHome }, + }); + const probeVersionDir = path.join(probeHome, relBase); + const probePlatforms = fs.existsSync(probeVersionDir) ? fs.readdirSync(probeVersionDir) : []; + for (const platform of probePlatforms) { + const candidate = path.join(probeVersionDir, platform, 'fts', 'libfts.lbug_extension'); + if (install.status === 0 && fs.existsSync(candidate)) { + extensionRelPath = path.join(relBase, platform, 'fts', 'libfts.lbug_extension'); + seedExtensionFile = candidate; + networkAvailable = true; + return; + } + } +}; + +type ExtensionState = 'valid' | 'broken' | 'missing'; + +/** Create a hermetic fake home whose `.lbdb` holds the requested extension state. */ +const makeHome = (state: ExtensionState): { home: string; extensionFile: string } => { + const home = makeTmpDir(`home-${state}`); + const extensionFile = path.join(home, extensionRelPath); + fs.mkdirSync(path.dirname(extensionFile), { recursive: true }); + if (state === 'valid' && seedExtensionFile) fs.copyFileSync(seedExtensionFile, extensionFile); + if (state === 'broken') fs.writeFileSync(extensionFile, 'not a shared library'); + return { home, extensionFile }; +}; + +/** Fresh git-initialised throwaway repo with a uniquely named symbol to search for. */ +const makeFixtureRepo = (label: string): string => { + const repo = path.join(makeTmpDir(`repo-${label}`), `fts-e2e-${label}`); + fs.mkdirSync(path.join(repo, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(repo, 'src', 'greeter.ts'), + 'export function greetE2eSymbol(name: string): string {\n' + + ' return `Hello, ${name}`;\n' + + '}\n' + + "greetE2eSymbol('world');\n", + ); + const gitEnv = { + ...process.env, + GIT_AUTHOR_NAME: 'test', + GIT_AUTHOR_EMAIL: 'test@test', + GIT_COMMITTER_NAME: 'test', + GIT_COMMITTER_EMAIL: 'test@test', + }; + spawnSync('git', ['init'], { cwd: repo, stdio: 'pipe' }); + spawnSync('git', ['add', '-A'], { cwd: repo, stdio: 'pipe' }); + spawnSync('git', ['commit', '-m', 'initial'], { cwd: repo, stdio: 'pipe', env: gitEnv }); + return repo; +}; + +interface CliResult { + status: number | null; + /** stdout + stderr combined — warn lines and progress renderer interleave streams. */ + output: string; +} + +const runCli = ( + args: string[], + cwd: string, + home: string, + policy: 'load-only' | 'auto', + timeoutMs = 180_000, +): CliResult => { + const result = spawnSync(process.execPath, ['--import', tsxImportUrl, cliEntry, ...args], { + cwd, + encoding: 'utf8', + timeout: timeoutMs, + stdio: ['pipe', 'pipe', 'pipe'], + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + GITNEXUS_HOME: path.join(home, '.gitnexus'), + GITNEXUS_LANG: 'en', + GITNEXUS_LBUG_EXTENSION_INSTALL: policy, + GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS: '60000', + // Skip analyzeCommand's ensureHeap re-exec, which would drop the tsx loader. + NODE_OPTIONS: `${process.env.NODE_OPTIONS || ''} --max-old-space-size=8192`.trim(), + }, + }); + return { status: result.status, output: `${result.stdout ?? ''}\n${result.stderr ?? ''}` }; +}; + +beforeAll(() => { + resolveSeedExtension(); + if (!seedExtensionFile && REQUIRE_FTS) { + throw new Error( + 'GITNEXUS_REQUIRE_FTS=1 but no FTS extension could be located or installed for the E2E suite.', + ); + } + // The self-heal cases need the real extension repo; probe it cheaply when + // the seed came from a local copy (the installer fallback already proved it). + return (async () => { + if (seedExtensionFile && !networkAvailable) { + try { + const res = await fetch('https://extension.ladybugdb.com/', { + method: 'HEAD', + signal: AbortSignal.timeout(5000), + }); + networkAvailable = res.ok; + } catch { + networkAvailable = false; + } + } + })(); +}, 180_000); + +afterAll(() => { + for (const dir of tmpDirs) cleanupTempDirSync(dir); +}); + +// Skip everything (visibly) when no valid extension exists and the machine is +// offline — mirrors the dynamic-skip convention in test/helpers/fts-availability.ts. +beforeEach((ctx) => { + if (!seedExtensionFile) ctx.skip(); +}); + +describe('happy path — extension pre-installed, fully offline (load-only)', () => { + let home: string; + let repo: string; + + beforeAll(() => { + // The file-level beforeEach skip fires only per-test; this hook runs first, + // so guard makeHome() (which needs extensionRelPath) when there is no seed. + if (!seedExtensionFile) return; + ({ home } = makeHome('valid')); + repo = makeFixtureRepo('happy'); + }); + + it('analyze builds the index with FTS and emits no degradation warning', () => { + const result = runCli(['analyze'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('indexed successfully'); + expect(result.output).not.toContain('FTS extension unavailable'); + expect(result.output).not.toContain('search is disabled'); + }, 180_000); + + it('query finds the symbol via BM25 with no degradation warning', () => { + const result = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('greetE2eSymbol'); + expect(result.output).not.toContain('keyword search degraded'); + }, 60_000); + + it('doctor reports a live-probed available FTS and a resolved LadybugDB version', () => { + const result = runCli(['doctor'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('Full-text search: available'); + // #2374: version used to print as "unknown" on every platform. + expect(result.output).toMatch(/LadybugDB:\s*\d+\.\d+\.\d+/); + }, 60_000); + + it('analyze --repair-fts rebuilds the search indexes offline', () => { + const result = runCli(['analyze', '--repair-fts'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('FTS indexes repaired successfully'); + }, 180_000); +}); + +describe('unhappy path — extension file present but broken (the #2374 report)', () => { + let home: string; + let repo: string; + + beforeAll(() => { + // See the happy-path note: skip setup when no seed extension is available + // so the per-test beforeEach skip is reached instead of throwing here. + if (!seedExtensionFile) return; + ({ home } = makeHome('broken')); + repo = makeFixtureRepo('broken'); + }); + + it('analyze degrades gracefully and names the real LOAD failure, not "not pre-installed"', () => { + const result = runCli(['analyze'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('indexed successfully'); + expect(result.output).toContain('FTS extension unavailable'); + // The load-side ground truth must survive to the user… + expect(result.output).toContain('LOAD fts failed'); + expect(result.output).toContain('Failed to load library'); + // …and the old misdiagnosis must not: the file IS pre-installed. + expect(result.output).not.toContain('not pre-installed'); + }, 180_000); + + it('analyze --repair-fts fails loudly with the live reason and an honest remedy', () => { + const result = runCli(['analyze', '--repair-fts'], repo, home, 'load-only'); + expect(result.status).not.toBe(0); + expect(result.output).toContain('Cannot repair FTS indexes'); + expect(result.output).toContain('FTS extension failed to load'); + expect(result.output).toContain('LOAD fts failed'); + // Old message sent users to doctor "to install it"; doctor never installed. + expect(result.output).not.toContain('doctor` to install'); + expect(result.output).toContain('gitnexus doctor'); + }, 180_000); + + it('query warns with the extension-load failure, not the misleading indexes-missing message', () => { + const result = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('FTS extension failed to load'); + expect(result.output).toContain('Failed to load library'); + expect(result.output).not.toContain('FTS indexes missing'); + }, 60_000); + + it('doctor live-probes FTS as unavailable and prints the real error', () => { + const result = runCli(['doctor'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('Full-text search: unavailable'); + expect(result.output).toContain('Failed to load library'); + }, 60_000); +}); + +describe('unhappy path — extension missing entirely', () => { + it('analyze degrades with a reason that distinguishes missing from broken', () => { + const { home } = makeHome('missing'); + const repo = makeFixtureRepo('missing'); + const result = runCli(['analyze'], repo, home, 'load-only'); + expect(result.status).toBe(0); + expect(result.output).toContain('FTS extension unavailable'); + expect(result.output).toContain('has not been installed'); + expect(result.output).not.toContain('Failed to load library'); + }, 180_000); +}); + +describe('self-heal over the network — FORCE INSTALL replaces a broken file (auto)', () => { + beforeEach((ctx) => { + if (!networkAvailable) ctx.skip(); + }); + + it('the reported journey heals: degraded analyze, then repair-fts with auto re-downloads and repairs', () => { + const { home, extensionFile } = makeHome('broken'); + const repo = makeFixtureRepo('heal'); + + const degraded = runCli(['analyze'], repo, home, 'load-only'); + expect(degraded.status).toBe(0); + expect(degraded.output).toContain('FTS extension unavailable'); + + // The reporter's exact failing command — plain INSTALL used to no-op + // over the broken file and this kept failing forever. + const repair = runCli(['analyze', '--repair-fts'], repo, home, 'auto'); + expect(repair.status).toBe(0); + expect(repair.output).toContain('FTS indexes repaired successfully'); + expect(fs.statSync(extensionFile).size).toBeGreaterThan(1024 * 1024); + + const query = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); + expect(query.status).toBe(0); + expect(query.output).toContain('greetE2eSymbol'); + expect(query.output).not.toContain('keyword search degraded'); + }, 600_000); + + it('a fresh machine with no extension installs it during analyze and gets full FTS', () => { + const { home, extensionFile } = makeHome('missing'); + const repo = makeFixtureRepo('fresh'); + const result = runCli(['analyze'], repo, home, 'auto'); + expect(result.status).toBe(0); + expect(result.output).toContain('indexed successfully'); + expect(result.output).not.toContain('FTS extension unavailable'); + expect(fs.existsSync(extensionFile)).toBe(true); + }, 600_000); +}); diff --git a/gitnexus/test/unit/fts-degraded-warning.test.ts b/gitnexus/test/unit/fts-degraded-warning.test.ts new file mode 100644 index 000000000..1739498be --- /dev/null +++ b/gitnexus/test/unit/fts-degraded-warning.test.ts @@ -0,0 +1,77 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { extensionManager, resetExtensionState } from '../../src/core/lbug/extension-loader.js'; +import { ftsDegradedWarning } from '../../src/core/search/fts-indexes.js'; + +afterEach(() => { + resetExtensionState(); +}); + +describe('ftsDegradedWarning (#2374)', () => { + it('reports missing indexes when the FTS extension loaded fine', async () => { + await extensionManager.ensure(vi.fn().mockResolvedValue({}), 'fts', 'FTS', { + policy: 'load-only', + }); + + expect(ftsDegradedWarning()).toContain('FTS indexes missing'); + }); + + it('reports the live load failure with its reason when the extension cannot load', async () => { + await extensionManager.ensure( + vi.fn().mockRejectedValue(new Error('invalid ELF header.')), + 'fts', + 'FTS', + { policy: 'load-only' }, + ); + + const warning = ftsDegradedWarning(); + expect(warning).toContain('FTS extension failed to load'); + expect(warning).toContain('invalid ELF header'); + expect(warning).toContain('gitnexus doctor'); + }); + + it('falls back to the indexes-missing message when no load was attempted in this process', () => { + expect(ftsDegradedWarning()).toContain('FTS indexes missing'); + }); + + it('redacts the absolute extension path from the warning but keeps the error class', async () => { + await extensionManager.ensure( + vi + .fn() + .mockRejectedValue( + new Error( + "Failed to load library '/home/alice/.lbdb/extension/0.18.0/linux_amd64/fts/libfts.lbug_extension': invalid ELF header", + ), + ), + 'fts', + 'FTS', + { policy: 'load-only' }, + ); + + const warning = ftsDegradedWarning(); + // The username / home dir / absolute path must not leak to HTTP or MCP clients. + expect(warning).not.toMatch(/\/home\/|\/Users\/|C:\\Users\\/); + // …but the actionable error class survives redaction. + expect(warning).toContain('FTS extension failed to load'); + expect(warning).toContain('Failed to load library'); + expect(warning).toContain('invalid ELF header'); + }); + + it('redacts Windows-style extension paths too', async () => { + await extensionManager.ensure( + vi + .fn() + .mockRejectedValue( + new Error( + "Failed to load library 'C:\\Users\\bob\\.lbdb\\extension\\0.18.0\\win_amd64\\fts\\libfts.lbug_extension': not a valid Win32 application", + ), + ), + 'fts', + 'FTS', + { policy: 'load-only' }, + ); + + const warning = ftsDegradedWarning(); + expect(warning).not.toMatch(/C:\\Users\\/); + expect(warning).toContain('not a valid Win32 application'); + }); +}); diff --git a/gitnexus/test/unit/install-duckdb-extension.test.ts b/gitnexus/test/unit/install-duckdb-extension.test.ts new file mode 100644 index 000000000..8b9e6b935 --- /dev/null +++ b/gitnexus/test/unit/install-duckdb-extension.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect, vi } from 'vitest'; +import { + chooseInstallVerb, + installDuckDbExtension, +} from '../../scripts/install-duckdb-extension.mjs'; + +/** + * Offline, network-free regression guard for the install-verb decision (#2374, + * PR #2375). A revert to unconditional INSTALL or unconditional FORCE INSTALL + * fails here on any runner — the self-heal e2e is network-gated and can silently + * skip, so this deterministic unit test is the real guard. + */ + +interface RecordingConn { + query: ReturnType; +} + +/** Injectable connection factory that records the SQL instead of touching lbug. */ +const recordingConnect = (): { conn: RecordingConn; dispose: () => Promise } => { + const conn: RecordingConn = { query: vi.fn(async () => undefined) }; + return { conn, dispose: async () => undefined }; +}; + +describe('chooseInstallVerb (#2374)', () => { + it.each([ + [ + 'IO exception: Failed to load library: /x/libfts.lbug_extension. invalid ELF header', + 'FORCE INSTALL', + ], + ['Failed to load library: /x/libfts.lbug_extension. file too short', 'FORCE INSTALL'], + ['Failed to load library: /x/libfts.dll: not a valid Win32 application', 'FORCE INSTALL'], + [ + 'Binder exception: Extension: fts is an official extension and has not been installed.', + 'INSTALL', + ], + ['Failed to load library: /x/libfts: libfoo.so: cannot open shared object file', 'INSTALL'], + ['some unrelated error', 'INSTALL'], + ])('maps %j to %s', (loadError, expected) => { + expect(chooseInstallVerb(loadError)).toBe(expected); + }); + + it('defaults to plain INSTALL when no load error is provided', () => { + expect(chooseInstallVerb(undefined)).toBe('INSTALL'); + }); +}); + +describe('installDuckDbExtension issues the chosen SQL (#2374)', () => { + it('issues FORCE INSTALL when the load error indicates file corruption', async () => { + const factory = recordingConnect(); + await installDuckDbExtension('fts', { + loadError: 'Failed to load library: /x/libfts.lbug_extension. invalid ELF header', + connect: () => factory, + }); + expect(factory.conn.query).toHaveBeenCalledWith('FORCE INSTALL fts'); + }); + + it('issues plain INSTALL for a missing extension file', async () => { + const factory = recordingConnect(); + await installDuckDbExtension('fts', { + loadError: 'Extension: fts is an official extension and has not been installed.', + connect: () => factory, + }); + expect(factory.conn.query).toHaveBeenCalledWith('INSTALL fts'); + }); + + it('issues LOAD EXTENSION and never an install verb in verifyOnly mode', async () => { + const factory = recordingConnect(); + await installDuckDbExtension('fts', { verifyOnly: true, connect: () => factory }); + expect(factory.conn.query).toHaveBeenCalledWith('LOAD EXTENSION fts'); + expect(factory.conn.query.mock.calls.some(([sql]) => String(sql).includes('INSTALL'))).toBe( + false, + ); + }); + + it('rejects an invalid extension name before opening any connection', async () => { + const connect = vi.fn(); + await expect(installDuckDbExtension('fts; DROP', { connect })).rejects.toThrow( + /Invalid DuckDB extension name/, + ); + expect(connect).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/lbug-extension-loader.test.ts b/gitnexus/test/unit/lbug-extension-loader.test.ts index 351a11608..8a0a05716 100644 --- a/gitnexus/test/unit/lbug-extension-loader.test.ts +++ b/gitnexus/test/unit/lbug-extension-loader.test.ts @@ -61,7 +61,9 @@ describe('ExtensionManager — install policies', () => { true, ); - expect(installExtension).toHaveBeenCalledWith('fts', 1234); + // The LOAD failure reason is threaded to the installer so it can pick + // INSTALL vs FORCE INSTALL from the error class (#2374, PR #2375). + expect(installExtension).toHaveBeenCalledWith('fts', 1234, 'Extension "fts" not found'); expect(query.mock.calls.map(([sql]) => sql)).toEqual([ 'LOAD EXTENSION fts', 'LOAD EXTENSION fts', @@ -132,6 +134,68 @@ describe('ExtensionManager — install policies', () => { }); }); +describe('ExtensionManager — reason strings carry the real LOAD error (#2374)', () => { + it('load-only failure reason includes the underlying LadybugDB error, collapsed to one line', async () => { + const warn = vi.fn(); + const manager = new ExtensionManager({ policy: 'load-only', warn }); + const query = vi + .fn() + .mockRejectedValue( + new Error( + 'IO exception: Failed to load library: /x/libfts.lbug_extension.\ninvalid ELF header', + ), + ); + + await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + + expect(manager.getCapabilities()).toEqual([ + { + name: 'fts', + loaded: false, + reason: expect.stringContaining( + 'LOAD fts failed: IO exception: Failed to load library: /x/libfts.lbug_extension. invalid ELF header', + ), + }, + ]); + expect(warn).toHaveBeenCalledWith(expect.stringContaining('invalid ELF header')); + }); + + it('failed-install reason includes both the install message and the original LOAD error', async () => { + const installExtension = vi.fn().mockResolvedValue(failedInstall); + const manager = new ExtensionManager({ policy: 'auto', installExtension, warn: noopWarn }); + const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); + + await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + + expect(manager.getCapabilities()).toEqual([ + { + name: 'fts', + loaded: false, + reason: 'install failed; LOAD fts had failed: Extension "fts" not found', + }, + ]); + }); + + it('post-install LOAD failure reason includes the retry error', async () => { + const installExtension = vi.fn().mockResolvedValue(okInstall); + const manager = new ExtensionManager({ policy: 'auto', installExtension, warn: noopWarn }); + const query = vi + .fn() + .mockRejectedValue(new Error('version mismatch: extension built for 0.17.0')); + + await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + + expect(manager.getCapabilities()).toEqual([ + { + name: 'fts', + loaded: false, + reason: + 'LOAD fts failed after successful INSTALL: version mismatch: extension built for 0.17.0', + }, + ]); + }); +}); + describe('ExtensionManager — caching', () => { it('caches install attempt outcome to avoid retrying within the same process', async () => { const installExtension = vi.fn().mockResolvedValue(timedOutInstall); diff --git a/gitnexus/test/unit/native-check-probe.test.ts b/gitnexus/test/unit/native-check-probe.test.ts new file mode 100644 index 000000000..3593977ab --- /dev/null +++ b/gitnexus/test/unit/native-check-probe.test.ts @@ -0,0 +1,92 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +/** + * Unit coverage for probeFtsExtensionLoad (#2374, PR #2375): the doctor FTS + * probe's outcomes without the real native module or network. @ladybugdb/core + * is mocked so query behavior (resolve / reject / never-settle) is controlled + * per test, and the timeout is exercised with a tiny injected budget. + */ + +const h = vi.hoisted(() => ({ + query: vi.fn<(sql: string) => Promise>(), + connCtor: vi.fn<() => void>(), + connClose: vi.fn<() => Promise>(async () => undefined), + dbClose: vi.fn<() => Promise>(async () => undefined), +})); + +vi.mock('@ladybugdb/core', () => { + class Database { + constructor(_path: string) {} + close = h.dbClose; + } + class Connection { + constructor(_db: unknown) { + h.connCtor(); + } + query = h.query; + close = h.connClose; + } + return { default: { Database, Connection } }; +}); + +import { probeFtsExtensionLoad } from '../../src/core/lbug/native-check.js'; + +const closeable = () => ({ close: vi.fn() }); + +beforeEach(() => { + h.query.mockReset(); + h.connCtor.mockReset(); + h.connClose.mockClear(); + h.dbClose.mockClear(); +}); + +describe('probeFtsExtensionLoad (#2374)', () => { + it('reports loaded and closes every result when LOAD succeeds (array result)', async () => { + const results = [closeable(), closeable()]; + h.query.mockResolvedValue(results); + + await expect(probeFtsExtensionLoad()).resolves.toEqual({ loaded: true }); + + expect(results.map((r) => r.close.mock.calls.length)).toEqual([1, 1]); + expect(h.connClose).toHaveBeenCalled(); + expect(h.dbClose).toHaveBeenCalled(); + }); + + it('reports loaded for a single (non-array) result', async () => { + h.query.mockResolvedValue(closeable()); + await expect(probeFtsExtensionLoad()).resolves.toEqual({ loaded: true }); + }); + + it('reports the collapsed reason when LOAD fails', async () => { + h.query.mockRejectedValue(new Error('IO exception:\n invalid ELF header')); + await expect(probeFtsExtensionLoad()).resolves.toMatchObject({ + loaded: false, + reason: 'IO exception: invalid ELF header', + }); + }); + + it('times out instead of hanging when the native call never settles', async () => { + h.query.mockReturnValue(new Promise(() => undefined)); + await expect(probeFtsExtensionLoad(20)).resolves.toMatchObject({ + loaded: false, + reason: expect.stringContaining('timed out'), + }); + }); + + it('still closes the db when the Connection ctor throws', async () => { + h.connCtor.mockImplementation(() => { + throw new Error('connection ctor failed'); + }); + await expect(probeFtsExtensionLoad()).resolves.toMatchObject({ loaded: false }); + expect(h.dbClose).toHaveBeenCalled(); + }); + + it('reports loaded even when a result close() throws', async () => { + h.query.mockResolvedValue({ + close: () => { + throw new Error('close boom'); + }, + }); + await expect(probeFtsExtensionLoad()).resolves.toEqual({ loaded: true }); + }); +}); diff --git a/gitnexus/test/unit/platform-capabilities.test.ts b/gitnexus/test/unit/platform-capabilities.test.ts index 44d18f14a..3b58d9d16 100644 --- a/gitnexus/test/unit/platform-capabilities.test.ts +++ b/gitnexus/test/unit/platform-capabilities.test.ts @@ -1,5 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { isVectorExtensionSupportedByPlatform } from '../../src/core/platform/capabilities.js'; +import { + getRuntimeFingerprint, + isVectorExtensionSupportedByPlatform, +} from '../../src/core/platform/capabilities.js'; describe('platform capabilities', () => { it('keeps Ladybug VECTOR disabled by default on Windows', () => { @@ -10,4 +13,8 @@ describe('platform capabilities', () => { expect(isVectorExtensionSupportedByPlatform('linux')).toBe(true); expect(isVectorExtensionSupportedByPlatform('darwin')).toBe(true); }); + + it('resolves the LadybugDB version even though @ladybugdb/core exports omit ./package.json (#2374)', () => { + expect(getRuntimeFingerprint().ladybugdb).toMatch(/^\d+\.\d+\.\d+/); + }); }); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index ce4493308..8a19ef951 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -20,6 +20,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { vi.doUnmock('../../src/core/search/fts-indexes.js'); vi.doUnmock('../../src/core/ingestion/pipeline.js'); vi.doUnmock('../../src/storage/repo-manager.js'); + vi.doUnmock('../../src/core/lbug/extension-loader.js'); vi.resetModules(); vi.clearAllMocks(); vi.unstubAllEnvs(); @@ -290,6 +291,15 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { createSearchFTSIndexes, verifySearchFTSIndexes: vi.fn(async () => []), })); + // Populate the live capability so the repair error actually interpolates the + // real LOAD reason (#2374). Without this the branch is vacuous — the reason + // is undefined and the assertion passes whether or not interpolation fires. + vi.doMock('../../src/core/lbug/extension-loader.js', async (importActual) => ({ + ...(await importActual()), + getExtensionCapabilities: () => [ + { name: 'fts', loaded: false, reason: 'LOAD fts failed: invalid ELF header' }, + ], + })); const tmpRepo = await createTempDir('gitnexus-run-analyze-repair-fts-unavailable-'); try { @@ -307,7 +317,11 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { await expect( runFullAnalysis(tmpRepo.dbPath, { repairFts: true }, { onProgress: () => {} }), - ).rejects.toThrow(/FTS extension is unavailable[\s\S]*gitnexus doctor/i); + // The specific reason must appear between the headline and the remedy — + // proving the interpolation fired, not just that the base message exists. + ).rejects.toThrow( + /FTS extension failed to load[\s\S]*invalid ELF header[\s\S]*gitnexus doctor/i, + ); // The guard fires before drop-then-create, so no index is dropped. expect(createSearchFTSIndexes).not.toHaveBeenCalled(); } finally { diff --git a/gitnexus/vitest.config.ts b/gitnexus/vitest.config.ts index 54421aba0..7702c6f2f 100644 --- a/gitnexus/vitest.config.ts +++ b/gitnexus/vitest.config.ts @@ -118,6 +118,7 @@ export default defineConfig({ 'test/integration/group/manifest-resolve-symbol-2325.test.ts', 'test/integration/group/http-route-resolve-symbol.test.ts', 'test/integration/skills-e2e.test.ts', + 'test/integration/fts-extension-e2e.test.ts', 'test/integration/fts-stemmer-sweep.test.ts', 'test/integration/lbug-multiwriter-deadlock.test.ts', ], @@ -127,7 +128,12 @@ export default defineConfig({ extends: true, test: { name: 'cli-e2e', - include: ['test/integration/skills-e2e.test.ts'], + include: [ + 'test/integration/skills-e2e.test.ts', + // Spawns the real CLI per test; runs sequentially (fileParallelism: + // false) so it doesn't aggravate the under-load timeout-flake class. + 'test/integration/fts-extension-e2e.test.ts', + ], fileParallelism: false, sequence: { groupOrder: 2 }, },