diff --git a/.gitattributes b/.gitattributes index eeb1a0976..15fad158a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -15,6 +15,7 @@ *.so binary *.dll binary *.dylib binary +*.lbug_extension binary # TypeScript sources are always text for diff purposes. Git's binary # heuristic fires when EITHER blob in a pair carries a NUL, so a source diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4a25e682d..535204a38 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -89,6 +89,10 @@ updates: # tree-sitter-cli follows the runtime's version cadence. Bump when # regenerating vendor/tree-sitter-proto/src/parser.c, not on a schedule. - dependency-name: tree-sitter-cli + # Pin @ladybugdb/core so a daily bump cannot ship a skewed FTS artifact. + # The extension version is a separate upstream constant, not derivable + # from the core version (see vendor/lbug-fts/manifest.json). + - dependency-name: '@ladybugdb/core' # gitnexus-web (thin frontend client). - package-ecosystem: npm diff --git a/.github/scripts/fetch-lbug-fts-artifacts.mjs b/.github/scripts/fetch-lbug-fts-artifacts.mjs new file mode 100644 index 000000000..8ad916ed3 --- /dev/null +++ b/.github/scripts/fetch-lbug-fts-artifacts.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node +/** + * Fetch Ladybug FTS artifacts into gitnexus/vendor/lbug-fts/prebuilds/. + * + * Lives outside the published package (`files` includes `scripts` wholesale). + * Reads versions, filename, and tuple→upstream-platform mapping from + * vendor/lbug-fts/manifest.json so the gate and runtime cannot drift. + * + * Usage: node .github/scripts/fetch-lbug-fts-artifacts.mjs + */ +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..'); +const VENDOR = path.join(REPO_ROOT, 'gitnexus', 'vendor', 'lbug-fts'); +const PREBUILDS = path.join(VENDOR, 'prebuilds'); +const MANIFEST_PATH = path.join(VENDOR, 'manifest.json'); + +/** Only the Ladybug official extension host — never a manifest-supplied origin. */ +const OFFICIAL_REPO = 'https://extension.ladybugdb.com/'; +const EXACT_VERSION = /^\d+\.\d+\.\d+$/; +const SAFE_UPSTREAM = /^(linux_amd64|linux_arm64|osx_amd64|osx_arm64|win_amd64)$/; + +/** + * Build the official artifact URL from allowlisted fields only. + * `officialRepo` in the manifest must match {@link OFFICIAL_REPO}; the + * origin itself is a constant so an edited manifest cannot redirect the fetch. + */ +export function officialArtifactUrl(manifest, upstreamPlatform) { + const officialRepo = String(manifest?.officialRepo ?? ''); + if (officialRepo !== OFFICIAL_REPO) { + throw new Error(`refusing unofficial FTS repo: '${officialRepo}'`); + } + const version = String(manifest?.extensionVersion ?? ''); + if (!EXACT_VERSION.test(version)) { + throw new Error(`unsafe extensionVersion: '${version}'`); + } + if (!SAFE_UPSTREAM.test(String(upstreamPlatform ?? ''))) { + throw new Error(`unsafe upstream platform: '${upstreamPlatform}'`); + } + const filename = String(manifest?.filename ?? ''); + if (!SAFE_FILENAME.test(filename)) { + throw new Error(`unsafe FTS artifact filename: '${filename}'`); + } + return `${OFFICIAL_REPO}v${version}/${upstreamPlatform}/fts/${filename}`; +} + +const sha256 = (buf) => createHash('sha256').update(buf).digest('hex'); + +const readExistingHash = (filePath) => { + if (!existsSync(filePath)) return null; + return sha256(readFileSync(filePath)); +}; + +export const supportedTuples = (manifest) => manifest.tuples.map((entry) => entry.tuple); + +const SAFE_TUPLE = /^(darwin|linux|win32)-(x64|arm64)$/; +const SAFE_FILENAME = /^[\w.-]+\.lbug_extension$/; + +/** Relative-path containment — not a prefix match (rejects `prebuilds-evil`). */ +const isPathInsideRoot = (root, candidate) => { + const relative = path.relative(root, candidate); + if (path.isAbsolute(relative)) return false; + return relative !== '' && !relative.startsWith(`..${path.sep}`) && relative !== '..'; +}; + +export function assertSafeArtifactDest({ prebuildsDir, tuple, filename }) { + if (!SAFE_TUPLE.test(String(tuple ?? ''))) { + throw new Error( + `unsafe FTS artifact tuple: '${tuple}' (expected (darwin|linux|win32)-(x64|arm64))`, + ); + } + if (!SAFE_FILENAME.test(String(filename ?? ''))) { + throw new Error(`unsafe FTS artifact filename: '${filename}' (expected *.lbug_extension)`); + } + const dest = path.join(prebuildsDir, tuple, filename); + if (!isPathInsideRoot(prebuildsDir, dest)) { + throw new Error(`FTS artifact dest is not inside prebuildsDir: ${dest}`); + } + return dest; +} + +async function fetchBuffer(url) { + // codeql[js/request-forgery] — origin is OFFICIAL_REPO; path segments are allowlisted. + // lgtm[js/request-forgery] + // codeql[js/file-access-to-http] — versions/platforms are regex-pinned, not raw file bytes. + const res = await fetch(url, { signal: AbortSignal.timeout(120_000) }); + if (!res.ok) { + throw new Error(`GET ${url} → ${res.status} ${res.statusText}`); + } + return Buffer.from(await res.arrayBuffer()); +} + +const writeAllowlistedArtifact = (prebuildsDir, dest, buf) => { + if (!isPathInsideRoot(prebuildsDir, dest)) { + throw new Error(`FTS artifact dest is not inside prebuildsDir: ${dest}`); + } + // codeql[js/http-to-file-access] — dest is assertSafeArtifactDest + containment-checked. + writeFileSync(dest, buf); +}; + +export async function refreshArtifacts({ + manifest = JSON.parse(readFileSync(MANIFEST_PATH, 'utf8')), + prebuildsDir = PREBUILDS, + download = fetchBuffer, +} = {}) { + mkdirSync(prebuildsDir, { recursive: true }); + const lines = []; + for (const { tuple, upstreamPlatform } of manifest.tuples) { + const dest = assertSafeArtifactDest({ + prebuildsDir, + tuple, + filename: manifest.filename, + }); + mkdirSync(path.dirname(dest), { recursive: true }); + const url = officialArtifactUrl(manifest, upstreamPlatform); + const previousHash = readExistingHash(dest); + const previousSize = previousHash ? readFileSync(dest).byteLength : 0; + const buf = await download(url); + const nextHash = sha256(buf); + writeAllowlistedArtifact(prebuildsDir, dest, buf); + const changed = previousHash !== nextHash; + console.log( + changed + ? `[fts-fetch] ${tuple}: ${previousHash ?? '(new)'} (${previousSize} B) → ${nextHash} (${buf.byteLength} B)` + : `[fts-fetch] ${tuple}: unchanged ${nextHash} (${buf.byteLength} B)`, + ); + lines.push(`${nextHash} ./${tuple}/${manifest.filename}`); + } + lines.sort(); + writeFileSync(path.join(prebuildsDir, 'SHA256SUMS'), `${lines.join('\n')}\n`); + return lines; +} + +const invokedDirectly = + process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url; +if (invokedDirectly) { + refreshArtifacts().catch((err) => { + console.error(`[fts-fetch] ${err instanceof Error ? err.message : err}`); + process.exit(1); + }); +} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b3103030e..c7c85ba03 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -77,6 +77,14 @@ jobs: # GitHub PR CodeQL gate, so this file is excluded to avoid # re-filing js/regex-injection on every push of the same line. - 'gitnexus/src/server/grep-params.ts' + # Tests construct tmpdir fixtures and pass them into production + # read-only probes (openSync(..., 'r')). CodeQL models that as + # js/insecure-temporary-file even though nothing is created. + - '**/test/**' + # CI vendor fetch: origin is the official Ladybug repo; dest is + # regex-pinned and containment-checked. Inline suppressions do + # not clear the PR CodeQL gate (same as grep-params.ts). + - '.github/scripts/fetch-lbug-fts-artifacts.mjs' - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9 diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 90271c2bd..49123fda0 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -18,7 +18,7 @@ "@tailwindcss/vite": "^4.3.3", "axios": "^1.20.0", "d3": "^7.9.0", - "dompurify": "^3.4.13", + "dompurify": "^3.4.15", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -28,7 +28,7 @@ "graphology-utils": "^2.3.0", "i18next": "^26.3.6", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.5.4", + "langchain": "^1.5.11", "lru-cache": "^11.5.2", "lucide-react": "^1.31.0", "mermaid": "^11.17.2", @@ -39,7 +39,7 @@ "react-i18next": "^17.0.13", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", - "react-zoom-pan-pinch": "^4.0.3", + "react-zoom-pan-pinch": "^4.2.0", "remark-gfm": "^4.0.1", "sigma": "^3.0.3", "tailwindcss": "^4.3.3", @@ -60,7 +60,7 @@ "@vercel/node": "^5.10.2", "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", - "jsdom": "^29.1.1", + "jsdom": "^30.0.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", "vite": "^8.1.5", @@ -119,56 +119,38 @@ } }, "node_modules/@asamuzakjp/css-color": { - "version": "5.1.11", - "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz", - "integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==", + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-6.0.7.tgz", + "integrity": "sha512-vC/bk1Lz7Tn/EfU9/apOTBk80/8dyGyWMowPoV1tJ52muDGsDqt2HPT2klrFUiY60MQmQv9q8yIht15JnBgDGw==", "dev": true, "license": "MIT", "dependencies": { - "@asamuzakjp/generational-cache": "^1.0.1", - "@csstools/css-calc": "^3.2.0", - "@csstools/css-color-parser": "^4.1.0", + "@csstools/css-calc": "^3.3.0", + "@csstools/css-color-parser": "^4.1.10", "@csstools/css-parser-algorithms": "^4.0.0", - "@csstools/css-tokenizer": "^4.0.0" + "@csstools/css-tokenizer": "^4.0.0", + "lru-cache": "^11.5.2" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": "^22.13.0 || >=24.0.0" } }, "node_modules/@asamuzakjp/dom-selector": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz", - "integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==", + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-8.3.2.tgz", + "integrity": "sha512-93Z1N+BQNXysodoicpOIyNh2drHfz/CTf9nnT0FEx72GJcIiwgydD7tGAr78j41LsYn3hlRn+LdGPuBLn1Bl8Q==", "dev": true, "license": "MIT", "dependencies": { - "@asamuzakjp/generational-cache": "^1.0.1", - "@asamuzakjp/nwsapi": "^2.3.9", "bidi-js": "^1.0.3", "css-tree": "^3.2.1", - "is-potential-custom-element-name": "^1.0.1" + "is-potential-custom-element-name": "^1.0.1", + "lru-cache": "^11.5.2" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": "^22.13.0 || >=24.0.0" } }, - "node_modules/@asamuzakjp/generational-cache": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz", - "integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" - } - }, - "node_modules/@asamuzakjp/nwsapi": { - "version": "2.3.9", - "resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz", - "integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==", - "dev": true, - "license": "MIT" - }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -320,9 +302,9 @@ "license": "Apache-2.0" }, "node_modules/@csstools/color-helpers": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz", - "integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==", + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.1.tgz", + "integrity": "sha512-gLNsunvwf3mCi5u5o46/Z/JcJMnhbHSaZ69rkgPzNM3J4s8hWwpPUQB6/tt0EDFyCiWzxANlx+2LJwpYj4zS1w==", "dev": true, "funding": [ { @@ -340,9 +322,9 @@ } }, "node_modules/@csstools/css-calc": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz", - "integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.3.0.tgz", + "integrity": "sha512-c5ihYsPkdG6JCkU2zTMm4+k6r7RXuGxtWYhu5DHMIiF1FHzrfmHL5so11AoFpUv/tu61xfcmT4AmKoFfMPoqdQ==", "dev": true, "funding": [ { @@ -364,9 +346,9 @@ } }, "node_modules/@csstools/css-color-parser": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz", - "integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==", + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.2.tgz", + "integrity": "sha512-3QKjR/vxyjcSXBLgb6lP0S3MGdvwbmqSsvLPbYdVORqPDc8FX1HAJ0Spk38bxaRXgvENTA47tlhhbb5Z2e8hEg==", "dev": true, "funding": [ { @@ -380,8 +362,8 @@ ], "license": "MIT", "dependencies": { - "@csstools/color-helpers": "^6.0.2", - "@csstools/css-calc": "^3.2.0" + "@csstools/color-helpers": "^6.1.1", + "@csstools/css-calc": "^3.3.0" }, "engines": { "node": ">=20.19.0" @@ -415,9 +397,9 @@ } }, "node_modules/@csstools/css-syntax-patches-for-csstree": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz", - "integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==", + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.12.tgz", + "integrity": "sha512-3vLQK+dXxhBMR2Wx99PTCifE+vHtW2ndZWyla8yK813ev6oGhyn8Lja8jCyGAWTJ+LEYZK7EVtJxrDj8ztevJw==", "dev": true, "funding": [ { @@ -960,9 +942,9 @@ } }, "node_modules/@exodus/bytes": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz", - "integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==", + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz", + "integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==", "dev": true, "license": "MIT", "engines": { @@ -1132,9 +1114,9 @@ } }, "node_modules/@langchain/core": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.9.tgz", - "integrity": "sha512-conzSEj9Zu1AyXJLXsSbgrtxtxinmI1yGqQ5CIJZSoV5rvv+yvQE/vgBnoySpBQ/bl3YPgj2FL/gbDjWykLSfg==", + "version": "1.2.10", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.10.tgz", + "integrity": "sha512-skr9zkyidZSzhoqhWINdULRstPT5aMD+w3tWtE3puxffWH7wxAp8tx1gMqadjKZwlgo6exlK0nHncFJv6m+3Cg==", "license": "MIT", "dependencies": { "@cfworker/json-schema": "^4.0.2", @@ -3125,9 +3107,9 @@ "license": "MIT" }, "node_modules/bidi-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", - "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.1.0.tgz", + "integrity": "sha512-fX1Onk0tdVPC7obPWB5EbJ1z7NVhLq4m2xZLq2YXBkxzMXIGRpNMU88n0EPgWseKl12J7zXs7qrDxPK4sRs2fg==", "dev": true, "license": "MIT", "dependencies": { @@ -4026,9 +4008,9 @@ "peer": true }, "node_modules/dompurify": { - "version": "3.4.13", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz", - "integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==", + "version": "3.4.15", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz", + "integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -5109,39 +5091,39 @@ "peer": true }, "node_modules/jsdom": { - "version": "29.1.1", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz", - "integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==", + "version": "30.0.1", + "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.0.1.tgz", + "integrity": "sha512-52v7mUVUfNQVYYqE1lcdaymWL0njO7lTLUog6ZvW2U5KsbiLk/GnZlVJ+qx0xfNJZ6Gn+KSpPNE52vurbxZwrA==", "dev": true, "license": "MIT", "dependencies": { - "@asamuzakjp/css-color": "^5.1.11", - "@asamuzakjp/dom-selector": "^7.1.1", + "@asamuzakjp/css-color": "^6.0.5", + "@asamuzakjp/dom-selector": "^8.3.0", "@bramus/specificity": "^2.4.2", - "@csstools/css-syntax-patches-for-csstree": "^1.1.3", - "@exodus/bytes": "^1.15.0", + "@csstools/css-syntax-patches-for-csstree": "^1.1.7", + "@exodus/bytes": "^1.15.1", "css-tree": "^3.2.1", "data-urls": "^7.0.0", "decimal.js": "^10.6.0", "html-encoding-sniffer": "^6.0.0", "is-potential-custom-element-name": "^1.0.1", - "lru-cache": "^11.3.5", + "lru-cache": "^11.5.2", "parse5": "^8.0.1", "saxes": "^6.0.0", "symbol-tree": "^3.2.4", - "tough-cookie": "^6.0.1", - "undici": "^7.25.0", + "tough-cookie": "^6.0.2", + "undici": "^8.9.0", "w3c-xmlserializer": "^5.0.0", "webidl-conversions": "^8.0.1", "whatwg-mimetype": "^5.0.0", - "whatwg-url": "^16.0.1", + "whatwg-url": "^17.1.0", "xml-name-validator": "^5.0.0" }, "engines": { - "node": "^20.19.0 || ^22.13.0 || >=24.0.0" + "node": "^22.22.2 || ^24.15.0 || >=26.0.0" }, "peerDependencies": { - "canvas": "^3.0.0" + "canvas": "^3.2.3" }, "peerDependenciesMeta": { "canvas": { @@ -5163,13 +5145,13 @@ } }, "node_modules/jsdom/node_modules/undici": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", - "integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==", + "version": "8.10.2", + "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", + "integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==", "dev": true, "license": "MIT", "engines": { - "node": ">=20.18.1" + "node": ">=22.19.0" } }, "node_modules/jsdom/node_modules/webidl-conversions": { @@ -5183,18 +5165,18 @@ } }, "node_modules/jsdom/node_modules/whatwg-url": { - "version": "16.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz", - "integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==", + "version": "17.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-17.1.0.tgz", + "integrity": "sha512-3GeworPmc2ZfEEHP7lEbUfBX/L75wdEsi0rLNhXcXxnoN5jyq0SL5gCy06SGW2cyTIZdTvWIDQNQoza++vKeaw==", "dev": true, "license": "MIT", "dependencies": { - "@exodus/bytes": "^1.11.0", + "@exodus/bytes": "^1.15.1", "tr46": "^6.0.0", "webidl-conversions": "^8.0.1" }, "engines": { - "node": "^20.19.0 || ^22.12.0 || >=24.0.0" + "node": "^22.14.0 || >=24.0.0" } }, "node_modules/json-schema-to-ts": { @@ -5248,13 +5230,13 @@ "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" }, "node_modules/langchain": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.4.tgz", - "integrity": "sha512-9Rq6Ih77UOy3+7bCbxMJS16MRUJwfxuljU0yW2KOXDgEKWE8cmaZJE6ONEy4HdWGMsbj3qyv3vD5UvV7fvNksg==", + "version": "1.5.11", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.11.tgz", + "integrity": "sha512-6Sx9N5ylAJ11WrP1QnJLSIo75UABZbshzTJgG28H4mXuGcDk+w+7ZaNLkmGIh9sy/3PZcYS8UrI2rvH6A8NYIg==", "license": "MIT", "dependencies": { - "@langchain/langgraph": "^1.4.7", - "@langchain/langgraph-checkpoint": "^1.1.3", + "@langchain/langgraph": "^1.4.13", + "@langchain/langgraph-checkpoint": "^1.1.5", "langsmith": ">=0.5.0 <1.0.0", "zod": "^3.25.76 || ^4" }, @@ -5262,7 +5244,7 @@ "node": ">=20" }, "peerDependencies": { - "@langchain/core": "^1.2.3" + "@langchain/core": "^1.2.10" } }, "node_modules/langsmith": { @@ -7382,9 +7364,9 @@ } }, "node_modules/react-zoom-pan-pinch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/react-zoom-pan-pinch/-/react-zoom-pan-pinch-4.0.3.tgz", - "integrity": "sha512-N2Hi6L78fFmhRra+ORpFSW7WST5x6kxpOPplIvtB0b7b+U2anpo1z1wLgaWRPS2kUSqcraRG+JgBCIlDJnqqAg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/react-zoom-pan-pinch/-/react-zoom-pan-pinch-4.2.0.tgz", + "integrity": "sha512-QSw1dvr6QGv5zHCUY7SEIY7EWRDbx1y6plylrWf2Nko9KOMW1MEoPC6oJ+Y/qOSeQjhvbkMV0m5bFF7pBxCH4A==", "license": "MIT", "engines": { "node": ">=8", @@ -7906,22 +7888,22 @@ } }, "node_modules/tldts": { - "version": "7.0.25", - "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.25.tgz", - "integrity": "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w==", + "version": "7.4.11", + "resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.11.tgz", + "integrity": "sha512-aBiNayCfTQxuIJBm06M+xR14cYaYlDlSXZbgsnKzKNxDKUVq7KFwTjwBSsb7m9Y5xO8WfPnBc63WaYFMTGlvqw==", "dev": true, "license": "MIT", "dependencies": { - "tldts-core": "^7.0.25" + "tldts-core": "^7.4.11" }, "bin": { "tldts": "bin/cli.js" } }, "node_modules/tldts-core": { - "version": "7.0.25", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.25.tgz", - "integrity": "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw==", + "version": "7.4.11", + "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.11.tgz", + "integrity": "sha512-CW3WN2rIIE/Of21mulhgnGOwoDyEFNygyIBOONSdyAuSATgMMUCpLeUlB+E8sAwA5xRV9hYPl+kyZ9citHCaKg==", "dev": true, "license": "MIT" }, @@ -7939,9 +7921,9 @@ } }, "node_modules/tough-cookie": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz", - "integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz", + "integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==", "dev": true, "license": "BSD-3-Clause", "dependencies": { diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index fee794c47..c1d4d6b49 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -28,7 +28,7 @@ "@tailwindcss/vite": "^4.3.3", "axios": "^1.20.0", "d3": "^7.9.0", - "dompurify": "^3.4.13", + "dompurify": "^3.4.15", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -38,7 +38,7 @@ "graphology-utils": "^2.3.0", "i18next": "^26.3.6", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.5.4", + "langchain": "^1.5.11", "lru-cache": "^11.5.2", "lucide-react": "^1.31.0", "mermaid": "^11.17.2", @@ -49,7 +49,7 @@ "react-i18next": "^17.0.13", "react-markdown": "^10.1.0", "react-syntax-highlighter": "^16.1.1", - "react-zoom-pan-pinch": "^4.0.3", + "react-zoom-pan-pinch": "^4.2.0", "remark-gfm": "^4.0.1", "sigma": "^3.0.3", "tailwindcss": "^4.3.3", @@ -70,7 +70,7 @@ "@vercel/node": "^5.10.2", "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", - "jsdom": "^29.1.1", + "jsdom": "^30.0.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", "vite": "^8.1.5", diff --git a/gitnexus/README.md b/gitnexus/README.md index 95b53cdcf..b09e846c4 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -518,11 +518,11 @@ truthy, when the install is not an npm global/local install (npx cache, dev checkout, Docker image — the Docker CLI image sets the opt-out itself), or when opted out: -| Variable | Effect | -| --- | --- | -| `GITNEXUS_NO_UPDATE_NOTIFIER` | Truthy (`1`, `true`, …) disables the update check on every surface. | -| `NO_UPDATE_NOTIFIER` | Cross-tool convention; honored the same way. | -| `npm_config_registry` | The check reads the `latest` dist-tag from this registry instead of `https://registry.npmjs.org`. Credentials are never sent, and registries that require authentication are not supported (the check silently skips). | +| Variable | Effect | +| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_NO_UPDATE_NOTIFIER` | Truthy (`1`, `true`, …) disables the update check on every surface. | +| `NO_UPDATE_NOTIFIER` | Cross-tool convention; honored the same way. | +| `npm_config_registry` | The check reads the `latest` dist-tag from this registry instead of `https://registry.npmjs.org`. Credentials are never sent, and registries that require authentication are not supported (the check silently skips). | Eval harnesses running a global install can set `GITNEXUS_NO_UPDATE_NOTIFIER` for a quiet registry. @@ -614,20 +614,17 @@ runtime dependencies Windows does not ship by default: 1. **Microsoft Visual C++ 2015-2022 Redistributable (x64)** — -2. **OpenSSL 3** — `libssl-3-x64.dll` and `libcrypto-3-x64.dll`, resolvable on `PATH` +2. **OpenSSL 3** — install it as a system runtime so `libssl-3-x64.dll` and + `libcrypto-3-x64.dll` resolve without borrowing them from another application. -The redistributable alone is **not** sufficient. If Git for Windows is installed you already have -the OpenSSL DLLs — run `gitnexus` from **Git Bash**, or prepend the directory to `PATH` in the -shell you use: +The redistributable alone is **not** sufficient. Do not prepend a third-party +application directory (including Git for Windows) to `PATH` to pick up those DLLs. -```powershell -$env:PATH = "C:\Program Files\Git\mingw64\bin;$env:PATH" -gitnexus analyze --repair-fts -``` - -Without them the index is still built, but without search tables, so `query` returns empty keyword -results until you re-run `gitnexus analyze --repair-fts` from a shell where the DLLs resolve -([#2669](https://github.com/abhigyanpatwari/GitNexus/issues/2669)). +Without both runtimes the index is still built, but without search tables, so +`query` returns empty keyword results until you install the prerequisites and +re-run `gitnexus analyze --repair-fts` +([#2669](https://github.com/abhigyanpatwari/GitNexus/issues/2669), +[#3218](https://github.com/abhigyanpatwari/GitNexus/issues/3218)). ### Installation fails with native module errors @@ -671,20 +668,20 @@ GitNexus uses optional DuckDB extensions for BM25 and vector search. The `gitnex Configure the behavior with these environment variables: -| Variable | Values | Default | Effect | -| -------------------------------------------- | ------------------------------ | ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `auto` | `auto` runs one bounded install if LOAD fails — a plain `INSTALL`, escalating to `FORCE INSTALL` only when the LOAD error shows the present extension file is broken. `load-only` only uses already-installed extensions (recommended for offline / firewalled environments). `never` skips optional extensions entirely. | -| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | -| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | -| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | -| `GITNEXUS_STORAGE_PATH` | absolute, non-empty directory | unset (repo-local) | Complete external index directory. This preserves the existing configuration semantics and takes precedence over `GITNEXUS_STORAGE_ROOT` when both are set. | -| `GITNEXUS_STORAGE_ROOT` | absolute, non-empty directory | unset (repo-local) | Absolute root directory for external indexes. GitNexus creates an isolated `-/` slot beneath it for each repository, then registers the resolved slot so `status`, MCP, and `serve` can reopen it later. | -| `GITNEXUS_CONTENT_RETENTION` | `full`, `symbol`, `none` | `full` | Source-text retention profile: `full` keeps file and symbol text, `symbol` keeps symbol snippets without full file content, and `none` keeps the structural graph without source body text. | -| `GITNEXUS_STREAM_GRAPH_EMIT` | `0`, `1` | `1` (on) | **On by default** on a full rebuild (`--force`); incremental runs ignore it. Holds structural relationships (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) as CSV-on-disk plus compact in-memory columns instead of as objects in three overlapping indexes, cutting peak in-memory graph heap by ~1.4x at no measurable CPU cost (measured A/B on a synthetic 400k-node / 1.08M-edge graph: 819 MB -> 584 MB, iteration at parity, scaling verified linear from 100k to 800k nodes, with every edge still visible through the graph interface; no end-to-end measurement on a real repository yet). Nothing is traded away — community detection, process extraction, PDG taint summaries and the local-symbol pruner all read a complete relationship set and behave identically. Set to `0` only to bisect a suspected streaming-related fault. | -| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` is the supported default. `icebug` and `auto` are **experimental** and currently behave identically: both try the optional `@ladybugmem/icebug` native Leiden over a CSR export and fall back to Graphology if it is not installed, cannot load, or lacks the deterministic thread/seed controls. Experimental engines partition differently, so community IDs are not comparable across engines. | -| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | -| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores that native unbounded default; invalid values warn and fall back to the default. During `analyze` the pool is right-sized to the graph and, on non-4 KiB-page hosts (Apple Silicon 16 KiB, Ascend/aarch64 64 KiB), scaled by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | -| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | +| Variable | Values | Default | Effect | +| -------------------------------------------- | ------------------------------ | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_LBUG_EXTENSION_INSTALL` | `auto`, `load-only`, `never` | `load-only` globally; `analyze` defaults to `auto` | The process-wide default is `load-only` so serve/MCP/query never install over the network. `gitnexus analyze` overrides to `auto` unless you set the env. FTS loads the packaged per-platform artifact first (macOS, Windows, and Linux), then a named `LOAD`, then `INSTALL` only under `auto`. `never` skips optional extensions entirely. | +| `GITNEXUS_LBUG_EXTENSION_INSTALL_TIMEOUT_MS` | positive integer | `15000` | Wall-clock budget for the out-of-process extension-install child before it is killed. | +| `GITNEXUS_FTS_STEMMER` | supported LadybugDB stemmer | `porter` | Stemmer used when rebuilding BM25/FTS indexes. Use `none` for CJK-heavy repositories, or a language stemmer such as `german`, `french`, or `spanish` when that better matches repository comments and identifiers. Re-run `gitnexus analyze --repair-fts` after changing it. | +| `GITNEXUS_FTS_CJK_SEGMENTATION` | `none`, `bigram` | `none` | `bigram` inserts overlapping character-bigram boundaries into Chinese/Japanese Han-ideograph spans in `content`/`description` before FTS indexing, so LadybugDB's space-only tokenizer can see sub-phrase word boundaries. Scoped to CJK Unified Ideographs only — Japanese Hiragana/Katakana and Korean Hangul are not currently segmented. Unlike `GITNEXUS_FTS_STEMMER`, this rewrites stored text — enabling it on an already-indexed repo requires a full `gitnexus analyze --force`; neither `--repair-fts` nor a plain incremental `analyze` applies it to previously-indexed files. Set the same value wherever `analyze` and search-serving processes (CLI query, MCP server, web server) run. | +| `GITNEXUS_STORAGE_PATH` | absolute, non-empty directory | unset (repo-local) | Complete external index directory. This preserves the existing configuration semantics and takes precedence over `GITNEXUS_STORAGE_ROOT` when both are set. | +| `GITNEXUS_STORAGE_ROOT` | absolute, non-empty directory | unset (repo-local) | Absolute root directory for external indexes. GitNexus creates an isolated `-/` slot beneath it for each repository, then registers the resolved slot so `status`, MCP, and `serve` can reopen it later. | +| `GITNEXUS_CONTENT_RETENTION` | `full`, `symbol`, `none` | `full` | Source-text retention profile: `full` keeps file and symbol text, `symbol` keeps symbol snippets without full file content, and `none` keeps the structural graph without source body text. | +| `GITNEXUS_STREAM_GRAPH_EMIT` | `0`, `1` | `1` (on) | **On by default** on a full rebuild (`--force`); incremental runs ignore it. Holds structural relationships (CALLS, IMPORTS, ACCESSES, CONTAINS, ...) as CSV-on-disk plus compact in-memory columns instead of as objects in three overlapping indexes, cutting peak in-memory graph heap by ~1.4x at no measurable CPU cost (measured A/B on a synthetic 400k-node / 1.08M-edge graph: 819 MB -> 584 MB, iteration at parity, scaling verified linear from 100k to 800k nodes, with every edge still visible through the graph interface; no end-to-end measurement on a real repository yet). Nothing is traded away — community detection, process extraction, PDG taint summaries and the local-symbol pruner all read a complete relationship set and behave identically. Set to `0` only to bisect a suspected streaming-related fault. | +| `GITNEXUS_COMMUNITY_ENGINE` | `graphology`, `icebug`, `auto` | `graphology` | Community-detection engine used during analyze. `graphology` is the supported default. `icebug` and `auto` are **experimental** and currently behave identically: both try the optional `@ladybugmem/icebug` native Leiden over a CSR export and fall back to Graphology if it is not installed, cannot load, or lacks the deterministic thread/seed controls. Experimental engines partition differently, so community IDs are not comparable across engines. | +| `GITNEXUS_WAL_CHECKPOINT_THRESHOLD` | integer `>= -1` | `67108864` (64 MiB) | LadybugDB WAL auto-checkpoint threshold during analyze (bytes). Auto-checkpoint remains enabled; `-1` keeps Ladybug's stock ~16 MiB. Larger thresholds reduce checkpoint frequency but increase the WAL size at rotation time — choose a smaller value on disk-constrained environments. | +| `GITNEXUS_LBUG_BUFFER_POOL_SIZE` | integer `>= 0` (bytes) | min(2 GiB, 80% RAM) | LadybugDB buffer-pool ceiling for every GitNexus database (analyze, MCP server, serve, group bridges). Bounded so a long-lived `gitnexus mcp` process or a large incremental `analyze` cannot grow toward LadybugDB's native 80%-of-RAM default and OOM the host (#2557). `0` restores that native unbounded default; invalid values warn and fall back to the default. During `analyze` the pool is right-sized to the graph and, on non-4 KiB-page hosts (Apple Silicon 16 KiB, Ascend/aarch64 64 KiB), scaled by the page-size granule ratio up to min(2 GiB × pageSize/4 KiB, 80% RAM) (#2631); this env var overrides all of that as an absolute value. | +| `GITNEXUS_LBUG_MAX_DB_SIZE` | positive integer (bytes) | `17179869184` (16 GiB) | Upper bound for a single LadybugDB database file. This is an mmap/disk-address-space ceiling, not a memory limit — it does not constrain the buffer pool (use `GITNEXUS_LBUG_BUFFER_POOL_SIZE` for that). Raise it when indexing genuinely huge monorepos; invalid values silently fall back to the default. | ```bash # Offline/airgapped: never reach the network for extensions diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 30d17220c..b5c17fb31 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -100,16 +100,16 @@ } }, "node_modules/@babel/generator": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.0.tgz", - "integrity": "sha512-NT9NrVwJsbSV6Y2FSstWa71EETOnzrjkL5/wX3D2mYHtKM+qvqB1DvR4D0Setb/gDBsHzRICifwEWMO8CnTF6g==", + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-8.0.5.tgz", + "integrity": "sha512-f/TuhuMAxJqhwxEGNsJrswuG9VHmh0oNFoQoo6TbpgtFAz9wYZXcTAcWZMHfp7ljesr0RG04bp3Aos9GI59L7w==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^8.0.0", - "@babel/types": "^8.0.0", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", + "@babel/parser": "^8.0.5", + "@babel/types": "^8.0.5", + "@jridgewell/gen-mapping": "0.4.0-beta.0", + "@jridgewell/trace-mapping": "^0.3.31", "@types/jsesc": "^2.5.0", "jsesc": "^3.0.2" }, @@ -148,13 +148,13 @@ } }, "node_modules/@babel/parser": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.4.tgz", - "integrity": "sha512-srpptsAkEbbNIC/q8nT7o+m6CQe8CJUTV/t7MYc9NnWlgYVtHOb7JH6SorxMhN0kuRJjVqXbKClG6xSbPtzz+g==", + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-8.0.5.tgz", + "integrity": "sha512-51RXvQNFakaS0bTpYiGkxNbUVwkPO4kONv6EVLorZABxsx+KZ6Z7uSYvi/wmKS/+X+rfj9RvOw0/ZNh+cmI0Rw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^8.0.4" + "@babel/types": "^8.0.5" }, "bin": { "parser": "bin/babel-parser.js" @@ -179,18 +179,18 @@ } }, "node_modules/@babel/traverse": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-8.0.4.tgz", - "integrity": "sha512-bZnmqzGG8UZneG1lLxBoWIH0G6Gr1D846Yu4/3XnY6FhCndMR49u26nTY08u/dAxWmLWF9vGQOuC+84FfIUoeg==", + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-8.0.5.tgz", + "integrity": "sha512-XFfnuvapSc/vJOcUO7kwORSvpBIvraofKEZ2dhT0PjiF21BRCD7YbAFC8UEeDJNeLoQz82/gVqzgX5hCzkCbdg==", "dev": true, "license": "MIT", "dependencies": { "@babel/code-frame": "^8.0.0", - "@babel/generator": "^8.0.0", + "@babel/generator": "^8.0.5", "@babel/helper-globals": "^8.0.0", - "@babel/parser": "^8.0.4", + "@babel/parser": "^8.0.5", "@babel/template": "^8.0.0", - "@babel/types": "^8.0.4", + "@babel/types": "^8.0.5", "obug": "^2.1.1" }, "engines": { @@ -198,9 +198,9 @@ } }, "node_modules/@babel/types": { - "version": "8.0.4", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.4.tgz", - "integrity": "sha512-eY+Yn3dCqTGmyiq2QRU66lA5FL8lqqqvecHt0fF3uHONIa7ToYsaCiWV8lOKqAs0Rb2SjixiKFROngnulPtt2g==", + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-8.0.5.tgz", + "integrity": "sha512-eVdMqi3ej5aHhyQ2Si6yD2cAWeV8FJK9UrhK5aL0Sd8hu5GhT+YswhVNbVheOGVYMg8kuGuMaUpkB3stjj4z8A==", "dev": true, "license": "MIT", "dependencies": { @@ -1230,13 +1230,13 @@ } }, "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "version": "0.4.0-beta.0", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.4.0-beta.0.tgz", + "integrity": "sha512-JdGNkbE4GlNPYQhM0L95fBQr7ctLZJ276QXQLTad4t1oSdnnCI3fDq9DW3BqYAWv8Wc3+HS+4Gsii1oPMCfz1w==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/sourcemap-codec": "^1.6.0-beta.0", "@jridgewell/trace-mapping": "^0.3.24" } }, @@ -1251,9 +1251,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", "dev": true, "license": "MIT" }, diff --git a/gitnexus/package.json b/gitnexus/package.json index 0b0108da4..f79854e9e 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -53,7 +53,7 @@ "postinstall": "node scripts/build-tree-sitter-grammars.cjs", "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", - "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web", + "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web", "version": "node scripts/sync-plugin-manifests.mjs" }, "dependencies": { diff --git a/gitnexus/scripts/assert-publish-fts-coverage.cjs b/gitnexus/scripts/assert-publish-fts-coverage.cjs new file mode 100644 index 000000000..469fc743d --- /dev/null +++ b/gitnexus/scripts/assert-publish-fts-coverage.cjs @@ -0,0 +1,246 @@ +#!/usr/bin/env node +/** + * Publish guard: core↔extension pairing plus vendored FTS artifact integrity. + * + * Does not shell out to `npm pack` (prepack re-entrancy; see the grammar gate). + * Checksums and the `files` allow-list are asserted as pure predicates so a + * future lean-publish narrowing cannot drop the artifacts silently. + */ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); + +const WIN32_ARM64 = 'win32-arm64'; +const SAFE_FILENAME = /^[\w.-]+\.lbug_extension$/; +const REQUIRED_SUPPORTED_TUPLES = [ + 'linux-x64', + 'linux-arm64', + 'darwin-x64', + 'darwin-arm64', + 'win32-x64', +]; + +/** + * Pure pairing core (exported for tests). Returns human-readable problem + * strings; an empty array means the core↔extension pin is consistent. + */ +const EXACT_CORE_PIN = /^\d+\.\d+\.\d+$/; + +function findPairingProblems({ + installedCoreVersion, + manifestCoreVersion, + manifestExtensionVersion, +}) { + const problems = []; + const installed = String(installedCoreVersion ?? ''); + const pinned = String(manifestCoreVersion ?? ''); + if (!EXACT_CORE_PIN.test(installed) || !EXACT_CORE_PIN.test(pinned)) { + problems.push( + `core pin must be exact x.y.z: installed '${installedCoreVersion ?? ''}' vs manifest '${manifestCoreVersion ?? ''}'`, + ); + return problems; + } + if (installed !== pinned) { + problems.push( + `core pin mismatch: installed ${installed} vs manifest ${pinned}` + + (manifestExtensionVersion ? ` (extension ${manifestExtensionVersion})` : ''), + ); + } + return problems; +} + +function readInstalledCoreVersion(pkg) { + const raw = pkg?.dependencies?.['@ladybugdb/core']; + return raw == null ? '' : String(raw); +} + +function normalizeFilesEntry(value) { + return String(value ?? '') + .replace(/\\/g, '/') + .replace(/\/+$/, '') + .replace(/\/\*\*?$/, ''); +} + +/** True when package.json `files` still ships the FTS prebuild tree. */ +function filesCoverFtsArtifacts(filesField) { + return (filesField || []).some((entry) => { + const n = normalizeFilesEntry(entry); + return ( + n === 'vendor' || + n === 'vendor/lbug-fts' || + n === 'vendor/lbug-fts/prebuilds' || + n === 'vendor/**/prebuilds' + ); + }); +} + +function supportedTuplesFromManifest(manifest) { + return (manifest?.tuples ?? []).map((entry) => entry.tuple); +} + +function unsupportedTuplesFromManifest(manifest) { + return (manifest?.unsupportedTuples ?? []).map((entry) => entry.tuple); +} + +function parseSha256Sums(text) { + const out = {}; + for (const line of String(text ?? '').split(/\r?\n/)) { + const m = /^([a-fA-F0-9]{64})\s+\.\/(\S+)$/.exec(line.trim()); + if (!m) continue; + out[m[2]] = m[1].toLowerCase(); + } + return out; +} + +function sha256File(filePath) { + return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex'); +} + +/** + * Integrity + coverage predicates. `artifactByTuple` is injected so tests + * never invoke pack and never need the real binaries. + */ +function findArtifactProblems({ + tuples, + unsupportedTuples, + filesField, + checksumByRelPath, + artifactByTuple, + filename, +}) { + const problems = []; + const filenameSafe = filename || 'libfts.lbug_extension'; + if (!SAFE_FILENAME.test(filenameSafe)) { + problems.push(`invalid FTS artifact filename: ${filenameSafe}`); + } + const listed = new Set(tuples || []); + for (const required of REQUIRED_SUPPORTED_TUPLES) { + if (!listed.has(required)) { + problems.push(`manifest.tuples is missing required ${required}`); + } + } + if (!tuples || tuples.length === 0) { + problems.push('manifest.tuples is empty — refusing to publish with 0 artifacts'); + } + if (!filesCoverFtsArtifacts(filesField)) { + problems.push('package.json files no longer covers vendor/lbug-fts/prebuilds'); + } + if (!(unsupportedTuples || []).includes(WIN32_ARM64)) { + problems.push('win32-arm64 must be declared unsupported (no upstream artifact)'); + } + if ((tuples || []).includes(WIN32_ARM64)) { + problems.push('win32-arm64 is listed as supported but has no upstream artifact'); + } + for (const tuple of tuples || []) { + const rel = `${tuple}/${filenameSafe}`; + const artifact = artifactByTuple?.[tuple]; + if (!artifact?.exists) { + problems.push(`missing artifact for ${tuple} (${rel})`); + continue; + } + const expected = checksumByRelPath?.[rel]; + if (!expected) { + problems.push(`missing SHA-256 for ${rel}`); + continue; + } + if (artifact.hash !== expected) { + problems.push( + `checksum mismatch for ${tuple}: expected ${expected} got ${artifact.hash}` + + (artifact.sizeBytes != null ? ` (${artifact.sizeBytes} bytes)` : ''), + ); + } + } + return problems; +} + +function readDiskArtifacts(prebuildsDir, tuples, filename) { + const artifactByTuple = {}; + for (const tuple of tuples) { + const filePath = path.join(prebuildsDir, tuple, filename); + if (!fs.existsSync(filePath)) { + artifactByTuple[tuple] = { exists: false }; + continue; + } + const buf = fs.readFileSync(filePath); + artifactByTuple[tuple] = { + exists: true, + hash: crypto.createHash('sha256').update(buf).digest('hex'), + sizeBytes: buf.byteLength, + }; + } + return artifactByTuple; +} + +function main() { + const gitnexusRoot = path.join(__dirname, '..'); + const pkg = JSON.parse(fs.readFileSync(path.join(gitnexusRoot, 'package.json'), 'utf8')); + const vendorDir = path.join(gitnexusRoot, 'vendor', 'lbug-fts'); + const manifestPath = path.join(vendorDir, 'manifest.json'); + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (err) { + console.error( + `[fts-pairing] Refusing to publish — cannot read ${manifestPath}: ${err.message}`, + ); + process.exit(1); + } + + const installedCoreVersion = readInstalledCoreVersion(pkg); + const pairing = findPairingProblems({ + installedCoreVersion, + manifestCoreVersion: manifest.coreVersion, + manifestExtensionVersion: manifest.extensionVersion, + }); + + const tuples = supportedTuplesFromManifest(manifest); + const unsupportedTuples = unsupportedTuplesFromManifest(manifest); + const filename = manifest.filename || 'libfts.lbug_extension'; + const prebuildsDir = path.join(vendorDir, 'prebuilds'); + let checksumByRelPath = {}; + try { + checksumByRelPath = parseSha256Sums( + fs.readFileSync(path.join(prebuildsDir, 'SHA256SUMS'), 'utf8'), + ); + } catch (err) { + pairing.push(`cannot read SHA256SUMS: ${err.message}`); + } + + const artifacts = findArtifactProblems({ + tuples, + unsupportedTuples, + filesField: pkg.files, + checksumByRelPath, + artifactByTuple: readDiskArtifacts(prebuildsDir, tuples, filename), + filename, + }); + + const problems = [...pairing, ...artifacts]; + if (problems.length > 0) { + console.error('[fts-pairing] Refusing to publish — FTS artifact coverage failed:'); + for (const p of problems) console.error(` - ${p}`); + console.error( + '\nFix: refresh vendor/lbug-fts via .github/scripts/fetch-lbug-fts-artifacts.mjs, ' + + 'or restore the core pin / files allow-list.', + ); + process.exit(1); + } + + console.log( + `[fts-pairing] OK — core ${installedCoreVersion} ↔ extension ${manifest.extensionVersion}; ` + + `${tuples.length} artifacts.`, + ); +} + +if (require.main === module) main(); + +module.exports = { + findPairingProblems, + findArtifactProblems, + filesCoverFtsArtifacts, + parseSha256Sums, + readInstalledCoreVersion, + supportedTuplesFromManifest, + unsupportedTuplesFromManifest, + sha256File, +}; diff --git a/gitnexus/scripts/cross-platform-shard.ts b/gitnexus/scripts/cross-platform-shard.ts index 0cd9fe8d9..9e8f2d8e2 100644 --- a/gitnexus/scripts/cross-platform-shard.ts +++ b/gitnexus/scripts/cross-platform-shard.ts @@ -55,8 +55,13 @@ export const WINDOWS_WEIGHTS_SEC: Readonly> = { // Upstream speedups may reduce these figures; retaining conservative weights // keeps the expensive suites distributed without changing the watchdog. 'test/unit/incremental-index-extension-dml-gate.test.ts': 414, - 'test/integration/skills-e2e.test.ts': 444, - 'test/integration/fts-extension-e2e.test.ts': 146, + // Re-measured on windows-latest run 34815870795 after vendored-first FTS + // rewrote the HOME-layout e2e (373s) and skills-e2e grew to 542s. The old + // 146s/444s entries packed both onto shard 2/3 and blew the 20-minute + // watchdog with one file still queued. + 'test/integration/skills-e2e.test.ts': 550, + 'test/integration/fts-extension-e2e.test.ts': 380, + 'test/integration/skip-fts.test.ts': 110, 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, 'test/integration/cli-limit-e2e.test.ts': 75, 'test/unit/hooks.test.ts': 26, diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index a69fac087..95f5df7ec 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -85,6 +85,20 @@ const PLATFORM_LOGIC = [ 'test/unit/lbug-config-pagesize.test.ts', 'test/unit/worker-pool-windows-quarantine.test.ts', 'test/unit/lbug-pool-fts-load.test.ts', + // U7 arm B: Windows FTS names a vendor-neutral OpenSSL/VC++ prerequisite + // and must never recommend borrowing Git for Windows DLLs. The file's + // assertions are unconditional so a skip-only suite cannot stay green. + 'test/integration/fts-windows-dependency.test.ts', + // Remedy-text suites: discoverability only. They pass explicit platform + // strings into pure classifiers and drive mocked rejections with hardcoded + // literals, so they assert identically on every runner. Registering them + // here does not claim Windows-specific behavioral coverage. + 'test/unit/extension-load-error.test.ts', + 'test/unit/fts-degraded-warning.test.ts', + // Vendored-root symlink containment uses realpathSync + path.relative; a + // prefix-only leak follows a Windows junction / POSIX symlink out of + // vendor/. Ubuntu-only would leave that guard unverified on the OS matrix. + 'test/unit/lbug-extension-loader.test.ts', // Global registry writes use the platform-specific index-lock backend // (Windows named pipe, Linux socket, or macOS file lock). This includes the // overlapping-registration regression from #2716 on every OS matrix. @@ -208,6 +222,7 @@ const SPAWN_CLI = [ // FTS extension lifecycle — the #2374 bug was Windows-reported, so this must // run on the Windows/macOS matrix, not just the Ubuntu full suite. 'test/integration/fts-extension-e2e.test.ts', + 'test/integration/fts-vendored-root-seam.test.ts', 'test/integration/server-http-startup.test.ts', 'test/integration/mcp/server-startup.test.ts', 'test/integration/analyze-heap-oom-e2e.test.ts', diff --git a/gitnexus/scripts/ensure-fts.ts b/gitnexus/scripts/ensure-fts.ts index 94f781374..f3d6199f2 100644 --- a/gitnexus/scripts/ensure-fts.ts +++ b/gitnexus/scripts/ensure-fts.ts @@ -1,14 +1,11 @@ /** - * Install the LadybugDB FTS and VECTOR extensions into the shared home (~/.lbdb) - * up front, so every test in a sharded CI run finds them regardless of shard. + * Make FTS and VECTOR resolvable for every shard before vitest starts. * - * FTS-dependent tests split two ways: the LOAD-path gate (skipUnlessFtsAvailable) - * self-installs on miss, but the FILE-path gate (requireFtsResourceOrSkip, e.g. - * extension-binary-real.test.ts) resolves the extension path at module load and - * cannot self-install. Sharding (and the balancing sequencer) can drop such a - * test into a shard with no installer sibling — this step removes that ordering - * dependency by installing FTS once before vitest starts. `auto` is LOAD-first, - * so a cache-warmed extension costs no network. + * After vendoring, FTS is ready when the packaged artifact exists — LOAD + * no longer writes `~/.lbdb`, and the FILE-path gates resolve that artifact + * (or a leftover home install). When no artifact is present yet, fall back + * to one bounded `auto` INSTALL so shards without an installer sibling still + * find a file. * * Best-effort: exits 0 on failure (offline etc.) — the per-test gates still * hard-fail under GITNEXUS_REQUIRE_FTS=1 if FTS is genuinely unavailable, which @@ -23,12 +20,17 @@ import { loadVectorExtension, closeLbug, } from '../src/core/lbug/lbug-adapter.js'; +import { resolveVendoredFtsPath } from '../src/core/lbug/vendored-extension-path.js'; const dir = mkdtempSync(join(tmpdir(), 'gn-ensure-fts-')); try { await initLbug(join(dir, 'ensure-fts.lbug')); - const ok = await loadFTSExtension(undefined, { policy: 'auto' }); - console.log(ok ? 'FTS extension ready.' : 'FTS extension unavailable (continuing).'); + if (resolveVendoredFtsPath()) { + console.log('FTS extension ready (vendored artifact).'); + } else { + const ok = await loadFTSExtension(undefined, { policy: 'auto' }); + console.log(ok ? 'FTS extension ready.' : 'FTS extension unavailable (continuing).'); + } // VECTOR rides the same pre-install (#2623): the win32 gate is gone, so the // vector suites genuinely run on Windows/macOS — installing once here means // every sharded test process LOADs from ~/.lbdb instead of racing its own diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 2849a2b96..0f034ee26 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -13,7 +13,7 @@ import os from 'os'; import { spawn } from 'child_process'; import v8 from 'v8'; import cliProgress from 'cli-progress'; -import { FTS_DISABLED_MESSAGE, isExplicitFtsDisablement } from '../core/search/fts-policy.js'; +import { formatAnalyzeFtsSkipSummary } from '../core/search/fts-policy.js'; import { isLbugReady, LbugWipeError } from '../core/lbug/lbug-adapter.js'; import { boundedCheckpointBeforeExit } from '../core/lbug/shutdown-helpers.js'; import { findUndeclaredRelationPairError } from '../core/lbug/rel-pair-routing.js'; @@ -1451,8 +1451,9 @@ const analyzeCommandImpl = async ( console.error = origError; bar.stop(); console.log(' Already up to date\n'); - if (isExplicitFtsDisablement(result.ftsSkipReason)) - console.log(` ${FTS_DISABLED_MESSAGE}\n`); + if (result.ftsSkipped) { + console.log(` ${formatAnalyzeFtsSkipSummary(result.ftsSkipReason)}\n`); + } if (runOptions.registryName) { console.log(` Registry name: ${result.repoName}\n`); } @@ -1610,28 +1611,9 @@ const analyzeCommandImpl = async ( // progress-bar log() that fired mid-run has already scrolled away, so the // degraded-search state must also appear in the final summary (#1161). if (result.ftsSkipped) { - // #2658 review L2: a build/verify failure is NOT an extension-unavailable - // problem — sending the user to install the extension is the wrong remedy. - if (isExplicitFtsDisablement(result.ftsSkipReason)) { - console.log(`\n ${FTS_DISABLED_MESSAGE}`); - } else if (result.ftsSkipReason === 'build-failed') { - console.log( - `\n Warning: full-text/BM25 search is disabled — the search index build failed this run.\n` + - ` The FTS extension is available; rerun \`gitnexus analyze --repair-fts\`. If it persists,\n` + - ` check the disk for space or corruption. Run \`gitnexus doctor\` for details.`, - ); - } else { - console.log( - // NOT "then rerun" (#2841 §5.C): this run stamped `lastCommit`, so a - // plain rerun on an unchanged tree takes the up-to-date fast path and - // returns before Phase 3 could rebuild anything — the advice would be - // ineffective exactly when the user follows it. `--repair-fts` is the - // verb that rebuilds the search indexes without re-parsing the repo. - `\n Warning: full-text/BM25 search is disabled — the LadybugDB FTS extension was unavailable.\n` + - ` Install it once with network access (GITNEXUS_LBUG_EXTENSION_INSTALL=auto), then run\n` + - ` \`gitnexus analyze --repair-fts\` to build the search indexes. Run \`gitnexus doctor\` for details.`, - ); - } + // Total switch (#2658 L2 + native-abort/tuple-missing): a new skip + // reason must not inherit the network-install remedy. + console.log(`\n ${formatAnalyzeFtsSkipSummary(result.ftsSkipReason)}`); } try { diff --git a/gitnexus/src/cli/doctor.ts b/gitnexus/src/cli/doctor.ts index aebfc7cf8..202baafbd 100644 --- a/gitnexus/src/cli/doctor.ts +++ b/gitnexus/src/cli/doctor.ts @@ -14,6 +14,7 @@ import { import { cudaRedirectDoctorStatus } from '../core/embeddings/onnxruntime-node-resolver.js'; import { checkLbugNative, + ftsAvailabilityLabel, type NativeCheckResult, probeFtsExtensionLoad, probeVectorExtensionLoad, @@ -23,8 +24,12 @@ import { getOsPageSize, isPageSizeAwareLadybug, } from '../core/lbug/lbug-config.js'; -import { diagnoseExtensionLoad } from '../core/lbug/extension-load-error.js'; -import { getExtensionInstallPolicy } from '../core/lbug/extension-loader.js'; +import { diagnoseExtensionLoad, extractExtensionPath } from '../core/lbug/extension-load-error.js'; +import { resolveFtsVersionPair } from '../core/lbug/vendored-extension-path.js'; +import { + getExtensionInstallPolicy, + resolveAnalyzeInstallPolicy, +} from '../core/lbug/extension-loader.js'; import { updateEligibleInstallSync } from '../core/install-context.js'; import { readValidatedUpdateCacheSync, type ValidatedUpdateCache } from '../core/update-cache.js'; import { t } from './i18n/index.js'; @@ -258,9 +263,7 @@ export const doctorCommand = async () => { const ftsProbe = nativeCheck.ok ? await probeFtsExtensionLoad() : { loaded: false, reason: 'LadybugDB native module (lbugjs.node) failed to load' }; - console.log( - ` ${label('doctor.labels.fullTextSearch', 18)}${ftsProbe.loaded ? 'available' : 'unavailable'}`, - ); + console.log(` ${label('doctor.labels.fullTextSearch', 18)}${ftsAvailabilityLabel(ftsProbe)}`); if (!ftsProbe.loaded && ftsProbe.reason) { console.log(` ${padDisplayEnd('', 18)}${ftsProbe.reason}`); // Add an actionable remedy for recognized failure classes (#2374). The @@ -268,7 +271,15 @@ export const doctorCommand = async () => { // ("specified module could not be found") is opaque, so name the fix (VC++ // redist, then OpenSSL) instead of leaving the user to reinstall in vain. // `unknown`'s remedy is "run doctor", which would be circular here. - const { kind, remedy } = diagnoseExtensionLoad(ftsProbe.reason); + // Policy `never` is not a load failure — skip structural diagnosis. + const { kind, remedy } = ftsProbe.suppressed + ? { kind: 'unknown' as const, remedy: '' } + : diagnoseExtensionLoad( + ftsProbe.reason, + 'FTS', + extractExtensionPath(ftsProbe.reason), + resolveFtsVersionPair(extractExtensionPath(ftsProbe.reason)), + ); if (kind !== 'unknown') { console.log(` ${padDisplayEnd('', 18)}${remedy}`); } @@ -304,14 +315,17 @@ export const doctorCommand = async () => { // Surface the optional-extension install policy so offline users can see // whether analyze/query will reach the network (extension.ladybugdb.com). // Literal label (like the 'native' line) to avoid adding i18n keys. - const installPolicy = getExtensionInstallPolicy(); - const policyHint = - installPolicy === 'load-only' + const serveQueryPolicy = getExtensionInstallPolicy(); + const analyzePolicy = resolveAnalyzeInstallPolicy(); + const policyHint = (policy: string) => + policy === 'load-only' ? ' (offline; load only, no network install)' - : installPolicy === 'never' + : policy === 'never' ? ' (optional extensions disabled)' : ' (installs missing extensions over network)'; - console.log(` ${padDisplayEnd('Ext install:', 18)}${installPolicy}${policyHint}`); + console.log( + ` ${padDisplayEnd('Ext install:', 18)}serve/query=${serveQueryPolicy}${policyHint(serveQueryPolicy)}; analyze=${analyzePolicy}${policyHint(analyzePolicy)}`, + ); console.log( ` ${label('doctor.labels.exactScanLimit', 18)}${t('doctor.chunks', { count: capabilities.exactScanLimit })}`, ); diff --git a/gitnexus/src/core/lbug/extension-load-error.ts b/gitnexus/src/core/lbug/extension-load-error.ts index f8213ef4d..3e08beb4d 100644 --- a/gitnexus/src/core/lbug/extension-load-error.ts +++ b/gitnexus/src/core/lbug/extension-load-error.ts @@ -29,8 +29,18 @@ export type ExtensionLoadErrorKind = | 'missing_file' | 'corrupt_file' | 'missing_dependency' + | 'version_skew' | 'unknown'; +export interface ExtensionVersionPair { + expected?: string; + found?: string; +} + +/** Kinds whose remedy must replace the generic network-install tail. */ +export const usesClassifiedLoadRemedy = (kind: ExtensionLoadErrorKind): boolean => + kind === 'missing_dependency' || kind === 'version_skew'; + export interface ExtensionLoadDiagnosis { readonly kind: ExtensionLoadErrorKind; /** Actionable, literal-English remedy suited to the class. */ @@ -111,6 +121,13 @@ const LOAD_FAILURE_WRAPPER = /failed to load library/i; const repairFtsHint = (label: string, lead: string): string => label === 'FTS' ? ` (${lead}\`gitnexus analyze --repair-fts\`)` : ''; +const VERSION_SKEW_HINT = 'This is a version mismatch, not a missing host runtime.'; + +const versionSkewRemedy = (label: string, expected: string, found: string): string => + `The ${label} extension version ${found} does not match the expected ${expected}. ` + + `Use a matching artifact${repairFtsHint(label, 'or ')} and run \`gitnexus doctor\`. ` + + VERSION_SKEW_HINT; + const missingFileRemedy = (label: string): string => `The ${label} extension is not installed. Re-run with network access and ` + `GITNEXUS_LBUG_EXTENSION_INSTALL=auto${repairFtsHint(label, 'or ')} to download it.`; @@ -127,25 +144,21 @@ const VC_REDIST_INSTALL_HINT = 'the Microsoft Visual C++ 2015-2022 Redistributable (x64) from ' + 'https://aka.ms/vs/17/release/vc_redist.x64.exe'; -// Git for Windows already ships the OpenSSL 3 DLLs in its mingw64 bin directory, -// so the identical command that fails in PowerShell succeeds in Git Bash (#2669 -// reporter, who had the VC++ redist installed and still failed until that -// directory was on PATH). Deliberately a fixed system path and never a -// user-profile one: remedy text is NOT path-redacted (fts-indexes.ts redacts -// only the reason), and fts-degraded-warning.test.ts asserts that no -// `C:\Users\…` path ever reaches a user through this surface. -const GIT_BASH_OPENSSL_HINT = - ' If Git for Windows is installed you already have those DLLs: run the same command from Git Bash, ' + - 'or prepend "C:\\Program Files\\Git\\mingw64\\bin" to PATH.'; +// U7 arm B (OQ1 unanswered; KTD13 forbids shipping OpenSSL DLLs without a +// named CVE owner). Name the system runtimes. Do not tell anyone to borrow +// DLLs from Git for Windows or prepend a third-party application directory. +const WINDOWS_OPENSSL_RUNTIME_HINT = + ' If the error persists, install OpenSSL 3 as a system runtime so ' + + 'libcrypto-3-x64.dll and libssl-3-x64.dll resolve without borrowing them ' + + 'from another application.'; // MSVC-first per DuckDB's canonical answer for this exact error; OpenSSL second. const windowsMissingDependencyRemedy = (label: string): string => `The ${label} extension is present but a required runtime library is missing (Windows error 126). ` + 'Reinstalling the extension will NOT help. Install ' + VC_REDIST_INSTALL_HINT + - '; if the error persists, the extension also needs OpenSSL 3 ' + - '(libcrypto-3-x64.dll / libssl-3-x64.dll) on the DLL search path.' + - GIT_BASH_OPENSSL_HINT; + '.' + + WINDOWS_OPENSSL_RUNTIME_HINT; const posixMissingDependencyRemedy = (label: string): string => `The ${label} extension is present but a shared library it depends on could not be loaded (named in ` + @@ -217,8 +230,7 @@ const structuralMissingDependencyRemedy = (label: string): string => `The ${label} extension file is valid, so the failure is a missing or incompatible runtime dependency, ` + 'not the extension itself — reinstalling will NOT help. On Windows, install ' + VC_REDIST_INSTALL_HINT + - ' and ensure OpenSSL 3 is available; on Linux/macOS install the shared library named in the error above.' + - GIT_BASH_OPENSSL_HINT; + ' and install OpenSSL 3 as a system runtime; on Linux/macOS install the shared library named in the error above.'; /** * Pull the extension file path out of lbug's load error. lbug's wrapper is @@ -357,10 +369,12 @@ export function inspectExtensionBinary( export function diagnoseExtensionLoad( reason: string | undefined | null, label: string = 'FTS', + explicitPath?: string | null, + versions?: ExtensionVersionPair, ): ExtensionLoadDiagnosis { const text = reason ?? ''; const stringResult = classifyExtensionLoadError(text, label); - const fileState = inspectExtensionBinary(extractExtensionPath(text)); + const fileState = inspectExtensionBinary(explicitPath ?? extractExtensionPath(text)); if (fileState === 'corrupt') { return { kind: 'corrupt_file', remedy: corruptFileRemedy(label) }; @@ -376,6 +390,12 @@ export function diagnoseExtensionLoad( if (stringResult.kind === 'corrupt_file') { return stringResult; } + if (versions?.expected && versions.found && versions.expected !== versions.found) { + return { + kind: 'version_skew', + remedy: versionSkewRemedy(label, versions.expected, versions.found), + }; + } // A structurally sound binary that still failed to load ⇒ a dependency/runtime // problem, decided WITHOUT the localized tail. Keep the string classifier's // sharper remedy when it recognized the specific case (e.g. English 126). diff --git a/gitnexus/src/core/lbug/extension-loader.ts b/gitnexus/src/core/lbug/extension-loader.ts index 0ad020519..575b3587d 100644 --- a/gitnexus/src/core/lbug/extension-loader.ts +++ b/gitnexus/src/core/lbug/extension-loader.ts @@ -1,9 +1,29 @@ import { spawn } from 'child_process'; import { fileURLToPath } from 'node:url'; import { LBUG_MAX_DB_SIZE } from './lbug-config.js'; -import { diagnoseExtensionLoad, type ExtensionLoadDiagnosis } from './extension-load-error.js'; +import { escapeCypherString } from './cypher-escape.js'; +import { + diagnoseExtensionLoad, + extractExtensionPath, + type ExtensionLoadDiagnosis, +} from './extension-load-error.js'; +import { + defaultVendorRoot, + isUnsupportedFtsTuple, + nodePlatformTuple, + resolveFtsVersionPair, + resolveVendoredFtsPath, +} from './vendored-extension-path.js'; import { logger } from '../logger.js'; +export type ExtensionAttemptSource = 'vendored' | 'named' | 'install'; + +/** Structured load attempt — source and tuple labels only, never a path. */ +export interface ExtensionLoadAttempt { + source: ExtensionAttemptSource; + tuple?: string; +} + const DEFAULT_EXTENSION_INSTALL_TIMEOUT_MS = 15_000; const EXTENSION_NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]*$/; @@ -37,6 +57,8 @@ export interface ExtensionCapability { * cached remedy instead of re-inspecting the extension file on every call (#2383 F3). */ diagnosis?: ExtensionLoadDiagnosis; + /** What this ensure() tried, labels only (KTD7). */ + attempts?: ExtensionLoadAttempt[]; } /** Per-call overrides applied on top of `ExtensionManager` defaults. */ @@ -55,6 +77,10 @@ export interface ExtensionEnsureOptions { * degradation goes unreported. */ quiet?: boolean; + /** Injected vendor tree for tests / e2e. Never an attacker-controlled env. */ + vendorRoot?: string; + /** Injected Node platform tuple (`linux-x64`). Defaults to this process. */ + platformTuple?: string; } export interface ExtensionManagerOptions { @@ -192,12 +218,13 @@ export const installDuckDbExtensionOutOfProcess = async ( /** * Centralized lifecycle manager for optional LadybugDB extensions. * - * Always tries `LOAD EXTENSION ` first — it is per-connection, - * idempotent, and never touches the network. If `LOAD` fails and the active - * policy permits, the manager runs a single bounded out-of-process `INSTALL` - * attempt per process and retries `LOAD`. Capability outcomes are cached so - * unavailable extensions degrade search features without ever blocking - * subsequent analyze or query calls. + * Tries `LOAD` first — it is per-connection, idempotent, and never + * touches the network. For FTS, a packaged vendored path is path-LOADed + * before the named `LOAD EXTENSION fts`. If `LOAD` fails and the active + * policy permits, the manager runs a single bounded out-of-process + * `INSTALL` attempt per process and retries `LOAD`. Capability outcomes + * are cached so unavailable extensions degrade search features without + * ever blocking subsequent analyze or query calls. * * Policy precedence (most specific wins): * per-call `opts.policy` → constructor `options.policy` → env → `load-only` @@ -244,28 +271,78 @@ export class ExtensionManager { const warn = this.options.warn ?? ((msg: string) => logger.warn(msg)); const quiet = opts.quiet === true; + const attempts: ExtensionLoadAttempt[] = []; + let lastInspectPath: string | null = null; + const versionsFor = (inspectPath: string | null) => + name === 'fts' ? resolveFtsVersionPair(inspectPath, opts.vendorRoot) : undefined; + if (policy === 'never') { - this.markUnavailable(name, label, 'extension install policy is "never"', warn, quiet); + this.markUnavailable( + name, + label, + 'extension install policy is "never"', + warn, + quiet, + attempts, + null, + ); return false; } + if (name === 'fts') { + const tuple = opts.platformTuple ?? nodePlatformTuple(); + const vendorRoot = opts.vendorRoot ?? defaultVendorRoot(); + const vendored = resolveVendoredFtsPath({ vendorRoot, tuple }); + if (vendored) { + attempts.push({ source: 'vendored', tuple }); + const vendoredError = await this.tryLoadPath(query, vendored); + if (vendoredError === null) { + this.markLoaded(name, attempts); + return true; + } + lastInspectPath = vendored; + } else if (isUnsupportedFtsTuple(tuple, vendorRoot)) { + attempts.push({ source: 'vendored', tuple }); + this.markUnavailable( + name, + label, + this.composeReason(`no packaged FTS artifact for ${tuple}`, attempts), + warn, + quiet, + attempts, + null, + ); + return false; + } + } + + attempts.push({ source: 'named' }); const loadError = await this.tryLoad(query, name); if (loadError === null) { - this.markLoaded(name); + this.markLoaded(name, attempts); return true; } + const namedPath = extractExtensionPath(loadError); + if (namedPath) lastInspectPath = namedPath; if (policy === 'load-only') { this.markUnavailable( name, label, - `load-only policy (no install attempted); LOAD ${name} failed: ${loadError}`, + this.composeReason( + `load-only policy (no install attempted); LOAD ${name} failed: ${loadError}`, + attempts, + ), warn, quiet, + attempts, + lastInspectPath, + versionsFor(lastInspectPath), ); return false; } + attempts.push({ source: 'install' }); let install = this.installAttempted.get(name); if (!install) { const installFn = this.options.installExtension ?? installDuckDbExtensionOutOfProcess; @@ -279,25 +356,31 @@ export class ExtensionManager { this.markUnavailable( name, label, - `${install.message}; LOAD ${name} had failed: ${loadError}`, + this.composeReason(`${install.message}; LOAD ${name} had failed: ${loadError}`, attempts), warn, quiet, + attempts, + lastInspectPath, + versionsFor(lastInspectPath), ); return false; } const retryError = await this.tryLoad(query, name); if (retryError === null) { - this.markLoaded(name); + this.markLoaded(name, attempts); return true; } this.markUnavailable( name, label, - `LOAD ${name} failed after successful INSTALL: ${retryError}`, + this.composeReason(`LOAD ${name} failed after successful INSTALL: ${retryError}`, attempts), warn, quiet, + attempts, + extractExtensionPath(retryError), + versionsFor(extractExtensionPath(retryError)), ); return false; } @@ -323,8 +406,36 @@ export class ExtensionManager { } } - private markLoaded(name: string): void { - this.capabilities.set(name, { name, loaded: true }); + private async tryLoadPath( + query: (sql: string) => Promise, + absPath: string, + ): Promise { + try { + await query(`LOAD EXTENSION '${escapeCypherString(absPath)}'`); + return null; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return alreadyAvailable(msg) ? null : oneLine(msg); + } + } + + private composeReason(base: string, attempts: ExtensionLoadAttempt[]): string { + if (!attempts.some((attempt) => attempt.source === 'vendored')) return base; + const trail = attempts + .map((attempt) => + attempt.source === 'vendored' ? `vendored ${attempt.tuple}` : attempt.source, + ) + .join(', '); + return `${base} (attempts: ${trail})`; + } + + private markLoaded(name: string, attempts: ExtensionLoadAttempt[] = []): void { + const record = attempts.some((attempt) => attempt.source === 'vendored') ? attempts : undefined; + this.capabilities.set(name, { + name, + loaded: true, + ...(record ? { attempts: record } : {}), + }); } private markUnavailable( @@ -333,14 +444,19 @@ export class ExtensionManager { reason: string, warn: (message: string) => void, quiet = false, + attempts: ExtensionLoadAttempt[] = [], + inspectPath: string | null = null, + versions?: { expected?: string; found?: string }, ): void { // Classify once here (the single load-failure sink, run per Database not per // request) so the hot per-request warning path does no file I/O (#2383 F3). + // Diagnose the LAST attempt's file, never a path scraped from concatenated text. this.capabilities.set(name, { name, loaded: false, reason, - diagnosis: diagnoseExtensionLoad(reason, label), + attempts, + diagnosis: diagnoseExtensionLoad(reason, label, inspectPath, versions), }); const message = `GitNexus: ${label} extension unavailable; continuing without ${label} features. ${reason}`; // A quiet probe must not register the dedup key: the owning caller may hit diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index ced1c48bd..621d26da6 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -12,7 +12,8 @@ import { escapeCypherString } from './cypher-escape.js'; import { withConnLock } from './conn-lock.js'; import { isWalDriverActive } from './wal-driver-state.js'; import { KnowledgeGraph } from '../graph/types.js'; -import type { ContentRetention } from '../../storage/repo-meta.js'; +import { loadMeta, type ContentRetention } from '../../storage/repo-meta.js'; +import { allowsFtsCrashWalPark, hasRecoveredInPlaceFtsAbort } from '../search/fts-crash-marker.js'; import { NODE_TABLES, REL_TABLE_NAME, @@ -42,7 +43,8 @@ import { } from './extension-loader.js'; // Remedy classification for LOAD failures (#2374/#2383). Pure + node:fs only, so // this adds no cycle: `extension-loader.ts` already depends on it. -import { diagnoseExtensionLoad } from './extension-load-error.js'; +import { diagnoseExtensionLoad, extractExtensionPath } from './extension-load-error.js'; +import { resolveFtsVersionPair } from './vendored-extension-path.js'; import { classifyDeleteAllError, closeLbugConnection, @@ -63,9 +65,11 @@ import { type LbugConnectionHandle, } from './lbug-config.js'; import { + assertReadOnlyFtsCrashSafe, cleanQuarantinedMissingShadowWals, finalizeLbugSidecarsAfterClose, guardWalQuarantine, + type WalCrashEvidence, isMissingShadowSidecarError, isReadOnlyShadowReplayError, lbugLockRemediation, @@ -280,10 +284,9 @@ const DB_LOCK_RETRY_DELAY_MS = 500; /** * Return true when the error message indicates a write was attempted against * a read-only LadybugDB connection. The MCP query pool opens DBs read-only, - * so any path that calls a `CREATE_*` procedure there will surface this - * (e.g. defensive `ensureFTSIndex` calls). Owners of the writable analyze - * path should ignore this error — index creation is owned by `gitnexus - * analyze` and either already happened or will happen on the next run. + * so any path that calls a `CREATE_*` procedure there will surface this. + * Index creation is owned by `gitnexus analyze` and either already happened + * or will happen on the next run. */ export const isReadOnlyDbError = (err: unknown): boolean => { // Walk the `cause` chain (bounded) so a wrapped read-only error — e.g. the @@ -546,8 +549,11 @@ const refuseLargeWalQuarantine = async ( dbPath: string, mode: 'read-only' | 'writable', triggeringErr: unknown, + crashEvidence?: WalCrashEvidence, ): Promise => { - await guardWalQuarantine(dbPath, mode, triggeringErr, logger); + // Latitude defaults to refusal. Only the analyze writer passes + // `fts-inplace-checkpointed`; serve never does (R9). + await guardWalQuarantine(dbPath, mode, triggeringErr, logger, crashEvidence); }; const reopenReadOnlyAfterMissingShadow = async ( @@ -578,11 +584,37 @@ const reopenReadOnlyAfterMissingShadow = async ( } }; +const writableFtsCrashWalEvidence = async ( + dbPath: string, +): Promise => { + try { + const meta = await loadMeta(path.dirname(dbPath)); + if ( + meta && + (allowsFtsCrashWalPark(meta.incrementalInProgress) || + hasRecoveredInPlaceFtsAbort(meta.capabilities?.fts)) + ) { + return { kind: 'fts-inplace-checkpointed' }; + } + } catch { + return undefined; + } + return undefined; +}; + const reopenWritableAfterMissingShadow = async ( dbPath: string, err: unknown, ): Promise => { - await refuseLargeWalQuarantine(dbPath, 'writable', err); + // Analyze writers may park a leftover in-place FTS abort WAL. Serve/embed + // refuse first via assertReadOnlyFtsCrashSafe and must not pass this + // evidence themselves (R9); read-only reopen never parks. + await refuseLargeWalQuarantine( + dbPath, + 'writable', + err, + await writableFtsCrashWalEvidence(dbPath), + ); try { await quarantineWalForMissingShadow(dbPath, { logger, @@ -814,6 +846,7 @@ const doInitLbug = async ( // create databases and don't need the lock. // --------------------------------------------------------------------------- if (readOnly) { + await assertReadOnlyFtsCrashSafe(dbPath); await preflightLbugSidecars(dbPath, { mode: 'read-only', logger, @@ -3408,8 +3441,8 @@ export const loadVectorExtension = async ( }; /** * Default stemmer for FTS indexes. Single source so the analyze path - * (`getSearchFTSStemmer`) and the read-only `createFTSIndex`/`ensureFTSIndex` - * defaults can never silently diverge. + * (`getSearchFTSStemmer`) and `createFTSIndex` defaults can never silently + * diverge. */ export const DEFAULT_FTS_STEMMER = 'porter'; @@ -3772,45 +3805,6 @@ export const ensureFtsRowDmlSafe = async ( return await loadFTSExtension(undefined, { policy: resolveAnalyzeInstallPolicy() }); }; -/** - * Lazy-create an FTS index, caching the fact in-process. - * - * Kept for writable maintenance paths that need to lazily materialize an - * index. Read-only query paths must not call this; production analysis owns - * creating the configured search indexes before the database is served. - * - * Safe to call repeatedly — the in-process Set guarantees only the first - * call hits LadybugDB. `closeLbug` clears the cache so re-init starts fresh. - * - * Defense in depth: if the active connection is read-only (e.g. the MCP - * pool adapter), `CREATE_FTS_INDEX` will fail with "Cannot execute write - * operations in a read-only database". Treat that as a no-op and cache - * the key so callers don't loop on a path that can never succeed here — - * the index is owned by `gitnexus analyze` (writable) and either already - * exists or will be created on the next analyze. - */ -export const ensureFTSIndex = async ( - tableName: string, - indexName: string, - properties: string[], - stemmer: string = DEFAULT_FTS_STEMMER, -): Promise => { - const key = ftsIndexKey(tableName, indexName); - if (ensuredFTSIndexes.has(key)) return; - try { - await createFTSIndex(tableName, indexName, properties, stemmer); - } catch (e) { - // Read-only DB: writable analyze owns index creation; silently skip - // and cache so callers don't loop on a path that can never succeed - // here (the MCP query pool opens DBs read-only by design). - if (isReadOnlyDbError(e)) { - ensuredFTSIndexes.add(key); - return; - } - throw e; - } -}; - export type FtsQueryFailureClass = 'missing-index' | 'missing-table' | 'other'; /** @@ -4068,7 +4062,15 @@ export const dropFTSIndex = async (tableName: string, indexName: string): Promis // extension binary is not re-inspected, falling back to a fresh structural // diagnosis when nothing recorded one. const ftsCapability = getFtsCapability(); - const { remedy } = ftsCapability?.diagnosis ?? diagnoseExtensionLoad(ftsCapability?.reason); + const inspectPath = extractExtensionPath(ftsCapability?.reason); + const { remedy } = + ftsCapability?.diagnosis ?? + diagnoseExtensionLoad( + ftsCapability?.reason, + 'FTS', + inspectPath, + resolveFtsVersionPair(inspectPath), + ); // Deliberately message-only: `remedy` is generated text (fixed system paths // at most), and LadybugDB's own path-bearing `reason` is NEVER interpolated // here — the #2374/#2375 redaction contract. diff --git a/gitnexus/src/core/lbug/native-check.ts b/gitnexus/src/core/lbug/native-check.ts index b70690691..7f4d81b3f 100644 --- a/gitnexus/src/core/lbug/native-check.ts +++ b/gitnexus/src/core/lbug/native-check.ts @@ -2,6 +2,8 @@ import fs from 'fs'; import path from 'path'; import { createRequire } from 'node:module'; import { spawnSync, type SpawnSyncReturns } from 'node:child_process'; +import { escapeCypherString } from './cypher-escape.js'; +import { resolveVendoredFtsPath } from './vendored-extension-path.js'; /** Cap the out-of-process native load probe so a hung filesystem cannot wedge a * CLI startup gate (same bounding rationale as the extension probe below). */ @@ -399,8 +401,18 @@ export interface FtsProbeResult { loaded: boolean; /** Collapsed LadybugDB error when `loaded` is false. */ reason?: string; + /** Policy `never` refused the probe — not a load failure. */ + suppressed?: boolean; } +export type FtsAvailabilityLabel = 'available' | 'unavailable' | 'suppressed'; + +export const ftsAvailabilityLabel = (probe: FtsProbeResult): FtsAvailabilityLabel => { + if (probe.loaded) return 'available'; + if (probe.suppressed) return 'suppressed'; + return 'unavailable'; +}; + /** Same shape for every optional extension; `FtsProbeResult` is the legacy name. */ export type ExtensionProbeResult = FtsProbeResult; @@ -441,10 +453,38 @@ const closeProbeResults = (result: unknown): void => { * cannot cancel an in-flight native call, so a future thread-blocking case * would need an out-of-process probe. */ +/** Local copy of the env policy parse — must not import extension-loader + * (that module statically pulls lbug-config, which would break doctor when + * the native addon is missing). */ +type ProbeInstallPolicy = 'auto' | 'load-only' | 'never'; + +const resolveProbeInstallPolicy = (): ProbeInstallPolicy => { + const raw = process.env.GITNEXUS_LBUG_EXTENSION_INSTALL; + if (raw === 'load-only' || raw === 'never' || raw === 'auto') return raw; + return 'load-only'; +}; + +export interface FtsProbeOptions { + policy?: ProbeInstallPolicy; + /** Injected vendored path. `null` skips path-LOAD; omit to resolve from disk. */ + vendoredPath?: string | null; +} + export async function probeFtsExtensionLoad( timeoutMs: number = DEFAULT_FTS_PROBE_TIMEOUT_MS, + opts?: FtsProbeOptions, ): Promise { - return await probeExtensionLoad('fts', timeoutMs); + const policy = opts?.policy ?? resolveProbeInstallPolicy(); + if (policy === 'never') { + return { + loaded: false, + suppressed: true, + reason: 'suppressed by policy GITNEXUS_LBUG_EXTENSION_INSTALL=never', + }; + } + const vendoredPath = + opts?.vendoredPath !== undefined ? opts.vendoredPath : resolveVendoredFtsPath(); + return await probeExtensionLoad('fts', timeoutMs, vendoredPath); } /** @@ -474,6 +514,7 @@ export async function probeVectorExtensionLoad( async function probeExtensionLoad( extension: 'fts' | 'vector', timeoutMs: number, + vendoredPath?: string | null, ): Promise { let timer: ReturnType | undefined; const timeout = new Promise((resolve) => { @@ -487,6 +528,12 @@ async function probeExtensionLoad( ); }); + const statements: string[] = []; + if (extension === 'fts' && vendoredPath) { + statements.push(`LOAD EXTENSION '${escapeCypherString(path.resolve(vendoredPath))}'`); + } + statements.push(`LOAD EXTENSION ${extension}`); + const probe = (async (): Promise => { try { const { default: lbug } = await import('@ladybugdb/core'); @@ -495,9 +542,18 @@ async function probeExtensionLoad( try { const conn = new lbug.Connection(db); try { - const result = await conn.query(`LOAD EXTENSION ${extension}`); - closeProbeResults(result); - return { loaded: true }; + let lastReason: string | undefined; + for (const sql of statements) { + try { + const result = await conn.query(sql); + closeProbeResults(result); + return { loaded: true }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + lastReason = message.replace(/\s+/g, ' ').trim(); + } + } + return { loaded: false, reason: lastReason }; } finally { await conn.close().catch(() => {}); } diff --git a/gitnexus/src/core/lbug/pool-adapter.ts b/gitnexus/src/core/lbug/pool-adapter.ts index cedc5ae45..09d28abf9 100644 --- a/gitnexus/src/core/lbug/pool-adapter.ts +++ b/gitnexus/src/core/lbug/pool-adapter.ts @@ -29,6 +29,8 @@ import { WAL_RECOVERY_SUGGESTION, } from './lbug-config.js'; import { + assertReadOnlyFtsCrashSafe, + FtsReaderUnrepairableError, guardWalQuarantine, isMissingFsError, isMissingShadowSidecarError, @@ -564,6 +566,9 @@ async function tryQuarantineForMissingShadow( // refuseLargeWalQuarantine (issue #2382 review, Finding B). Kept OUTSIDE the // try so the actionable recovery message propagates to the MCP caller rather // than being re-wrapped as a rename failure. + // Never pass crash evidence: the pool is a reader/MCP surface and must + // keep today's large-WAL refusal (R9). Analyze parks via the dirty-recovery + // family before it opens. await guardWalQuarantine(dbPath, opts.reason, opts.err, poolSidecarLogger); try { const quarantinePath = await quarantineWalForMissingShadow(dbPath, { @@ -625,6 +630,7 @@ async function openReadOnlyDatabase(dbPath: string): Promise { let db: lbug.Database | undefined; silenceStdout(); try { + await assertReadOnlyFtsCrashSafe(dbPath); await preflightLbugSidecars(dbPath, { mode: 'read-only', logger: poolSidecarLogger, @@ -842,6 +848,9 @@ async function doInitLbug(repoId: string, dbPath: string): Promise + `Cannot open ${path.basename(dbPath)} read-only after an in-place FTS abort. ` + + `The leftover WAL would replay and kill this process. ` + + `Run \`${FTS_READER_REPAIR_COMMAND}\` after stopping any GitNexus MCP or serve process.`; + +export class FtsReaderUnrepairableError extends Error { + readonly code = 'FTS_READER_UNREPAIRABLE' as const; + constructor(dbPath: string) { + super(ftsReaderRefuseMessage(dbPath)); + this.name = 'FtsReaderUnrepairableError'; + } +} + +const sidecarHasLiveWal = (state: LbugSidecarState): boolean => + state.kind === 'orphan-wal' || + state.kind === 'tiny-orphan-wal' || + state.kind === 'wal-with-shadow'; + +/** + * Advisory reader gate (KTD10 / R9b). When meta names an in-place FTS + * abort (live dirty flag, or a persisted in-place `native-abort`) and a + * WAL is still live, refuse before the native open. Does not write, + * rename, or repair. Parking still requires the conjunctive + * `allowsFtsCrashWalPark` warrant. Missing or unreadable meta falls + * through to today's open path. + */ +export const assertReadOnlyFtsCrashSafe = async (dbPath: string): Promise => { + let meta; + try { + meta = await loadMeta(path.dirname(dbPath)); + } catch { + return; + } + if (!meta || !shouldRefuseFtsCrashWal(meta.incrementalInProgress, meta.capabilities?.fts)) { + return; + } + const state = await inspectLbugSidecars(dbPath); + if (!sidecarHasLiveWal(state)) return; + throw new FtsReaderUnrepairableError(dbPath); +}; + +export const ftsCrashParkFailureMessage = (failedPath: string, err?: unknown): string => { + const detail = err instanceof Error ? err.message : err != null ? String(err) : ''; + return ( + `Cannot park ${path.basename(failedPath)} after an in-place FTS abort` + + (detail ? ` (${detail})` : '') + + `. The database was not opened. Run \`${CLEAN_LBUG_SIDECARS_COMMAND}\` ` + + 'after stopping any GitNexus MCP or serve process, then retry ' + + '`gitnexus analyze` or `gitnexus analyze --repair-fts`.' + ); +}; + /** * Counter-based warn anti-spam (PR #1747 review, Finding 6). * @@ -297,6 +364,7 @@ export const guardWalQuarantine = async ( mode: string, triggeringErr: unknown, logger: SidecarRecoveryLogger, + crashEvidence?: WalCrashEvidence, ): Promise => { const state = await inspectLbugSidecars(dbPath); if (state.kind === 'wal-with-shadow') { @@ -310,6 +378,15 @@ export const guardWalQuarantine = async ( throw new Error(presentShadowUnreachableMessage(dbPath, triggeringErr)); } if (state.kind === 'orphan-wal') { + if (crashEvidence?.kind === 'fts-inplace-checkpointed') { + const { failed } = await quarantineSidecarsForDirtyRecovery(dbPath, (message) => + logger.warn(message), + ); + if (failed.length > 0) { + throw new Error(ftsCrashParkFailureMessage(failed[0]!)); + } + return; + } warnOnce( logger, `${dbPath}:large-wal-refuse:${mode}`, @@ -551,6 +628,12 @@ const dirtyRecoveryParkedNames = (dbPath: string): string[] => * every subsequent open this run performs is replay-free — or the entry is * in `failed` and the caller MUST abort before any DB open. * + * Retention: these parks are not reclaimed on the next writable open + * (unlike missing-shadow quarantines). FTS-phase parks stay until + * `gitnexus clean --lbug-sidecars` or the next park overwrites the same + * fixed `.dirty-recovery` name. That is intentional — the parked bytes + * are the only forensic copy of a proven in-place abort. + * * @returns `moved` — destination paths now holding the parked bytes; * `removed` — source sidecars whose bytes are GONE (forensics lost, replay * risk eliminated); `failed` — source sidecars still in place: a diff --git a/gitnexus/src/core/lbug/vendored-extension-path.ts b/gitnexus/src/core/lbug/vendored-extension-path.ts new file mode 100644 index 000000000..ce5706981 --- /dev/null +++ b/gitnexus/src/core/lbug/vendored-extension-path.ts @@ -0,0 +1,132 @@ +import { existsSync, readFileSync, realpathSync } from 'node:fs'; +import path from 'node:path'; +import { VENDOR_ROOT } from '../vendor-root.js'; + +/** + * Resolve the packaged FTS extension for this process's Node platform tuple. + * + * Lives here (not in extension-loader) so the doctor startup probe can share + * the same path without importing the loader, which statically pulls lbug-config. + */ +const DEFAULT_FILENAME = 'libfts.lbug_extension'; + +export const defaultVendorRoot = (): string => VENDOR_ROOT; + +export const nodePlatformTuple = ( + platform: NodeJS.Platform = process.platform, + arch: string = process.arch, +): string => `${platform}-${arch}`; + +export interface FtsArtifactManifest { + coreVersion?: string; + extensionVersion?: string; + filename?: string; + unsupportedTuples?: Array<{ tuple: string; reason?: string }>; +} + +const asOptionalString = (value: unknown): string | undefined => + typeof value === 'string' && value.length > 0 ? value : undefined; + +/** Drop non-array / non-object entries so `.some(entry => entry.tuple)` cannot throw. */ +const asUnsupportedTuples = (value: unknown): FtsArtifactManifest['unsupportedTuples'] => { + if (!Array.isArray(value)) return undefined; + const entries: Array<{ tuple: string; reason?: string }> = []; + for (const entry of value) { + if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) continue; + const tuple = (entry as { tuple?: unknown }).tuple; + if (typeof tuple !== 'string' || tuple.length === 0) continue; + const reason = (entry as { reason?: unknown }).reason; + entries.push({ + tuple, + ...(typeof reason === 'string' ? { reason } : {}), + }); + } + return entries; +}; + +export const readFtsArtifactManifest = ( + vendorRoot: string = defaultVendorRoot(), +): FtsArtifactManifest => { + const manifestPath = path.join(vendorRoot, 'lbug-fts', 'manifest.json'); + try { + const parsed: unknown = JSON.parse(readFileSync(manifestPath, 'utf8')); + if (parsed !== null && typeof parsed === 'object' && !Array.isArray(parsed)) { + const rec = parsed as Record; + return { + coreVersion: asOptionalString(rec.coreVersion), + extensionVersion: asOptionalString(rec.extensionVersion), + filename: asOptionalString(rec.filename), + unsupportedTuples: asUnsupportedTuples(rec.unsupportedTuples), + }; + } + return {}; + } catch { + return {}; + } +}; + +export const isUnsupportedFtsTuple = ( + tuple: string, + vendorRoot: string = defaultVendorRoot(), +): boolean => + (readFtsArtifactManifest(vendorRoot).unsupportedTuples ?? []).some( + (entry) => entry.tuple === tuple, + ); + +/** Relative-path containment — not a prefix match (rejects `vendor-evil`). */ +export const isPathInsideRoot = (root: string, candidate: string): boolean => { + const relative = path.relative(root, candidate); + if (path.isAbsolute(relative)) return false; + return relative !== '' && !relative.startsWith(`..${path.sep}`) && relative !== '..'; +}; + +export const validateVendoredExtensionPath = ( + candidate: string, + vendorRoot: string, +): string | null => { + let realFile: string; + let realRoot: string; + try { + realFile = realpathSync(candidate); + realRoot = realpathSync(vendorRoot); + } catch { + return null; + } + if (!/\.lbug_extension$/i.test(realFile)) return null; + if (!isPathInsideRoot(realRoot, realFile)) return null; + return realFile; +}; + +export const inferExtensionVersionFromPath = ( + filePath: string | null | undefined, +): string | undefined => { + if (!filePath) return undefined; + const home = /[/\\]extension[/\\](\d+\.\d+\.\d+)[/\\]/.exec(filePath); + return home?.[1]; +}; + +export const resolveFtsVersionPair = ( + inspectPath?: string | null, + vendorRoot?: string, +): { expected?: string; found?: string } => { + // Ladybug's home path is `~/.lbdb/extension//…`. Compare that + // directory to the packaged core pin, not the (often different) artifact + // version, or a matching runtime looks skewed. + const expected = readFtsArtifactManifest(vendorRoot).coreVersion; + const found = inferExtensionVersionFromPath(inspectPath); + return { expected, found }; +}; + +export const resolveVendoredFtsPath = (opts?: { + tuple?: string; + vendorRoot?: string; + filename?: string; +}): string | null => { + const vendorRoot = opts?.vendorRoot ?? defaultVendorRoot(); + const tuple = opts?.tuple ?? nodePlatformTuple(); + const filename = + opts?.filename ?? readFtsArtifactManifest(vendorRoot).filename ?? DEFAULT_FILENAME; + const candidate = path.resolve(vendorRoot, 'lbug-fts', 'prebuilds', tuple, filename); + if (!existsSync(candidate)) return null; + return validateVendoredExtensionPath(candidate, vendorRoot); +}; diff --git a/gitnexus/src/core/lbug/wal-checkpoint-driver.ts b/gitnexus/src/core/lbug/wal-checkpoint-driver.ts index 09665127f..6d7669757 100644 --- a/gitnexus/src/core/lbug/wal-checkpoint-driver.ts +++ b/gitnexus/src/core/lbug/wal-checkpoint-driver.ts @@ -130,11 +130,15 @@ export const runCheckpointWithRetry = async ( * * Honors the `GITNEXUS_WAL_MANUAL_CHECKPOINT=0` opt-out so operators can * disable the manual path if it ever interacts badly with a future - * Ladybug release. + * Ladybug release. Returns true only when a CHECKPOINT actually flushed + * (`tryFlushWAL` → true). Opt-out and a no-op flush (no open connection) + * both return false so an FTS park warrant cannot treat a skipped + * checkpoint as success. */ -export const checkpointOnce = async (): Promise => { - if (!isManualCheckpointEnabled()) return; - await runCheckpointWithRetry(); +export const checkpointOnce = async (): Promise => { + if (!isManualCheckpointEnabled()) return false; + const { flushed } = await runCheckpointWithRetry(); + return flushed; }; /** Default cadence (ms) for the periodic driver. */ diff --git a/gitnexus/src/core/run-analyze.ts b/gitnexus/src/core/run-analyze.ts index ce98c98ed..d0d497e4c 100644 --- a/gitnexus/src/core/run-analyze.ts +++ b/gitnexus/src/core/run-analyze.ts @@ -19,6 +19,17 @@ import { DEFAULT_VECTOR_SEARCH_CAPABILITY, type FtsSkipReason, } from './search/fts-policy.js'; +import { + allowsFtsCrashWalPark, + buildFtsDirtyStamp, + inferNativeAbortSkip, + isBoundaryCheckpointFatal, + isFtsStagingDirty, + resolveFtsWritePlan, + shouldRefuseFtsCrashWal, + shouldRefuseRepairFtsWhileDirty, + shouldStampFtsDirtyPhase, +} from './search/fts-crash-marker.js'; import { PDG_EDGE_TYPES } from './lbug/pdg-emit-sink.js'; import path from 'path'; import fs from 'fs/promises'; @@ -108,13 +119,19 @@ import { getFtsCapability, resolveAnalyzeInstallPolicy, } from './lbug/extension-loader.js'; -import { diagnoseExtensionLoad } from './lbug/extension-load-error.js'; +import { + diagnoseExtensionLoad, + extractExtensionPath, + usesClassifiedLoadRemedy, +} from './lbug/extension-load-error.js'; +import { resolveFtsVersionPair } from './lbug/vendored-extension-path.js'; import { startWalCheckpointDriver, checkpointOnce, type WalCheckpointDriver, } from './lbug/wal-checkpoint-driver.js'; import { + ftsCrashParkFailureMessage, quarantineSidecarsForDirtyRecovery, inspectLbugSidecars, } from './lbug/sidecar-recovery.js'; @@ -590,8 +607,10 @@ export interface AnalyzeResult { * `extension-unavailable` (the LadybugDB FTS extension could not load — the * offline-first case, remedied by installing it) vs `build-failed` (the * extension loaded but the index build/verify failed non-fatally — remedied by - * `--repair-fts`, not by installing the extension). Lets the CLI show the - * correct recovery hint instead of always blaming a missing extension. + * `--repair-fts`, not by installing the extension) vs `native-abort` (inferred + * on the next run from an FTS-phase crash) vs `tuple-missing` (no packaged + * artifact for this platform). Lets the CLI show the correct recovery hint + * instead of always blaming a missing extension. * `disabled-by-flag` and `disabled-by-env` record intentional opt-out; * neither calls for extension installation or repair. */ @@ -1268,6 +1287,13 @@ async function runFullAnalysisInner( const existingMeta = loadedMeta ? withExplicitFtsDisablement(loadedMeta, ftsDisabledReason) : undefined; + // KTD6: the dying process writes nothing. Infer skip from the FTS-phase + // dirty flag (or a persisted native-abort skipReason) BEFORE later + // saveMeta calls overwrite the on-disk phase. + const priorFtsNativeAbort = inferNativeAbortSkip( + existingMeta?.incrementalInProgress, + existingMeta?.capabilities?.fts?.skipReason, + ); // Claim a fresh, ownership-validated slot before the pipeline writes caches. // A later registry-name collision or pipeline failure can otherwise leave @@ -1284,6 +1310,7 @@ async function runFullAnalysisInner( } // ── FTS-only repair path ──────────────────────────────────────────── + const requestedRepairFts = Boolean(options.repairFts); if ( options.repairFts && existingMeta && @@ -1301,13 +1328,17 @@ async function runFullAnalysisInner( 'Run `gitnexus analyze` first to create the initial index, then retry `--repair-fts`.', ); } - if (existingMeta.incrementalInProgress) { - // #2409 / tri-review 4669518496 (R6): a dirty flag means the previous - // run died mid-writeback — the graph may be half-written and its WAL - // possibly poisoned. This branch returns early, so the dirty-recovery - // sidecar quarantine below would never run: repairing FTS now would - // open the DB and replay that WAL pre-quarantine, and even a - // survivable open would certify FTS over a half-written graph. + if (shouldRefuseRepairFtsWhileDirty(existingMeta.incrementalInProgress)) { + // #2409 / tri-review 4669518496 (R6): a non-FTS dirty flag means the + // previous run died mid-writeback — the graph may be half-written and + // its WAL possibly poisoned. This branch returns early, so the + // dirty-recovery sidecar quarantine below would never run: repairing + // FTS now would open the DB and replay that WAL pre-quarantine, and + // even a survivable open would certify FTS over a half-written graph. + // An FTS-phase flag with a successful checkpoint is different (KTD4): + // the graph-boundary checkpoint already ran, so `--repair-fts` must + // stay usable (R8). Missing/failed checkpoint is treated like a + // half-written graph. throw new Error( 'Cannot repair FTS indexes: the index is mid-incremental-recovery ' + '(a previous analyze run did not complete cleanly). ' + @@ -1344,6 +1375,22 @@ async function runFullAnalysisInner( ); } await ensureWritableStorage(); + // P1 R8: park a poisoned live WAL before opening. Staging never parks — + // the live index next to an unpublished staging file must replay its WAL. + const repairDirty = existingMeta.incrementalInProgress; + if (shouldRefuseFtsCrashWal(repairDirty, existingMeta.capabilities?.fts)) { + const { + moved: repairParked, + removed: repairRemoved, + failed: repairParkFailed, + } = await quarantineSidecarsForDirtyRecovery(lbugPath, (message) => log(` ${message}`)); + if (repairParkFailed.length > 0) { + throw new Error(ftsCrashParkFailureMessage(repairParkFailed[0]!)); + } + if (repairParked.length + repairRemoved.length > 0) { + log('Parked leftover WAL/shadow from the previous in-place FTS abort before --repair-fts.'); + } + } try { await initAnalysisLbug(lbugPath); // Gate on FTS availability BEFORE touching any index. createSearchFTSIndexes @@ -1371,12 +1418,17 @@ async function runFullAnalysisInner( // by re-installing — the file is already present. Route that class to the // classified remedy (install VC++ redist / OpenSSL) instead of the old // "retry the network install" text that trapped the user in a loop. - const { kind, remedy } = diagnoseExtensionLoad(rawFtsReason); - const remedyTail = - kind === 'missing_dependency' - ? ` ${remedy}` - : '. Retry with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to install it, ' + - 'or pre-install the extension file; run `gitnexus doctor` for live FTS status.'; + const inspectPath = extractExtensionPath(rawFtsReason); + const { kind, remedy } = diagnoseExtensionLoad( + rawFtsReason, + 'FTS', + inspectPath, + resolveFtsVersionPair(inspectPath), + ); + const remedyTail = usesClassifiedLoadRemedy(kind) + ? ` ${remedy}` + : '. Retry with network access and GITNEXUS_LBUG_EXTENSION_INSTALL=auto to install it, ' + + 'or pre-install the extension file; run `gitnexus doctor` for live FTS status.'; throw new Error( 'Cannot repair FTS indexes: the LadybugDB FTS extension failed to load' + (ftsReason ? ` — ${ftsReason}` : '') + @@ -1384,6 +1436,25 @@ async function runFullAnalysisInner( ); } progress('fts', 85, 'Repairing search indexes...'); + // Restamp before CREATE so a second native abort still has in-place + // FTS dirty evidence after persist cleared the first stamp. + try { + const latestBeforeCreate = (await loadMeta(metaDir)) ?? existingMeta; + await saveMeta(metaDir, { + ...latestBeforeCreate, + incrementalInProgress: buildFtsDirtyStamp({ + prior: latestBeforeCreate.incrementalInProgress, + writePlan: 'in-place', + checkpointSucceeded: true, + }), + }); + } catch (err) { + log( + `FTS dirty restamp write failed (non-critical, continuing with repair${ + err instanceof Error ? `: ${err.message}` : '' + }).`, + ); + } const repairFailures = await createSearchFTSIndexes({ indexes: ftsIndexes, onIndexStart: options.verbose @@ -1433,6 +1504,7 @@ async function runFullAnalysisInner( const latestMeta = (await loadMeta(metaDir)) ?? existingMeta; await saveMeta(metaDir, { ...latestMeta, + incrementalInProgress: undefined, capabilities: { graph: latestMeta.capabilities?.graph ?? DEFAULT_GRAPH_CAPABILITY, fts: { provider: 'ladybugdb-fts', status: 'available' }, @@ -1519,10 +1591,12 @@ async function runFullAnalysisInner( } } - // ── Crash recovery: dirty flag forces full rebuild ──────────────── - // If the previous incremental run set incrementalInProgress and didn't - // clear it, the on-disk index may be in a half-state. Cheapest path - // back to a known-good index is to wipe + rebuild from scratch. + // ── Crash recovery ──────────────────────────────────────────────── + // A non-FTS dirty flag (or an FTS abort that never checkpointed) still + // forces a wipe + rebuild. An in-place FTS abort AFTER a successful + // graph-boundary checkpoint (Windows full rebuild, POSIX incremental) + // parks the live WAL and keeps the graph. A staging FTS abort never + // parks the live WAL — that index must replay its own delta. if (existingMeta?.incrementalInProgress) { const dirty = existingMeta.incrementalInProgress; const dirtyDetails = @@ -1548,52 +1622,92 @@ async function runFullAnalysisInner( .filter(Boolean) .join(', ') : 'legacy dirty flag'; - log( - // "analyze run", not "incremental run" — since #2099 F1 the flag is a - // generic dirty marker written by BOTH writeback branches. - 'Previous analyze run did not complete cleanly (incrementalInProgress flag set); ' + - `last dirty state: ${dirtyDetails}; ` + - 'forcing full rebuild to restore a known-good index.', - ); - options = { ...options, force: true }; - // Reload meta after clearing the flag in-memory; we still want fileHashes - // for the post-rebuild meta carry-over, but force=true ensures the - // rebuild path executes. - // - // #2409 defect 2: the crashed writeback's WAL can be poisoned — replaying - // it kills the process natively, and the first DB open of this recovery - // run (the embedding-cache preservation open below) happens BEFORE the - // rebuild wipe that would discard it. Park the WAL/shadow sidecars aside - // now, while nothing is open, so every open in this run is replay-free. - // The rebuild wipes the DB regardless, so no committed data is at stake. - await ensureWritableStorage(); - const { removed, failed } = await quarantineSidecarsForDirtyRecovery(lbugPath, log); - if (removed.length > 0) { + + const persistFtsNativeAbortRecovery = async (): Promise => { + existingMeta.incrementalInProgress = undefined; + existingMeta.capabilities = { + ...existingMeta.capabilities, + graph: existingMeta.capabilities?.graph ?? DEFAULT_GRAPH_CAPABILITY, + fts: { + provider: existingMeta.capabilities?.fts?.provider ?? 'ladybugdb-fts', + status: 'unavailable', + skipReason: 'native-abort', + ...(dirty.writePlan ? { writePlan: dirty.writePlan } : {}), + }, + vectorSearch: existingMeta.capabilities?.vectorSearch ?? DEFAULT_VECTOR_SEARCH_CAPABILITY, + }; + await saveMeta(metaDir, existingMeta); + }; + + if (allowsFtsCrashWalPark(dirty)) { log( - `Dirty-state recovery discarded ${removed.map((p) => path.basename(p)).join(', ')} ` + - 'from the interrupted run (the file could not be moved aside, so its bytes were ' + - 'removed — post-mortem forensics lost). Recovery proceeds with full embedding ' + - 'preservation.', + 'Previous analyze run aborted during in-place FTS after a successful graph checkpoint ' + + `(${dirtyDetails}); parking leftover WAL/shadow so the existing graph can reopen. ` + + 'Search indexes stay skipped until `gitnexus analyze --repair-fts`.', ); - } - if (failed.length > 0) { - // FIX 1 (this shipping review, replacing the tri-review 4669518496 - // P2-3 drop-shape design): under a persistent lock the old drop-shape - // run derived its embedding mode as "drop", ran the WHOLE pipeline, - // and then died at the rebuild wipe on the very same handle — wasting - // minutes and zeroing embeddings on the way. A possibly-poisoned - // sidecar still sits next to the DB (any pre-wipe open would replay it - // and die), so failing here, in seconds, with the same actionable - // typed error the wipe would eventually throw is strictly better — - // and the CLI's LbugWipeError handler already renders it - // (recoveryHint 'lbug-wipe-failed'). The message is self-contained - // (headline + paths + lock guidance) because serve forwards only - // err.message over worker IPC. - throw new LbugWipeError(failed, { - headline: - "Cannot start dirty-state recovery — the interrupted run's LadybugDB sidecars " + - 'could neither be moved aside nor removed:', - }); + await ensureWritableStorage(); + const { failed } = await quarantineSidecarsForDirtyRecovery(lbugPath, (message) => + log(` ${message}`), + ); + if (failed.length > 0) { + throw new Error(ftsCrashParkFailureMessage(failed[0]!)); + } + await persistFtsNativeAbortRecovery(); + } else if (isFtsStagingDirty(dirty)) { + log( + 'Previous analyze run aborted during FTS after a staging writeback ' + + `(${dirtyDetails}); leaving the live index WAL in place so the unpublished ` + + 'staging file can still replay. Not forcing a rebuild.', + ); + await persistFtsNativeAbortRecovery(); + } else { + log( + // "analyze run", not "incremental run" — since #2099 F1 the flag is a + // generic dirty marker written by BOTH writeback branches. + 'Previous analyze run did not complete cleanly (incrementalInProgress flag set); ' + + `last dirty state: ${dirtyDetails}; ` + + 'forcing full rebuild to restore a known-good index.', + ); + options = { ...options, force: true }; + // Reload meta after clearing the flag in-memory; we still want fileHashes + // for the post-rebuild meta carry-over, but force=true ensures the + // rebuild path executes. + // + // #2409 defect 2: the crashed writeback's WAL can be poisoned — replaying + // it kills the process natively, and the first DB open of this recovery + // run (the embedding-cache preservation open below) happens BEFORE the + // rebuild wipe that would discard it. Park the WAL/shadow sidecars aside + // now, while nothing is open, so every open in this run is replay-free. + // The rebuild wipes the DB regardless, so no committed data is at stake. + await ensureWritableStorage(); + const { removed, failed } = await quarantineSidecarsForDirtyRecovery(lbugPath, log); + if (removed.length > 0) { + log( + `Dirty-state recovery discarded ${removed.map((p) => path.basename(p)).join(', ')} ` + + 'from the interrupted run (the file could not be moved aside, so its bytes were ' + + 'removed — post-mortem forensics lost). Recovery proceeds with full embedding ' + + 'preservation.', + ); + } + if (failed.length > 0) { + // FIX 1 (this shipping review, replacing the tri-review 4669518496 + // P2-3 drop-shape design): under a persistent lock the old drop-shape + // run derived its embedding mode as "drop", ran the WHOLE pipeline, + // and then died at the rebuild wipe on the very same handle — wasting + // minutes and zeroing embeddings on the way. A possibly-poisoned + // sidecar still sits next to the DB (any pre-wipe open would replay it + // and die), so failing here, in seconds, with the same actionable + // typed error the wipe would eventually throw is strictly better — + // and the CLI's LbugWipeError handler already renders it + // (recoveryHint 'lbug-wipe-failed'). The message is self-contained + // (headline + paths + lock guidance) because serve forwards only + // err.message over worker IPC. + throw new LbugWipeError(failed, { + headline: + "Cannot start dirty-state recovery — the interrupted run's LadybugDB sidecars " + + 'could neither be moved aside nor removed:', + }); + } } } @@ -2034,6 +2148,9 @@ async function runFullAnalysisInner( stats: existingMeta.stats ?? {}, alreadyUpToDate: true, ...(ftsDisabledReason ? { ftsSkipped: true, ftsSkipReason: ftsDisabledReason } : {}), + ...(!ftsDisabledReason && priorFtsNativeAbort + ? { ftsSkipped: true, ftsSkipReason: 'native-abort' } + : {}), isPrimaryBranch: !placement.branch, }; } @@ -2990,7 +3107,15 @@ async function runFullAnalysisInner( : undefined, ] .filter((e): e is { reason: string | undefined; label: string } => e !== undefined) - .map(({ reason, label }) => diagnoseExtensionLoad(reason, label).remedy); + .map(({ reason, label }) => { + const inspectPath = extractExtensionPath(reason); + return diagnoseExtensionLoad( + reason, + label, + inspectPath, + label === 'FTS' ? resolveFtsVersionPair(inspectPath) : undefined, + ).remedy; + }); log( `Incremental: ${escalationCauses.join('; and ')} — switching to a full DB write ` + `(wipe + bulk COPY) for this run; file-level incremental bookkeeping is unaffected.` + @@ -3266,13 +3391,6 @@ async function runFullAnalysisInner( await restoreDerivedRels(derivedSnapshot); } } - - // Boundary drain (#2409): checkpoint at the end of the incremental - // writeback so the WAL it accumulated never lingers into the FTS and - // embedding phases — a later crash leaves only post-checkpoint WAL for - // the next open to replay. Near-instant when the periodic driver has - // kept up; rides the driver's bounded retry via runCheckpointWithRetry. - await checkpointOnce(); } else { // ── Full rebuild ─────────────────────────────────────────────── // Pass the streamed PDG-emit manifest (#2202) so the BasicBlock layer that @@ -3294,6 +3412,43 @@ async function runFullAnalysisInner( ); } + // Converged graph-boundary drain: incremental used to checkpoint here; + // full rebuild did not. One site so FTS always starts after a settled + // plan (post-escalation `buildPath`) and a recorded checkpoint outcome. + const ftsWritePlan = resolveFtsWritePlan(buildPath, lbugPath); + let boundaryCheckpointSucceeded = false; + try { + boundaryCheckpointSucceeded = await checkpointOnce(); + } catch (error) { + if (isBoundaryCheckpointFatal(ftsWritePlan)) { + throw error; + } + const detail = error instanceof Error ? error.message : String(error); + log( + `Boundary WAL checkpoint failed on the in-place FTS path (best-effort): ${detail}. ` + + 'Continuing; recovery will treat the graph-boundary checkpoint as unsuccessful.', + ); + } + if (shouldStampFtsDirtyPhase(ftsWritePlan)) { + // Lift the prior-meta precondition: a first-ever in-place run (Windows + // full rebuild, or any in-place incremental) must stamp too. Staging + // never stamps — an abort there abandons the unpublished file. + const latestMeta = (await loadMeta(metaDir)) ?? existingMeta; + const base: RepoMeta = latestMeta ?? { + repoPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + }; + await saveMeta(metaDir, { + ...base, + incrementalInProgress: buildFtsDirtyStamp({ + prior: base.incrementalInProgress, + writePlan: 'in-place', + checkpointSucceeded: boundaryCheckpointSucceeded, + }), + }); + } + // ── Phase 3: FTS (85–90%) ───────────────────────────────────────── // The analyze (write) path owns building the search indexes, so it uses // the `auto` install policy (LOAD-first, then one bounded INSTALL) — @@ -3322,12 +3477,30 @@ async function runFullAnalysisInner( let ftsReady = ftsAvailable; // Why FTS ended up skipped (#2658 review L2): an explicit opt-out // (`disabled-by-flag` / `disabled-by-env`, #3091) when one was recorded, + // else tuple-missing when the loader reported no packaged artifact, // else extension-unavailable up front, or build-failed in the degrade // branch below. + const tupleMissing = + !ftsAvailable && + !ftsDisabledReason && + /no packaged FTS artifact/i.test(getFtsCapability()?.reason ?? ''); let ftsSkipReason: FtsSkipReason | undefined = ftsAvailable ? undefined - : (ftsDisabledReason ?? 'extension-unavailable'); - if (ftsAvailable) { + : (ftsDisabledReason ?? (tupleMissing ? 'tuple-missing' : 'extension-unavailable')); + if (ftsAvailable && priorFtsNativeAbort && !requestedRepairFts) { + // KTD6 / R11: do not retry CREATE_FTS_INDEX after a native abort — + // the same content can kill the process again. `--repair-fts` is the + // explicit retry: its early-return path never reaches this branch, + // and a retention-mismatch rewrite that started as `--repair-fts` + // still creates indexes. + ftsReady = false; + ftsSkipReason = 'native-abort'; + log( + 'FTS index build skipped — a previous analyze aborted while building search indexes. ' + + 'Graph analysis completed. Run `gitnexus analyze --repair-fts` to retry.', + ); + progress('fts', 90, 'Search indexes skipped (previous native abort)'); + } else if (ftsAvailable) { // Degrade rather than throw: createSearchFTSIndexes re-tokenizes every // stored row on every run, so a native tokenizer error on a single // pre-existing row (#2544/#2546) must not discard this run's otherwise- @@ -3379,9 +3552,15 @@ async function runFullAnalysisInner( // Same #2383 mock seam as the repair path above — keep the exported // `getExtensionCapabilities()` lookup here. const ftsReason = getExtensionCapabilities().find((c) => c.name === 'fts')?.reason; - const { kind, remedy } = diagnoseExtensionLoad(ftsReason); + const inspectPath = extractExtensionPath(ftsReason); + const { kind, remedy } = diagnoseExtensionLoad( + ftsReason, + 'FTS', + inspectPath, + resolveFtsVersionPair(inspectPath), + ); log( - kind === 'missing_dependency' + usesClassifiedLoadRemedy(kind) ? `${FTS_UNAVAILABLE_LEAD} ${remedy}` : FTS_UNAVAILABLE_MESSAGE, ); @@ -4147,6 +4326,14 @@ async function runFullAnalysisInner( // 'unavailable', and the next run does it again. Stamping the // discriminator the run already computed makes the read exact instead. skipReason: ftsReady ? undefined : ftsSkipReason, + // Keep the abort write plan after persist cleared the dirty flag + // so a leftover live WAL is still refuse-able. Successful FTS + // drops it with skipReason. + ...(!ftsReady && + ftsSkipReason === 'native-abort' && + existingMeta?.capabilities?.fts?.writePlan + ? { writePlan: existingMeta.capabilities.fts.writePlan } + : {}), }, vectorSearch: { provider: effectiveSemanticMode === 'vector-index' ? 'ladybugdb-vector' : 'exact-scan', diff --git a/gitnexus/src/core/search/fts-crash-marker.ts b/gitnexus/src/core/search/fts-crash-marker.ts new file mode 100644 index 000000000..6495583fe --- /dev/null +++ b/gitnexus/src/core/search/fts-crash-marker.ts @@ -0,0 +1,115 @@ +/** + * FTS-phase dirty-flag policy (KTD4 / KTD6). + * + * A native abort during CREATE_FTS_INDEX kills the process before JS can + * persist a skip reason. The next run infers `native-abort` from this phase + * value, or from a persisted `capabilities.fts.skipReason` of `native-abort` + * after recovery clears the dirty flag. Tests induce the flag through + * saveMeta — they cannot be a green real-abort of analyze. + */ +import type { RepoMeta } from '../../storage/repo-meta.js'; + +export const FTS_DIRTY_PHASE = 'fts' as const; + +export type FtsWritePlan = 'in-place' | 'staging'; + +export type IncrementalDirtyState = NonNullable; + +export const resolveFtsWritePlan = (buildPath: string, livePath: string): FtsWritePlan => + buildPath === livePath ? 'in-place' : 'staging'; + +export const shouldStampFtsDirtyPhase = (writePlan: FtsWritePlan): boolean => + writePlan === 'in-place'; + +/** Staging throws (abandons the unpublished file). In-place is best-effort. */ +export const isBoundaryCheckpointFatal = (writePlan: FtsWritePlan): boolean => + writePlan === 'staging'; + +export const isFtsDirtyPhase = ( + dirty: RepoMeta['incrementalInProgress'] | undefined, +): dirty is IncrementalDirtyState & { phase: typeof FTS_DIRTY_PHASE } => + dirty?.phase === FTS_DIRTY_PHASE; + +/** + * Half-written graph, or FTS-phase without a successful checkpoint, blocks + * `--repair-fts`. FTS-phase + `checkpointSucceeded === true` is admitted + * (in-place or staging). Staging still must not park. + */ +export const shouldRefuseRepairFtsWhileDirty = ( + dirty: RepoMeta['incrementalInProgress'] | undefined, +): boolean => dirty != null && (!isFtsDirtyPhase(dirty) || dirty.checkpointSucceeded !== true); + +export const inferNativeAbortSkip = ( + dirty: RepoMeta['incrementalInProgress'] | undefined, + skipReason?: string, +): boolean => isFtsDirtyPhase(dirty) || skipReason === 'native-abort'; + +/** + * KTD5 conjunctive warrant for parking a live WAL: FTS phase, in-place + * write plan (Windows full rebuild, POSIX incremental, escalated-in-place), + * and a successful graph-boundary checkpoint. Staging never qualifies — + * the live index next to an unpublished staging file must keep its WAL. + */ +export const allowsFtsCrashWalPark = ( + dirty: RepoMeta['incrementalInProgress'] | undefined, +): boolean => + isFtsDirtyPhase(dirty) && dirty.writePlan === 'in-place' && dirty.checkpointSucceeded === true; + +export const isInPlaceFtsDirty = ( + dirty: RepoMeta['incrementalInProgress'] | undefined, +): dirty is IncrementalDirtyState & { phase: typeof FTS_DIRTY_PHASE; writePlan: 'in-place' } => + isFtsDirtyPhase(dirty) && dirty.writePlan === 'in-place'; + +/** Persisted FTS capability fields used as crash evidence after the dirty flag is cleared. */ +export type PersistedFtsCrashEvidence = { + skipReason?: string; + writePlan?: FtsWritePlan; +}; + +export const hasRecoveredInPlaceFtsAbort = (fts: PersistedFtsCrashEvidence | undefined): boolean => + fts?.skipReason === 'native-abort' && fts.writePlan === 'in-place'; + +/** + * Reader / `--repair-fts` refuse-or-park warrant. Any in-place FTS dirty + * flag is enough — a failed graph-boundary checkpoint still leaves CREATE + * able to abort with a live WAL. After persist clears the flag, a + * `native-abort` skip plus persisted `writePlan: 'in-place'` is the same + * evidence. Staging persist also writes `native-abort` and must not match. + */ +export const shouldRefuseFtsCrashWal = ( + dirty: RepoMeta['incrementalInProgress'] | undefined, + fts?: PersistedFtsCrashEvidence, +): boolean => isInPlaceFtsDirty(dirty) || hasRecoveredInPlaceFtsAbort(fts); + +export const isFtsStagingDirty = (dirty: RepoMeta['incrementalInProgress'] | undefined): boolean => + isFtsDirtyPhase(dirty) && dirty.writePlan === 'staging'; + +export const buildFtsDirtyStamp = (args: { + prior?: IncrementalDirtyState; + now?: number; + writePlan: 'in-place'; + checkpointSucceeded: boolean; +}): IncrementalDirtyState => { + const now = args.now ?? Date.now(); + const prior = args.prior; + return { + startedAt: prior?.startedAt ?? now, + updatedAt: now, + toWriteCount: prior?.toWriteCount ?? 0, + phase: FTS_DIRTY_PHASE, + writePlan: args.writePlan, + checkpointSucceeded: args.checkpointSucceeded, + ...(prior?.directWriteCount !== undefined ? { directWriteCount: prior.directWriteCount } : {}), + ...(prior?.importerExpansion !== undefined + ? { importerExpansion: prior.importerExpansion } + : {}), + ...(prior?.effectiveWriteCount !== undefined + ? { effectiveWriteCount: prior.effectiveWriteCount } + : {}), + ...(prior?.deleteCount !== undefined ? { deleteCount: prior.deleteCount } : {}), + ...(prior?.shadowSeedCount !== undefined ? { shadowSeedCount: prior.shadowSeedCount } : {}), + ...(prior?.droppedImporterChunks !== undefined + ? { droppedImporterChunks: prior.droppedImporterChunks } + : {}), + }; +}; diff --git a/gitnexus/src/core/search/fts-indexes.ts b/gitnexus/src/core/search/fts-indexes.ts index 4591923cd..2070fadf3 100644 --- a/gitnexus/src/core/search/fts-indexes.ts +++ b/gitnexus/src/core/search/fts-indexes.ts @@ -9,7 +9,10 @@ import { } from '../lbug/lbug-adapter.js'; import { getFtsCapability } from '../lbug/extension-loader.js'; import { FTS_DISABLED_MESSAGE, type FtsDisabledReason } from './fts-policy.js'; -import { classifyExtensionLoadError } from '../lbug/extension-load-error.js'; +import { + classifyExtensionLoadError, + usesClassifiedLoadRemedy, +} from '../lbug/extension-load-error.js'; import { FTS_INDEXES, type FTSIndexDefinition } from './fts-schema.js'; /** @@ -89,10 +92,9 @@ export const ftsDegradedWarning = ( // per-request path (HTTP /api/search + MCP query) does NO file I/O (#2383 F3); // fall back to the pure, no-I/O string classifier if it is somehow absent. const { kind, remedy } = fts.diagnosis ?? classifyExtensionLoadError(fts.reason); - const tail = - kind === 'missing_dependency' - ? ` ${remedy}` - : '. Run `gitnexus doctor` for details, then `gitnexus analyze --repair-fts` with network access to reinstall.'; + const tail = usesClassifiedLoadRemedy(kind) + ? ` ${remedy}` + : '. Run `gitnexus doctor` for details, then `gitnexus analyze --repair-fts` with network access to reinstall.'; return ( 'FTS extension failed to load — keyword search degraded' + (reason ? ` (${reason})` : '') + diff --git a/gitnexus/src/core/search/fts-policy.ts b/gitnexus/src/core/search/fts-policy.ts index 9cc12e252..bd3606140 100644 --- a/gitnexus/src/core/search/fts-policy.ts +++ b/gitnexus/src/core/search/fts-policy.ts @@ -1,7 +1,34 @@ -import type { RepoMeta } from '../../storage/repo-meta.js'; +import type { PersistedFtsSkipReason, RepoMeta } from '../../storage/repo-meta.js'; export type FtsDisabledReason = 'disabled-by-flag' | 'disabled-by-env'; -export type FtsSkipReason = FtsDisabledReason | 'extension-unavailable' | 'build-failed'; + +/** Failure-like skip reasons. Must not join `FtsDisabledReason` — that + * predicate is a hand-written `string` check the compiler does not verify. */ +export type FtsFailureSkipReason = + | 'extension-unavailable' + | 'build-failed' + | 'native-abort' + | 'tuple-missing'; + +export type FtsSkipReason = FtsDisabledReason | FtsFailureSkipReason; + +type _FtsSkipReasonStorageParity = FtsSkipReason extends PersistedFtsSkipReason + ? PersistedFtsSkipReason extends FtsSkipReason + ? true + : never + : never; +const _ftsSkipReasonStorageParity: _FtsSkipReasonStorageParity = true; +void _ftsSkipReasonStorageParity; + +/** Runtime list for storage/core parity tests. Order is not significant. */ +export const FTS_SKIP_REASONS: readonly FtsSkipReason[] = [ + 'disabled-by-flag', + 'disabled-by-env', + 'extension-unavailable', + 'build-failed', + 'native-abort', + 'tuple-missing', +]; type RepoCapabilities = NonNullable; @@ -74,3 +101,48 @@ export function withExplicitFtsDisablement( export const FTS_DISABLED_MESSAGE = 'FTS disabled for this index. To enable keyword search, run gitnexus analyze ' + 'without --skip-fts and with GITNEXUS_SKIP_FTS unset.'; + +const FTS_BUILD_FAILED_MESSAGE = + 'Warning: full-text/BM25 search is disabled — the search index build failed this run.\n' + + ' The FTS extension is available; rerun `gitnexus analyze --repair-fts`. If it persists,\n' + + ' check the disk for space or corruption. Run `gitnexus doctor` for details.'; + +const FTS_EXTENSION_UNAVAILABLE_MESSAGE = + 'Warning: full-text/BM25 search is disabled — the LadybugDB FTS extension was unavailable.\n' + + ' Install it once with network access (GITNEXUS_LBUG_EXTENSION_INSTALL=auto), then run\n' + + ' `gitnexus analyze --repair-fts` to build the search indexes. Run `gitnexus doctor` for details.'; + +const FTS_NATIVE_ABORT_MESSAGE = + 'Warning: full-text/BM25 search is disabled — a previous analyze aborted while building the search indexes.\n' + + ' Rerun `gitnexus analyze --repair-fts` to recover. Run `gitnexus doctor` for details.'; + +const FTS_TUPLE_MISSING_MESSAGE = + 'Warning: full-text/BM25 search is disabled — no packaged FTS artifact is available for this platform.\n' + + ' Keyword search is unavailable on this host. Run `gitnexus doctor` for details.'; + +const assertNever = (value: never): never => { + throw new Error(`unhandled FTS skip reason: ${String(value)}`); +}; + +/** + * CLI analyze summary copy for a skipped FTS build. Total switch so a new + * member cannot silently inherit the network-install remedy. + */ +export const formatAnalyzeFtsSkipSummary = (reason: FtsSkipReason | undefined): string => { + if (reason === undefined) return FTS_EXTENSION_UNAVAILABLE_MESSAGE; + switch (reason) { + case 'disabled-by-flag': + case 'disabled-by-env': + return FTS_DISABLED_MESSAGE; + case 'build-failed': + return FTS_BUILD_FAILED_MESSAGE; + case 'native-abort': + return FTS_NATIVE_ABORT_MESSAGE; + case 'tuple-missing': + return FTS_TUPLE_MISSING_MESSAGE; + case 'extension-unavailable': + return FTS_EXTENSION_UNAVAILABLE_MESSAGE; + default: + return assertNever(reason); + } +}; diff --git a/gitnexus/src/core/tree-sitter/vendored-grammars.ts b/gitnexus/src/core/tree-sitter/vendored-grammars.ts index 932ec1a70..bdff7c61d 100644 --- a/gitnexus/src/core/tree-sitter/vendored-grammars.ts +++ b/gitnexus/src/core/tree-sitter/vendored-grammars.ts @@ -1,25 +1,10 @@ import { createRequire } from 'node:module'; -import { fileURLToPath } from 'node:url'; import path from 'node:path'; +import { VENDOR_ROOT } from '../vendor-root.js'; const _require = createRequire(import.meta.url); -/** - * Absolute path to the vendored grammar tree (`/vendor`). - * - * This module compiles to `/dist/core/tree-sitter/vendored-grammars.js` - * and runs from `/src/core/tree-sitter/...` under tsx in dev — both sit - * three directories below the package root, and the build (`tsc`) never bundles, - * so `import.meta.url` resolves the same way in both. `vendor/` ships in the - * published package via package.json `files`. - */ -export const VENDOR_ROOT = path.resolve( - path.dirname(fileURLToPath(import.meta.url)), - '..', - '..', - '..', - 'vendor', -); +export { VENDOR_ROOT }; /** * The tree-sitter grammars GitNexus vendors inside its own package (NOT npm diff --git a/gitnexus/src/core/vendor-root.ts b/gitnexus/src/core/vendor-root.ts new file mode 100644 index 000000000..9d952030a --- /dev/null +++ b/gitnexus/src/core/vendor-root.ts @@ -0,0 +1,16 @@ +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +/** + * Absolute path to the published `vendor/` tree (`/vendor`). + * + * This module compiles to `/dist/core/vendor-root.js` and runs from + * `/src/core/` under tsx — both sit two directories below the package + * root. Shared so grammar loaders and the FTS artifact resolver cannot drift. + */ +export const VENDOR_ROOT = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..', + '..', + 'vendor', +); diff --git a/gitnexus/src/mcp/server.ts b/gitnexus/src/mcp/server.ts index f95edcec4..d44f8c0f2 100644 --- a/gitnexus/src/mcp/server.ts +++ b/gitnexus/src/mcp/server.ts @@ -93,6 +93,16 @@ function getNextStepHint(toolName: string, args: Record | undefined } } +/** Include a string `Error.code` in MCP error text when present. */ +function formatMcpToolError(error: unknown): string { + const message = error instanceof Error ? error.message : 'Unknown error'; + const code = + typeof error === 'object' && error !== null && 'code' in error + ? (error as { code: unknown }).code + : undefined; + return typeof code === 'string' ? `Error [${code}]: ${message}` : `Error: ${message}`; +} + /** * Create a configured MCP Server with all handlers registered. * Transport-agnostic — caller connects the desired transport. @@ -171,13 +181,13 @@ export function createMCPServer( }, ], }; - } catch (err: any) { + } catch (err: unknown) { return { contents: [ { uri, mimeType: 'text/plain', - text: `Error: ${err.message}`, + text: formatMcpToolError(err), }, ], }; @@ -241,12 +251,11 @@ export function createMCPServer( ], }; } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error'; return { content: [ { type: 'text', - text: applyMcpMaxTokens(`Error: ${message}`, maxTokens), + text: applyMcpMaxTokens(formatMcpToolError(error), maxTokens), }, ], isError: true, diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index 393782e5f..1e3bd4762 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -38,6 +38,7 @@ import { withLbugDb, isReadOnlyDbError, } from '../core/lbug/lbug-adapter.js'; +import { assertReadOnlyFtsCrashSafe } from '../core/lbug/sidecar-recovery.js'; import { isValidQueryParams } from '../core/lbug/query-params.js'; import { NODE_TABLES, type GraphNode, type GraphRelationship } from 'gitnexus-shared'; import { searchFTSFromLbug } from '../core/search/bm25-index.js'; @@ -585,6 +586,12 @@ export const streamGraphNdjson = async ( }); }; +const httpErrorBody = (err: any, fallback: string): { error: string; code?: string } => { + const body: { error: string; code?: string } = { error: err.message || fallback }; + if (typeof err?.code === 'string') body.code = err.code; + return body; +}; + const statusFromError = (err: any): number => { // Validation helpers throw BadRequestError / ForbiddenError with a typed // .status field — honor it before falling back to message-string matching. @@ -862,7 +869,7 @@ export const handleQueryRequest = async ( res.status(403).json({ error: 'Write queries are not allowed via the HTTP API' }); return; } - res.status(500).json({ error: err.message || 'Query failed' }); + res.status(500).json(httpErrorBody(err, 'Query failed')); } }; @@ -1358,17 +1365,17 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => if (err instanceof ClientDisconnectedError) { return; } - const message = err.message || 'Failed to build graph'; + const body = httpErrorBody(err, 'Failed to build graph'); if (res.headersSent) { try { - res.write(JSON.stringify({ type: 'error', error: message }) + '\n'); + res.write(JSON.stringify({ type: 'error', ...body }) + '\n'); } catch { // Best-effort only after streaming has started. } res.end(); return; } - res.status(500).json({ error: message }); + res.status(500).json(body); } }); @@ -1552,7 +1559,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => res.json(response); } catch (err: any) { if (sendStorageRequirementHttp(err, res)) return; - res.status(500).json({ error: err.message || 'Search failed' }); + res.status(500).json(httpErrorBody(err, 'Search failed')); } }); @@ -1623,7 +1630,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => res.json({ results, ...(timedOut ? { timedOut: true } : {}) }); } catch (err: any) { if (sendStorageRequirementHttp(err, res)) return; - res.status(statusFromError(err)).json({ error: err.message || 'Grep failed' }); + res.status(statusFromError(err)).json(httpErrorBody(err, 'Grep failed')); } }); @@ -1633,7 +1640,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const result = await backend.queryProcesses(requestedRepo(req)); res.json(result); } catch (err: any) { - res.status(statusFromError(err)).json({ error: err.message || 'Failed to query processes' }); + res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query processes')); } }); @@ -1653,9 +1660,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => } res.json(result); } catch (err: any) { - res - .status(statusFromError(err)) - .json({ error: err.message || 'Failed to query process detail' }); + res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query process detail')); } }); @@ -1665,7 +1670,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const result = await backend.queryClusters(requestedRepo(req)); res.json(result); } catch (err: any) { - res.status(statusFromError(err)).json({ error: err.message || 'Failed to query clusters' }); + res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query clusters')); } }); @@ -1685,9 +1690,7 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => } res.json(result); } catch (err: any) { - res - .status(statusFromError(err)) - .json({ error: err.message || 'Failed to query cluster detail' }); + res.status(statusFromError(err)).json(httpErrorBody(err, 'Failed to query cluster detail')); } }); @@ -1998,6 +2001,10 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => const lbugPath = path.join(storagePath, LBUG_DIRECTORY); const ftsSession = await loadFtsSession(storagePath); let embeddingMeta = ftsSession.meta; + // Writable embed still replays a leftover FTS-abort WAL. Refuse + // here — doInitLbug only gates the readOnly path, and analyze + // writers must still be able to park/rebuild. + await assertReadOnlyFtsCrashSafe(lbugPath); await withLbugDb( lbugPath, async () => { diff --git a/gitnexus/src/storage/repo-meta.ts b/gitnexus/src/storage/repo-meta.ts index 363900b4c..159e6a4a8 100644 --- a/gitnexus/src/storage/repo-meta.ts +++ b/gitnexus/src/storage/repo-meta.ts @@ -86,6 +86,18 @@ export interface AnalyzerRunnerIdentity { }; } +/** + * Hand-mirrored `FtsSkipReason` from core/search/fts-policy.ts so storage + * takes no core import. Change both declarations together. + */ +export type PersistedFtsSkipReason = + | 'extension-unavailable' + | 'build-failed' + | 'disabled-by-flag' + | 'disabled-by-env' + | 'native-abort' + | 'tuple-missing'; + export interface RepoMeta { repoPath: string; /** Complete index directory selected for this successful analysis. */ @@ -184,17 +196,28 @@ export interface RepoMeta { * `--repair-fts` or a content change addresses it. * - `disabled-by-flag` / `disabled-by-env` — deliberate opt-out. * A later analyze without the opt-out rebuilds FTS at the same commit. + * - `native-abort` — inferred on the next run from an FTS-phase dirty + * flag after the previous process died in the native FTS build. + * - `tuple-missing` — no packaged artifact for this platform tuple. + * The tuple itself is not persisted (closed enum; live messages name it). * * Collapsing both into `status: 'unavailable'` is exactly what made that * loop reachable. ABSENT on indexes written before #2841 and on the * `--repair-fts` stamp (which writes `status: 'available'`); `undefined` * therefore reads as "cause unknown" and keeps the pre-#2841 behaviour. + * No schema version: meta reads are unchecked casts. An older binary + * still sees the raw JSON string for an unknown member; it does not + * drop the field to `undefined` at parse time. */ - skipReason?: - | 'extension-unavailable' - | 'build-failed' - | 'disabled-by-flag' - | 'disabled-by-env'; + skipReason?: PersistedFtsSkipReason; + /** + * Write plan of the analyze that aborted, persisted with + * `skipReason: 'native-abort'` so readers can refuse a leftover live + * WAL after recovery clears `incrementalInProgress`. Staging persist + * also writes `native-abort` and must keep its live WAL. Absent on + * older indexes and on non-abort skips. + */ + writePlan?: 'in-place' | 'staging'; }; vectorSearch: { provider: string; @@ -403,8 +426,17 @@ export interface RepoMeta { /** Number of files in the writable set, for diagnostic logs. * `0` on the full-rebuild path (no incremental write set exists). */ toWriteCount: number; - /** Last completed writeback phase before the process stopped. */ + /** + * Last completed writeback phase before the process stopped. + * `'fts'` is the graph-boundary / FTS-build marker: recovery may act + * more narrowly than a bare dirty flag, and `--repair-fts` must not + * treat it as a half-written graph. + */ phase?: string; + /** Settled write plan at the FTS boundary (in-place vs unpublished staging). */ + writePlan?: 'in-place' | 'staging'; + /** Whether the converged graph-boundary CHECKPOINT succeeded. */ + checkpointSucceeded?: boolean; /** Directly changed/added files before importer expansion. */ directWriteCount?: number; /** Extra files pulled into the writable set by importer BFS. */ diff --git a/gitnexus/test/helpers/fts-availability.ts b/gitnexus/test/helpers/fts-availability.ts index 30ba05b8a..8ecfe6843 100644 --- a/gitnexus/test/helpers/fts-availability.ts +++ b/gitnexus/test/helpers/fts-availability.ts @@ -1,5 +1,7 @@ import { existsSync, readdirSync, statSync } from 'node:fs'; +import { homedir } from 'node:os'; import { join } from 'node:path'; +import { resolveVendoredFtsPath } from '../../src/core/lbug/vendored-extension-path.js'; /** A valid `libfts.lbug_extension` is ~2.2MB; anything smaller is truncated/corrupt. */ const MIN_VALID_FTS_EXTENSION_BYTES = 1024 * 1024; @@ -41,6 +43,19 @@ export const findInstalledFtsExtension = (extensionRoot: string): string | null } }; +/** + * Resolve the FTS extension the same way doctor/analyze will after vendoring: + * packaged artifact first, then a `~/.lbdb` install. File-path CI gates must + * accept the vendored file so they stay green when ensure-fts no longer + * populates the home cache. + */ +export const resolveFtsExtension = (opts?: { + vendorRoot?: string; + homeExtensionRoot?: string; +}): string | null => + resolveVendoredFtsPath({ vendorRoot: opts?.vendorRoot }) ?? + findInstalledFtsExtension(opts?.homeExtensionRoot ?? join(homedir(), '.lbdb', 'extension')); + export const FTS_UNAVAILABLE_NOTE = 'FTS extension unavailable (load-only policy; LOAD failed on this machine)'; @@ -83,12 +98,12 @@ export const skipUnlessFtsAvailable = async (ctx: { }; /** - * Skip a structural FTS test when a required on-disk artifact (the installed - * extension file, the native addon) is not resolvable — but HARD-FAIL under - * GITNEXUS_REQUIRE_FTS=1 (#2299, #2383 F6d) so it never silently vanishes from a - * green CI run. Used by tests that inspect the extension *file* directly and so - * need its path rather than a loaded connection (skipUnlessFtsAvailable needs an - * initialized LadybugDB, which those tests do not set up). + * Skip a structural FTS test when a required on-disk artifact (the vendored + * extension, a home install, or the native addon) is not resolvable — but + * HARD-FAIL under GITNEXUS_REQUIRE_FTS=1 (#2299, #2383 F6d) so it never + * silently vanishes from a green CI run. Used by tests that inspect the + * extension *file* directly and so need its path rather than a loaded + * connection (skipUnlessFtsAvailable needs an initialized LadybugDB). */ export const requireFtsResourceOrSkip = ( ctx: { skip: (note?: string) => void }, diff --git a/gitnexus/test/integration/cli-e2e.test.ts b/gitnexus/test/integration/cli-e2e.test.ts index 9998a1425..c9da12b86 100644 --- a/gitnexus/test/integration/cli-e2e.test.ts +++ b/gitnexus/test/integration/cli-e2e.test.ts @@ -360,7 +360,7 @@ describe('CLI end-to-end', () => { const repoParent = path.dirname(repo); try { - const result = runCliWithEnv(['analyze'], repo, { GITNEXUS_HOME: gnHome }, 60000); + const result = runCliWithEnv(['analyze'], repo, { GITNEXUS_HOME: gnHome }, 90_000); expect( result.status, @@ -419,7 +419,7 @@ describe('CLI end-to-end', () => { cleanupTempDirSync(gnHome); cleanupTempDirSync(repoParent); } - }, 60_000); + }, 90_000); it('already-up-to-date analyze fails when registry entry is missing (#1169)', () => { const gnHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-1169-fastpath-home-')); diff --git a/gitnexus/test/integration/extension-binary-real.test.ts b/gitnexus/test/integration/extension-binary-real.test.ts index f46958a2b..e98823d78 100644 --- a/gitnexus/test/integration/extension-binary-real.test.ts +++ b/gitnexus/test/integration/extension-binary-real.test.ts @@ -6,17 +6,14 @@ import { rmSync, writeFileSync, } from 'node:fs'; -import { homedir, tmpdir } from 'node:os'; +import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, describe, expect, it } from 'vitest'; import { diagnoseExtensionLoad, inspectExtensionBinary, } from '../../src/core/lbug/extension-load-error.js'; -import { - findInstalledFtsExtension, - requireFtsResourceOrSkip, -} from '../helpers/fts-availability.js'; +import { requireFtsResourceOrSkip, resolveFtsExtension } from '../helpers/fts-availability.js'; /** * #2374: exercise the language-independent structural classifier against REAL @@ -47,13 +44,12 @@ function resolveLbugNative(): string | null { } /** - * The actual installed FTS extension binary for the running lbug version. - * `os.homedir()` already honors `$HOME` (POSIX) / `%USERPROFILE%` (Windows) — - * the same resolution LadybugDB's native layer uses — so it stays correct - * under the hermetic-home overrides other tests in this suite set via env vars. + * The FTS extension for this platform: packaged artifact first, then a + * `~/.lbdb` install. `resolveFtsExtension` honors the same home as LadybugDB + * (`$HOME` / `%USERPROFILE%`) so hermetic-home overrides still apply. */ function resolveInstalledFtsExtension(): string | null { - return findInstalledFtsExtension(join(homedir(), '.lbdb', 'extension')); + return resolveFtsExtension(); } const lbugNative = resolveLbugNative(); diff --git a/gitnexus/test/integration/fts-extension-e2e.test.ts b/gitnexus/test/integration/fts-extension-e2e.test.ts index d029f0b70..47f79d137 100644 --- a/gitnexus/test/integration/fts-extension-e2e.test.ts +++ b/gitnexus/test/integration/fts-extension-e2e.test.ts @@ -3,20 +3,21 @@ * * Everything real, nothing mocked: each test spawns the actual CLI entry as a * child process, LadybugDB loads the actual extension shared library from - * disk, and the real out-of-process installer downloads the real extension in - * the network-gated cases. + * disk. The packaged vendor artifact is the first LOAD path; HOME copies + * are no longer required for a green FTS run. * - * Isolation: LadybugDB resolves its extension directory from the process HOME - * (USERPROFILE on Windows), so every scenario owns a hermetic fake home with - * its own `.lbdb/extension///fts/` state — the machine's - * real ~/.lbdb is never read or written. GITNEXUS_HOME additionally isolates - * the registry (#829), following cli-e2e.test.ts conventions. + * Isolation: GITNEXUS_HOME isolates the registry (#829). LadybugDB still + * resolves `~/.lbdb` from HOME, and every scenario owns a hermetic fake home + * so the machine's real ~/.lbdb is never written. Analyze now path-LOADs the + * packaged vendor artifact first, so a broken or missing HOME copy is no + * longer an FTS outage when the packaged file is present. Vendor-broken + * coverage lives in `fts-vendored-root-seam.test.ts` (injected vendorRoot). * - * Scenario matrix (the #2374 report, codified): - * - happy: valid extension pre-installed, offline (load-only) - * - unhappy: extension file present but broken — the reporter's exact state - * - unhappy: extension file missing entirely (distinguishable reason) - * - heal: FORCE INSTALL replaces a broken file over the network (auto) + * Scenario matrix: + * - happy: valid HOME copy, offline (load-only) — vendor or HOME loads + * - packaged vendor survives a broken or missing HOME copy + * - #2841: HOME copy vanishes between runs — incremental stays incremental + * - auto: same vendor survivorship under the install policy */ import { describe, it, expect, beforeAll, beforeEach, afterAll } from 'vitest'; import { CLI_SPAWN_PREFIX } from '../helpers/cli-entry.js'; @@ -26,6 +27,11 @@ import fs from 'fs'; import os from 'os'; import { getExtensionInstallChildProcessArgs } from '../../src/core/lbug/extension-loader.js'; +import { + defaultVendorRoot, + nodePlatformTuple, + resolveVendoredFtsPath, +} from '../../src/core/lbug/vendored-extension-path.js'; import { cleanupTempDirSync } from '../helpers/test-db.js'; import { findInstalledFtsExtension } from '../helpers/fts-availability.js'; @@ -33,8 +39,6 @@ import { findInstalledFtsExtension } from '../helpers/fts-availability.js'; let extensionRelPath: string; /** Canonical valid extension bytes (path to a known-good file). */ let seedExtensionFile: string | null = null; -/** Real reachability of the extension repo — gates the auto-install cases. */ -let networkAvailable = false; const REQUIRE_FTS = process.env.GITNEXUS_REQUIRE_FTS === '1'; const tmpDirs: string[] = []; @@ -46,12 +50,39 @@ const makeTmpDir = (label: string): string => { }; /** - * Locate a known-good extension file for the running LadybugDB version. - * Prefers a copy already installed under the machine's real home (pure file - * read, offline); falls back to one real out-of-process install into a probe - * home — the production installer script, not a reimplementation. + * Locate a known-good extension file for HOME-copy fixtures. + * Prefers the packaged vendor artifact (offline, no HOME/network), then a + * copy already installed under the machine's real home, then one real + * out-of-process install into a probe home. */ +/** Ladybug HOME layout: `~/.lbdb/extension///fts/`. */ +const ladybugHomeExtensionRelPath = (filename: string): string | null => { + try { + const raw = JSON.parse( + fs.readFileSync(path.join(defaultVendorRoot(), 'lbug-fts', 'manifest.json'), 'utf8'), + ) as { + coreVersion?: string; + tuples?: Array<{ tuple: string; upstreamPlatform: string }>; + }; + const upstream = raw.tuples?.find( + (entry) => entry.tuple === nodePlatformTuple(), + )?.upstreamPlatform; + if (!raw.coreVersion || !upstream) return null; + return path.join('.lbdb', 'extension', raw.coreVersion, upstream, 'fts', filename); + } catch { + return null; + } +}; + const resolveSeedExtension = (): void => { + const packaged = resolveVendoredFtsPath(); + if (packaged) { + extensionRelPath = + ladybugHomeExtensionRelPath(path.basename(packaged)) ?? + path.join('.lbdb', 'extension', 'vendor-seed', 'fts', path.basename(packaged)); + seedExtensionFile = packaged; + return; + } const realExtensionRoot = path.join(os.homedir(), '.lbdb', 'extension'); const installed = findInstalledFtsExtension(realExtensionRoot); if (installed) { @@ -71,7 +102,6 @@ const resolveSeedExtension = (): void => { if (install.status === 0 && probeInstalled) { extensionRelPath = path.relative(probeHome, probeInstalled); seedExtensionFile = probeInstalled; - networkAvailable = true; return; } }; @@ -152,21 +182,6 @@ beforeAll(() => { 'GITNEXUS_REQUIRE_FTS=1 but no FTS extension could be located or installed for the E2E suite.', ); } - // The self-heal cases need the real extension repo; probe it cheaply when - // the seed came from a local copy (the installer fallback already proved it). - return (async () => { - if (seedExtensionFile && !networkAvailable) { - try { - const res = await fetch('https://extension.ladybugdb.com/', { - method: 'HEAD', - signal: AbortSignal.timeout(5000), - }); - networkAvailable = res.ok; - } catch { - networkAvailable = false; - } - } - })(); }, 180_000); afterAll(() => { @@ -221,7 +236,7 @@ describe('happy path — extension pre-installed, fully offline (load-only)', () }, 180_000); }); -describe('unhappy path — extension file present but broken (the #2374 report)', () => { +describe('packaged vendor survives a broken or missing home copy', () => { let home: string; let repo: string; @@ -233,68 +248,47 @@ describe('unhappy path — extension file present but broken (the #2374 report)' repo = makeFixtureRepo('broken'); }); - it('analyze degrades gracefully and names the real LOAD failure, not "not pre-installed"', () => { + it('analyze stays FTS-available when ~/.lbdb is broken', () => { const result = runCli(['analyze'], repo, home, 'load-only'); expect(result.status).toBe(0); expect(result.output).toContain('indexed successfully'); - expect(result.output).toContain('FTS extension unavailable'); - // The load-side ground truth must survive to the user… - expect(result.output).toContain('LOAD fts failed'); - expect(result.output).toContain('Failed to load library'); - // …and the old misdiagnosis must not: the file IS pre-installed. - expect(result.output).not.toContain('not pre-installed'); + expect(result.output).not.toContain('FTS extension unavailable'); + expect(result.output).not.toContain('search is disabled'); }, 180_000); - it('analyze --repair-fts fails loudly with the live reason and an honest remedy', () => { + it('analyze --repair-fts succeeds from the packaged artifact', () => { const result = runCli(['analyze', '--repair-fts'], repo, home, 'load-only'); - expect(result.status).not.toBe(0); - expect(result.output).toContain('Cannot repair FTS indexes'); - expect(result.output).toContain('FTS extension failed to load'); - expect(result.output).toContain('LOAD fts failed'); - // Old message sent users to doctor "to install it"; doctor never installed. - expect(result.output).not.toContain('doctor` to install'); - expect(result.output).toContain('gitnexus doctor'); - // #2374 (U2): a corrupt file classifies as corrupt_file, so the Windows - // missing-dependency remedy must not misfire on the repair path either. - expect(result.output).not.toContain('Visual C++'); + expect(result.status).toBe(0); + expect(result.output).toContain('FTS indexes repaired successfully'); }, 180_000); - it('query warns with the extension-load failure, not the misleading indexes-missing message', () => { + it('query finds the symbol with no HOME-copy degradation warning', () => { const result = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); expect(result.status).toBe(0); - expect(result.output).toContain('FTS extension failed to load'); - expect(result.output).toContain('Failed to load library'); - expect(result.output).not.toContain('FTS indexes missing'); + expect(result.output).toContain('greetE2eSymbol'); + expect(result.output).not.toContain('keyword search degraded'); + expect(result.output).not.toContain('FTS extension failed to load'); }, 60_000); - it('doctor live-probes FTS as unavailable, prints the real error and an actionable remedy', () => { + it('doctor reports a live-probed available FTS despite a broken HOME copy', () => { const result = runCli(['doctor'], repo, home, 'load-only'); expect(result.status).toBe(0); - expect(result.output).toContain('Full-text search: unavailable'); - expect(result.output).toContain('Failed to load library'); - // #2374 (U2): doctor routes the reason through the classifier and prints a - // remedy. A broken file is corrupt_file → re-download guidance; the Windows - // missing-dependency remedy (VC++/OpenSSL) must NOT misfire on a corrupt file - // — the catch-all guard, verified end-to-end through the real CLI. - expect(result.output).toContain('Re-download it with network access'); - expect(result.output).not.toContain('Visual C++'); + expect(result.output).toContain('Full-text search: available'); }, 60_000); -}); -describe('unhappy path — extension missing entirely', () => { - it('analyze degrades with a reason that distinguishes missing from broken', () => { - const { home } = makeHome('missing'); - const repo = makeFixtureRepo('missing'); - const result = runCli(['analyze'], repo, home, 'load-only'); + it('analyze stays FTS-available when ~/.lbdb is missing entirely', () => { + const missing = makeHome('missing'); + const missingRepo = makeFixtureRepo('missing'); + const result = runCli(['analyze'], missingRepo, missing.home, 'load-only'); expect(result.status).toBe(0); - expect(result.output).toContain('FTS extension unavailable'); - expect(result.output).toContain('has not been installed'); - expect(result.output).not.toContain('Failed to load library'); + expect(result.output).toContain('indexed successfully'); + expect(result.output).not.toContain('FTS extension unavailable'); + expect(result.output).not.toContain('has not been installed'); }, 180_000); }); -describe('regression — the extension disappears between analyze runs (#2841)', () => { - it('the incremental run completes with a full DB write instead of an opaque Binder exception', (ctx) => { +describe('regression — the home copy disappears between analyze runs (#2841)', () => { + it('the incremental run completes without a Binder exception or a full-DB escalation', (ctx) => { const { home, extensionFile } = makeHome('valid'); const repo = makeFixtureRepo('vanishing-extension'); @@ -336,36 +330,28 @@ describe('regression — the extension disappears between analyze runs (#2841)', const second = runCli(['analyze'], repo, home, 'load-only'); // Pre-fix: exit 1 with "Binder exception: Trying to delete from an index on // table File but its extension is not loaded" and no mention of FTS at all. + // Packaged vendor still loads after HOME vanishes, so incremental stays + // incremental (no Binder, no full-DB escalation). expect(second.status).toBe(0); expect(second.output).not.toContain('its extension is not loaded'); - expect(second.output).toContain('full DB write'); - expect(second.output).toContain('FTS'); + expect(second.output).not.toContain('full DB write'); + expect(second.output).not.toContain('forcing full rebuild'); + expect(second.output).toMatch(/Incremental:|indexed successfully/); }, 400_000); }); -describe('self-heal over the network — FORCE INSTALL replaces a broken file (auto)', () => { - beforeEach((ctx) => { - // The platform matrix already exercises offline FTS load/diagnostic paths - // against real macOS/Windows binaries. Keep network redownload coverage on - // Ubuntu, where the full test job has the most stable extension fetch path. - if (process.platform !== 'linux') ctx.skip(); - if (!networkAvailable) ctx.skip(); - }); - - it('the reported journey heals: degraded analyze, then repair-fts with auto re-downloads and repairs', () => { - const { home, extensionFile } = makeHome('broken'); +describe('auto policy — packaged vendor does not need a HOME reinstall', () => { + it('analyze --repair-fts with auto succeeds from the packaged artifact when HOME is broken', () => { + const { home } = makeHome('broken'); const repo = makeFixtureRepo('heal'); - const degraded = runCli(['analyze'], repo, home, 'load-only'); - expect(degraded.status).toBe(0); - expect(degraded.output).toContain('FTS extension unavailable'); + const first = runCli(['analyze'], repo, home, 'load-only'); + expect(first.status).toBe(0); + expect(first.output).not.toContain('FTS extension unavailable'); - // The reporter's exact failing command — plain INSTALL used to no-op - // over the broken file and this kept failing forever. const repair = runCli(['analyze', '--repair-fts'], repo, home, 'auto'); expect(repair.status).toBe(0); expect(repair.output).toContain('FTS indexes repaired successfully'); - expect(fs.statSync(extensionFile).size).toBeGreaterThan(1024 * 1024); const query = runCli(['query', 'greetE2eSymbol'], repo, home, 'load-only'); expect(query.status).toBe(0); @@ -373,13 +359,12 @@ describe('self-heal over the network — FORCE INSTALL replaces a broken file (a expect(query.output).not.toContain('keyword search degraded'); }, 600_000); - it('a fresh machine with no extension installs it during analyze and gets full FTS', () => { - const { home, extensionFile } = makeHome('missing'); + it('a fresh machine with no HOME copy still gets full FTS under load-only', () => { + const { home } = makeHome('missing'); const repo = makeFixtureRepo('fresh'); - const result = runCli(['analyze'], repo, home, 'auto'); + const result = runCli(['analyze'], repo, home, 'load-only'); expect(result.status).toBe(0); expect(result.output).toContain('indexed successfully'); expect(result.output).not.toContain('FTS extension unavailable'); - expect(fs.existsSync(extensionFile)).toBe(true); }, 600_000); }); diff --git a/gitnexus/test/integration/fts-vendored-root-seam.test.ts b/gitnexus/test/integration/fts-vendored-root-seam.test.ts new file mode 100644 index 000000000..53a6aeb8a --- /dev/null +++ b/gitnexus/test/integration/fts-vendored-root-seam.test.ts @@ -0,0 +1,86 @@ +/** + * U9: injected vendored-root seam. Never an environment variable — an + * attacker-controlled env would be a path into an in-process native load. + */ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { extensionManager, resetExtensionState } from '../../src/core/lbug/extension-loader.js'; +import { diagnoseExtensionLoad } from '../../src/core/lbug/extension-load-error.js'; +import { + defaultVendorRoot, + nodePlatformTuple, +} from '../../src/core/lbug/vendored-extension-path.js'; +import { cleanupTempDirSync } from '../helpers/test-db.js'; + +const tmpDirs: string[] = []; + +const makeVendorTree = (state: 'valid' | 'truncated'): { vendorRoot: string; dest: string } => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-fts-vendor with space-')); + tmpDirs.push(dir); + const vendorRoot = path.join(dir, 'vendor'); + const dest = path.join( + vendorRoot, + 'lbug-fts', + 'prebuilds', + nodePlatformTuple(), + 'libfts.lbug_extension', + ); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + const manifestSrc = path.join(defaultVendorRoot(), 'lbug-fts', 'manifest.json'); + if (fs.existsSync(manifestSrc)) { + fs.copyFileSync(manifestSrc, path.join(vendorRoot, 'lbug-fts', 'manifest.json')); + } + const packaged = path.join( + defaultVendorRoot(), + 'lbug-fts', + 'prebuilds', + nodePlatformTuple(), + 'libfts.lbug_extension', + ); + if (state === 'valid' && fs.existsSync(packaged)) fs.copyFileSync(packaged, dest); + if (state === 'truncated') fs.writeFileSync(dest, Buffer.alloc(64, 0)); + return { vendorRoot, dest }; +}; + +afterEach(() => { + resetExtensionState(); + while (tmpDirs.length > 0) { + const dir = tmpDirs.pop(); + if (dir) cleanupTempDirSync(dir); + } +}); + +describe('vendored-root seam (injected parameter, never env)', () => { + it('loads from an injected vendor tree without attempting a network install', async (ctx) => { + const { vendorRoot, dest } = makeVendorTree('valid'); + if (!fs.existsSync(dest)) { + ctx.skip(); + return; + } + const query = vi.fn().mockResolvedValue({}); + const ok = await extensionManager.ensure(query, 'fts', 'FTS', { + vendorRoot, + policy: 'load-only', + }); + expect(ok).toBe(true); + expect(query).toHaveBeenCalled(); + const sql = String(query.mock.calls[0]?.[0] ?? ''); + expect(sql).toMatch(/LOAD/i); + expect(sql).toMatch(/libfts\.lbug_extension/); + expect(query.mock.calls.some(([text]) => String(text).includes('INSTALL'))).toBe(false); + }); + + it('reports a corrupt diagnosis when the injected artifact is truncated', async () => { + const { vendorRoot, dest } = makeVendorTree('truncated'); + const reason = `Failed to load library '${dest}': invalid ELF header`; + const query = vi.fn().mockRejectedValue(new Error(reason)); + const ok = await extensionManager.ensure(query, 'fts', 'FTS', { + vendorRoot, + policy: 'load-only', + }); + expect(ok).toBe(false); + expect(diagnoseExtensionLoad(reason).kind).toBe('corrupt_file'); + }); +}); diff --git a/gitnexus/test/integration/fts-windows-dependency.test.ts b/gitnexus/test/integration/fts-windows-dependency.test.ts new file mode 100644 index 000000000..ba05d6bc1 --- /dev/null +++ b/gitnexus/test/integration/fts-windows-dependency.test.ts @@ -0,0 +1,32 @@ +/** + * U7 Windows FTS arm. OQ1 (does path-LOAD search the extension directory for + * transitive DLLs?) is unanswered on this Linux workspace, so the shipping-DLL + * arm is closed — KTD13 forbids merging shipped OpenSSL without a CVE owner. + * The recorded arm is the vendor-neutral prerequisite. + * + * Registered in scripts/cross-platform-tests.ts so windows-latest must run it. + * Assertions are unconditional: a skip-only suite would stay green if Windows + * never ran. + */ +import { describe, expect, it } from 'vitest'; +import { classifyExtensionLoadError } from '../../src/core/lbug/extension-load-error.js'; + +export const WINDOWS_FTS_ARM = 'prerequisite' as const; + +const BORROWED_DLL_HINT = /Git Bash|mingw64|Program Files\\Git|prepend/i; + +describe('Windows FTS dependency arm (U7)', () => { + it('records the prerequisite arm unconditionally', () => { + expect(WINDOWS_FTS_ARM).toBe('prerequisite'); + }); + + it('names vendor-neutral runtimes and never a third-party application directory', () => { + const { remedy } = classifyExtensionLoadError( + 'needed by extension: fts. Error: The specified module could not be found.', + ); + expect(remedy).toMatch(/Visual C\+\+/); + expect(remedy).toMatch(/OpenSSL 3/); + expect(remedy).toMatch(/system runtime/); + expect(remedy).not.toMatch(BORROWED_DLL_HINT); + }); +}); diff --git a/gitnexus/test/integration/lbug-core-adapter.test.ts b/gitnexus/test/integration/lbug-core-adapter.test.ts index f66d25e3b..814fd60a4 100644 --- a/gitnexus/test/integration/lbug-core-adapter.test.ts +++ b/gitnexus/test/integration/lbug-core-adapter.test.ts @@ -399,21 +399,6 @@ withTestLbugDB( ).resolves.toBeUndefined(); }); - it('ensureFTSIndex is idempotent and caches across writable calls (#1224)', async (ctx) => { - await skipUnlessFtsAvailable(ctx); - const { ensureFTSIndex } = await import('../../src/core/lbug/lbug-adapter.js'); - - // First call creates the index. Second call must short-circuit on the - // in-process cache — guarantees the read-only guard added in #1224 - // still respects the success path. - await expect( - ensureFTSIndex('Function', 'function_fts_ensure', ['name', 'content']), - ).resolves.toBeUndefined(); - await expect( - ensureFTSIndex('Function', 'function_fts_ensure', ['name', 'content']), - ).resolves.toBeUndefined(); - }); - it('getLbugStats returns valid counts', async () => { const { getLbugStats } = await import('../../src/core/lbug/lbug-adapter.js'); diff --git a/gitnexus/test/integration/lbug-delete-nodes-for-files.test.ts b/gitnexus/test/integration/lbug-delete-nodes-for-files.test.ts index 6620d3b73..5686ace62 100644 --- a/gitnexus/test/integration/lbug-delete-nodes-for-files.test.ts +++ b/gitnexus/test/integration/lbug-delete-nodes-for-files.test.ts @@ -517,7 +517,10 @@ withTestLbugDB('embedding-row-dml-vector-gate', (handle) => { process.env.GITNEXUS_LBUG_EXTENSION_INSTALL = 'load-only'; await withUnreadableIndexCatalog(async (seen) => { await ensureFtsRowDmlSafe(); - expect(seen.some((s) => /^\s*LOAD EXTENSION fts\b/i.test(s))).toBe(true); + const isFtsLoad = (sql: string): boolean => + /^\s*LOAD\s+EXTENSION\b/i.test(sql) && + (/\bfts\b/i.test(sql) || /libfts\.lbug_extension/i.test(sql)); + expect(seen.some(isFtsLoad)).toBe(true); // …and it did not charge the caller a VECTOR load it never needed. expect(seen.some((s) => /^\s*LOAD EXTENSION vector\b/i.test(s))).toBe(false); }); diff --git a/gitnexus/test/unit/api-fts-mode.test.ts b/gitnexus/test/unit/api-fts-mode.test.ts index 85f36f77b..632bd37d3 100644 --- a/gitnexus/test/unit/api-fts-mode.test.ts +++ b/gitnexus/test/unit/api-fts-mode.test.ts @@ -3,7 +3,7 @@ import { EventEmitter } from 'node:events'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; const mocks = vi.hoisted(() => ({ loadMeta: vi.fn(), @@ -73,6 +73,7 @@ vi.mock('../../src/server/analyze-job.js', () => ({ import { createServer } from '../../src/server/api.js'; import { FTS_DISABLED_MESSAGE } from '../../src/core/search/fts-policy.js'; +import { extensionManager, resetExtensionState } from '../../src/core/lbug/extension-loader.js'; const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-mode-fixture-')); const entry = { @@ -182,6 +183,16 @@ const cases = [ fts: { provider: 'ladybugdb-fts', status: 'degraded', skipReason: 'build-failed' }, skip: false, }, + { + name: 'native-abort', + fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: 'native-abort' }, + skip: false, + }, + { + name: 'tuple-missing', + fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: 'tuple-missing' }, + skip: false, + }, ] as const; describe('serve uses one metadata-derived FTS mode on every DB-open path', () => { @@ -251,6 +262,31 @@ describe('serve uses one metadata-derived FTS mode on every DB-open path', () => ); }); +describe('GET /api/search FTS warning redaction', () => { + afterEach(() => { + resetExtensionState(); + }); + + it('redacts a space-containing vendor path from the HTTP response body', async () => { + const spaced = '/tmp/fts vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension'; + await extensionManager.ensure( + vi + .fn() + .mockRejectedValue(new Error(`Failed to load library '${spaced}': invalid ELF header`)), + 'fts', + 'FTS', + { policy: 'load-only', vendorRoot: '/tmp/empty-vendor-root' }, + ); + mocks.loadMeta.mockResolvedValue({ + capabilities: { fts: { provider: 'ladybugdb-fts', status: 'available' } }, + }); + mocks.search.mockResolvedValue({ results: [], ftsAvailable: false }); + const response = await invoke('/api/search'); + expect(String(response.body.warning)).toContain('invalid ELF header'); + expect(String(response.body.warning)).not.toMatch(/fts vendor|\/tmp\/|C:\\Users\\/); + }); +}); + describe('GET /api/repos catalog validation', () => { it('lists registered repos with validate: true', async () => { mocks.loadMeta.mockResolvedValue({}); diff --git a/gitnexus/test/unit/api-readonly-wiring.test.ts b/gitnexus/test/unit/api-readonly-wiring.test.ts index 70466556a..e53489966 100644 --- a/gitnexus/test/unit/api-readonly-wiring.test.ts +++ b/gitnexus/test/unit/api-readonly-wiring.test.ts @@ -67,6 +67,20 @@ describe('api read-only endpoint wiring', () => { ); }); + it('/api/embed refuses an in-place FTS crash WAL before the writable open', async () => { + const source = await readSource(); + const embedSection = source.match( + /\/\/ Run embedding pipeline asynchronously[\s\S]*?skipFtsOption\(ftsSession\.skipFts\)/, + ); + expect(embedSection).not.toBeNull(); + const gateIdx = embedSection![0].indexOf('assertReadOnlyFtsCrashSafe(lbugPath)'); + const openIdx = embedSection![0].indexOf('await withLbugDb('); + expect(gateIdx).toBeGreaterThan(-1); + expect(openIdx).toBeGreaterThan(gateIdx); + expect(embedSection![0]).not.toMatch(/fts-inplace-checkpointed/); + expect(embedSection![0]).not.toMatch(/readOnly:\s*true/); + }); + it('/api/embed remains write-mode (writes embeddings — must not be flipped to readOnly)', async () => { const source = await readSource(); // Negative assertion: no `readOnly: true` between the embed job's diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index 78c786004..dd3cde207 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -801,6 +801,34 @@ describe('LocalBackend.callTool', () => { expect((result as any).warning).toMatch(/gitnexus analyze --repair-fts/); }); + it('redacts a space-containing vendor path from the MCP query warning', async () => { + const { extensionManager, resetExtensionState } = + await import('../../src/core/lbug/extension-loader.js'); + const spaced = '/tmp/fts vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension'; + await extensionManager.ensure( + vi + .fn() + .mockRejectedValue(new Error(`Failed to load library '${spaced}': invalid ELF header`)), + 'fts', + 'FTS', + { policy: 'load-only', vendorRoot: '/tmp/empty-vendor-root' }, + ); + const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js'); + vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ results: [], ftsAvailable: false }); + (executeParameterized as any).mockResolvedValue([]); + + try { + const result = await backend.callTool('query', { query: 'ProcessActivity' }); + expect(result).toHaveProperty('warning'); + expect(String((result as { warning?: string }).warning)).toContain('invalid ELF header'); + expect(String((result as { warning?: string }).warning)).not.toMatch( + /fts vendor|\/tmp\/|C:\\Users\\/, + ); + } finally { + resetExtensionState(); + } + }); + it('does not include warning when ftsAvailable is true with zero results', async () => { const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js'); vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ results: [], ftsAvailable: true }); diff --git a/gitnexus/test/unit/cross-platform-shard.test.ts b/gitnexus/test/unit/cross-platform-shard.test.ts index 30a6ecd1d..66ea6a430 100644 --- a/gitnexus/test/unit/cross-platform-shard.test.ts +++ b/gitnexus/test/unit/cross-platform-shard.test.ts @@ -34,10 +34,11 @@ describe('cross-platform shard partition', () => { // in the scheduling table. Keep the observed profile independent of the // table so deleting a weight cannot make this regression pass again. const observed: Readonly> = { - 'test/integration/skills-e2e.test.ts': 444, + 'test/integration/skills-e2e.test.ts': 550, 'test/unit/incremental-index-extension-dml-gate.test.ts': 414, - 'test/integration/fts-extension-e2e.test.ts': 146, + 'test/integration/fts-extension-e2e.test.ts': 380, 'test/integration/analyze-wal-checkpoint-failure.test.ts': 86, + 'test/integration/skip-fts.test.ts': 110, }; const floor = weightOf('test/unmeasured.test.ts'); const loads = allShards(ALL_CROSS_PLATFORM, SHARD_TOTAL).map((files) => @@ -54,6 +55,13 @@ describe('cross-platform shard partition', () => { ].map((file) => shards.findIndex((files) => files.includes(file))); expect(heavyweightLocations).not.toContain(-1); expect(new Set(heavyweightLocations).size).toBe(SHARD_TOTAL); + expect( + shards.filter( + (s) => + s.includes('test/integration/skills-e2e.test.ts') && + s.includes('test/integration/fts-extension-e2e.test.ts'), + ), + ).toEqual([]); }); it('covers every file exactly once, with no overlap between shards', () => { diff --git a/gitnexus/test/unit/doctor-format.test.ts b/gitnexus/test/unit/doctor-format.test.ts index 6c9da7752..662e8e4ef 100644 --- a/gitnexus/test/unit/doctor-format.test.ts +++ b/gitnexus/test/unit/doctor-format.test.ts @@ -11,11 +11,20 @@ import { import { setCliLanguage, type SupportedCliLanguage } from '../../src/cli/i18n/index.js'; import type { NativeCheckResult } from '../../src/core/lbug/native-check.js'; -const nativeProbeState = vi.hoisted(() => ({ vectorLoaded: true })); +const nativeProbeState = vi.hoisted(() => ({ + vectorLoaded: true, + fts: { loaded: true } as { + loaded: boolean; + suppressed?: boolean; + reason?: string; + }, +})); vi.mock('../../src/core/lbug/native-check.js', () => ({ checkLbugNative: () => ({ ok: true, binaryPath: '/synthetic/lbugjs.node' }), - probeFtsExtensionLoad: async () => ({ loaded: true }), + ftsAvailabilityLabel: (probe: { loaded: boolean; suppressed?: boolean }) => + probe.loaded ? 'available' : probe.suppressed ? 'suppressed' : 'unavailable', + probeFtsExtensionLoad: async () => nativeProbeState.fts, probeVectorExtensionLoad: async () => nativeProbeState.vectorLoaded ? { loaded: true } @@ -67,6 +76,7 @@ describe('doctor VECTOR capability claims', () => { afterEach(() => { nativeProbeState.vectorLoaded = true; + nativeProbeState.fts = { loaded: true }; setCliLanguage(null); vi.restoreAllMocks(); for (const key of ENV_KEYS) { @@ -102,6 +112,38 @@ describe('doctor VECTOR capability claims', () => { }); }); +describe('doctor FTS policy claims (U12)', () => { + afterEach(() => { + nativeProbeState.fts = { loaded: true }; + setCliLanguage(null); + vi.restoreAllMocks(); + }); + + it('reports suppressed-by-policy, not unavailable, when the probe is suppressed', async () => { + nativeProbeState.fts = { + loaded: false, + suppressed: true, + reason: 'suppressed by policy GITNEXUS_LBUG_EXTENSION_INSTALL=never', + }; + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + await doctorCommand(); + const output = log.mock.calls.map((args) => args.map(String).join(' ')).join('\n'); + + expect(output).toMatch(/Full-text search:\s+suppressed/); + expect(output).toContain('suppressed by policy GITNEXUS_LBUG_EXTENSION_INSTALL=never'); + expect(output).not.toMatch(/Full-text search:\s+unavailable/); + }); + + it('prints both serve/query and analyze extension install policies', async () => { + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + await doctorCommand(); + const output = log.mock.calls.map((args) => args.map(String).join(' ')).join('\n'); + + expect(output).toMatch(/serve\/query=/); + expect(output).toMatch(/analyze=/); + }); +}); + describe('doctor embedding-runtime support status', () => { it('flags local embeddings as unavailable on macOS Intel (darwin/x64)', () => { const { status, detail } = localEmbeddingDoctorStatus({ diff --git a/gitnexus/test/unit/drop-fts-index-error-classification.test.ts b/gitnexus/test/unit/drop-fts-index-error-classification.test.ts index 272e54f73..17ca4b953 100644 --- a/gitnexus/test/unit/drop-fts-index-error-classification.test.ts +++ b/gitnexus/test/unit/drop-fts-index-error-classification.test.ts @@ -10,6 +10,8 @@ * specific engine failure was not achieved during investigation, but the * classifier's behavior for it is still provable from the message alone. */ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; import { isBenignDropFtsIndexError, dropFTSIndex } from '../../src/core/lbug/lbug-adapter.js'; import { withTestLbugDB } from '../helpers/test-indexed-db.js'; @@ -204,7 +206,10 @@ describe('dropFTSIndex with the FTS extension unloaded (#2841)', () => { sql: string, ...rest: unknown[] ) { - if (/^\s*LOAD EXTENSION fts\b/i.test(sql)) { + if ( + /^\s*LOAD\s+EXTENSION\b/i.test(sql) && + (/\bfts\b/i.test(sql) || /libfts\.lbug_extension/i.test(sql)) + ) { return Promise.reject(new Error(FORCED_LOAD_FAILURE)); } return originalQuery.call(this, sql, ...rest); @@ -237,3 +242,20 @@ describe('dropFTSIndex with the FTS extension unloaded (#2841)', () => { } }, 120_000); }); + +describe('dropFTSIndex fallback diagnosis wiring', () => { + it('extracts the inspect path before resolveFtsVersionPair', () => { + const source = readFileSync( + path.join(__dirname, '..', '..', 'src', 'core', 'lbug', 'lbug-adapter.ts'), + 'utf8', + ); + const drop = source.slice(source.indexOf('export const dropFTSIndex')); + const inspectAt = drop.indexOf('extractExtensionPath(ftsCapability?.reason)'); + const diagnoseAt = drop.indexOf('diagnoseExtensionLoad('); + const pairAt = drop.indexOf('resolveFtsVersionPair(inspectPath)'); + expect(inspectAt).toBeGreaterThan(-1); + expect(diagnoseAt).toBeGreaterThan(inspectAt); + expect(pairAt).toBeGreaterThan(diagnoseAt); + expect(drop).not.toContain('resolveFtsVersionPair(undefined)'); + }); +}); diff --git a/gitnexus/test/unit/extension-load-error.test.ts b/gitnexus/test/unit/extension-load-error.test.ts index 4cbba9233..751e74bc5 100644 --- a/gitnexus/test/unit/extension-load-error.test.ts +++ b/gitnexus/test/unit/extension-load-error.test.ts @@ -7,6 +7,7 @@ import { classifyExtensionLoadError, diagnoseExtensionLoad, extractExtensionPath, + usesClassifiedLoadRemedy, type ExtensionLoadErrorKind, } from '../../src/core/lbug/extension-load-error.js'; @@ -154,9 +155,8 @@ describe('classifyExtensionLoadError', () => { expect(remedy).toMatch(/will NOT help/); // Must not resurrect the old, wrong "retry the network install" instruction. expect(remedy).not.toMatch(/Retry with network access/i); - // #2669: the zero-install path — Git for Windows already ships those DLLs. - expect(remedy).toMatch(/Git Bash/); - expect(remedy).toMatch(/mingw64/); + expect(remedy).toMatch(/system runtime/); + expect(remedy).not.toMatch(/Git Bash|mingw64|Program Files\\Git/i); // Never a user-profile path: remedy text reaches /api/search unredacted. expect(remedy).not.toMatch(/C:\\Users\\/); }); @@ -323,8 +323,8 @@ describe('diagnoseExtensionLoad (structural, language-independent)', () => { const { kind, remedy } = diagnoseExtensionLoad(reason); expect(kind).toBe('missing_dependency'); expect(remedy).toMatch(/vc_redist\.x64\.exe/); - // #2669: the structural remedy carries the same zero-install hint. - expect(remedy).toMatch(/Git Bash/); + expect(remedy).toMatch(/OpenSSL 3/); + expect(remedy).not.toMatch(/Git Bash|mingw64|Program Files\\Git/i); expect(remedy).not.toMatch(/C:\\Users\\/); } finally { rmSync(dir, { recursive: true, force: true }); @@ -355,4 +355,67 @@ describe('diagnoseExtensionLoad (structural, language-independent)', () => { ), ).toMatchObject({ kind: 'missing_dependency' }); }); + + it('a valid artifact with mismatched versions is version_skew, not missing_dependency (U3)', () => { + const dir = mkdtempSync(join(tmpdir(), 'ext-diag-skew-')); + const file = join(dir, 'libfts.lbug_extension'); + writeFileSync(file, buildHostValidBinary()); + try { + const reason = `Failed to load library: ${file} which is needed by extension: fts. Error: `; + const { kind, remedy } = diagnoseExtensionLoad(reason, 'FTS', file, { + expected: '0.18.1', + found: '0.17.0', + }); + expect(kind).toBe('version_skew'); + expect(remedy).toContain('0.18.1'); + expect(remedy).toContain('0.17.0'); + expect(remedy).not.toMatch(/VC\+\+|OpenSSL|vcredist/i); + expect(remedy).not.toContain(file); + expect(usesClassifiedLoadRemedy(kind)).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('matching versions plus a Windows 126 signature stay missing_dependency', () => { + const dir = mkdtempSync(join(tmpdir(), 'ext-diag-match-')); + const file = join(dir, 'libfts.lbug_extension'); + writeFileSync(file, buildHostValidBinary()); + try { + const reason = `Failed to load library: ${file} which is needed by extension: fts. Error: The specified module could not be found.`; + expect( + diagnoseExtensionLoad(reason, 'FTS', file, { expected: '0.18.1', found: '0.18.1' }), + ).toMatchObject({ kind: 'missing_dependency' }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a header-valid file the loader calls "file too short" stays corrupt_file even when versions also differ', () => { + const dir = mkdtempSync(join(tmpdir(), 'ext-diag-valid-trunc-skew-')); + const file = join(dir, 'libfts.lbug_extension'); + writeFileSync(file, buildHostValidBinary()); + try { + const reason = `Failed to load library: ${file} which is needed by extension: fts. Error: file too short`; + expect( + diagnoseExtensionLoad(reason, 'FTS', file, { expected: '0.18.1', found: '0.17.0' }), + ).toMatchObject({ kind: 'corrupt_file' }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('a truncated artifact stays corrupt even when versions also differ', () => { + const dir = mkdtempSync(join(tmpdir(), 'ext-diag-trunc-skew-')); + const file = join(dir, 'libfts.lbug_extension'); + writeFileSync(file, Buffer.from('short')); + try { + const reason = `Failed to load library: ${file} which is needed by extension: fts. Error: file too short`; + expect( + diagnoseExtensionLoad(reason, 'FTS', file, { expected: '0.18.1', found: '0.17.0' }), + ).toMatchObject({ kind: 'corrupt_file' }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/gitnexus/test/unit/fts-artifact-coverage.test.ts b/gitnexus/test/unit/fts-artifact-coverage.test.ts new file mode 100644 index 000000000..35f252f20 --- /dev/null +++ b/gitnexus/test/unit/fts-artifact-coverage.test.ts @@ -0,0 +1,306 @@ +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; +import { load } from 'js-yaml'; +import { + assertSafeArtifactDest, + officialArtifactUrl, +} from '../../../.github/scripts/fetch-lbug-fts-artifacts.mjs'; + +/** + * Coverage for the FTS pairing gate `scripts/assert-publish-fts-coverage.cjs`. + * + * U13: a core bump must not ship a skewed extension artifact. The gate is CJS + * with a pure pairing predicate; this suite imports that predicate and also + * asserts the Dependabot ignore over the parsed config so removing it fails + * a test rather than silently re-enabling daily bumps. + */ +const requireCjs = createRequire(import.meta.url); +const SCRIPT = fileURLToPath( + new URL('../../scripts/assert-publish-fts-coverage.cjs', import.meta.url), +); +const { + findPairingProblems, + findArtifactProblems, + filesCoverFtsArtifacts, + parseSha256Sums, + supportedTuplesFromManifest, +} = requireCjs(SCRIPT); + +const REPO_ROOT = fileURLToPath(new URL('../../../', import.meta.url)); +const GITNEXUS_ROOT = fileURLToPath(new URL('../../', import.meta.url)); + +describe('findPairingProblems (pure pairing core)', () => { + it('passes when the manifest pin matches the installed core', () => { + expect( + findPairingProblems({ + installedCoreVersion: '0.18.3', + manifestCoreVersion: '0.18.3', + manifestExtensionVersion: '0.18.1', + }), + ).toEqual([]); + }); + + it('fails when the installed core is bumped without a manifest update, naming both versions', () => { + const problems = findPairingProblems({ + installedCoreVersion: '0.18.4', + manifestCoreVersion: '0.18.3', + manifestExtensionVersion: '0.18.1', + }); + expect(problems).toHaveLength(1); + expect(problems[0]).toContain('0.18.4'); + expect(problems[0]).toContain('0.18.3'); + }); + + it('fails when a caret prefix is stripped-equivalent but not an exact x.y.z pin', () => { + const problems = findPairingProblems({ + installedCoreVersion: '^0.18.3', + manifestCoreVersion: '0.18.3', + manifestExtensionVersion: '0.18.1', + }); + expect(problems.length).toBeGreaterThan(0); + }); +}); + +describe('Dependabot ignore for @ladybugdb/core', () => { + it('ignores the core package in the gitnexus npm ecosystem so daily bumps stay off', () => { + const raw = readFileSync(path.join(REPO_ROOT, '.github/dependabot.yml'), 'utf8'); + const parsed = load(raw) as { + updates?: Array<{ + 'package-ecosystem'?: string; + directory?: string; + ignore?: Array<{ 'dependency-name'?: string }>; + }>; + }; + const gitnexusNpm = (parsed.updates ?? []).find( + (u) => u['package-ecosystem'] === 'npm' && u.directory === '/gitnexus', + ); + expect(gitnexusNpm, 'expected an npm ecosystem entry for /gitnexus').toBeDefined(); + const names = (gitnexusNpm?.ignore ?? []).map((i) => i['dependency-name']); + expect(names).toContain('@ladybugdb/core'); + }); +}); + +describe('real repo pairing (guards against a silent core bump)', () => { + it('the script exits 0 against the committed repo state', () => { + const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 20_000 }); + expect(r.status, r.stderr + r.stdout).toBe(0); + expect(r.stdout).toContain('[fts-pairing] OK'); + }); + + it('reads the installed core from package.json, not from a network pin', () => { + const pkg = JSON.parse(readFileSync(path.join(GITNEXUS_ROOT, 'package.json'), 'utf8')) as { + dependencies: Record; + files: string[]; + }; + const manifest = JSON.parse( + readFileSync(path.join(GITNEXUS_ROOT, 'vendor/lbug-fts/manifest.json'), 'utf8'), + ) as { coreVersion: string; extensionVersion: string }; + expect(pkg.dependencies['@ladybugdb/core']).toBe(manifest.coreVersion); + expect(manifest.extensionVersion).toMatch(/^\d+\.\d+\.\d+$/); + expect(pkg.files).toContain('vendor'); + }); +}); + +const HASH_A = 'a'.repeat(64); +const HASH_B = 'b'.repeat(64); +const FILENAME = 'libfts.lbug_extension'; +const TUPLES = ['linux-x64', 'linux-arm64', 'darwin-x64', 'darwin-arm64', 'win32-x64'] as const; + +const presentArtifacts = Object.fromEntries( + TUPLES.map((tuple) => [tuple, { exists: true, hash: HASH_A, sizeBytes: 100 }]), +); +const matchingChecksums = Object.fromEntries( + TUPLES.map((tuple) => [`${tuple}/${FILENAME}`, HASH_A]), +); + +describe('findArtifactProblems (U1 integrity gate)', () => { + it('passes when every tuple exists, hashes match, files cover vendor, and win32-arm64 is unsupported', () => { + expect( + findArtifactProblems({ + tuples: [...TUPLES], + unsupportedTuples: ['win32-arm64'], + filesField: ['dist', 'vendor'], + checksumByRelPath: matchingChecksums, + artifactByTuple: presentArtifacts, + filename: FILENAME, + }), + ).toEqual([]); + }); + + it('fails when a tuple directory is removed', () => { + const { 'linux-arm64': _removed, ...rest } = presentArtifacts; + const problems = findArtifactProblems({ + tuples: [...TUPLES], + unsupportedTuples: ['win32-arm64'], + filesField: ['vendor'], + checksumByRelPath: matchingChecksums, + artifactByTuple: rest, + filename: FILENAME, + }); + expect(problems.some((p) => p.includes('missing artifact for linux-arm64'))).toBe(true); + }); + + it('fails when a checksum is edited to a wrong value', () => { + const problems = findArtifactProblems({ + tuples: [...TUPLES], + unsupportedTuples: ['win32-arm64'], + filesField: ['vendor'], + checksumByRelPath: { ...matchingChecksums, [`linux-x64/${FILENAME}`]: HASH_B }, + artifactByTuple: presentArtifacts, + filename: FILENAME, + }); + expect(problems.some((p) => p.includes('checksum mismatch for linux-x64'))).toBe(true); + expect(problems.some((p) => p.includes(HASH_B) && p.includes(HASH_A))).toBe(true); + }); + + it('passes with win32-arm64 absent because it is declared unsupported', () => { + expect(presentArtifacts['win32-arm64']).toBeUndefined(); + expect( + findArtifactProblems({ + tuples: [...TUPLES], + unsupportedTuples: ['win32-arm64'], + filesField: ['vendor'], + checksumByRelPath: matchingChecksums, + artifactByTuple: presentArtifacts, + filename: FILENAME, + }), + ).toEqual([]); + }); + + it('fails when a files entry stops covering the artifact path', () => { + const problems = findArtifactProblems({ + tuples: [...TUPLES], + unsupportedTuples: ['win32-arm64'], + filesField: ['dist', 'vendor/**/package.json'], + checksumByRelPath: matchingChecksums, + artifactByTuple: presentArtifacts, + filename: FILENAME, + }); + expect(problems.some((p) => p.includes('files no longer covers'))).toBe(true); + }); + + it('fails when tuples is empty even if files and checksums look fine', () => { + const problems = findArtifactProblems({ + tuples: [], + unsupportedTuples: ['win32-arm64'], + filesField: ['vendor'], + checksumByRelPath: matchingChecksums, + artifactByTuple: presentArtifacts, + filename: FILENAME, + }); + expect(problems.some((p) => p.includes('manifest.tuples is empty'))).toBe(true); + }); + + it('fails when the manifest filename is not a .lbug_extension', () => { + const problems = findArtifactProblems({ + tuples: [...TUPLES], + unsupportedTuples: ['win32-arm64'], + filesField: ['vendor'], + checksumByRelPath: matchingChecksums, + artifactByTuple: presentArtifacts, + filename: '../../manifest.json', + }); + expect(problems.some((p) => p.includes('invalid FTS artifact filename'))).toBe(true); + }); + + it('fails when a required supported tuple is omitted from the manifest list', () => { + const problems = findArtifactProblems({ + tuples: TUPLES.filter((t) => t !== 'darwin-arm64'), + unsupportedTuples: ['win32-arm64'], + filesField: ['vendor'], + checksumByRelPath: matchingChecksums, + artifactByTuple: presentArtifacts, + filename: FILENAME, + }); + expect(problems.some((p) => p.includes('missing required darwin-arm64'))).toBe(true); + }); +}); + +describe('assertSafeArtifactDest (fetch-script path allowlist)', () => { + const prebuildsDir = path.join(GITNEXUS_ROOT, 'vendor', 'lbug-fts', 'prebuilds'); + + it('rejects a path-escaping tuple', () => { + expect(() => + assertSafeArtifactDest({ + prebuildsDir, + tuple: '../evil', + filename: FILENAME, + }), + ).toThrow(/unsafe FTS artifact tuple/); + }); + + it('rejects a filename that is not a .lbug_extension', () => { + expect(() => + assertSafeArtifactDest({ + prebuildsDir, + tuple: 'linux-x64', + filename: 'not-an-extension', + }), + ).toThrow(/unsafe FTS artifact filename/); + }); +}); + +describe('officialArtifactUrl (fetch-script origin pin)', () => { + const valid = { + officialRepo: 'https://extension.ladybugdb.com/', + extensionVersion: '0.18.1', + filename: FILENAME, + }; + + it('builds a URL only for the official host and allowlisted path segments', () => { + expect(officialArtifactUrl(valid, 'linux_amd64')).toBe( + `https://extension.ladybugdb.com/v0.18.1/linux_amd64/fts/${FILENAME}`, + ); + }); + + it('refuses a redirected officialRepo', () => { + expect(() => + officialArtifactUrl({ ...valid, officialRepo: 'https://evil.example/' }, 'linux_amd64'), + ).toThrow(/unofficial FTS repo/); + }); +}); + +describe('filesCoverFtsArtifacts', () => { + it('accepts a broad vendor entry and the lean-publish prebuilds glob', () => { + expect(filesCoverFtsArtifacts(['vendor'])).toBe(true); + expect(filesCoverFtsArtifacts(['vendor/**/prebuilds/**'])).toBe(true); + expect(filesCoverFtsArtifacts(['dist'])).toBe(false); + }); +}); + +describe('committed FTS artifacts and fetch-script placement', () => { + it('every manifest-listed file exists with a matching SHA-256', () => { + const manifest = JSON.parse( + readFileSync(path.join(GITNEXUS_ROOT, 'vendor/lbug-fts/manifest.json'), 'utf8'), + ); + const tuples = supportedTuplesFromManifest(manifest); + const sums = parseSha256Sums( + readFileSync(path.join(GITNEXUS_ROOT, 'vendor/lbug-fts/prebuilds/SHA256SUMS'), 'utf8'), + ); + expect(tuples).toEqual([...TUPLES]); + for (const tuple of tuples) { + const rel = `${tuple}/${manifest.filename}`; + const filePath = path.join(GITNEXUS_ROOT, 'vendor/lbug-fts/prebuilds', rel); + expect(existsSync(filePath), rel).toBe(true); + const actual = createHash('sha256').update(readFileSync(filePath)).digest('hex'); + expect(sums[rel], rel).toBe(actual); + expect(readFileSync(filePath).byteLength).toBeGreaterThan(1024 * 1024); + } + }); + + it('keeps the fetch script outside the published package', () => { + const pkg = JSON.parse(readFileSync(path.join(GITNEXUS_ROOT, 'package.json'), 'utf8')) as { + files: string[]; + }; + expect(pkg.files).not.toContain('.github'); + expect(pkg.files.some((f) => String(f).includes('fetch-lbug-fts'))).toBe(false); + expect( + readFileSync(path.join(REPO_ROOT, '.github/scripts/fetch-lbug-fts-artifacts.mjs'), 'utf8'), + ).toContain('vendor/lbug-fts/prebuilds'); + }); +}); diff --git a/gitnexus/test/unit/fts-availability-resolve.test.ts b/gitnexus/test/unit/fts-availability-resolve.test.ts new file mode 100644 index 000000000..aa6669d23 --- /dev/null +++ b/gitnexus/test/unit/fts-availability-resolve.test.ts @@ -0,0 +1,52 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { requireFtsResourceOrSkip, resolveFtsExtension } from '../helpers/fts-availability.js'; + +const tmpRoots: string[] = []; + +const makeRoot = (): string => { + const root = mkdtempSync(join(tmpdir(), 'gn-fts-gate-')); + tmpRoots.push(root); + return root; +}; + +afterEach(() => { + delete process.env.GITNEXUS_REQUIRE_FTS; + for (const root of tmpRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +describe('resolveFtsExtension (U12 CI gates)', () => { + it('reports available from a vendored artifact when ~/.lbdb is empty', () => { + const vendorRoot = makeRoot(); + const emptyHome = join(makeRoot(), 'extension'); + const tuple = `${process.platform}-${process.arch}`; + const dir = join(vendorRoot, 'lbug-fts', 'prebuilds', tuple); + mkdirSync(dir, { recursive: true }); + const artifact = join(dir, 'libfts.lbug_extension'); + writeFileSync(artifact, 'placeholder'); + + const resolved = resolveFtsExtension({ vendorRoot, homeExtensionRoot: emptyHome }); + expect(resolved).toBe(artifact); + + process.env.GITNEXUS_REQUIRE_FTS = '1'; + expect(() => + requireFtsResourceOrSkip({ skip: () => undefined }, resolved, 'installed FTS extension'), + ).not.toThrow(); + }); + + it('still throws under GITNEXUS_REQUIRE_FTS=1 when nothing resolves', () => { + const vendorRoot = makeRoot(); + const emptyHome = join(makeRoot(), 'extension'); + const resolved = resolveFtsExtension({ vendorRoot, homeExtensionRoot: emptyHome }); + expect(resolved).toBeNull(); + + process.env.GITNEXUS_REQUIRE_FTS = '1'; + expect(() => + requireFtsResourceOrSkip({ skip: () => undefined }, resolved, 'installed FTS extension'), + ).toThrow(/GITNEXUS_REQUIRE_FTS=1/); + }); +}); diff --git a/gitnexus/test/unit/fts-degraded-warning.test.ts b/gitnexus/test/unit/fts-degraded-warning.test.ts index 42425be99..dbfdac8c7 100644 --- a/gitnexus/test/unit/fts-degraded-warning.test.ts +++ b/gitnexus/test/unit/fts-degraded-warning.test.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { extensionManager, @@ -126,18 +129,25 @@ describe('ftsDegradedWarning (#2374)', () => { }); it('keeps the reinstall guidance for a never-installed extension', async () => { - await extensionManager.ensure( - vi - .fn() - .mockRejectedValue( - new Error('Extension "fts" is an official extension and has not been installed.'), - ), - 'fts', - 'FTS', - { policy: 'load-only' }, - ); + // Empty vendor root so named "not installed" is not reclassified against + // a packaged, structurally valid artifact (that path is missing_dependency). + const vendorRoot = mkdtempSync(path.join(tmpdir(), 'gn-fts-empty-vendor-')); + try { + await extensionManager.ensure( + vi + .fn() + .mockRejectedValue( + new Error('Extension "fts" is an official extension and has not been installed.'), + ), + 'fts', + 'FTS', + { policy: 'load-only', vendorRoot }, + ); - expect(ftsDegradedWarning()).toContain('--repair-fts'); + expect(ftsDegradedWarning()).toContain('--repair-fts'); + } finally { + rmSync(vendorRoot, { recursive: true, force: true }); + } }); it('caches the load diagnosis on the capability so the warning does no per-request I/O (#2383 F3)', async () => { diff --git a/gitnexus/test/unit/fts-policy.test.ts b/gitnexus/test/unit/fts-policy.test.ts index e3d9a9d7a..b62120579 100644 --- a/gitnexus/test/unit/fts-policy.test.ts +++ b/gitnexus/test/unit/fts-policy.test.ts @@ -4,13 +4,16 @@ import os from 'node:os'; import path from 'node:path'; import { FTS_DISABLED_MESSAGE, + FTS_SKIP_REASONS, + formatAnalyzeFtsSkipSummary, getFtsDisabledReason, isExplicitFtsDisablement, resolveFtsDisableReason, withExplicitFtsDisablement, + type FtsSkipReason, } from '../../src/core/search/fts-policy.js'; -import type { RepoMeta } from '../../src/storage/repo-meta.js'; -import { ftsDegradedWarning } from '../../src/core/search/fts-indexes.js'; +import type { PersistedFtsSkipReason, RepoMeta } from '../../src/storage/repo-meta.js'; +import { classifyFtsBuildError, ftsDegradedWarning } from '../../src/core/search/fts-indexes.js'; import { searchFTSFromLbug } from '../../src/core/search/bm25-index.js'; import { hybridSearch } from '../../src/core/search/hybrid-search.js'; import { extensionManager, resetExtensionState } from '../../src/core/lbug/extension-loader.js'; @@ -33,11 +36,19 @@ describe('explicit FTS opt-out', () => { expect(isExplicitFtsDisablement('disabled-by-flag')).toBe(true); expect(isExplicitFtsDisablement('disabled-by-env')).toBe(true); expect(isExplicitFtsDisablement('build-failed')).toBe(false); + expect(isExplicitFtsDisablement('native-abort')).toBe(false); + expect(isExplicitFtsDisablement('tuple-missing')).toBe(false); }); it('does not infer intent from a failed or legacy index', () => { expect(getFtsDisabledReason(undefined)).toBeUndefined(); - for (const skipReason of [undefined, 'build-failed', 'extension-unavailable'] as const) { + for (const skipReason of [ + undefined, + 'build-failed', + 'extension-unavailable', + 'native-abort', + 'tuple-missing', + ] as const) { expect( getFtsDisabledReason({ provider: 'ladybugdb-fts', status: 'unavailable', skipReason }), ).toBeUndefined(); @@ -150,3 +161,99 @@ describe('explicit FTS opt-out', () => { expect(executeQuery).not.toHaveBeenCalled(); }); }); + +describe('FTS skip-reason members (U6)', () => { + type SameSkipReason = FtsSkipReason extends PersistedFtsSkipReason + ? PersistedFtsSkipReason extends FtsSkipReason + ? true + : never + : never; + const _storageMirrorsCore: SameSkipReason = true; + void _storageMirrorsCore; + + it('round-trips native-abort and tuple-missing through the capability stamp', () => { + const base = { + indexedAt: '2026-01-01T00:00:00.000Z', + lastCommit: 'abc', + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' as const }, + fts: { provider: 'ladybugdb-fts', status: 'available' as const }, + vectorSearch: { + provider: 'ladybugdb-vector', + status: 'vector-index' as const, + exactScanLimit: 10, + }, + }, + } as RepoMeta; + + for (const reason of ['native-abort', 'tuple-missing'] as const) { + const stamped: RepoMeta = { + ...base, + capabilities: { + ...base.capabilities!, + fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: reason }, + }, + }; + expect(stamped.capabilities?.fts?.skipReason).toBe(reason); + expect(isExplicitFtsDisablement(stamped.capabilities?.fts?.skipReason)).toBe(false); + } + }); + + it('lets the storage union accept every core-side member', () => { + for (const reason of FTS_SKIP_REASONS) { + const persisted: PersistedFtsSkipReason = reason; + const core: FtsSkipReason = persisted; + expect(core).toBe(reason); + } + }); + + it('keeps explicit disablement ahead of the new failure members', () => { + expect(resolveFtsDisableReason(true, '1')).toBe('disabled-by-flag'); + expect(isExplicitFtsDisablement(resolveFtsDisableReason(true))).toBe(true); + expect(formatAnalyzeFtsSkipSummary('disabled-by-flag')).toBe(FTS_DISABLED_MESSAGE); + expect(formatAnalyzeFtsSkipSummary('native-abort')).not.toContain( + 'GITNEXUS_LBUG_EXTENSION_INSTALL=auto', + ); + expect(formatAnalyzeFtsSkipSummary('tuple-missing')).not.toContain( + 'GITNEXUS_LBUG_EXTENSION_INSTALL=auto', + ); + }); + + it('still classifies a message-bearing tokenizer failure as capability', () => { + expect(classifyFtsBuildError('Runtime exception: Failed calling LOWER: Invalid UTF-8.')).toBe( + 'capability', + ); + }); + + it('names each skip reason instead of falling through to the network-install remedy', () => { + const nativeAbort = formatAnalyzeFtsSkipSummary('native-abort'); + expect(nativeAbort).toMatch(/aborted while building/); + expect(nativeAbort.replaceAll('gitnexus analyze --repair-fts', '')).not.toContain( + 'gitnexus analyze', + ); + expect(formatAnalyzeFtsSkipSummary('tuple-missing')).toMatch(/no packaged FTS artifact/); + expect(formatAnalyzeFtsSkipSummary('build-failed')).toMatch(/search index build failed/); + expect(formatAnalyzeFtsSkipSummary('extension-unavailable')).toContain( + 'GITNEXUS_LBUG_EXTENSION_INSTALL=auto', + ); + expect(formatAnalyzeFtsSkipSummary(undefined)).toContain( + 'GITNEXUS_LBUG_EXTENSION_INSTALL=auto', + ); + }); + + it('prints the skip summary on the already-up-to-date CLI path whenever FTS was skipped', async () => { + const { readFile } = await import('node:fs/promises'); + const { fileURLToPath } = await import('node:url'); + const analyzeSrc = await readFile( + fileURLToPath(new URL('../../src/cli/analyze.ts', import.meta.url)), + 'utf8', + ); + const alreadyUpToDate = analyzeSrc.match( + /Already up to date[\s\S]{0,400}if \(runOptions\.registryName\)/, + ); + expect(alreadyUpToDate).not.toBeNull(); + expect(alreadyUpToDate![0]).toContain('if (result.ftsSkipped)'); + expect(alreadyUpToDate![0]).toContain('formatAnalyzeFtsSkipSummary(result.ftsSkipReason)'); + expect(alreadyUpToDate![0]).not.toContain('isExplicitFtsDisablement'); + }); +}); diff --git a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts index ec42adc3d..fa3190364 100644 --- a/gitnexus/test/unit/group/sync-windowed-resolution.test.ts +++ b/gitnexus/test/unit/group/sync-windowed-resolution.test.ts @@ -156,6 +156,14 @@ vi.mock('../../../src/core/lbug/sidecar-recovery.js', () => ({ quarantineWalForMissingShadow: vi.fn().mockResolvedValue(''), renameFailureMessage: vi.fn((p: string) => `rename failed for ${p}`), statIfExists: vi.fn().mockResolvedValue(null), + assertReadOnlyFtsCrashSafe: vi.fn().mockResolvedValue(undefined), + FtsReaderUnrepairableError: class FtsReaderUnrepairableError extends Error { + readonly code = 'FTS_READER_UNREPAIRABLE' as const; + constructor(dbPath = '') { + super(dbPath); + this.name = 'FtsReaderUnrepairableError'; + } + }, })); // The registry read happens in syncGroup's else branch; resolveRepoHandle is diff --git a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts index 13a1c3940..0ee1bc6d6 100644 --- a/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts +++ b/gitnexus/test/unit/lbug-adapter-wal-schema.test.ts @@ -67,6 +67,9 @@ function makeFsMock(dbPath: string) { mkdir: vi.fn(async () => {}), open: makeOpenMock(), readdir: vi.fn(async () => []), + readFile: vi.fn(async () => { + throw ENOENT; + }), }, }; } @@ -120,6 +123,32 @@ describe('doInitLbug WAL corruption guard — structural', () => { expect(walGuardIdx).toBeLessThan(warnIdx); }); + it('refuses a read-only FTS crash before preflight', () => { + const readOnlyBlock = adapterSource.slice(adapterSource.indexOf('if (readOnly)')); + const refuseIdx = readOnlyBlock.indexOf('assertReadOnlyFtsCrashSafe(dbPath)'); + const preflightIdx = readOnlyBlock.indexOf('preflightLbugSidecars'); + expect(refuseIdx).toBeGreaterThan(-1); + expect(preflightIdx).toBeGreaterThan(refuseIdx); + }); + + it('writable missing-shadow reopen can pass FTS crash evidence; read-only must not', () => { + const evidenceStart = adapterSource.indexOf('const writableFtsCrashWalEvidence'); + const writableStart = adapterSource.indexOf('const reopenWritableAfterMissingShadow'); + const readOnlyStart = adapterSource.indexOf('const reopenReadOnlyAfterMissingShadow'); + expect(evidenceStart).toBeGreaterThan(-1); + expect(writableStart).toBeGreaterThan(evidenceStart); + expect(readOnlyStart).toBeGreaterThan(-1); + expect(adapterSource.slice(evidenceStart, writableStart + 600)).toMatch( + /fts-inplace-checkpointed/, + ); + expect(adapterSource.slice(writableStart, writableStart + 600)).toMatch( + /writableFtsCrashWalEvidence/, + ); + expect(adapterSource.slice(readOnlyStart, evidenceStart)).not.toMatch( + /fts-inplace-checkpointed/, + ); + }); + it('imports throwIfStorageVersionMismatch and uses it in the schema catch', () => { expect(adapterSource).toMatch(/throwIfStorageVersionMismatch/); expect(schemaLoopBody).toMatch(/isStorageVersionMismatchError\(err\)/); @@ -641,6 +670,9 @@ function makeFsMockWithWalSize( mkdir: vi.fn(async () => {}), open: makeOpenMock(), readdir: vi.fn(async () => []), + readFile: vi.fn(async () => { + throw ENOENT; + }), }, }; } diff --git a/gitnexus/test/unit/lbug-extension-loader.test.ts b/gitnexus/test/unit/lbug-extension-loader.test.ts index a3e52e917..acdfe7760 100644 --- a/gitnexus/test/unit/lbug-extension-loader.test.ts +++ b/gitnexus/test/unit/lbug-extension-loader.test.ts @@ -1,4 +1,8 @@ -import { describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterAll, describe, expect, it, vi } from 'vitest'; +import { escapeCypherString } from '../../src/core/lbug/cypher-escape.js'; import { ExtensionManager, getExtensionInstallChildProcessArgs, @@ -6,6 +10,14 @@ import { getExtensionInstallTimeoutMs, type ExtensionInstallResult, } from '../../src/core/lbug/extension-loader.js'; +import { diagnoseExtensionLoad } from '../../src/core/lbug/extension-load-error.js'; + +const emptyVendor = mkdtempSync(path.join(tmpdir(), 'gn-fts-empty-vendor-')); +const noVendored = { vendorRoot: emptyVendor }; + +afterAll(() => { + rmSync(emptyVendor, { recursive: true, force: true }); +}); const okInstall: ExtensionInstallResult = { success: true, @@ -31,7 +43,7 @@ describe('ExtensionManager — LOAD-first behavior', () => { const manager = new ExtensionManager({ policy: 'auto', installExtension }); const query = vi.fn().mockResolvedValue({}); - await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(true); + await expect(manager.ensure(query, 'fts', 'FTS', noVendored)).resolves.toBe(true); expect(query.mock.calls.map(([sql]) => sql)).toEqual(['LOAD EXTENSION fts']); expect(installExtension).not.toHaveBeenCalled(); @@ -43,7 +55,7 @@ describe('ExtensionManager — LOAD-first behavior', () => { const manager = new ExtensionManager({ policy: 'auto', installExtension }); const query = vi.fn().mockRejectedValue(new Error('Extension fts is already loaded')); - await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(true); + await expect(manager.ensure(query, 'fts', 'FTS', noVendored)).resolves.toBe(true); expect(installExtension).not.toHaveBeenCalled(); }); }); @@ -57,9 +69,9 @@ describe('ExtensionManager — install policies', () => { .mockRejectedValueOnce(new Error('Extension "fts" not found')) .mockResolvedValueOnce({}); - await expect(manager.ensure(query, 'fts', 'FTS', { installTimeoutMs: 1234 })).resolves.toBe( - true, - ); + await expect( + manager.ensure(query, 'fts', 'FTS', { ...noVendored, installTimeoutMs: 1234 }), + ).resolves.toBe(true); // The LOAD failure reason is threaded to the installer so it can pick // INSTALL vs FORCE INSTALL from the error class (#2374, PR #2375). @@ -77,7 +89,7 @@ describe('ExtensionManager — install policies', () => { const manager = new ExtensionManager({ policy: 'load-only', installExtension, warn }); const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); - await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + await expect(manager.ensure(query, 'fts', 'FTS', noVendored)).resolves.toBe(false); expect(installExtension).not.toHaveBeenCalled(); expect(warn).toHaveBeenCalledWith(expect.stringContaining('continuing without FTS features')); @@ -146,7 +158,7 @@ describe('ExtensionManager — reason strings carry the real LOAD error (#2374)' ), ); - await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + await expect(manager.ensure(query, 'fts', 'FTS', noVendored)).resolves.toBe(false); expect(manager.getCapabilities()).toMatchObject([ { @@ -165,7 +177,7 @@ describe('ExtensionManager — reason strings carry the real LOAD error (#2374)' const manager = new ExtensionManager({ policy: 'auto', installExtension, warn: noopWarn }); const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); - await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + await expect(manager.ensure(query, 'fts', 'FTS', noVendored)).resolves.toBe(false); expect(manager.getCapabilities()).toMatchObject([ { @@ -183,7 +195,7 @@ describe('ExtensionManager — reason strings carry the real LOAD error (#2374)' .fn() .mockRejectedValue(new Error('version mismatch: extension built for 0.17.0')); - await expect(manager.ensure(query, 'fts', 'FTS')).resolves.toBe(false); + await expect(manager.ensure(query, 'fts', 'FTS', noVendored)).resolves.toBe(false); expect(manager.getCapabilities()).toMatchObject([ { @@ -221,13 +233,13 @@ describe('ExtensionManager — caching', () => { }); const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); - await manager.ensure(query, 'fts', 'FTS'); + await manager.ensure(query, 'fts', 'FTS', noVendored); expect(manager.getCapabilities()).toHaveLength(1); manager.reset(); expect(manager.getCapabilities()).toEqual([]); - await manager.ensure(query, 'fts', 'FTS'); + await manager.ensure(query, 'fts', 'FTS', noVendored); expect(installExtension).toHaveBeenCalledTimes(2); }); }); @@ -238,7 +250,7 @@ describe('ExtensionManager — observability', () => { const okQuery = vi.fn().mockResolvedValue({}); const failQuery = vi.fn().mockRejectedValue(new Error('Extension "vector" not found')); - await manager.ensure(okQuery, 'fts', 'FTS'); + await manager.ensure(okQuery, 'fts', 'FTS', noVendored); await manager.ensure(failQuery, 'vector', 'VECTOR'); expect(manager.getCapabilities()).toMatchObject([ @@ -253,8 +265,8 @@ describe('ExtensionManager — observability', () => { const manager = new ExtensionManager({ policy: 'load-only', installExtension, warn }); const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); - await manager.ensure(query, 'fts', 'FTS'); - await manager.ensure(query, 'fts', 'FTS'); + await manager.ensure(query, 'fts', 'FTS', noVendored); + await manager.ensure(query, 'fts', 'FTS', noVendored); expect(warn).toHaveBeenCalledTimes(1); }); @@ -273,11 +285,13 @@ describe('ExtensionManager — observability', () => { .mockResolvedValueOnce({}); await expect( - manager.ensure(query, 'fts', 'FTS', { policy: 'load-only', quiet: true }), + manager.ensure(query, 'fts', 'FTS', { ...noVendored, policy: 'load-only', quiet: true }), ).resolves.toBe(false); expect(warn).not.toHaveBeenCalled(); - await expect(manager.ensure(query, 'fts', 'FTS', { policy: 'auto' })).resolves.toBe(true); + await expect( + manager.ensure(query, 'fts', 'FTS', { ...noVendored, policy: 'auto' }), + ).resolves.toBe(true); expect(warn).not.toHaveBeenCalled(); expect(manager.getCapabilities()).toEqual([{ name: 'fts', loaded: true }]); }); @@ -287,8 +301,8 @@ describe('ExtensionManager — observability', () => { const manager = new ExtensionManager({ policy: 'load-only', warn }); const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); - await manager.ensure(query, 'fts', 'FTS', { quiet: true }); - await manager.ensure(query, 'fts', 'FTS'); + await manager.ensure(query, 'fts', 'FTS', { ...noVendored, quiet: true }); + await manager.ensure(query, 'fts', 'FTS', noVendored); expect(warn).toHaveBeenCalledTimes(1); expect(warn).toHaveBeenCalledWith(expect.stringContaining('continuing without FTS features')); @@ -413,3 +427,264 @@ describe('getExtensionInstallTimeoutMs', () => { } }); }); + +const buildHostValidBinary = (): Buffer => { + const arm = process.arch === 'arm64'; + if (process.platform === 'win32') { + const peOff = 0x80; + const b = Buffer.alloc(peOff + 8); + b[0] = 0x4d; + b[1] = 0x5a; + b.writeUInt32LE(peOff, 0x3c); + b[peOff] = 0x50; + b[peOff + 1] = 0x45; + b.writeUInt16LE(arm ? 0xaa64 : 0x8664, peOff + 4); + return b; + } + if (process.platform === 'darwin') { + const b = Buffer.alloc(32); + b.writeUInt32LE(0xfeedfacf, 0); + b.writeUInt32LE(arm ? 0x0100000c : 0x01000007, 4); + return b; + } + const b = Buffer.alloc(64); + b[0] = 0x7f; + b[1] = 0x45; + b[2] = 0x4c; + b[3] = 0x46; + b[4] = 2; + b[5] = 1; + b.writeUInt16LE(arm ? 0xb7 : 0x3e, 18); + return b; +}; + +const writeVendorArtifact = ( + root: string, + tuple: string, + filename = 'libfts.lbug_extension', +): string => { + const dir = path.join(root, 'lbug-fts', 'prebuilds', tuple); + mkdirSync(dir, { recursive: true }); + const artifact = path.join(dir, filename); + writeFileSync(artifact, 'placeholder'); + writeFileSync( + path.join(root, 'lbug-fts', 'manifest.json'), + JSON.stringify({ + filename, + unsupportedTuples: [{ tuple: 'win32-arm64', reason: 'none' }], + }), + ); + return artifact; +}; + +describe('ExtensionManager — vendored-first FTS (U2)', () => { + const tmpRoots: string[] = []; + const makeRoot = (): string => { + const root = mkdtempSync(path.join(tmpdir(), 'gn-fts-vendor-')); + tmpRoots.push(root); + return root; + }; + + afterAll(() => { + for (const root of tmpRoots) rmSync(root, { recursive: true, force: true }); + }); + + it('loads a present artifact without spawning an installer child', async () => { + const vendorRoot = makeRoot(); + const artifact = writeVendorArtifact(vendorRoot, 'linux-x64'); + const installExtension = vi.fn(); + const manager = new ExtensionManager({ policy: 'auto', installExtension }); + const query = vi.fn().mockResolvedValue({}); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(true); + + expect(query).toHaveBeenCalledTimes(1); + expect(query.mock.calls[0][0]).toBe( + `LOAD EXTENSION '${escapeCypherString(realpathSync(artifact))}'`, + ); + expect(installExtension).not.toHaveBeenCalled(); + expect(JSON.stringify(manager.getCapabilities())).not.toContain(vendorRoot); + }); + + it('still loads under load-only when the artifact is present', async () => { + const vendorRoot = makeRoot(); + writeVendorArtifact(vendorRoot, 'linux-x64'); + const installExtension = vi.fn(); + const manager = new ExtensionManager({ policy: 'load-only', installExtension }); + const query = vi.fn().mockResolvedValue({}); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(true); + expect(installExtension).not.toHaveBeenCalled(); + }); + + it('attempts nothing under never, even with a packaged artifact', async () => { + const vendorRoot = makeRoot(); + writeVendorArtifact(vendorRoot, 'linux-x64'); + const query = vi.fn(); + const installExtension = vi.fn(); + const manager = new ExtensionManager({ policy: 'never', installExtension, warn: noopWarn }); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(false); + expect(query).not.toHaveBeenCalled(); + expect(installExtension).not.toHaveBeenCalled(); + expect(manager.getCapabilities()[0]?.reason).toContain('never'); + }); + + it('fails closed on an unsupported tuple without named LOAD or INSTALL', async () => { + const vendorRoot = makeRoot(); + writeVendorArtifact(vendorRoot, 'linux-x64'); + const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); + const installExtension = vi.fn(); + const manager = new ExtensionManager({ + policy: 'load-only', + installExtension, + warn: noopWarn, + }); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'win32-arm64' }), + ).resolves.toBe(false); + expect(query).not.toHaveBeenCalled(); + expect(installExtension).not.toHaveBeenCalled(); + expect(manager.getCapabilities()[0]?.reason).toContain('win32-arm64'); + expect(manager.getCapabilities()[0]?.reason).toContain('no packaged FTS artifact'); + expect(manager.getCapabilities()[0]?.reason).not.toContain(vendorRoot); + }); + + it('does not INSTALL on an unsupported tuple under auto policy', async () => { + const vendorRoot = makeRoot(); + writeVendorArtifact(vendorRoot, 'linux-x64'); + const query = vi.fn(); + const installExtension = vi.fn(); + const manager = new ExtensionManager({ policy: 'auto', installExtension, warn: noopWarn }); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'win32-arm64' }), + ).resolves.toBe(false); + expect(query).not.toHaveBeenCalled(); + expect(installExtension).not.toHaveBeenCalled(); + expect(manager.getCapabilities()[0]?.reason).toContain('win32-arm64'); + expect(manager.getCapabilities()[0]?.reason).toContain('no packaged FTS artifact'); + expect(manager.getCapabilities()[0]?.reason).not.toContain(vendorRoot); + }); + + it('keeps the vendored inspect path when named LOAD has no .lbug_extension path', async () => { + const vendorRoot = makeRoot(); + const artifact = writeVendorArtifact(vendorRoot, 'linux-x64'); + writeFileSync(artifact, buildHostValidBinary()); + const query = vi + .fn() + .mockRejectedValueOnce( + new Error( + `Failed to load library: ${artifact} which is needed by extension: fts. Error: 126 The specified module could not be found.`, + ), + ) + .mockRejectedValueOnce(new Error('Extension "fts" has not been installed.')); + const installExtension = vi.fn(); + const manager = new ExtensionManager({ + policy: 'load-only', + installExtension, + warn: noopWarn, + }); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(false); + + expect(installExtension).not.toHaveBeenCalled(); + const cap = manager.getCapabilities()[0]; + expect(cap?.diagnosis?.kind).toBe('missing_dependency'); + expect(cap?.diagnosis?.remedy).toMatch(/OpenSSL|VC\+\+|runtime/i); + }); + + it('diagnoses a truncated home copy as corrupt, not missing_dependency (KTD7)', async () => { + const vendorRoot = makeRoot(); + const artifact = writeVendorArtifact(vendorRoot, 'linux-x64'); + const homeCopy = path.join(makeRoot(), 'truncated.lbug_extension'); + writeFileSync(homeCopy, 'short'); + const query = vi + .fn() + .mockRejectedValueOnce( + new Error(`Failed to load library: ${artifact} which is needed by extension: fts`), + ) + .mockRejectedValueOnce( + new Error( + `Failed to load library: ${homeCopy} which is needed by extension: fts. file too short`, + ), + ); + const manager = new ExtensionManager({ + policy: 'auto', + installExtension: vi.fn().mockResolvedValue(failedInstall), + warn: noopWarn, + }); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(false); + + const cap = manager.getCapabilities()[0]; + expect(cap?.reason).not.toContain(artifact); + expect(cap?.diagnosis?.kind).toBe('corrupt_file'); + expect(diagnoseExtensionLoad(cap?.reason, 'FTS', homeCopy).kind).toBe('corrupt_file'); + }); + + it('escapes a vendored path that contains a quote', async () => { + const vendorRoot = mkdtempSync(path.join(tmpdir(), "gn-fts-it's-")); + tmpRoots.push(vendorRoot); + const artifact = writeVendorArtifact(vendorRoot, 'linux-x64'); + const query = vi.fn().mockResolvedValue({}); + const manager = new ExtensionManager({ policy: 'load-only', warn: noopWarn }); + + await expect( + manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(true); + expect(query.mock.calls[0][0]).toBe( + `LOAD EXTENSION '${escapeCypherString(realpathSync(artifact))}'`, + ); + }); + + it('rejects a sibling directory that only shares the vendor prefix', async () => { + const parent = makeRoot(); + const vendorRoot = path.join(parent, 'vendor'); + const evil = path.join(parent, 'vendor-evil', 'lbug-fts', 'prebuilds', 'linux-x64'); + mkdirSync(path.join(vendorRoot, 'lbug-fts'), { recursive: true }); + mkdirSync(evil, { recursive: true }); + writeFileSync(path.join(evil, 'libfts.lbug_extension'), 'evil'); + writeFileSync( + path.join(vendorRoot, 'lbug-fts', 'manifest.json'), + JSON.stringify({ filename: 'libfts.lbug_extension' }), + ); + mkdirSync(path.join(vendorRoot, 'lbug-fts', 'prebuilds', 'linux-x64'), { recursive: true }); + // Candidate must exist so resolveVendoredFtsPath reaches realpath containment + // (a prefix-only leak would follow this symlink into vendor-evil). + symlinkSync( + path.join(evil, 'libfts.lbug_extension'), + path.join(vendorRoot, 'lbug-fts', 'prebuilds', 'linux-x64', 'libfts.lbug_extension'), + ); + + const query = vi.fn().mockRejectedValue(new Error('Extension "fts" not found')); + const manager = new ExtensionManager({ policy: 'load-only', warn: noopWarn }); + await manager.ensure(query, 'fts', 'FTS', { vendorRoot, platformTuple: 'linux-x64' }); + expect(query.mock.calls.map(([sql]) => sql)).toEqual(['LOAD EXTENSION fts']); + expect(String(query.mock.calls[0][0])).not.toContain('vendor-evil'); + }); + + it('does not try a vendored path for VECTOR and records no tuple', async () => { + const vendorRoot = makeRoot(); + writeVendorArtifact(vendorRoot, 'linux-x64'); + const query = vi.fn().mockResolvedValue({}); + const manager = new ExtensionManager({ policy: 'auto' }); + + await expect( + manager.ensure(query, 'vector', 'VECTOR', { vendorRoot, platformTuple: 'linux-x64' }), + ).resolves.toBe(true); + expect(query.mock.calls.map(([sql]) => sql)).toEqual(['LOAD EXTENSION vector']); + expect(manager.getCapabilities()[0]?.attempts).toBeUndefined(); + }); +}); diff --git a/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts b/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts index 5cc2fc32f..64cd767d5 100644 --- a/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts +++ b/gitnexus/test/unit/lbug-pool-evict-reopen-race.test.ts @@ -69,6 +69,14 @@ vi.mock('../../src/core/lbug/sidecar-recovery.js', () => ({ .mockResolvedValue({ moved: [], removed: [], failed: [] }), renameFailureMessage: vi.fn((p: string) => `rename failed for ${p}`), statIfExists: vi.fn().mockResolvedValue(null), + assertReadOnlyFtsCrashSafe: vi.fn().mockResolvedValue(undefined), + FtsReaderUnrepairableError: class FtsReaderUnrepairableError extends Error { + readonly code = 'FTS_READER_UNREPAIRABLE' as const; + constructor(dbPath = '') { + super(dbPath); + this.name = 'FtsReaderUnrepairableError'; + } + }, })); const { initLbug, closeLbug, isLbugReady, unpinRepo } = diff --git a/gitnexus/test/unit/lbug-pool-pinning.test.ts b/gitnexus/test/unit/lbug-pool-pinning.test.ts index 4b5b5ce24..d6fb1dbf8 100644 --- a/gitnexus/test/unit/lbug-pool-pinning.test.ts +++ b/gitnexus/test/unit/lbug-pool-pinning.test.ts @@ -71,6 +71,14 @@ vi.mock('../../src/core/lbug/sidecar-recovery.js', () => ({ .mockResolvedValue({ moved: [], removed: [], failed: [] }), renameFailureMessage: vi.fn((p: string) => `rename failed for ${p}`), statIfExists: vi.fn().mockResolvedValue(null), + assertReadOnlyFtsCrashSafe: vi.fn().mockResolvedValue(undefined), + FtsReaderUnrepairableError: class FtsReaderUnrepairableError extends Error { + readonly code = 'FTS_READER_UNREPAIRABLE' as const; + constructor(dbPath = '') { + super(dbPath); + this.name = 'FtsReaderUnrepairableError'; + } + }, })); const { initLbug, initLbugWithDb, closeLbug, isLbugReady, pinRepo, unpinRepo } = diff --git a/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts b/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts index f5b4946b9..aef8867de 100644 --- a/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts +++ b/gitnexus/test/unit/lbug-pool-storage-version-lock-retry.test.ts @@ -53,6 +53,14 @@ vi.mock('../../src/core/lbug/sidecar-recovery.js', () => ({ .mockResolvedValue({ moved: [], removed: [], failed: [] }), renameFailureMessage: vi.fn((p: string) => `rename failed for ${p}`), statIfExists: vi.fn().mockResolvedValue(null), + assertReadOnlyFtsCrashSafe: vi.fn().mockResolvedValue(undefined), + FtsReaderUnrepairableError: class FtsReaderUnrepairableError extends Error { + readonly code = 'FTS_READER_UNREPAIRABLE' as const; + constructor(dbPath = '') { + super(dbPath); + this.name = 'FtsReaderUnrepairableError'; + } + }, })); const { initLbug, closeLbug, isLbugReady, unpinRepo } = diff --git a/gitnexus/test/unit/lbug-readonly-error.test.ts b/gitnexus/test/unit/lbug-readonly-error.test.ts index 514ddc07d..51b03cae7 100644 --- a/gitnexus/test/unit/lbug-readonly-error.test.ts +++ b/gitnexus/test/unit/lbug-readonly-error.test.ts @@ -1,20 +1,25 @@ /** * Regression Tests: read-only DB error discriminator (#1224) * - * The MCP query pool opens LadybugDB read-only. Defensive callers of - * `ensureFTSIndex` from that pool used to spam stderr with five - * "Cannot execute write operations in a read-only database" warnings - * per query because the cache was invalidated each time. The fix: - * `ensureFTSIndex` now treats the read-only error as a no-op and - * caches the key — but to do that it relies on a precise discriminator - * that does NOT swallow lock / busy / "already exists" errors. + * The MCP query pool opens LadybugDB read-only. A write there surfaces + * "Cannot execute write operations in a read-only database". The + * discriminator must stay precise so lock / busy / "already exists" + * errors are not swallowed. * * This file unit-tests the discriminator directly so future refactors * keep the contract. */ +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { describe, it, expect } from 'vitest'; import { isReadOnlyDbError } from '../../src/core/lbug/lbug-adapter.js'; +const adapterSource = readFileSync( + path.join(path.dirname(fileURLToPath(import.meta.url)), '../../src/core/lbug/lbug-adapter.ts'), + 'utf8', +); + describe('isReadOnlyDbError', () => { it('matches the canonical LadybugDB read-only message verbatim', () => { const err = new Error( @@ -69,7 +74,7 @@ describe('isReadOnlyDbError', () => { expect(isReadOnlyDbError(wrapped)).toBe(false); }); - it('does NOT match unrelated errors that the ensure path must still surface', () => { + it('does NOT match unrelated errors that create/drop must still surface', () => { // Lock contention — handled separately by isDbBusyError; must not be // silenced by the read-only filter. expect(isReadOnlyDbError(new Error('Could not set lock on file'))).toBe(false); @@ -80,4 +85,14 @@ describe('isReadOnlyDbError', () => { // Generic transient error. expect(isReadOnlyDbError(new Error('Connection refused'))).toBe(false); }); + + it('has no production ensureFTSIndex residue (#1500)', () => { + expect(adapterSource).not.toContain('ensureFTSIndex'); + expect(adapterSource).toContain('export const createFTSIndex'); + const start = adapterSource.indexOf('export const createFTSIndex'); + const nextExport = adapterSource.indexOf('\nexport const', start + 1); + const createBody = adapterSource.slice(start, nextExport); + expect(createBody).toContain("includes('already exists')"); + expect(createBody).not.toContain('isReadOnlyDbError'); + }); }); diff --git a/gitnexus/test/unit/native-check-probe.test.ts b/gitnexus/test/unit/native-check-probe.test.ts index 8d18663aa..8cb6f5d9a 100644 --- a/gitnexus/test/unit/native-check-probe.test.ts +++ b/gitnexus/test/unit/native-check-probe.test.ts @@ -1,3 +1,4 @@ +import path from 'node:path'; import { describe, it, expect, vi, beforeEach } from 'vitest'; /** @@ -30,6 +31,7 @@ vi.mock('@ladybugdb/core', () => { }); import { + ftsAvailabilityLabel, probeFtsExtensionLoad, probeVectorExtensionLoad, } from '../../src/core/lbug/native-check.js'; @@ -92,6 +94,45 @@ describe('probeFtsExtensionLoad (#2374)', () => { }); await expect(probeFtsExtensionLoad()).resolves.toEqual({ loaded: true }); }); + + it('reports suppressed-by-policy under never without issuing LOAD', async () => { + await expect(probeFtsExtensionLoad(undefined, { policy: 'never' })).resolves.toEqual({ + loaded: false, + suppressed: true, + reason: 'suppressed by policy GITNEXUS_LBUG_EXTENSION_INSTALL=never', + }); + expect(h.query).not.toHaveBeenCalled(); + expect(ftsAvailabilityLabel({ loaded: false, suppressed: true })).toBe('suppressed'); + }); + + it('tries a vendored path LOAD before the name-only LOAD', async () => { + h.query.mockResolvedValue(closeable()); + const vendoredPath = '/tmp/gn-fts-vendor/libfts.lbug_extension'; + + await expect( + probeFtsExtensionLoad(undefined, { vendoredPath, policy: 'load-only' }), + ).resolves.toEqual({ loaded: true }); + + expect(h.query).toHaveBeenCalledWith(`LOAD EXTENSION '${path.resolve(vendoredPath)}'`); + expect(h.query).not.toHaveBeenCalledWith('LOAD EXTENSION fts'); + }); + + it('falls back to name-only LOAD when the vendored path LOAD fails', async () => { + const vendoredPath = '/tmp/gn-fts-vendor/libfts.lbug_extension'; + h.query + .mockRejectedValueOnce(new Error('IO exception: missing vendored file')) + .mockResolvedValueOnce(closeable()); + + await expect( + probeFtsExtensionLoad(undefined, { + vendoredPath, + policy: 'load-only', + }), + ).resolves.toEqual({ loaded: true }); + + expect(h.query).toHaveBeenNthCalledWith(1, `LOAD EXTENSION '${path.resolve(vendoredPath)}'`); + expect(h.query).toHaveBeenNthCalledWith(2, 'LOAD EXTENSION fts'); + }); }); describe('probeVectorExtensionLoad (#2623 follow-up)', () => { diff --git a/gitnexus/test/unit/pool-wal-recovery.test.ts b/gitnexus/test/unit/pool-wal-recovery.test.ts index 6ae7cfe06..e11d0d5f3 100644 --- a/gitnexus/test/unit/pool-wal-recovery.test.ts +++ b/gitnexus/test/unit/pool-wal-recovery.test.ts @@ -16,6 +16,10 @@ vi.mock('fs/promises', () => ({ stat: vi.fn().mockResolvedValue({}), unlink: vi.fn().mockResolvedValue(undefined), rename: vi.fn().mockResolvedValue(undefined), + readFile: vi.fn(async () => { + const err = Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + throw err; + }), }, })); @@ -57,6 +61,7 @@ vi.mock('../../src/mcp/stdio-capture.js', () => ({ getActiveStdoutWrite: vi.fn(() => vi.fn()), })); +import { readFileSync } from 'node:fs'; import fs from 'fs/promises'; import { createLbugDatabase } from '../../src/core/lbug/lbug-config.js'; @@ -92,6 +97,10 @@ describe('WAL corruption recovery in doInitLbug (#1402)', () => { (createLbugDatabase as any).mockReset(); (fs.stat as any).mockReset(); (fs.rename as any).mockReset(); + (fs.readFile as any).mockReset(); + (fs.readFile as any).mockImplementation(async () => { + throw ENOENT_STAT; + }); mockInit.mockReset(); mockClose.mockReset(); connectionQueryMock.mockReset(); @@ -325,6 +334,10 @@ describe('Pool-adapter missing-shadow quarantine: TOCTOU + permission classifica (createLbugDatabase as any).mockReset(); (fs.stat as any).mockReset(); (fs.rename as any).mockReset(); + (fs.readFile as any).mockReset(); + (fs.readFile as any).mockImplementation(async () => { + throw ENOENT_STAT; + }); mockInit.mockReset(); mockClose.mockReset(); connectionQueryMock.mockReset(); @@ -512,4 +525,43 @@ describe('Pool-adapter missing-shadow quarantine: TOCTOU + permission classifica await expect(initLbug('test-repo-pool-large-wal', dbPath)).rejects.toThrow(/Rebuild the index/); expect(fs.rename).not.toHaveBeenCalled(); }); + + it('refuses a large orphan WAL with FTS crash evidence before the native open', async () => { + const { initLbug } = await import('../../src/core/lbug/pool-adapter.js'); + const { FtsReaderUnrepairableError } = await import('../../src/core/lbug/sidecar-recovery.js'); + const dbPath = '/tmp/test-pool-fts-reader-refuse/lbug'; + + (fs.stat as any).mockImplementation(async (p: string) => { + if (p.endsWith('.shadow')) throw ENOENT_STAT; + if (p.endsWith('.wal')) return { size: 8192 }; + return { size: 0 }; + }); + (fs.readFile as any).mockResolvedValue( + JSON.stringify({ + incrementalInProgress: { + startedAt: 1, + toWriteCount: 0, + phase: 'fts', + writePlan: 'in-place', + checkpointSucceeded: true, + }, + }), + ); + + await expect(initLbug('test-repo-pool-fts-reader-refuse', dbPath)).rejects.toBeInstanceOf( + FtsReaderUnrepairableError, + ); + expect(createLbugDatabase).not.toHaveBeenCalled(); + expect(fs.rename).not.toHaveBeenCalled(); + expect(fs.unlink).not.toHaveBeenCalled(); + }); + + it('never threads FTS crash evidence into the pool reader path', () => { + const src = readFileSync( + new URL('../../src/core/lbug/pool-adapter.ts', import.meta.url), + 'utf8', + ); + expect(src).toMatch(/Never pass crash evidence/); + expect(src).not.toMatch(/fts-inplace-checkpointed/); + }); }); diff --git a/gitnexus/test/unit/repo-manager-reconcile.test.ts b/gitnexus/test/unit/repo-manager-reconcile.test.ts index c0b647ea7..4e6ab26d6 100644 --- a/gitnexus/test/unit/repo-manager-reconcile.test.ts +++ b/gitnexus/test/unit/repo-manager-reconcile.test.ts @@ -308,6 +308,7 @@ describe('runFullAnalysis metadata reconciliation (mocked pipeline)', () => { queryImporters: vi.fn(async () => []), queryImportersBatch: vi.fn(async () => []), loadFTSExtension: vi.fn(async () => false), + tryFlushWAL: vi.fn(async () => true), })); vi.doMock('../../src/core/search/fts-indexes.js', () => ({ initialiseSearchFTSStemmer: vi.fn(() => 'porter'), diff --git a/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts b/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts new file mode 100644 index 000000000..387e85a75 --- /dev/null +++ b/gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts @@ -0,0 +1,1198 @@ +/** + * U4: FTS-phase dirty flag, converged boundary checkpoint, and `--repair-fts` + * admission. A true native abort kills the process before JS can write a skip + * reason (KTD6); these tests induce the phase through saveMeta / in-run stamps + * rather than killing analyze. + */ +import { execSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import fs from 'fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + getStoragePaths, + loadMeta, + saveMeta, + type RepoMeta, +} from '../../src/storage/repo-manager.js'; +import { createTempDir } from '../helpers/test-db.js'; +import { ANALYSIS_FEATURES } from '../../src/core/analysis-feature-registry.js'; +import { resolveAnalysisFeatureVersions } from '../../src/core/analysis-features.js'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import { resolveAnalyzerRunnerIdentity } from '../../src/core/analyzer-identity.js'; +import { EMBEDDING_DIMS, SCHEMA_FINGERPRINT } from '../../src/core/lbug/schema.js'; +import { getSearchFTSCjkSegmentation } from '../../src/core/search/cjk-segmentation.js'; +import { + FTS_DIRTY_PHASE, + allowsFtsCrashWalPark, + buildFtsDirtyStamp, + hasRecoveredInPlaceFtsAbort, + inferNativeAbortSkip, + isBoundaryCheckpointFatal, + isFtsStagingDirty, + isInPlaceFtsDirty, + resolveFtsWritePlan, + shouldRefuseFtsCrashWal, + shouldRefuseRepairFtsWhileDirty, + shouldStampFtsDirtyPhase, +} from '../../src/core/search/fts-crash-marker.js'; + +const RUN_ANALYZE_URL = new URL('../../src/core/run-analyze.ts', import.meta.url); +const RUN_ANALYZE_SRC = fileURLToPath(RUN_ANALYZE_URL); + +const REL_FILE = 'src/a.ts'; + +const createPlaceholderGraphStore = async (lbugPath: string): Promise => { + await fs.writeFile(lbugPath, 'fixture'); +}; + +const seedGitFile = async (repoPath: string): Promise => { + await fs.mkdir(path.join(repoPath, 'src'), { recursive: true }); + await fs.writeFile(path.join(repoPath, REL_FILE), 'export const a = 1;\n'); + execSync('git init', { cwd: repoPath, stdio: 'pipe' }); + execSync('git -c user.name=test -c user.email=t@t -c commit.gpgsign=false add -A', { + cwd: repoPath, + stdio: 'pipe', + }); + execSync('git -c user.name=test -c user.email=t@t -c commit.gpgsign=false commit -q -m init', { + cwd: repoPath, + stdio: 'pipe', + }); +}; + +const incrementalMeta = (repoPath: string): RepoMeta => ({ + repoPath, + lastCommit: 'stale-not-head', + indexedAt: new Date().toISOString(), + stats: {}, + fileHashes: { [REL_FILE]: 'stale-hash' }, + schemaFingerprint: SCHEMA_FINGERPRINT, + analysisFeatures: resolveAnalysisFeatureVersions(ANALYSIS_FEATURES, [REL_FILE]), + cjkSegmentation: getSearchFTSCjkSegmentation(), + embeddingDims: EMBEDDING_DIMS, + runnerIdentity: resolveAnalyzerRunnerIdentity(RUN_ANALYZE_URL.href), +}); + +const headCommit = (repoPath: string): string => + execSync('git rev-parse HEAD', { cwd: repoPath, encoding: 'utf8' }).trim(); + +const GRAPH_BYTES = 'QUERYABLE_GRAPH_BYTES_U5'; +const WAL_PATTERN = Buffer.alloc(8192, 0xab); + +const ftsInPlaceDirty = { + startedAt: Date.now() - 60_000, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place' as const, + checkpointSucceeded: true, +}; + +const fileGraph = () => { + const graph = createKnowledgeGraph(); + graph.addNode({ + id: 'file:src/a.ts', + label: 'File', + properties: { filePath: REL_FILE }, + }); + return graph; +}; + +const mockLbugAdapter = async () => { + const actual = await vi.importActual( + '../../src/core/lbug/lbug-adapter.js', + ); + return { + ...actual, + initLbug: vi.fn(async () => undefined), + loadGraphToLbug: vi.fn(async () => undefined), + getLbugStats: vi.fn(async () => ({ nodes: 1, edges: 0, communities: 0, processes: 0 })), + executeQuery: vi.fn(async () => []), + executeWithReusedStatement: vi.fn(async () => []), + closeLbug: vi.fn(async () => undefined), + wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), + loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), + deleteNodesForFile: vi.fn(async () => undefined), + deleteNodesForFiles: vi.fn(async () => undefined), + nodeTablesWithRowsForFiles: vi.fn(async () => []), + snapshotDerivedRelsForFiles: vi.fn(async () => []), + restoreDerivedRels: vi.fn(async () => undefined), + deleteAllCommunitiesAndProcesses: vi.fn(async () => undefined), + deleteAllInterprocTaintPaths: vi.fn(async () => undefined), + deleteAllCallSummaries: vi.fn(async () => undefined), + deleteAllInjects: vi.fn(async () => undefined), + deleteAllAdvisedBy: vi.fn(async () => undefined), + deleteAllDestinations: vi.fn(async () => undefined), + deleteSpringAopEvidenceNodes: vi.fn(async () => undefined), + deleteSpringAutoConfigurationDeclarations: vi.fn(async () => undefined), + deleteSpringAutoConfigurationSyntheticClasses: vi.fn(async () => undefined), + queryImporters: vi.fn(async () => []), + queryImportersBatch: vi.fn(async () => []), + loadFTSExtension: vi.fn(async () => true), + readIndexCatalogSnapshot: vi.fn(async () => []), + ensureEmbeddingRowDmlSafe: vi.fn(async () => true), + ensureFtsRowDmlSafe: vi.fn(async () => true), + }; +}; + +describe('FTS crash-marker policy (characterization)', () => { + it('stamps only the in-place write plan', () => { + expect(resolveFtsWritePlan('/idx/lbug', '/idx/lbug')).toBe('in-place'); + expect(resolveFtsWritePlan('/idx/lbug.staging.abc', '/idx/lbug')).toBe('staging'); + expect(shouldStampFtsDirtyPhase('in-place')).toBe(true); + expect(shouldStampFtsDirtyPhase('staging')).toBe(false); + }); + + it('mirrors staging-versus-in-place checkpoint fatality', () => { + expect(isBoundaryCheckpointFatal('staging')).toBe(true); + expect(isBoundaryCheckpointFatal('in-place')).toBe(false); + }); + + it('admits --repair-fts only for an FTS phase with a successful checkpoint', () => { + expect(shouldRefuseRepairFtsWhileDirty(undefined)).toBe(false); + expect( + shouldRefuseRepairFtsWhileDirty({ + startedAt: 1, + toWriteCount: 3, + phase: 'load-graph', + }), + ).toBe(true); + expect( + shouldRefuseRepairFtsWhileDirty({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + }), + ).toBe(true); + expect( + shouldRefuseRepairFtsWhileDirty({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + checkpointSucceeded: true, + }), + ).toBe(false); + expect(inferNativeAbortSkip({ startedAt: 1, toWriteCount: 0, phase: 'full-rebuild' })).toBe( + false, + ); + expect(inferNativeAbortSkip({ startedAt: 1, toWriteCount: 0, phase: FTS_DIRTY_PHASE })).toBe( + true, + ); + }); + + it('infers native-abort from a persisted skipReason after the dirty flag is gone', () => { + expect(inferNativeAbortSkip(undefined, 'native-abort')).toBe(true); + expect(inferNativeAbortSkip(undefined, 'tuple-missing')).toBe(false); + expect(inferNativeAbortSkip(undefined, 'extension-unavailable')).toBe(false); + expect(inferNativeAbortSkip(undefined, 'build-failed')).toBe(false); + expect(inferNativeAbortSkip(undefined, 'disabled-by-flag')).toBe(false); + expect(inferNativeAbortSkip(undefined)).toBe(false); + }); + + it('warrants a live WAL park only for in-place FTS after a successful checkpoint', () => { + expect( + allowsFtsCrashWalPark({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: true, + }), + ).toBe(true); + expect( + allowsFtsCrashWalPark({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'staging', + checkpointSucceeded: true, + }), + ).toBe(false); + expect( + allowsFtsCrashWalPark({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: false, + }), + ).toBe(false); + expect( + allowsFtsCrashWalPark({ + startedAt: 1, + toWriteCount: 3, + phase: 'load-graph', + writePlan: 'in-place', + checkpointSucceeded: true, + }), + ).toBe(false); + expect( + isInPlaceFtsDirty({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: false, + }), + ).toBe(true); + expect( + shouldRefuseFtsCrashWal({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: false, + }), + ).toBe(true); + expect(hasRecoveredInPlaceFtsAbort({ skipReason: 'native-abort', writePlan: 'in-place' })).toBe( + true, + ); + expect(hasRecoveredInPlaceFtsAbort({ skipReason: 'native-abort', writePlan: 'staging' })).toBe( + false, + ); + expect(hasRecoveredInPlaceFtsAbort({ skipReason: 'native-abort' })).toBe(false); + expect( + shouldRefuseFtsCrashWal(undefined, { skipReason: 'native-abort', writePlan: 'in-place' }), + ).toBe(true); + expect( + shouldRefuseFtsCrashWal(undefined, { skipReason: 'native-abort', writePlan: 'staging' }), + ).toBe(false); + expect( + isFtsStagingDirty({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'staging', + }), + ).toBe(true); + expect( + isFtsStagingDirty({ + startedAt: 1, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + }), + ).toBe(false); + }); + + it('lifts the prior-meta precondition on the in-place stamp', () => { + const stamp = buildFtsDirtyStamp({ + writePlan: 'in-place', + checkpointSucceeded: true, + now: 42, + }); + expect(stamp).toMatchObject({ + startedAt: 42, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: true, + toWriteCount: 0, + }); + }); + + it('stamps after the escalation valve in source order', () => { + const src = readFileSync(RUN_ANALYZE_SRC, 'utf8'); + const valve = src.indexOf("saveIncrementalDirtyState('escalated-full-write'"); + const stamp = src.indexOf('shouldStampFtsDirtyPhase(ftsWritePlan)'); + expect(valve).toBeGreaterThan(-1); + expect(stamp).toBeGreaterThan(valve); + }); +}); + +describe('runFullAnalysis FTS crash marker', () => { + afterEach(() => { + vi.doUnmock('../../src/core/lbug/lbug-adapter.js'); + vi.doUnmock('../../src/core/search/fts-indexes.js'); + vi.doUnmock('../../src/core/ingestion/pipeline.js'); + vi.doUnmock('../../src/storage/repo-manager.js'); + vi.doUnmock('../../src/core/lbug/wal-checkpoint-driver.js'); + vi.restoreAllMocks(); + vi.resetModules(); + vi.clearAllMocks(); + vi.unstubAllEnvs(); + }); + + it('stamps phase fts during an in-place incremental build and clears it after a clean run', async () => { + const sequence: string[] = []; + const checkpointOnce = vi.fn(async () => { + sequence.push('checkpoint'); + return true; + }); + let midBuild: RepoMeta | null = null; + vi.doMock('../../src/core/lbug/wal-checkpoint-driver.js', async (importActual) => ({ + ...(await importActual()), + checkpointOnce, + })); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + missingSearchFTSIndexTables: vi.fn(async () => []), + dropSearchFTSIndexes: vi.fn(async () => undefined), + buildSearchIndexesOrDegrade: vi.fn(async () => { + sequence.push('build'); + return { ok: true }; + }), + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: fileGraph(), + })), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => { + const actual = await importActual(); + return { + ...actual, + saveMeta: async (...args: Parameters) => { + const result = await actual.saveMeta(...args); + if (args[1].incrementalInProgress?.phase === FTS_DIRTY_PHASE) { + sequence.push('stamp-fts'); + midBuild = args[1]; + } + return result; + }, + }; + }); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-inplace-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, incrementalMeta(tmpRepo.dbPath)); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(result.ftsSkipped).not.toBe(true); + expect(midBuild?.incrementalInProgress).toMatchObject({ + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: true, + }); + expect(sequence.indexOf('checkpoint')).toBeLessThan(sequence.indexOf('stamp-fts')); + expect(sequence.indexOf('stamp-fts')).toBeLessThan(sequence.indexOf('build')); + expect(checkpointOnce).toHaveBeenCalled(); + + const finalMeta = await loadMeta(storagePath); + expect(finalMeta?.incrementalInProgress).toBeUndefined(); + } finally { + await tmpRepo.cleanup(); + } + }); + + it.skipIf(process.platform === 'win32')( + 'does not stamp an FTS phase on a staging plan', + async () => { + const phases: Array = []; + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + buildSearchIndexesOrDegrade: vi.fn(async () => ({ ok: true })), + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + })), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => { + const actual = await importActual(); + return { + ...actual, + saveMeta: async (...args: Parameters) => { + phases.push(args[1].incrementalInProgress?.phase); + return actual.saveMeta(...args); + }, + }; + }); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-staging-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await runFullAnalysis( + tmpRepo.dbPath, + { force: true, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(phases).toContain('full-rebuild'); + expect(phases).not.toContain(FTS_DIRTY_PHASE); + } finally { + await tmpRepo.cleanup(); + } + }, + ); + + it('clears the FTS phase on the degrade path as well as on success', async () => { + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + missingSearchFTSIndexTables: vi.fn(async () => []), + dropSearchFTSIndexes: vi.fn(async () => undefined), + buildSearchIndexesOrDegrade: vi.fn(async () => ({ + ok: false, + error: 'tokenizer failed', + failureClass: 'capability' as const, + })), + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: fileGraph(), + })), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-degrade-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, incrementalMeta(tmpRepo.dbPath)); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('build-failed'); + const finalMeta = await loadMeta(storagePath); + expect(finalMeta?.incrementalInProgress).toBeUndefined(); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('lets --repair-fts run when the dirty flag is the FTS phase', async () => { + const createSearchFTSIndexes = vi.fn(async () => []); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + createSearchFTSIndexes, + verifySearchFTSIndexes: vi.fn(async () => []), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => ({ + ...(await importActual()), + ensureGitNexusIgnored: vi.fn(async () => undefined), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-repair-admit-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: 'abc', + indexedAt: new Date().toISOString(), + stats: {}, + incrementalInProgress: { + startedAt: Date.now() - 60_000, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: true, + }, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { onProgress: () => {} }, + ); + expect(result.ftsRepairedOnly).toBe(true); + expect(createSearchFTSIndexes).toHaveBeenCalled(); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('refuses --repair-fts for an FTS-phase crash without a successful checkpoint', async () => { + const initLbug = vi.fn(async () => undefined); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + initLbug, + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-repair-refuse-nocheckpoint-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + incrementalInProgress: { + startedAt: Date.now() - 60_000, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: false, + }, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await expect( + runFullAnalysis(tmpRepo.dbPath, { repairFts: true }, { onProgress: () => {} }), + ).rejects.toThrow(/mid-incremental-recovery[\s\S]*gitnexus analyze/); + expect(initLbug).not.toHaveBeenCalled(); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('still refuses --repair-fts for a half-written graph phase', async () => { + const tmpRepo = await createTempDir('gitnexus-fts-crash-repair-refuse-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + incrementalInProgress: { + startedAt: Date.now() - 60_000, + toWriteCount: 12, + phase: 'load-graph', + }, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await expect( + runFullAnalysis(tmpRepo.dbPath, { repairFts: true }, { onProgress: () => {} }), + ).rejects.toThrow(/mid-incremental-recovery[\s\S]*gitnexus analyze/); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('infers native-abort and skips CREATE after an FTS-phase crash', async () => { + const buildSearchIndexesOrDegrade = vi.fn(async () => ({ ok: true })); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + buildSearchIndexesOrDegrade, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + })), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-infer-skip-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + incrementalInProgress: { + startedAt: Date.now() - 60_000, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: true, + }, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { force: true, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(buildSearchIndexesOrDegrade).not.toHaveBeenCalled(); + expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('native-abort'); + const finalMeta = await loadMeta(storagePath); + expect(finalMeta?.incrementalInProgress).toBeUndefined(); + expect(finalMeta?.capabilities?.fts).toMatchObject({ + status: 'unavailable', + skipReason: 'native-abort', + writePlan: 'in-place', + }); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('skips CREATE from a persisted native-abort skipReason after the dirty flag is gone', async () => { + const buildSearchIndexesOrDegrade = vi.fn(async () => ({ ok: true })); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + buildSearchIndexesOrDegrade, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + })), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-skipreason-persist-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: 'native-abort' }, + vectorSearch: { provider: 'exact-scan', status: 'unavailable', exactScanLimit: 0 }, + }, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { force: true, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(buildSearchIndexesOrDegrade).not.toHaveBeenCalled(); + expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('native-abort'); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('still creates FTS indexes when --repair-fts is rewritten by a retention mismatch', async () => { + const buildSearchIndexesOrDegrade = vi.fn(async () => ({ ok: true })); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + missingSearchFTSIndexTables: vi.fn(async () => []), + dropSearchFTSIndexes: vi.fn(async () => undefined), + buildSearchIndexesOrDegrade, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + })), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-repair-retention-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + contentRetention: 'symbol', + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { provider: 'ladybugdb-fts', status: 'unavailable', skipReason: 'native-abort' }, + vectorSearch: { provider: 'exact-scan', status: 'unavailable', exactScanLimit: 0 }, + }, + }); + await createPlaceholderGraphStore(lbugPath); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(buildSearchIndexesOrDegrade).toHaveBeenCalled(); + expect(result.ftsSkipReason).not.toBe('native-abort'); + } finally { + await tmpRepo.cleanup(); + } + }); + + it.skipIf(process.platform === 'win32')( + 'treats a boundary checkpoint failure as fatal on staging', + async () => { + const checkpointError = new Error('checkpoint rename failed'); + const checkpointOnce = vi.fn(async () => { + throw checkpointError; + }); + const buildSearchIndexesOrDegrade = vi.fn(async () => ({ ok: true })); + vi.doMock('../../src/core/lbug/wal-checkpoint-driver.js', async (importActual) => ({ + ...(await importActual()), + checkpointOnce, + })); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + buildSearchIndexesOrDegrade, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: { forEachNode: () => undefined }, + })), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-ckpt-staging-'); + try { + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: '', + indexedAt: new Date().toISOString(), + stats: {}, + }); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { force: true, skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ), + ).rejects.toBe(checkpointError); + expect(buildSearchIndexesOrDegrade).not.toHaveBeenCalled(); + } finally { + await tmpRepo.cleanup(); + } + }, + ); + + it('treats a boundary checkpoint failure as best-effort on an in-place plan', async () => { + const checkpointOnce = vi.fn(async () => { + throw new Error('checkpoint rename failed'); + }); + let stamped: RepoMeta['incrementalInProgress']; + vi.doMock('../../src/core/lbug/wal-checkpoint-driver.js', async (importActual) => ({ + ...(await importActual()), + checkpointOnce, + })); + vi.doMock('../../src/core/lbug/lbug-adapter.js', mockLbugAdapter); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + missingSearchFTSIndexTables: vi.fn(async () => []), + dropSearchFTSIndexes: vi.fn(async () => undefined), + buildSearchIndexesOrDegrade: vi.fn(async () => ({ ok: true })), + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: fileGraph(), + })), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => { + const actual = await importActual(); + return { + ...actual, + saveMeta: async (...args: Parameters) => { + if (args[1].incrementalInProgress?.phase === FTS_DIRTY_PHASE) { + stamped = args[1].incrementalInProgress; + } + return actual.saveMeta(...args); + }, + }; + }); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-ckpt-inplace-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, incrementalMeta(tmpRepo.dbPath)); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + expect(result.ftsSkipped).not.toBe(true); + expect(stamped).toMatchObject({ + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: false, + }); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('parks an in-place FTS crash WAL and keeps the graph on the next analyze', async () => { + const wipeLbugDbFiles = vi.fn(async () => undefined); + const runPipelineFromRepo = vi.fn(async () => { + throw new Error('pipeline must not run on FTS-park survivorship'); + }); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + wipeLbugDbFiles, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ runPipelineFromRepo })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-survivorship-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + ...incrementalMeta(tmpRepo.dbPath), + lastCommit: headCommit(tmpRepo.dbPath), + incrementalInProgress: ftsInPlaceDirty, + }); + await fs.writeFile(lbugPath, GRAPH_BYTES); + await fs.writeFile(`${lbugPath}.wal`, WAL_PATTERN); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(result.alreadyUpToDate).toBe(true); + expect(result.ftsSkipped).toBe(true); + expect(result.ftsSkipReason).toBe('native-abort'); + expect(wipeLbugDbFiles).not.toHaveBeenCalled(); + expect(runPipelineFromRepo).not.toHaveBeenCalled(); + expect(await fs.readFile(lbugPath, 'utf8')).toBe(GRAPH_BYTES); + expect(Buffer.compare(await fs.readFile(`${lbugPath}.wal.dirty-recovery`), WAL_PATTERN)).toBe( + 0, + ); + await expect(fs.stat(`${lbugPath}.wal`)).rejects.toMatchObject({ code: 'ENOENT' }); + const finalMeta = await loadMeta(storagePath); + expect(finalMeta?.incrementalInProgress).toBeUndefined(); + expect(finalMeta?.capabilities?.fts).toMatchObject({ + status: 'unavailable', + skipReason: 'native-abort', + writePlan: 'in-place', + }); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('does not keep the graph for a non-FTS dirty flag', async () => { + const wipeLbugDbFiles = vi.fn(async () => undefined); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + wipeLbugDbFiles, + })); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + buildSearchIndexesOrDegrade: vi.fn(async () => ({ ok: true })), + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ + runPipelineFromRepo: vi.fn(async (repoPath: string) => ({ + repoPath, + graph: fileGraph(), + })), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-nonfts-dirty-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + ...incrementalMeta(tmpRepo.dbPath), + lastCommit: headCommit(tmpRepo.dbPath), + incrementalInProgress: { + startedAt: Date.now() - 60_000, + toWriteCount: 12, + phase: 'load-graph', + }, + }); + await fs.writeFile(lbugPath, GRAPH_BYTES); + await fs.writeFile(`${lbugPath}.wal`, WAL_PATTERN); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(result.alreadyUpToDate).not.toBe(true); + expect(result.ftsSkipReason).not.toBe('native-abort'); + expect(wipeLbugDbFiles).toHaveBeenCalled(); + expect(Buffer.compare(await fs.readFile(`${lbugPath}.wal.dirty-recovery`), WAL_PATTERN)).toBe( + 0, + ); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('leaves a staging FTS abort WAL on the live index', async () => { + // Defensive: production never stamps phase=fts on a staging plan + // (`shouldStampFtsDirtyPhase('staging')` is false). The park warrant + // must still refuse this combination if it appears on disk. + const wipeLbugDbFiles = vi.fn(async () => undefined); + const runPipelineFromRepo = vi.fn(async () => { + throw new Error('pipeline must not run on staging FTS recover'); + }); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + wipeLbugDbFiles, + })); + vi.doMock('../../src/core/ingestion/pipeline.js', () => ({ runPipelineFromRepo })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-staging-wal-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + ...incrementalMeta(tmpRepo.dbPath), + lastCommit: headCommit(tmpRepo.dbPath), + incrementalInProgress: { + startedAt: Date.now() - 60_000, + toWriteCount: 0, + phase: FTS_DIRTY_PHASE, + writePlan: 'staging', + checkpointSucceeded: true, + }, + }); + await fs.writeFile(lbugPath, GRAPH_BYTES); + await fs.writeFile(`${lbugPath}.wal`, WAL_PATTERN); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {}, onLog: () => {} }, + ); + + expect(result.alreadyUpToDate).toBe(true); + expect(result.ftsSkipReason).toBe('native-abort'); + expect(wipeLbugDbFiles).not.toHaveBeenCalled(); + expect(Buffer.compare(await fs.readFile(`${lbugPath}.wal`), WAL_PATTERN)).toBe(0); + await expect(fs.stat(`${lbugPath}.wal.dirty-recovery`)).rejects.toMatchObject({ + code: 'ENOENT', + }); + } finally { + await tmpRepo.cleanup(); + } + }); + + it('parks a live WAL before --repair-fts opens the DB', async () => { + const initLbug = vi.fn(async () => undefined); + const createSearchFTSIndexes = vi.fn(async () => { + const midMeta = await loadMeta(getStoragePaths(tmpRepo.dbPath).storagePath); + expect(midMeta?.incrementalInProgress).toMatchObject({ + phase: FTS_DIRTY_PHASE, + writePlan: 'in-place', + checkpointSucceeded: true, + }); + return []; + }); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + initLbug, + })); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + createSearchFTSIndexes, + verifySearchFTSIndexes: vi.fn(async () => []), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => ({ + ...(await importActual()), + ensureGitNexusIgnored: vi.fn(async () => undefined), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-repair-park-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: 'abc', + indexedAt: new Date().toISOString(), + stats: {}, + incrementalInProgress: ftsInPlaceDirty, + }); + await createPlaceholderGraphStore(lbugPath); + await fs.writeFile(`${lbugPath}.wal`, WAL_PATTERN); + + const renameSpy = vi.spyOn(fs, 'rename'); + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { onProgress: () => {} }, + ); + expect(result.ftsRepairedOnly).toBe(true); + expect(createSearchFTSIndexes).toHaveBeenCalled(); + expect(initLbug).toHaveBeenCalled(); + expect(Buffer.compare(await fs.readFile(`${lbugPath}.wal.dirty-recovery`), WAL_PATTERN)).toBe( + 0, + ); + const initOrder = initLbug.mock.invocationCallOrder[0] ?? 0; + expect(initOrder).toBeGreaterThan(0); + const parkIdx = renameSpy.mock.calls.findIndex(([, to]) => + String(to).includes('.dirty-recovery'), + ); + expect(parkIdx).toBeGreaterThanOrEqual(0); + expect(renameSpy.mock.invocationCallOrder[parkIdx] ?? 0).toBeLessThan(initOrder); + const finalMeta = await loadMeta(storagePath); + expect(finalMeta?.incrementalInProgress).toBeUndefined(); + expect(finalMeta?.capabilities?.fts).toMatchObject({ status: 'available' }); + } finally { + vi.restoreAllMocks(); + await tmpRepo.cleanup(); + } + }); + + it('parks a live WAL after persist cleared the dirty flag when writePlan is in-place', async () => { + const initLbug = vi.fn(async () => undefined); + const createSearchFTSIndexes = vi.fn(async () => []); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + initLbug, + })); + vi.doMock('../../src/core/search/fts-indexes.js', async (importActual) => ({ + ...(await importActual()), + initialiseSearchFTSStemmer: vi.fn(() => 'porter'), + createSearchFTSIndexes, + verifySearchFTSIndexes: vi.fn(async () => []), + })); + vi.doMock('../../src/storage/repo-manager.js', async (importActual) => ({ + ...(await importActual()), + ensureGitNexusIgnored: vi.fn(async () => undefined), + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-repair-persisted-'); + try { + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + repoPath: tmpRepo.dbPath, + lastCommit: 'abc', + indexedAt: new Date().toISOString(), + stats: {}, + capabilities: { + graph: { provider: 'ladybugdb', status: 'available' }, + fts: { + provider: 'ladybugdb-fts', + status: 'unavailable', + skipReason: 'native-abort', + writePlan: 'in-place', + }, + vectorSearch: { provider: 'exact-scan', status: 'unavailable', exactScanLimit: 0 }, + }, + }); + await createPlaceholderGraphStore(lbugPath); + await fs.writeFile(`${lbugPath}.wal`, WAL_PATTERN); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + const result = await runFullAnalysis( + tmpRepo.dbPath, + { repairFts: true }, + { onProgress: () => {} }, + ); + expect(result.ftsRepairedOnly).toBe(true); + expect(createSearchFTSIndexes).toHaveBeenCalled(); + expect(initLbug).toHaveBeenCalled(); + expect(Buffer.compare(await fs.readFile(`${lbugPath}.wal.dirty-recovery`), WAL_PATTERN)).toBe( + 0, + ); + await expect(fs.stat(`${lbugPath}.wal`)).rejects.toMatchObject({ code: 'ENOENT' }); + } finally { + vi.restoreAllMocks(); + await tmpRepo.cleanup(); + } + }); + + it('refuses to open the DB when an FTS-phase park cannot move the WAL', async () => { + const initLbug = vi.fn(async () => undefined); + vi.doMock('../../src/core/lbug/lbug-adapter.js', async () => ({ + ...(await mockLbugAdapter()), + initLbug, + })); + + const tmpRepo = await createTempDir('gitnexus-fts-crash-park-fail-'); + try { + await seedGitFile(tmpRepo.dbPath); + const { storagePath, lbugPath } = getStoragePaths(tmpRepo.dbPath); + await fs.mkdir(storagePath, { recursive: true }); + await saveMeta(storagePath, { + ...incrementalMeta(tmpRepo.dbPath), + lastCommit: headCommit(tmpRepo.dbPath), + incrementalInProgress: ftsInPlaceDirty, + }); + await fs.writeFile(lbugPath, GRAPH_BYTES); + await fs.writeFile(`${lbugPath}.wal`, WAL_PATTERN); + + const originalRename: typeof fs.rename = fs.rename; + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + if (String(to).includes('.dirty-recovery')) { + const err = new Error('resource busy or locked') as NodeJS.ErrnoException; + err.code = 'EBUSY'; + throw err; + } + return originalRename(from, to); + }); + const originalRm: typeof fs.rm = fs.rm; + vi.spyOn(fs, 'rm').mockImplementation(async (p, opts) => { + if (String(p) === `${lbugPath}.wal`) { + const err = new Error('resource busy or locked') as NodeJS.ErrnoException; + err.code = 'EBUSY'; + throw err; + } + return originalRm(p, opts); + }); + + const { runFullAnalysis } = await import('../../src/core/run-analyze.js'); + await expect( + runFullAnalysis( + tmpRepo.dbPath, + { skipAgentsMd: true, skipSkills: true }, + { onProgress: () => {} }, + ), + ).rejects.toThrow(/gitnexus clean --lbug-sidecars/); + expect(initLbug).not.toHaveBeenCalled(); + expect(await fs.readFile(lbugPath, 'utf8')).toBe(GRAPH_BYTES); + expect(Buffer.compare(await fs.readFile(`${lbugPath}.wal`), WAL_PATTERN)).toBe(0); + } finally { + vi.restoreAllMocks(); + await tmpRepo.cleanup(); + } + }); +}); diff --git a/gitnexus/test/unit/run-analyze-fts-repair.test.ts b/gitnexus/test/unit/run-analyze-fts-repair.test.ts index 0327c5a07..875e7dc47 100644 --- a/gitnexus/test/unit/run-analyze-fts-repair.test.ts +++ b/gitnexus/test/unit/run-analyze-fts-repair.test.ts @@ -222,6 +222,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -278,6 +279,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { executeWithReusedStatement: vi.fn(async () => []), closeLbug: vi.fn(async () => undefined), wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), deleteNodesForFiles: vi.fn(async () => undefined), @@ -568,6 +570,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -633,6 +636,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -701,6 +705,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -772,6 +777,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -843,6 +849,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { executeWithReusedStatement: vi.fn(async () => []), closeLbug: vi.fn(async () => undefined), wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), deleteNodesForFiles: vi.fn(async () => undefined), @@ -922,6 +929,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -995,6 +1003,7 @@ describe('runFullAnalysis FTS repair and verification failure paths', () => { // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [] })), deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -1113,6 +1122,7 @@ describe('runFullAnalysis wipe-and-restore vector-index stamp (tri-review 466951 // Full-rebuild wipe is loud now (#2409, tri-review 4669518496 P2-4) — // run-analyze calls this on every full-path analyze. wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), // ≥1 cached row with a real-dims embedding: the harness default (empty // cache) would leave restoredEmbeddingCount at 0 and the recreation // gate shut — this test would then assert nothing. @@ -1266,6 +1276,7 @@ describe('runFullAnalysis dirty-recovery parking failure fails fast (this shippi executeWithReusedStatement: vi.fn(async () => []), closeLbug: vi.fn(async () => undefined), wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings, deleteNodesForFile: vi.fn(async () => undefined), // Batched incremental APIs (#2409) — consumed UNCONDITIONALLY by @@ -1551,6 +1562,7 @@ describe('runFullAnalysis Phase 5 embedding gate (#2790)', () => { executeWithReusedStatement: vi.fn(async () => []), closeLbug: vi.fn(async () => undefined), wipeLbugDbFiles: vi.fn(async () => undefined), + tryFlushWAL: vi.fn(async () => true), loadCachedEmbeddings: vi.fn(async () => ({ embeddingNodeIds: new Set(), embeddings: [], diff --git a/gitnexus/test/unit/run-analyze.test.ts b/gitnexus/test/unit/run-analyze.test.ts index 6c579ef83..3dbf0e3f5 100644 --- a/gitnexus/test/unit/run-analyze.test.ts +++ b/gitnexus/test/unit/run-analyze.test.ts @@ -649,14 +649,21 @@ describe('run-analyze module', () => { ); expect(recovered.alreadyUpToDate).not.toBe(true); - // The #2790 symptom line must NOT appear: pre-fix the advanced hashes - // diffed to zero and the run "preserved" every stale row instead. - expect(recoveryLogs).not.toContainEqual(expect.stringContaining('skipping wipe')); - // The crash-recovery contract survived the checkpoint, so the dirty flag - // is what drives the rebuild. - expect(recoveryLogs).toContainEqual( - expect.stringContaining('forcing full rebuild to restore a known-good index'), - ); + // #2790 was: hashes advanced mid-run, changed=0, "skipping wipe" preserved + // the OLD graph. An FTS-phase stamp after the graph write can now recover + // via incremental (graph already mutated) instead of a forced wipe — that + // is not the #2790 bug as long as lastCommit is still stale and the + // incremental write set is non-empty. A forced rebuild also heals. + const skipWipe = recoveryLogs.find((message) => message.includes('skipping wipe')); + if (skipWipe) { + // #2790 was changed=0/added=0/deleted=0 over the old graph. A write + // set with added files is a real incremental, not that bug. + expect(skipWipe).not.toMatch(/changed=0, added=0, deleted=0/); + } else { + expect(recoveryLogs).toContainEqual( + expect.stringContaining('forcing full rebuild to restore a known-good index'), + ); + } const healed = await loadMeta(storagePath); expect(healed).toMatchObject({ lastCommit: commitB }); expect(healed?.embeddingCheckpoint).toBeUndefined(); diff --git a/gitnexus/test/unit/server.test.ts b/gitnexus/test/unit/server.test.ts index 945414da1..c1b2a63df 100644 --- a/gitnexus/test/unit/server.test.ts +++ b/gitnexus/test/unit/server.test.ts @@ -457,6 +457,17 @@ describe('MCP output budgets', () => { // ─── Tool handler error handling ────────────────────────────────────── describe('server error handling', () => { + it('includes string Error.code in tool error text', async () => { + const err = Object.assign(new Error('reader refuse'), { code: 'FTS_READER_UNREPAIRABLE' }); + const backend = createMockBackend({ + callTool: vi.fn().mockRejectedValue(err), + }); + const { text, isError } = await callToolThroughServer(backend, 'context', { name: 'auth' }); + expect(isError).toBe(true); + expect(text).toContain('FTS_READER_UNREPAIRABLE'); + expect(text).toContain('reader refuse'); + }); + it('createMCPServer does not throw for valid backend', () => { const backend = createMockBackend(); expect(() => createMCPServer(backend)).not.toThrow(); diff --git a/gitnexus/test/unit/sidecar-recovery.test.ts b/gitnexus/test/unit/sidecar-recovery.test.ts index 382ae77df..b1634cdf2 100644 --- a/gitnexus/test/unit/sidecar-recovery.test.ts +++ b/gitnexus/test/unit/sidecar-recovery.test.ts @@ -5,6 +5,8 @@ import path from 'node:path'; import { readFileSync } from 'node:fs'; import { _resetSidecarRecoveryWarningsForTest, + assertReadOnlyFtsCrashSafe, + FtsReaderUnrepairableError, cleanParkedDirtyRecoverySidecars, cleanParkedLbugSidecars, cleanQuarantinedMissingShadowWals, @@ -417,6 +419,189 @@ describe('LadybugDB sidecar recovery', () => { expect(log.warn).toHaveBeenCalledTimes(1); expect(log.debug).toHaveBeenCalled(); }); + + it('parks a large orphan WAL only with fts-inplace-checkpointed evidence', async () => { + const wal = Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab); + await fs.writeFile(`${dbPath}.wal`, wal); + await guardWalQuarantine(dbPath, 'writable', new Error('trigger'), logger(), { + kind: 'fts-inplace-checkpointed', + }); + expect(Buffer.compare(readFileSync(`${dbPath}.wal.dirty-recovery`), wal)).toBe(0); + await expect(fs.stat(`${dbPath}.wal`)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('still refuses a large orphan WAL when crash evidence is omitted', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1)); + await expect( + guardWalQuarantine(dbPath, 'writable', new Error('trigger'), logger()), + ).rejects.toThrow(/Rebuild the index/); + await expect(fs.stat(`${dbPath}.wal`)).resolves.toBeDefined(); + }); + + it('still refuses a present shadow even with crash evidence', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1)); + await fs.writeFile(`${dbPath}.shadow`, Buffer.alloc(64)); + await expect( + guardWalQuarantine(dbPath, 'writable', new Error('trigger'), logger(), { + kind: 'fts-inplace-checkpointed', + }), + ).rejects.toThrow(/present but unreachable/); + await expect(fs.stat(`${dbPath}.wal`)).resolves.toBeDefined(); + }); + + it('names gitnexus clean --lbug-sidecars when an evidenced park fails', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + const originalRename: typeof fs.rename = fs.rename; + vi.spyOn(fs, 'rename').mockImplementation(async (from, to) => { + if (String(to).includes('.dirty-recovery')) { + const err = new Error('resource busy or locked') as NodeJS.ErrnoException; + err.code = 'EBUSY'; + throw err; + } + return originalRename(from, to); + }); + const originalRm: typeof fs.rm = fs.rm; + vi.spyOn(fs, 'rm').mockImplementation(async (p, opts) => { + if (String(p) === `${dbPath}.wal`) { + const err = new Error('resource busy or locked') as NodeJS.ErrnoException; + err.code = 'EBUSY'; + throw err; + } + return originalRm(p, opts); + }); + await expect( + guardWalQuarantine(dbPath, 'writable', new Error('trigger'), logger(), { + kind: 'fts-inplace-checkpointed', + }), + ).rejects.toThrow(/gitnexus clean --lbug-sidecars/); + }); + }); + + describe('assertReadOnlyFtsCrashSafe (KTD10 reader refuse)', () => { + const ftsDirtyMeta = { + incrementalInProgress: { + startedAt: 1, + toWriteCount: 0, + phase: 'fts', + writePlan: 'in-place', + checkpointSucceeded: true, + }, + }; + + const snapshotDir = async (): Promise => { + const names = (await fs.readdir(dir)).sort(); + const parts = await Promise.all( + names.map(async (name) => { + const bytes = await fs.readFile(path.join(dir, name)); + return `${name}:${bytes.length}:${Buffer.from(bytes).toString('hex').slice(0, 32)}`; + }), + ); + return parts.join('|'); + }; + + it('refuses a large orphan WAL when FTS crash evidence is on disk, without touching files', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + await fs.writeFile(path.join(dir, 'gitnexus.json'), JSON.stringify(ftsDirtyMeta)); + const before = await snapshotDir(); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).rejects.toBeInstanceOf( + FtsReaderUnrepairableError, + ); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).rejects.toThrow( + /gitnexus analyze --repair-fts/, + ); + expect(await snapshotDir()).toBe(before); + }); + + it('falls through when meta is missing so today large-WAL readers stay unchanged', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + const before = await snapshotDir(); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).resolves.toBeUndefined(); + expect(await snapshotDir()).toBe(before); + }); + + it('falls through when meta is corrupt', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + await fs.writeFile(path.join(dir, 'gitnexus.json'), '{not-json'); + const before = await snapshotDir(); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).resolves.toBeUndefined(); + expect(await snapshotDir()).toBe(before); + }); + + it('falls through for a non-FTS dirty flag', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + await fs.writeFile( + path.join(dir, 'gitnexus.json'), + JSON.stringify({ + incrementalInProgress: { startedAt: 1, toWriteCount: 3, phase: 'load-graph' }, + }), + ); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).resolves.toBeUndefined(); + }); + + it('refuses an in-place FTS dirty WAL even when the boundary checkpoint failed', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + await fs.writeFile( + path.join(dir, 'gitnexus.json'), + JSON.stringify({ + incrementalInProgress: { + startedAt: 1, + toWriteCount: 0, + phase: 'fts', + writePlan: 'in-place', + checkpointSucceeded: false, + }, + }), + ); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).rejects.toBeInstanceOf( + FtsReaderUnrepairableError, + ); + }); + + it('refuses a persisted in-place native-abort plus a live WAL after the dirty flag is gone', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + await fs.writeFile( + path.join(dir, 'gitnexus.json'), + JSON.stringify({ + capabilities: { + fts: { + provider: 'ladybugdb-fts', + status: 'unavailable', + skipReason: 'native-abort', + writePlan: 'in-place', + }, + }, + }), + ); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).rejects.toBeInstanceOf( + FtsReaderUnrepairableError, + ); + }); + + it('falls through for a persisted staging native-abort so the live WAL can replay', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES + 1, 0xab)); + await fs.writeFile( + path.join(dir, 'gitnexus.json'), + JSON.stringify({ + capabilities: { + fts: { + provider: 'ladybugdb-fts', + status: 'unavailable', + skipReason: 'native-abort', + writePlan: 'staging', + }, + }, + }), + ); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).resolves.toBeUndefined(); + }); + + it('refuses a tiny orphan WAL when in-place FTS crash evidence is on disk', async () => { + await fs.writeFile(`${dbPath}.wal`, Buffer.alloc(TINY_ORPHAN_WAL_BYTES, 0xab)); + await fs.writeFile(path.join(dir, 'gitnexus.json'), JSON.stringify(ftsDirtyMeta)); + await expect(assertReadOnlyFtsCrashSafe(dbPath)).rejects.toBeInstanceOf( + FtsReaderUnrepairableError, + ); + }); }); describe('presentShadowUnreachableMessage (present-but-locked, not missing — S2)', () => { diff --git a/gitnexus/test/unit/vendored-extension-path.test.ts b/gitnexus/test/unit/vendored-extension-path.test.ts new file mode 100644 index 000000000..ca380f032 --- /dev/null +++ b/gitnexus/test/unit/vendored-extension-path.test.ts @@ -0,0 +1,140 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + inferExtensionVersionFromPath, + isPathInsideRoot, + nodePlatformTuple, + readFtsArtifactManifest, + resolveFtsVersionPair, + resolveVendoredFtsPath, + isUnsupportedFtsTuple, + validateVendoredExtensionPath, +} from '../../src/core/lbug/vendored-extension-path.js'; + +const tmpRoots: string[] = []; + +const makeVendorRoot = (): string => { + const root = mkdtempSync(join(tmpdir(), 'gn-fts-vendor-')); + tmpRoots.push(root); + return root; +}; + +afterEach(() => { + for (const root of tmpRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } +}); + +describe('resolveVendoredFtsPath', () => { + it('returns null when the tuple directory has no artifact', () => { + const vendorRoot = makeVendorRoot(); + expect(resolveVendoredFtsPath({ vendorRoot, tuple: 'linux-x64' })).toBeNull(); + }); + + it('returns the absolute artifact path when the file exists', () => { + const vendorRoot = makeVendorRoot(); + const tuple = 'linux-x64'; + const dir = join(vendorRoot, 'lbug-fts', 'prebuilds', tuple); + mkdirSync(dir, { recursive: true }); + const artifact = join(dir, 'libfts.lbug_extension'); + writeFileSync(artifact, 'placeholder'); + + expect(resolveVendoredFtsPath({ vendorRoot, tuple })).toBe(artifact); + }); + + it('reads the filename from manifest.json when present', () => { + const vendorRoot = makeVendorRoot(); + mkdirSync(join(vendorRoot, 'lbug-fts'), { recursive: true }); + writeFileSync( + join(vendorRoot, 'lbug-fts', 'manifest.json'), + JSON.stringify({ filename: 'custom.lbug_extension' }), + ); + const dir = join(vendorRoot, 'lbug-fts', 'prebuilds', 'darwin-arm64'); + mkdirSync(dir, { recursive: true }); + const artifact = join(dir, 'custom.lbug_extension'); + writeFileSync(artifact, 'placeholder'); + + expect(resolveVendoredFtsPath({ vendorRoot, tuple: 'darwin-arm64' })).toBe(artifact); + expect(readFtsArtifactManifest(vendorRoot).filename).toBe('custom.lbug_extension'); + }); + + it('joins Node platform and arch as the tuple', () => { + expect(nodePlatformTuple('win32', 'x64')).toBe('win32-x64'); + expect(nodePlatformTuple('linux', 'arm64')).toBe('linux-arm64'); + }); + + it('rejects a sibling directory that only shares the vendor prefix', () => { + const parent = makeVendorRoot(); + const vendorRoot = join(parent, 'vendor'); + const evil = join(parent, 'vendor-evil', 'libfts.lbug_extension'); + mkdirSync(join(parent, 'vendor-evil'), { recursive: true }); + mkdirSync(vendorRoot, { recursive: true }); + writeFileSync(evil, 'placeholder'); + expect(isPathInsideRoot(vendorRoot, evil)).toBe(false); + expect(validateVendoredExtensionPath(evil, vendorRoot)).toBeNull(); + }); +}); + +describe('resolveFtsVersionPair', () => { + it('reads expected from the manifest and found from a Ladybug home path', () => { + const vendorRoot = makeVendorRoot(); + mkdirSync(join(vendorRoot, 'lbug-fts'), { recursive: true }); + writeFileSync( + join(vendorRoot, 'lbug-fts', 'manifest.json'), + JSON.stringify({ coreVersion: '0.18.3', extensionVersion: '0.18.1' }), + ); + expect( + inferExtensionVersionFromPath( + '/home/alice/.lbdb/extension/0.17.0/linux_amd64/fts/libfts.lbug_extension', + ), + ).toBe('0.17.0'); + expect( + resolveFtsVersionPair( + 'C:\\Users\\bob\\.lbdb\\extension\\0.17.0\\win_amd64\\fts\\libfts.lbug_extension', + vendorRoot, + ), + ).toEqual({ expected: '0.18.3', found: '0.17.0' }); + }); + + it('leaves found unset when a packaged lbug-fts path has no version segment', () => { + const vendorRoot = makeVendorRoot(); + mkdirSync(join(vendorRoot, 'lbug-fts'), { recursive: true }); + writeFileSync( + join(vendorRoot, 'lbug-fts', 'manifest.json'), + JSON.stringify({ coreVersion: '0.18.3', extensionVersion: '0.18.1' }), + ); + expect( + resolveFtsVersionPair( + join(vendorRoot, 'lbug-fts', 'prebuilds', 'linux-x64', 'libfts.lbug_extension'), + vendorRoot, + ), + ).toEqual({ expected: '0.18.3', found: undefined }); + }); +}); + +describe('readFtsArtifactManifest', () => { + it.each(['null', '[]'] as const)('returns {} when manifest.json is %s', (contents) => { + const vendorRoot = makeVendorRoot(); + mkdirSync(join(vendorRoot, 'lbug-fts'), { recursive: true }); + writeFileSync(join(vendorRoot, 'lbug-fts', 'manifest.json'), contents); + expect(readFtsArtifactManifest(vendorRoot)).toEqual({}); + }); + + it('sanitizes a non-array unsupportedTuples so lookup does not throw', () => { + const vendorRoot = makeVendorRoot(); + mkdirSync(join(vendorRoot, 'lbug-fts'), { recursive: true }); + writeFileSync( + join(vendorRoot, 'lbug-fts', 'manifest.json'), + JSON.stringify({ + coreVersion: 1, + filename: '', + unsupportedTuples: {}, + }), + ); + expect(readFtsArtifactManifest(vendorRoot)).toEqual({}); + expect(() => isUnsupportedFtsTuple('linux-x64', vendorRoot)).not.toThrow(); + expect(isUnsupportedFtsTuple('linux-x64', vendorRoot)).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/wal-checkpoint-driver.test.ts b/gitnexus/test/unit/wal-checkpoint-driver.test.ts index 2079c74dc..9deda8609 100644 --- a/gitnexus/test/unit/wal-checkpoint-driver.test.ts +++ b/gitnexus/test/unit/wal-checkpoint-driver.test.ts @@ -10,7 +10,9 @@ * native engine lives in `test/integration/analyze-wal-checkpoint-failure.test.ts`. */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import * as lbugAdapter from '../../src/core/lbug/lbug-adapter.js'; import { + checkpointOnce, isManualCheckpointEnabled, runCheckpointWithRetry, startWalCheckpointDriver, @@ -133,6 +135,32 @@ describe('isManualCheckpointEnabled — env var parsing', () => { }); }); +describe('checkpointOnce — opt-out', () => { + let originalEnv: string | undefined; + beforeEach(() => { + originalEnv = process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; + }); + afterEach(() => { + if (originalEnv === undefined) delete process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; + else process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT = originalEnv; + vi.restoreAllMocks(); + }); + + it('resolves false and does not call CHECKPOINT when GITNEXUS_WAL_MANUAL_CHECKPOINT=0', async () => { + process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT = '0'; + const flush = vi.spyOn(lbugAdapter, 'tryFlushWAL'); + await expect(checkpointOnce()).resolves.toBe(false); + expect(flush).not.toHaveBeenCalled(); + }); + + it('returns the flushed warrant, not a hardcoded success', async () => { + delete process.env.GITNEXUS_WAL_MANUAL_CHECKPOINT; + const flush = vi.spyOn(lbugAdapter, 'tryFlushWAL').mockResolvedValue(false); + await expect(checkpointOnce()).resolves.toBe(false); + expect(flush).toHaveBeenCalled(); + }); +}); + describe('startWalCheckpointDriver — lifecycle', () => { let originalEnv: string | undefined; beforeEach(() => { diff --git a/gitnexus/vendor/lbug-fts/LICENSE b/gitnexus/vendor/lbug-fts/LICENSE new file mode 100644 index 000000000..107021a6d --- /dev/null +++ b/gitnexus/vendor/lbug-fts/LICENSE @@ -0,0 +1,27 @@ +The files under `prebuilds/` are the LadybugDB FTS extension, redistributed +unchanged from https://extension.ladybugdb.com/. LadybugDB is MIT-licensed. +There is no upstream-published digest for these artifacts; GitNexus records +SHA-256 locally in `prebuilds/SHA256SUMS`. + +MIT License + +Copyright (c) 2022-2025 Kùzu Inc. +Copyright (c) 2025-2026 Ladybug Memory Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gitnexus/vendor/lbug-fts/manifest.json b/gitnexus/vendor/lbug-fts/manifest.json new file mode 100644 index 000000000..6d93ddd03 --- /dev/null +++ b/gitnexus/vendor/lbug-fts/manifest.json @@ -0,0 +1,21 @@ +{ + "coreVersion": "0.18.3", + "extensionVersion": "0.18.1", + "filename": "libfts.lbug_extension", + "officialRepo": "https://extension.ladybugdb.com/", + "urlTemplate": "{officialRepo}v{extensionVersion}/{upstreamPlatform}/fts/{filename}", + "upstreamDigestPublished": false, + "tuples": [ + { "tuple": "linux-x64", "upstreamPlatform": "linux_amd64" }, + { "tuple": "linux-arm64", "upstreamPlatform": "linux_arm64" }, + { "tuple": "darwin-x64", "upstreamPlatform": "osx_amd64" }, + { "tuple": "darwin-arm64", "upstreamPlatform": "osx_arm64" }, + { "tuple": "win32-x64", "upstreamPlatform": "win_amd64" } + ], + "unsupportedTuples": [ + { + "tuple": "win32-arm64", + "reason": "no upstream-published artifact at vendor-refresh time" + } + ] +} diff --git a/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS b/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS new file mode 100644 index 000000000..3aa968510 --- /dev/null +++ b/gitnexus/vendor/lbug-fts/prebuilds/SHA256SUMS @@ -0,0 +1,5 @@ +01cf0d4debae1b0c7c182bf78747ee1b148a76667bbaa5bb0cb3cc848998028d ./win32-x64/libfts.lbug_extension +4af2602007a4a02d5b18d0b6ecb20b1cc2f493242a915faf7df1c033136107b1 ./linux-arm64/libfts.lbug_extension +cf687fda0f82bfbe116e775f2f17c39faf45be6300fd1937c2b1afd21142c121 ./linux-x64/libfts.lbug_extension +d3564c05ec28a0293a5488eaff2c06591624ac01a44af183d8cfacd92a29d785 ./darwin-arm64/libfts.lbug_extension +d8589c0a91c6667e959da6a81f712f69b330d0563602933da983edeebee60fc6 ./darwin-x64/libfts.lbug_extension diff --git a/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension new file mode 100644 index 000000000..31f85a0da Binary files /dev/null and b/gitnexus/vendor/lbug-fts/prebuilds/darwin-arm64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension new file mode 100644 index 000000000..b0fd3b7fd Binary files /dev/null and b/gitnexus/vendor/lbug-fts/prebuilds/darwin-x64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension new file mode 100644 index 000000000..41f2fa575 Binary files /dev/null and b/gitnexus/vendor/lbug-fts/prebuilds/linux-arm64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension new file mode 100644 index 000000000..0971ff059 Binary files /dev/null and b/gitnexus/vendor/lbug-fts/prebuilds/linux-x64/libfts.lbug_extension differ diff --git a/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension b/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension new file mode 100644 index 000000000..093f8b7f9 Binary files /dev/null and b/gitnexus/vendor/lbug-fts/prebuilds/win32-x64/libfts.lbug_extension differ