scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-01 02:01:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge upstream/main and bound incremental COPY workers

Browse source
This commit is contained in:
azizur100389 2026-09-15 14:09:38 +01:00
commit 81a47c8a78
53 changed files with 2658 additions and 1950 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/docker.yml vendored
View file

@ -74,8 +74,10 @@ jobs:
- name: gitnexus-web
dockerfile: Dockerfile.web
slug: gitnexus-web
# CLI / `gitnexus serve` backend. Heavy native deps (tree-sitter,
# onnxruntime-node) live only in this image.
# CLI / `gitnexus serve` backend. Tree-sitter natives live in this
# image. onnxruntime-node is opt-in (`gitnexus embeddings install` or
# a bind-mounted prefix / GITNEXUS_EMBEDDING_URL); npm is stripped
# at runtime so the image cannot auto-heal the embedding stack.
- name: gitnexus
dockerfile: Dockerfile.cli
slug: gitnexus

5
Dockerfile.cli
View file

@ -6,8 +6,11 @@ ARG TARGETPLATFORM
ARG NPM_VERSION=11.14.1
# -- Builder -----------------------------------------------------------
# Native modules (tree-sitter-*, onnxruntime-node, node-gyp builds for
# Native modules (tree-sitter-*, node-gyp builds for
# tree-sitter-proto / tree-sitter-swift) require python3 + a C/C++ toolchain.
# onnxruntime-node is not installed by `npm ci`; local embeddings need
# `gitnexus embeddings install` (or HTTP env / a bind-mounted prefix). The
# runtime stage strips npm, so this image cannot auto-heal the stack.
# node:22-bookworm-slim
FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder
ARG NPM_VERSION

5
README.md
View file

@ -72,7 +72,7 @@ That's it. `analyze` indexes the codebase, installs agent skills, registers Clau
> **No C++ toolchain?** Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` — those four languages won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild.
> **Behind an HTTP proxy / regional firewall?** `onnxruntime-node`'s postinstall downloads optional CUDA binaries from `api.nuget.org` and ignores `HTTP_PROXY`/`HTTPS_PROXY` ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). The embedding stack is an optional dependency, so a failed download no longer breaks the install — and it self-heals: the first `gitnexus analyze --embeddings` (or `gitnexus embeddings install`) fetches the stack through your npm registry config (mirrors/proxies apply, no NuGet) into `~/.gitnexus/embedding-runtime` (override with `GITNEXUS_EMBEDDING_RUNTIME_DIR`). The on-demand prefix needs Node with `module.registerHooks` (≥ 22.15 on 22.x, ≥ 23.5 on 23.x); on older Node, keep the stack in the install itself with `ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus` (works on every supported Node).
> **Local embeddings are opt-in.** Default `npm install` does not fetch `@huggingface/transformers` or `onnxruntime-node`. Run `gitnexus embeddings install` (or `gitnexus analyze --embeddings`, which auto-heals) to fetch the stack through your npm registry config into `~/.gitnexus/embedding-runtime`. CUDA GPU binaries still use NuGet via `--cuda` ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). The prefix needs Node with `module.registerHooks` (≥ 22.15 on 22.x, ≥ 23.5 on 23.x). A leftover 1.6.12 package-first tree is residual until a clean reinstall; `--force` only refreshes prefix overrides.
> **About `tree-sitter-kotlin`:** like Dart/Proto/Swift, Kotlin is a **vendored** grammar (under `gitnexus/vendor/tree-sitter-kotlin`). Upstream ships **source only** (no prebuilt binaries), so GitNexus cross-builds the platform prebuilds itself (via the `build-tree-sitter-prebuilds` GitHub Actions workflow) and vendors them — the same uniform pipeline used for Dart, Proto, and Swift. `node-gyp-build` selects the right `.node` at require time, so **no C/C++ toolchain is needed**. If no prebuild matches your platform-arch, only Kotlin (`.kt`/`.kts`) parsing is unavailable; the rest of `gitnexus` is unaffected.
@ -587,6 +587,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max
| `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. |
| `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. |
| `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS` | unset | When truthy (`1`/`true`/`yes`), per-attempt HTTP embedding timeouts (`TimeoutError` on fetch or body read) go through the bounded `GITNEXUS_EMBEDDING_MAX_ATTEMPTS` retry loop instead of failing the job. Any other value leaves it off, so cloud/default timeouts remain terminal. | Local accelerators that drop a device lock when the client disconnects and succeed on the next request (observed with FastFlowLM on Ryzen AI). |
| `GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS` | `180000` (3 minutes) | Per-request IPC timeout for local embedding sidecar embed batches. On overrun the parent SIGKILLs the sidecar child and rejects the batch. Init still uses the HF download budget (`HF_DOWNLOAD_TIMEOUT_MS` × attempts), not this knob. | Large embed batches or slow local ONNX inference cause sidecar request timeouts during `analyze --embeddings`, `embeddings sync`, serve, or MCP. |
| `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. |
| `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. |
| `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. |
@ -960,7 +961,7 @@ docker compose --env-file .env up -d
Files:
- [Dockerfile.web](Dockerfile.web) — builds `gitnexus-shared` and `gitnexus-web`, then serves the production frontend.
- [Dockerfile.cli](Dockerfile.cli) — builds the CLI/server (with its native deps) and runs `gitnexus serve --host 0.0.0.0`.
- [Dockerfile.cli](Dockerfile.cli) — builds the CLI/server (with its native deps) and runs `gitnexus serve --host 0.0.0.0`. Local embeddings are **not** in the image (`onnxruntime-node` is opt-in; runtime npm is stripped). Bind-mount a prefix or set `GITNEXUS_EMBEDDING_URL`.
- [docker-compose.yaml](docker-compose.yaml) — starts both signed images side by side.
- [.env.example](.env.example) — overrides for image names, container names, ports, and the workspace mount.

196
gitnexus-web/package-lock.json generated
View file

@ -18,7 +18,7 @@
"@tailwindcss/vite": "^4.3.3",
"axios": "^1.20.0",
"d3": "^7.9.0",
"dompurify": "^3.4.13",
"dompurify": "^3.4.15",
"gitnexus-shared": "file:../gitnexus-shared",
"graphology": "^0.26.0",
"graphology-indices": "^0.17.0",
@ -28,7 +28,7 @@
"graphology-utils": "^2.3.0",
"i18next": "^26.3.6",
"i18next-browser-languagedetector": "^8.2.1",
"langchain": "^1.5.4",
"langchain": "^1.5.11",
"lru-cache": "^11.5.2",
"lucide-react": "^1.31.0",
"mermaid": "^11.17.2",
@ -39,12 +39,12 @@
"react-i18next": "^17.0.13",
"react-markdown": "^10.1.0",
"react-syntax-highlighter": "^16.1.1",
"react-zoom-pan-pinch": "^4.0.3",
"react-zoom-pan-pinch": "^4.2.0",
"remark-gfm": "^4.0.1",
"sigma": "^3.0.3",
"tailwindcss": "^4.3.3",
"uuid": "^14.0.2",
"zod": "^4.4.3"
"zod": "^4.5.4"
},
"devDependencies": {
"@babel/types": "^8.0.4",
@ -60,7 +60,7 @@
"@vercel/node": "^5.10.2",
"@vitejs/plugin-react": "^6.1.1",
"@vitest/coverage-v8": "^4.1.11",
"jsdom": "^29.1.1",
"jsdom": "^30.0.1",
"tree-sitter-wasms": "^0.1.13",
"typescript": "^5.4.5",
"vite": "^8.1.5",
@ -119,56 +119,38 @@
}
},
"node_modules/@asamuzakjp/css-color": {
"version": "5.1.11",
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-5.1.11.tgz",
"integrity": "sha512-KVw6qIiCTUQhByfTd78h2yD1/00waTmm9uy/R7Ck/ctUyAPj+AEDLkQIdJW0T8+qGgj3j5bpNKK7Q3G+LedJWg==",
"version": "6.0.7",
"resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-6.0.7.tgz",
"integrity": "sha512-vC/bk1Lz7Tn/EfU9/apOTBk80/8dyGyWMowPoV1tJ52muDGsDqt2HPT2klrFUiY60MQmQv9q8yIht15JnBgDGw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@asamuzakjp/generational-cache": "^1.0.1",
"@csstools/css-calc": "^3.2.0",
"@csstools/css-color-parser": "^4.1.0",
"@csstools/css-calc": "^3.3.0",
"@csstools/css-color-parser": "^4.1.10",
"@csstools/css-parser-algorithms": "^4.0.0",
"@csstools/css-tokenizer": "^4.0.0"
"@csstools/css-tokenizer": "^4.0.0",
"lru-cache": "^11.5.2"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
"node": "^22.13.0 || >=24.0.0"
}
},
"node_modules/@asamuzakjp/dom-selector": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-7.1.1.tgz",
"integrity": "sha512-67RZDnYRc8H/8MLDgQCDE//zoqVFwajkepHZgmXrbwybzXOEwOWGPYGmALYl9J2DOLfFPPs6kKCqmbzV895hTQ==",
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-8.3.2.tgz",
"integrity": "sha512-93Z1N+BQNXysodoicpOIyNh2drHfz/CTf9nnT0FEx72GJcIiwgydD7tGAr78j41LsYn3hlRn+LdGPuBLn1Bl8Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"@asamuzakjp/generational-cache": "^1.0.1",
"@asamuzakjp/nwsapi": "^2.3.9",
"bidi-js": "^1.0.3",
"css-tree": "^3.2.1",
"is-potential-custom-element-name": "^1.0.1"
"is-potential-custom-element-name": "^1.0.1",
"lru-cache": "^11.5.2"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
"node": "^22.13.0 || >=24.0.0"
}
},
"node_modules/@asamuzakjp/generational-cache": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@asamuzakjp/generational-cache/-/generational-cache-1.0.1.tgz",
"integrity": "sha512-wajfB8KqzMCN2KGNFdLkReeHncd0AslUSrvHVvvYWuU8ghncRJoA50kT3zP9MVL0+9g4/67H+cdvBskj9THPzg==",
"dev": true,
"license": "MIT",
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
}
},
"node_modules/@asamuzakjp/nwsapi": {
"version": "2.3.9",
"resolved": "https://registry.npmjs.org/@asamuzakjp/nwsapi/-/nwsapi-2.3.9.tgz",
"integrity": "sha512-n8GuYSrI9bF7FFZ/SjhwevlHc8xaVlb/7HmHelnc/PZXBD2ZR49NnN9sMMuDdEGPeeRQ5d0hqlSlEpgCX3Wl0Q==",
"dev": true,
"license": "MIT"
},
"node_modules/@babel/code-frame": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz",
@ -320,9 +302,9 @@
"license": "Apache-2.0"
},
"node_modules/@csstools/color-helpers": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.0.2.tgz",
"integrity": "sha512-LMGQLS9EuADloEFkcTBR3BwV/CGHV7zyDxVRtVDTwdI2Ca4it0CCVTT9wCkxSgokjE5Ho41hEPgb8OEUwoXr6Q==",
"version": "6.1.1",
"resolved": "https://registry.npmjs.org/@csstools/color-helpers/-/color-helpers-6.1.1.tgz",
"integrity": "sha512-gLNsunvwf3mCi5u5o46/Z/JcJMnhbHSaZ69rkgPzNM3J4s8hWwpPUQB6/tt0EDFyCiWzxANlx+2LJwpYj4zS1w==",
"dev": true,
"funding": [
{
@ -340,9 +322,9 @@
}
},
"node_modules/@csstools/css-calc": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.2.0.tgz",
"integrity": "sha512-bR9e6o2BDB12jzN/gIbjHa5wLJ4UjD1CB9pM7ehlc0ddk6EBz+yYS1EV2MF55/HUxrHcB/hehAyt5vhsA3hx7w==",
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/@csstools/css-calc/-/css-calc-3.3.0.tgz",
"integrity": "sha512-c5ihYsPkdG6JCkU2zTMm4+k6r7RXuGxtWYhu5DHMIiF1FHzrfmHL5so11AoFpUv/tu61xfcmT4AmKoFfMPoqdQ==",
"dev": true,
"funding": [
{
@ -364,9 +346,9 @@
}
},
"node_modules/@csstools/css-color-parser": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.1.0.tgz",
"integrity": "sha512-U0KhLYmy2GVj6q4T3WaAe6NPuFYCPQoE3b0dRGxejWDgcPp8TP7S5rVdM5ZrFaqu4N67X8YaPBw14dQSYx3IyQ==",
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/@csstools/css-color-parser/-/css-color-parser-4.2.2.tgz",
"integrity": "sha512-3QKjR/vxyjcSXBLgb6lP0S3MGdvwbmqSsvLPbYdVORqPDc8FX1HAJ0Spk38bxaRXgvENTA47tlhhbb5Z2e8hEg==",
"dev": true,
"funding": [
{
@ -380,8 +362,8 @@
],
"license": "MIT",
"dependencies": {
"@csstools/color-helpers": "^6.0.2",
"@csstools/css-calc": "^3.2.0"
"@csstools/color-helpers": "^6.1.1",
"@csstools/css-calc": "^3.3.0"
},
"engines": {
"node": ">=20.19.0"
@ -415,9 +397,9 @@
}
},
"node_modules/@csstools/css-syntax-patches-for-csstree": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.3.tgz",
"integrity": "sha512-SH60bMfrRCJF3morcdk57WklujF4Jr/EsQUzqkarfHXEFcAR1gg7fS/chAE922Sehgzc1/+Tz5H3Ypa1HiEKrg==",
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/@csstools/css-syntax-patches-for-csstree/-/css-syntax-patches-for-csstree-1.1.12.tgz",
"integrity": "sha512-3vLQK+dXxhBMR2Wx99PTCifE+vHtW2ndZWyla8yK813ev6oGhyn8Lja8jCyGAWTJ+LEYZK7EVtJxrDj8ztevJw==",
"dev": true,
"funding": [
{
@ -960,9 +942,9 @@
}
},
"node_modules/@exodus/bytes": {
"version": "1.15.0",
"resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.0.tgz",
"integrity": "sha512-UY0nlA+feH81UGSHv92sLEPLCeZFjXOuHhrIo0HQydScuQc8s0A7kL/UdgwgDq8g8ilksmuoF35YVTNphV2aBQ==",
"version": "1.15.1",
"resolved": "https://registry.npmjs.org/@exodus/bytes/-/bytes-1.15.1.tgz",
"integrity": "sha512-S6mL0yNB/Abt9Ei4tq8gDhcczc4S3+vQ4ra7vxnAf+YHC02srtqxKKZghx2Dq6p0e66THKwR6r8N6P95wEty7Q==",
"dev": true,
"license": "MIT",
"engines": {
@ -1132,9 +1114,9 @@
}
},
"node_modules/@langchain/core": {
"version": "1.2.9",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.9.tgz",
"integrity": "sha512-conzSEj9Zu1AyXJLXsSbgrtxtxinmI1yGqQ5CIJZSoV5rvv+yvQE/vgBnoySpBQ/bl3YPgj2FL/gbDjWykLSfg==",
"version": "1.2.10",
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.10.tgz",
"integrity": "sha512-skr9zkyidZSzhoqhWINdULRstPT5aMD+w3tWtE3puxffWH7wxAp8tx1gMqadjKZwlgo6exlK0nHncFJv6m+3Cg==",
"license": "MIT",
"dependencies": {
"@cfworker/json-schema": "^4.0.2",
@ -3125,9 +3107,9 @@
"license": "MIT"
},
"node_modules/bidi-js": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz",
"integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==",
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.1.0.tgz",
"integrity": "sha512-fX1Onk0tdVPC7obPWB5EbJ1z7NVhLq4m2xZLq2YXBkxzMXIGRpNMU88n0EPgWseKl12J7zXs7qrDxPK4sRs2fg==",
"dev": true,
"license": "MIT",
"dependencies": {
@ -4026,9 +4008,9 @@
"peer": true
},
"node_modules/dompurify": {
"version": "3.4.13",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.13.tgz",
"integrity": "sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ==",
"version": "3.4.15",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.15.tgz",
"integrity": "sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==",
"license": "(MPL-2.0 OR Apache-2.0)",
"optionalDependencies": {
"@types/trusted-types": "^2.0.7"
@ -5109,39 +5091,39 @@
"peer": true
},
"node_modules/jsdom": {
"version": "29.1.1",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-29.1.1.tgz",
"integrity": "sha512-ECi4Fi2f7BdJtUKTflYRTiaMxIB0O6zfR1fX0GXpUrf6flp8QIYn1UT20YQqdSOfk2dfkCwS8LAFoJDEppNK5Q==",
"version": "30.0.1",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-30.0.1.tgz",
"integrity": "sha512-52v7mUVUfNQVYYqE1lcdaymWL0njO7lTLUog6ZvW2U5KsbiLk/GnZlVJ+qx0xfNJZ6Gn+KSpPNE52vurbxZwrA==",
"dev": true,
"license": "MIT",
"dependencies": {
"@asamuzakjp/css-color": "^5.1.11",
"@asamuzakjp/dom-selector": "^7.1.1",
"@asamuzakjp/css-color": "^6.0.5",
"@asamuzakjp/dom-selector": "^8.3.0",
"@bramus/specificity": "^2.4.2",
"@csstools/css-syntax-patches-for-csstree": "^1.1.3",
"@exodus/bytes": "^1.15.0",
"@csstools/css-syntax-patches-for-csstree": "^1.1.7",
"@exodus/bytes": "^1.15.1",
"css-tree": "^3.2.1",
"data-urls": "^7.0.0",
"decimal.js": "^10.6.0",
"html-encoding-sniffer": "^6.0.0",
"is-potential-custom-element-name": "^1.0.1",
"lru-cache": "^11.3.5",
"lru-cache": "^11.5.2",
"parse5": "^8.0.1",
"saxes": "^6.0.0",
"symbol-tree": "^3.2.4",
"tough-cookie": "^6.0.1",
"undici": "^7.25.0",
"tough-cookie": "^6.0.2",
"undici": "^8.9.0",
"w3c-xmlserializer": "^5.0.0",
"webidl-conversions": "^8.0.1",
"whatwg-mimetype": "^5.0.0",
"whatwg-url": "^16.0.1",
"whatwg-url": "^17.1.0",
"xml-name-validator": "^5.0.0"
},
"engines": {
"node": "^20.19.0 || ^22.13.0 || >=24.0.0"
"node": "^22.22.2 || ^24.15.0 || >=26.0.0"
},
"peerDependencies": {
"canvas": "^3.0.0"
"canvas": "^3.2.3"
},
"peerDependenciesMeta": {
"canvas": {
@ -5163,13 +5145,13 @@
}
},
"node_modules/jsdom/node_modules/undici": {
"version": "7.29.0",
"resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz",
"integrity": "sha512-IDxfleLmmbSskfWSUATiN1nfn2rDuvnMOqb5CWR92iIfojA0Ud+ulOAAEQ57LPr9rWmsreUyf5lwyao+7GNNVw==",
"version": "8.10.2",
"resolved": "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz",
"integrity": "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=20.18.1"
"node": ">=22.19.0"
}
},
"node_modules/jsdom/node_modules/webidl-conversions": {
@ -5183,18 +5165,18 @@
}
},
"node_modules/jsdom/node_modules/whatwg-url": {
"version": "16.0.1",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-16.0.1.tgz",
"integrity": "sha512-1to4zXBxmXHV3IiSSEInrreIlu02vUOvrhxJJH5vcxYTBDAx51cqZiKdyTxlecdKNSjj8EcxGBxNf6Vg+945gw==",
"version": "17.1.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-17.1.0.tgz",
"integrity": "sha512-3GeworPmc2ZfEEHP7lEbUfBX/L75wdEsi0rLNhXcXxnoN5jyq0SL5gCy06SGW2cyTIZdTvWIDQNQoza++vKeaw==",
"dev": true,
"license": "MIT",
"dependencies": {
"@exodus/bytes": "^1.11.0",
"@exodus/bytes": "^1.15.1",
"tr46": "^6.0.0",
"webidl-conversions": "^8.0.1"
},
"engines": {
"node": "^20.19.0 || ^22.12.0 || >=24.0.0"
"node": "^22.14.0 || >=24.0.0"
}
},
"node_modules/json-schema-to-ts": {
@ -5248,13 +5230,13 @@
"integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw=="
},
"node_modules/langchain": {
"version": "1.5.4",
"resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.4.tgz",
"integrity": "sha512-9Rq6Ih77UOy3+7bCbxMJS16MRUJwfxuljU0yW2KOXDgEKWE8cmaZJE6ONEy4HdWGMsbj3qyv3vD5UvV7fvNksg==",
"version": "1.5.11",
"resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.11.tgz",
"integrity": "sha512-6Sx9N5ylAJ11WrP1QnJLSIo75UABZbshzTJgG28H4mXuGcDk+w+7ZaNLkmGIh9sy/3PZcYS8UrI2rvH6A8NYIg==",
"license": "MIT",
"dependencies": {
"@langchain/langgraph": "^1.4.7",
"@langchain/langgraph-checkpoint": "^1.1.3",
"@langchain/langgraph": "^1.4.13",
"@langchain/langgraph-checkpoint": "^1.1.5",
"langsmith": ">=0.5.0 <1.0.0",
"zod": "^3.25.76 || ^4"
},
@ -5262,7 +5244,7 @@
"node": ">=20"
},
"peerDependencies": {
"@langchain/core": "^1.2.3"
"@langchain/core": "^1.2.10"
}
},
"node_modules/langsmith": {
@ -7382,9 +7364,9 @@
}
},
"node_modules/react-zoom-pan-pinch": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/react-zoom-pan-pinch/-/react-zoom-pan-pinch-4.0.3.tgz",
"integrity": "sha512-N2Hi6L78fFmhRra+ORpFSW7WST5x6kxpOPplIvtB0b7b+U2anpo1z1wLgaWRPS2kUSqcraRG+JgBCIlDJnqqAg==",
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/react-zoom-pan-pinch/-/react-zoom-pan-pinch-4.2.0.tgz",
"integrity": "sha512-QSw1dvr6QGv5zHCUY7SEIY7EWRDbx1y6plylrWf2Nko9KOMW1MEoPC6oJ+Y/qOSeQjhvbkMV0m5bFF7pBxCH4A==",
"license": "MIT",
"engines": {
"node": ">=8",
@ -7906,22 +7888,22 @@
}
},
"node_modules/tldts": {
"version": "7.0.25",
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.0.25.tgz",
"integrity": "sha512-keinCnPbwXEUG3ilrWQZU+CqcTTzHq9m2HhoUP2l7Xmi8l1LuijAXLpAJ5zRW+ifKTNscs4NdCkfkDCBYm352w==",
"version": "7.4.11",
"resolved": "https://registry.npmjs.org/tldts/-/tldts-7.4.11.tgz",
"integrity": "sha512-aBiNayCfTQxuIJBm06M+xR14cYaYlDlSXZbgsnKzKNxDKUVq7KFwTjwBSsb7m9Y5xO8WfPnBc63WaYFMTGlvqw==",
"dev": true,
"license": "MIT",
"dependencies": {
"tldts-core": "^7.0.25"
"tldts-core": "^7.4.11"
},
"bin": {
"tldts": "bin/cli.js"
}
},
"node_modules/tldts-core": {
"version": "7.0.25",
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.0.25.tgz",
"integrity": "sha512-ZjCZK0rppSBu7rjHYDYsEaMOIbbT+nWF57hKkv4IUmZWBNrBWBOjIElc0mKRgLM8bm7x/BBlof6t2gi/Oq/Asw==",
"version": "7.4.11",
"resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-7.4.11.tgz",
"integrity": "sha512-CW3WN2rIIE/Of21mulhgnGOwoDyEFNygyIBOONSdyAuSATgMMUCpLeUlB+E8sAwA5xRV9hYPl+kyZ9citHCaKg==",
"dev": true,
"license": "MIT"
},
@ -7939,9 +7921,9 @@
}
},
"node_modules/tough-cookie": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.1.tgz",
"integrity": "sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==",
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-6.0.2.tgz",
"integrity": "sha512-exgYmnmL/sJpR3upZfXG5PoatXQii55xAiXGXzY+sROLZ/Y+SLcp9PgJNI9Vz37HpQ74WvDcLT8eqm+kV3FzrA==",
"dev": true,
"license": "BSD-3-Clause",
"dependencies": {
@ -8508,9 +8490,9 @@
}
},
"node_modules/zod": {
"version": "4.4.3",
"resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz",
"integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==",
"version": "4.5.4",
"resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz",
"integrity": "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/colinhacks"

10
gitnexus-web/package.json
View file

@ -28,7 +28,7 @@
"@tailwindcss/vite": "^4.3.3",
"axios": "^1.20.0",
"d3": "^7.9.0",
"dompurify": "^3.4.13",
"dompurify": "^3.4.15",
"gitnexus-shared": "file:../gitnexus-shared",
"graphology": "^0.26.0",
"graphology-indices": "^0.17.0",
@ -38,7 +38,7 @@
"graphology-utils": "^2.3.0",
"i18next": "^26.3.6",
"i18next-browser-languagedetector": "^8.2.1",
"langchain": "^1.5.4",
"langchain": "^1.5.11",
"lru-cache": "^11.5.2",
"lucide-react": "^1.31.0",
"mermaid": "^11.17.2",
@ -49,12 +49,12 @@
"react-i18next": "^17.0.13",
"react-markdown": "^10.1.0",
"react-syntax-highlighter": "^16.1.1",
"react-zoom-pan-pinch": "^4.0.3",
"react-zoom-pan-pinch": "^4.2.0",
"remark-gfm": "^4.0.1",
"sigma": "^3.0.3",
"tailwindcss": "^4.3.3",
"uuid": "^14.0.2",
"zod": "^4.4.3"
"zod": "^4.5.4"
},
"devDependencies": {
"@babel/types": "^8.0.4",
@ -70,7 +70,7 @@
"@vercel/node": "^5.10.2",
"@vitejs/plugin-react": "^6.1.1",
"@vitest/coverage-v8": "^4.1.11",
"jsdom": "^29.1.1",
"jsdom": "^30.0.1",
"tree-sitter-wasms": "^0.1.13",
"typescript": "^5.4.5",
"vite": "^8.1.5",

29
gitnexus/.npmignore
View file

@ -13,25 +13,22 @@ node_modules/
vendor/**/node_modules
vendor/**/build
# ── Lean publish (FUTURE optimization — NOT done here) ─────────────────────────
# Once the build-tree-sitter-prebuilds workflow has committed 6/6 prebuilds for
# EVERY vendored grammar (c, dart, proto, kotlin, swift), the ~50 MB of generated
# source (parser.c etc.) can be dropped from the tarball — node-gyp-build never
# needs the source when a prebuild matches.
# ── Lean publish (done via package.json `files`, not this file) ───────────────
# Every vendored grammar now has 6/6 prebuilds, so the published tarball drops
# generated source (parser.c etc.). node-gyp-build loads prebuilds without it.
#
# IMPORTANT: this CANNOT be done from this file. package.json's `files: ["vendor"]`
# IMPORTANT: this CANNOT be done from this file. A broad `files: ["vendor"]`
# allow-list OVERRIDES .npmignore for the vendor/ subtree (verified: an active
# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). To slim
# the tarball, narrow the `files` field instead — replace the blanket "vendor"
# with the non-source subpaths only (vendor/**/prebuilds/**,
# vendor/**/bindings/node/index.*, vendor/**/src/node-types.json,
# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md).
# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). The
# lever is the `files` field — non-source subpaths only (vendor/**/prebuilds/**,
# vendor/**/bindings/node/index.js, vendor/**/src/node-types.json,
# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md, plus Leiden
# and the FTS manifest).
#
# Whatever the mechanism, the prepack guard
# (scripts/assert-publish-grammar-coverage.cjs, also `npm run
# assert-publish-coverage`) inspects the EFFECTIVE `npm pack` file list and FAILS
# the publish whenever a grammar with <6 prebuilds loses a source-build input — so
# the slim can never silently ship a dead grammar. Do not bypass it.
# The prepack guard (scripts/assert-publish-grammar-coverage.cjs, also
# `npm run assert-publish-coverage`) reads `files` and FAILS the publish
# whenever a grammar with <6 prebuilds loses a source-build input. Do not
# bypass it. Do not add a bare `vendor` entry back.
# Package lock (consumers use their own)
package-lock.json

6
gitnexus/README.md
View file

@ -531,7 +531,7 @@ for a quiet registry.
### `Cannot destructure property 'package' of 'node.target' as it is null`
This error comes from **npm 11.x's arborist** while installing gitnexus (often via `npx`), before gitnexus code runs. It is triggered by platform-filtered `optionalDependencies` in native packages such as `onnxruntime-node` / `@huggingface/transformers` (used when indexing with `--embeddings`). GitNexus cannot catch it at runtime — use one of these workarounds:
This error comes from **npm 11.x's arborist** while installing a package with platform-filtered `optionalDependencies` (often via `npx`), before gitnexus code runs. Default `npm install` / `npx gitnexus` no longer fetch `@huggingface/transformers` or `onnxruntime-node`; those packages appear only if you run `gitnexus embeddings install` (or you still have a leftover 1.6.12 package-first tree). Other native optionals can still trigger the same arborist crash. GitNexus cannot catch it at runtime — use one of these workarounds:
```bash
pnpm --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter dlx gitnexus@latest analyze # auto-selected when pnpm + npm 11+
@ -645,7 +645,7 @@ npm install -g gitnexus
`onnxruntime-node`'s postinstall downloads optional CUDA GPU binaries from `api.nuget.org` — outside the npm registry, so registry mirrors don't cover it, and its proxy layer (`global-agent`) ignores the standard `HTTP_PROXY`/`HTTPS_PROXY` variables and rejects 302 redirects ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)).
Since the packages are optional dependencies, a failed download no longer breaks `npm install -g gitnexus` — npm skips the embedding stack and everything else works. The stack then **self-heals on demand**: the first `gitnexus analyze --embeddings` (or an explicit `gitnexus embeddings install`) fetches it through your configured npm registry — mirrors and proxies apply, no NuGet download involved — into `~/.gitnexus/embedding-runtime`.
Default `npm install -g gitnexus` no longer fetches the embedding stack. **Opt in on demand**: the first `gitnexus analyze --embeddings` (or an explicit `gitnexus embeddings install`) fetches it through your configured npm registry — mirrors and proxies apply, no NuGet download involved — into `~/.gitnexus/embedding-runtime`. A leftover 1.6.12 package-first tree in gitnexus `node_modules` is residual until a clean reinstall; `--force` only refreshes prefix overrides.
```bash
# heal a proxy-degraded install manually (CPU embeddings; registry-only)
@ -660,7 +660,7 @@ GLOBAL_AGENT_HTTPS_PROXY=<proxy-url> gitnexus embeddings install --cuda
The prefix defaults to `~/.gitnexus/embedding-runtime`; set `GITNEXUS_EMBEDDING_RUNTIME_DIR` to install it elsewhere (e.g. a writable path in a container).
> **Node requirement for the on-demand prefix:** the self-heal loads the prefixed packages via `module.registerHooks`, available on Node **≥ 22.15** (on the 22.x line) or **≥ 23.5** (on the 23.x line). On an older Node the packages install but can't be loaded from the prefix — reinstall them into the install itself instead (works on every supported Node): `ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus` (Windows: `set ONNXRUNTIME_NODE_INSTALL=skip && npm install -g gitnexus`). Skipping only the CUDA download keeps full CPU embeddings (CPU embeddings don't need it). Check the result any time with `gitnexus doctor` (Embeddings → Support line).
> **Node requirement for the on-demand prefix:** the prefix loads via `module.registerHooks`, available on Node **≥ 22.15** (on the 22.x line) or **≥ 23.5** (on the 23.x line). On an older Node the packages install but can't be loaded from the prefix — upgrade Node, then run `gitnexus embeddings install`. Check the result any time with `gitnexus doctor` (Embeddings → Support line).
### Analyze warns about unavailable FTS or VECTOR extensions

981
gitnexus/package-lock.json generated
View file

File diff suppressed because it is too large Load diff

19
gitnexus/package.json
View file

@ -35,8 +35,16 @@
"hooks",
"scripts",
"skills",
"vendor",
"web"
"web",
"vendor/**/prebuilds/**",
"vendor/**/bindings/node/index.js",
"vendor/**/src/node-types.json",
"vendor/**/package.json",
"vendor/**/LICENSE",
"vendor/**/README.md",
"vendor/leiden/index.cjs",
"vendor/leiden/utils.cjs",
"vendor/lbug-fts/manifest.json"
],
"scripts": {
"build": "node scripts/build.js",
@ -97,7 +105,7 @@
"tree-sitter-typescript": "^0.23.2",
"uuid": "^14.0.0"
},
"optionalDependencies": {
"gitnexusEmbeddingStack": {
"@huggingface/transformers": "^4.1.0",
"onnxruntime-node": "^1.24.0"
},
@ -125,10 +133,7 @@
},
"overrides": {
"adm-zip": ">=0.6.0",
"sharp": ">=0.35.0",
"@huggingface/transformers": {
"onnxruntime-node": "$onnxruntime-node"
}
"sharp": ">=0.35.0"
},
"engines": {
"node": "^22.18.0 || >=24.11.0"

106
gitnexus/scripts/assert-publish-grammar-coverage.cjs
View file

@ -2,20 +2,19 @@
/**
* Publish guard: every vendored tree-sitter grammar must ship a loadable binding.
*
* The npm tarball includes gitnexus/vendor/ (package.json `files`). A grammar is
* "covered" on a platform-arch tuple if EITHER a prebuild ships for it OR the
* grammar's full source-build set ships (so the install can source-build it,
* toolchain permitting). A future lean publish — dropping the ~50 MB of generated
* source to ship prebuilds only — is safe ONLY once every grammar has all six
* prebuilds; doing it while any grammar still lacks a prebuild would ship a
* grammar with NO loadable binding (neither prebuild nor buildable source) → that
* language is silently dead for users.
* The npm tarball ships a lean vendor/ allow-list (package.json `files`):
* prebuilds, the bindings entry, node-types.json, and package metadata — not
* generated parser.c. A grammar is "covered" on a platform-arch tuple if EITHER
* a prebuild ships for it OR the grammar's full source-build set ships (so the
* install can source-build it, toolchain permitting). Lean publish is safe ONLY
* when every grammar has all six prebuilds; dropping source while any grammar
* still lacks a prebuild would ship a grammar with NO loadable binding.
*
* HOW SOURCE INCLUSION IS DECIDED. The `files` allow-list OVERRIDES `.npmignore`
* for the vendored subtree (verified: an active "vendor/(star-star)/src/parser.c"
* in .npmignore does NOT drop it from `npm pack`). So `.npmignore` can never
* exclude vendored source — the ONLY lever is the `files` field. A broad `vendor`
* ships the whole subtree (source + prebuilds); a lean publish narrows `files` to
* ships the whole subtree (source + prebuilds); lean publish narrows `files` to
* non-source subpaths. This guard therefore reads `files` directly rather than
* shelling out to `npm pack` (which, in prepack, would re-enter this guard and,
* on npm versions that don't honor --ignore-scripts for prepare/prepack, run the
@ -55,13 +54,75 @@ const SOURCE_BUILD_REL = [
* then rely on prebuilds.
*/
function filesShipsVendorSource(filesField) {
return (filesField || []).some((f) => {
const n = String(f)
.replace(/\\/g, '/')
.replace(/\/+$/, '')
.replace(/\/\*\*?$/, '');
return n === 'vendor';
});
return filesEntries(filesField).includes('vendor');
}
function normalizeFilesEntry(value) {
return String(value ?? '')
.replace(/\\/g, '/')
.replace(/\/+$/, '')
.replace(/\/\*\*?$/, '');
}
function filesEntries(filesField) {
return (filesField || []).map(normalizeFilesEntry);
}
function filesCoverGrammarPrebuilds(entries, grammarName) {
if (entries.includes('vendor') || entries.includes('vendor/**/prebuilds')) return true;
return (
entries.includes(`vendor/${grammarName}/prebuilds`) || entries.includes(`vendor/${grammarName}`)
);
}
function filesCoverGrammarBindings(entries, grammarName) {
if (entries.includes('vendor') || entries.includes('vendor/**/bindings/node/index.js')) {
return true;
}
return (
entries.includes(`vendor/${grammarName}/bindings/node/index.js`) ||
entries.includes(`vendor/${grammarName}`)
);
}
function filesCoverGrammarPackageJson(entries, grammarName) {
if (entries.includes('vendor') || entries.includes('vendor/**/package.json')) return true;
return (
entries.includes(`vendor/${grammarName}/package.json`) ||
entries.includes(`vendor/${grammarName}`)
);
}
function filesCoverLeiden(entries) {
if (entries.includes('vendor') || entries.includes('vendor/leiden')) return true;
return entries.includes('vendor/leiden/index.cjs') && entries.includes('vendor/leiden/utils.cjs');
}
/**
* Packed-tarball coverage from `files` globs — not on-disk prebuild counts.
* Lean publish can leave 6/6 `.node` files in the checkout while omitting
* them from the pack list.
*/
function findPackedFilesProblems({ filesField, grammarNames }) {
const entries = filesEntries(filesField);
const problems = [];
for (const name of grammarNames || []) {
if (!filesCoverGrammarPrebuilds(entries, name)) {
problems.push(`${name}: package.json files does not cover vendor/${name}/prebuilds`);
}
if (!filesCoverGrammarBindings(entries, name)) {
problems.push(
`${name}: package.json files does not cover vendor/${name}/bindings/node/index.js`,
);
}
if (!filesCoverGrammarPackageJson(entries, name)) {
problems.push(`${name}: package.json files does not cover vendor/${name}/package.json`);
}
}
if (!filesCoverLeiden(entries)) {
problems.push('package.json files does not cover vendor/leiden/index.cjs and utils.cjs');
}
return problems;
}
/** The on-disk source-build inputs for a grammar (relative paths). */
@ -171,7 +232,13 @@ function main() {
process.exit(1);
}
const problems = findCoverageProblems({ grammars });
const problems = [
...findCoverageProblems({ grammars }),
...findPackedFilesProblems({
filesField: pkg.files,
grammarNames: grammars.map((g) => g.name),
}),
];
if (problems.length > 0) {
console.error('[publish-guard] Refusing to publish — a vendored grammar would ship unusable:');
for (const p of problems) console.error(` - ${p}`);
@ -193,8 +260,13 @@ if (require.main === module) main();
module.exports = {
findCoverageProblems,
findPackedFilesProblems,
findStrayBuildArtifacts,
filesShipsVendorSource,
filesCoverGrammarPrebuilds,
filesCoverGrammarBindings,
filesCoverGrammarPackageJson,
filesCoverLeiden,
isBuildableFromSource,
sourceBuildSet,
countPrebuiltTuples,

38
gitnexus/src/cli/analyze.ts
View file

@ -69,9 +69,9 @@ import {
safeUrl,
} from '../core/embeddings/http-client.js';
import {
assessLocalEmbeddingRuntime,
isLocalEmbeddingRuntimeBlockerMessage,
isMissingLocalEmbeddingStackMessage,
localEmbeddingPrefixUnloadableMessage,
localEmbeddingStackMissingMessage,
} from '../core/embeddings/runtime-support.js';
import {
@ -79,8 +79,6 @@ import {
getEmbeddingInstallTimeoutMs,
getEmbeddingRuntimeDir,
installEmbeddingRuntime,
isPrefixRuntimeLoadable,
resolveEmbeddingRuntime,
} from '../core/embeddings/runtime-install.js';
import { warnIfNpm11NpxRisk } from './resolve-invocation.js';
@ -1128,14 +1126,18 @@ const analyzeCommandImpl = async (
);
}
// On-demand embedding runtime (#2370): when the optional stack was pruned at
// install time (proxy-blocked NuGet download in onnxruntime-node's
// postinstall), heal it here instead of failing later in the pipeline. The
// install goes through the user's npm registry config (mirrors/proxies
// apply) with --ignore-scripts, so no NuGet download is attempted. Runs
// before bar.start() like the sibling validations above.
// Local embeddings: refuse Intel Mac / unloadable prefix before any registry
// download, then auto-heal a missing stack. Analyze uses a short install
// timeout so a blackholed proxy cannot stall the index run.
if (embeddingsEnabled && !isHttpMode()) {
const resolved = resolveEmbeddingRuntime();
const assessment = assessLocalEmbeddingRuntime();
if (assessment.status === 'blocked') {
cliError(` ${assessment.message.replace(/\n/g, '\n ')}\n`, {
recoveryHint: 'local-embedding-unsupported',
});
process.exitCode = 1;
return;
}
// Resolved-but-unloadable (a populated prefix on a Node with no
// module.registerHooks), or nothing installed on such a Node: fail fast with
// capability guidance instead of dying mid-pipeline over an unusable prefix
@ -1143,28 +1145,20 @@ const analyzeCommandImpl = async (
// never needs the hook, so it is excluded. --embeddings was explicitly
// requested and this failure is deterministic, so fail fast rather than
// silently degrading to BM25 (distinct from a transient install timeout).
if (!isPrefixRuntimeLoadable() && (resolved === null || resolved.source === 'runtime-prefix')) {
cliError(` ${localEmbeddingPrefixUnloadableMessage().replace(/\n/g, '\n ')}\n`, {
if (assessment.status === 'prefix-unloadable') {
cliError(` ${assessment.message.replace(/\n/g, '\n ')}\n`, {
recoveryHint: 'local-embedding-stack-missing',
});
process.exitCode = 1;
return;
}
// On-demand embedding runtime (#2370): when the optional stack was pruned at
// install time (proxy-blocked NuGet download in onnxruntime-node's
// postinstall), heal it here instead of failing later in the pipeline. The
// install goes through the user's npm registry config (mirrors/proxies
// apply) with --ignore-scripts, so no NuGet download is attempted.
if (resolved === null) {
if (assessment.status === 'needs-install') {
console.log(
` Local embedding runtime is not installed (optional packages were skipped at install time).\n` +
` Local embedding runtime is not installed.\n` +
` Downloading it now from your npm registry into ${getEmbeddingRuntimeDir()} …\n` +
` (one-time; rerun manually anytime with \`gitnexus embeddings install\`)\n`,
);
try {
// Short deadline (env override still wins): analyze is interactive, so a
// blackholed proxy must not stall the whole index run for the 10-minute
// default — fail over to the guidance below instead.
await installEmbeddingRuntime(
{},
getEmbeddingInstallTimeoutMs(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS),

4
gitnexus/src/cli/doctor.ts
View file

@ -108,12 +108,10 @@ export function localEmbeddingDoctorStatus(opts: {
if (blocker) {
return { status: `✗ local embeddings unavailable on ${platform}/${arch}`, detail: blocker };
}
// The stack is an optionalDependency — npm prunes it when onnxruntime-node's
// postinstall can't download its CUDA binaries (proxy/firewall, #2370).
const resolution = opts.resolution !== undefined ? opts.resolution : resolveEmbeddingRuntime();
if (resolution === null) {
return {
status: '✗ optional embedding stack not installed',
status: '✗ local embedding stack not installed',
detail: localEmbeddingStackMissingMessage(),
};
}

39
gitnexus/src/cli/embeddings-sync.ts
View file

@ -11,8 +11,6 @@ import {
fetchExistingEmbeddingHashes,
initLbug,
} from '../core/lbug/lbug-adapter.js';
import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js';
import { resolveEmbeddingIdentity } from '../core/embeddings/embedding-identity.js';
import {
decideEmbeddingResume,
mintInterruptedCheckpoint,
@ -27,6 +25,18 @@ import {
measurePersistedEmbeddingCount,
persistedEmbeddingCountOrUndefined,
} from '../core/embedding-count.js';
import { isHttpMode } from '../core/embeddings/http-client.js';
import {
ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS,
getEmbeddingInstallTimeoutMs,
getEmbeddingRuntimeDir,
installEmbeddingRuntime,
} from '../core/embeddings/runtime-install.js';
import {
assessLocalEmbeddingRuntime,
localEmbeddingStackMissingMessage,
} from '../core/embeddings/runtime-support.js';
import { reapEmbeddingSidecarSafely } from '../core/embeddings/embedding-sidecar-reap.js';
/** Add missing embeddings directly to a healthy index, checkpointing periodically. */
export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> => {
@ -62,6 +72,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
);
}
const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js');
const identity = resolveEmbeddingIdentity();
let forceReembedNodeIds: ReadonlySet<string> | undefined;
let resumedFrom: EmbeddingCheckpoint | undefined;
@ -111,6 +122,28 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
);
}
if (!isHttpMode()) {
const assessment = assessLocalEmbeddingRuntime();
if (assessment.status === 'blocked' || assessment.status === 'prefix-unloadable') {
throw new Error(assessment.message);
}
if (assessment.status === 'needs-install') {
cliInfo(`Local embedding runtime is not installed.`);
cliInfo(`Downloading it now from your npm registry into ${getEmbeddingRuntimeDir()} …`);
try {
await installEmbeddingRuntime(
{},
getEmbeddingInstallTimeoutMs(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS),
);
} catch (err) {
throw new Error(
`Could not install the embedding runtime: ${err instanceof Error ? err.message : String(err)}\n\n` +
localEmbeddingStackMissingMessage(),
);
}
}
}
await initLbug(lbugPath);
try {
const existing = await fetchExistingEmbeddingHashes(executeQuery);
@ -139,6 +172,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
cliInfo(`Embedding ${repoPath}`);
cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`);
const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
const result = await runEmbeddingPipeline(
executeQuery,
executeWithReusedStatement,
@ -190,6 +224,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise<void> =
cliInfo(`Embeddings ready: ${embeddings}`);
} finally {
await closeLbug().catch(() => {});
await reapEmbeddingSidecarSafely();
}
} finally {
lock.release();

28
gitnexus/src/cli/embeddings.ts
View file

@ -6,7 +6,10 @@ import {
isPrefixRuntimeLoadable,
resolveEmbeddingRuntime,
} from '../core/embeddings/runtime-install.js';
import { localEmbeddingPrefixUnloadableMessage } from '../core/embeddings/runtime-support.js';
import {
getLocalEmbeddingRuntimeBlocker,
localEmbeddingPrefixUnloadableMessage,
} from '../core/embeddings/runtime-support.js';
export interface EmbeddingsInstallOptions {
cuda?: boolean;
@ -14,20 +17,29 @@ export interface EmbeddingsInstallOptions {
}
/**
* `gitnexus embeddings install [--cuda] [--force]` — fetch the optional local
* embedding stack on demand (#2370). Goes through the user's npm registry
* config (mirrors/proxies apply); with --cuda it additionally runs
* onnxruntime-node's postinstall to download the CUDA GPU binaries from NuGet
* (set GLOBAL_AGENT_HTTPS_PROXY behind a proxy).
* `gitnexus embeddings install [--cuda] [--force]` — fetch the local
* embedding stack. Default npm install does not include it. Goes through the
* user's npm registry config (mirrors/proxies apply); with --cuda it
* additionally runs onnxruntime-node's postinstall to download the CUDA GPU
* binaries from NuGet (set GLOBAL_AGENT_HTTPS_PROXY behind a proxy).
* `--force` refreshes prefix overrides; it does not replace a leftover
* 1.6.12 package-first tree in gitnexus node_modules.
*/
export const embeddingsInstallCommand = async (
options: EmbeddingsInstallOptions = {},
): Promise<void> => {
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
cliError(`${runtimeBlocker}\n`, { recoveryHint: 'local-embedding-unsupported' });
process.exitCode = 1;
return;
}
const resolved = resolveEmbeddingRuntime();
if (resolved?.source === 'package' && !options.force) {
cliInfo(
'The embedding stack is already installed with gitnexus itself — nothing to do.\n' +
'(Use --force to install a copy into the runtime prefix anyway.)',
'The embedding stack already resolves from this gitnexus install (leftover package-first tree) — nothing to do.\n' +
'(Use --force to refresh prefix overrides; a clean reinstall removes leftover packages.)',
);
return;
}

2
gitnexus/src/cli/i18n/en.ts
View file

@ -183,7 +183,7 @@ export const en = {
'Install the latest published GitNexus globally (`npm i -g gitnexus@<x.y.z>`).',
'help.command.embeddings.description': 'Manage the on-demand local embedding runtime',
'help.command.embeddings.install.description':
'Install the local embedding stack (@huggingface/transformers + onnxruntime-node) on demand. Heals installs where npm skipped the optional packages (e.g. behind an HTTP proxy, #2370). Downloads only from your configured npm registry — mirrors and proxies apply.',
'Install the local embedding stack (@huggingface/transformers + onnxruntime-node) on demand. The stack is not part of a default npm install. CPU installs download only from your configured npm registry — mirrors and proxies apply. `--cuda` additionally runs onnxruntime-node postinstall, which fetches CUDA binaries from NuGet (set GLOBAL_AGENT_HTTPS_PROXY behind a proxy).',
'help.command.clean.description': 'Delete GitNexus index for current repo',
'help.command.remove.description':
'Delete the GitNexus index for a registered repo (by alias, name, or absolute path). Unlike `clean`, does not require being inside the repo. Idempotent on unknown targets.',

2
gitnexus/src/cli/i18n/zh-CN.ts
View file

@ -177,7 +177,7 @@ export const zhCN = {
'通过 npm 全局安装最新发布的 GitNexus（`npm i -g gitnexus@<x.y.z>`）。',
'help.command.embeddings.description': '管理按需安装的本地嵌入运行时',
'help.command.embeddings.install.description':
'按需安装本地嵌入组件（@huggingface/transformers + onnxruntime-node）。修复 npm 跳过可选包的安装（例如在 HTTP 代理后，#2370）。仅从你配置的 npm registry 下载 — 镜像和代理均生效。',
'按需安装本地嵌入组件（@huggingface/transformers + onnxruntime-node）。默认 npm 安装不包含该组件。CPU 安装仅从你配置的 npm registry 下载 — 镜像和代理均生效。`--cuda` 还会运行 onnxruntime-node 的 postinstall，从 NuGet 获取 CUDA 二进制（代理后需设置 GLOBAL_AGENT_HTTPS_PROXY）。',
'help.command.clean.description': '删除当前仓库的 GitNexus 索引',
'help.command.remove.description':
'删除已注册仓库的 GitNexus 索引（按别名、名称或绝对路径）。与 `clean` 不同，不要求位于仓库内；未知目标会幂等处理。',

368
gitnexus/src/core/embeddings/embedder.ts
View file

@ -1,75 +1,44 @@
/**
* Embedder Module
* Embedder façade — HTTP or embedding sidecar.
*
* Singleton factory for transformers.js embedding pipeline.
* Handles model loading, caching, and both single and batch embedding operations.
*
* Uses snowflake-arctic-embed-xs by default (22M params, 384 dims, ~90MB)
* This module must not import the Hugging Face transformers package, the
* ONNX Node binding, the ONNX resolvers, or the child-only local init
* module. Local inference runs in the sidecar child; the parent keeps
* Ladybug writes.
*/
// Suppress ONNX Runtime native warnings (e.g. VerifyEachNodeIsAssignedToAnEp)
// Must be set BEFORE onnxruntime-node is imported by transformers.js
// Level 3 = Error only (skips Warning/Info)
if (!process.env.ORT_LOG_LEVEL) {
process.env.ORT_LOG_LEVEL = '3';
}
// Type-only import: erased at compile time so loading this module never pulls
// in @huggingface/transformers (and its native onnxruntime-node binding) at
// runtime. The runtime values (pipeline, env) are dynamically imported inside
// initEmbedder, after the platform guard has passed (#1515).
import type { FeatureExtractionPipeline, ProgressInfo } from '@huggingface/transformers';
import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
import {
DEFAULT_EMBEDDING_CONFIG,
type EmbeddingConfig,
type ModelProgressCallback,
} from './types.js';
import {
isHttpMode,
getHttpDimensions,
httpEmbed,
type EmbeddingRequestOptions,
} from './http-client.js';
import { resolveEmbeddingConfig } from './config.js';
import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
import { assessLocalEmbeddingRuntime, getLocalEmbeddingRuntimeBlocker } from './runtime-support.js';
import {
getLocalEmbeddingRuntimeBlocker,
getMissingLocalEmbeddingStackMessage,
} from './runtime-support.js';
import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js';
import { ensureEmbeddingStackResolvable } from './runtime-install.js';
import {
ensureOnnxRuntimeNodeMatchesSystem,
isEffectiveCudaAvailable,
} from './onnxruntime-node-resolver.js';
import { logger } from '../logger.js';
ensureEmbeddingSidecar,
getSidecarDevice,
reapEmbeddingSidecarAndWait,
sidecarEmbedBatch,
} from './embedding-sidecar-client.js';
import type { EmbeddingSidecarDevice } from './embedding-sidecar-protocol.js';
// Module-level state for singleton pattern
let embedderInstance: FeatureExtractionPipeline | null = null;
let isInitializing = false;
let initPromise: Promise<FeatureExtractionPipeline> | null = null;
let currentDevice: 'dml' | 'cuda' | 'cpu' | 'wasm' | null = null;
export type { ModelProgressCallback } from './types.js';
/**
* Progress callback type for model loading
*/
export type ModelProgressCallback = (progress: ModelProgress) => void;
export const getCurrentDevice = (): EmbeddingSidecarDevice | null => {
if (isHttpMode()) return null;
return getSidecarDevice();
};
/**
* Get the current device being used for inference
*/
export const getCurrentDevice = (): 'dml' | 'cuda' | 'cpu' | 'wasm' | null => currentDevice;
/**
* Initialize the embedding model
* Uses singleton pattern - only loads once, subsequent calls return cached instance
*
* @param onProgress - Optional callback for model download progress
* @param config - Optional configuration override
* @param forceDevice - Force a specific device
* @returns Promise resolving to the embedder pipeline
*/
export const initEmbedder = async (
onProgress?: ModelProgressCallback,
config: Partial<EmbeddingConfig> = {},
forceDevice?: 'dml' | 'cuda' | 'cpu' | 'wasm',
): Promise<FeatureExtractionPipeline> => {
forceDevice?: EmbeddingSidecarDevice,
): Promise<{ device: EmbeddingSidecarDevice }> => {
if (isHttpMode()) {
throw new Error(
'initEmbedder() should not be called in HTTP mode. ' +
@ -77,203 +46,39 @@ export const initEmbedder = async (
);
}
// Fail fast on platforms where the bundled native ONNX Runtime binding is not
// shipped (macOS Intel, #1515). Must run before any transformers.js /
// onnxruntime-node import or resolution — otherwise the native module load
// crashes with a raw "Cannot find module ...onnxruntime_binding.node" that
// ONNX_WEB_BACKEND=wasm cannot rescue (#1516). HTTP mode was already handled
// above, so this only blocks the local-runtime path.
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
throw new Error(runtimeBlocker);
}
// Return existing instance if available
if (embedderInstance) {
return embedderInstance;
return ensureEmbeddingSidecar({
onProgress,
embeddingConfig: config,
forceDevice,
});
};
export const getEmbedder = (): never => {
if (isHttpMode()) {
throw new Error(
'getEmbedder() is not available in HTTP embedding mode. Use embedText()/embedBatch() instead.',
);
}
// If already initializing, wait for that promise
if (isInitializing && initPromise) {
return initPromise;
}
isInitializing = true;
const finalConfig = resolveEmbeddingConfig(config);
// CUDA is probe-gated because ONNX Runtime can crash in native code when
// provider libraries are missing. DirectML stays opt-in for the same reason.
// Probe for CUDA first — ONNX Runtime crashes (uncatchable native error)
// if we attempt CUDA without the required shared libraries
const gpuDevice = isEffectiveCudaAvailable() ? 'cuda' : 'cpu';
const requestedDevice =
forceDevice || (finalConfig.device === 'auto' ? gpuDevice : finalConfig.device);
initPromise = (async () => {
try {
// Lazy-load transformers.js only after the runtime guard has passed, so
// unsupported platforms never reach the native ONNX import (#1515).
// Registered FIRST so it sits last in the hook chain (registerHooks runs
// the most recent hook first): when the optional stack was pruned at
// install time (#2370), its bare specifiers fall back to the on-demand
// runtime prefix.
ensureEmbeddingStackResolvable();
// Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common`
// import is unresolvable; register the fallback resolver first (#307).
ensureOnnxRuntimeCommonResolvable();
// Registered AFTER the common fallback so this hook resolves FIRST (Node
// runs the most-recently-registered hook first): on CUDA-13 hosts it
// redirects onnxruntime-node (and its version-matched onnxruntime-common)
// to the CUDA-13 build before transformers imports them. No-op on matching
// layouts, non-CUDA, Windows/DirectML, and macOS.
ensureOnnxRuntimeNodeMatchesSystem();
// The stack is an optionalDependency: npm prunes it when onnxruntime-node's
// postinstall can't reach api.nuget.org (#2370). Rethrow with actionable
// reinstall guidance instead of a raw ERR_MODULE_NOT_FOUND.
const { pipeline, env } = await import('@huggingface/transformers').catch((err: unknown) => {
const missing = getMissingLocalEmbeddingStackMessage(err);
if (missing) throw new Error(missing);
throw err;
});
// Configure transformers.js environment
env.allowLocalModels = false;
// Bridge user-controlled env vars to transformers.js: HF_HOME →
// env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
// applyHfEnvOverrides so the MCP embedder entry point behaves
// identically.
applyHfEnvOverrides(env);
const isDev = process.env.NODE_ENV === 'development';
if (isDev) {
logger.info(`🧠 Loading embedding model: ${finalConfig.modelId}`);
}
const progressCallback = onProgress
? (data: ProgressInfo) => {
const progress: ModelProgress = {
// Map the `progress_total` aggregate event (not in ModelProgress.status)
// back to 'progress' so callers don't need to handle it separately.
status:
data.status === 'progress_total'
? 'progress'
: ((data.status as ModelProgress['status']) ?? 'progress'),
file: 'file' in data ? data.file : undefined,
progress: 'progress' in data ? data.progress : undefined,
loaded: 'loaded' in data ? data.loaded : undefined,
total: 'total' in data ? data.total : undefined,
};
onProgress(progress);
}
: undefined;
// Try GPU first if auto, fall back to CPU
// Windows: dml (DirectML/DirectX12), Linux: cuda
const devicesToTry: Array<'dml' | 'cuda' | 'cpu' | 'wasm'> =
requestedDevice === 'dml' || requestedDevice === 'cuda'
? [requestedDevice, 'cpu']
: [requestedDevice as 'cpu' | 'wasm'];
for (const device of devicesToTry) {
try {
if (isDev && device === 'dml') {
logger.info('🔧 Trying DirectML (DirectX12) GPU backend...');
} else if (isDev && device === 'cuda') {
logger.info('🔧 Trying CUDA GPU backend...');
} else if (isDev && device === 'cpu') {
logger.info('🔧 Using CPU backend...');
} else if (isDev && device === 'wasm') {
logger.info('🔧 Using WASM backend (slower)...');
}
embedderInstance = await withHfDownloadRetry(
() =>
pipeline('feature-extraction', finalConfig.modelId, {
device: device,
dtype: 'fp32',
progress_callback: progressCallback,
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: finalConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
},
}),
{
onRetry: isDev
? (attempt, max, err) =>
logger.warn(
{ attempt, max, err: err.message },
`⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`,
)
: undefined,
},
);
currentDevice = device;
if (isDev) {
const label =
device === 'dml'
? 'GPU (DirectML/DirectX12)'
: device === 'cuda'
? 'GPU (CUDA)'
: device.toUpperCase();
logger.info(`✅ Using ${label} backend`);
logger.info('✅ Embedding model loaded successfully');
}
return embedderInstance!;
} catch (deviceError) {
// Network errors and circuit-open errors are not device-specific —
// they will fail the same way on every device. Rethrow immediately
// with actionable HF_ENDPOINT guidance rather than silently falling
// back to the next device.
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
if (isHfDownloadFailure(errMsg)) {
const endpointHint = process.env.HF_ENDPOINT
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
: `huggingface.co may be unreachable from your network.\n` +
` Set HF_ENDPOINT to a mirror and retry:\n` +
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
}
if (isDev && (device === 'cuda' || device === 'dml')) {
const gpuType = device === 'dml' ? 'DirectML' : 'CUDA';
logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`);
}
// Continue to next device in list
if (device === devicesToTry[devicesToTry.length - 1]) {
throw deviceError; // Last device failed, propagate error
}
}
}
throw new Error('No suitable device found for embedding model');
} catch (error) {
isInitializing = false;
initPromise = null;
embedderInstance = null;
throw error;
} finally {
isInitializing = false;
}
})();
return initPromise;
throw new Error(
'getEmbedder() is not available. Local inference runs in the embedding sidecar. Use embedText()/embedBatch() instead.',
);
};
/**
* Check if the embedder is initialized and ready
* Ready when HTTP embeddings are configured, or local runtime assessment
* is `ready` (blocker / prefix-unloadable / missing-stack are not ready).
* Sidecar liveness is not required. Resolution alone is not enough: a leftover
* 1.6.12 package-first tree on darwin/x64 still resolves, then embedText throws.
*/
export const isEmbedderReady = (): boolean => {
return isHttpMode() || embedderInstance !== null;
return isHttpMode() || assessLocalEmbeddingRuntime().status === 'ready';
};
/**
* Get the effective embedding dimensions.
* In HTTP mode, uses GITNEXUS_EMBEDDING_DIMS if set, otherwise the default.
*/
export const getEmbeddingDimensions = (): number => {
if (isHttpMode()) {
return getHttpDimensions() ?? DEFAULT_EMBEDDING_CONFIG.dimensions;
@ -281,27 +86,6 @@ export const getEmbeddingDimensions = (): number => {
return DEFAULT_EMBEDDING_CONFIG.dimensions;
};
/**
* Get the embedder instance (throws if not initialized)
*/
export const getEmbedder = (): FeatureExtractionPipeline => {
if (isHttpMode()) {
throw new Error(
'getEmbedder() is not available in HTTP embedding mode. Use embedText()/embedBatch() instead.',
);
}
if (!embedderInstance) {
throw new Error('Embedder not initialized. Call initEmbedder() first.');
}
return embedderInstance;
};
/**
* Embed a single text string
*
* @param text - Text to embed
* @returns Float32Array of embedding vector
*/
export const embedText = async (
text: string,
options: EmbeddingRequestOptions = {},
@ -312,24 +96,15 @@ export const embedText = async (
return vec;
}
const embedder = getEmbedder();
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
throw new Error(runtimeBlocker);
}
const result = await embedder(text, {
pooling: 'mean',
normalize: true,
});
// Result is a Tensor, convert to Float32Array
return new Float32Array(result.data as ArrayLike<number>);
const [vec] = await sidecarEmbedBatch([text], options);
return vec;
};
/**
* Embed multiple texts in a single batch
* More efficient than calling embedText multiple times
*
* @param texts - Array of texts to embed
* @returns Array of Float32Array embedding vectors
*/
export const embedBatch = async (
texts: string[],
options: EmbeddingRequestOptions = {},
@ -343,52 +118,21 @@ export const embedBatch = async (
return httpEmbed(texts, options);
}
const embedder = getEmbedder();
// Process batch
const result = await embedder(texts, {
pooling: 'mean',
normalize: true,
});
options.signal?.throwIfAborted();
// Result shape is [batch_size, dimensions]
// Need to split into individual vectors
const data = result.data as ArrayLike<number>;
const dimensions = DEFAULT_EMBEDDING_CONFIG.dimensions;
const embeddings: Float32Array[] = [];
for (let i = 0; i < texts.length; i++) {
const start = i * dimensions;
const end = start + dimensions;
embeddings.push(new Float32Array(Array.prototype.slice.call(data, start, end)));
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
throw new Error(runtimeBlocker);
}
return embeddings;
return sidecarEmbedBatch(texts, options);
};
/**
* Convert Float32Array to regular number array (for LadybugDB storage)
*/
export const embeddingToArray = (embedding: Float32Array): number[] => {
return Array.from(embedding);
};
/**
* Cleanup the embedder (free memory)
* Call this when done with embeddings
* Reap the sidecar. Never runs ONNX dispose in this process.
*/
export const disposeEmbedder = async (): Promise<void> => {
if (embedderInstance) {
// transformers.js pipelines may have a dispose method
try {
if ('dispose' in embedderInstance && typeof embedderInstance.dispose === 'function') {
await embedderInstance.dispose();
}
} catch {
// Ignore disposal errors
}
embedderInstance = null;
initPromise = null;
}
await reapEmbeddingSidecarAndWait();
};

227
gitnexus/src/core/embeddings/embedding-local-init.ts Normal file
View file

@ -0,0 +1,227 @@
/**
* Child-only local ONNX embedder.
*
* Imported solely by `embedding-sidecar.ts`. Parent processes must not import
* this module — it loads transformers.js / onnxruntime-node after the platform
* guard, which is what isolation is meant to keep out of analyze/serve/MCP.
*/
if (!process.env.ORT_LOG_LEVEL) {
process.env.ORT_LOG_LEVEL = '3';
}
import type { FeatureExtractionPipeline, ProgressInfo } from '@huggingface/transformers';
import {
DEFAULT_EMBEDDING_CONFIG,
type EmbeddingConfig,
type ModelProgress,
type ModelProgressCallback,
} from './types.js';
import type { EmbeddingSidecarDevice } from './embedding-sidecar-protocol.js';
import { resolveEmbeddingConfig } from './config.js';
import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
import {
getLocalEmbeddingRuntimeBlocker,
getMissingLocalEmbeddingStackMessage,
} from './runtime-support.js';
import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js';
import { ensureEmbeddingStackResolvable } from './runtime-install.js';
import {
ensureOnnxRuntimeNodeMatchesSystem,
isEffectiveCudaAvailable,
} from './onnxruntime-node-resolver.js';
import { logger } from '../logger.js';
let embedderInstance: FeatureExtractionPipeline | null = null;
let initPromise: Promise<FeatureExtractionPipeline> | null = null;
let currentDevice: EmbeddingSidecarDevice | null = null;
let activeDimensions = DEFAULT_EMBEDDING_CONFIG.dimensions;
const formatDeviceLabel = (device: EmbeddingSidecarDevice): string => {
switch (device) {
case 'dml':
return 'GPU (DirectML/DirectX12)';
case 'cuda':
return 'GPU (CUDA)';
default:
return device.toUpperCase();
}
};
export const getCurrentDevice = (): EmbeddingSidecarDevice | null => currentDevice;
export const initLocalEmbedder = async (
onProgress?: ModelProgressCallback,
config: Partial<EmbeddingConfig> = {},
forceDevice?: EmbeddingSidecarDevice,
): Promise<FeatureExtractionPipeline> => {
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
throw new Error(runtimeBlocker);
}
if (embedderInstance) {
return embedderInstance;
}
if (initPromise) {
return initPromise;
}
const finalConfig = resolveEmbeddingConfig(config);
const gpuDevice: EmbeddingSidecarDevice = isEffectiveCudaAvailable()
? 'cuda'
: process.platform === 'win32'
? 'dml'
: 'cpu';
const requestedDevice =
forceDevice || (finalConfig.device === 'auto' ? gpuDevice : finalConfig.device);
initPromise = (async () => {
try {
ensureEmbeddingStackResolvable();
ensureOnnxRuntimeCommonResolvable();
ensureOnnxRuntimeNodeMatchesSystem();
const { pipeline, env } = await import('@huggingface/transformers').catch((err: unknown) => {
const missing = getMissingLocalEmbeddingStackMessage(err);
if (missing) throw new Error(missing);
throw err;
});
env.allowLocalModels = false;
applyHfEnvOverrides(env);
const isDev = process.env.NODE_ENV === 'development';
if (isDev) {
logger.info(`🧠 Loading embedding model: ${finalConfig.modelId}`);
}
const progressCallback = onProgress
? (data: ProgressInfo) => {
const progress: ModelProgress = {
status:
data.status === 'progress_total'
? 'progress'
: ((data.status as ModelProgress['status']) ?? 'progress'),
file: 'file' in data ? data.file : undefined,
progress: 'progress' in data ? data.progress : undefined,
loaded: 'loaded' in data ? data.loaded : undefined,
total: 'total' in data ? data.total : undefined,
};
onProgress(progress);
}
: undefined;
const devicesToTry: EmbeddingSidecarDevice[] =
requestedDevice === 'dml' || requestedDevice === 'cuda'
? [requestedDevice, 'cpu']
: [requestedDevice];
for (const device of devicesToTry) {
try {
if (isDev && device === 'dml') {
logger.info('🔧 Trying DirectML (DirectX12) GPU backend...');
} else if (isDev && device === 'cuda') {
logger.info('🔧 Trying CUDA GPU backend...');
} else if (isDev && device === 'cpu') {
logger.info('🔧 Using CPU backend...');
} else if (isDev && device === 'wasm') {
logger.info('🔧 Using WASM backend (slower)...');
}
embedderInstance = await withHfDownloadRetry(
() =>
pipeline('feature-extraction', finalConfig.modelId, {
device: device,
dtype: 'fp32',
progress_callback: progressCallback,
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: finalConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
},
}),
{
onRetry: isDev
? (attempt, max, err) =>
logger.warn(
{ attempt, max, err: err.message },
`⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`,
)
: undefined,
},
);
currentDevice = device;
activeDimensions = finalConfig.dimensions;
if (isDev) {
logger.info(`✅ Using ${formatDeviceLabel(device)} backend`);
logger.info('✅ Embedding model loaded successfully');
}
return embedderInstance!;
} catch (deviceError) {
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
if (isHfDownloadFailure(errMsg)) {
const endpointHint = process.env.HF_ENDPOINT
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
: `huggingface.co may be unreachable from your network.\n` +
` Set HF_ENDPOINT to a mirror and retry:\n` +
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
}
if (isDev && (device === 'cuda' || device === 'dml')) {
const gpuType = device === 'dml' ? 'DirectML' : 'CUDA';
logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`);
}
if (device === devicesToTry[devicesToTry.length - 1]) {
throw deviceError;
}
}
}
throw new Error('No suitable device found for embedding model');
} catch (error) {
initPromise = null;
embedderInstance = null;
throw error;
}
})();
return initPromise;
};
const getLocalEmbedder = (): FeatureExtractionPipeline => {
if (!embedderInstance) {
throw new Error('Embedder not initialized. Call initLocalEmbedder() first.');
}
return embedderInstance;
};
export const localEmbedBatch = async (texts: string[]): Promise<Float32Array[]> => {
if (texts.length === 0) {
return [];
}
const embedder = getLocalEmbedder();
const result = await embedder(texts, {
pooling: 'mean',
normalize: true,
});
const data = result.data as ArrayLike<number>;
const dims = (result as { dims?: number[] }).dims;
const dimensions =
typeof dims?.[dims.length - 1] === 'number' ? dims[dims.length - 1] : activeDimensions;
const embeddings: Float32Array[] = [];
for (let i = 0; i < texts.length; i++) {
const start = i * dimensions;
const end = start + dimensions;
embeddings.push(new Float32Array(Array.prototype.slice.call(data, start, end)));
}
return embeddings;
};

19
gitnexus/src/core/embeddings/embedding-pipeline.ts
View file

@ -17,6 +17,7 @@ import {
embeddingToArray,
isEmbedderReady,
} from './embedder.js';
import { isHttpMode } from './http-client.js';
import { generateEmbeddingText } from './text-generator.js';
import { chunkNode, characterChunk } from './chunker.js';
import { extractStructuralNames } from './structural-extractor.js';
@ -560,7 +561,7 @@ export const runEmbeddingPipeline = async (
modelDownloadPercent: 0,
});
if (!isEmbedderReady()) {
if (!isHttpMode()) {
await initEmbedder((modelProgress: ModelProgress) => {
const downloadPercent = modelProgress.progress ?? 0;
onProgress({
@ -1009,6 +1010,13 @@ export const semanticSearch = async (
k: number = 10,
maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE),
): Promise<SemanticSearchResult[]> => {
// determinism: probe — existence only. Only `exists.length` is read; which
// row LIMIT 1 returns cannot change whether the table is empty.
const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`);
if (!exists.length) {
return [];
}
if (!isEmbedderReady()) {
throw new Error('Embedding model not initialized. Run embedding pipeline first.');
}
@ -1054,12 +1062,11 @@ export const semanticSearch = async (
}
if (bestChunks.size === 0) {
// The Cypher only. NOT `measurePersistedEmbeddingCount`: its tri-state
// exists so a publisher never writes a fabricated 0, whereas here `?? 0`
// is the right answer — an unknown count simply skips the exact scan.
// NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher
// never writes a fabricated 0, whereas here `?? 0` is the right answer —
// an unknown count simply skips the exact scan.
const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER);
const countRow = countRows[0];
const embeddingCount = Number(countRow?.cnt ?? countRow?.[0] ?? 0);
const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0);
const exactLimit = getExactScanLimit();
if (embeddingCount > 0 && embeddingCount <= exactLimit) {
const rows = await executeQuery(`

437
gitnexus/src/core/embeddings/embedding-sidecar-client.ts Normal file
View file

@ -0,0 +1,437 @@
/**
* Parent-side embedding sidecar client.
*
* Forks the sidecar over IPC and never imports the local ONNX init
* path, the embeddings barrel, or the pipeline module. Stdio must not inherit
* parent stdout (MCP JSON-RPC).
*/
import { fork, type ChildProcess, type ForkOptions } from 'node:child_process';
import { createRequire } from 'node:module';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { HF_BASE_DELAY_MS, resolveHfEnvMaxAttempts, resolveHfEnvTimeoutMs } from './hf-env.js';
import {
EMBEDDING_SIDECAR_DIED_LEAD,
getLocalEmbeddingRuntimeBlocker,
LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD,
} from './runtime-support.js';
import type { EmbeddingConfig, ModelProgress } from './types.js';
import type {
EmbeddingSidecarDevice,
SidecarRequest,
SidecarRequestBody,
SidecarResponse,
} from './embedding-sidecar-protocol.js';
import { logger } from '../logger.js';
export type ForkImpl = (
modulePath: string,
args: readonly string[],
options: ForkOptions,
) => ChildProcess;
const DEFAULT_EMBED_STALL_MS = 3 * 60 * 1000;
const MAX_RECREATES = 1;
let forkImpl: ForkImpl = fork;
let child: ChildProcess | null = null;
let ready = false;
let nextId = 1;
let recreatesUsed = 0;
let deathSeen = false;
let localUnavailable = false;
let device: EmbeddingSidecarDevice = 'cpu';
let ensureChain: Promise<void> | null = null;
let lastInitOptions:
| {
embeddingConfig?: Partial<EmbeddingConfig>;
forceDevice?: EmbeddingSidecarDevice;
}
| undefined;
let exitHooked = false;
let progressSink: ((progress: ModelProgress) => void) | undefined;
const pending = new Map<
number,
{
resolve: (value: SidecarResponse) => void;
reject: (error: Error) => void;
timer: NodeJS.Timeout;
}
>();
export const _setForkForTests = (impl: ForkImpl | null): void => {
forkImpl = impl ?? fork;
};
export const _resetEmbeddingSidecarForTests = (): void => {
reapEmbeddingSidecar();
recreatesUsed = 0;
deathSeen = false;
localUnavailable = false;
ready = false;
nextId = 1;
device = 'cpu';
ensureChain = null;
lastInitOptions = undefined;
};
const sidecarScriptPath = (): string => {
const callerPath = fileURLToPath(import.meta.url);
const isDev = callerPath.endsWith('.ts');
const file = isDev ? 'embedding-sidecar.ts' : 'embedding-sidecar.js';
return path.join(path.dirname(callerPath), file);
};
const tsxHookArgs = (): string[] => {
const callerPath = fileURLToPath(import.meta.url);
if (!callerPath.endsWith('.ts')) return [];
const require = createRequire(import.meta.url);
return ['--import', pathToFileURL(require.resolve('tsx/esm')).href];
};
const childEnv = (): NodeJS.ProcessEnv => {
const env = { ...process.env };
delete env.GITNEXUS_EMBEDDING_URL;
return env;
};
/** Parent timer starts before `child.send()`; child starts each download timeout after IPC. */
export const SIDECAR_INIT_IPC_SLACK_MS = 10_000;
export const sidecarInitTimeoutMs = (): number => {
const perAttempt = resolveHfEnvTimeoutMs();
const attempts = resolveHfEnvMaxAttempts();
// Child retries use exponential waits between attempts (HF_BASE_DELAY_MS * 2^i).
const backoffMs = attempts > 1 ? HF_BASE_DELAY_MS * (2 ** (attempts - 1) - 1) : 0;
return perAttempt * attempts + backoffMs + SIDECAR_INIT_IPC_SLACK_MS;
};
export const sidecarEmbedTimeoutMs = (): number => {
const raw = Number(process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS);
if (Number.isFinite(raw) && raw > 0) return raw;
return DEFAULT_EMBED_STALL_MS;
};
export class EmbeddingSidecarDeadError extends Error {
readonly code: number | null;
readonly signal: NodeJS.Signals | null;
constructor(code: number | null, signal: NodeJS.Signals | null) {
const detail = signal ? `signal ${signal}` : `exit ${code ?? 'unknown'}`;
super(`${EMBEDDING_SIDECAR_DIED_LEAD} (${detail})`);
this.name = 'EmbeddingSidecarDeadError';
this.code = code;
this.signal = signal;
}
}
const NATIVE_ABORT_SIGNALS = new Set<NodeJS.Signals>(['SIGSEGV', 'SIGABRT', 'SIGBUS', 'SIGILL']);
const localUnavailableError = (): Error => new Error(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD);
const noteChildDeath = (signal?: NodeJS.Signals | null): void => {
deathSeen = true;
if (signal && NATIVE_ABORT_SIGNALS.has(signal)) {
localUnavailable = true;
}
};
const rejectAll = (error: Error): void => {
const waiters = [...pending.values()];
for (const waiter of waiters) {
waiter.reject(error);
}
};
const attachChild = (proc: ChildProcess): void => {
proc.stderr?.on('data', (chunk: Buffer | string) => {
logger.debug({ sidecar: true }, String(chunk).trimEnd());
});
proc.on('message', (msg: SidecarResponse) => {
if (msg.type === 'progress') {
progressSink?.({
status: msg.status,
progress: msg.progress,
});
return;
}
const waiter = pending.get(msg.id);
if (!waiter) return;
waiter.resolve(msg);
});
proc.on('close', (code, signal) => {
if (child !== proc) return;
noteChildDeath(signal);
child = null;
ready = false;
rejectAll(new EmbeddingSidecarDeadError(code, signal));
});
proc.on('error', (err) => {
if (child !== proc) return;
noteChildDeath(null);
child = null;
ready = false;
rejectAll(err instanceof Error ? err : new Error(String(err)));
});
};
const request = (
msg: SidecarRequestBody,
timeoutMs: number,
signal?: AbortSignal,
): Promise<SidecarResponse> => {
if (!child) return Promise.reject(new Error('Embedding sidecar is not running'));
try {
signal?.throwIfAborted();
} catch (err) {
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
}
const id = nextId++;
return new Promise((resolve, reject) => {
let settled = false;
const onAbort = (): void => {
finish(() => {
try {
signal!.throwIfAborted();
} catch (err) {
reject(err instanceof Error ? err : new Error(String(err)));
}
});
};
const finish = (fn: () => void): void => {
if (settled) return;
settled = true;
clearTimeout(timer);
pending.delete(id);
signal?.removeEventListener('abort', onAbort);
fn();
};
const timer = setTimeout(() => {
finish(() => {
child?.kill('SIGKILL');
reject(new Error(`Embedding sidecar request timed out after ${timeoutMs}ms (${msg.type})`));
});
}, timeoutMs);
pending.set(id, {
resolve: (value) => finish(() => resolve(value)),
reject: (error) => finish(() => reject(error)),
timer,
});
signal?.addEventListener('abort', onAbort, { once: true });
if (signal?.aborted) {
onAbort();
return;
}
try {
child!.send({ ...msg, id } as SidecarRequest);
} catch (err) {
finish(() => {
reject(err instanceof Error ? err : new Error(String(err)));
});
}
});
};
const spawnSidecar = (): ChildProcess => {
const proc = forkImpl(sidecarScriptPath(), [], {
execArgv: tsxHookArgs(),
stdio: ['ignore', 'ignore', 'pipe', 'ipc'],
env: childEnv(),
});
if (!exitHooked) {
exitHooked = true;
process.on('exit', () => {
reapEmbeddingSidecar();
});
}
return proc;
};
export const reapEmbeddingSidecar = (): void => {
if (!child) return;
const proc = child;
child = null;
ready = false;
rejectAll(new Error('Embedding sidecar reaped'));
try {
proc.kill('SIGKILL');
} catch {
// already gone
}
};
/** Reap and wait for the killed child's close/error so an awaited dispose is a real boundary. */
export const reapEmbeddingSidecarAndWait = async (timeoutMs = 5_000): Promise<void> => {
const proc = child;
if (!proc) return;
const closed = new Promise<void>((resolve) => {
const finish = (): void => resolve();
proc.once('close', finish);
proc.once('error', finish);
});
reapEmbeddingSidecar();
let timer: ReturnType<typeof setTimeout> | undefined;
try {
await Promise.race([
closed,
new Promise<void>((resolve) => {
timer = setTimeout(resolve, timeoutMs);
}),
]);
} finally {
if (timer !== undefined) clearTimeout(timer);
}
};
export const getSidecarDevice = (): EmbeddingSidecarDevice | null => (ready ? device : null);
type EnsureSidecarOptions = {
onProgress?: (progress: ModelProgress) => void;
embeddingConfig?: Partial<EmbeddingConfig>;
forceDevice?: EmbeddingSidecarDevice;
signal?: AbortSignal;
};
const markUnavailableIfBudgetSpent = (): void => {
if (deathSeen && recreatesUsed >= MAX_RECREATES) {
localUnavailable = true;
}
};
const spawnAndInit = async (options?: EnsureSidecarOptions): Promise<void> => {
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
throw new Error(runtimeBlocker);
}
markUnavailableIfBudgetSpent();
if (localUnavailable) throw localUnavailableError();
if (deathSeen) {
recreatesUsed += 1;
deathSeen = false;
}
const persisted = {
embeddingConfig: options?.embeddingConfig ?? lastInitOptions?.embeddingConfig,
forceDevice: options?.forceDevice ?? lastInitOptions?.forceDevice,
};
child = spawnSidecar();
attachChild(child);
progressSink = options?.onProgress;
try {
const response = await request(
{
type: 'init',
embeddingConfig: persisted.embeddingConfig,
forceDevice: persisted.forceDevice,
},
sidecarInitTimeoutMs(),
);
if (response.type === 'error') throw new Error(response.message);
if (response.type !== 'ready') {
throw new Error(`Unexpected sidecar response: ${response.type}`);
}
ready = true;
device = response.device;
lastInitOptions = persisted;
} catch (err) {
reapEmbeddingSidecar();
throw err;
} finally {
progressSink = undefined;
}
};
const rejectConflictingForceDevice = (forceDevice?: EmbeddingSidecarDevice): void => {
if (forceDevice && forceDevice !== device) {
throw new Error(
`Embedding sidecar already initialized on ${device}; cannot switch to ${forceDevice}`,
);
}
};
const raceAbort = async <T>(promise: Promise<T>, signal?: AbortSignal): Promise<T> => {
if (!signal) return promise;
try {
signal.throwIfAborted();
} catch (err) {
return Promise.reject(err instanceof Error ? err : new Error(String(err)));
}
return new Promise<T>((resolve, reject) => {
const onAbort = (): void => {
signal.removeEventListener('abort', onAbort);
try {
signal.throwIfAborted();
} catch (err) {
reject(err instanceof Error ? err : new Error(String(err)));
}
};
signal.addEventListener('abort', onAbort, { once: true });
promise.then(
(value) => {
signal.removeEventListener('abort', onAbort);
resolve(value);
},
(err) => {
signal.removeEventListener('abort', onAbort);
reject(err);
},
);
});
};
export const ensureEmbeddingSidecar = async (
options?: EnsureSidecarOptions,
): Promise<{ device: EmbeddingSidecarDevice }> => {
if (localUnavailable) throw localUnavailableError();
if (ready && child) {
rejectConflictingForceDevice(options?.forceDevice);
return { device };
}
if (!ensureChain) {
const { signal: _ignored, ...initOptions } = options ?? {};
ensureChain = spawnAndInit(initOptions).finally(() => {
ensureChain = null;
});
}
await raceAbort(ensureChain, options?.signal);
rejectConflictingForceDevice(options?.forceDevice);
return { device };
};
const vectorsFromEmbedResponse = (response: SidecarResponse): Float32Array[] => {
if (response.type === 'error') throw new Error(response.message);
if (response.type !== 'vectors') {
throw new Error(`Unexpected sidecar response: ${response.type}`);
}
return response.vectors.map((row) => Float32Array.from(row));
};
export const sidecarEmbedBatch = async (
texts: string[],
options?: { signal?: AbortSignal },
): Promise<Float32Array[]> => {
if (texts.length === 0) return [];
options?.signal?.throwIfAborted();
if (localUnavailable) throw localUnavailableError();
if (!ready || !child) {
await ensureEmbeddingSidecar({ signal: options?.signal });
}
try {
return vectorsFromEmbedResponse(
await request({ type: 'embed', texts }, sidecarEmbedTimeoutMs(), options?.signal),
);
} catch (err) {
if (!(err instanceof EmbeddingSidecarDeadError)) throw err;
await ensureEmbeddingSidecar({ signal: options?.signal });
return vectorsFromEmbedResponse(
await request({ type: 'embed', texts }, sidecarEmbedTimeoutMs(), options?.signal),
);
}
};

29
gitnexus/src/core/embeddings/embedding-sidecar-protocol.ts Normal file
View file

@ -0,0 +1,29 @@
/**
* Embedding sidecar IPC types.
*
* Declarations only — consumers `import type` so this leaf never pulls the
* local ONNX init path or the parent façade. Mirrors `analyze-worker-protocol.ts`.
*/
import type { EmbeddingConfig, ModelProgress } from './types.js';
export type EmbeddingSidecarDevice = 'dml' | 'cuda' | 'cpu' | 'wasm';
export type SidecarRequest =
| {
id: number;
type: 'init';
embeddingConfig?: Partial<EmbeddingConfig>;
forceDevice?: EmbeddingSidecarDevice;
}
| { id: number; type: 'embed'; texts: string[] };
type DistributiveOmit<T, K extends keyof T> = T extends unknown ? Omit<T, K> : never;
export type SidecarRequestBody = DistributiveOmit<SidecarRequest, 'id'>;
export type SidecarResponse =
| { id: number; type: 'ready'; device: EmbeddingSidecarDevice }
| { id: number; type: 'progress'; progress: number; status: ModelProgress['status'] }
| { id: number; type: 'vectors'; vectors: number[][] }
| { id: number; type: 'error'; message: string };

12
gitnexus/src/core/embeddings/embedding-sidecar-reap.ts Normal file
View file

@ -0,0 +1,12 @@
/**
* Best-effort sidecar reap for shutdown paths that must not hide the
* caller's error, and that must not statically import the sidecar client.
*/
export const reapEmbeddingSidecarSafely = async (): Promise<void> => {
try {
const { reapEmbeddingSidecar } = await import('./embedding-sidecar-client.js');
reapEmbeddingSidecar();
} catch {
// Reap failure must not hide a pipeline error.
}
};

59
gitnexus/src/core/embeddings/embedding-sidecar.ts Normal file
View file

@ -0,0 +1,59 @@
/**
* Embedding sidecar entry — `child_process.fork()` target.
*
* Loads the local ONNX stack in this process only and returns vectors over IPC.
* The parent keeps Ladybug writes. Do not import the parent façade from here.
*/
import { getCurrentDevice, initLocalEmbedder, localEmbedBatch } from './embedding-local-init.js';
import type { SidecarRequest, SidecarResponse } from './embedding-sidecar-protocol.js';
const send = (msg: SidecarResponse): void => {
process.send?.(msg);
};
process.on('message', (msg: SidecarRequest) => {
void handle(msg);
});
async function handle(msg: SidecarRequest): Promise<void> {
try {
switch (msg.type) {
case 'init': {
await initLocalEmbedder(
(progress) => {
send({
id: msg.id,
type: 'progress',
progress: progress.progress ?? 0,
status: progress.status,
});
},
msg.embeddingConfig,
msg.forceDevice,
);
send({
id: msg.id,
type: 'ready',
device: getCurrentDevice() ?? 'cpu',
});
return;
}
case 'embed': {
const vectors = await localEmbedBatch(msg.texts);
send({
id: msg.id,
type: 'vectors',
vectors: vectors.map((row) => Array.from(row)),
});
return;
}
}
} catch (err) {
send({
id: msg.id,
type: 'error',
message: err instanceof Error ? err.message : String(err),
});
}
}

34
gitnexus/src/core/embeddings/hf-env.ts
View file

@ -22,6 +22,22 @@ export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000;
/** Upper bound clamped on the env-override attempt count. */
export const HF_MAX_ATTEMPTS_CAP = 10;
/** Per-attempt timeout from `HF_DOWNLOAD_TIMEOUT_MS`, else the built-in default. */
export const resolveHfEnvTimeoutMs = (): number => {
const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS);
return Number.isFinite(envTimeout) && envTimeout > 0
? Math.min(envTimeout, HF_MAX_TIMEOUT_MS)
: HF_DOWNLOAD_TIMEOUT_MS;
};
/** Attempt count from `HF_MAX_ATTEMPTS` (finite values that floor to ≥1), else the default. */
export const resolveHfEnvMaxAttempts = (): number => {
const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS);
if (!Number.isFinite(envMaxAttempts)) return HF_MAX_ATTEMPTS;
const attempts = Math.floor(envMaxAttempts);
return attempts >= 1 ? Math.min(attempts, HF_MAX_ATTEMPTS_CAP) : HF_MAX_ATTEMPTS;
};
/**
* @internal Exported only for unit tests and the two embedder entry points
* (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
@ -222,22 +238,12 @@ export async function withHfDownloadRetry<T>(
// Upper bounds are clamped to prevent accidental runaway configuration:
// - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min)
// - maxAttempts is floored (fractional values → integer) and capped at
// HF_MAX_ATTEMPTS_CAP (10). Values ≤ 0, NaN, or Infinity fall back to
// the built-in defaults.
const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS);
const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS);
const resolvedTimeout =
Number.isFinite(envTimeout) && envTimeout > 0
? Math.min(envTimeout, HF_MAX_TIMEOUT_MS)
: HF_DOWNLOAD_TIMEOUT_MS;
const resolvedMaxAttempts =
Number.isFinite(envMaxAttempts) && envMaxAttempts > 0
? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP)
: HF_MAX_ATTEMPTS;
// HF_MAX_ATTEMPTS_CAP (10). Values that floor below 1, NaN, or Infinity
// fall back to the built-in defaults.
const {
maxAttempts = resolvedMaxAttempts,
maxAttempts = resolveHfEnvMaxAttempts(),
baseDelayMs = HF_BASE_DELAY_MS,
timeoutMs = resolvedTimeout,
timeoutMs = resolveHfEnvTimeoutMs(),
circuit = hfDownloadCircuit,
onRetry,
} = options;

82
gitnexus/src/core/embeddings/runtime-install.ts
View file

@ -1,25 +1,22 @@
/**
* On-demand install of the optional local embedding stack (#2370).
* On-demand install of the local embedding stack.
*
* `@huggingface/transformers` and `onnxruntime-node` are optionalDependencies:
* npm prunes them (instead of failing the whole install) when
* `onnxruntime-node`'s postinstall cannot download its CUDA binaries from
* api.nuget.org — common behind HTTP proxies and regional firewalls, where
* that download ignores standard proxy env vars and 302 redirects.
* This overrides #2370: `@huggingface/transformers` and `onnxruntime-node` are
* no longer optionalDependencies. Default `npm install` does not fetch them.
* Operators opt in with `gitnexus embeddings install` (or analyze/sync auto-heal),
* which writes a prefix `package.json` (pins + overrides) and installs into
* `~/.gitnexus/embedding-runtime`. Version pins live in package.json
* `gitnexusEmbeddingStack` so they ship in the tarball without installing the
* packages. `onnxruntime-common` stays a regular dependency (#307).
*
* This module heals such an install without a reinstall: it fetches the stack
* into a user-level runtime prefix (`~/.gitnexus/embedding-runtime`) straight
* from the user's configured npm registry — honouring their mirror and proxy
* settings, the part of their network setup that demonstrably works — with
* `--ignore-scripts`, so no NuGet download is attempted at all. The CPU ONNX
* binding ships inside the npm tarball; only CUDA GPU acceleration needs the
* postinstall, and `installEmbeddingRuntime({ cuda: true })` opts into it.
*
* Resolution is package-first: a normally-installed stack always wins, and the
* runtime prefix is only consulted when the bare specifier does not resolve.
* Resolution is still package-first: a leftover 1.6.12 tree that already has
* the packages in gitnexus `node_modules` wins until a clean reinstall.
* `--force` refreshes prefix overrides; it does not remove a leftover
* package-first tree.
*/
import { createRequire } from 'node:module';
import { spawn, execFileSync, type ChildProcess } from 'node:child_process';
import { mkdirSync, writeFileSync } from 'node:fs';
import { pathToFileURL } from 'node:url';
import { homedir } from 'node:os';
import { join, resolve } from 'node:path';
@ -90,22 +87,42 @@ export const getEmbeddingRuntimeDir = (): string => {
return override ? resolve(override) : join(homedir(), '.gitnexus', 'embedding-runtime');
};
const EMBEDDING_STACK_NAMES = ['@huggingface/transformers', 'onnxruntime-node'] as const;
/**
* The version specs to install — read from gitnexus' own package.json
* `optionalDependencies` so the on-demand install can never drift from what a
* normal install would have provided. (The manifest ships in the tarball even
* when npm pruned the packages themselves.)
* The version specs to install — read from `gitnexusEmbeddingStack` on
* gitnexus' package.json so the on-demand prefix cannot drift from the
* committed pins after the packages left optionalDependencies.
*/
export const getEmbeddingStackSpecs = (): Record<string, string> => {
const manifest = require('../../../package.json') as {
optionalDependencies?: Record<string, string>;
gitnexusEmbeddingStack?: Record<string, string>;
};
const optional = manifest.optionalDependencies ?? {};
return Object.fromEntries(
['@huggingface/transformers', 'onnxruntime-node']
.filter((name) => optional[name] !== undefined)
.map((name) => [name, optional[name]]),
);
const stack = manifest.gitnexusEmbeddingStack ?? {};
const missing = EMBEDDING_STACK_NAMES.filter((name) => !stack[name]);
if (missing.length > 0) {
throw new Error(`package.json gitnexusEmbeddingStack is missing: ${missing.join(', ')}`);
}
return Object.fromEntries(EMBEDDING_STACK_NAMES.map((name) => [name, stack[name]]));
};
export const writeEmbeddingRuntimePrefixManifest = (): void => {
const dir = getEmbeddingRuntimeDir();
mkdirSync(dir, { recursive: true });
const specs = getEmbeddingStackSpecs();
const manifest = {
name: 'gitnexus-embedding-runtime',
private: true,
dependencies: specs,
overrides: {
'adm-zip': '>=0.6.0',
sharp: '>=0.35.0',
'@huggingface/transformers': {
'onnxruntime-node': specs['onnxruntime-node'],
},
},
};
writeFileSync(join(dir, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`);
};
export interface EmbeddingRuntimeResolution {
@ -125,7 +142,15 @@ export interface EmbeddingRuntimeResolution {
export const isPrefixRuntimeLoadable = (): boolean => typeof getRegisterHooks() === 'function';
/** Resolution anchored inside the runtime prefix (`<dir>/node_modules`). */
const prefixRequire = () => createRequire(join(getEmbeddingRuntimeDir(), 'noop.js'));
let cachedPrefixRequire: { dir: string; req: ReturnType<typeof createRequire> } | null = null;
const prefixRequire = (): ReturnType<typeof createRequire> => {
const dir = getEmbeddingRuntimeDir();
if (cachedPrefixRequire?.dir === dir) return cachedPrefixRequire.req;
const req = createRequire(join(dir, 'noop.js'));
cachedPrefixRequire = { dir, req };
return req;
};
/**
* True when BOTH load-bearing stack packages resolve from `req`. Probing
@ -349,6 +374,7 @@ export const installEmbeddingRuntime = async (
opts: EmbeddingInstallOptions = {},
timeoutMs: number = getEmbeddingInstallTimeoutMs(),
): Promise<void> => {
writeEmbeddingRuntimePrefixManifest();
const { args, env } = buildEmbeddingInstallCommand(opts);
await new Promise<void>((resolve, reject) => {
// Windows `npm` is a `.cmd` shim, so the spawn must go through a shell.

91
gitnexus/src/core/embeddings/runtime-support.ts
View file

@ -17,7 +17,7 @@
* (The runtime-install import below only resolves paths — it never loads the
* embedding stack.)
*/
import { resolveEmbeddingRuntime } from './runtime-install.js';
import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime } from './runtime-install.js';
/**
* Stable lead line of the macOS-Intel blocker message. Also used to recognise
@ -63,8 +63,9 @@ export const getLocalEmbeddingRuntimeBlocker = (
' - Run analyze without --embeddings (all other indexing still works).',
' - Point GITNEXUS_EMBEDDING_URL (with GITNEXUS_EMBEDDING_MODEL) at an',
' OpenAI-compatible /v1/embeddings endpoint to embed over HTTP.',
' - Run GitNexus on Linux or in Docker, where the native binding ships.',
' - Run GitNexus on Apple Silicon (darwin/arm64), which ships a binding.',
' - Run GitNexus on Linux or Apple Silicon (darwin/arm64), then',
' `gitnexus embeddings install`. Official CLI Docker images no longer',
' ship onnxruntime-node; bind-mount a prefix or use HTTP.',
' - Use a future GitNexus build that restores darwin/x64 ONNX support.',
].join('\n');
}
@ -87,37 +88,32 @@ export const isLocalEmbeddingRuntimeBlockerMessage = (message: string): boolean
* line (see {@link isMissingLocalEmbeddingStackMessage}).
*/
const LOCAL_EMBEDDING_STACK_MISSING_LEAD =
'Local semantic embeddings are unavailable: the optional embedding stack is not installed.';
'Local semantic embeddings are unavailable: the local embedding stack is not installed.';
/**
* The full guidance shown when the optional local embedding stack
* (`@huggingface/transformers` → `onnxruntime-node`) is missing at runtime.
*
* Both packages are `optionalDependencies` (#2370): `onnxruntime-node`'s
* postinstall downloads CUDA support binaries from api.nuget.org, which fails
* behind HTTP proxies and regional firewalls (its `global-agent` proxy layer
* ignores the standard HTTP_PROXY/HTTPS_PROXY vars and rejects 302 redirects).
* npm then skips the optional subtree instead of failing the whole install —
* every GitNexus feature except local embeddings keeps working.
* Guidance when transformers / onnxruntime-node are not resolvable.
* Default npm install no longer fetches those packages. Primary heal is
* `gitnexus embeddings install`. A leftover 1.6.12 package-first tree in
* gitnexus node_modules is residual until a clean reinstall; `--force`
* only refreshes the prefix overrides.
*/
export const localEmbeddingStackMissingMessage = (): string =>
[
LOCAL_EMBEDDING_STACK_MISSING_LEAD,
'npm skipped the optional packages @huggingface/transformers / onnxruntime-node',
"during install — usually because onnxruntime-node's postinstall could not",
'download its CUDA support binaries from api.nuget.org (common behind HTTP',
'proxies and regional firewalls, #2370). Everything except local embeddings',
'still works.',
'@huggingface/transformers and onnxruntime-node are not part of a default',
'gitnexus install. Everything except local embeddings still works.',
'',
'To enable local embeddings:',
' - Run `gitnexus embeddings install` — fetches the stack on demand through',
' your npm registry config (mirrors and proxies apply; no NuGet download).',
' `gitnexus analyze --embeddings` does this automatically.',
' - Run `gitnexus embeddings install` — fetches the stack through your npm',
' registry config into ~/.gitnexus/embedding-runtime (or',
' GITNEXUS_EMBEDDING_RUNTIME_DIR when set; mirrors and proxies',
' apply; no NuGet download). `gitnexus analyze --embeddings` and',
' `gitnexus embeddings sync` do this automatically.',
' Add --cuda on CUDA GPU hosts (behind a proxy, also set',
' GLOBAL_AGENT_HTTPS_PROXY=<proxy-url> for the NuGet download).',
' - Or reinstall with the CUDA download skipped (CPU embeddings need no CUDA):',
' ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus',
' (Windows: set ONNXRUNTIME_NODE_INSTALL=skip && npm install -g gitnexus)',
' - A leftover 1.6.12 install that still has those packages under',
' gitnexus node_modules is residual. `--force` only refreshes prefix',
' overrides; remove leftover packages with a clean reinstall.',
' - Or point GITNEXUS_EMBEDDING_URL (with GITNEXUS_EMBEDDING_MODEL) at an',
' OpenAI-compatible /v1/embeddings endpoint to embed over HTTP.',
].join('\n');
@ -140,13 +136,33 @@ export const localEmbeddingPrefixUnloadableMessage = (): string =>
[
LOCAL_EMBEDDING_PREFIX_UNLOADABLE_LEAD,
'The runtime prefix loads via module.registerHooks, which needs Node',
'>= 22.15 (on the 22.x line) or >= 23.5 (on the 23.x line). Either:',
' - Upgrade Node to a build that has module.registerHooks, or',
' - Reinstall the packages normally (works on every supported Node):',
' ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus',
' (Windows: set ONNXRUNTIME_NODE_INSTALL=skip && npm install -g gitnexus)',
'>= 22.15 (on the 22.x line) or >= 23.5 (on the 23.x line). Upgrade this',
'Node to a build that has module.registerHooks. Installing the prefix from',
'another Node cannot add that API here. A leftover 1.6.12 package-first',
'tree still loads without the hook; the prefix path does not.',
].join('\n');
export type LocalEmbeddingRuntimeAssessment =
| { status: 'blocked'; message: string }
| { status: 'prefix-unloadable'; message: string }
| { status: 'needs-install' }
| { status: 'ready' };
/**
* Shared local-runtime preflight for analyze and embeddings-sync.
* Callers keep their own error routing (CLI vs thrown Error).
*/
export const assessLocalEmbeddingRuntime = (): LocalEmbeddingRuntimeAssessment => {
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) return { status: 'blocked', message: runtimeBlocker };
const resolved = resolveEmbeddingRuntime();
if (!isPrefixRuntimeLoadable() && (resolved === null || resolved.source === 'runtime-prefix')) {
return { status: 'prefix-unloadable', message: localEmbeddingPrefixUnloadableMessage() };
}
if (resolved === null) return { status: 'needs-install' };
return { status: 'ready' };
};
/** Module specifiers whose absence means the optional embedding stack was pruned. */
const EMBEDDING_STACK_SPECIFIERS = ['@huggingface/transformers', 'onnxruntime-node'] as const;
@ -176,11 +192,26 @@ export const getMissingLocalEmbeddingStackMessage = (err: unknown): string | nul
export const isMissingLocalEmbeddingStackMessage = (message: string): boolean =>
message.includes(LOCAL_EMBEDDING_STACK_MISSING_LEAD);
/** Lead line when the embedding sidecar has been marked permanently unavailable. */
export const LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD =
'Local embeddings are unavailable after the sidecar aborted';
/** Lead of `EmbeddingSidecarDeadError` — native abort / unexpected child exit. */
export const EMBEDDING_SIDECAR_DIED_LEAD = 'Embedding sidecar died';
/**
* True when `message` is a sidecar-abort or sidecar-dead error. MCP `query()`
* must treat these like a missing stack so agents see the degradation.
*/
export const isLocalEmbeddingSidecarAbortMessage = (message: string): boolean =>
message.includes(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD) ||
message.includes(EMBEDDING_SIDECAR_DIED_LEAD);
/**
* True when the optional local embedding stack resolves from this install —
* either the normally-installed packages or the on-demand runtime prefix.
* Resolution only — nothing is imported, so this is safe on every platform
* (including macOS Intel, where *loading* onnxruntime-node would crash).
* Used by `doctor` to surface a pruned optional install (#2370) up front.
* Used by `doctor` to surface a missing local stack (default install excludes it).
*/
export const isLocalEmbeddingStackInstalled = (): boolean => resolveEmbeddingRuntime() !== null;

2
gitnexus/src/core/embeddings/types.ts
View file

@ -318,6 +318,8 @@ export interface ModelProgress {
total?: number;
}
export type ModelProgressCallback = (progress: ModelProgress) => void;
export interface ChunkSearchRow {
nodeId: string;
chunkIndex: number;

15
gitnexus/src/core/run-analyze.ts
View file

@ -3288,8 +3288,6 @@ async function runFullAnalysisInner(
? await snapshotDerivedRelsForFiles(filesToDelete, [...tablesWithRows])
: [];
await dropSearchFTSIndexes(indexCatalogRows, ftsIndexes, incrementalFtsRebuildTables);
// Release dropped search-index storage before the next bulk COPY.
await checkpointOnce();
// 1b. Remove the write set's existing rows — batched (#2409): one
// DETACH DELETE per table per 200-file chunk. The former per-file
// loop issued a count + delete per table per FILE — ~13k
@ -3389,6 +3387,18 @@ async function runFullAnalysisInner(
effectiveWriteCount: effectiveWriteSet.size,
deleteCount: filesToDelete.length,
});
// PARALLEL=false serializes CSV reading, not native COPY worker state.
// On a 32-thread host that scratch allocation exhausts the 256 MiB pool
// alongside retained FTS indexes, even for a one-row incremental write.
// Bound COPY only; restore the caller's setting before FTS construction.
// A failed COPY goes through the outer connection-cleanup handler.
const copyThreads = Number(
(await executeQuery("CALL current_setting('threads') RETURN *"))[0]?.threads,
);
if (!Number.isSafeInteger(copyThreads) || copyThreads < 1) {
throw new Error('Could not read the LadybugDB execution thread count before COPY');
}
await executeQuery('CALL threads=1');
await loadGraphToLbug(
subgraph,
pipelineResult.repoPath,
@ -3402,6 +3412,7 @@ async function runFullAnalysisInner(
undefined,
contentRetention,
);
await executeQuery(`CALL threads=${copyThreads}`);
if (preserveDerivedLayer && derivedSnapshot.length > 0) {
await restoreDerivedRels(derivedSnapshot);
}

19
gitnexus/src/core/search/hybrid-search.ts
View file

@ -177,13 +177,16 @@ export const hybridSearch = async (
// index recorded an explicit FTS opt-out (`disabledReason`, #3091).
// If FTS fails (e.g. extension not loaded in MCP process), fall back to
// semantic-only search instead of crashing with "bm25Results is not iterable".
let bm25Results: BM25SearchResult[] = [];
try {
const ftsResponse = await searchFTSFromLbug(query, limit, undefined, disabledReason);
bm25Results = ftsResponse?.results ?? [];
} catch {
// FTS unavailable — continue with semantic-only search
}
const semanticResults = await semanticSearch(executeQuery, query, limit);
const [bm25Results, semanticResults] = await Promise.all([
(async (): Promise<BM25SearchResult[]> => {
try {
const ftsResponse = await searchFTSFromLbug(query, limit, undefined, disabledReason);
return ftsResponse?.results ?? [];
} catch {
return [];
}
})(),
semanticSearch(executeQuery, query, limit).catch(() => []),
]);
return mergeWithRRF(bm25Results, semanticResults, limit);
};

219
gitnexus/src/mcp/core/embedder.ts
View file

@ -1,217 +1,36 @@
/**
* Embedder Module (Read-Only)
* MCP embedder façade — HTTP or the shared embedding sidecar client.
*
* Singleton factory for transformers.js embedding pipeline.
* For MCP, we only need to compute query embeddings, not batch embed.
* Local ONNX inference lives in the sidecar child. This module must not import
* the Hugging Face transformers package or the ONNX Node binding.
*/
// Type-only import: erased at compile time so loading this module never pulls
// in @huggingface/transformers (and its native onnxruntime-node binding) at
// runtime. The runtime values (pipeline, env) are dynamically imported inside
// initEmbedder, after the platform guard has passed (#1515).
import type { FeatureExtractionPipeline } from '@huggingface/transformers';
import {
isHttpMode,
getHttpDimensions,
httpEmbedQuery,
} from '../../core/embeddings/http-client.js';
import { resolveEmbeddingConfig } from '../../core/embeddings/config.js';
import {
applyHfEnvOverrides,
isHfDownloadFailure,
withHfDownloadRetry,
} from '../../core/embeddings/hf-env.js';
import {
getLocalEmbeddingRuntimeBlocker,
getMissingLocalEmbeddingStackMessage,
} from '../../core/embeddings/runtime-support.js';
import { ensureOnnxRuntimeCommonResolvable } from '../../core/embeddings/onnxruntime-common-resolver.js';
import { ensureEmbeddingStackResolvable } from '../../core/embeddings/runtime-install.js';
import { ensureOnnxRuntimeNodeMatchesSystem } from '../../core/embeddings/onnxruntime-node-resolver.js';
import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js';
disposeEmbedder as disposeCoreEmbedder,
embedText,
embeddingToArray,
getEmbeddingDimensions,
initEmbedder as initCoreEmbedder,
isEmbedderReady as isCoreEmbedderReady,
} from '../../core/embeddings/embedder.js';
import { httpEmbedQuery, isHttpMode } from '../../core/embeddings/http-client.js';
import { logger } from '../../core/logger.js';
// Model config
const MODEL_ID = 'Snowflake/snowflake-arctic-embed-xs';
export const initEmbedder = initCoreEmbedder;
// Module-level state for singleton pattern
let embedderInstance: FeatureExtractionPipeline | null = null;
let isInitializing = false;
let initPromise: Promise<FeatureExtractionPipeline> | null = null;
export const isEmbedderReady = isCoreEmbedderReady;
/**
* Initialize the embedding model (lazy, on first search)
*/
export const initEmbedder = async (): Promise<FeatureExtractionPipeline> => {
if (isHttpMode()) {
throw new Error('initEmbedder() should not be called in HTTP mode.');
}
// Fail fast on platforms where the bundled native ONNX Runtime binding is not
// shipped (macOS Intel, #1515). Must run before any transformers.js /
// onnxruntime-node import or resolution — otherwise the native module load
// crashes with a raw "Cannot find module ...onnxruntime_binding.node" that
// ONNX_WEB_BACKEND=wasm cannot rescue (#1516).
const runtimeBlocker = getLocalEmbeddingRuntimeBlocker();
if (runtimeBlocker) {
throw new Error(runtimeBlocker);
}
if (embedderInstance) {
return embedderInstance;
}
if (isInitializing && initPromise) {
return initPromise;
}
isInitializing = true;
initPromise = (async () => {
try {
// Lazy-load transformers.js only after the runtime guard has passed, so
// unsupported platforms never reach the native ONNX import (#1515).
// Registered FIRST so it sits last in the hook chain (registerHooks runs
// the most recent hook first): when the optional stack was pruned at
// install time (#2370), its bare specifiers fall back to the on-demand
// runtime prefix.
ensureEmbeddingStackResolvable();
// Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common`
// import is unresolvable; register the fallback resolver first (#307).
ensureOnnxRuntimeCommonResolvable();
// Registered AFTER the common fallback so this hook resolves FIRST (Node
// runs the most-recently-registered hook first): on CUDA-13 hosts it
// redirects onnxruntime-node to the system-matched build before
// transformers imports it. No-op on matching layouts, non-CUDA,
// Windows/DirectML, and macOS. Mirrors the core embedder's call site so
// MCP query-time embedding gets the same CUDA-13 fix.
ensureOnnxRuntimeNodeMatchesSystem();
// The stack is an optionalDependency: npm prunes it when onnxruntime-node's
// postinstall can't reach api.nuget.org (#2370). Rethrow with actionable
// reinstall guidance instead of a raw ERR_MODULE_NOT_FOUND.
const { pipeline, env } = await import('@huggingface/transformers').catch((err: unknown) => {
const missing = getMissingLocalEmbeddingStackMessage(err);
if (missing) throw new Error(missing);
throw err;
});
env.allowLocalModels = false;
// Bridge user-controlled env vars to transformers.js: HF_HOME →
// env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
// applyHfEnvOverrides so this MCP entry point behaves identically to
// the analyze pipeline embedder.
applyHfEnvOverrides(env);
const embeddingConfig = resolveEmbeddingConfig();
logger.info('GitNexus: Loading embedding model (first search may take a moment)...');
const devicesToTry: Array<'dml' | 'cuda' | 'cpu'> =
embeddingConfig.device === 'dml' || embeddingConfig.device === 'cuda'
? [embeddingConfig.device, 'cpu']
: ['cpu'];
for (const device of devicesToTry) {
try {
// Silence stdout and stderr during model load — ONNX Runtime and transformers.js
// may write progress/init messages that corrupt MCP stdio protocol or produce
// noisy warnings (e.g. node assignment to execution providers).
// Use the centralized silenceStdout() to avoid conflicts with pool-adapter's
// own stdout patching (independent patching caused restore-order bugs).
silenceStdout();
process.stderr.write = (() => true) as any;
try {
embedderInstance = await withHfDownloadRetry(() =>
pipeline('feature-extraction', MODEL_ID, {
device: device,
dtype: 'fp32',
session_options: {
logSeverityLevel: 3,
intraOpNumThreads: embeddingConfig.threads,
interOpNumThreads: 1,
executionMode: 'sequential',
},
}),
);
} finally {
restoreStdout();
process.stderr.write = realStderrWrite;
}
logger.info({ device }, 'GitNexus: Embedding model loaded');
return embedderInstance!;
} catch (deviceError) {
// Network errors and circuit-open errors are not device-specific —
// they will fail the same way on every device. Rethrow immediately
// with actionable HF_ENDPOINT guidance rather than silently falling
// back to the next device.
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
if (isHfDownloadFailure(errMsg)) {
const endpointHint = process.env.HF_ENDPOINT
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
: `huggingface.co may be unreachable from your network.\n` +
` Set HF_ENDPOINT to a mirror and retry:\n` +
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
}
if (device === 'cpu') throw new Error('Failed to load embedding model');
}
}
throw new Error('No suitable device found');
} catch (error) {
isInitializing = false;
initPromise = null;
embedderInstance = null;
throw error;
} finally {
isInitializing = false;
}
})();
return initPromise;
};
/**
* Check if embedder is ready
*/
export const isEmbedderReady = (): boolean => isHttpMode() || embedderInstance !== null;
/**
* Embed a query text for semantic search
*/
export const embedQuery = async (query: string): Promise<number[]> => {
if (isHttpMode()) {
return httpEmbedQuery(query);
}
const embedder = await initEmbedder();
const result = await embedder(query, {
pooling: 'mean',
normalize: true,
});
return Array.from(result.data as ArrayLike<number>);
return embeddingToArray(await embedText(query));
};
/**
* Get embedding dimensions
* Query-vector width for CAST. HTTP uses GITNEXUS_EMBEDDING_DIMS when set;
* local is the model default. Do not bind this to schema EMBEDDING_DIMS.
*/
export const getEmbeddingDims = (): number => {
return getHttpDimensions() ?? 384;
};
export const getEmbeddingDims = (): number => getEmbeddingDimensions();
/**
* Cleanup embedder
*/
export const disposeEmbedder = async (): Promise<void> => {
if (embedderInstance) {
try {
if ('dispose' in embedderInstance && typeof embedderInstance.dispose === 'function') {
await embedderInstance.dispose();
}
} catch {}
embedderInstance = null;
initPromise = null;
}
};
/** Reap the sidecar. Never runs ONNX dispose in this process. */
export const disposeEmbedder = disposeCoreEmbedder;

51
gitnexus/src/mcp/local/local-backend.ts
View file

@ -62,6 +62,7 @@ import {
} from '../../core/group/service.js';
import { resolveAtGroupMemberRepoPath } from '../../core/group/resolve-at-member.js';
import { collectBestChunks } from '../../core/embeddings/types.js';
import { reapEmbeddingSidecarSafely } from '../../core/embeddings/embedding-sidecar-reap.js';
import {
DEFAULT_MCP_VECTOR_MAX_DISTANCE,
getVectorMaxDistance,
@ -111,6 +112,7 @@ import {
import { logger } from '../../core/logger.js';
import {
isLocalEmbeddingRuntimeBlockerMessage,
isLocalEmbeddingSidecarAbortMessage,
isMissingLocalEmbeddingStackMessage,
} from '../../core/embeddings/runtime-support.js';
import {
@ -2962,9 +2964,10 @@ export class LocalBackend {
// over a single `current` phase slot.
const searchLimit = processLimit * maxSymbolsPerProcess; // fetch enough raw results
const ftsDisabledReason = getFtsDisabledReason(meta?.capabilities?.fts);
const vectorDegraded = { reason: undefined as string | undefined };
const [bm25SearchResult, semanticResults] = await Promise.all([
timer.time('bm25', this.bm25Search(repo, searchQuery, searchLimit, ftsDisabledReason)),
timer.time('vector', this.semanticSearch(repo, searchQuery, searchLimit)),
timer.time('vector', this.semanticSearch(repo, searchQuery, searchLimit, vectorDegraded)),
]);
// Guard against undefined results (#1489) — when FTS is entirely
@ -3427,6 +3430,9 @@ export class LocalBackend {
'Keyword results are unaffected.',
);
}
if (vectorDegraded.reason) {
warnings.push(vectorDegraded.reason);
}
if (enrichmentDegraded) {
warnings.push(
'Symbol enrichment partially failed — some process/cohesion/content data may be missing from these results (see server logs).',
@ -3569,7 +3575,12 @@ export class LocalBackend {
/**
* Semantic vector search helper
*/
private async semanticSearch(repo: RepoHandle, query: string, limit: number): Promise<any[]> {
private async semanticSearch(
repo: RepoHandle,
query: string,
limit: number,
degraded?: { reason?: string },
): Promise<any[]> {
// Whether THIS call produced a query vector — see `lastQueryEmbeddingDims`.
// A local flag, not a re-read of the map: the map may still hold an earlier
// call's width, and the catch below must only clear an entry it did not set.
@ -3735,18 +3746,21 @@ export class LocalBackend {
// the width IS still the live one). Clearing only in the former case
// keeps the recorded width a fact rather than a leftover (#2798).
if (embeddedDims === undefined) this.lastQueryEmbeddingDims.delete(repo.lbugPath);
// Embeddings disabled is the common, silent case. But a pruned or
// Node-unloadable optional stack (#2370/#2372) also lands here — surface it
// Embeddings disabled is the common, silent case. But a missing or
// Node-unloadable local stack (#2370/#2372) also lands here — surface it
// once so semantic search doesn't silently degrade to BM25 with no hint
// (the exact silent-degradation mode #2370 exists to fix). Emitted once per
// LocalBackend instance to keep stderr quiet on hot paths (like the VECTOR
// fallback above). All other errors stay silent, as before.
const message = err instanceof Error ? err.message : '';
if (
!this.warnedMissingEmbeddingStack &&
(isMissingLocalEmbeddingStackMessage(message) ||
isLocalEmbeddingRuntimeBlockerMessage(message))
) {
const isDegradedVectorError =
isMissingLocalEmbeddingStackMessage(message) ||
isLocalEmbeddingRuntimeBlockerMessage(message) ||
isLocalEmbeddingSidecarAbortMessage(message);
if (isDegradedVectorError) {
if (degraded) degraded.reason = message;
}
if (!this.warnedMissingEmbeddingStack && isDegradedVectorError) {
this.warnedMissingEmbeddingStack = true;
logger.warn(`GitNexus [query:vector]: ${message}`);
}
@ -9387,14 +9401,15 @@ export class LocalBackend {
}
async disconnect(): Promise<void> {
await closeLbug(); // close all connections
// Note: we intentionally do NOT call disposeEmbedder() here.
// ONNX Runtime's native cleanup segfaults on macOS and some Linux configs,
// and importing the embedder module on Node v24+ crashes if onnxruntime
// was never loaded during the session. Since process.exit(0) follows
// immediately after disconnect(), the OS reclaims everything. See #38, #89.
this.repos.clear();
this.contextCache.clear();
this.initializedRepos.clear();
try {
await closeLbug(); // close all connections
} finally {
// Reap even when Ladybug close rejects. Do not run ONNX dispose in this
// process (native dispose can SIGSEGV). The reap helper does not load ONNX.
await reapEmbeddingSidecarSafely();
this.repos.clear();
this.contextCache.clear();
this.initializedRepos.clear();
}
}
}

16
gitnexus/src/server/analyze-worker.ts
View file

@ -14,6 +14,7 @@
import type { ParentMessage, WorkerMessage } from './analyze-worker-protocol.js';
import { runWorkerAnalysis, createTerminalClaim } from './analyze-worker-core.js';
import { reapEmbeddingSidecar } from '../core/embeddings/embedding-sidecar-client.js';
type BoundedCheckpointBeforeExit =
typeof import('../core/lbug/shutdown-helpers.js').boundedCheckpointBeforeExit;
@ -26,6 +27,11 @@ type BoundedCheckpointBeforeExit =
// directly, so nothing else belongs in this list.
export type { CompleteMessage, WorkerMessage } from './analyze-worker-protocol.js';
function exitReapingSidecar(code: number): never {
reapEmbeddingSidecar();
process.exit(code);
}
function send(msg: WorkerMessage) {
// No try/catch: if the IPC channel is gone, process.send throws
// (ERR_IPC_CHANNEL_CLOSED) and that failure must NOT be swallowed. Every caller
@ -50,7 +56,7 @@ process.on('uncaughtException', (err: unknown) => {
const message = err instanceof Error ? err.message : 'Uncaught exception in worker';
send({ type: 'error', message });
} finally {
setTimeout(() => process.exit(1), 500);
setTimeout(() => exitReapingSidecar(1), 500);
}
});
@ -59,7 +65,7 @@ process.on('unhandledRejection', (reason: unknown) => {
const message = reason instanceof Error ? reason.message : 'Unhandled rejection in worker';
send({ type: 'error', message });
} finally {
setTimeout(() => process.exit(1), 500);
setTimeout(() => exitReapingSidecar(1), 500);
}
});
@ -76,13 +82,13 @@ function requestWorkerCancellation(source: string): void {
}
if (!started) {
// No analysis has started, so no native work needs a safe-point handshake.
process.exit(0);
exitReapingSidecar(0);
}
}
function exitAfterCancellation(): void {
if (!boundedCheckpointBeforeExit) {
process.exit(0);
exitReapingSidecar(0);
return;
}
void boundedCheckpointBeforeExit({
@ -154,6 +160,6 @@ process.on('message', async (msg: ParentMessage) => {
if (cancellationRequested) exitAfterCancellation();
// Normal terminal outcomes still need the existing process exit because
// LadybugDB stays live.
else setTimeout(() => process.exit(0), 500);
else setTimeout(() => exitReapingSidecar(0), 500);
}
});

92
gitnexus/test/integration/xaml-search.test.ts
View file

@ -18,7 +18,6 @@ import * as pool from '../../src/core/lbug/pool-adapter.js';
import { LocalBackend } from '../../src/mcp/local/local-backend.js';
import * as adapter from '../../src/core/lbug/lbug-adapter.js';
import * as ftsIndexes from '../../src/core/search/fts-indexes.js';
import * as checkpoints from '../../src/core/lbug/wal-checkpoint-driver.js';
import { batchInsertEmbeddings } from '../../src/core/embeddings/embedding-pipeline.js';
import { EMBEDDING_DIMS } from '../../src/core/lbug/schema.js';
@ -31,6 +30,87 @@ afterEach(async () => {
});
describe('persisted XAML declarations (#3202)', () => {
it('bounds incremental COPY threads and restores the setting without dropping untouched FTS indexes', async () => {
const repo = await setupMiniRepo();
const home = await createTempDir();
fixtures.push(repo, home);
vi.stubEnv('GITNEXUS_HOME', home.dbPath);
const file = path.join(repo.dbPath, 'Threaded.xaml');
const document = (name: string) =>
`<Grid xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" x:Name="${name}" />`;
await fs.writeFile(file, document('OriginalThreadedView'));
const options = { skipAgentsMd: true, registryName: 'copy-thread-fixture' };
const originalCopy = adapter.loadGraphToLbug;
const copyThreads: number[] = [];
vi.spyOn(adapter, 'loadGraphToLbug').mockImplementation(async (...args) => {
const rows = await adapter.executeQuery("CALL current_setting('threads') RETURN *");
copyThreads.push(Number(rows[0].threads));
return originalCopy(...args);
});
await runFullAnalysis(repo.dbPath, options, { onProgress() {} });
const statements = vi.spyOn(adapter, 'executeQuery');
const drops = vi.spyOn(ftsIndexes, 'dropSearchFTSIndexes');
await fs.writeFile(file, document('ChangedThreadedView'));
const result = await runFullAnalysis(repo.dbPath, options, { onProgress() {} });
expect(result.incrementalStats?.writeMode).toBe('incremental');
expect(result.ftsSkipped).toBe(false);
expect(copyThreads).toEqual([expect.any(Number), 1]);
expect(copyThreads[0]).toBeGreaterThan(0);
expect(
statements.mock.calls
.map(([query]) => query)
.filter((query) => query.startsWith('CALL threads=')),
).toEqual(['CALL threads=1', `CALL threads=${copyThreads[0]}`]);
expect(drops).toHaveBeenCalledOnce();
const tables = drops.mock.calls[0][2];
expect(tables).toBeDefined();
expect(tables?.has('Section')).toBe(true);
expect(tables?.has('Function')).toBe(false);
const { lbugPath } = getStoragePaths(repo.dbPath);
await initLbug(lbugPath);
expect((await searchFTSFromLbug('ChangedThreadedView')).results).toEqual(
expect.arrayContaining([expect.objectContaining({ filePath: 'Threaded.xaml' })]),
);
expect((await searchFTSFromLbug('OriginalThreadedView')).results).toEqual([]);
const rows = await adapter.executeQuery('CALL SHOW_INDEXES() RETURN *');
expect(rows.some((row) => adapter.indexRowName(row) === 'function_fts')).toBe(true);
}, 180_000);
it('closes the capped COPY connection and preserves the dirty receipt on failure', async () => {
const repo = await setupMiniRepo();
const home = await createTempDir();
fixtures.push(repo, home);
vi.stubEnv('GITNEXUS_HOME', home.dbPath);
const file = path.join(repo.dbPath, 'CopyFailure.xaml');
await fs.writeFile(
file,
'<Grid xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" x:Name="BeforeFailure" />',
);
const options = { skipAgentsMd: true, registryName: 'copy-thread-failure-fixture' };
await runFullAnalysis(repo.dbPath, options, { onProgress() {} });
const { storagePath } = getStoragePaths(repo.dbPath);
const before = await loadMeta(storagePath);
expect(before).not.toBeNull();
const failure = new Error('Injected incremental COPY failure');
const copy = vi.spyOn(adapter, 'loadGraphToLbug').mockImplementationOnce(async () => {
const rows = await adapter.executeQuery("CALL current_setting('threads') RETURN *");
expect(Number(rows[0].threads)).toBe(1);
throw failure;
});
await fs.writeFile(
file,
'<Grid xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" x:Name="AfterFailure" />',
);
await expect(runFullAnalysis(repo.dbPath, options, { onProgress() {} })).rejects.toBe(failure);
expect(copy).toHaveBeenCalledOnce();
expect(adapter.isLbugReady()).toBe(false);
expect(await loadMeta(storagePath)).toMatchObject({
incrementalInProgress: { phase: 'load-graph' },
indexedAt: before?.indexedAt,
fileHashes: before?.fileHashes,
});
}, 180_000);
it.each(['node-id', 'file-path'])(
'preserves existing code symbol types through %s lookup',
async (lookup) => {
@ -88,7 +168,7 @@ describe('persisted XAML declarations (#3202)', () => {
180_000,
);
it('keeps graph and embedding rows intact when the post-drop checkpoint fails', async () => {
it('keeps graph and embedding rows intact when dropping FTS indexes fails', async () => {
const repo = await setupMiniRepo();
const home = await createTempDir();
fixtures.push(repo, home);
@ -132,15 +212,11 @@ describe('persisted XAML declarations (#3202)', () => {
const originalDrop = ftsIndexes.dropSearchFTSIndexes;
let dropped = false;
const failure = new Error('Injected FTS index drop failure');
vi.spyOn(ftsIndexes, 'dropSearchFTSIndexes').mockImplementation(async (...args) => {
await originalDrop(...args);
dropped = true;
});
const failure = new Error('Injected post-drop checkpoint failure');
const originalCheckpoint = checkpoints.checkpointOnce;
vi.spyOn(checkpoints, 'checkpointOnce').mockImplementation(async () => {
if (dropped) throw failure;
await originalCheckpoint();
throw failure;
});
const deletes = vi.spyOn(adapter, 'deleteNodesForFiles');
const copies = vi.spyOn(adapter, 'loadGraphToLbug');

173
gitnexus/test/unit/analyze-local-embedding-error.test.ts
View file

@ -15,6 +15,17 @@
*/
import { beforeEach, describe, expect, it, vi } from 'vitest';
import { getLocalEmbeddingRuntimeBlocker } from '../../src/core/embeddings/runtime-support.js';
import type { LoggerCapture } from '../../src/core/logger.js';
async function withCapturedLogger<T>(fn: (cap: LoggerCapture) => Promise<T>): Promise<T> {
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
try {
return await fn(cap);
} finally {
cap.restore();
}
}
const runFullAnalysisMock = vi.fn();
// Controllable so the dual-match scenario can force the network heuristic to
@ -101,38 +112,32 @@ describe('analyzeCommand local-embedding-runtime error handling', () => {
it('routes the blocker to a clean local-embedding-unsupported message (exit 1)', async () => {
runFullAnalysisMock.mockRejectedValue(new Error(blockerMessage));
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await analyzeCommand(undefined, { embeddings: true });
expect(process.exitCode).toBe(1);
expect(process.exitCode).toBe(1);
const records = cap.records();
const blockerRecord = records.find((r) => r.recoveryHint === 'local-embedding-unsupported');
expect(blockerRecord).toBeDefined();
expect(typeof blockerRecord?.msg === 'string' && blockerRecord.msg).toMatch(/macOS Intel/);
cap.restore();
const records = cap.records();
const blockerRecord = records.find((r) => r.recoveryHint === 'local-embedding-unsupported');
expect(blockerRecord).toBeDefined();
expect(typeof blockerRecord?.msg === 'string' && blockerRecord.msg).toMatch(/macOS Intel/);
});
});
it('does NOT fall through to the module-not-found "installation may be corrupt" hint', async () => {
runFullAnalysisMock.mockRejectedValue(new Error(blockerMessage));
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await analyzeCommand(undefined, { embeddings: true });
const records = cap.records();
const corruptRecord = records.find(
(r) => typeof r.msg === 'string' && r.msg.includes('installation may be corrupt'),
);
expect(corruptRecord).toBeUndefined();
cap.restore();
const records = cap.records();
const corruptRecord = records.find(
(r) => typeof r.msg === 'string' && r.msg.includes('installation may be corrupt'),
);
expect(corruptRecord).toBeUndefined();
});
});
it('wins over the HF-download branch even when isHfDownloadFailure also matches (R4 ordering)', async () => {
@ -142,20 +147,17 @@ describe('analyzeCommand local-embedding-runtime error handling', () => {
isHfDownloadFailureMock.mockReturnValue(true);
runFullAnalysisMock.mockRejectedValue(new Error(blockerMessage));
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await analyzeCommand(undefined, { embeddings: true });
expect(process.exitCode).toBe(1);
expect(process.exitCode).toBe(1);
const records = cap.records();
expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(true);
// The HF-download branch must NOT have fired.
expect(records.some((r) => r.recoveryHint === 'hf-endpoint-unreachable')).toBe(false);
cap.restore();
const records = cap.records();
expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(true);
// The HF-download branch must NOT have fired.
expect(records.some((r) => r.recoveryHint === 'hf-endpoint-unreachable')).toBe(false);
});
});
it('does NOT route unrelated errors through the local-embedding branch', async () => {
@ -163,18 +165,15 @@ describe('analyzeCommand local-embedding-runtime error handling', () => {
new Error('Some unexpected failure unrelated to embeddings'),
);
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await analyzeCommand(undefined, { embeddings: true });
expect(process.exitCode).toBe(1);
expect(process.exitCode).toBe(1);
const records = cap.records();
expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(false);
cap.restore();
const records = cap.records();
expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(false);
});
});
});
@ -190,37 +189,58 @@ describe('analyzeCommand — prefix-runtime capability gate (#2372)', () => {
process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=8192`.trim();
});
it('does not spawn npm on darwin/x64 even when the stack is missing', async () => {
const orig = { platform: process.platform, arch: process.arch };
Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true });
Object.defineProperty(process, 'arch', { value: 'x64', configurable: true });
resolveEmbeddingRuntimeMock.mockReturnValue(null);
isPrefixRuntimeLoadableMock.mockReturnValue(true);
try {
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
expect(process.exitCode).toBe(1);
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(
true,
);
});
} finally {
Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true });
Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true });
}
});
it('fails fast without installing when nothing is installed and the prefix is unloadable', async () => {
resolveEmbeddingRuntimeMock.mockReturnValue(null);
isPrefixRuntimeLoadableMock.mockReturnValue(false);
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
expect(process.exitCode).toBe(1);
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
const record = cap.records().find((r) => r.recoveryHint === 'local-embedding-stack-missing');
expect(typeof record?.msg === 'string' && record.msg).toMatch(/module\.registerHooks/);
cap.restore();
expect(process.exitCode).toBe(1);
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
const record = cap.records().find((r) => r.recoveryHint === 'local-embedding-stack-missing');
expect(typeof record?.msg === 'string' && record.msg).toMatch(/module\.registerHooks/);
});
});
it('fails fast on a resolved-but-unloadable prefix (the previously-uncaught state)', async () => {
resolveEmbeddingRuntimeMock.mockReturnValue({ source: 'runtime-prefix' });
isPrefixRuntimeLoadableMock.mockReturnValue(false);
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await withCapturedLogger(async (cap) => {
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
expect(process.exitCode).toBe(1);
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-stack-missing')).toBe(
true,
);
cap.restore();
expect(process.exitCode).toBe(1);
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-stack-missing')).toBe(
true,
);
});
});
it('installs with a shorter-than-default timeout when nothing is installed and the prefix is loadable', async () => {
@ -229,18 +249,17 @@ describe('analyzeCommand — prefix-runtime capability gate (#2372)', () => {
// Reject afterwards so analyze bails right after the install, isolating the gate.
runFullAnalysisMock.mockRejectedValue(new Error('stop after install'));
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS } =
await import('../../src/core/embeddings/runtime-install.js');
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
await withCapturedLogger(async () => {
const { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS } =
await import('../../src/core/embeddings/runtime-install.js');
const { analyzeCommand } = await import('../../src/cli/analyze.js');
await analyzeCommand(undefined, { embeddings: true });
expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1);
// analyze must pass the shorter deadline so a blackholed proxy can't stall
// the run for the 10-minute default.
const timeoutArg = installEmbeddingRuntimeMock.mock.calls[0][1] as number;
expect(timeoutArg).toBe(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS);
cap.restore();
expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1);
// analyze must pass the shorter deadline so a blackholed proxy can't stall
// the run for the 10-minute default.
const timeoutArg = installEmbeddingRuntimeMock.mock.calls[0][1] as number;
expect(timeoutArg).toBe(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS);
});
});
});

32
gitnexus/test/unit/analyze-worker-reap.test.ts Normal file
View file

@ -0,0 +1,32 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
const reapEmbeddingSidecarMock = vi.hoisted(() => vi.fn());
vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({
reapEmbeddingSidecar: () => reapEmbeddingSidecarMock(),
}));
describe('analyze-worker exitReapingSidecar', () => {
let exitSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
reapEmbeddingSidecarMock.mockReset();
exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never);
});
afterEach(() => {
exitSpy.mockRestore();
});
it('reaps the sidecar before process.exit on cancel-before-start', async () => {
await import('../../src/server/analyze-worker.js');
process.emit('message', { type: 'cancel' });
await vi.waitFor(() => {
expect(reapEmbeddingSidecarMock).toHaveBeenCalled();
expect(exitSpy).toHaveBeenCalledWith(0);
});
const reapOrder = reapEmbeddingSidecarMock.mock.invocationCallOrder[0];
const exitOrder = exitSpy.mock.invocationCallOrder[0];
expect(reapOrder).toBeLessThan(exitOrder);
});
});

144
gitnexus/test/unit/assert-publish-grammar-coverage.test.ts
View file

@ -2,9 +2,10 @@ import { describe, it, expect } from 'vitest';
import { spawnSync } from 'node:child_process';
import { createRequire } from 'node:module';
import { fileURLToPath } from 'node:url';
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { globSync } from 'glob';
/**
* Coverage for the publish guard `scripts/assert-publish-grammar-coverage.cjs`.
@ -21,8 +22,12 @@ const requireCjs = createRequire(import.meta.url);
const SCRIPT = fileURLToPath(
new URL('../../scripts/assert-publish-grammar-coverage.cjs', import.meta.url),
);
const { findCoverageProblems, filesShipsVendorSource, findStrayBuildArtifacts } =
requireCjs(SCRIPT);
const {
findCoverageProblems,
findPackedFilesProblems,
filesShipsVendorSource,
findStrayBuildArtifacts,
} = requireCjs(SCRIPT);
describe('findCoverageProblems (pure decision core)', () => {
it('passes when source ships, even with incomplete prebuilds (transitional state)', () => {
@ -55,6 +60,67 @@ describe('findCoverageProblems (pure decision core)', () => {
});
});
describe('findPackedFilesProblems (files globs, not on-disk counts)', () => {
const grammars = ['tree-sitter-c', 'tree-sitter-kotlin'];
const lean = [
'vendor/**/prebuilds/**',
'vendor/**/bindings/node/index.js',
'vendor/**/package.json',
'vendor/leiden/index.cjs',
'vendor/leiden/utils.cjs',
];
it('passes the current lean files list', () => {
expect(findPackedFilesProblems({ filesField: lean, grammarNames: grammars })).toEqual([]);
});
it('fails when files only covers FTS plus one grammar prebuild', () => {
const problems = findPackedFilesProblems({
filesField: ['vendor/lbug-fts/prebuilds/**', 'vendor/tree-sitter-c/prebuilds/**'],
grammarNames: grammars,
});
expect(problems.some((p: string) => p.includes('tree-sitter-kotlin'))).toBe(true);
expect(problems.some((p: string) => p.includes('bindings/node/index.js'))).toBe(true);
expect(problems.some((p: string) => p.includes('leiden'))).toBe(true);
});
it('fails when only one grammar binding is listed explicitly', () => {
const problems = findPackedFilesProblems({
filesField: [
'vendor/**/prebuilds/**',
'vendor/tree-sitter-c/bindings/node/index.js',
'vendor/**/package.json',
'vendor/leiden/index.cjs',
'vendor/leiden/utils.cjs',
],
grammarNames: grammars,
});
expect(problems.some((p: string) => p.includes('tree-sitter-kotlin'))).toBe(true);
expect(problems.some((p: string) => p.includes('bindings/node/index.js'))).toBe(true);
});
it('fails when per-grammar package.json is omitted', () => {
const problems = findPackedFilesProblems({
filesField: [
'vendor/**/prebuilds/**',
'vendor/**/bindings/node/index.js',
'vendor/leiden/index.cjs',
'vendor/leiden/utils.cjs',
],
grammarNames: grammars,
});
expect(problems.some((p: string) => p.includes('package.json'))).toBe(true);
});
it('fails when Leiden entrypoints are dropped', () => {
const problems = findPackedFilesProblems({
filesField: ['vendor/**/prebuilds/**', 'vendor/**/bindings/node/index.js'],
grammarNames: grammars,
});
expect(problems.some((p: string) => p.includes('leiden'))).toBe(true);
});
});
describe('filesShipsVendorSource', () => {
it('ships when a broad vendor entry is present', () => {
expect(filesShipsVendorSource(['dist', 'vendor', 'web'])).toBe(true);
@ -114,11 +180,81 @@ describe('findStrayBuildArtifacts (stray vendor build dirs that would ship + sha
});
});
describe('real repo publish-safety (guards against premature files narrowing)', () => {
describe('real repo publish-safety (lean files + 6/6 prebuilds)', () => {
const GITNEXUS_ROOT = fileURLToPath(new URL('../..', import.meta.url));
const VENDORED = [
'tree-sitter-c',
'tree-sitter-dart',
'tree-sitter-kotlin',
'tree-sitter-objc',
'tree-sitter-proto',
'tree-sitter-swift',
'tree-sitter-zig',
] as const;
const TUPLES = [
'linux-x64',
'linux-arm64',
'darwin-x64',
'darwin-arm64',
'win32-x64',
'win32-arm64',
] as const;
const simulatePackedVendorFiles = (filesField: string[]): string[] => {
const packed = new Set<string>();
for (const pattern of filesField) {
const n = String(pattern).replace(/\\/g, '/');
if (!n.startsWith('vendor')) continue;
for (const match of globSync(n, { cwd: GITNEXUS_ROOT, nodir: true, dot: false })) {
packed.add(match.replace(/\\/g, '/'));
}
}
return [...packed];
};
it('the script exits 0 against the committed repo state', () => {
// Deterministic: reads package.json + walks vendor/ — no npm pack, fast.
const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 20_000 });
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain('[publish-guard] OK');
expect(r.stdout).toContain('prebuilds-only');
});
it('package.json files is lean: no bare vendor, no shipped parser.c, load files remain', () => {
const pkg = JSON.parse(readFileSync(path.join(GITNEXUS_ROOT, 'package.json'), 'utf8')) as {
files: string[];
};
expect(filesShipsVendorSource(pkg.files)).toBe(false);
expect(pkg.files).not.toContain('vendor');
expect(pkg.files).toContain('vendor/**/prebuilds/**');
expect(pkg.files).toContain('vendor/leiden/index.cjs');
expect(pkg.files).toContain('vendor/leiden/utils.cjs');
expect(pkg.files).toContain('vendor/lbug-fts/manifest.json');
const packed = simulatePackedVendorFiles(pkg.files);
expect(packed.some((p) => /vendor\/tree-sitter-[^/]+\/src\/parser\.c$/.test(p))).toBe(false);
expect(packed.some((p) => /vendor\/tree-sitter-[^/]+\/src\/scanner\.c$/.test(p))).toBe(false);
expect(packed.some((p) => p.endsWith('binding.gyp'))).toBe(false);
for (const name of VENDORED) {
expect(packed, name).toContain(`vendor/${name}/package.json`);
expect(packed, name).toContain(`vendor/${name}/bindings/node/index.js`);
expect(packed, name).toContain(`vendor/${name}/src/node-types.json`);
for (const tuple of TUPLES) {
expect(
packed.some(
(p) => p.startsWith(`vendor/${name}/prebuilds/${tuple}/`) && p.endsWith('.node'),
),
`${name} ${tuple}`,
).toBe(true);
}
}
expect(packed).toContain('vendor/leiden/index.cjs');
expect(packed).toContain('vendor/leiden/utils.cjs');
expect(packed).toContain('vendor/lbug-fts/manifest.json');
for (const tuple of ['linux-x64', 'linux-arm64', 'darwin-x64', 'darwin-arm64', 'win32-x64']) {
expect(packed).toContain(`vendor/lbug-fts/prebuilds/${tuple}/libfts.lbug_extension`);
}
});
});

3
gitnexus/test/unit/cli-commands.test.ts
View file

@ -109,7 +109,8 @@ describe('CLI commands', () => {
// (vendored-grammars.ts), so postinstall no longer materializes anything.
expect(pkg.default.scripts.postinstall).not.toContain('materialize-vendor-grammars.cjs');
expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-grammars.cjs');
expect(pkg.default.files).toContain('vendor');
expect(pkg.default.files).toContain('vendor/**/prebuilds/**');
expect(pkg.default.files).not.toContain('vendor');
});
it('declares node-gyp-build/node-addon-api as regular dependencies (runtime-load contract)', async () => {

14
gitnexus/test/unit/doctor-format.test.ts
View file

@ -163,7 +163,12 @@ describe('doctor embedding-runtime support status', () => {
['linux', 'x64'],
['win32', 'x64'],
] as Array<[NodeJS.Platform, NodeJS.Architecture]>) {
const { status, detail } = localEmbeddingDoctorStatus({ httpMode: false, platform, arch });
const { status, detail } = localEmbeddingDoctorStatus({
httpMode: false,
platform,
arch,
resolution: { source: 'package' },
});
expect(status).toBe('✓ local embeddings supported');
expect(detail).toBeNull();
}
@ -179,15 +184,16 @@ describe('doctor embedding-runtime support status', () => {
expect(detail).toBeNull();
});
it('flags a pruned optional embedding stack with reinstall guidance (#2370)', () => {
it('flags a missing local embedding stack with install guidance', () => {
const { status, detail } = localEmbeddingDoctorStatus({
httpMode: false,
platform: 'linux',
arch: 'x64',
resolution: null,
});
expect(status).toBe('✗ optional embedding stack not installed');
expect(detail).toContain('ONNXRUNTIME_NODE_INSTALL=skip');
expect(status).toBe('✗ local embedding stack not installed');
expect(detail).toContain('gitnexus embeddings install');
expect(detail).not.toContain('ONNXRUNTIME_NODE_INSTALL=skip');
});
it('reports a package-sourced stack as supported regardless of Node loadability', () => {

26
gitnexus/test/unit/embedder.test.ts
View file

@ -1,4 +1,9 @@
import { describe, it, expect } from 'vitest';
import { readFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { describe, expect, it } from 'vitest';
import { isHttpMode } from '../../src/core/embeddings/http-client.js';
import { assessLocalEmbeddingRuntime } from '../../src/core/embeddings/runtime-support.js';
import { getEmbeddingDims, isEmbedderReady } from '../../src/mcp/core/embedder.js';
describe('embedder', () => {
@ -9,8 +14,23 @@ describe('embedder', () => {
});
describe('isEmbedderReady', () => {
it('returns false before initialization', () => {
expect(isEmbedderReady()).toBe(false);
it('follows HTTP mode or a ready local runtime, not resolution alone', () => {
expect(isEmbedderReady()).toBe(
isHttpMode() || assessLocalEmbeddingRuntime().status === 'ready',
);
});
});
it('does not import the Hugging Face transformers package', () => {
const src = readFileSync(
path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../src/mcp/core/embedder.ts'),
'utf8',
);
expect(src).not.toMatch(/from\s+['"]@huggingface\/transformers['"]/);
expect(src).not.toMatch(/import\s*\(\s*['"]@huggingface\/transformers['"]/);
expect(src).not.toMatch(/from\s+['"]onnxruntime-node['"]/);
expect(src).not.toMatch(/import\s*\(\s*['"]onnxruntime-node['"]/);
expect(src).not.toMatch(/from\s+['"].*embedding-local-init['"]/);
expect(src).not.toMatch(/import\s*\(\s*['"].*embedding-local-init['"]/);
});
});

76
gitnexus/test/unit/embedding-runtime-install.test.ts
View file

@ -3,6 +3,17 @@ import { EventEmitter } from 'node:events';
import { homedir } from 'node:os';
import { join, resolve } from 'node:path';
import { createRequire } from 'node:module';
const { mkdirSyncMock, writeFileSyncMock } = vi.hoisted(() => ({
mkdirSyncMock: vi.fn(),
writeFileSyncMock: vi.fn(),
}));
vi.mock('node:fs', async (importOriginal) => ({
...(await importOriginal<typeof import('node:fs')>()),
mkdirSync: (...args: unknown[]) => mkdirSyncMock(...args),
writeFileSync: (...args: unknown[]) => writeFileSyncMock(...args),
}));
import {
ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS,
buildEmbeddingInstallCommand,
@ -43,6 +54,8 @@ const savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
beforeEach(() => {
for (const key of ENV_KEYS) delete process.env[key];
mkdirSyncMock.mockReset();
writeFileSyncMock.mockReset();
});
afterEach(() => {
@ -76,16 +89,23 @@ describe('getEmbeddingRuntimeDir', () => {
});
describe('getEmbeddingStackSpecs', () => {
it('mirrors the optionalDependencies manifest exactly (drift guard, #2370)', () => {
it('mirrors gitnexusEmbeddingStack exactly and keeps the packages off optionalDependencies', () => {
const manifest = require('../../package.json') as {
optionalDependencies: Record<string, string>;
gitnexusEmbeddingStack: Record<string, string>;
optionalDependencies?: Record<string, string>;
dependencies: Record<string, string>;
overrides: Record<string, unknown>;
};
expect(getEmbeddingStackSpecs()).toEqual({
'@huggingface/transformers': manifest.optionalDependencies['@huggingface/transformers'],
'onnxruntime-node': manifest.optionalDependencies['onnxruntime-node'],
'@huggingface/transformers': manifest.gitnexusEmbeddingStack['@huggingface/transformers'],
'onnxruntime-node': manifest.gitnexusEmbeddingStack['onnxruntime-node'],
});
expect(manifest.optionalDependencies['@huggingface/transformers']).toBeDefined();
expect(manifest.optionalDependencies['onnxruntime-node']).toBeDefined();
expect(manifest.gitnexusEmbeddingStack['@huggingface/transformers']).toBeDefined();
expect(manifest.gitnexusEmbeddingStack['onnxruntime-node']).toBeDefined();
expect(manifest.optionalDependencies?.['@huggingface/transformers']).toBeUndefined();
expect(manifest.optionalDependencies?.['onnxruntime-node']).toBeUndefined();
expect(manifest.dependencies['onnxruntime-common']).toBeDefined();
expect(JSON.stringify(manifest.overrides)).not.toContain('$onnxruntime-node');
});
});
@ -121,9 +141,18 @@ describe('buildEmbeddingInstallCommand', () => {
});
describe('resolveEmbeddingRuntime', () => {
it('finds the normally-installed stack (package source wins over the prefix)', () => {
it('returns package when the leftover tree is present, otherwise null on a clean prefix', () => {
process.env.GITNEXUS_EMBEDDING_RUNTIME_DIR = '/nonexistent/for/this/test';
expect(resolveEmbeddingRuntime()).toEqual({ source: 'package' });
// After U4 a default install has no stack. A leftover 1.6.12 package-first
// tree in this workspace's node_modules still wins until a clean reinstall.
let leftoverPackageTree = true;
try {
require.resolve('@huggingface/transformers');
require.resolve('onnxruntime-node');
} catch {
leftoverPackageTree = false;
}
expect(resolveEmbeddingRuntime()).toEqual(leftoverPackageTree ? { source: 'package' } : null);
});
});
@ -191,6 +220,37 @@ describe('quoteWin32Arg', () => {
});
});
describe('installEmbeddingRuntime prefix manifest', () => {
it('writes pins and overrides to prefix package.json before npm spawn', async () => {
process.env.GITNEXUS_EMBEDDING_RUNTIME_DIR = resolve('/custom/runtime');
const child = new FakeChild();
spawnMock.mockReturnValue(child);
const pending = installEmbeddingRuntime({}, 10_000);
expect(mkdirSyncMock).toHaveBeenCalled();
expect(writeFileSyncMock).toHaveBeenCalled();
const [pathArg, body] = writeFileSyncMock.mock.calls[0] as [string, string];
expect(pathArg).toBe(join(resolve('/custom/runtime'), 'package.json'));
const written = JSON.parse(body) as {
dependencies: Record<string, string>;
overrides: {
'adm-zip': string;
sharp: string;
'@huggingface/transformers': { 'onnxruntime-node': string };
};
};
const specs = getEmbeddingStackSpecs();
expect(written.dependencies).toEqual(specs);
expect(written.overrides['adm-zip']).toBe('>=0.6.0');
expect(written.overrides.sharp).toBe('>=0.35.0');
expect(written.overrides['@huggingface/transformers']['onnxruntime-node']).toBe(
specs['onnxruntime-node'],
);
expect(spawnMock).toHaveBeenCalled();
child.emit('close', 0, null);
await pending;
});
});
describe('installEmbeddingRuntime — spawn lifecycle', () => {
beforeEach(() => {
vi.useFakeTimers();

8
gitnexus/test/unit/embedding-runtime-resolution.test.ts
View file

@ -5,10 +5,10 @@ import { describe, it, expect, vi, afterEach } from 'vitest';
*
* The package tier uses runtime-install's module-scope require (anchored at its
* own `import.meta.url`); the prefix tier uses a require anchored at
* `<prefix>/noop.js`. In dev/CI both optional deps ARE really installed, so the
* package tier can never miss with the real require — we mock `createRequire` to
* route each anchor to a fake whose `.resolve()` is driven by a fixture map,
* exercising the partial / full / missing permutations.
* `<prefix>/noop.js`. After U4 a clean tree has no embedding packages, but a
* leftover 1.6.12 package-first install still can. We mock `createRequire` so
* the package tier is driven by a fixture map, exercising the partial / full /
* missing permutations without depending on this workspace's node_modules.
*
* This file has ZERO static import of runtime-install.js (the dual-instance
* rule): every load goes through the dynamic-import harness, so no real

81
gitnexus/test/unit/embedding-runtime-support.test.ts
View file

@ -3,9 +3,12 @@ import {
getLocalEmbeddingRuntimeBlocker,
getMissingLocalEmbeddingStackMessage,
isLocalEmbeddingRuntimeBlockerMessage,
isLocalEmbeddingSidecarAbortMessage,
isLocalEmbeddingStackInstalled,
isMissingLocalEmbeddingStackMessage,
LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD,
localEmbeddingStackMissingMessage,
localEmbeddingPrefixUnloadableMessage,
} from '../../src/core/embeddings/runtime-support.js';
/**
@ -26,11 +29,11 @@ vi.mock('@huggingface/transformers', () => {
});
/**
* Spy for the CUDA-13 build-matching resolver hook. Both local embedders must
* call this before importing transformers.js — mocked (rather than exercising
* the real resolver's env/subprocess probing) to keep this suite fast and
* platform-independent; `onnxruntime-node-resolver.test.ts` covers the
* resolver's own decision logic.
* Spy for the CUDA-13 build-matching resolver hook. Child `initLocalEmbedder`
* must call this before importing transformers.js — mocked (rather than
* exercising the real resolver's env/subprocess probing) to keep this suite
* fast and platform-independent; `onnxruntime-node-resolver.test.ts` covers
* the resolver's own decision logic.
*/
const { resolverHookInstalled } = vi.hoisted(() => ({ resolverHookInstalled: vi.fn() }));
@ -39,6 +42,14 @@ vi.mock('../../src/core/embeddings/onnxruntime-node-resolver.js', () => ({
isEffectiveCudaAvailable: () => false,
}));
vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({
ensureEmbeddingSidecar: vi.fn(async () => ({ device: 'cpu' })),
getSidecarDevice: () => 'cpu',
reapEmbeddingSidecar: vi.fn(),
reapEmbeddingSidecarAndWait: vi.fn(async () => {}),
sidecarEmbedBatch: vi.fn(async (texts: string[]) => texts.map(() => new Float32Array(384))),
}));
/**
* Mock `module.registerHooks` with a spy (#2372). Without this, a successful
* local `initEmbedder()` calls the REAL `ensureEmbeddingStackResolvable` /
@ -118,8 +129,8 @@ describe('getLocalEmbeddingRuntimeBlocker', () => {
// Safe alternatives
expect(text).toMatch(/without --embeddings/);
expect(text).toContain('GITNEXUS_EMBEDDING_URL');
expect(text).toMatch(/Linux or in Docker/);
expect(text).toMatch(/Apple Silicon/);
expect(text).toMatch(/Linux or Apple Silicon/);
expect(text).toMatch(/Official CLI Docker images no longer/);
// Addresses the GitNexus device knob too, not only ONNX_WEB_BACKEND (R3 / #1987)
expect(text).toContain('GITNEXUS_EMBEDDING_DEVICE');
});
@ -211,16 +222,27 @@ describe('getMissingLocalEmbeddingStackMessage (#2370 pruned optional stack)', (
it('produces guidance naming every recovery path', () => {
const msg = localEmbeddingStackMissingMessage();
expect(msg).toContain('gitnexus embeddings install');
expect(msg).toContain('ONNXRUNTIME_NODE_INSTALL=skip');
expect(msg).toContain('GITNEXUS_EMBEDDING_RUNTIME_DIR');
expect(msg).not.toContain('ONNXRUNTIME_NODE_INSTALL=skip');
expect(msg).toContain('GLOBAL_AGENT_HTTPS_PROXY');
expect(msg).toContain('GITNEXUS_EMBEDDING_URL');
expect(msg).toContain('#2370');
expect(msg).toContain('1.6.12');
// Must not trip analyze.ts's generic "installation may be corrupt" branch.
expect(msg).not.toMatch(/Cannot find (module|package)/);
expect(msg).not.toContain('MODULE_NOT_FOUND');
});
});
describe('localEmbeddingPrefixUnloadableMessage', () => {
it('tells the user to upgrade this Node, not to install from another Node and retry', () => {
const msg = localEmbeddingPrefixUnloadableMessage();
expect(msg).toContain('module.registerHooks');
expect(msg).toContain('Upgrade this');
expect(msg).toContain('cannot add that API here');
expect(msg).not.toMatch(/then retry/i);
});
});
describe('isMissingLocalEmbeddingStackMessage', () => {
it('recognises its own message and rejects the platform blocker and unrelated errors', () => {
expect(isMissingLocalEmbeddingStackMessage(localEmbeddingStackMissingMessage())).toBe(true);
@ -231,10 +253,19 @@ describe('isMissingLocalEmbeddingStackMessage', () => {
});
});
describe('isLocalEmbeddingSidecarAbortMessage', () => {
it('recognises sidecar abort and sidecar-dead text', () => {
expect(isLocalEmbeddingSidecarAbortMessage(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD)).toBe(true);
expect(isLocalEmbeddingSidecarAbortMessage('Embedding sidecar died (signal SIGSEGV)')).toBe(
true,
);
expect(isLocalEmbeddingSidecarAbortMessage(localEmbeddingStackMissingMessage())).toBe(false);
});
});
describe('isLocalEmbeddingStackInstalled', () => {
it('resolves the optional stack in the dev workspace without importing it', () => {
expect(isLocalEmbeddingStackInstalled()).toBe(true);
// Resolution only — the transformers.js import spy must not fire.
it('probes stack resolution without importing transformers.js', () => {
isLocalEmbeddingStackInstalled();
expect(transformersImported).not.toHaveBeenCalled();
});
});
@ -386,46 +417,40 @@ describe('MCP embedQuery on darwin/x64', () => {
});
describe('CUDA-13 resolver hook installation (both local-embedding entrypoints)', () => {
// Regression guard for the two local embedders drifting apart (gitnexus PR #2341
// follow-up): both `core/embeddings/embedder.ts` and `mcp/core/embedder.ts` must
// install the CUDA-build-matching redirect during a successful local init. (The
// source itself places the call before `await import('@huggingface/transformers')`
// — not re-asserted here via mock call-order, since the hoisted `@huggingface/
// transformers` mock's factory only fires once per file run for this external
// package, making a second per-test "called fresh" assertion on it unreliable.)
it('core embedder installs the resolver hook on a successful local init', async () => {
// Parent façade must not install CUDA / transformers hooks; only the
// sidecar child local-init path does.
it('core embedder does not install the resolver hook in the parent on local init', async () => {
const restore = stubPlatform('linux', 'x64');
try {
const { initEmbedder } = await import('../../src/core/embeddings/embedder.js');
await expect(initEmbedder()).resolves.toBeDefined();
expect(resolverHookInstalled).toHaveBeenCalled();
expect(resolverHookInstalled).not.toHaveBeenCalled();
expect(transformersImported).not.toHaveBeenCalled();
} finally {
restore();
}
});
it('MCP embedder installs the resolver hook on a successful local init', async () => {
it('MCP embedder does not install the resolver hook in the parent on local init', async () => {
const restore = stubPlatform('linux', 'x64');
try {
const { initEmbedder } = await import('../../src/mcp/core/embedder.js');
await expect(initEmbedder()).resolves.toBeDefined();
expect(resolverHookInstalled).toHaveBeenCalled();
expect(resolverHookInstalled).not.toHaveBeenCalled();
expect(transformersImported).not.toHaveBeenCalled();
} finally {
restore();
}
});
it('registers the runtime-prefix fallback through the mocked registerHooks, not the real global API (#2372)', async () => {
// The whole point of the node:module mock: a successful local init exercises
// ensureEmbeddingStackResolvable's registration via the spy, so no real
// process-global resolution hook leaks into other tests in the worker.
it('does not register process-global resolution hooks in the parent on local init (#2372)', async () => {
const restore = stubPlatform('linux', 'x64');
try {
const { initEmbedder } = await import('../../src/core/embeddings/embedder.js');
await expect(initEmbedder()).resolves.toBeDefined();
expect(registerHooksSpy).toHaveBeenCalled();
expect(registerHooksSpy).not.toHaveBeenCalled();
} finally {
restore();
}

431
gitnexus/test/unit/embedding-sidecar-client.test.ts Normal file
View file

@ -0,0 +1,431 @@
import { EventEmitter } from 'node:events';
import { readFileSync } from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { ChildProcess } from 'node:child_process';
import type {
SidecarRequest,
SidecarResponse,
} from '../../src/core/embeddings/embedding-sidecar-protocol.js';
const embeddingsDir = path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
'../../src/core/embeddings',
);
class FakeChild extends EventEmitter {
killed = false;
connected = true;
stdout = new EventEmitter();
stderr = new EventEmitter();
send = vi.fn((msg: SidecarRequest) => {
queueMicrotask(() => {
const response = this.respond(msg);
if (response) this.emit('message', response);
});
return true;
});
kill = vi.fn((signal?: NodeJS.Signals) => {
this.killed = true;
this.emit('close', signal === 'SIGKILL' ? null : 0, signal ?? null);
return true;
});
unref = vi.fn();
respond(msg: SidecarRequest): SidecarResponse | undefined {
if (msg.type === 'init') {
return { id: msg.id, type: 'ready', device: 'cpu' };
}
if (msg.type === 'embed') {
return {
id: msg.id,
type: 'vectors',
vectors: msg.texts.map(() => [0.25, 0.5, 0.75]),
};
}
return undefined;
}
}
describe('embedding sidecar client', () => {
const originalUrl = process.env.GITNEXUS_EMBEDDING_URL;
const originalModel = process.env.GITNEXUS_EMBEDDING_MODEL;
const originalHfTimeout = process.env.HF_DOWNLOAD_TIMEOUT_MS;
const originalHfAttempts = process.env.HF_MAX_ATTEMPTS;
const originalSidecarTimeout = process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS;
const hostPlatform = process.platform;
const hostArch = process.arch;
let forkMock: ReturnType<typeof vi.fn>;
let children: FakeChild[];
beforeEach(async () => {
delete process.env.GITNEXUS_EMBEDDING_URL;
// Local-success cases must not inherit a darwin/x64 host blocker.
Object.defineProperty(process, 'platform', { value: 'linux', configurable: true });
Object.defineProperty(process, 'arch', { value: 'x64', configurable: true });
children = [];
forkMock = vi.fn((_script: string, _args: string[], _opts: unknown) => {
const child = new FakeChild();
children.push(child);
return child as unknown as ChildProcess;
});
const client = await import('../../src/core/embeddings/embedding-sidecar-client.js');
client._resetEmbeddingSidecarForTests();
client._setForkForTests(forkMock);
});
afterEach(async () => {
const client = await import('../../src/core/embeddings/embedding-sidecar-client.js');
client._resetEmbeddingSidecarForTests();
client._setForkForTests(null);
if (originalUrl === undefined) delete process.env.GITNEXUS_EMBEDDING_URL;
else process.env.GITNEXUS_EMBEDDING_URL = originalUrl;
if (originalModel === undefined) delete process.env.GITNEXUS_EMBEDDING_MODEL;
else process.env.GITNEXUS_EMBEDDING_MODEL = originalModel;
if (originalHfTimeout === undefined) delete process.env.HF_DOWNLOAD_TIMEOUT_MS;
else process.env.HF_DOWNLOAD_TIMEOUT_MS = originalHfTimeout;
if (originalHfAttempts === undefined) delete process.env.HF_MAX_ATTEMPTS;
else process.env.HF_MAX_ATTEMPTS = originalHfAttempts;
if (originalSidecarTimeout === undefined)
delete process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS;
else process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS = originalSidecarTimeout;
Object.defineProperty(process, 'platform', { value: hostPlatform, configurable: true });
Object.defineProperty(process, 'arch', { value: hostArch, configurable: true });
vi.useRealTimers();
vi.unstubAllGlobals();
});
it('strips GITNEXUS_EMBEDDING_URL and does not inherit stdout', async () => {
process.env.GITNEXUS_EMBEDDING_URL = 'http://custom.example/v1';
const { ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await ensureEmbeddingSidecar();
expect(forkMock).toHaveBeenCalledTimes(1);
const opts = forkMock.mock.calls[0][2] as {
env: NodeJS.ProcessEnv;
stdio: unknown;
};
expect(opts.env.GITNEXUS_EMBEDDING_URL).toBeUndefined();
expect(opts.stdio).toEqual(['ignore', 'ignore', 'pipe', 'ipc']);
});
it('reports no sidecar device until init is ready', async () => {
const { getSidecarDevice, ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
expect(getSidecarDevice()).toBeNull();
await ensureEmbeddingSidecar();
expect(getSidecarDevice()).toBe('cpu');
});
it('rejects a forceDevice that conflicts with the initialized sidecar', async () => {
const { ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await expect(ensureEmbeddingSidecar()).resolves.toEqual({ device: 'cpu' });
await expect(ensureEmbeddingSidecar({ forceDevice: 'cuda' })).rejects.toThrow(
/already initialized on cpu; cannot switch to cuda/,
);
await expect(ensureEmbeddingSidecar({ forceDevice: 'cpu' })).resolves.toEqual({
device: 'cpu',
});
});
it('forks once for two batches', async () => {
const { sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
const first = await sidecarEmbedBatch(['a']);
const second = await sidecarEmbedBatch(['b']);
expect(first).toHaveLength(1);
expect(second).toHaveLength(1);
expect(forkMock).toHaveBeenCalledTimes(1);
});
it('does not fork on darwin/x64', async () => {
const orig = { platform: process.platform, arch: process.arch };
Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true });
Object.defineProperty(process, 'arch', { value: 'x64', configurable: true });
try {
const { ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await expect(ensureEmbeddingSidecar()).rejects.toThrow(/macOS Intel/);
expect(forkMock).not.toHaveBeenCalled();
} finally {
Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true });
Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true });
}
});
it('does not fork on an empty batch', async () => {
const { sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await expect(sidecarEmbedBatch([])).resolves.toEqual([]);
expect(forkMock).not.toHaveBeenCalled();
});
it('does not fork in HTTP mode', async () => {
process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
process.env.GITNEXUS_EMBEDDING_MODEL = 'test-model';
const mockVec = Array.from({ length: 384 }, (_, i) => i / 384);
vi.stubGlobal(
'fetch',
vi.fn(async (_url: string, init: { body: string }) => {
const n = (JSON.parse(init.body) as { input: string[] }).input.length;
return {
ok: true,
json: async () => ({ data: Array.from({ length: n }, () => ({ embedding: mockVec })) }),
};
}),
);
const { embedBatch, isEmbedderReady } = await import('../../src/core/embeddings/embedder.js');
expect(isEmbedderReady()).toBe(true);
const batch = await embedBatch(['hello']);
expect(batch).toHaveLength(1);
expect(forkMock).not.toHaveBeenCalled();
});
it('marks local embeddings unavailable on native abort and does not respawn', async () => {
const { sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await sidecarEmbedBatch(['first']);
expect(forkMock).toHaveBeenCalledTimes(1);
children[0].emit('close', null, 'SIGSEGV');
await expect(sidecarEmbedBatch(['second'])).rejects.toThrow(
/unavailable after the sidecar aborted/,
);
expect(forkMock).toHaveBeenCalledTimes(1);
});
it('does not respawn after init-time SIGSEGV', async () => {
forkMock.mockImplementation(() => {
const child = new FakeChild();
children.push(child);
child.send = vi.fn(() => {
queueMicrotask(() => child.emit('close', null, 'SIGSEGV'));
return true;
});
return child as unknown as ChildProcess;
});
const { ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await expect(ensureEmbeddingSidecar()).rejects.toThrow(/Embedding sidecar died/);
expect(forkMock).toHaveBeenCalledTimes(1);
await expect(ensureEmbeddingSidecar()).rejects.toThrow(/unavailable after the sidecar aborted/);
expect(forkMock).toHaveBeenCalledTimes(1);
});
it('SIGKILLs a stalled embed request and rejects with a timeout', async () => {
process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS = '50';
vi.useFakeTimers();
const { sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await sidecarEmbedBatch(['warmup']);
children[0].send = vi.fn(() => true);
const pending = sidecarEmbedBatch(['stalled']);
const assertion = expect(pending).rejects.toThrow(/timed out after 50ms \(embed\)/);
await vi.advanceTimersByTimeAsync(50);
await assertion;
expect(children[0].kill).toHaveBeenCalledWith('SIGKILL');
});
it('rejects embedBatch when aborted while waiting on an outstanding embed request', async () => {
const { embedBatch } = await import('../../src/core/embeddings/embedder.js');
await embedBatch(['warmup']);
children[0].send = vi.fn(() => true);
const controller = new AbortController();
const pending = embedBatch(['stalled'], { signal: controller.signal });
await vi.waitFor(() => expect(children[0].send).toHaveBeenCalled());
controller.abort();
await expect(pending).rejects.toThrow();
});
it('cleans up a pending waiter when IPC send throws', async () => {
process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS = '50';
const { sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await sidecarEmbedBatch(['warmup']);
vi.useFakeTimers();
children[0].send = vi.fn(() => {
throw new Error('Channel closed');
});
await expect(sidecarEmbedBatch(['x'])).rejects.toThrow(/Channel closed/);
expect(children[0].kill).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(50);
expect(children[0].kill).not.toHaveBeenCalled();
});
it('lets a joining caller abort without cancelling shared sidecar init', async () => {
let releaseInit: (() => void) | undefined;
forkMock.mockImplementation(() => {
const child = new FakeChild();
children.push(child);
child.send = vi.fn((msg: SidecarRequest) => {
if (msg.type === 'init') {
releaseInit = () => child.emit('message', { id: msg.id, type: 'ready', device: 'cpu' });
return true;
}
queueMicrotask(() => {
const response = child.respond(msg);
if (response) child.emit('message', response);
});
return true;
});
return child as unknown as ChildProcess;
});
const { ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
const first = ensureEmbeddingSidecar();
await vi.waitFor(() => expect(releaseInit).toBeDefined());
const joining = new AbortController();
const second = ensureEmbeddingSidecar({ signal: joining.signal });
joining.abort();
await expect(second).rejects.toThrow();
releaseInit!();
await expect(first).resolves.toEqual({ device: 'cpu' });
expect(forkMock).toHaveBeenCalledTimes(1);
});
it('does not cancel shared sidecar init when the first waiter aborts', async () => {
let releaseInit: (() => void) | undefined;
forkMock.mockImplementation(() => {
const child = new FakeChild();
children.push(child);
child.send = vi.fn((msg: SidecarRequest) => {
if (msg.type === 'init') {
releaseInit = () => child.emit('message', { id: msg.id, type: 'ready', device: 'cpu' });
return true;
}
queueMicrotask(() => {
const response = child.respond(msg);
if (response) child.emit('message', response);
});
return true;
});
return child as unknown as ChildProcess;
});
const { ensureEmbeddingSidecar } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
const firstAbort = new AbortController();
const first = ensureEmbeddingSidecar({ signal: firstAbort.signal });
await vi.waitFor(() => expect(releaseInit).toBeDefined());
const second = ensureEmbeddingSidecar();
firstAbort.abort();
await expect(first).rejects.toThrow();
releaseInit!();
await expect(second).resolves.toEqual({ device: 'cpu' });
expect(forkMock).toHaveBeenCalledTimes(1);
});
it('rejects an aborted embed wait without killing the sidecar', async () => {
const { sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await sidecarEmbedBatch(['warmup']);
children[0].send = vi.fn(() => true);
const controller = new AbortController();
const pending = sidecarEmbedBatch(['stalled'], { signal: controller.signal });
await vi.waitFor(() => expect(children[0].send).toHaveBeenCalled());
controller.abort();
await expect(pending).rejects.toThrow();
expect(children[0].kill).not.toHaveBeenCalled();
});
it('respawns a dead sidecar with the last init embeddingConfig and forceDevice', async () => {
const { ensureEmbeddingSidecar, sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await ensureEmbeddingSidecar({
embeddingConfig: { dimensions: 768 },
forceDevice: 'cpu',
});
expect(forkMock).toHaveBeenCalledTimes(1);
expect(children[0].send.mock.calls[0][0]).toMatchObject({
type: 'init',
embeddingConfig: { dimensions: 768 },
forceDevice: 'cpu',
});
children[0].emit('close', 1, null);
await sidecarEmbedBatch(['again']);
expect(forkMock).toHaveBeenCalledTimes(2);
expect(children[1].send.mock.calls[0][0]).toMatchObject({
type: 'init',
embeddingConfig: { dimensions: 768 },
forceDevice: 'cpu',
});
});
it('does not use worker_threads or import the embeddings barrel', () => {
const clientSrc = readFileSync(path.join(embeddingsDir, 'embedding-sidecar-client.ts'), 'utf8');
const façadeSrc = readFileSync(path.join(embeddingsDir, 'embedder.ts'), 'utf8');
const importLines = clientSrc
.split('\n')
.filter(
(line) =>
/^\s*import\b/.test(line) || /^\s*\} from /.test(line) || /import\s*\(/.test(line),
)
.join('\n');
expect(clientSrc).not.toMatch(/worker_threads/);
expect(clientSrc).not.toMatch(/new Worker\b/);
expect(importLines).not.toContain('embedding-pipeline');
expect(importLines).not.toContain('embedding-identity');
expect(importLines).not.toContain('./index.js');
expect(façadeSrc).not.toMatch(/from\s+['"]@huggingface\/transformers['"]/);
expect(façadeSrc).not.toMatch(/import\s*\(\s*['"]@huggingface\/transformers['"]/);
expect(façadeSrc).not.toMatch(/from\s+['"]onnxruntime-node['"]/);
expect(façadeSrc).not.toMatch(/import\s*\(\s*['"]onnxruntime-node['"]/);
expect(façadeSrc).not.toMatch(/from\s+['"].*onnxruntime-common-resolver['"]/);
expect(façadeSrc).not.toMatch(/import\s*\(\s*['"].*onnxruntime-common-resolver['"]/);
expect(façadeSrc).not.toMatch(/from\s+['"].*embedding-local-init['"]/);
expect(façadeSrc).not.toMatch(/import\s*\(\s*['"].*embedding-local-init['"]/);
});
it('sizes the init deadline from the HF download budget, not a 15s process lifetime', async () => {
const { SIDECAR_INIT_IPC_SLACK_MS, sidecarInitTimeoutMs } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
delete process.env.HF_DOWNLOAD_TIMEOUT_MS;
delete process.env.HF_MAX_ATTEMPTS;
// 3 attempts × 5 min plus 2s + 4s exponential backoff, plus IPC slack.
expect(sidecarInitTimeoutMs()).toBe(
5 * 60 * 1_000 * 3 + 2_000 + 4_000 + SIDECAR_INIT_IPC_SLACK_MS,
);
expect(sidecarInitTimeoutMs()).toBeGreaterThan(15_000);
process.env.HF_DOWNLOAD_TIMEOUT_MS = '120000';
process.env.HF_MAX_ATTEMPTS = '2';
expect(sidecarInitTimeoutMs()).toBe(240_000 + 2_000 + SIDECAR_INIT_IPC_SLACK_MS);
process.env.HF_DOWNLOAD_TIMEOUT_MS = String(60 * 60 * 1_000);
process.env.HF_MAX_ATTEMPTS = '1';
expect(sidecarInitTimeoutMs()).toBe(30 * 60 * 1_000 + SIDECAR_INIT_IPC_SLACK_MS);
process.env.HF_DOWNLOAD_TIMEOUT_MS = '120000';
process.env.HF_MAX_ATTEMPTS = '9.5';
expect(sidecarInitTimeoutMs()).toBe(
120_000 * 9 + 2_000 * (2 ** 8 - 1) + SIDECAR_INIT_IPC_SLACK_MS,
);
});
it('clears the reap wait timeout once the child closes', async () => {
vi.useFakeTimers();
const { ensureEmbeddingSidecar, reapEmbeddingSidecarAndWait } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await ensureEmbeddingSidecar();
await reapEmbeddingSidecarAndWait(5_000);
expect(vi.getTimerCount()).toBe(0);
});
it('does not let a reaped child reset its replacement', async () => {
const { ensureEmbeddingSidecar, reapEmbeddingSidecar, sidecarEmbedBatch } =
await import('../../src/core/embeddings/embedding-sidecar-client.js');
await ensureEmbeddingSidecar();
const first = children[0];
reapEmbeddingSidecar();
await ensureEmbeddingSidecar();
expect(forkMock).toHaveBeenCalledTimes(2);
first.emit('error', new Error('late error from reaped child'));
first.emit('close', 1, null);
await expect(sidecarEmbedBatch(['still-alive'])).resolves.toHaveLength(1);
expect(forkMock).toHaveBeenCalledTimes(2);
});
});

102
gitnexus/test/unit/embeddings-install-command.test.ts
View file

@ -9,6 +9,7 @@
* capture logger records, assert on process.exitCode + recoveryHint/msg.
*/
import { beforeEach, describe, expect, it, vi } from 'vitest';
import type { LoggerCapture } from '../../src/core/logger.js';
const resolveEmbeddingRuntimeMock = vi.fn<() => { source: string } | null>();
const isPrefixRuntimeLoadableMock = vi.fn(() => true);
@ -23,12 +24,19 @@ vi.mock('../../src/core/embeddings/runtime-install.js', async (importOriginal) =
getEmbeddingStackSpecs: () => ({ '@huggingface/transformers': '^4.1.0' }),
}));
async function run(options: { cuda?: boolean; force?: boolean } = {}) {
async function withInstallCapture(
options: { cuda?: boolean; force?: boolean } = {},
assert: (cap: LoggerCapture) => void | Promise<void>,
): Promise<void> {
const { _captureLogger } = await import('../../src/core/logger.js');
const cap = _captureLogger();
const { embeddingsInstallCommand } = await import('../../src/cli/embeddings.js');
await embeddingsInstallCommand(options);
return cap;
try {
const { embeddingsInstallCommand } = await import('../../src/cli/embeddings.js');
await embeddingsInstallCommand(options);
await assert(cap);
} finally {
cap.restore();
}
}
describe('embeddingsInstallCommand outcomes (#2372)', () => {
@ -40,26 +48,52 @@ describe('embeddingsInstallCommand outcomes (#2372)', () => {
process.exitCode = undefined;
});
it('refuses to spawn npm on darwin/x64', async () => {
const orig = { platform: process.platform, arch: process.arch };
Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true });
Object.defineProperty(process, 'arch', { value: 'x64', configurable: true });
resolveEmbeddingRuntimeMock.mockReturnValue(null);
try {
await withInstallCapture({}, (cap) => {
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
expect(process.exitCode).toBe(1);
expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(
true,
);
});
} finally {
Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true });
Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true });
}
});
it('package-sourced --force still installs so prefix overrides refresh', async () => {
resolveEmbeddingRuntimeMock.mockReturnValue({ source: 'package' });
await withInstallCapture({ force: true }, () => {
expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1);
});
});
it('already-installed package source without --force: no install, "nothing to do"', async () => {
resolveEmbeddingRuntimeMock.mockReturnValue({ source: 'package' });
const cap = await run();
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
expect(
cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('nothing to do')),
).toBe(true);
cap.restore();
await withInstallCapture({}, (cap) => {
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
expect(
cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('nothing to do')),
).toBe(true);
});
});
it('post-check resolves nothing: exit 1 and the ✗ message', async () => {
// First call (pre-check) not package, so it installs; post-check returns null.
resolveEmbeddingRuntimeMock.mockReturnValueOnce(null).mockReturnValueOnce(null);
const cap = await run();
expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1);
expect(process.exitCode).toBe(1);
expect(
cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('does not resolve')),
).toBe(true);
cap.restore();
await withInstallCapture({}, (cap) => {
expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1);
expect(process.exitCode).toBe(1);
expect(
cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('does not resolve')),
).toBe(true);
});
});
it('post-check runtime-prefix + loadable: unqualified ✓, exit unset', async () => {
@ -67,10 +101,12 @@ describe('embeddingsInstallCommand outcomes (#2372)', () => {
source: 'runtime-prefix',
});
isPrefixRuntimeLoadableMock.mockReturnValue(true);
const cap = await run();
expect(process.exitCode).toBeUndefined();
expect(cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('✓'))).toBe(true);
cap.restore();
await withInstallCapture({}, (cap) => {
expect(process.exitCode).toBeUndefined();
expect(cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('✓'))).toBe(
true,
);
});
});
it('post-check runtime-prefix + not loadable: capability warning, no false ✓, exit unset', async () => {
@ -78,17 +114,17 @@ describe('embeddingsInstallCommand outcomes (#2372)', () => {
source: 'runtime-prefix',
});
isPrefixRuntimeLoadableMock.mockReturnValue(false);
const cap = await run();
// install itself succeeded, so exit code stays unset...
expect(process.exitCode).toBeUndefined();
const records = cap.records();
// ...but the message names the capability requirement, not an unqualified ✓.
expect(
records.some((r) => typeof r.msg === 'string' && r.msg.includes('module.registerHooks')),
).toBe(true);
expect(records.some((r) => typeof r.msg === 'string' && r.msg.includes('is ready'))).toBe(
false,
);
cap.restore();
await withInstallCapture({}, (cap) => {
// install itself succeeded, so exit code stays unset...
expect(process.exitCode).toBeUndefined();
const records = cap.records();
// ...but the message names the capability requirement, not an unqualified ✓.
expect(
records.some((r) => typeof r.msg === 'string' && r.msg.includes('module.registerHooks')),
).toBe(true);
expect(records.some((r) => typeof r.msg === 'string' && r.msg.includes('is ready'))).toBe(
false,
);
});
});
});

86
gitnexus/test/unit/embeddings-sync-command.test.ts
View file

@ -21,6 +21,10 @@ const {
fetchExistingEmbeddingHashesMock,
runEmbeddingPipelineMock,
resolveEmbeddingIdentityMock,
installEmbeddingRuntimeMock,
resolveEmbeddingRuntimeMock,
isPrefixRuntimeLoadableMock,
reapEmbeddingSidecarMock,
} = vi.hoisted(() => ({
acquireIndexLockMock: vi.fn(),
releaseMock: vi.fn(),
@ -34,6 +38,10 @@ const {
fetchExistingEmbeddingHashesMock: vi.fn(),
runEmbeddingPipelineMock: vi.fn(),
resolveEmbeddingIdentityMock: vi.fn(),
installEmbeddingRuntimeMock: vi.fn(),
resolveEmbeddingRuntimeMock: vi.fn(),
isPrefixRuntimeLoadableMock: vi.fn(),
reapEmbeddingSidecarMock: vi.fn(),
}));
vi.mock('../../src/storage/git.js', () => ({
@ -67,6 +75,17 @@ vi.mock('../../src/core/embeddings/embedding-identity.js', () => ({
resolveEmbeddingIdentity: () => resolveEmbeddingIdentityMock(),
}));
vi.mock('../../src/core/embeddings/runtime-install.js', async (importOriginal) => ({
...(await importOriginal<typeof import('../../src/core/embeddings/runtime-install.js')>()),
installEmbeddingRuntime: (...args: unknown[]) => installEmbeddingRuntimeMock(...args),
resolveEmbeddingRuntime: () => resolveEmbeddingRuntimeMock(),
isPrefixRuntimeLoadable: () => isPrefixRuntimeLoadableMock(),
}));
vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({
reapEmbeddingSidecar: () => reapEmbeddingSidecarMock(),
}));
const IDENTITY = { model: 'test-model', dimensions: 768, provider: 'local' } as const;
const BASE_META = {
@ -96,6 +115,8 @@ async function run(inputPath = '/tmp/emb-sync-repo') {
describe('embeddingsSyncCommand writer safety (#3065)', () => {
const tmpDirs: string[] = [];
const originalEmbeddingUrl = process.env.GITNEXUS_EMBEDDING_URL;
const originalEmbeddingModel = process.env.GITNEXUS_EMBEDDING_MODEL;
async function store(kind: 'file' | 'missing' | 'dir' = 'file') {
const dir = await mkdtemp(path.join(tmpdir(), 'emb-sync-'));
@ -126,9 +147,19 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => {
failedNodeIds: [],
});
resolveEmbeddingIdentityMock.mockReset().mockReturnValue({ ...IDENTITY });
installEmbeddingRuntimeMock.mockReset().mockResolvedValue(undefined);
resolveEmbeddingRuntimeMock.mockReset().mockReturnValue({ source: 'package' });
isPrefixRuntimeLoadableMock.mockReset().mockReturnValue(true);
reapEmbeddingSidecarMock.mockReset();
delete process.env.GITNEXUS_EMBEDDING_URL;
delete process.env.GITNEXUS_EMBEDDING_MODEL;
});
afterEach(async () => {
if (originalEmbeddingUrl === undefined) delete process.env.GITNEXUS_EMBEDDING_URL;
else process.env.GITNEXUS_EMBEDDING_URL = originalEmbeddingUrl;
if (originalEmbeddingModel === undefined) delete process.env.GITNEXUS_EMBEDDING_MODEL;
else process.env.GITNEXUS_EMBEDDING_MODEL = originalEmbeddingModel;
await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
});
@ -354,6 +385,61 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => {
expect(releaseMock).toHaveBeenCalled();
});
it('keeps the pipeline error when sidecar reap also throws', async () => {
await store();
runEmbeddingPipelineMock.mockRejectedValue(new Error('pipeline boom'));
reapEmbeddingSidecarMock.mockImplementation(() => {
throw new Error('reap boom');
});
await expect(run()).rejects.toThrow('pipeline boom');
expect(releaseMock).toHaveBeenCalled();
});
it('does not spawn npm on darwin/x64', async () => {
await store();
resolveEmbeddingRuntimeMock.mockReturnValue(null);
const orig = { platform: process.platform, arch: process.arch };
Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true });
Object.defineProperty(process, 'arch', { value: 'x64', configurable: true });
try {
await expect(run()).rejects.toThrow(/macOS Intel/);
expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled();
} finally {
Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true });
Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true });
}
});
it('auto-heals a missing stack without requesting CUDA binaries', async () => {
await store();
resolveEmbeddingRuntimeMock.mockReturnValue(null);
await run();
expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1);
expect(installEmbeddingRuntimeMock.mock.calls[0]?.[0]).toEqual({});
expect(installEmbeddingRuntimeMock.mock.calls[0]?.[0]).not.toMatchObject({ cuda: true });
});
it('wraps a failed auto-install with the missing-stack recovery path', async () => {
await store();
resolveEmbeddingRuntimeMock.mockReturnValue(null);
installEmbeddingRuntimeMock.mockRejectedValue(new Error('npm install timed out'));
await expect(run()).rejects.toThrow(
/Could not install the embedding runtime[\s\S]*gitnexus embeddings install/,
);
expect(initLbugMock).not.toHaveBeenCalled();
});
it('does not statically import the embedding pipeline', async () => {
const { readFileSync } = await import('node:fs');
const src = readFileSync(new URL('../../src/cli/embeddings-sync.ts', import.meta.url), 'utf8');
expect(src).not.toMatch(/^import .*embedding-pipeline/m);
expect(src).toContain("await import('../core/embeddings/embedding-pipeline.js')");
});
it('loads existing hashes without materializing cached vectors', async () => {
await store();
const hashes = new Map([

4
gitnexus/test/unit/fts-artifact-coverage.test.ts
View file

@ -102,7 +102,9 @@ describe('real repo pairing (guards against a silent core bump)', () => {
) as { coreVersion: string; extensionVersion: string };
expect(pkg.dependencies['@ladybugdb/core']).toBe(manifest.coreVersion);
expect(manifest.extensionVersion).toMatch(/^\d+\.\d+\.\d+$/);
expect(pkg.files).toContain('vendor');
expect(filesCoverFtsArtifacts(pkg.files)).toBe(true);
expect(pkg.files).toContain('vendor/**/prebuilds/**');
expect(pkg.files).not.toContain('vendor');
});
});

10
gitnexus/test/unit/hf-env.test.ts
View file

@ -326,6 +326,16 @@ describe('withHfDownloadRetry env overrides', () => {
expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS);
});
it('HF_MAX_ATTEMPTS=0.5 floors below 1 and falls back to the built-in default', async () => {
process.env.HF_MAX_ATTEMPTS = '0.5';
const fn = vi.fn().mockRejectedValue(new Error('fetch failed'));
const cb = new CircuitBreaker({ failureThreshold: 99_999 });
await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow(
'fetch failed',
);
expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS);
});
it('HF_MAX_ATTEMPTS=0 falls back to the built-in default', async () => {
process.env.HF_MAX_ATTEMPTS = '0';
const fn = vi.fn().mockRejectedValue(new Error('fetch failed'));

9
gitnexus/test/unit/http-embedder.test.ts
View file

@ -61,8 +61,13 @@ describe('HTTP embedding backend', () => {
expect(getEmbeddingDims()).toBe(384);
});
it('returns false before initialization', () => {
expect(isEmbedderReady()).toBe(false);
it('is ready from HTTP mode or a ready local runtime, not resolution alone', async () => {
const { isHttpMode } = await import('../../src/core/embeddings/http-client.js');
const { assessLocalEmbeddingRuntime } =
await import('../../src/core/embeddings/runtime-support.js');
expect(isEmbedderReady()).toBe(
isHttpMode() || assessLocalEmbeddingRuntime().status === 'ready',
);
});
it('returns true when HTTP environment variables are set', async () => {

16
gitnexus/test/unit/hybrid-search.test.ts
View file

@ -199,4 +199,20 @@ describe('hybridSearch — FTS failure fallback (#1489)', () => {
expect(results[0].filePath).toBe('src/fts-hit.ts');
expect(results[0].sources).toEqual(['bm25']);
});
it('keeps BM25 results when semantic search throws', async () => {
const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js');
vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({
results: [{ filePath: 'src/fts-hit.ts', score: 5, rank: 1 }],
ftsAvailable: true,
});
const mockExecuteQuery = vi.fn().mockResolvedValue([]);
const mockSemanticSearch = vi.fn().mockRejectedValue(new Error('sidecar aborted'));
const results = await hybridSearch('test query', 10, mockExecuteQuery, mockSemanticSearch);
expect(results).toHaveLength(1);
expect(results[0].filePath).toBe('src/fts-hit.ts');
expect(results[0].sources).toEqual(['bm25']);
});
});

31
gitnexus/test/unit/local-backend-disconnect-reap.test.ts Normal file
View file

@ -0,0 +1,31 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { closeLbugMock, reapEmbeddingSidecarMock } = vi.hoisted(() => ({
closeLbugMock: vi.fn(),
reapEmbeddingSidecarMock: vi.fn(),
}));
vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => ({
...(await importOriginal<typeof import('../../src/core/lbug/pool-adapter.js')>()),
closeLbug: (...args: unknown[]) => closeLbugMock(...args),
}));
vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({
reapEmbeddingSidecar: () => reapEmbeddingSidecarMock(),
}));
import { LocalBackend } from '../../src/mcp/local/local-backend.js';
describe('LocalBackend.disconnect sidecar reap', () => {
beforeEach(() => {
closeLbugMock.mockReset();
reapEmbeddingSidecarMock.mockReset();
});
it('reaps the sidecar when closeLbug rejects', async () => {
closeLbugMock.mockRejectedValue(new Error('ladybug close failed'));
const backend = new LocalBackend();
await expect(backend.disconnect()).rejects.toThrow('ladybug close failed');
expect(reapEmbeddingSidecarMock).toHaveBeenCalled();
});
});

52
gitnexus/test/unit/local-backend-semantic-warn.test.ts
View file

@ -1,13 +1,16 @@
/**
* Tests that MCP semantic search surfaces a pruned/unloadable optional embedding
* stack once instead of silently degrading to BM25 (#2372) — the silent-
* degradation mode #2370 exists to fix. executeQuery is mocked to report a
* populated embedding table so execution reaches the embedder import, which is
* mocked to throw the missing-stack message.
* Tests that MCP semantic search surfaces a missing local embedding stack once
* instead of silently degrading to BM25 (#2372) — the silent-degradation mode
* #2370 exists to fix. executeQuery is mocked to report a populated embedding
* table so execution reaches the embedder import, which is mocked to throw the
* missing-stack message (R20 copy: default install no longer ships the stack).
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { _captureLogger, type LoggerCapture } from '../../src/core/logger.js';
import { localEmbeddingStackMissingMessage } from '../../src/core/embeddings/runtime-support.js';
import {
LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD,
localEmbeddingStackMissingMessage,
} from '../../src/core/embeddings/runtime-support.js';
const executeQueryMock = vi.fn();
const embedQueryMock = vi.fn();
@ -24,10 +27,15 @@ vi.mock('../../src/mcp/core/embedder.js', () => ({
import { LocalBackend } from '../../src/mcp/local/local-backend.js';
interface SemanticSearchable {
semanticSearch(repo: { lbugPath: string }, query: string, limit: number): Promise<unknown[]>;
semanticSearch(
repo: { lbugPath: string },
query: string,
limit: number,
degraded?: { reason?: string },
): Promise<unknown[]>;
}
const callSemanticSearch = (b: LocalBackend): Promise<unknown[]> =>
(b as unknown as SemanticSearchable).semanticSearch({ lbugPath: '/tmp/x' }, 'q', 5);
const callSemanticSearch = (b: LocalBackend, degraded?: { reason?: string }): Promise<unknown[]> =>
(b as unknown as SemanticSearchable).semanticSearch({ lbugPath: '/tmp/x' }, 'q', 5, degraded);
const stackWarns = (cap: LoggerCapture): number =>
cap
@ -36,7 +44,7 @@ const stackWarns = (cap: LoggerCapture): number =>
(r) =>
typeof r.msg === 'string' &&
r.msg.includes('query:vector') &&
r.msg.includes('optional embedding stack'),
r.msg.includes('local embedding stack is not installed'),
).length;
describe('LocalBackend.semanticSearch — missing-stack warning (#2372)', () => {
@ -49,10 +57,30 @@ describe('LocalBackend.semanticSearch — missing-stack warning (#2372)', () =>
embedQueryMock.mockRejectedValue(new Error(localEmbeddingStackMissingMessage()));
const backend = new LocalBackend();
const cap = _captureLogger();
const first = { reason: undefined as string | undefined };
const second = { reason: undefined as string | undefined };
try {
expect(await callSemanticSearch(backend)).toEqual([]);
expect(await callSemanticSearch(backend)).toEqual([]);
expect(await callSemanticSearch(backend, first)).toEqual([]);
expect(await callSemanticSearch(backend, second)).toEqual([]);
expect(stackWarns(cap)).toBe(1); // once per LocalBackend instance
expect(first.reason).toContain('local embedding stack is not installed');
expect(second.reason).toContain('local embedding stack is not installed');
} finally {
cap.restore();
}
});
it('stashes sidecar-abort text for query() warnings', async () => {
embedQueryMock.mockRejectedValue(new Error(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD));
const backend = new LocalBackend();
const cap = _captureLogger();
const degraded = { reason: undefined as string | undefined };
try {
expect(await callSemanticSearch(backend, degraded)).toEqual([]);
expect(degraded.reason).toBe(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD);
expect(
cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('sidecar aborted')),
).toBe(true);
} finally {
cap.restore();
}

4
gitnexus/test/unit/run-analyze-fts-crash-marker.test.ts
View file

@ -107,7 +107,9 @@ const mockLbugAdapter = async () => {
initLbug: vi.fn(async () => undefined),
loadGraphToLbug: vi.fn(async () => undefined),
getLbugStats: vi.fn(async () => ({ nodes: 1, edges: 0, communities: 0, processes: 0 })),
executeQuery: vi.fn(async () => []),
executeQuery: vi.fn(async (query: string) =>
query === "CALL current_setting('threads') RETURN *" ? [{ threads: '4' }] : [],
),
executeWithReusedStatement: vi.fn(async () => []),
closeLbug: vi.fn(async () => undefined),
wipeLbugDbFiles: vi.fn(async () => undefined),

58
gitnexus/test/unit/semantic-search-ready.test.ts Normal file
View file

@ -0,0 +1,58 @@
import { beforeEach, describe, expect, it, vi } from 'vitest';
const { embedTextMock, isEmbedderReadyMock, loadVectorExtensionMock } = vi.hoisted(() => ({
embedTextMock: vi.fn(),
isEmbedderReadyMock: vi.fn(),
loadVectorExtensionMock: vi.fn(async () => false),
}));
vi.mock('../../src/core/embeddings/embedder.js', () => ({
initEmbedder: vi.fn(),
embedBatch: vi.fn(),
embedText: (...args: unknown[]) => embedTextMock(...args),
embeddingToArray: (embedding: Float32Array) => Array.from(embedding),
isEmbedderReady: () => isEmbedderReadyMock(),
}));
vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({
loadVectorExtension: (...args: unknown[]) => loadVectorExtensionMock(...args),
createVectorIndex: vi.fn(),
}));
describe('semanticSearch ready-check (KTD11)', () => {
beforeEach(() => {
embedTextMock.mockReset();
isEmbedderReadyMock.mockReset();
loadVectorExtensionMock.mockReset().mockResolvedValue(false);
});
it('does not throw when the stack is resolvable and no in-process singleton exists', async () => {
isEmbedderReadyMock.mockReturnValue(true);
embedTextMock.mockResolvedValue(new Float32Array(384));
const executeQuery = vi.fn(async (cypher: string) => {
if (cypher.includes('RETURN 1 AS ok')) return [{ ok: 1 }];
if (cypher.includes('count(e) AS cnt')) return [{ cnt: 2 }];
return [];
});
const { semanticSearch } = await import('../../src/core/embeddings/embedding-pipeline.js');
await expect(semanticSearch(executeQuery, 'find auth', 5)).resolves.toEqual([]);
expect(embedTextMock).toHaveBeenCalledTimes(1);
expect(embedTextMock.mock.calls[0]?.[0]).toBe('find auth');
});
it('does not request a query vector when the embedding table is empty', async () => {
isEmbedderReadyMock.mockReturnValue(true);
embedTextMock.mockResolvedValue(new Float32Array(384));
const executeQuery = vi.fn(async (cypher: string) => {
if (cypher.includes('RETURN 1 AS ok')) return [];
if (cypher.includes('count(e) AS cnt')) return [{ cnt: 0 }];
return [];
});
const { semanticSearch } = await import('../../src/core/embeddings/embedding-pipeline.js');
await expect(semanticSearch(executeQuery, 'find auth', 5)).resolves.toEqual([]);
expect(embedTextMock).not.toHaveBeenCalled();
expect(loadVectorExtensionMock).not.toHaveBeenCalled();
});
});