diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0334531c1..19519cd7c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -74,8 +74,10 @@ jobs: - name: gitnexus-web dockerfile: Dockerfile.web slug: gitnexus-web - # CLI / `gitnexus serve` backend. Heavy native deps (tree-sitter, - # onnxruntime-node) live only in this image. + # CLI / `gitnexus serve` backend. Tree-sitter natives live in this + # image. onnxruntime-node is opt-in (`gitnexus embeddings install` or + # a bind-mounted prefix / GITNEXUS_EMBEDDING_URL); npm is stripped + # at runtime so the image cannot auto-heal the embedding stack. - name: gitnexus dockerfile: Dockerfile.cli slug: gitnexus diff --git a/Dockerfile.cli b/Dockerfile.cli index eb3cfb5ea..50b1a14ba 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -6,8 +6,11 @@ ARG TARGETPLATFORM ARG NPM_VERSION=11.14.1 # -- Builder ----------------------------------------------------------- -# Native modules (tree-sitter-*, onnxruntime-node, node-gyp builds for +# Native modules (tree-sitter-*, node-gyp builds for # tree-sitter-proto / tree-sitter-swift) require python3 + a C/C++ toolchain. +# onnxruntime-node is not installed by `npm ci`; local embeddings need +# `gitnexus embeddings install` (or HTTP env / a bind-mounted prefix). The +# runtime stage strips npm, so this image cannot auto-heal the stack. # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder ARG NPM_VERSION diff --git a/README.md b/README.md index 0d8ae457c..e798c7284 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ That's it. `analyze` indexes the codebase, installs agent skills, registers Clau > **No C++ toolchain?** Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` — those four languages won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild. -> **Behind an HTTP proxy / regional firewall?** `onnxruntime-node`'s postinstall downloads optional CUDA binaries from `api.nuget.org` and ignores `HTTP_PROXY`/`HTTPS_PROXY` ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). The embedding stack is an optional dependency, so a failed download no longer breaks the install — and it self-heals: the first `gitnexus analyze --embeddings` (or `gitnexus embeddings install`) fetches the stack through your npm registry config (mirrors/proxies apply, no NuGet) into `~/.gitnexus/embedding-runtime` (override with `GITNEXUS_EMBEDDING_RUNTIME_DIR`). The on-demand prefix needs Node with `module.registerHooks` (≥ 22.15 on 22.x, ≥ 23.5 on 23.x); on older Node, keep the stack in the install itself with `ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus` (works on every supported Node). +> **Local embeddings are opt-in.** Default `npm install` does not fetch `@huggingface/transformers` or `onnxruntime-node`. Run `gitnexus embeddings install` (or `gitnexus analyze --embeddings`, which auto-heals) to fetch the stack through your npm registry config into `~/.gitnexus/embedding-runtime`. CUDA GPU binaries still use NuGet via `--cuda` ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). The prefix needs Node with `module.registerHooks` (≥ 22.15 on 22.x, ≥ 23.5 on 23.x). A leftover 1.6.12 package-first tree is residual until a clean reinstall; `--force` only refreshes prefix overrides. > **About `tree-sitter-kotlin`:** like Dart/Proto/Swift, Kotlin is a **vendored** grammar (under `gitnexus/vendor/tree-sitter-kotlin`). Upstream ships **source only** (no prebuilt binaries), so GitNexus cross-builds the platform prebuilds itself (via the `build-tree-sitter-prebuilds` GitHub Actions workflow) and vendors them — the same uniform pipeline used for Dart, Proto, and Swift. `node-gyp-build` selects the right `.node` at require time, so **no C/C++ toolchain is needed**. If no prebuild matches your platform-arch, only Kotlin (`.kt`/`.kts`) parsing is unavailable; the rest of `gitnexus` is unaffected. @@ -587,6 +587,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | | `GITNEXUS_EMBEDDING_RETRY_TIMEOUTS` | unset | When truthy (`1`/`true`/`yes`), per-attempt HTTP embedding timeouts (`TimeoutError` on fetch or body read) go through the bounded `GITNEXUS_EMBEDDING_MAX_ATTEMPTS` retry loop instead of failing the job. Any other value leaves it off, so cloud/default timeouts remain terminal. | Local accelerators that drop a device lock when the client disconnects and succeed on the next request (observed with FastFlowLM on Ryzen AI). | +| `GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS` | `180000` (3 minutes) | Per-request IPC timeout for local embedding sidecar embed batches. On overrun the parent SIGKILLs the sidecar child and rejects the batch. Init still uses the HF download budget (`HF_DOWNLOAD_TIMEOUT_MS` × attempts), not this knob. | Large embed batches or slow local ONNX inference cause sidecar request timeouts during `analyze --embeddings`, `embeddings sync`, serve, or MCP. | | `GITNEXUS_ANALYZER_IDENTITY_IN_PROCESS_GUARDS` | unset | When truthy (`1`/`true`/`yes`), forces in-process cache-guard validation once a batch has ≥128 requests. In-process mode also auto-selects when `packageRoot`/`buildRoot` fail `W_OK` with `EACCES`/`EROFS`. Otherwise those large batches use a Node subprocess probe. Batches under 128 always stay in-process. | Trusted or read-only installs where two identity subprocess spawns per analyze dominate wall time; leave unset to keep the default isolation path on writable trees. | | `GITNEXUS_RESOLVE_DEF_GRAPH_ID_MEMO` | on (unset) | Memoizes `resolveDefGraphId` per `nodeLookup` instance (WeakMap). Enabled by default. Set to `0`/`false`/`off`/`no` to disable and recompute on every call (debug / bisect memo bugs). | Suspecting stale graph-id resolution after a lookup rebuild, or comparing memo vs uncached cost on a large index. | | `GITNEXUS_AUTH_TOKEN` | unset | Bearer token required when `eval-server` binds beyond loopback. May also be read from `.env.local` or `.env`; shell values take precedence. | Exposing the evaluation HTTP tools to a container, VM, or LAN. | @@ -958,7 +959,7 @@ docker compose --env-file .env up -d Files: - [Dockerfile.web](Dockerfile.web) — builds `gitnexus-shared` and `gitnexus-web`, then serves the production frontend. -- [Dockerfile.cli](Dockerfile.cli) — builds the CLI/server (with its native deps) and runs `gitnexus serve --host 0.0.0.0`. +- [Dockerfile.cli](Dockerfile.cli) — builds the CLI/server (with its native deps) and runs `gitnexus serve --host 0.0.0.0`. Local embeddings are **not** in the image (`onnxruntime-node` is opt-in; runtime npm is stripped). Bind-mount a prefix or set `GITNEXUS_EMBEDDING_URL`. - [docker-compose.yaml](docker-compose.yaml) — starts both signed images side by side. - [.env.example](.env.example) — overrides for image names, container names, ports, and the workspace mount. diff --git a/gitnexus/.npmignore b/gitnexus/.npmignore index bf2c8b7c9..f3ea41532 100644 --- a/gitnexus/.npmignore +++ b/gitnexus/.npmignore @@ -13,25 +13,22 @@ node_modules/ vendor/**/node_modules vendor/**/build -# ── Lean publish (FUTURE optimization — NOT done here) ───────────────────────── -# Once the build-tree-sitter-prebuilds workflow has committed 6/6 prebuilds for -# EVERY vendored grammar (c, dart, proto, kotlin, swift), the ~50 MB of generated -# source (parser.c etc.) can be dropped from the tarball — node-gyp-build never -# needs the source when a prebuild matches. +# ── Lean publish (done via package.json `files`, not this file) ─────────────── +# Every vendored grammar now has 6/6 prebuilds, so the published tarball drops +# generated source (parser.c etc.). node-gyp-build loads prebuilds without it. # -# IMPORTANT: this CANNOT be done from this file. package.json's `files: ["vendor"]` +# IMPORTANT: this CANNOT be done from this file. A broad `files: ["vendor"]` # allow-list OVERRIDES .npmignore for the vendor/ subtree (verified: an active -# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). To slim -# the tarball, narrow the `files` field instead — replace the blanket "vendor" -# with the non-source subpaths only (vendor/**/prebuilds/**, -# vendor/**/bindings/node/index.*, vendor/**/src/node-types.json, -# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md). +# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). The +# lever is the `files` field — non-source subpaths only (vendor/**/prebuilds/**, +# vendor/**/bindings/node/index.js, vendor/**/src/node-types.json, +# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md, plus Leiden +# and the FTS manifest). # -# Whatever the mechanism, the prepack guard -# (scripts/assert-publish-grammar-coverage.cjs, also `npm run -# assert-publish-coverage`) inspects the EFFECTIVE `npm pack` file list and FAILS -# the publish whenever a grammar with <6 prebuilds loses a source-build input — so -# the slim can never silently ship a dead grammar. Do not bypass it. +# The prepack guard (scripts/assert-publish-grammar-coverage.cjs, also +# `npm run assert-publish-coverage`) reads `files` and FAILS the publish +# whenever a grammar with <6 prebuilds loses a source-build input. Do not +# bypass it. Do not add a bare `vendor` entry back. # Package lock (consumers use their own) package-lock.json diff --git a/gitnexus/README.md b/gitnexus/README.md index b09e846c4..816ca0ef0 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -531,7 +531,7 @@ for a quiet registry. ### `Cannot destructure property 'package' of 'node.target' as it is null` -This error comes from **npm 11.x's arborist** while installing gitnexus (often via `npx`), before gitnexus code runs. It is triggered by platform-filtered `optionalDependencies` in native packages such as `onnxruntime-node` / `@huggingface/transformers` (used when indexing with `--embeddings`). GitNexus cannot catch it at runtime — use one of these workarounds: +This error comes from **npm 11.x's arborist** while installing a package with platform-filtered `optionalDependencies` (often via `npx`), before gitnexus code runs. Default `npm install` / `npx gitnexus` no longer fetch `@huggingface/transformers` or `onnxruntime-node`; those packages appear only if you run `gitnexus embeddings install` (or you still have a leftover 1.6.12 package-first tree). Other native optionals can still trigger the same arborist crash. GitNexus cannot catch it at runtime — use one of these workarounds: ```bash pnpm --allow-build=@ladybugdb/core --allow-build=gitnexus --allow-build=tree-sitter dlx gitnexus@latest analyze # auto-selected when pnpm + npm 11+ @@ -645,7 +645,7 @@ npm install -g gitnexus `onnxruntime-node`'s postinstall downloads optional CUDA GPU binaries from `api.nuget.org` — outside the npm registry, so registry mirrors don't cover it, and its proxy layer (`global-agent`) ignores the standard `HTTP_PROXY`/`HTTPS_PROXY` variables and rejects 302 redirects ([#2370](https://github.com/abhigyanpatwari/GitNexus/issues/2370)). -Since the packages are optional dependencies, a failed download no longer breaks `npm install -g gitnexus` — npm skips the embedding stack and everything else works. The stack then **self-heals on demand**: the first `gitnexus analyze --embeddings` (or an explicit `gitnexus embeddings install`) fetches it through your configured npm registry — mirrors and proxies apply, no NuGet download involved — into `~/.gitnexus/embedding-runtime`. +Default `npm install -g gitnexus` no longer fetches the embedding stack. **Opt in on demand**: the first `gitnexus analyze --embeddings` (or an explicit `gitnexus embeddings install`) fetches it through your configured npm registry — mirrors and proxies apply, no NuGet download involved — into `~/.gitnexus/embedding-runtime`. A leftover 1.6.12 package-first tree in gitnexus `node_modules` is residual until a clean reinstall; `--force` only refreshes prefix overrides. ```bash # heal a proxy-degraded install manually (CPU embeddings; registry-only) @@ -660,7 +660,7 @@ GLOBAL_AGENT_HTTPS_PROXY= gitnexus embeddings install --cuda The prefix defaults to `~/.gitnexus/embedding-runtime`; set `GITNEXUS_EMBEDDING_RUNTIME_DIR` to install it elsewhere (e.g. a writable path in a container). -> **Node requirement for the on-demand prefix:** the self-heal loads the prefixed packages via `module.registerHooks`, available on Node **≥ 22.15** (on the 22.x line) or **≥ 23.5** (on the 23.x line). On an older Node the packages install but can't be loaded from the prefix — reinstall them into the install itself instead (works on every supported Node): `ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus` (Windows: `set ONNXRUNTIME_NODE_INSTALL=skip && npm install -g gitnexus`). Skipping only the CUDA download keeps full CPU embeddings (CPU embeddings don't need it). Check the result any time with `gitnexus doctor` (Embeddings → Support line). +> **Node requirement for the on-demand prefix:** the prefix loads via `module.registerHooks`, available on Node **≥ 22.15** (on the 22.x line) or **≥ 23.5** (on the 23.x line). On an older Node the packages install but can't be loaded from the prefix — upgrade Node, then run `gitnexus embeddings install`. Check the result any time with `gitnexus doctor` (Embeddings → Support line). ### Analyze warns about unavailable FTS or VECTOR extensions diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index f6766fd4f..6dbb880c6 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -72,10 +72,6 @@ }, "engines": { "node": "^22.18.0 || >=24.11.0" - }, - "optionalDependencies": { - "@huggingface/transformers": "^4.1.0", - "onnxruntime-node": "^1.24.0" } }, "../gitnexus-shared": { @@ -221,16 +217,6 @@ "node": ">=18" } }, - "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.1", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", @@ -685,538 +671,6 @@ "hono": "^4" } }, - "node_modules/@huggingface/jinja": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.6.tgz", - "integrity": "sha512-MyMWyLnjqo+KRJYSH7oWNbsOn5onuIvfXYPcc0WOGxU0eHUV7oAYUoQTl2BMdu7ml+ea/bu11UM+EshbeHwtIA==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/@huggingface/tokenizers": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", - "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", - "license": "Apache-2.0", - "optional": true - }, - "node_modules/@huggingface/transformers": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", - "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@huggingface/jinja": "^0.5.6", - "@huggingface/tokenizers": "^0.1.3", - "onnxruntime-node": "1.24.3", - "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", - "sharp": "^0.34.5" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.3.tgz", - "integrity": "sha512-RMnFX7YQsMoh7lWfcM4NEHHymBX/rLuKNPVM84XE9ONPcaSCDgE7CHIHpSgPcO2xcRthgBy1HfNO319mwhIAkg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.2" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.3.tgz", - "integrity": "sha512-Xo+5uFBtLN0BKqieTxiFzFPQAUlBbbH5iBKyRX/z1JrbnYsHTfKJnUfL8+p2TPXr1pXqao4eeL4Rl144uDpK9w==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.2" - } - }, - "node_modules/@img/sharp-freebsd-wasm32": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.3.tgz", - "integrity": "sha512-lUxcqWIj2wMQ9BrwNjngcr1gWUr5xgaGThBRqPPalIC2n67Cqj1uPh8NnA/ZhAg8hUbKl+kVHKwgUIwe6ZYPrg==", - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "dependencies": { - "@img/sharp-wasm32": "0.35.3" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.2.tgz", - "integrity": "sha512-9J6ypZFpQBj4YnePGoq/S38w6nz+vqg5WZLrLGY4YuSemdMq47GMLBPO42MzwdGwpg/agZ7xzZcFHa48xlywfg==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.2.tgz", - "integrity": "sha512-m2pW1n6cns9VaubNwsZ+c3CRYjxNQWgJ5gPlnL1nbBcpkBvFm6SCFN5o0psFHI8w9n11NKhFkeEDns98tiqbEw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.2.tgz", - "integrity": "sha512-1eMLzy92I4J6rmi4mAT8yC3HxOtniyGELlzGbNMLLeqe052ahFQ0h6LFq+lh5DsDIdYViIDst08abvSbcEdLXQ==", - "cpu": [ - "arm" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.2.tgz", - "integrity": "sha512-dqVSFynCox4C/J8kT16V7SIFAns0IjgLwkvYT7p8LQVmJ5OS5b6tI9IGflxTeuBS//zXeFIUbwt5dwxyZ17cnA==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.2.tgz", - "integrity": "sha512-3z0NHDxD6n5I9gc05U1eW1AyRm+Gznzq3naMrthPNqE6oYykcogW0l/jfpJdjYnuNl8R7yI9pNbE1XiUeyq0Aw==", - "cpu": [ - "ppc64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.2.tgz", - "integrity": "sha512-bsb4rI+NldGOsXuej2r8OdSS8+zXDVaCWxyWrcv6kneTOlgAHtZABRzBBCwdsPiD90J4myNJuHpg6kA20ImW/w==", - "cpu": [ - "riscv64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.2.tgz", - "integrity": "sha512-/ABshyj8gCpyIrNXnHn4LorDJ0HHm1VhXPBlxZ8zAtfVPAaSafXPGn+sUSIRiwaSBy0mmFjSjiXI5mkcwdChKQ==", - "cpu": [ - "s390x" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.2.tgz", - "integrity": "sha512-ITPEtgffGJ0S6G9dRyw/366tJQqFRcHWPHhC+Stpg3Z8AEMrDrTr2lhdz4f/Y/HMbRh//7Z5mBzEpVdi62Oc3w==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.2.tgz", - "integrity": "sha512-zE9EdiUzUmg5mDT5a1rk5fYJ6GWPloTwWBYDS14naqHsL+EaMpDj1AWnpLgh3u0YCORv2Tt50wrcrpYqkP97Kw==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.2.tgz", - "integrity": "sha512-m0lrLiUt+lBYnCFr8qV/65yMR4E/c7/wf78I5eKTdkEakFAlZ9QlzEM3QIhhAwVeUhLAHLcCq7a7Vszq/oFNZQ==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.3.tgz", - "integrity": "sha512-affVWCTLooy8TSxbDx2qkzuDeaWLNVBA+P//FNBirHsXpP2fuBhk5AuboYUnrDnzoXes8GFjpTx0SBFOCRg+FA==", - "cpu": [ - "arm" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.2" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.3.tgz", - "integrity": "sha512-QgKDspHPnrU+GQ55XPhGwyhC8acLVOOSyAvo1oVfFmrIXLkDNmGWzAfDZ4xK8oSA1qBQrALcHX0G5UZni/SuFQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.2" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.3.tgz", - "integrity": "sha512-sMd8rDxmpLOwv/7N44klFjOD5DUO7FLdjiXDI0hoxYaf7Ar262dQIEkosE98bps+5HPLtp/EvNqeqQtOycP/IA==", - "cpu": [ - "ppc64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.2" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.3.tgz", - "integrity": "sha512-0Eob78yjlYPfL5vMNWAW55l3R9Y6BQS/gOfe0ZcP9mEz9ohhKSt4im1hayiknXgf8AWrFqMvJcKIdmLmEe7yeQ==", - "cpu": [ - "riscv64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.2" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.3.tgz", - "integrity": "sha512-KgAxQ0DxpNOq1rG2t5cgTgShJFGSuU7XO45cqC+1NVOuZnP6tlgZRuSYOfNupGkHID0o3cJOsw4DVeJpMovcGw==", - "cpu": [ - "s390x" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.2" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.3.tgz", - "integrity": "sha512-8pqvxubL2PGdhlPy6GLqzDYMUjyRmKAwKHYKixpdJYBUK7PJ0C029XdsnpFIdgRZG68fZiGdHVWcKPvtiPB4cA==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.2" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.3.tgz", - "integrity": "sha512-Vz0iQjzzcSX3HCbfwFfCSG/9SCIqyO0mH2sXyiHaAYfBk0cRsCWXRyQYX0ovCK/PAQBbTzQ0dsPQHh5MAFL59w==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.2" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.3.tgz", - "integrity": "sha512-6O1NPKcDVj9QEdg7Hx549EX8U0rp6yXQERqru6yRN7fGBn32UvIRJUlWnk+8xDCiG76hXVBbX82NZ/ZKr0euIg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.2" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.3.tgz", - "integrity": "sha512-cZ0XkcYGpHZkqW6iCkqTcmUC0CD9DhD5d/qeZlZkfRBn6GnHniZXLUo5+9xw8Iv76YE6LQFN9YNBlKREcCG76w==", - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.11.1" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-webcontainers-wasm32": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.3.tgz", - "integrity": "sha512-2rnq7bX3NzeR2T4YWgz8qiG4h3TSdMe+vN1iQXpJleSJ3SM5zQ8Fy2SyyXAWlbxpEZ2Y+Z4u1BePgJEYbSy80Q==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.3" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.3.tgz", - "integrity": "sha512-4bPwFdMbeC4JQ8L8LOyWp6nsHcboP5fxkp6iPOXz2Vg49R42TuMs2whkJ5OAP4/Ul035qOzy0AecOF9VOscn4w==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.3.tgz", - "integrity": "sha512-r53mXsBN6lFUDiST764SvgwUdHAqM4rPAiDzAmf4fLoB6X/rkfyTrLCg6+g17wJJiCmB3JYgHuUldCWUIRFSXw==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.3.tgz", - "integrity": "sha512-D4y1vNeZrIIJCN+uHaWVtH86B+aCrdMYYjicy9pXHvbGZeGYLLSd3wdVuC37FxVXlU1ARsk84eKWfWMXGYEqvA==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -1426,72 +880,6 @@ "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", "license": "MIT" }, - "node_modules/@protobufjs/aspromise": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", - "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/base64": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", - "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/codegen": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", - "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/eventemitter": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", - "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/fetch": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", - "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.1" - } - }, - "node_modules/@protobufjs/float": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", - "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/path": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", - "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/pool": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", - "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", - "license": "BSD-3-Clause", - "optional": true - }, - "node_modules/@protobufjs/utf8": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.1.tgz", - "integrity": "sha512-oOAWABowe8EAbMyWKM0tYDKi8Yaox52D+HWZhAIJqQXbqe0xI/GV7FhLWqlEKreMkfDjshR5FKgi3mnle0h6Eg==", - "license": "BSD-3-Clause", - "optional": true - }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.4.tgz", @@ -1891,7 +1279,7 @@ "version": "26.5.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-26.5.0.tgz", "integrity": "sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~8.9.0" @@ -2099,16 +1487,6 @@ "node": ">= 0.6" } }, - "node_modules/adm-zip": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.6.0.tgz", - "integrity": "sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=14.0" - } - }, "node_modules/ajv": { "version": "8.18.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", @@ -2685,42 +2063,6 @@ "node": ">=4.0.0" } }, - "node_modules/define-data-property": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", - "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "license": "MIT", - "optional": true, - "dependencies": { - "es-define-property": "^1.0.0", - "es-errors": "^1.3.0", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/define-properties": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", - "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "license": "MIT", - "optional": true, - "dependencies": { - "define-data-property": "^1.0.1", - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -2734,7 +2076,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -2878,19 +2220,6 @@ "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", "license": "MIT" }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", - "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -3270,39 +2599,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/global-agent": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-4.1.3.tgz", - "integrity": "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==", - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "globalthis": "^1.0.2", - "matcher": "^4.0.0", - "semver": "^7.3.5", - "serialize-error": "^8.1.0" - }, - "engines": { - "node": ">=10.0" - } - }, - "node_modules/globalthis": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", - "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "define-properties": "^1.2.1", - "gopd": "^1.0.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/gopd": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", @@ -3380,13 +2676,6 @@ "node": "^22.0.0 || ^24.0.0 || ^25.0.0 || >=26.0.0" } }, - "node_modules/guid-typescript": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", - "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", - "license": "ISC", - "optional": true - }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -3396,19 +2685,6 @@ "node": ">=8" } }, - "node_modules/has-property-descriptors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", - "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "license": "MIT", - "optional": true, - "dependencies": { - "es-define-property": "^1.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -3970,13 +3246,6 @@ "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", "license": "MIT" }, - "node_modules/long": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", - "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", - "license": "Apache-2.0", - "optional": true - }, "node_modules/lru-cache": { "version": "11.5.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", @@ -4074,22 +3343,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/matcher": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/matcher/-/matcher-4.0.0.tgz", - "integrity": "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "escape-string-regexp": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -4280,16 +3533,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/obliterator": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", @@ -4343,46 +3586,6 @@ "integrity": "sha512-/F63/e2VJoaVXGGNu6S5QH7jivBThGO95OzAVXXQ8hTta/b1QxI8udHa6cI3+3mAb5WWIIaMMwfZw01oivjJ1g==", "license": "MIT" }, - "node_modules/onnxruntime-node": { - "version": "1.29.0", - "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.29.0.tgz", - "integrity": "sha512-WjiVVB72riILz8HbYvxvmjKyE/WmkYoSfKY++axo5jAR609HQg8MwiG/HhShpTcJfmmAdzxxmB+MMST3A+SiPA==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "win32", - "darwin", - "linux" - ], - "dependencies": { - "adm-zip": "^0.6.0", - "global-agent": "^4.1.3", - "onnxruntime-common": "1.29.0" - } - }, - "node_modules/onnxruntime-web": { - "version": "1.26.0-dev.20260416-b7804b056c", - "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", - "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", - "license": "MIT", - "optional": true, - "dependencies": { - "flatbuffers": "^25.1.24", - "guid-typescript": "^1.0.9", - "long": "^5.2.3", - "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", - "platform": "^1.3.6", - "protobufjs": "^7.2.4" - } - }, - "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { - "version": "1.24.0-dev.20251116-b39e144322", - "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", - "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", - "license": "MIT", - "optional": true - }, "node_modules/pandemonium": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/pandemonium/-/pandemonium-2.4.1.tgz", @@ -4559,13 +3762,6 @@ "node": ">=16.20.0" } }, - "node_modules/platform": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", - "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", - "license": "MIT", - "optional": true - }, "node_modules/postcss": { "version": "8.5.26", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", @@ -4611,30 +3807,6 @@ ], "license": "MIT" }, - "node_modules/protobufjs": { - "version": "7.6.6", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.6.tgz", - "integrity": "sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==", - "hasInstallScript": true, - "license": "BSD-3-Clause", - "optional": true, - "dependencies": { - "@protobufjs/aspromise": "^1.1.2", - "@protobufjs/base64": "^1.1.2", - "@protobufjs/codegen": "^2.0.5", - "@protobufjs/eventemitter": "^1.1.1", - "@protobufjs/fetch": "^1.1.1", - "@protobufjs/float": "^1.0.2", - "@protobufjs/path": "^1.1.2", - "@protobufjs/pool": "^1.1.0", - "@protobufjs/utf8": "^1.1.1", - "@types/node": ">=13.7.0", - "long": "^5.3.2" - }, - "engines": { - "node": ">=12.0.0" - } - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -4887,22 +4059,6 @@ "url": "https://opencollective.com/express" } }, - "node_modules/serialize-error": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", - "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/serve-static": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", @@ -4928,56 +4084,6 @@ "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", "license": "ISC" }, - "node_modules/sharp": { - "version": "0.35.3", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.3.tgz", - "integrity": "sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@img/colour": "^1.1.0", - "detect-libc": "^2.1.2", - "semver": "^7.8.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.3", - "@img/sharp-darwin-x64": "0.35.3", - "@img/sharp-freebsd-wasm32": "0.35.3", - "@img/sharp-libvips-darwin-arm64": "1.3.2", - "@img/sharp-libvips-darwin-x64": "1.3.2", - "@img/sharp-libvips-linux-arm": "1.3.2", - "@img/sharp-libvips-linux-arm64": "1.3.2", - "@img/sharp-libvips-linux-ppc64": "1.3.2", - "@img/sharp-libvips-linux-riscv64": "1.3.2", - "@img/sharp-libvips-linux-s390x": "1.3.2", - "@img/sharp-libvips-linux-x64": "1.3.2", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.2", - "@img/sharp-libvips-linuxmusl-x64": "1.3.2", - "@img/sharp-linux-arm": "0.35.3", - "@img/sharp-linux-arm64": "0.35.3", - "@img/sharp-linux-ppc64": "0.35.3", - "@img/sharp-linux-riscv64": "0.35.3", - "@img/sharp-linux-s390x": "0.35.3", - "@img/sharp-linux-x64": "0.35.3", - "@img/sharp-linuxmusl-arm64": "0.35.3", - "@img/sharp-linuxmusl-x64": "0.35.3", - "@img/sharp-webcontainers-wasm32": "0.35.3", - "@img/sharp-win32-arm64": "0.35.3", - "@img/sharp-win32-ia32": "0.35.3", - "@img/sharp-win32-x64": "0.35.3" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -5520,19 +4626,6 @@ "fsevents": "~2.3.3" } }, - "node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "license": "(MIT OR CC0-1.0)", - "optional": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/type-is": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-2.1.0.tgz", @@ -5591,7 +4684,7 @@ "version": "8.9.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.9.0.tgz", "integrity": "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg==", - "devOptional": true, + "dev": true, "license": "MIT" }, "node_modules/universalify": { diff --git a/gitnexus/package.json b/gitnexus/package.json index f79854e9e..73b76a481 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -35,8 +35,16 @@ "hooks", "scripts", "skills", - "vendor", - "web" + "web", + "vendor/**/prebuilds/**", + "vendor/**/bindings/node/index.js", + "vendor/**/src/node-types.json", + "vendor/**/package.json", + "vendor/**/LICENSE", + "vendor/**/README.md", + "vendor/leiden/index.cjs", + "vendor/leiden/utils.cjs", + "vendor/lbug-fts/manifest.json" ], "scripts": { "build": "node scripts/build.js", @@ -97,7 +105,7 @@ "tree-sitter-typescript": "^0.23.2", "uuid": "^14.0.0" }, - "optionalDependencies": { + "gitnexusEmbeddingStack": { "@huggingface/transformers": "^4.1.0", "onnxruntime-node": "^1.24.0" }, @@ -125,10 +133,7 @@ }, "overrides": { "adm-zip": ">=0.6.0", - "sharp": ">=0.35.0", - "@huggingface/transformers": { - "onnxruntime-node": "$onnxruntime-node" - } + "sharp": ">=0.35.0" }, "engines": { "node": "^22.18.0 || >=24.11.0" diff --git a/gitnexus/scripts/assert-publish-grammar-coverage.cjs b/gitnexus/scripts/assert-publish-grammar-coverage.cjs index b523f68e8..0a2b5b50e 100644 --- a/gitnexus/scripts/assert-publish-grammar-coverage.cjs +++ b/gitnexus/scripts/assert-publish-grammar-coverage.cjs @@ -2,20 +2,19 @@ /** * Publish guard: every vendored tree-sitter grammar must ship a loadable binding. * - * The npm tarball includes gitnexus/vendor/ (package.json `files`). A grammar is - * "covered" on a platform-arch tuple if EITHER a prebuild ships for it OR the - * grammar's full source-build set ships (so the install can source-build it, - * toolchain permitting). A future lean publish — dropping the ~50 MB of generated - * source to ship prebuilds only — is safe ONLY once every grammar has all six - * prebuilds; doing it while any grammar still lacks a prebuild would ship a - * grammar with NO loadable binding (neither prebuild nor buildable source) → that - * language is silently dead for users. + * The npm tarball ships a lean vendor/ allow-list (package.json `files`): + * prebuilds, the bindings entry, node-types.json, and package metadata — not + * generated parser.c. A grammar is "covered" on a platform-arch tuple if EITHER + * a prebuild ships for it OR the grammar's full source-build set ships (so the + * install can source-build it, toolchain permitting). Lean publish is safe ONLY + * when every grammar has all six prebuilds; dropping source while any grammar + * still lacks a prebuild would ship a grammar with NO loadable binding. * * HOW SOURCE INCLUSION IS DECIDED. The `files` allow-list OVERRIDES `.npmignore` * for the vendored subtree (verified: an active "vendor/(star-star)/src/parser.c" * in .npmignore does NOT drop it from `npm pack`). So `.npmignore` can never * exclude vendored source — the ONLY lever is the `files` field. A broad `vendor` - * ships the whole subtree (source + prebuilds); a lean publish narrows `files` to + * ships the whole subtree (source + prebuilds); lean publish narrows `files` to * non-source subpaths. This guard therefore reads `files` directly rather than * shelling out to `npm pack` (which, in prepack, would re-enter this guard and, * on npm versions that don't honor --ignore-scripts for prepare/prepack, run the @@ -55,13 +54,75 @@ const SOURCE_BUILD_REL = [ * then rely on prebuilds. */ function filesShipsVendorSource(filesField) { - return (filesField || []).some((f) => { - const n = String(f) - .replace(/\\/g, '/') - .replace(/\/+$/, '') - .replace(/\/\*\*?$/, ''); - return n === 'vendor'; - }); + return filesEntries(filesField).includes('vendor'); +} + +function normalizeFilesEntry(value) { + return String(value ?? '') + .replace(/\\/g, '/') + .replace(/\/+$/, '') + .replace(/\/\*\*?$/, ''); +} + +function filesEntries(filesField) { + return (filesField || []).map(normalizeFilesEntry); +} + +function filesCoverGrammarPrebuilds(entries, grammarName) { + if (entries.includes('vendor') || entries.includes('vendor/**/prebuilds')) return true; + return ( + entries.includes(`vendor/${grammarName}/prebuilds`) || entries.includes(`vendor/${grammarName}`) + ); +} + +function filesCoverGrammarBindings(entries, grammarName) { + if (entries.includes('vendor') || entries.includes('vendor/**/bindings/node/index.js')) { + return true; + } + return ( + entries.includes(`vendor/${grammarName}/bindings/node/index.js`) || + entries.includes(`vendor/${grammarName}`) + ); +} + +function filesCoverGrammarPackageJson(entries, grammarName) { + if (entries.includes('vendor') || entries.includes('vendor/**/package.json')) return true; + return ( + entries.includes(`vendor/${grammarName}/package.json`) || + entries.includes(`vendor/${grammarName}`) + ); +} + +function filesCoverLeiden(entries) { + if (entries.includes('vendor') || entries.includes('vendor/leiden')) return true; + return entries.includes('vendor/leiden/index.cjs') && entries.includes('vendor/leiden/utils.cjs'); +} + +/** + * Packed-tarball coverage from `files` globs — not on-disk prebuild counts. + * Lean publish can leave 6/6 `.node` files in the checkout while omitting + * them from the pack list. + */ +function findPackedFilesProblems({ filesField, grammarNames }) { + const entries = filesEntries(filesField); + const problems = []; + for (const name of grammarNames || []) { + if (!filesCoverGrammarPrebuilds(entries, name)) { + problems.push(`${name}: package.json files does not cover vendor/${name}/prebuilds`); + } + if (!filesCoverGrammarBindings(entries, name)) { + problems.push( + `${name}: package.json files does not cover vendor/${name}/bindings/node/index.js`, + ); + } + if (!filesCoverGrammarPackageJson(entries, name)) { + problems.push(`${name}: package.json files does not cover vendor/${name}/package.json`); + } + } + if (!filesCoverLeiden(entries)) { + problems.push('package.json files does not cover vendor/leiden/index.cjs and utils.cjs'); + } + return problems; } /** The on-disk source-build inputs for a grammar (relative paths). */ @@ -171,7 +232,13 @@ function main() { process.exit(1); } - const problems = findCoverageProblems({ grammars }); + const problems = [ + ...findCoverageProblems({ grammars }), + ...findPackedFilesProblems({ + filesField: pkg.files, + grammarNames: grammars.map((g) => g.name), + }), + ]; if (problems.length > 0) { console.error('[publish-guard] Refusing to publish — a vendored grammar would ship unusable:'); for (const p of problems) console.error(` - ${p}`); @@ -193,8 +260,13 @@ if (require.main === module) main(); module.exports = { findCoverageProblems, + findPackedFilesProblems, findStrayBuildArtifacts, filesShipsVendorSource, + filesCoverGrammarPrebuilds, + filesCoverGrammarBindings, + filesCoverGrammarPackageJson, + filesCoverLeiden, isBuildableFromSource, sourceBuildSet, countPrebuiltTuples, diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 0f034ee26..04efddfd8 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -69,9 +69,9 @@ import { safeUrl, } from '../core/embeddings/http-client.js'; import { + assessLocalEmbeddingRuntime, isLocalEmbeddingRuntimeBlockerMessage, isMissingLocalEmbeddingStackMessage, - localEmbeddingPrefixUnloadableMessage, localEmbeddingStackMissingMessage, } from '../core/embeddings/runtime-support.js'; import { @@ -79,8 +79,6 @@ import { getEmbeddingInstallTimeoutMs, getEmbeddingRuntimeDir, installEmbeddingRuntime, - isPrefixRuntimeLoadable, - resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js'; import { warnIfNpm11NpxRisk } from './resolve-invocation.js'; @@ -1128,14 +1126,18 @@ const analyzeCommandImpl = async ( ); } - // On-demand embedding runtime (#2370): when the optional stack was pruned at - // install time (proxy-blocked NuGet download in onnxruntime-node's - // postinstall), heal it here instead of failing later in the pipeline. The - // install goes through the user's npm registry config (mirrors/proxies - // apply) with --ignore-scripts, so no NuGet download is attempted. Runs - // before bar.start() like the sibling validations above. + // Local embeddings: refuse Intel Mac / unloadable prefix before any registry + // download, then auto-heal a missing stack. Analyze uses a short install + // timeout so a blackholed proxy cannot stall the index run. if (embeddingsEnabled && !isHttpMode()) { - const resolved = resolveEmbeddingRuntime(); + const assessment = assessLocalEmbeddingRuntime(); + if (assessment.status === 'blocked') { + cliError(` ${assessment.message.replace(/\n/g, '\n ')}\n`, { + recoveryHint: 'local-embedding-unsupported', + }); + process.exitCode = 1; + return; + } // Resolved-but-unloadable (a populated prefix on a Node with no // module.registerHooks), or nothing installed on such a Node: fail fast with // capability guidance instead of dying mid-pipeline over an unusable prefix @@ -1143,28 +1145,20 @@ const analyzeCommandImpl = async ( // never needs the hook, so it is excluded. --embeddings was explicitly // requested and this failure is deterministic, so fail fast rather than // silently degrading to BM25 (distinct from a transient install timeout). - if (!isPrefixRuntimeLoadable() && (resolved === null || resolved.source === 'runtime-prefix')) { - cliError(` ${localEmbeddingPrefixUnloadableMessage().replace(/\n/g, '\n ')}\n`, { + if (assessment.status === 'prefix-unloadable') { + cliError(` ${assessment.message.replace(/\n/g, '\n ')}\n`, { recoveryHint: 'local-embedding-stack-missing', }); process.exitCode = 1; return; } - // On-demand embedding runtime (#2370): when the optional stack was pruned at - // install time (proxy-blocked NuGet download in onnxruntime-node's - // postinstall), heal it here instead of failing later in the pipeline. The - // install goes through the user's npm registry config (mirrors/proxies - // apply) with --ignore-scripts, so no NuGet download is attempted. - if (resolved === null) { + if (assessment.status === 'needs-install') { console.log( - ` Local embedding runtime is not installed (optional packages were skipped at install time).\n` + + ` Local embedding runtime is not installed.\n` + ` Downloading it now from your npm registry into ${getEmbeddingRuntimeDir()} …\n` + ` (one-time; rerun manually anytime with \`gitnexus embeddings install\`)\n`, ); try { - // Short deadline (env override still wins): analyze is interactive, so a - // blackholed proxy must not stall the whole index run for the 10-minute - // default — fail over to the guidance below instead. await installEmbeddingRuntime( {}, getEmbeddingInstallTimeoutMs(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS), diff --git a/gitnexus/src/cli/doctor.ts b/gitnexus/src/cli/doctor.ts index 202baafbd..909bd945d 100644 --- a/gitnexus/src/cli/doctor.ts +++ b/gitnexus/src/cli/doctor.ts @@ -108,12 +108,10 @@ export function localEmbeddingDoctorStatus(opts: { if (blocker) { return { status: `✗ local embeddings unavailable on ${platform}/${arch}`, detail: blocker }; } - // The stack is an optionalDependency — npm prunes it when onnxruntime-node's - // postinstall can't download its CUDA binaries (proxy/firewall, #2370). const resolution = opts.resolution !== undefined ? opts.resolution : resolveEmbeddingRuntime(); if (resolution === null) { return { - status: '✗ optional embedding stack not installed', + status: '✗ local embedding stack not installed', detail: localEmbeddingStackMissingMessage(), }; } diff --git a/gitnexus/src/cli/embeddings-sync.ts b/gitnexus/src/cli/embeddings-sync.ts index d45d059ed..ff00eeb57 100644 --- a/gitnexus/src/cli/embeddings-sync.ts +++ b/gitnexus/src/cli/embeddings-sync.ts @@ -11,8 +11,6 @@ import { fetchExistingEmbeddingHashes, initLbug, } from '../core/lbug/lbug-adapter.js'; -import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js'; -import { resolveEmbeddingIdentity } from '../core/embeddings/embedding-identity.js'; import { decideEmbeddingResume, mintInterruptedCheckpoint, @@ -27,6 +25,18 @@ import { measurePersistedEmbeddingCount, persistedEmbeddingCountOrUndefined, } from '../core/embedding-count.js'; +import { isHttpMode } from '../core/embeddings/http-client.js'; +import { + ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS, + getEmbeddingInstallTimeoutMs, + getEmbeddingRuntimeDir, + installEmbeddingRuntime, +} from '../core/embeddings/runtime-install.js'; +import { + assessLocalEmbeddingRuntime, + localEmbeddingStackMissingMessage, +} from '../core/embeddings/runtime-support.js'; +import { reapEmbeddingSidecarSafely } from '../core/embeddings/embedding-sidecar-reap.js'; /** Add missing embeddings directly to a healthy index, checkpointing periodically. */ export const embeddingsSyncCommand = async (inputPath?: string): Promise => { @@ -62,6 +72,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = ); } + const { resolveEmbeddingIdentity } = await import('../core/embeddings/embedding-identity.js'); const identity = resolveEmbeddingIdentity(); let forceReembedNodeIds: ReadonlySet | undefined; let resumedFrom: EmbeddingCheckpoint | undefined; @@ -111,6 +122,28 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = ); } + if (!isHttpMode()) { + const assessment = assessLocalEmbeddingRuntime(); + if (assessment.status === 'blocked' || assessment.status === 'prefix-unloadable') { + throw new Error(assessment.message); + } + if (assessment.status === 'needs-install') { + cliInfo(`Local embedding runtime is not installed.`); + cliInfo(`Downloading it now from your npm registry into ${getEmbeddingRuntimeDir()} …`); + try { + await installEmbeddingRuntime( + {}, + getEmbeddingInstallTimeoutMs(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS), + ); + } catch (err) { + throw new Error( + `Could not install the embedding runtime: ${err instanceof Error ? err.message : String(err)}\n\n` + + localEmbeddingStackMissingMessage(), + ); + } + } + } + await initLbug(lbugPath); try { const existing = await fetchExistingEmbeddingHashes(executeQuery); @@ -139,6 +172,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = cliInfo(`Embedding ${repoPath}`); cliInfo(`Checkpointed nodes already present: ${existing?.size ?? 0}`); + const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js'); const result = await runEmbeddingPipeline( executeQuery, executeWithReusedStatement, @@ -190,6 +224,7 @@ export const embeddingsSyncCommand = async (inputPath?: string): Promise = cliInfo(`Embeddings ready: ${embeddings}`); } finally { await closeLbug().catch(() => {}); + await reapEmbeddingSidecarSafely(); } } finally { lock.release(); diff --git a/gitnexus/src/cli/embeddings.ts b/gitnexus/src/cli/embeddings.ts index a63b342c4..ed447db5d 100644 --- a/gitnexus/src/cli/embeddings.ts +++ b/gitnexus/src/cli/embeddings.ts @@ -6,7 +6,10 @@ import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime, } from '../core/embeddings/runtime-install.js'; -import { localEmbeddingPrefixUnloadableMessage } from '../core/embeddings/runtime-support.js'; +import { + getLocalEmbeddingRuntimeBlocker, + localEmbeddingPrefixUnloadableMessage, +} from '../core/embeddings/runtime-support.js'; export interface EmbeddingsInstallOptions { cuda?: boolean; @@ -14,20 +17,29 @@ export interface EmbeddingsInstallOptions { } /** - * `gitnexus embeddings install [--cuda] [--force]` — fetch the optional local - * embedding stack on demand (#2370). Goes through the user's npm registry - * config (mirrors/proxies apply); with --cuda it additionally runs - * onnxruntime-node's postinstall to download the CUDA GPU binaries from NuGet - * (set GLOBAL_AGENT_HTTPS_PROXY behind a proxy). + * `gitnexus embeddings install [--cuda] [--force]` — fetch the local + * embedding stack. Default npm install does not include it. Goes through the + * user's npm registry config (mirrors/proxies apply); with --cuda it + * additionally runs onnxruntime-node's postinstall to download the CUDA GPU + * binaries from NuGet (set GLOBAL_AGENT_HTTPS_PROXY behind a proxy). + * `--force` refreshes prefix overrides; it does not replace a leftover + * 1.6.12 package-first tree in gitnexus node_modules. */ export const embeddingsInstallCommand = async ( options: EmbeddingsInstallOptions = {}, ): Promise => { + const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); + if (runtimeBlocker) { + cliError(`${runtimeBlocker}\n`, { recoveryHint: 'local-embedding-unsupported' }); + process.exitCode = 1; + return; + } + const resolved = resolveEmbeddingRuntime(); if (resolved?.source === 'package' && !options.force) { cliInfo( - 'The embedding stack is already installed with gitnexus itself — nothing to do.\n' + - '(Use --force to install a copy into the runtime prefix anyway.)', + 'The embedding stack already resolves from this gitnexus install (leftover package-first tree) — nothing to do.\n' + + '(Use --force to refresh prefix overrides; a clean reinstall removes leftover packages.)', ); return; } diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index d280b6a2d..8dc7f45d7 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -183,7 +183,7 @@ export const en = { 'Install the latest published GitNexus globally (`npm i -g gitnexus@`).', 'help.command.embeddings.description': 'Manage the on-demand local embedding runtime', 'help.command.embeddings.install.description': - 'Install the local embedding stack (@huggingface/transformers + onnxruntime-node) on demand. Heals installs where npm skipped the optional packages (e.g. behind an HTTP proxy, #2370). Downloads only from your configured npm registry — mirrors and proxies apply.', + 'Install the local embedding stack (@huggingface/transformers + onnxruntime-node) on demand. The stack is not part of a default npm install. CPU installs download only from your configured npm registry — mirrors and proxies apply. `--cuda` additionally runs onnxruntime-node postinstall, which fetches CUDA binaries from NuGet (set GLOBAL_AGENT_HTTPS_PROXY behind a proxy).', 'help.command.clean.description': 'Delete GitNexus index for current repo', 'help.command.remove.description': 'Delete the GitNexus index for a registered repo (by alias, name, or absolute path). Unlike `clean`, does not require being inside the repo. Idempotent on unknown targets.', diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 97a6c2eff..bef315752 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -177,7 +177,7 @@ export const zhCN = { '通过 npm 全局安装最新发布的 GitNexus（`npm i -g gitnexus@`）。', 'help.command.embeddings.description': '管理按需安装的本地嵌入运行时', 'help.command.embeddings.install.description': - '按需安装本地嵌入组件（@huggingface/transformers + onnxruntime-node）。修复 npm 跳过可选包的安装（例如在 HTTP 代理后，#2370）。仅从你配置的 npm registry 下载 — 镜像和代理均生效。', + '按需安装本地嵌入组件（@huggingface/transformers + onnxruntime-node）。默认 npm 安装不包含该组件。CPU 安装仅从你配置的 npm registry 下载 — 镜像和代理均生效。`--cuda` 还会运行 onnxruntime-node 的 postinstall，从 NuGet 获取 CUDA 二进制（代理后需设置 GLOBAL_AGENT_HTTPS_PROXY）。', 'help.command.clean.description': '删除当前仓库的 GitNexus 索引', 'help.command.remove.description': '删除已注册仓库的 GitNexus 索引（按别名、名称或绝对路径）。与 `clean` 不同，不要求位于仓库内；未知目标会幂等处理。', diff --git a/gitnexus/src/core/embeddings/embedder.ts b/gitnexus/src/core/embeddings/embedder.ts index e41c4f4c9..c9d188f49 100644 --- a/gitnexus/src/core/embeddings/embedder.ts +++ b/gitnexus/src/core/embeddings/embedder.ts @@ -1,75 +1,44 @@ /** - * Embedder Module + * Embedder façade — HTTP or embedding sidecar. * - * Singleton factory for transformers.js embedding pipeline. - * Handles model loading, caching, and both single and batch embedding operations. - * - * Uses snowflake-arctic-embed-xs by default (22M params, 384 dims, ~90MB) + * This module must not import the Hugging Face transformers package, the + * ONNX Node binding, the ONNX resolvers, or the child-only local init + * module. Local inference runs in the sidecar child; the parent keeps + * Ladybug writes. */ -// Suppress ONNX Runtime native warnings (e.g. VerifyEachNodeIsAssignedToAnEp) -// Must be set BEFORE onnxruntime-node is imported by transformers.js -// Level 3 = Error only (skips Warning/Info) -if (!process.env.ORT_LOG_LEVEL) { - process.env.ORT_LOG_LEVEL = '3'; -} - -// Type-only import: erased at compile time so loading this module never pulls -// in @huggingface/transformers (and its native onnxruntime-node binding) at -// runtime. The runtime values (pipeline, env) are dynamically imported inside -// initEmbedder, after the platform guard has passed (#1515). -import type { FeatureExtractionPipeline, ProgressInfo } from '@huggingface/transformers'; -import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js'; +import { + DEFAULT_EMBEDDING_CONFIG, + type EmbeddingConfig, + type ModelProgressCallback, +} from './types.js'; import { isHttpMode, getHttpDimensions, httpEmbed, type EmbeddingRequestOptions, } from './http-client.js'; -import { resolveEmbeddingConfig } from './config.js'; -import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js'; +import { assessLocalEmbeddingRuntime, getLocalEmbeddingRuntimeBlocker } from './runtime-support.js'; import { - getLocalEmbeddingRuntimeBlocker, - getMissingLocalEmbeddingStackMessage, -} from './runtime-support.js'; -import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js'; -import { ensureEmbeddingStackResolvable } from './runtime-install.js'; -import { - ensureOnnxRuntimeNodeMatchesSystem, - isEffectiveCudaAvailable, -} from './onnxruntime-node-resolver.js'; -import { logger } from '../logger.js'; + ensureEmbeddingSidecar, + getSidecarDevice, + reapEmbeddingSidecarAndWait, + sidecarEmbedBatch, +} from './embedding-sidecar-client.js'; +import type { EmbeddingSidecarDevice } from './embedding-sidecar-protocol.js'; -// Module-level state for singleton pattern -let embedderInstance: FeatureExtractionPipeline | null = null; -let isInitializing = false; -let initPromise: Promise | null = null; -let currentDevice: 'dml' | 'cuda' | 'cpu' | 'wasm' | null = null; +export type { ModelProgressCallback } from './types.js'; -/** - * Progress callback type for model loading - */ -export type ModelProgressCallback = (progress: ModelProgress) => void; +export const getCurrentDevice = (): EmbeddingSidecarDevice | null => { + if (isHttpMode()) return null; + return getSidecarDevice(); +}; -/** - * Get the current device being used for inference - */ -export const getCurrentDevice = (): 'dml' | 'cuda' | 'cpu' | 'wasm' | null => currentDevice; - -/** - * Initialize the embedding model - * Uses singleton pattern - only loads once, subsequent calls return cached instance - * - * @param onProgress - Optional callback for model download progress - * @param config - Optional configuration override - * @param forceDevice - Force a specific device - * @returns Promise resolving to the embedder pipeline - */ export const initEmbedder = async ( onProgress?: ModelProgressCallback, config: Partial = {}, - forceDevice?: 'dml' | 'cuda' | 'cpu' | 'wasm', -): Promise => { + forceDevice?: EmbeddingSidecarDevice, +): Promise<{ device: EmbeddingSidecarDevice }> => { if (isHttpMode()) { throw new Error( 'initEmbedder() should not be called in HTTP mode. ' + @@ -77,203 +46,39 @@ export const initEmbedder = async ( ); } - // Fail fast on platforms where the bundled native ONNX Runtime binding is not - // shipped (macOS Intel, #1515). Must run before any transformers.js / - // onnxruntime-node import or resolution — otherwise the native module load - // crashes with a raw "Cannot find module ...onnxruntime_binding.node" that - // ONNX_WEB_BACKEND=wasm cannot rescue (#1516). HTTP mode was already handled - // above, so this only blocks the local-runtime path. const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); if (runtimeBlocker) { throw new Error(runtimeBlocker); } - // Return existing instance if available - if (embedderInstance) { - return embedderInstance; + return ensureEmbeddingSidecar({ + onProgress, + embeddingConfig: config, + forceDevice, + }); +}; + +export const getEmbedder = (): never => { + if (isHttpMode()) { + throw new Error( + 'getEmbedder() is not available in HTTP embedding mode. Use embedText()/embedBatch() instead.', + ); } - - // If already initializing, wait for that promise - if (isInitializing && initPromise) { - return initPromise; - } - - isInitializing = true; - - const finalConfig = resolveEmbeddingConfig(config); - // CUDA is probe-gated because ONNX Runtime can crash in native code when - // provider libraries are missing. DirectML stays opt-in for the same reason. - // Probe for CUDA first — ONNX Runtime crashes (uncatchable native error) - // if we attempt CUDA without the required shared libraries - const gpuDevice = isEffectiveCudaAvailable() ? 'cuda' : 'cpu'; - const requestedDevice = - forceDevice || (finalConfig.device === 'auto' ? gpuDevice : finalConfig.device); - - initPromise = (async () => { - try { - // Lazy-load transformers.js only after the runtime guard has passed, so - // unsupported platforms never reach the native ONNX import (#1515). - // Registered FIRST so it sits last in the hook chain (registerHooks runs - // the most recent hook first): when the optional stack was pruned at - // install time (#2370), its bare specifiers fall back to the on-demand - // runtime prefix. - ensureEmbeddingStackResolvable(); - // Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common` - // import is unresolvable; register the fallback resolver first (#307). - ensureOnnxRuntimeCommonResolvable(); - // Registered AFTER the common fallback so this hook resolves FIRST (Node - // runs the most-recently-registered hook first): on CUDA-13 hosts it - // redirects onnxruntime-node (and its version-matched onnxruntime-common) - // to the CUDA-13 build before transformers imports them. No-op on matching - // layouts, non-CUDA, Windows/DirectML, and macOS. - ensureOnnxRuntimeNodeMatchesSystem(); - // The stack is an optionalDependency: npm prunes it when onnxruntime-node's - // postinstall can't reach api.nuget.org (#2370). Rethrow with actionable - // reinstall guidance instead of a raw ERR_MODULE_NOT_FOUND. - const { pipeline, env } = await import('@huggingface/transformers').catch((err: unknown) => { - const missing = getMissingLocalEmbeddingStackMessage(err); - if (missing) throw new Error(missing); - throw err; - }); - - // Configure transformers.js environment - env.allowLocalModels = false; - // Bridge user-controlled env vars to transformers.js: HF_HOME → - // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in - // applyHfEnvOverrides so the MCP embedder entry point behaves - // identically. - applyHfEnvOverrides(env); - - const isDev = process.env.NODE_ENV === 'development'; - if (isDev) { - logger.info(`🧠 Loading embedding model: ${finalConfig.modelId}`); - } - - const progressCallback = onProgress - ? (data: ProgressInfo) => { - const progress: ModelProgress = { - // Map the `progress_total` aggregate event (not in ModelProgress.status) - // back to 'progress' so callers don't need to handle it separately. - status: - data.status === 'progress_total' - ? 'progress' - : ((data.status as ModelProgress['status']) ?? 'progress'), - file: 'file' in data ? data.file : undefined, - progress: 'progress' in data ? data.progress : undefined, - loaded: 'loaded' in data ? data.loaded : undefined, - total: 'total' in data ? data.total : undefined, - }; - onProgress(progress); - } - : undefined; - - // Try GPU first if auto, fall back to CPU - // Windows: dml (DirectML/DirectX12), Linux: cuda - const devicesToTry: Array<'dml' | 'cuda' | 'cpu' | 'wasm'> = - requestedDevice === 'dml' || requestedDevice === 'cuda' - ? [requestedDevice, 'cpu'] - : [requestedDevice as 'cpu' | 'wasm']; - - for (const device of devicesToTry) { - try { - if (isDev && device === 'dml') { - logger.info('🔧 Trying DirectML (DirectX12) GPU backend...'); - } else if (isDev && device === 'cuda') { - logger.info('🔧 Trying CUDA GPU backend...'); - } else if (isDev && device === 'cpu') { - logger.info('🔧 Using CPU backend...'); - } else if (isDev && device === 'wasm') { - logger.info('🔧 Using WASM backend (slower)...'); - } - - embedderInstance = await withHfDownloadRetry( - () => - pipeline('feature-extraction', finalConfig.modelId, { - device: device, - dtype: 'fp32', - progress_callback: progressCallback, - session_options: { - logSeverityLevel: 3, - intraOpNumThreads: finalConfig.threads, - interOpNumThreads: 1, - executionMode: 'sequential', - }, - }), - { - onRetry: isDev - ? (attempt, max, err) => - logger.warn( - { attempt, max, err: err.message }, - `⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`, - ) - : undefined, - }, - ); - currentDevice = device; - - if (isDev) { - const label = - device === 'dml' - ? 'GPU (DirectML/DirectX12)' - : device === 'cuda' - ? 'GPU (CUDA)' - : device.toUpperCase(); - logger.info(`✅ Using ${label} backend`); - logger.info('✅ Embedding model loaded successfully'); - } - - return embedderInstance!; - } catch (deviceError) { - // Network errors and circuit-open errors are not device-specific — - // they will fail the same way on every device. Rethrow immediately - // with actionable HF_ENDPOINT guidance rather than silently falling - // back to the next device. - const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError); - if (isHfDownloadFailure(errMsg)) { - const endpointHint = process.env.HF_ENDPOINT - ? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.` - : `huggingface.co may be unreachable from your network.\n` + - ` Set HF_ENDPOINT to a mirror and retry:\n` + - ` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` + - ` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`; - throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`); - } - if (isDev && (device === 'cuda' || device === 'dml')) { - const gpuType = device === 'dml' ? 'DirectML' : 'CUDA'; - logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`); - } - // Continue to next device in list - if (device === devicesToTry[devicesToTry.length - 1]) { - throw deviceError; // Last device failed, propagate error - } - } - } - - throw new Error('No suitable device found for embedding model'); - } catch (error) { - isInitializing = false; - initPromise = null; - embedderInstance = null; - throw error; - } finally { - isInitializing = false; - } - })(); - - return initPromise; + throw new Error( + 'getEmbedder() is not available. Local inference runs in the embedding sidecar. Use embedText()/embedBatch() instead.', + ); }; /** - * Check if the embedder is initialized and ready + * Ready when HTTP embeddings are configured, or local runtime assessment + * is `ready` (blocker / prefix-unloadable / missing-stack are not ready). + * Sidecar liveness is not required. Resolution alone is not enough: a leftover + * 1.6.12 package-first tree on darwin/x64 still resolves, then embedText throws. */ export const isEmbedderReady = (): boolean => { - return isHttpMode() || embedderInstance !== null; + return isHttpMode() || assessLocalEmbeddingRuntime().status === 'ready'; }; -/** - * Get the effective embedding dimensions. - * In HTTP mode, uses GITNEXUS_EMBEDDING_DIMS if set, otherwise the default. - */ export const getEmbeddingDimensions = (): number => { if (isHttpMode()) { return getHttpDimensions() ?? DEFAULT_EMBEDDING_CONFIG.dimensions; @@ -281,27 +86,6 @@ export const getEmbeddingDimensions = (): number => { return DEFAULT_EMBEDDING_CONFIG.dimensions; }; -/** - * Get the embedder instance (throws if not initialized) - */ -export const getEmbedder = (): FeatureExtractionPipeline => { - if (isHttpMode()) { - throw new Error( - 'getEmbedder() is not available in HTTP embedding mode. Use embedText()/embedBatch() instead.', - ); - } - if (!embedderInstance) { - throw new Error('Embedder not initialized. Call initEmbedder() first.'); - } - return embedderInstance; -}; - -/** - * Embed a single text string - * - * @param text - Text to embed - * @returns Float32Array of embedding vector - */ export const embedText = async ( text: string, options: EmbeddingRequestOptions = {}, @@ -312,24 +96,15 @@ export const embedText = async ( return vec; } - const embedder = getEmbedder(); + const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); + if (runtimeBlocker) { + throw new Error(runtimeBlocker); + } - const result = await embedder(text, { - pooling: 'mean', - normalize: true, - }); - - // Result is a Tensor, convert to Float32Array - return new Float32Array(result.data as ArrayLike); + const [vec] = await sidecarEmbedBatch([text], options); + return vec; }; -/** - * Embed multiple texts in a single batch - * More efficient than calling embedText multiple times - * - * @param texts - Array of texts to embed - * @returns Array of Float32Array embedding vectors - */ export const embedBatch = async ( texts: string[], options: EmbeddingRequestOptions = {}, @@ -343,52 +118,21 @@ export const embedBatch = async ( return httpEmbed(texts, options); } - const embedder = getEmbedder(); - - // Process batch - const result = await embedder(texts, { - pooling: 'mean', - normalize: true, - }); - options.signal?.throwIfAborted(); - - // Result shape is [batch_size, dimensions] - // Need to split into individual vectors - const data = result.data as ArrayLike; - const dimensions = DEFAULT_EMBEDDING_CONFIG.dimensions; - const embeddings: Float32Array[] = []; - - for (let i = 0; i < texts.length; i++) { - const start = i * dimensions; - const end = start + dimensions; - embeddings.push(new Float32Array(Array.prototype.slice.call(data, start, end))); + const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); + if (runtimeBlocker) { + throw new Error(runtimeBlocker); } - return embeddings; + return sidecarEmbedBatch(texts, options); }; -/** - * Convert Float32Array to regular number array (for LadybugDB storage) - */ export const embeddingToArray = (embedding: Float32Array): number[] => { return Array.from(embedding); }; /** - * Cleanup the embedder (free memory) - * Call this when done with embeddings + * Reap the sidecar. Never runs ONNX dispose in this process. */ export const disposeEmbedder = async (): Promise => { - if (embedderInstance) { - // transformers.js pipelines may have a dispose method - try { - if ('dispose' in embedderInstance && typeof embedderInstance.dispose === 'function') { - await embedderInstance.dispose(); - } - } catch { - // Ignore disposal errors - } - embedderInstance = null; - initPromise = null; - } + await reapEmbeddingSidecarAndWait(); }; diff --git a/gitnexus/src/core/embeddings/embedding-local-init.ts b/gitnexus/src/core/embeddings/embedding-local-init.ts new file mode 100644 index 000000000..8e7c62784 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-local-init.ts @@ -0,0 +1,227 @@ +/** + * Child-only local ONNX embedder. + * + * Imported solely by `embedding-sidecar.ts`. Parent processes must not import + * this module — it loads transformers.js / onnxruntime-node after the platform + * guard, which is what isolation is meant to keep out of analyze/serve/MCP. + */ + +if (!process.env.ORT_LOG_LEVEL) { + process.env.ORT_LOG_LEVEL = '3'; +} + +import type { FeatureExtractionPipeline, ProgressInfo } from '@huggingface/transformers'; +import { + DEFAULT_EMBEDDING_CONFIG, + type EmbeddingConfig, + type ModelProgress, + type ModelProgressCallback, +} from './types.js'; +import type { EmbeddingSidecarDevice } from './embedding-sidecar-protocol.js'; +import { resolveEmbeddingConfig } from './config.js'; +import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js'; +import { + getLocalEmbeddingRuntimeBlocker, + getMissingLocalEmbeddingStackMessage, +} from './runtime-support.js'; +import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js'; +import { ensureEmbeddingStackResolvable } from './runtime-install.js'; +import { + ensureOnnxRuntimeNodeMatchesSystem, + isEffectiveCudaAvailable, +} from './onnxruntime-node-resolver.js'; +import { logger } from '../logger.js'; + +let embedderInstance: FeatureExtractionPipeline | null = null; +let initPromise: Promise | null = null; +let currentDevice: EmbeddingSidecarDevice | null = null; +let activeDimensions = DEFAULT_EMBEDDING_CONFIG.dimensions; + +const formatDeviceLabel = (device: EmbeddingSidecarDevice): string => { + switch (device) { + case 'dml': + return 'GPU (DirectML/DirectX12)'; + case 'cuda': + return 'GPU (CUDA)'; + default: + return device.toUpperCase(); + } +}; + +export const getCurrentDevice = (): EmbeddingSidecarDevice | null => currentDevice; + +export const initLocalEmbedder = async ( + onProgress?: ModelProgressCallback, + config: Partial = {}, + forceDevice?: EmbeddingSidecarDevice, +): Promise => { + const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); + if (runtimeBlocker) { + throw new Error(runtimeBlocker); + } + + if (embedderInstance) { + return embedderInstance; + } + + if (initPromise) { + return initPromise; + } + + const finalConfig = resolveEmbeddingConfig(config); + const gpuDevice: EmbeddingSidecarDevice = isEffectiveCudaAvailable() + ? 'cuda' + : process.platform === 'win32' + ? 'dml' + : 'cpu'; + const requestedDevice = + forceDevice || (finalConfig.device === 'auto' ? gpuDevice : finalConfig.device); + + initPromise = (async () => { + try { + ensureEmbeddingStackResolvable(); + ensureOnnxRuntimeCommonResolvable(); + ensureOnnxRuntimeNodeMatchesSystem(); + const { pipeline, env } = await import('@huggingface/transformers').catch((err: unknown) => { + const missing = getMissingLocalEmbeddingStackMessage(err); + if (missing) throw new Error(missing); + throw err; + }); + + env.allowLocalModels = false; + applyHfEnvOverrides(env); + + const isDev = process.env.NODE_ENV === 'development'; + if (isDev) { + logger.info(`🧠 Loading embedding model: ${finalConfig.modelId}`); + } + + const progressCallback = onProgress + ? (data: ProgressInfo) => { + const progress: ModelProgress = { + status: + data.status === 'progress_total' + ? 'progress' + : ((data.status as ModelProgress['status']) ?? 'progress'), + file: 'file' in data ? data.file : undefined, + progress: 'progress' in data ? data.progress : undefined, + loaded: 'loaded' in data ? data.loaded : undefined, + total: 'total' in data ? data.total : undefined, + }; + onProgress(progress); + } + : undefined; + + const devicesToTry: EmbeddingSidecarDevice[] = + requestedDevice === 'dml' || requestedDevice === 'cuda' + ? [requestedDevice, 'cpu'] + : [requestedDevice]; + + for (const device of devicesToTry) { + try { + if (isDev && device === 'dml') { + logger.info('🔧 Trying DirectML (DirectX12) GPU backend...'); + } else if (isDev && device === 'cuda') { + logger.info('🔧 Trying CUDA GPU backend...'); + } else if (isDev && device === 'cpu') { + logger.info('🔧 Using CPU backend...'); + } else if (isDev && device === 'wasm') { + logger.info('🔧 Using WASM backend (slower)...'); + } + + embedderInstance = await withHfDownloadRetry( + () => + pipeline('feature-extraction', finalConfig.modelId, { + device: device, + dtype: 'fp32', + progress_callback: progressCallback, + session_options: { + logSeverityLevel: 3, + intraOpNumThreads: finalConfig.threads, + interOpNumThreads: 1, + executionMode: 'sequential', + }, + }), + { + onRetry: isDev + ? (attempt, max, err) => + logger.warn( + { attempt, max, err: err.message }, + `⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`, + ) + : undefined, + }, + ); + currentDevice = device; + activeDimensions = finalConfig.dimensions; + + if (isDev) { + logger.info(`✅ Using ${formatDeviceLabel(device)} backend`); + logger.info('✅ Embedding model loaded successfully'); + } + + return embedderInstance!; + } catch (deviceError) { + const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError); + if (isHfDownloadFailure(errMsg)) { + const endpointHint = process.env.HF_ENDPOINT + ? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.` + : `huggingface.co may be unreachable from your network.\n` + + ` Set HF_ENDPOINT to a mirror and retry:\n` + + ` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` + + ` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`; + throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`); + } + if (isDev && (device === 'cuda' || device === 'dml')) { + const gpuType = device === 'dml' ? 'DirectML' : 'CUDA'; + logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`); + } + if (device === devicesToTry[devicesToTry.length - 1]) { + throw deviceError; + } + } + } + + throw new Error('No suitable device found for embedding model'); + } catch (error) { + initPromise = null; + embedderInstance = null; + throw error; + } + })(); + + return initPromise; +}; + +const getLocalEmbedder = (): FeatureExtractionPipeline => { + if (!embedderInstance) { + throw new Error('Embedder not initialized. Call initLocalEmbedder() first.'); + } + return embedderInstance; +}; + +export const localEmbedBatch = async (texts: string[]): Promise => { + if (texts.length === 0) { + return []; + } + + const embedder = getLocalEmbedder(); + const result = await embedder(texts, { + pooling: 'mean', + normalize: true, + }); + + const data = result.data as ArrayLike; + const dims = (result as { dims?: number[] }).dims; + const dimensions = + typeof dims?.[dims.length - 1] === 'number' ? dims[dims.length - 1] : activeDimensions; + const embeddings: Float32Array[] = []; + + for (let i = 0; i < texts.length; i++) { + const start = i * dimensions; + const end = start + dimensions; + embeddings.push(new Float32Array(Array.prototype.slice.call(data, start, end))); + } + + return embeddings; +}; diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 605bf8457..78aec5b02 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -17,6 +17,7 @@ import { embeddingToArray, isEmbedderReady, } from './embedder.js'; +import { isHttpMode } from './http-client.js'; import { generateEmbeddingText } from './text-generator.js'; import { chunkNode, characterChunk } from './chunker.js'; import { extractStructuralNames } from './structural-extractor.js'; @@ -560,7 +561,7 @@ export const runEmbeddingPipeline = async ( modelDownloadPercent: 0, }); - if (!isEmbedderReady()) { + if (!isHttpMode()) { await initEmbedder((modelProgress: ModelProgress) => { const downloadPercent = modelProgress.progress ?? 0; onProgress({ @@ -1009,6 +1010,13 @@ export const semanticSearch = async ( k: number = 10, maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE), ): Promise => { + // determinism: probe — existence only. Only `exists.length` is read; which + // row LIMIT 1 returns cannot change whether the table is empty. + const exists = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN 1 AS ok LIMIT 1`); + if (!exists.length) { + return []; + } + if (!isEmbedderReady()) { throw new Error('Embedding model not initialized. Run embedding pipeline first.'); } @@ -1054,12 +1062,11 @@ export const semanticSearch = async ( } if (bestChunks.size === 0) { - // The Cypher only. NOT `measurePersistedEmbeddingCount`: its tri-state - // exists so a publisher never writes a fabricated 0, whereas here `?? 0` - // is the right answer — an unknown count simply skips the exact scan. + // NOT `measurePersistedEmbeddingCount`: its tri-state exists so a publisher + // never writes a fabricated 0, whereas here `?? 0` is the right answer — + // an unknown count simply skips the exact scan. const countRows = await executeQuery(EMBEDDING_COUNT_CYPHER); - const countRow = countRows[0]; - const embeddingCount = Number(countRow?.cnt ?? countRow?.[0] ?? 0); + const embeddingCount = Number(countRows[0]?.cnt ?? countRows[0]?.[0] ?? 0); const exactLimit = getExactScanLimit(); if (embeddingCount > 0 && embeddingCount <= exactLimit) { const rows = await executeQuery(` diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-client.ts b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts new file mode 100644 index 000000000..ce8e1e2f0 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-sidecar-client.ts @@ -0,0 +1,437 @@ +/** + * Parent-side embedding sidecar client. + * + * Forks the sidecar over IPC and never imports the local ONNX init + * path, the embeddings barrel, or the pipeline module. Stdio must not inherit + * parent stdout (MCP JSON-RPC). + */ + +import { fork, type ChildProcess, type ForkOptions } from 'node:child_process'; +import { createRequire } from 'node:module'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { HF_BASE_DELAY_MS, resolveHfEnvMaxAttempts, resolveHfEnvTimeoutMs } from './hf-env.js'; +import { + EMBEDDING_SIDECAR_DIED_LEAD, + getLocalEmbeddingRuntimeBlocker, + LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD, +} from './runtime-support.js'; +import type { EmbeddingConfig, ModelProgress } from './types.js'; +import type { + EmbeddingSidecarDevice, + SidecarRequest, + SidecarRequestBody, + SidecarResponse, +} from './embedding-sidecar-protocol.js'; +import { logger } from '../logger.js'; + +export type ForkImpl = ( + modulePath: string, + args: readonly string[], + options: ForkOptions, +) => ChildProcess; + +const DEFAULT_EMBED_STALL_MS = 3 * 60 * 1000; +const MAX_RECREATES = 1; + +let forkImpl: ForkImpl = fork; +let child: ChildProcess | null = null; +let ready = false; +let nextId = 1; +let recreatesUsed = 0; +let deathSeen = false; +let localUnavailable = false; +let device: EmbeddingSidecarDevice = 'cpu'; +let ensureChain: Promise | null = null; +let lastInitOptions: + | { + embeddingConfig?: Partial; + forceDevice?: EmbeddingSidecarDevice; + } + | undefined; +let exitHooked = false; +let progressSink: ((progress: ModelProgress) => void) | undefined; + +const pending = new Map< + number, + { + resolve: (value: SidecarResponse) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + } +>(); + +export const _setForkForTests = (impl: ForkImpl | null): void => { + forkImpl = impl ?? fork; +}; + +export const _resetEmbeddingSidecarForTests = (): void => { + reapEmbeddingSidecar(); + recreatesUsed = 0; + deathSeen = false; + localUnavailable = false; + ready = false; + nextId = 1; + device = 'cpu'; + ensureChain = null; + lastInitOptions = undefined; +}; + +const sidecarScriptPath = (): string => { + const callerPath = fileURLToPath(import.meta.url); + const isDev = callerPath.endsWith('.ts'); + const file = isDev ? 'embedding-sidecar.ts' : 'embedding-sidecar.js'; + return path.join(path.dirname(callerPath), file); +}; + +const tsxHookArgs = (): string[] => { + const callerPath = fileURLToPath(import.meta.url); + if (!callerPath.endsWith('.ts')) return []; + const require = createRequire(import.meta.url); + return ['--import', pathToFileURL(require.resolve('tsx/esm')).href]; +}; + +const childEnv = (): NodeJS.ProcessEnv => { + const env = { ...process.env }; + delete env.GITNEXUS_EMBEDDING_URL; + return env; +}; + +/** Parent timer starts before `child.send()`; child starts each download timeout after IPC. */ +export const SIDECAR_INIT_IPC_SLACK_MS = 10_000; + +export const sidecarInitTimeoutMs = (): number => { + const perAttempt = resolveHfEnvTimeoutMs(); + const attempts = resolveHfEnvMaxAttempts(); + // Child retries use exponential waits between attempts (HF_BASE_DELAY_MS * 2^i). + const backoffMs = attempts > 1 ? HF_BASE_DELAY_MS * (2 ** (attempts - 1) - 1) : 0; + return perAttempt * attempts + backoffMs + SIDECAR_INIT_IPC_SLACK_MS; +}; + +export const sidecarEmbedTimeoutMs = (): number => { + const raw = Number(process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS); + if (Number.isFinite(raw) && raw > 0) return raw; + return DEFAULT_EMBED_STALL_MS; +}; + +export class EmbeddingSidecarDeadError extends Error { + readonly code: number | null; + readonly signal: NodeJS.Signals | null; + + constructor(code: number | null, signal: NodeJS.Signals | null) { + const detail = signal ? `signal ${signal}` : `exit ${code ?? 'unknown'}`; + super(`${EMBEDDING_SIDECAR_DIED_LEAD} (${detail})`); + this.name = 'EmbeddingSidecarDeadError'; + this.code = code; + this.signal = signal; + } +} + +const NATIVE_ABORT_SIGNALS = new Set(['SIGSEGV', 'SIGABRT', 'SIGBUS', 'SIGILL']); + +const localUnavailableError = (): Error => new Error(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD); + +const noteChildDeath = (signal?: NodeJS.Signals | null): void => { + deathSeen = true; + if (signal && NATIVE_ABORT_SIGNALS.has(signal)) { + localUnavailable = true; + } +}; + +const rejectAll = (error: Error): void => { + const waiters = [...pending.values()]; + for (const waiter of waiters) { + waiter.reject(error); + } +}; + +const attachChild = (proc: ChildProcess): void => { + proc.stderr?.on('data', (chunk: Buffer | string) => { + logger.debug({ sidecar: true }, String(chunk).trimEnd()); + }); + proc.on('message', (msg: SidecarResponse) => { + if (msg.type === 'progress') { + progressSink?.({ + status: msg.status, + progress: msg.progress, + }); + return; + } + const waiter = pending.get(msg.id); + if (!waiter) return; + waiter.resolve(msg); + }); + proc.on('close', (code, signal) => { + if (child !== proc) return; + noteChildDeath(signal); + child = null; + ready = false; + rejectAll(new EmbeddingSidecarDeadError(code, signal)); + }); + proc.on('error', (err) => { + if (child !== proc) return; + noteChildDeath(null); + child = null; + ready = false; + rejectAll(err instanceof Error ? err : new Error(String(err))); + }); +}; + +const request = ( + msg: SidecarRequestBody, + timeoutMs: number, + signal?: AbortSignal, +): Promise => { + if (!child) return Promise.reject(new Error('Embedding sidecar is not running')); + try { + signal?.throwIfAborted(); + } catch (err) { + return Promise.reject(err instanceof Error ? err : new Error(String(err))); + } + const id = nextId++; + return new Promise((resolve, reject) => { + let settled = false; + const onAbort = (): void => { + finish(() => { + try { + signal!.throwIfAborted(); + } catch (err) { + reject(err instanceof Error ? err : new Error(String(err))); + } + }); + }; + const finish = (fn: () => void): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + pending.delete(id); + signal?.removeEventListener('abort', onAbort); + fn(); + }; + const timer = setTimeout(() => { + finish(() => { + child?.kill('SIGKILL'); + reject(new Error(`Embedding sidecar request timed out after ${timeoutMs}ms (${msg.type})`)); + }); + }, timeoutMs); + pending.set(id, { + resolve: (value) => finish(() => resolve(value)), + reject: (error) => finish(() => reject(error)), + timer, + }); + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) { + onAbort(); + return; + } + try { + child!.send({ ...msg, id } as SidecarRequest); + } catch (err) { + finish(() => { + reject(err instanceof Error ? err : new Error(String(err))); + }); + } + }); +}; + +const spawnSidecar = (): ChildProcess => { + const proc = forkImpl(sidecarScriptPath(), [], { + execArgv: tsxHookArgs(), + stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + env: childEnv(), + }); + if (!exitHooked) { + exitHooked = true; + process.on('exit', () => { + reapEmbeddingSidecar(); + }); + } + return proc; +}; + +export const reapEmbeddingSidecar = (): void => { + if (!child) return; + const proc = child; + child = null; + ready = false; + rejectAll(new Error('Embedding sidecar reaped')); + try { + proc.kill('SIGKILL'); + } catch { + // already gone + } +}; + +/** Reap and wait for the killed child's close/error so an awaited dispose is a real boundary. */ +export const reapEmbeddingSidecarAndWait = async (timeoutMs = 5_000): Promise => { + const proc = child; + if (!proc) return; + const closed = new Promise((resolve) => { + const finish = (): void => resolve(); + proc.once('close', finish); + proc.once('error', finish); + }); + reapEmbeddingSidecar(); + let timer: ReturnType | undefined; + try { + await Promise.race([ + closed, + new Promise((resolve) => { + timer = setTimeout(resolve, timeoutMs); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +}; + +export const getSidecarDevice = (): EmbeddingSidecarDevice | null => (ready ? device : null); + +type EnsureSidecarOptions = { + onProgress?: (progress: ModelProgress) => void; + embeddingConfig?: Partial; + forceDevice?: EmbeddingSidecarDevice; + signal?: AbortSignal; +}; + +const markUnavailableIfBudgetSpent = (): void => { + if (deathSeen && recreatesUsed >= MAX_RECREATES) { + localUnavailable = true; + } +}; + +const spawnAndInit = async (options?: EnsureSidecarOptions): Promise => { + const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); + if (runtimeBlocker) { + throw new Error(runtimeBlocker); + } + markUnavailableIfBudgetSpent(); + if (localUnavailable) throw localUnavailableError(); + + if (deathSeen) { + recreatesUsed += 1; + deathSeen = false; + } + + const persisted = { + embeddingConfig: options?.embeddingConfig ?? lastInitOptions?.embeddingConfig, + forceDevice: options?.forceDevice ?? lastInitOptions?.forceDevice, + }; + + child = spawnSidecar(); + attachChild(child); + progressSink = options?.onProgress; + try { + const response = await request( + { + type: 'init', + embeddingConfig: persisted.embeddingConfig, + forceDevice: persisted.forceDevice, + }, + sidecarInitTimeoutMs(), + ); + if (response.type === 'error') throw new Error(response.message); + if (response.type !== 'ready') { + throw new Error(`Unexpected sidecar response: ${response.type}`); + } + ready = true; + device = response.device; + lastInitOptions = persisted; + } catch (err) { + reapEmbeddingSidecar(); + throw err; + } finally { + progressSink = undefined; + } +}; + +const rejectConflictingForceDevice = (forceDevice?: EmbeddingSidecarDevice): void => { + if (forceDevice && forceDevice !== device) { + throw new Error( + `Embedding sidecar already initialized on ${device}; cannot switch to ${forceDevice}`, + ); + } +}; + +const raceAbort = async (promise: Promise, signal?: AbortSignal): Promise => { + if (!signal) return promise; + try { + signal.throwIfAborted(); + } catch (err) { + return Promise.reject(err instanceof Error ? err : new Error(String(err))); + } + return new Promise((resolve, reject) => { + const onAbort = (): void => { + signal.removeEventListener('abort', onAbort); + try { + signal.throwIfAborted(); + } catch (err) { + reject(err instanceof Error ? err : new Error(String(err))); + } + }; + signal.addEventListener('abort', onAbort, { once: true }); + promise.then( + (value) => { + signal.removeEventListener('abort', onAbort); + resolve(value); + }, + (err) => { + signal.removeEventListener('abort', onAbort); + reject(err); + }, + ); + }); +}; + +export const ensureEmbeddingSidecar = async ( + options?: EnsureSidecarOptions, +): Promise<{ device: EmbeddingSidecarDevice }> => { + if (localUnavailable) throw localUnavailableError(); + if (ready && child) { + rejectConflictingForceDevice(options?.forceDevice); + return { device }; + } + + if (!ensureChain) { + const { signal: _ignored, ...initOptions } = options ?? {}; + ensureChain = spawnAndInit(initOptions).finally(() => { + ensureChain = null; + }); + } + await raceAbort(ensureChain, options?.signal); + rejectConflictingForceDevice(options?.forceDevice); + return { device }; +}; + +const vectorsFromEmbedResponse = (response: SidecarResponse): Float32Array[] => { + if (response.type === 'error') throw new Error(response.message); + if (response.type !== 'vectors') { + throw new Error(`Unexpected sidecar response: ${response.type}`); + } + return response.vectors.map((row) => Float32Array.from(row)); +}; + +export const sidecarEmbedBatch = async ( + texts: string[], + options?: { signal?: AbortSignal }, +): Promise => { + if (texts.length === 0) return []; + options?.signal?.throwIfAborted(); + if (localUnavailable) throw localUnavailableError(); + + if (!ready || !child) { + await ensureEmbeddingSidecar({ signal: options?.signal }); + } + + try { + return vectorsFromEmbedResponse( + await request({ type: 'embed', texts }, sidecarEmbedTimeoutMs(), options?.signal), + ); + } catch (err) { + if (!(err instanceof EmbeddingSidecarDeadError)) throw err; + await ensureEmbeddingSidecar({ signal: options?.signal }); + return vectorsFromEmbedResponse( + await request({ type: 'embed', texts }, sidecarEmbedTimeoutMs(), options?.signal), + ); + } +}; diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-protocol.ts b/gitnexus/src/core/embeddings/embedding-sidecar-protocol.ts new file mode 100644 index 000000000..ffcfd471e --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-sidecar-protocol.ts @@ -0,0 +1,29 @@ +/** + * Embedding sidecar IPC types. + * + * Declarations only — consumers `import type` so this leaf never pulls the + * local ONNX init path or the parent façade. Mirrors `analyze-worker-protocol.ts`. + */ + +import type { EmbeddingConfig, ModelProgress } from './types.js'; + +export type EmbeddingSidecarDevice = 'dml' | 'cuda' | 'cpu' | 'wasm'; + +export type SidecarRequest = + | { + id: number; + type: 'init'; + embeddingConfig?: Partial; + forceDevice?: EmbeddingSidecarDevice; + } + | { id: number; type: 'embed'; texts: string[] }; + +type DistributiveOmit = T extends unknown ? Omit : never; + +export type SidecarRequestBody = DistributiveOmit; + +export type SidecarResponse = + | { id: number; type: 'ready'; device: EmbeddingSidecarDevice } + | { id: number; type: 'progress'; progress: number; status: ModelProgress['status'] } + | { id: number; type: 'vectors'; vectors: number[][] } + | { id: number; type: 'error'; message: string }; diff --git a/gitnexus/src/core/embeddings/embedding-sidecar-reap.ts b/gitnexus/src/core/embeddings/embedding-sidecar-reap.ts new file mode 100644 index 000000000..c4ce88c1b --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-sidecar-reap.ts @@ -0,0 +1,12 @@ +/** + * Best-effort sidecar reap for shutdown paths that must not hide the + * caller's error, and that must not statically import the sidecar client. + */ +export const reapEmbeddingSidecarSafely = async (): Promise => { + try { + const { reapEmbeddingSidecar } = await import('./embedding-sidecar-client.js'); + reapEmbeddingSidecar(); + } catch { + // Reap failure must not hide a pipeline error. + } +}; diff --git a/gitnexus/src/core/embeddings/embedding-sidecar.ts b/gitnexus/src/core/embeddings/embedding-sidecar.ts new file mode 100644 index 000000000..70f3aa936 --- /dev/null +++ b/gitnexus/src/core/embeddings/embedding-sidecar.ts @@ -0,0 +1,59 @@ +/** + * Embedding sidecar entry — `child_process.fork()` target. + * + * Loads the local ONNX stack in this process only and returns vectors over IPC. + * The parent keeps Ladybug writes. Do not import the parent façade from here. + */ + +import { getCurrentDevice, initLocalEmbedder, localEmbedBatch } from './embedding-local-init.js'; +import type { SidecarRequest, SidecarResponse } from './embedding-sidecar-protocol.js'; + +const send = (msg: SidecarResponse): void => { + process.send?.(msg); +}; + +process.on('message', (msg: SidecarRequest) => { + void handle(msg); +}); + +async function handle(msg: SidecarRequest): Promise { + try { + switch (msg.type) { + case 'init': { + await initLocalEmbedder( + (progress) => { + send({ + id: msg.id, + type: 'progress', + progress: progress.progress ?? 0, + status: progress.status, + }); + }, + msg.embeddingConfig, + msg.forceDevice, + ); + send({ + id: msg.id, + type: 'ready', + device: getCurrentDevice() ?? 'cpu', + }); + return; + } + case 'embed': { + const vectors = await localEmbedBatch(msg.texts); + send({ + id: msg.id, + type: 'vectors', + vectors: vectors.map((row) => Array.from(row)), + }); + return; + } + } + } catch (err) { + send({ + id: msg.id, + type: 'error', + message: err instanceof Error ? err.message : String(err), + }); + } +} diff --git a/gitnexus/src/core/embeddings/hf-env.ts b/gitnexus/src/core/embeddings/hf-env.ts index 5ae6d89ae..266eafaf8 100644 --- a/gitnexus/src/core/embeddings/hf-env.ts +++ b/gitnexus/src/core/embeddings/hf-env.ts @@ -22,6 +22,22 @@ export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000; /** Upper bound clamped on the env-override attempt count. */ export const HF_MAX_ATTEMPTS_CAP = 10; +/** Per-attempt timeout from `HF_DOWNLOAD_TIMEOUT_MS`, else the built-in default. */ +export const resolveHfEnvTimeoutMs = (): number => { + const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS); + return Number.isFinite(envTimeout) && envTimeout > 0 + ? Math.min(envTimeout, HF_MAX_TIMEOUT_MS) + : HF_DOWNLOAD_TIMEOUT_MS; +}; + +/** Attempt count from `HF_MAX_ATTEMPTS` (finite values that floor to ≥1), else the default. */ +export const resolveHfEnvMaxAttempts = (): number => { + const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS); + if (!Number.isFinite(envMaxAttempts)) return HF_MAX_ATTEMPTS; + const attempts = Math.floor(envMaxAttempts); + return attempts >= 1 ? Math.min(attempts, HF_MAX_ATTEMPTS_CAP) : HF_MAX_ATTEMPTS; +}; + /** * @internal Exported only for unit tests and the two embedder entry points * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the @@ -222,22 +238,12 @@ export async function withHfDownloadRetry( // Upper bounds are clamped to prevent accidental runaway configuration: // - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min) // - maxAttempts is floored (fractional values → integer) and capped at - // HF_MAX_ATTEMPTS_CAP (10). Values ≤ 0, NaN, or Infinity fall back to - // the built-in defaults. - const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS); - const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS); - const resolvedTimeout = - Number.isFinite(envTimeout) && envTimeout > 0 - ? Math.min(envTimeout, HF_MAX_TIMEOUT_MS) - : HF_DOWNLOAD_TIMEOUT_MS; - const resolvedMaxAttempts = - Number.isFinite(envMaxAttempts) && envMaxAttempts > 0 - ? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP) - : HF_MAX_ATTEMPTS; + // HF_MAX_ATTEMPTS_CAP (10). Values that floor below 1, NaN, or Infinity + // fall back to the built-in defaults. const { - maxAttempts = resolvedMaxAttempts, + maxAttempts = resolveHfEnvMaxAttempts(), baseDelayMs = HF_BASE_DELAY_MS, - timeoutMs = resolvedTimeout, + timeoutMs = resolveHfEnvTimeoutMs(), circuit = hfDownloadCircuit, onRetry, } = options; diff --git a/gitnexus/src/core/embeddings/runtime-install.ts b/gitnexus/src/core/embeddings/runtime-install.ts index e52274746..c75653401 100644 --- a/gitnexus/src/core/embeddings/runtime-install.ts +++ b/gitnexus/src/core/embeddings/runtime-install.ts @@ -1,25 +1,22 @@ /** - * On-demand install of the optional local embedding stack (#2370). + * On-demand install of the local embedding stack. * - * `@huggingface/transformers` and `onnxruntime-node` are optionalDependencies: - * npm prunes them (instead of failing the whole install) when - * `onnxruntime-node`'s postinstall cannot download its CUDA binaries from - * api.nuget.org — common behind HTTP proxies and regional firewalls, where - * that download ignores standard proxy env vars and 302 redirects. + * This overrides #2370: `@huggingface/transformers` and `onnxruntime-node` are + * no longer optionalDependencies. Default `npm install` does not fetch them. + * Operators opt in with `gitnexus embeddings install` (or analyze/sync auto-heal), + * which writes a prefix `package.json` (pins + overrides) and installs into + * `~/.gitnexus/embedding-runtime`. Version pins live in package.json + * `gitnexusEmbeddingStack` so they ship in the tarball without installing the + * packages. `onnxruntime-common` stays a regular dependency (#307). * - * This module heals such an install without a reinstall: it fetches the stack - * into a user-level runtime prefix (`~/.gitnexus/embedding-runtime`) straight - * from the user's configured npm registry — honouring their mirror and proxy - * settings, the part of their network setup that demonstrably works — with - * `--ignore-scripts`, so no NuGet download is attempted at all. The CPU ONNX - * binding ships inside the npm tarball; only CUDA GPU acceleration needs the - * postinstall, and `installEmbeddingRuntime({ cuda: true })` opts into it. - * - * Resolution is package-first: a normally-installed stack always wins, and the - * runtime prefix is only consulted when the bare specifier does not resolve. + * Resolution is still package-first: a leftover 1.6.12 tree that already has + * the packages in gitnexus `node_modules` wins until a clean reinstall. + * `--force` refreshes prefix overrides; it does not remove a leftover + * package-first tree. */ import { createRequire } from 'node:module'; import { spawn, execFileSync, type ChildProcess } from 'node:child_process'; +import { mkdirSync, writeFileSync } from 'node:fs'; import { pathToFileURL } from 'node:url'; import { homedir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -90,22 +87,42 @@ export const getEmbeddingRuntimeDir = (): string => { return override ? resolve(override) : join(homedir(), '.gitnexus', 'embedding-runtime'); }; +const EMBEDDING_STACK_NAMES = ['@huggingface/transformers', 'onnxruntime-node'] as const; + /** - * The version specs to install — read from gitnexus' own package.json - * `optionalDependencies` so the on-demand install can never drift from what a - * normal install would have provided. (The manifest ships in the tarball even - * when npm pruned the packages themselves.) + * The version specs to install — read from `gitnexusEmbeddingStack` on + * gitnexus' package.json so the on-demand prefix cannot drift from the + * committed pins after the packages left optionalDependencies. */ export const getEmbeddingStackSpecs = (): Record => { const manifest = require('../../../package.json') as { - optionalDependencies?: Record; + gitnexusEmbeddingStack?: Record; }; - const optional = manifest.optionalDependencies ?? {}; - return Object.fromEntries( - ['@huggingface/transformers', 'onnxruntime-node'] - .filter((name) => optional[name] !== undefined) - .map((name) => [name, optional[name]]), - ); + const stack = manifest.gitnexusEmbeddingStack ?? {}; + const missing = EMBEDDING_STACK_NAMES.filter((name) => !stack[name]); + if (missing.length > 0) { + throw new Error(`package.json gitnexusEmbeddingStack is missing: ${missing.join(', ')}`); + } + return Object.fromEntries(EMBEDDING_STACK_NAMES.map((name) => [name, stack[name]])); +}; + +export const writeEmbeddingRuntimePrefixManifest = (): void => { + const dir = getEmbeddingRuntimeDir(); + mkdirSync(dir, { recursive: true }); + const specs = getEmbeddingStackSpecs(); + const manifest = { + name: 'gitnexus-embedding-runtime', + private: true, + dependencies: specs, + overrides: { + 'adm-zip': '>=0.6.0', + sharp: '>=0.35.0', + '@huggingface/transformers': { + 'onnxruntime-node': specs['onnxruntime-node'], + }, + }, + }; + writeFileSync(join(dir, 'package.json'), `${JSON.stringify(manifest, null, 2)}\n`); }; export interface EmbeddingRuntimeResolution { @@ -125,7 +142,15 @@ export interface EmbeddingRuntimeResolution { export const isPrefixRuntimeLoadable = (): boolean => typeof getRegisterHooks() === 'function'; /** Resolution anchored inside the runtime prefix (`/node_modules`). */ -const prefixRequire = () => createRequire(join(getEmbeddingRuntimeDir(), 'noop.js')); +let cachedPrefixRequire: { dir: string; req: ReturnType } | null = null; + +const prefixRequire = (): ReturnType => { + const dir = getEmbeddingRuntimeDir(); + if (cachedPrefixRequire?.dir === dir) return cachedPrefixRequire.req; + const req = createRequire(join(dir, 'noop.js')); + cachedPrefixRequire = { dir, req }; + return req; +}; /** * True when BOTH load-bearing stack packages resolve from `req`. Probing @@ -349,6 +374,7 @@ export const installEmbeddingRuntime = async ( opts: EmbeddingInstallOptions = {}, timeoutMs: number = getEmbeddingInstallTimeoutMs(), ): Promise => { + writeEmbeddingRuntimePrefixManifest(); const { args, env } = buildEmbeddingInstallCommand(opts); await new Promise((resolve, reject) => { // Windows `npm` is a `.cmd` shim, so the spawn must go through a shell. diff --git a/gitnexus/src/core/embeddings/runtime-support.ts b/gitnexus/src/core/embeddings/runtime-support.ts index 11c4c98dd..ac691ee16 100644 --- a/gitnexus/src/core/embeddings/runtime-support.ts +++ b/gitnexus/src/core/embeddings/runtime-support.ts @@ -17,7 +17,7 @@ * (The runtime-install import below only resolves paths — it never loads the * embedding stack.) */ -import { resolveEmbeddingRuntime } from './runtime-install.js'; +import { isPrefixRuntimeLoadable, resolveEmbeddingRuntime } from './runtime-install.js'; /** * Stable lead line of the macOS-Intel blocker message. Also used to recognise @@ -63,8 +63,9 @@ export const getLocalEmbeddingRuntimeBlocker = ( ' - Run analyze without --embeddings (all other indexing still works).', ' - Point GITNEXUS_EMBEDDING_URL (with GITNEXUS_EMBEDDING_MODEL) at an', ' OpenAI-compatible /v1/embeddings endpoint to embed over HTTP.', - ' - Run GitNexus on Linux or in Docker, where the native binding ships.', - ' - Run GitNexus on Apple Silicon (darwin/arm64), which ships a binding.', + ' - Run GitNexus on Linux or Apple Silicon (darwin/arm64), then', + ' `gitnexus embeddings install`. Official CLI Docker images no longer', + ' ship onnxruntime-node; bind-mount a prefix or use HTTP.', ' - Use a future GitNexus build that restores darwin/x64 ONNX support.', ].join('\n'); } @@ -87,37 +88,32 @@ export const isLocalEmbeddingRuntimeBlockerMessage = (message: string): boolean * line (see {@link isMissingLocalEmbeddingStackMessage}). */ const LOCAL_EMBEDDING_STACK_MISSING_LEAD = - 'Local semantic embeddings are unavailable: the optional embedding stack is not installed.'; + 'Local semantic embeddings are unavailable: the local embedding stack is not installed.'; /** - * The full guidance shown when the optional local embedding stack - * (`@huggingface/transformers` → `onnxruntime-node`) is missing at runtime. - * - * Both packages are `optionalDependencies` (#2370): `onnxruntime-node`'s - * postinstall downloads CUDA support binaries from api.nuget.org, which fails - * behind HTTP proxies and regional firewalls (its `global-agent` proxy layer - * ignores the standard HTTP_PROXY/HTTPS_PROXY vars and rejects 302 redirects). - * npm then skips the optional subtree instead of failing the whole install — - * every GitNexus feature except local embeddings keeps working. + * Guidance when transformers / onnxruntime-node are not resolvable. + * Default npm install no longer fetches those packages. Primary heal is + * `gitnexus embeddings install`. A leftover 1.6.12 package-first tree in + * gitnexus node_modules is residual until a clean reinstall; `--force` + * only refreshes the prefix overrides. */ export const localEmbeddingStackMissingMessage = (): string => [ LOCAL_EMBEDDING_STACK_MISSING_LEAD, - 'npm skipped the optional packages @huggingface/transformers / onnxruntime-node', - "during install — usually because onnxruntime-node's postinstall could not", - 'download its CUDA support binaries from api.nuget.org (common behind HTTP', - 'proxies and regional firewalls, #2370). Everything except local embeddings', - 'still works.', + '@huggingface/transformers and onnxruntime-node are not part of a default', + 'gitnexus install. Everything except local embeddings still works.', '', 'To enable local embeddings:', - ' - Run `gitnexus embeddings install` — fetches the stack on demand through', - ' your npm registry config (mirrors and proxies apply; no NuGet download).', - ' `gitnexus analyze --embeddings` does this automatically.', + ' - Run `gitnexus embeddings install` — fetches the stack through your npm', + ' registry config into ~/.gitnexus/embedding-runtime (or', + ' GITNEXUS_EMBEDDING_RUNTIME_DIR when set; mirrors and proxies', + ' apply; no NuGet download). `gitnexus analyze --embeddings` and', + ' `gitnexus embeddings sync` do this automatically.', ' Add --cuda on CUDA GPU hosts (behind a proxy, also set', ' GLOBAL_AGENT_HTTPS_PROXY= for the NuGet download).', - ' - Or reinstall with the CUDA download skipped (CPU embeddings need no CUDA):', - ' ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus', - ' (Windows: set ONNXRUNTIME_NODE_INSTALL=skip && npm install -g gitnexus)', + ' - A leftover 1.6.12 install that still has those packages under', + ' gitnexus node_modules is residual. `--force` only refreshes prefix', + ' overrides; remove leftover packages with a clean reinstall.', ' - Or point GITNEXUS_EMBEDDING_URL (with GITNEXUS_EMBEDDING_MODEL) at an', ' OpenAI-compatible /v1/embeddings endpoint to embed over HTTP.', ].join('\n'); @@ -140,13 +136,33 @@ export const localEmbeddingPrefixUnloadableMessage = (): string => [ LOCAL_EMBEDDING_PREFIX_UNLOADABLE_LEAD, 'The runtime prefix loads via module.registerHooks, which needs Node', - '>= 22.15 (on the 22.x line) or >= 23.5 (on the 23.x line). Either:', - ' - Upgrade Node to a build that has module.registerHooks, or', - ' - Reinstall the packages normally (works on every supported Node):', - ' ONNXRUNTIME_NODE_INSTALL=skip npm install -g gitnexus', - ' (Windows: set ONNXRUNTIME_NODE_INSTALL=skip && npm install -g gitnexus)', + '>= 22.15 (on the 22.x line) or >= 23.5 (on the 23.x line). Upgrade this', + 'Node to a build that has module.registerHooks. Installing the prefix from', + 'another Node cannot add that API here. A leftover 1.6.12 package-first', + 'tree still loads without the hook; the prefix path does not.', ].join('\n'); +export type LocalEmbeddingRuntimeAssessment = + | { status: 'blocked'; message: string } + | { status: 'prefix-unloadable'; message: string } + | { status: 'needs-install' } + | { status: 'ready' }; + +/** + * Shared local-runtime preflight for analyze and embeddings-sync. + * Callers keep their own error routing (CLI vs thrown Error). + */ +export const assessLocalEmbeddingRuntime = (): LocalEmbeddingRuntimeAssessment => { + const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); + if (runtimeBlocker) return { status: 'blocked', message: runtimeBlocker }; + const resolved = resolveEmbeddingRuntime(); + if (!isPrefixRuntimeLoadable() && (resolved === null || resolved.source === 'runtime-prefix')) { + return { status: 'prefix-unloadable', message: localEmbeddingPrefixUnloadableMessage() }; + } + if (resolved === null) return { status: 'needs-install' }; + return { status: 'ready' }; +}; + /** Module specifiers whose absence means the optional embedding stack was pruned. */ const EMBEDDING_STACK_SPECIFIERS = ['@huggingface/transformers', 'onnxruntime-node'] as const; @@ -176,11 +192,26 @@ export const getMissingLocalEmbeddingStackMessage = (err: unknown): string | nul export const isMissingLocalEmbeddingStackMessage = (message: string): boolean => message.includes(LOCAL_EMBEDDING_STACK_MISSING_LEAD); +/** Lead line when the embedding sidecar has been marked permanently unavailable. */ +export const LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD = + 'Local embeddings are unavailable after the sidecar aborted'; + +/** Lead of `EmbeddingSidecarDeadError` — native abort / unexpected child exit. */ +export const EMBEDDING_SIDECAR_DIED_LEAD = 'Embedding sidecar died'; + +/** + * True when `message` is a sidecar-abort or sidecar-dead error. MCP `query()` + * must treat these like a missing stack so agents see the degradation. + */ +export const isLocalEmbeddingSidecarAbortMessage = (message: string): boolean => + message.includes(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD) || + message.includes(EMBEDDING_SIDECAR_DIED_LEAD); + /** * True when the optional local embedding stack resolves from this install — * either the normally-installed packages or the on-demand runtime prefix. * Resolution only — nothing is imported, so this is safe on every platform * (including macOS Intel, where *loading* onnxruntime-node would crash). - * Used by `doctor` to surface a pruned optional install (#2370) up front. + * Used by `doctor` to surface a missing local stack (default install excludes it). */ export const isLocalEmbeddingStackInstalled = (): boolean => resolveEmbeddingRuntime() !== null; diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index 759e19812..b4e13ab54 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -318,6 +318,8 @@ export interface ModelProgress { total?: number; } +export type ModelProgressCallback = (progress: ModelProgress) => void; + export interface ChunkSearchRow { nodeId: string; chunkIndex: number; diff --git a/gitnexus/src/core/search/hybrid-search.ts b/gitnexus/src/core/search/hybrid-search.ts index 7c09971dc..9b76101ac 100644 --- a/gitnexus/src/core/search/hybrid-search.ts +++ b/gitnexus/src/core/search/hybrid-search.ts @@ -177,13 +177,16 @@ export const hybridSearch = async ( // index recorded an explicit FTS opt-out (`disabledReason`, #3091). // If FTS fails (e.g. extension not loaded in MCP process), fall back to // semantic-only search instead of crashing with "bm25Results is not iterable". - let bm25Results: BM25SearchResult[] = []; - try { - const ftsResponse = await searchFTSFromLbug(query, limit, undefined, disabledReason); - bm25Results = ftsResponse?.results ?? []; - } catch { - // FTS unavailable — continue with semantic-only search - } - const semanticResults = await semanticSearch(executeQuery, query, limit); + const [bm25Results, semanticResults] = await Promise.all([ + (async (): Promise => { + try { + const ftsResponse = await searchFTSFromLbug(query, limit, undefined, disabledReason); + return ftsResponse?.results ?? []; + } catch { + return []; + } + })(), + semanticSearch(executeQuery, query, limit).catch(() => []), + ]); return mergeWithRRF(bm25Results, semanticResults, limit); }; diff --git a/gitnexus/src/mcp/core/embedder.ts b/gitnexus/src/mcp/core/embedder.ts index 663d74b77..efdadb4e8 100644 --- a/gitnexus/src/mcp/core/embedder.ts +++ b/gitnexus/src/mcp/core/embedder.ts @@ -1,217 +1,36 @@ /** - * Embedder Module (Read-Only) + * MCP embedder façade — HTTP or the shared embedding sidecar client. * - * Singleton factory for transformers.js embedding pipeline. - * For MCP, we only need to compute query embeddings, not batch embed. + * Local ONNX inference lives in the sidecar child. This module must not import + * the Hugging Face transformers package or the ONNX Node binding. */ -// Type-only import: erased at compile time so loading this module never pulls -// in @huggingface/transformers (and its native onnxruntime-node binding) at -// runtime. The runtime values (pipeline, env) are dynamically imported inside -// initEmbedder, after the platform guard has passed (#1515). -import type { FeatureExtractionPipeline } from '@huggingface/transformers'; import { - isHttpMode, - getHttpDimensions, - httpEmbedQuery, -} from '../../core/embeddings/http-client.js'; -import { resolveEmbeddingConfig } from '../../core/embeddings/config.js'; -import { - applyHfEnvOverrides, - isHfDownloadFailure, - withHfDownloadRetry, -} from '../../core/embeddings/hf-env.js'; -import { - getLocalEmbeddingRuntimeBlocker, - getMissingLocalEmbeddingStackMessage, -} from '../../core/embeddings/runtime-support.js'; -import { ensureOnnxRuntimeCommonResolvable } from '../../core/embeddings/onnxruntime-common-resolver.js'; -import { ensureEmbeddingStackResolvable } from '../../core/embeddings/runtime-install.js'; -import { ensureOnnxRuntimeNodeMatchesSystem } from '../../core/embeddings/onnxruntime-node-resolver.js'; -import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js'; + disposeEmbedder as disposeCoreEmbedder, + embedText, + embeddingToArray, + getEmbeddingDimensions, + initEmbedder as initCoreEmbedder, + isEmbedderReady as isCoreEmbedderReady, +} from '../../core/embeddings/embedder.js'; +import { httpEmbedQuery, isHttpMode } from '../../core/embeddings/http-client.js'; -import { logger } from '../../core/logger.js'; -// Model config -const MODEL_ID = 'Snowflake/snowflake-arctic-embed-xs'; +export const initEmbedder = initCoreEmbedder; -// Module-level state for singleton pattern -let embedderInstance: FeatureExtractionPipeline | null = null; -let isInitializing = false; -let initPromise: Promise | null = null; +export const isEmbedderReady = isCoreEmbedderReady; -/** - * Initialize the embedding model (lazy, on first search) - */ -export const initEmbedder = async (): Promise => { - if (isHttpMode()) { - throw new Error('initEmbedder() should not be called in HTTP mode.'); - } - - // Fail fast on platforms where the bundled native ONNX Runtime binding is not - // shipped (macOS Intel, #1515). Must run before any transformers.js / - // onnxruntime-node import or resolution — otherwise the native module load - // crashes with a raw "Cannot find module ...onnxruntime_binding.node" that - // ONNX_WEB_BACKEND=wasm cannot rescue (#1516). - const runtimeBlocker = getLocalEmbeddingRuntimeBlocker(); - if (runtimeBlocker) { - throw new Error(runtimeBlocker); - } - - if (embedderInstance) { - return embedderInstance; - } - - if (isInitializing && initPromise) { - return initPromise; - } - - isInitializing = true; - - initPromise = (async () => { - try { - // Lazy-load transformers.js only after the runtime guard has passed, so - // unsupported platforms never reach the native ONNX import (#1515). - // Registered FIRST so it sits last in the hook chain (registerHooks runs - // the most recent hook first): when the optional stack was pruned at - // install time (#2370), its bare specifiers fall back to the on-demand - // runtime prefix. - ensureEmbeddingStackResolvable(); - // Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common` - // import is unresolvable; register the fallback resolver first (#307). - ensureOnnxRuntimeCommonResolvable(); - // Registered AFTER the common fallback so this hook resolves FIRST (Node - // runs the most-recently-registered hook first): on CUDA-13 hosts it - // redirects onnxruntime-node to the system-matched build before - // transformers imports it. No-op on matching layouts, non-CUDA, - // Windows/DirectML, and macOS. Mirrors the core embedder's call site so - // MCP query-time embedding gets the same CUDA-13 fix. - ensureOnnxRuntimeNodeMatchesSystem(); - // The stack is an optionalDependency: npm prunes it when onnxruntime-node's - // postinstall can't reach api.nuget.org (#2370). Rethrow with actionable - // reinstall guidance instead of a raw ERR_MODULE_NOT_FOUND. - const { pipeline, env } = await import('@huggingface/transformers').catch((err: unknown) => { - const missing = getMissingLocalEmbeddingStackMessage(err); - if (missing) throw new Error(missing); - throw err; - }); - - env.allowLocalModels = false; - // Bridge user-controlled env vars to transformers.js: HF_HOME → - // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in - // applyHfEnvOverrides so this MCP entry point behaves identically to - // the analyze pipeline embedder. - applyHfEnvOverrides(env); - const embeddingConfig = resolveEmbeddingConfig(); - - logger.info('GitNexus: Loading embedding model (first search may take a moment)...'); - - const devicesToTry: Array<'dml' | 'cuda' | 'cpu'> = - embeddingConfig.device === 'dml' || embeddingConfig.device === 'cuda' - ? [embeddingConfig.device, 'cpu'] - : ['cpu']; - - for (const device of devicesToTry) { - try { - // Silence stdout and stderr during model load — ONNX Runtime and transformers.js - // may write progress/init messages that corrupt MCP stdio protocol or produce - // noisy warnings (e.g. node assignment to execution providers). - // Use the centralized silenceStdout() to avoid conflicts with pool-adapter's - // own stdout patching (independent patching caused restore-order bugs). - silenceStdout(); - process.stderr.write = (() => true) as any; - try { - embedderInstance = await withHfDownloadRetry(() => - pipeline('feature-extraction', MODEL_ID, { - device: device, - dtype: 'fp32', - session_options: { - logSeverityLevel: 3, - intraOpNumThreads: embeddingConfig.threads, - interOpNumThreads: 1, - executionMode: 'sequential', - }, - }), - ); - } finally { - restoreStdout(); - process.stderr.write = realStderrWrite; - } - logger.info({ device }, 'GitNexus: Embedding model loaded'); - return embedderInstance!; - } catch (deviceError) { - // Network errors and circuit-open errors are not device-specific — - // they will fail the same way on every device. Rethrow immediately - // with actionable HF_ENDPOINT guidance rather than silently falling - // back to the next device. - const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError); - if (isHfDownloadFailure(errMsg)) { - const endpointHint = process.env.HF_ENDPOINT - ? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.` - : `huggingface.co may be unreachable from your network.\n` + - ` Set HF_ENDPOINT to a mirror and retry:\n` + - ` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` + - ` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`; - throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`); - } - if (device === 'cpu') throw new Error('Failed to load embedding model'); - } - } - - throw new Error('No suitable device found'); - } catch (error) { - isInitializing = false; - initPromise = null; - embedderInstance = null; - throw error; - } finally { - isInitializing = false; - } - })(); - - return initPromise; -}; - -/** - * Check if embedder is ready - */ -export const isEmbedderReady = (): boolean => isHttpMode() || embedderInstance !== null; - -/** - * Embed a query text for semantic search - */ export const embedQuery = async (query: string): Promise => { if (isHttpMode()) { return httpEmbedQuery(query); } - - const embedder = await initEmbedder(); - - const result = await embedder(query, { - pooling: 'mean', - normalize: true, - }); - - return Array.from(result.data as ArrayLike); + return embeddingToArray(await embedText(query)); }; /** - * Get embedding dimensions + * Query-vector width for CAST. HTTP uses GITNEXUS_EMBEDDING_DIMS when set; + * local is the model default. Do not bind this to schema EMBEDDING_DIMS. */ -export const getEmbeddingDims = (): number => { - return getHttpDimensions() ?? 384; -}; +export const getEmbeddingDims = (): number => getEmbeddingDimensions(); -/** - * Cleanup embedder - */ -export const disposeEmbedder = async (): Promise => { - if (embedderInstance) { - try { - if ('dispose' in embedderInstance && typeof embedderInstance.dispose === 'function') { - await embedderInstance.dispose(); - } - } catch {} - embedderInstance = null; - initPromise = null; - } -}; +/** Reap the sidecar. Never runs ONNX dispose in this process. */ +export const disposeEmbedder = disposeCoreEmbedder; diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index b3486865b..5a3f9748d 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -62,6 +62,7 @@ import { } from '../../core/group/service.js'; import { resolveAtGroupMemberRepoPath } from '../../core/group/resolve-at-member.js'; import { collectBestChunks } from '../../core/embeddings/types.js'; +import { reapEmbeddingSidecarSafely } from '../../core/embeddings/embedding-sidecar-reap.js'; import { DEFAULT_MCP_VECTOR_MAX_DISTANCE, getVectorMaxDistance, @@ -111,6 +112,7 @@ import { import { logger } from '../../core/logger.js'; import { isLocalEmbeddingRuntimeBlockerMessage, + isLocalEmbeddingSidecarAbortMessage, isMissingLocalEmbeddingStackMessage, } from '../../core/embeddings/runtime-support.js'; import { @@ -2962,9 +2964,10 @@ export class LocalBackend { // over a single `current` phase slot. const searchLimit = processLimit * maxSymbolsPerProcess; // fetch enough raw results const ftsDisabledReason = getFtsDisabledReason(meta?.capabilities?.fts); + const vectorDegraded = { reason: undefined as string | undefined }; const [bm25SearchResult, semanticResults] = await Promise.all([ timer.time('bm25', this.bm25Search(repo, searchQuery, searchLimit, ftsDisabledReason)), - timer.time('vector', this.semanticSearch(repo, searchQuery, searchLimit)), + timer.time('vector', this.semanticSearch(repo, searchQuery, searchLimit, vectorDegraded)), ]); // Guard against undefined results (#1489) — when FTS is entirely @@ -3427,6 +3430,9 @@ export class LocalBackend { 'Keyword results are unaffected.', ); } + if (vectorDegraded.reason) { + warnings.push(vectorDegraded.reason); + } if (enrichmentDegraded) { warnings.push( 'Symbol enrichment partially failed — some process/cohesion/content data may be missing from these results (see server logs).', @@ -3569,7 +3575,12 @@ export class LocalBackend { /** * Semantic vector search helper */ - private async semanticSearch(repo: RepoHandle, query: string, limit: number): Promise { + private async semanticSearch( + repo: RepoHandle, + query: string, + limit: number, + degraded?: { reason?: string }, + ): Promise { // Whether THIS call produced a query vector — see `lastQueryEmbeddingDims`. // A local flag, not a re-read of the map: the map may still hold an earlier // call's width, and the catch below must only clear an entry it did not set. @@ -3735,18 +3746,21 @@ export class LocalBackend { // the width IS still the live one). Clearing only in the former case // keeps the recorded width a fact rather than a leftover (#2798). if (embeddedDims === undefined) this.lastQueryEmbeddingDims.delete(repo.lbugPath); - // Embeddings disabled is the common, silent case. But a pruned or - // Node-unloadable optional stack (#2370/#2372) also lands here — surface it + // Embeddings disabled is the common, silent case. But a missing or + // Node-unloadable local stack (#2370/#2372) also lands here — surface it // once so semantic search doesn't silently degrade to BM25 with no hint // (the exact silent-degradation mode #2370 exists to fix). Emitted once per // LocalBackend instance to keep stderr quiet on hot paths (like the VECTOR // fallback above). All other errors stay silent, as before. const message = err instanceof Error ? err.message : ''; - if ( - !this.warnedMissingEmbeddingStack && - (isMissingLocalEmbeddingStackMessage(message) || - isLocalEmbeddingRuntimeBlockerMessage(message)) - ) { + const isDegradedVectorError = + isMissingLocalEmbeddingStackMessage(message) || + isLocalEmbeddingRuntimeBlockerMessage(message) || + isLocalEmbeddingSidecarAbortMessage(message); + if (isDegradedVectorError) { + if (degraded) degraded.reason = message; + } + if (!this.warnedMissingEmbeddingStack && isDegradedVectorError) { this.warnedMissingEmbeddingStack = true; logger.warn(`GitNexus [query:vector]: ${message}`); } @@ -9387,14 +9401,15 @@ export class LocalBackend { } async disconnect(): Promise { - await closeLbug(); // close all connections - // Note: we intentionally do NOT call disposeEmbedder() here. - // ONNX Runtime's native cleanup segfaults on macOS and some Linux configs, - // and importing the embedder module on Node v24+ crashes if onnxruntime - // was never loaded during the session. Since process.exit(0) follows - // immediately after disconnect(), the OS reclaims everything. See #38, #89. - this.repos.clear(); - this.contextCache.clear(); - this.initializedRepos.clear(); + try { + await closeLbug(); // close all connections + } finally { + // Reap even when Ladybug close rejects. Do not run ONNX dispose in this + // process (native dispose can SIGSEGV). The reap helper does not load ONNX. + await reapEmbeddingSidecarSafely(); + this.repos.clear(); + this.contextCache.clear(); + this.initializedRepos.clear(); + } } } diff --git a/gitnexus/src/server/analyze-worker.ts b/gitnexus/src/server/analyze-worker.ts index d41a4b34d..766cf0022 100644 --- a/gitnexus/src/server/analyze-worker.ts +++ b/gitnexus/src/server/analyze-worker.ts @@ -14,6 +14,7 @@ import type { ParentMessage, WorkerMessage } from './analyze-worker-protocol.js'; import { runWorkerAnalysis, createTerminalClaim } from './analyze-worker-core.js'; +import { reapEmbeddingSidecar } from '../core/embeddings/embedding-sidecar-client.js'; type BoundedCheckpointBeforeExit = typeof import('../core/lbug/shutdown-helpers.js').boundedCheckpointBeforeExit; @@ -26,6 +27,11 @@ type BoundedCheckpointBeforeExit = // directly, so nothing else belongs in this list. export type { CompleteMessage, WorkerMessage } from './analyze-worker-protocol.js'; +function exitReapingSidecar(code: number): never { + reapEmbeddingSidecar(); + process.exit(code); +} + function send(msg: WorkerMessage) { // No try/catch: if the IPC channel is gone, process.send throws // (ERR_IPC_CHANNEL_CLOSED) and that failure must NOT be swallowed. Every caller @@ -50,7 +56,7 @@ process.on('uncaughtException', (err: unknown) => { const message = err instanceof Error ? err.message : 'Uncaught exception in worker'; send({ type: 'error', message }); } finally { - setTimeout(() => process.exit(1), 500); + setTimeout(() => exitReapingSidecar(1), 500); } }); @@ -59,7 +65,7 @@ process.on('unhandledRejection', (reason: unknown) => { const message = reason instanceof Error ? reason.message : 'Unhandled rejection in worker'; send({ type: 'error', message }); } finally { - setTimeout(() => process.exit(1), 500); + setTimeout(() => exitReapingSidecar(1), 500); } }); @@ -76,13 +82,13 @@ function requestWorkerCancellation(source: string): void { } if (!started) { // No analysis has started, so no native work needs a safe-point handshake. - process.exit(0); + exitReapingSidecar(0); } } function exitAfterCancellation(): void { if (!boundedCheckpointBeforeExit) { - process.exit(0); + exitReapingSidecar(0); return; } void boundedCheckpointBeforeExit({ @@ -154,6 +160,6 @@ process.on('message', async (msg: ParentMessage) => { if (cancellationRequested) exitAfterCancellation(); // Normal terminal outcomes still need the existing process exit because // LadybugDB stays live. - else setTimeout(() => process.exit(0), 500); + else setTimeout(() => exitReapingSidecar(0), 500); } }); diff --git a/gitnexus/test/unit/analyze-local-embedding-error.test.ts b/gitnexus/test/unit/analyze-local-embedding-error.test.ts index fcdc2a95d..e67f43fe5 100644 --- a/gitnexus/test/unit/analyze-local-embedding-error.test.ts +++ b/gitnexus/test/unit/analyze-local-embedding-error.test.ts @@ -15,6 +15,17 @@ */ import { beforeEach, describe, expect, it, vi } from 'vitest'; import { getLocalEmbeddingRuntimeBlocker } from '../../src/core/embeddings/runtime-support.js'; +import type { LoggerCapture } from '../../src/core/logger.js'; + +async function withCapturedLogger(fn: (cap: LoggerCapture) => Promise): Promise { + const { _captureLogger } = await import('../../src/core/logger.js'); + const cap = _captureLogger(); + try { + return await fn(cap); + } finally { + cap.restore(); + } +} const runFullAnalysisMock = vi.fn(); // Controllable so the dual-match scenario can force the network heuristic to @@ -101,38 +112,32 @@ describe('analyzeCommand local-embedding-runtime error handling', () => { it('routes the blocker to a clean local-embedding-unsupported message (exit 1)', async () => { runFullAnalysisMock.mockRejectedValue(new Error(blockerMessage)); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - await analyzeCommand(undefined, { embeddings: true }); + expect(process.exitCode).toBe(1); - expect(process.exitCode).toBe(1); - - const records = cap.records(); - const blockerRecord = records.find((r) => r.recoveryHint === 'local-embedding-unsupported'); - expect(blockerRecord).toBeDefined(); - expect(typeof blockerRecord?.msg === 'string' && blockerRecord.msg).toMatch(/macOS Intel/); - - cap.restore(); + const records = cap.records(); + const blockerRecord = records.find((r) => r.recoveryHint === 'local-embedding-unsupported'); + expect(blockerRecord).toBeDefined(); + expect(typeof blockerRecord?.msg === 'string' && blockerRecord.msg).toMatch(/macOS Intel/); + }); }); it('does NOT fall through to the module-not-found "installation may be corrupt" hint', async () => { runFullAnalysisMock.mockRejectedValue(new Error(blockerMessage)); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - await analyzeCommand(undefined, { embeddings: true }); - - const records = cap.records(); - const corruptRecord = records.find( - (r) => typeof r.msg === 'string' && r.msg.includes('installation may be corrupt'), - ); - expect(corruptRecord).toBeUndefined(); - - cap.restore(); + const records = cap.records(); + const corruptRecord = records.find( + (r) => typeof r.msg === 'string' && r.msg.includes('installation may be corrupt'), + ); + expect(corruptRecord).toBeUndefined(); + }); }); it('wins over the HF-download branch even when isHfDownloadFailure also matches (R4 ordering)', async () => { @@ -142,20 +147,17 @@ describe('analyzeCommand local-embedding-runtime error handling', () => { isHfDownloadFailureMock.mockReturnValue(true); runFullAnalysisMock.mockRejectedValue(new Error(blockerMessage)); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - await analyzeCommand(undefined, { embeddings: true }); + expect(process.exitCode).toBe(1); - expect(process.exitCode).toBe(1); - - const records = cap.records(); - expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(true); - // The HF-download branch must NOT have fired. - expect(records.some((r) => r.recoveryHint === 'hf-endpoint-unreachable')).toBe(false); - - cap.restore(); + const records = cap.records(); + expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(true); + // The HF-download branch must NOT have fired. + expect(records.some((r) => r.recoveryHint === 'hf-endpoint-unreachable')).toBe(false); + }); }); it('does NOT route unrelated errors through the local-embedding branch', async () => { @@ -163,18 +165,15 @@ describe('analyzeCommand local-embedding-runtime error handling', () => { new Error('Some unexpected failure unrelated to embeddings'), ); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - await analyzeCommand(undefined, { embeddings: true }); + expect(process.exitCode).toBe(1); - expect(process.exitCode).toBe(1); - - const records = cap.records(); - expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(false); - - cap.restore(); + const records = cap.records(); + expect(records.some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe(false); + }); }); }); @@ -190,37 +189,58 @@ describe('analyzeCommand — prefix-runtime capability gate (#2372)', () => { process.env.NODE_OPTIONS = `${process.env.NODE_OPTIONS ?? ''} --max-old-space-size=8192`.trim(); }); + it('does not spawn npm on darwin/x64 even when the stack is missing', async () => { + const orig = { platform: process.platform, arch: process.arch }; + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }); + resolveEmbeddingRuntimeMock.mockReturnValue(null); + isPrefixRuntimeLoadableMock.mockReturnValue(true); + try { + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); + + expect(process.exitCode).toBe(1); + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe( + true, + ); + }); + } finally { + Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true }); + Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true }); + } + }); + it('fails fast without installing when nothing is installed and the prefix is unloadable', async () => { resolveEmbeddingRuntimeMock.mockReturnValue(null); isPrefixRuntimeLoadableMock.mockReturnValue(false); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); - await analyzeCommand(undefined, { embeddings: true }); + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - expect(process.exitCode).toBe(1); - expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); - const record = cap.records().find((r) => r.recoveryHint === 'local-embedding-stack-missing'); - expect(typeof record?.msg === 'string' && record.msg).toMatch(/module\.registerHooks/); - cap.restore(); + expect(process.exitCode).toBe(1); + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + const record = cap.records().find((r) => r.recoveryHint === 'local-embedding-stack-missing'); + expect(typeof record?.msg === 'string' && record.msg).toMatch(/module\.registerHooks/); + }); }); it('fails fast on a resolved-but-unloadable prefix (the previously-uncaught state)', async () => { resolveEmbeddingRuntimeMock.mockReturnValue({ source: 'runtime-prefix' }); isPrefixRuntimeLoadableMock.mockReturnValue(false); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); - await analyzeCommand(undefined, { embeddings: true }); + await withCapturedLogger(async (cap) => { + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - expect(process.exitCode).toBe(1); - expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); - expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-stack-missing')).toBe( - true, - ); - cap.restore(); + expect(process.exitCode).toBe(1); + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-stack-missing')).toBe( + true, + ); + }); }); it('installs with a shorter-than-default timeout when nothing is installed and the prefix is loadable', async () => { @@ -229,18 +249,17 @@ describe('analyzeCommand — prefix-runtime capability gate (#2372)', () => { // Reject afterwards so analyze bails right after the install, isolating the gate. runFullAnalysisMock.mockRejectedValue(new Error('stop after install')); - const { _captureLogger } = await import('../../src/core/logger.js'); - const cap = _captureLogger(); - const { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS } = - await import('../../src/core/embeddings/runtime-install.js'); - const { analyzeCommand } = await import('../../src/cli/analyze.js'); - await analyzeCommand(undefined, { embeddings: true }); + await withCapturedLogger(async () => { + const { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS } = + await import('../../src/core/embeddings/runtime-install.js'); + const { analyzeCommand } = await import('../../src/cli/analyze.js'); + await analyzeCommand(undefined, { embeddings: true }); - expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1); - // analyze must pass the shorter deadline so a blackholed proxy can't stall - // the run for the 10-minute default. - const timeoutArg = installEmbeddingRuntimeMock.mock.calls[0][1] as number; - expect(timeoutArg).toBe(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS); - cap.restore(); + expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1); + // analyze must pass the shorter deadline so a blackholed proxy can't stall + // the run for the 10-minute default. + const timeoutArg = installEmbeddingRuntimeMock.mock.calls[0][1] as number; + expect(timeoutArg).toBe(ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS); + }); }); }); diff --git a/gitnexus/test/unit/analyze-worker-reap.test.ts b/gitnexus/test/unit/analyze-worker-reap.test.ts new file mode 100644 index 000000000..78a89aa95 --- /dev/null +++ b/gitnexus/test/unit/analyze-worker-reap.test.ts @@ -0,0 +1,32 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const reapEmbeddingSidecarMock = vi.hoisted(() => vi.fn()); + +vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ + reapEmbeddingSidecar: () => reapEmbeddingSidecarMock(), +})); + +describe('analyze-worker exitReapingSidecar', () => { + let exitSpy: ReturnType; + + beforeEach(() => { + reapEmbeddingSidecarMock.mockReset(); + exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never); + }); + + afterEach(() => { + exitSpy.mockRestore(); + }); + + it('reaps the sidecar before process.exit on cancel-before-start', async () => { + await import('../../src/server/analyze-worker.js'); + process.emit('message', { type: 'cancel' }); + await vi.waitFor(() => { + expect(reapEmbeddingSidecarMock).toHaveBeenCalled(); + expect(exitSpy).toHaveBeenCalledWith(0); + }); + const reapOrder = reapEmbeddingSidecarMock.mock.invocationCallOrder[0]; + const exitOrder = exitSpy.mock.invocationCallOrder[0]; + expect(reapOrder).toBeLessThan(exitOrder); + }); +}); diff --git a/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts b/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts index 421772281..cc10fd337 100644 --- a/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts +++ b/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts @@ -2,9 +2,10 @@ import { describe, it, expect } from 'vitest'; import { spawnSync } from 'node:child_process'; import { createRequire } from 'node:module'; import { fileURLToPath } from 'node:url'; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; +import { globSync } from 'glob'; /** * Coverage for the publish guard `scripts/assert-publish-grammar-coverage.cjs`. @@ -21,8 +22,12 @@ const requireCjs = createRequire(import.meta.url); const SCRIPT = fileURLToPath( new URL('../../scripts/assert-publish-grammar-coverage.cjs', import.meta.url), ); -const { findCoverageProblems, filesShipsVendorSource, findStrayBuildArtifacts } = - requireCjs(SCRIPT); +const { + findCoverageProblems, + findPackedFilesProblems, + filesShipsVendorSource, + findStrayBuildArtifacts, +} = requireCjs(SCRIPT); describe('findCoverageProblems (pure decision core)', () => { it('passes when source ships, even with incomplete prebuilds (transitional state)', () => { @@ -55,6 +60,67 @@ describe('findCoverageProblems (pure decision core)', () => { }); }); +describe('findPackedFilesProblems (files globs, not on-disk counts)', () => { + const grammars = ['tree-sitter-c', 'tree-sitter-kotlin']; + const lean = [ + 'vendor/**/prebuilds/**', + 'vendor/**/bindings/node/index.js', + 'vendor/**/package.json', + 'vendor/leiden/index.cjs', + 'vendor/leiden/utils.cjs', + ]; + + it('passes the current lean files list', () => { + expect(findPackedFilesProblems({ filesField: lean, grammarNames: grammars })).toEqual([]); + }); + + it('fails when files only covers FTS plus one grammar prebuild', () => { + const problems = findPackedFilesProblems({ + filesField: ['vendor/lbug-fts/prebuilds/**', 'vendor/tree-sitter-c/prebuilds/**'], + grammarNames: grammars, + }); + expect(problems.some((p: string) => p.includes('tree-sitter-kotlin'))).toBe(true); + expect(problems.some((p: string) => p.includes('bindings/node/index.js'))).toBe(true); + expect(problems.some((p: string) => p.includes('leiden'))).toBe(true); + }); + + it('fails when only one grammar binding is listed explicitly', () => { + const problems = findPackedFilesProblems({ + filesField: [ + 'vendor/**/prebuilds/**', + 'vendor/tree-sitter-c/bindings/node/index.js', + 'vendor/**/package.json', + 'vendor/leiden/index.cjs', + 'vendor/leiden/utils.cjs', + ], + grammarNames: grammars, + }); + expect(problems.some((p: string) => p.includes('tree-sitter-kotlin'))).toBe(true); + expect(problems.some((p: string) => p.includes('bindings/node/index.js'))).toBe(true); + }); + + it('fails when per-grammar package.json is omitted', () => { + const problems = findPackedFilesProblems({ + filesField: [ + 'vendor/**/prebuilds/**', + 'vendor/**/bindings/node/index.js', + 'vendor/leiden/index.cjs', + 'vendor/leiden/utils.cjs', + ], + grammarNames: grammars, + }); + expect(problems.some((p: string) => p.includes('package.json'))).toBe(true); + }); + + it('fails when Leiden entrypoints are dropped', () => { + const problems = findPackedFilesProblems({ + filesField: ['vendor/**/prebuilds/**', 'vendor/**/bindings/node/index.js'], + grammarNames: grammars, + }); + expect(problems.some((p: string) => p.includes('leiden'))).toBe(true); + }); +}); + describe('filesShipsVendorSource', () => { it('ships when a broad vendor entry is present', () => { expect(filesShipsVendorSource(['dist', 'vendor', 'web'])).toBe(true); @@ -114,11 +180,81 @@ describe('findStrayBuildArtifacts (stray vendor build dirs that would ship + sha }); }); -describe('real repo publish-safety (guards against premature files narrowing)', () => { +describe('real repo publish-safety (lean files + 6/6 prebuilds)', () => { + const GITNEXUS_ROOT = fileURLToPath(new URL('../..', import.meta.url)); + const VENDORED = [ + 'tree-sitter-c', + 'tree-sitter-dart', + 'tree-sitter-kotlin', + 'tree-sitter-objc', + 'tree-sitter-proto', + 'tree-sitter-swift', + 'tree-sitter-zig', + ] as const; + const TUPLES = [ + 'linux-x64', + 'linux-arm64', + 'darwin-x64', + 'darwin-arm64', + 'win32-x64', + 'win32-arm64', + ] as const; + + const simulatePackedVendorFiles = (filesField: string[]): string[] => { + const packed = new Set(); + for (const pattern of filesField) { + const n = String(pattern).replace(/\\/g, '/'); + if (!n.startsWith('vendor')) continue; + for (const match of globSync(n, { cwd: GITNEXUS_ROOT, nodir: true, dot: false })) { + packed.add(match.replace(/\\/g, '/')); + } + } + return [...packed]; + }; + it('the script exits 0 against the committed repo state', () => { // Deterministic: reads package.json + walks vendor/ — no npm pack, fast. const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 20_000 }); expect(r.status, r.stderr).toBe(0); expect(r.stdout).toContain('[publish-guard] OK'); + expect(r.stdout).toContain('prebuilds-only'); + }); + + it('package.json files is lean: no bare vendor, no shipped parser.c, load files remain', () => { + const pkg = JSON.parse(readFileSync(path.join(GITNEXUS_ROOT, 'package.json'), 'utf8')) as { + files: string[]; + }; + expect(filesShipsVendorSource(pkg.files)).toBe(false); + expect(pkg.files).not.toContain('vendor'); + expect(pkg.files).toContain('vendor/**/prebuilds/**'); + expect(pkg.files).toContain('vendor/leiden/index.cjs'); + expect(pkg.files).toContain('vendor/leiden/utils.cjs'); + expect(pkg.files).toContain('vendor/lbug-fts/manifest.json'); + + const packed = simulatePackedVendorFiles(pkg.files); + expect(packed.some((p) => /vendor\/tree-sitter-[^/]+\/src\/parser\.c$/.test(p))).toBe(false); + expect(packed.some((p) => /vendor\/tree-sitter-[^/]+\/src\/scanner\.c$/.test(p))).toBe(false); + expect(packed.some((p) => p.endsWith('binding.gyp'))).toBe(false); + + for (const name of VENDORED) { + expect(packed, name).toContain(`vendor/${name}/package.json`); + expect(packed, name).toContain(`vendor/${name}/bindings/node/index.js`); + expect(packed, name).toContain(`vendor/${name}/src/node-types.json`); + for (const tuple of TUPLES) { + expect( + packed.some( + (p) => p.startsWith(`vendor/${name}/prebuilds/${tuple}/`) && p.endsWith('.node'), + ), + `${name} ${tuple}`, + ).toBe(true); + } + } + + expect(packed).toContain('vendor/leiden/index.cjs'); + expect(packed).toContain('vendor/leiden/utils.cjs'); + expect(packed).toContain('vendor/lbug-fts/manifest.json'); + for (const tuple of ['linux-x64', 'linux-arm64', 'darwin-x64', 'darwin-arm64', 'win32-x64']) { + expect(packed).toContain(`vendor/lbug-fts/prebuilds/${tuple}/libfts.lbug_extension`); + } }); }); diff --git a/gitnexus/test/unit/cli-commands.test.ts b/gitnexus/test/unit/cli-commands.test.ts index e750666fc..ade958fe2 100644 --- a/gitnexus/test/unit/cli-commands.test.ts +++ b/gitnexus/test/unit/cli-commands.test.ts @@ -109,7 +109,8 @@ describe('CLI commands', () => { // (vendored-grammars.ts), so postinstall no longer materializes anything. expect(pkg.default.scripts.postinstall).not.toContain('materialize-vendor-grammars.cjs'); expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-grammars.cjs'); - expect(pkg.default.files).toContain('vendor'); + expect(pkg.default.files).toContain('vendor/**/prebuilds/**'); + expect(pkg.default.files).not.toContain('vendor'); }); it('declares node-gyp-build/node-addon-api as regular dependencies (runtime-load contract)', async () => { diff --git a/gitnexus/test/unit/doctor-format.test.ts b/gitnexus/test/unit/doctor-format.test.ts index 662e8e4ef..bc6d3f394 100644 --- a/gitnexus/test/unit/doctor-format.test.ts +++ b/gitnexus/test/unit/doctor-format.test.ts @@ -163,7 +163,12 @@ describe('doctor embedding-runtime support status', () => { ['linux', 'x64'], ['win32', 'x64'], ] as Array<[NodeJS.Platform, NodeJS.Architecture]>) { - const { status, detail } = localEmbeddingDoctorStatus({ httpMode: false, platform, arch }); + const { status, detail } = localEmbeddingDoctorStatus({ + httpMode: false, + platform, + arch, + resolution: { source: 'package' }, + }); expect(status).toBe('✓ local embeddings supported'); expect(detail).toBeNull(); } @@ -179,15 +184,16 @@ describe('doctor embedding-runtime support status', () => { expect(detail).toBeNull(); }); - it('flags a pruned optional embedding stack with reinstall guidance (#2370)', () => { + it('flags a missing local embedding stack with install guidance', () => { const { status, detail } = localEmbeddingDoctorStatus({ httpMode: false, platform: 'linux', arch: 'x64', resolution: null, }); - expect(status).toBe('✗ optional embedding stack not installed'); - expect(detail).toContain('ONNXRUNTIME_NODE_INSTALL=skip'); + expect(status).toBe('✗ local embedding stack not installed'); + expect(detail).toContain('gitnexus embeddings install'); + expect(detail).not.toContain('ONNXRUNTIME_NODE_INSTALL=skip'); }); it('reports a package-sourced stack as supported regardless of Node loadability', () => { diff --git a/gitnexus/test/unit/embedder.test.ts b/gitnexus/test/unit/embedder.test.ts index 20eaf8c59..66560c09b 100644 --- a/gitnexus/test/unit/embedder.test.ts +++ b/gitnexus/test/unit/embedder.test.ts @@ -1,4 +1,9 @@ -import { describe, it, expect } from 'vitest'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { isHttpMode } from '../../src/core/embeddings/http-client.js'; +import { assessLocalEmbeddingRuntime } from '../../src/core/embeddings/runtime-support.js'; import { getEmbeddingDims, isEmbedderReady } from '../../src/mcp/core/embedder.js'; describe('embedder', () => { @@ -9,8 +14,23 @@ describe('embedder', () => { }); describe('isEmbedderReady', () => { - it('returns false before initialization', () => { - expect(isEmbedderReady()).toBe(false); + it('follows HTTP mode or a ready local runtime, not resolution alone', () => { + expect(isEmbedderReady()).toBe( + isHttpMode() || assessLocalEmbeddingRuntime().status === 'ready', + ); }); }); + + it('does not import the Hugging Face transformers package', () => { + const src = readFileSync( + path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../../src/mcp/core/embedder.ts'), + 'utf8', + ); + expect(src).not.toMatch(/from\s+['"]@huggingface\/transformers['"]/); + expect(src).not.toMatch(/import\s*\(\s*['"]@huggingface\/transformers['"]/); + expect(src).not.toMatch(/from\s+['"]onnxruntime-node['"]/); + expect(src).not.toMatch(/import\s*\(\s*['"]onnxruntime-node['"]/); + expect(src).not.toMatch(/from\s+['"].*embedding-local-init['"]/); + expect(src).not.toMatch(/import\s*\(\s*['"].*embedding-local-init['"]/); + }); }); diff --git a/gitnexus/test/unit/embedding-runtime-install.test.ts b/gitnexus/test/unit/embedding-runtime-install.test.ts index 547d18f13..aac5deba2 100644 --- a/gitnexus/test/unit/embedding-runtime-install.test.ts +++ b/gitnexus/test/unit/embedding-runtime-install.test.ts @@ -3,6 +3,17 @@ import { EventEmitter } from 'node:events'; import { homedir } from 'node:os'; import { join, resolve } from 'node:path'; import { createRequire } from 'node:module'; + +const { mkdirSyncMock, writeFileSyncMock } = vi.hoisted(() => ({ + mkdirSyncMock: vi.fn(), + writeFileSyncMock: vi.fn(), +})); + +vi.mock('node:fs', async (importOriginal) => ({ + ...(await importOriginal()), + mkdirSync: (...args: unknown[]) => mkdirSyncMock(...args), + writeFileSync: (...args: unknown[]) => writeFileSyncMock(...args), +})); import { ANALYZE_EMBEDDING_INSTALL_TIMEOUT_MS, buildEmbeddingInstallCommand, @@ -43,6 +54,8 @@ const savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); beforeEach(() => { for (const key of ENV_KEYS) delete process.env[key]; + mkdirSyncMock.mockReset(); + writeFileSyncMock.mockReset(); }); afterEach(() => { @@ -76,16 +89,23 @@ describe('getEmbeddingRuntimeDir', () => { }); describe('getEmbeddingStackSpecs', () => { - it('mirrors the optionalDependencies manifest exactly (drift guard, #2370)', () => { + it('mirrors gitnexusEmbeddingStack exactly and keeps the packages off optionalDependencies', () => { const manifest = require('../../package.json') as { - optionalDependencies: Record; + gitnexusEmbeddingStack: Record; + optionalDependencies?: Record; + dependencies: Record; + overrides: Record; }; expect(getEmbeddingStackSpecs()).toEqual({ - '@huggingface/transformers': manifest.optionalDependencies['@huggingface/transformers'], - 'onnxruntime-node': manifest.optionalDependencies['onnxruntime-node'], + '@huggingface/transformers': manifest.gitnexusEmbeddingStack['@huggingface/transformers'], + 'onnxruntime-node': manifest.gitnexusEmbeddingStack['onnxruntime-node'], }); - expect(manifest.optionalDependencies['@huggingface/transformers']).toBeDefined(); - expect(manifest.optionalDependencies['onnxruntime-node']).toBeDefined(); + expect(manifest.gitnexusEmbeddingStack['@huggingface/transformers']).toBeDefined(); + expect(manifest.gitnexusEmbeddingStack['onnxruntime-node']).toBeDefined(); + expect(manifest.optionalDependencies?.['@huggingface/transformers']).toBeUndefined(); + expect(manifest.optionalDependencies?.['onnxruntime-node']).toBeUndefined(); + expect(manifest.dependencies['onnxruntime-common']).toBeDefined(); + expect(JSON.stringify(manifest.overrides)).not.toContain('$onnxruntime-node'); }); }); @@ -121,9 +141,18 @@ describe('buildEmbeddingInstallCommand', () => { }); describe('resolveEmbeddingRuntime', () => { - it('finds the normally-installed stack (package source wins over the prefix)', () => { + it('returns package when the leftover tree is present, otherwise null on a clean prefix', () => { process.env.GITNEXUS_EMBEDDING_RUNTIME_DIR = '/nonexistent/for/this/test'; - expect(resolveEmbeddingRuntime()).toEqual({ source: 'package' }); + // After U4 a default install has no stack. A leftover 1.6.12 package-first + // tree in this workspace's node_modules still wins until a clean reinstall. + let leftoverPackageTree = true; + try { + require.resolve('@huggingface/transformers'); + require.resolve('onnxruntime-node'); + } catch { + leftoverPackageTree = false; + } + expect(resolveEmbeddingRuntime()).toEqual(leftoverPackageTree ? { source: 'package' } : null); }); }); @@ -191,6 +220,37 @@ describe('quoteWin32Arg', () => { }); }); +describe('installEmbeddingRuntime prefix manifest', () => { + it('writes pins and overrides to prefix package.json before npm spawn', async () => { + process.env.GITNEXUS_EMBEDDING_RUNTIME_DIR = resolve('/custom/runtime'); + const child = new FakeChild(); + spawnMock.mockReturnValue(child); + const pending = installEmbeddingRuntime({}, 10_000); + expect(mkdirSyncMock).toHaveBeenCalled(); + expect(writeFileSyncMock).toHaveBeenCalled(); + const [pathArg, body] = writeFileSyncMock.mock.calls[0] as [string, string]; + expect(pathArg).toBe(join(resolve('/custom/runtime'), 'package.json')); + const written = JSON.parse(body) as { + dependencies: Record; + overrides: { + 'adm-zip': string; + sharp: string; + '@huggingface/transformers': { 'onnxruntime-node': string }; + }; + }; + const specs = getEmbeddingStackSpecs(); + expect(written.dependencies).toEqual(specs); + expect(written.overrides['adm-zip']).toBe('>=0.6.0'); + expect(written.overrides.sharp).toBe('>=0.35.0'); + expect(written.overrides['@huggingface/transformers']['onnxruntime-node']).toBe( + specs['onnxruntime-node'], + ); + expect(spawnMock).toHaveBeenCalled(); + child.emit('close', 0, null); + await pending; + }); +}); + describe('installEmbeddingRuntime — spawn lifecycle', () => { beforeEach(() => { vi.useFakeTimers(); diff --git a/gitnexus/test/unit/embedding-runtime-resolution.test.ts b/gitnexus/test/unit/embedding-runtime-resolution.test.ts index 949d53dbd..7b558a419 100644 --- a/gitnexus/test/unit/embedding-runtime-resolution.test.ts +++ b/gitnexus/test/unit/embedding-runtime-resolution.test.ts @@ -5,10 +5,10 @@ import { describe, it, expect, vi, afterEach } from 'vitest'; * * The package tier uses runtime-install's module-scope require (anchored at its * own `import.meta.url`); the prefix tier uses a require anchored at - * `/noop.js`. In dev/CI both optional deps ARE really installed, so the - * package tier can never miss with the real require — we mock `createRequire` to - * route each anchor to a fake whose `.resolve()` is driven by a fixture map, - * exercising the partial / full / missing permutations. + * `/noop.js`. After U4 a clean tree has no embedding packages, but a + * leftover 1.6.12 package-first install still can. We mock `createRequire` so + * the package tier is driven by a fixture map, exercising the partial / full / + * missing permutations without depending on this workspace's node_modules. * * This file has ZERO static import of runtime-install.js (the dual-instance * rule): every load goes through the dynamic-import harness, so no real diff --git a/gitnexus/test/unit/embedding-runtime-support.test.ts b/gitnexus/test/unit/embedding-runtime-support.test.ts index 4b9fdfba9..2d440518f 100644 --- a/gitnexus/test/unit/embedding-runtime-support.test.ts +++ b/gitnexus/test/unit/embedding-runtime-support.test.ts @@ -3,9 +3,12 @@ import { getLocalEmbeddingRuntimeBlocker, getMissingLocalEmbeddingStackMessage, isLocalEmbeddingRuntimeBlockerMessage, + isLocalEmbeddingSidecarAbortMessage, isLocalEmbeddingStackInstalled, isMissingLocalEmbeddingStackMessage, + LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD, localEmbeddingStackMissingMessage, + localEmbeddingPrefixUnloadableMessage, } from '../../src/core/embeddings/runtime-support.js'; /** @@ -26,11 +29,11 @@ vi.mock('@huggingface/transformers', () => { }); /** - * Spy for the CUDA-13 build-matching resolver hook. Both local embedders must - * call this before importing transformers.js — mocked (rather than exercising - * the real resolver's env/subprocess probing) to keep this suite fast and - * platform-independent; `onnxruntime-node-resolver.test.ts` covers the - * resolver's own decision logic. + * Spy for the CUDA-13 build-matching resolver hook. Child `initLocalEmbedder` + * must call this before importing transformers.js — mocked (rather than + * exercising the real resolver's env/subprocess probing) to keep this suite + * fast and platform-independent; `onnxruntime-node-resolver.test.ts` covers + * the resolver's own decision logic. */ const { resolverHookInstalled } = vi.hoisted(() => ({ resolverHookInstalled: vi.fn() })); @@ -39,6 +42,14 @@ vi.mock('../../src/core/embeddings/onnxruntime-node-resolver.js', () => ({ isEffectiveCudaAvailable: () => false, })); +vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ + ensureEmbeddingSidecar: vi.fn(async () => ({ device: 'cpu' })), + getSidecarDevice: () => 'cpu', + reapEmbeddingSidecar: vi.fn(), + reapEmbeddingSidecarAndWait: vi.fn(async () => {}), + sidecarEmbedBatch: vi.fn(async (texts: string[]) => texts.map(() => new Float32Array(384))), +})); + /** * Mock `module.registerHooks` with a spy (#2372). Without this, a successful * local `initEmbedder()` calls the REAL `ensureEmbeddingStackResolvable` / @@ -118,8 +129,8 @@ describe('getLocalEmbeddingRuntimeBlocker', () => { // Safe alternatives expect(text).toMatch(/without --embeddings/); expect(text).toContain('GITNEXUS_EMBEDDING_URL'); - expect(text).toMatch(/Linux or in Docker/); - expect(text).toMatch(/Apple Silicon/); + expect(text).toMatch(/Linux or Apple Silicon/); + expect(text).toMatch(/Official CLI Docker images no longer/); // Addresses the GitNexus device knob too, not only ONNX_WEB_BACKEND (R3 / #1987) expect(text).toContain('GITNEXUS_EMBEDDING_DEVICE'); }); @@ -211,16 +222,27 @@ describe('getMissingLocalEmbeddingStackMessage (#2370 pruned optional stack)', ( it('produces guidance naming every recovery path', () => { const msg = localEmbeddingStackMissingMessage(); expect(msg).toContain('gitnexus embeddings install'); - expect(msg).toContain('ONNXRUNTIME_NODE_INSTALL=skip'); + expect(msg).toContain('GITNEXUS_EMBEDDING_RUNTIME_DIR'); + expect(msg).not.toContain('ONNXRUNTIME_NODE_INSTALL=skip'); expect(msg).toContain('GLOBAL_AGENT_HTTPS_PROXY'); expect(msg).toContain('GITNEXUS_EMBEDDING_URL'); - expect(msg).toContain('#2370'); + expect(msg).toContain('1.6.12'); // Must not trip analyze.ts's generic "installation may be corrupt" branch. expect(msg).not.toMatch(/Cannot find (module|package)/); expect(msg).not.toContain('MODULE_NOT_FOUND'); }); }); +describe('localEmbeddingPrefixUnloadableMessage', () => { + it('tells the user to upgrade this Node, not to install from another Node and retry', () => { + const msg = localEmbeddingPrefixUnloadableMessage(); + expect(msg).toContain('module.registerHooks'); + expect(msg).toContain('Upgrade this'); + expect(msg).toContain('cannot add that API here'); + expect(msg).not.toMatch(/then retry/i); + }); +}); + describe('isMissingLocalEmbeddingStackMessage', () => { it('recognises its own message and rejects the platform blocker and unrelated errors', () => { expect(isMissingLocalEmbeddingStackMessage(localEmbeddingStackMissingMessage())).toBe(true); @@ -231,10 +253,19 @@ describe('isMissingLocalEmbeddingStackMessage', () => { }); }); +describe('isLocalEmbeddingSidecarAbortMessage', () => { + it('recognises sidecar abort and sidecar-dead text', () => { + expect(isLocalEmbeddingSidecarAbortMessage(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD)).toBe(true); + expect(isLocalEmbeddingSidecarAbortMessage('Embedding sidecar died (signal SIGSEGV)')).toBe( + true, + ); + expect(isLocalEmbeddingSidecarAbortMessage(localEmbeddingStackMissingMessage())).toBe(false); + }); +}); + describe('isLocalEmbeddingStackInstalled', () => { - it('resolves the optional stack in the dev workspace without importing it', () => { - expect(isLocalEmbeddingStackInstalled()).toBe(true); - // Resolution only — the transformers.js import spy must not fire. + it('probes stack resolution without importing transformers.js', () => { + isLocalEmbeddingStackInstalled(); expect(transformersImported).not.toHaveBeenCalled(); }); }); @@ -386,46 +417,40 @@ describe('MCP embedQuery on darwin/x64', () => { }); describe('CUDA-13 resolver hook installation (both local-embedding entrypoints)', () => { - // Regression guard for the two local embedders drifting apart (gitnexus PR #2341 - // follow-up): both `core/embeddings/embedder.ts` and `mcp/core/embedder.ts` must - // install the CUDA-build-matching redirect during a successful local init. (The - // source itself places the call before `await import('@huggingface/transformers')` - // — not re-asserted here via mock call-order, since the hoisted `@huggingface/ - // transformers` mock's factory only fires once per file run for this external - // package, making a second per-test "called fresh" assertion on it unreliable.) - it('core embedder installs the resolver hook on a successful local init', async () => { + // Parent façade must not install CUDA / transformers hooks; only the + // sidecar child local-init path does. + it('core embedder does not install the resolver hook in the parent on local init', async () => { const restore = stubPlatform('linux', 'x64'); try { const { initEmbedder } = await import('../../src/core/embeddings/embedder.js'); await expect(initEmbedder()).resolves.toBeDefined(); - expect(resolverHookInstalled).toHaveBeenCalled(); + expect(resolverHookInstalled).not.toHaveBeenCalled(); + expect(transformersImported).not.toHaveBeenCalled(); } finally { restore(); } }); - it('MCP embedder installs the resolver hook on a successful local init', async () => { + it('MCP embedder does not install the resolver hook in the parent on local init', async () => { const restore = stubPlatform('linux', 'x64'); try { const { initEmbedder } = await import('../../src/mcp/core/embedder.js'); await expect(initEmbedder()).resolves.toBeDefined(); - expect(resolverHookInstalled).toHaveBeenCalled(); + expect(resolverHookInstalled).not.toHaveBeenCalled(); + expect(transformersImported).not.toHaveBeenCalled(); } finally { restore(); } }); - it('registers the runtime-prefix fallback through the mocked registerHooks, not the real global API (#2372)', async () => { - // The whole point of the node:module mock: a successful local init exercises - // ensureEmbeddingStackResolvable's registration via the spy, so no real - // process-global resolution hook leaks into other tests in the worker. + it('does not register process-global resolution hooks in the parent on local init (#2372)', async () => { const restore = stubPlatform('linux', 'x64'); try { const { initEmbedder } = await import('../../src/core/embeddings/embedder.js'); await expect(initEmbedder()).resolves.toBeDefined(); - expect(registerHooksSpy).toHaveBeenCalled(); + expect(registerHooksSpy).not.toHaveBeenCalled(); } finally { restore(); } diff --git a/gitnexus/test/unit/embedding-sidecar-client.test.ts b/gitnexus/test/unit/embedding-sidecar-client.test.ts new file mode 100644 index 000000000..0bdf5745f --- /dev/null +++ b/gitnexus/test/unit/embedding-sidecar-client.test.ts @@ -0,0 +1,431 @@ +import { EventEmitter } from 'node:events'; +import { readFileSync } from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { ChildProcess } from 'node:child_process'; +import type { + SidecarRequest, + SidecarResponse, +} from '../../src/core/embeddings/embedding-sidecar-protocol.js'; + +const embeddingsDir = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../src/core/embeddings', +); + +class FakeChild extends EventEmitter { + killed = false; + connected = true; + stdout = new EventEmitter(); + stderr = new EventEmitter(); + send = vi.fn((msg: SidecarRequest) => { + queueMicrotask(() => { + const response = this.respond(msg); + if (response) this.emit('message', response); + }); + return true; + }); + kill = vi.fn((signal?: NodeJS.Signals) => { + this.killed = true; + this.emit('close', signal === 'SIGKILL' ? null : 0, signal ?? null); + return true; + }); + unref = vi.fn(); + + respond(msg: SidecarRequest): SidecarResponse | undefined { + if (msg.type === 'init') { + return { id: msg.id, type: 'ready', device: 'cpu' }; + } + if (msg.type === 'embed') { + return { + id: msg.id, + type: 'vectors', + vectors: msg.texts.map(() => [0.25, 0.5, 0.75]), + }; + } + return undefined; + } +} + +describe('embedding sidecar client', () => { + const originalUrl = process.env.GITNEXUS_EMBEDDING_URL; + const originalModel = process.env.GITNEXUS_EMBEDDING_MODEL; + const originalHfTimeout = process.env.HF_DOWNLOAD_TIMEOUT_MS; + const originalHfAttempts = process.env.HF_MAX_ATTEMPTS; + const originalSidecarTimeout = process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS; + const hostPlatform = process.platform; + const hostArch = process.arch; + + let forkMock: ReturnType; + let children: FakeChild[]; + + beforeEach(async () => { + delete process.env.GITNEXUS_EMBEDDING_URL; + // Local-success cases must not inherit a darwin/x64 host blocker. + Object.defineProperty(process, 'platform', { value: 'linux', configurable: true }); + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }); + children = []; + forkMock = vi.fn((_script: string, _args: string[], _opts: unknown) => { + const child = new FakeChild(); + children.push(child); + return child as unknown as ChildProcess; + }); + const client = await import('../../src/core/embeddings/embedding-sidecar-client.js'); + client._resetEmbeddingSidecarForTests(); + client._setForkForTests(forkMock); + }); + + afterEach(async () => { + const client = await import('../../src/core/embeddings/embedding-sidecar-client.js'); + client._resetEmbeddingSidecarForTests(); + client._setForkForTests(null); + if (originalUrl === undefined) delete process.env.GITNEXUS_EMBEDDING_URL; + else process.env.GITNEXUS_EMBEDDING_URL = originalUrl; + if (originalModel === undefined) delete process.env.GITNEXUS_EMBEDDING_MODEL; + else process.env.GITNEXUS_EMBEDDING_MODEL = originalModel; + if (originalHfTimeout === undefined) delete process.env.HF_DOWNLOAD_TIMEOUT_MS; + else process.env.HF_DOWNLOAD_TIMEOUT_MS = originalHfTimeout; + if (originalHfAttempts === undefined) delete process.env.HF_MAX_ATTEMPTS; + else process.env.HF_MAX_ATTEMPTS = originalHfAttempts; + if (originalSidecarTimeout === undefined) + delete process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS; + else process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS = originalSidecarTimeout; + Object.defineProperty(process, 'platform', { value: hostPlatform, configurable: true }); + Object.defineProperty(process, 'arch', { value: hostArch, configurable: true }); + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it('strips GITNEXUS_EMBEDDING_URL and does not inherit stdout', async () => { + process.env.GITNEXUS_EMBEDDING_URL = 'http://custom.example/v1'; + const { ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await ensureEmbeddingSidecar(); + + expect(forkMock).toHaveBeenCalledTimes(1); + const opts = forkMock.mock.calls[0][2] as { + env: NodeJS.ProcessEnv; + stdio: unknown; + }; + expect(opts.env.GITNEXUS_EMBEDDING_URL).toBeUndefined(); + expect(opts.stdio).toEqual(['ignore', 'ignore', 'pipe', 'ipc']); + }); + + it('reports no sidecar device until init is ready', async () => { + const { getSidecarDevice, ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + expect(getSidecarDevice()).toBeNull(); + await ensureEmbeddingSidecar(); + expect(getSidecarDevice()).toBe('cpu'); + }); + + it('rejects a forceDevice that conflicts with the initialized sidecar', async () => { + const { ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await expect(ensureEmbeddingSidecar()).resolves.toEqual({ device: 'cpu' }); + await expect(ensureEmbeddingSidecar({ forceDevice: 'cuda' })).rejects.toThrow( + /already initialized on cpu; cannot switch to cuda/, + ); + await expect(ensureEmbeddingSidecar({ forceDevice: 'cpu' })).resolves.toEqual({ + device: 'cpu', + }); + }); + + it('forks once for two batches', async () => { + const { sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const first = await sidecarEmbedBatch(['a']); + const second = await sidecarEmbedBatch(['b']); + expect(first).toHaveLength(1); + expect(second).toHaveLength(1); + expect(forkMock).toHaveBeenCalledTimes(1); + }); + + it('does not fork on darwin/x64', async () => { + const orig = { platform: process.platform, arch: process.arch }; + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }); + try { + const { ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await expect(ensureEmbeddingSidecar()).rejects.toThrow(/macOS Intel/); + expect(forkMock).not.toHaveBeenCalled(); + } finally { + Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true }); + Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true }); + } + }); + + it('does not fork on an empty batch', async () => { + const { sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await expect(sidecarEmbedBatch([])).resolves.toEqual([]); + expect(forkMock).not.toHaveBeenCalled(); + }); + + it('does not fork in HTTP mode', async () => { + process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1'; + process.env.GITNEXUS_EMBEDDING_MODEL = 'test-model'; + const mockVec = Array.from({ length: 384 }, (_, i) => i / 384); + vi.stubGlobal( + 'fetch', + vi.fn(async (_url: string, init: { body: string }) => { + const n = (JSON.parse(init.body) as { input: string[] }).input.length; + return { + ok: true, + json: async () => ({ data: Array.from({ length: n }, () => ({ embedding: mockVec })) }), + }; + }), + ); + const { embedBatch, isEmbedderReady } = await import('../../src/core/embeddings/embedder.js'); + expect(isEmbedderReady()).toBe(true); + const batch = await embedBatch(['hello']); + expect(batch).toHaveLength(1); + expect(forkMock).not.toHaveBeenCalled(); + }); + + it('marks local embeddings unavailable on native abort and does not respawn', async () => { + const { sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await sidecarEmbedBatch(['first']); + expect(forkMock).toHaveBeenCalledTimes(1); + + children[0].emit('close', null, 'SIGSEGV'); + + await expect(sidecarEmbedBatch(['second'])).rejects.toThrow( + /unavailable after the sidecar aborted/, + ); + expect(forkMock).toHaveBeenCalledTimes(1); + }); + + it('does not respawn after init-time SIGSEGV', async () => { + forkMock.mockImplementation(() => { + const child = new FakeChild(); + children.push(child); + child.send = vi.fn(() => { + queueMicrotask(() => child.emit('close', null, 'SIGSEGV')); + return true; + }); + return child as unknown as ChildProcess; + }); + const { ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await expect(ensureEmbeddingSidecar()).rejects.toThrow(/Embedding sidecar died/); + expect(forkMock).toHaveBeenCalledTimes(1); + await expect(ensureEmbeddingSidecar()).rejects.toThrow(/unavailable after the sidecar aborted/); + expect(forkMock).toHaveBeenCalledTimes(1); + }); + + it('SIGKILLs a stalled embed request and rejects with a timeout', async () => { + process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS = '50'; + vi.useFakeTimers(); + const { sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await sidecarEmbedBatch(['warmup']); + children[0].send = vi.fn(() => true); + const pending = sidecarEmbedBatch(['stalled']); + const assertion = expect(pending).rejects.toThrow(/timed out after 50ms \(embed\)/); + await vi.advanceTimersByTimeAsync(50); + await assertion; + expect(children[0].kill).toHaveBeenCalledWith('SIGKILL'); + }); + + it('rejects embedBatch when aborted while waiting on an outstanding embed request', async () => { + const { embedBatch } = await import('../../src/core/embeddings/embedder.js'); + await embedBatch(['warmup']); + children[0].send = vi.fn(() => true); + const controller = new AbortController(); + const pending = embedBatch(['stalled'], { signal: controller.signal }); + await vi.waitFor(() => expect(children[0].send).toHaveBeenCalled()); + controller.abort(); + await expect(pending).rejects.toThrow(); + }); + + it('cleans up a pending waiter when IPC send throws', async () => { + process.env.GITNEXUS_EMBEDDING_SIDECAR_TIMEOUT_MS = '50'; + const { sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await sidecarEmbedBatch(['warmup']); + vi.useFakeTimers(); + children[0].send = vi.fn(() => { + throw new Error('Channel closed'); + }); + await expect(sidecarEmbedBatch(['x'])).rejects.toThrow(/Channel closed/); + expect(children[0].kill).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(50); + expect(children[0].kill).not.toHaveBeenCalled(); + }); + + it('lets a joining caller abort without cancelling shared sidecar init', async () => { + let releaseInit: (() => void) | undefined; + forkMock.mockImplementation(() => { + const child = new FakeChild(); + children.push(child); + child.send = vi.fn((msg: SidecarRequest) => { + if (msg.type === 'init') { + releaseInit = () => child.emit('message', { id: msg.id, type: 'ready', device: 'cpu' }); + return true; + } + queueMicrotask(() => { + const response = child.respond(msg); + if (response) child.emit('message', response); + }); + return true; + }); + return child as unknown as ChildProcess; + }); + const { ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const first = ensureEmbeddingSidecar(); + await vi.waitFor(() => expect(releaseInit).toBeDefined()); + const joining = new AbortController(); + const second = ensureEmbeddingSidecar({ signal: joining.signal }); + joining.abort(); + await expect(second).rejects.toThrow(); + releaseInit!(); + await expect(first).resolves.toEqual({ device: 'cpu' }); + expect(forkMock).toHaveBeenCalledTimes(1); + }); + + it('does not cancel shared sidecar init when the first waiter aborts', async () => { + let releaseInit: (() => void) | undefined; + forkMock.mockImplementation(() => { + const child = new FakeChild(); + children.push(child); + child.send = vi.fn((msg: SidecarRequest) => { + if (msg.type === 'init') { + releaseInit = () => child.emit('message', { id: msg.id, type: 'ready', device: 'cpu' }); + return true; + } + queueMicrotask(() => { + const response = child.respond(msg); + if (response) child.emit('message', response); + }); + return true; + }); + return child as unknown as ChildProcess; + }); + const { ensureEmbeddingSidecar } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + const firstAbort = new AbortController(); + const first = ensureEmbeddingSidecar({ signal: firstAbort.signal }); + await vi.waitFor(() => expect(releaseInit).toBeDefined()); + const second = ensureEmbeddingSidecar(); + firstAbort.abort(); + await expect(first).rejects.toThrow(); + releaseInit!(); + await expect(second).resolves.toEqual({ device: 'cpu' }); + expect(forkMock).toHaveBeenCalledTimes(1); + }); + + it('rejects an aborted embed wait without killing the sidecar', async () => { + const { sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await sidecarEmbedBatch(['warmup']); + children[0].send = vi.fn(() => true); + const controller = new AbortController(); + const pending = sidecarEmbedBatch(['stalled'], { signal: controller.signal }); + await vi.waitFor(() => expect(children[0].send).toHaveBeenCalled()); + controller.abort(); + await expect(pending).rejects.toThrow(); + expect(children[0].kill).not.toHaveBeenCalled(); + }); + + it('respawns a dead sidecar with the last init embeddingConfig and forceDevice', async () => { + const { ensureEmbeddingSidecar, sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await ensureEmbeddingSidecar({ + embeddingConfig: { dimensions: 768 }, + forceDevice: 'cpu', + }); + expect(forkMock).toHaveBeenCalledTimes(1); + expect(children[0].send.mock.calls[0][0]).toMatchObject({ + type: 'init', + embeddingConfig: { dimensions: 768 }, + forceDevice: 'cpu', + }); + children[0].emit('close', 1, null); + await sidecarEmbedBatch(['again']); + expect(forkMock).toHaveBeenCalledTimes(2); + expect(children[1].send.mock.calls[0][0]).toMatchObject({ + type: 'init', + embeddingConfig: { dimensions: 768 }, + forceDevice: 'cpu', + }); + }); + + it('does not use worker_threads or import the embeddings barrel', () => { + const clientSrc = readFileSync(path.join(embeddingsDir, 'embedding-sidecar-client.ts'), 'utf8'); + const façadeSrc = readFileSync(path.join(embeddingsDir, 'embedder.ts'), 'utf8'); + const importLines = clientSrc + .split('\n') + .filter( + (line) => + /^\s*import\b/.test(line) || /^\s*\} from /.test(line) || /import\s*\(/.test(line), + ) + .join('\n'); + expect(clientSrc).not.toMatch(/worker_threads/); + expect(clientSrc).not.toMatch(/new Worker\b/); + expect(importLines).not.toContain('embedding-pipeline'); + expect(importLines).not.toContain('embedding-identity'); + expect(importLines).not.toContain('./index.js'); + expect(façadeSrc).not.toMatch(/from\s+['"]@huggingface\/transformers['"]/); + expect(façadeSrc).not.toMatch(/import\s*\(\s*['"]@huggingface\/transformers['"]/); + expect(façadeSrc).not.toMatch(/from\s+['"]onnxruntime-node['"]/); + expect(façadeSrc).not.toMatch(/import\s*\(\s*['"]onnxruntime-node['"]/); + expect(façadeSrc).not.toMatch(/from\s+['"].*onnxruntime-common-resolver['"]/); + expect(façadeSrc).not.toMatch(/import\s*\(\s*['"].*onnxruntime-common-resolver['"]/); + expect(façadeSrc).not.toMatch(/from\s+['"].*embedding-local-init['"]/); + expect(façadeSrc).not.toMatch(/import\s*\(\s*['"].*embedding-local-init['"]/); + }); + + it('sizes the init deadline from the HF download budget, not a 15s process lifetime', async () => { + const { SIDECAR_INIT_IPC_SLACK_MS, sidecarInitTimeoutMs } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + delete process.env.HF_DOWNLOAD_TIMEOUT_MS; + delete process.env.HF_MAX_ATTEMPTS; + // 3 attempts × 5 min plus 2s + 4s exponential backoff, plus IPC slack. + expect(sidecarInitTimeoutMs()).toBe( + 5 * 60 * 1_000 * 3 + 2_000 + 4_000 + SIDECAR_INIT_IPC_SLACK_MS, + ); + expect(sidecarInitTimeoutMs()).toBeGreaterThan(15_000); + + process.env.HF_DOWNLOAD_TIMEOUT_MS = '120000'; + process.env.HF_MAX_ATTEMPTS = '2'; + expect(sidecarInitTimeoutMs()).toBe(240_000 + 2_000 + SIDECAR_INIT_IPC_SLACK_MS); + + process.env.HF_DOWNLOAD_TIMEOUT_MS = String(60 * 60 * 1_000); + process.env.HF_MAX_ATTEMPTS = '1'; + expect(sidecarInitTimeoutMs()).toBe(30 * 60 * 1_000 + SIDECAR_INIT_IPC_SLACK_MS); + + process.env.HF_DOWNLOAD_TIMEOUT_MS = '120000'; + process.env.HF_MAX_ATTEMPTS = '9.5'; + expect(sidecarInitTimeoutMs()).toBe( + 120_000 * 9 + 2_000 * (2 ** 8 - 1) + SIDECAR_INIT_IPC_SLACK_MS, + ); + }); + + it('clears the reap wait timeout once the child closes', async () => { + vi.useFakeTimers(); + const { ensureEmbeddingSidecar, reapEmbeddingSidecarAndWait } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await ensureEmbeddingSidecar(); + await reapEmbeddingSidecarAndWait(5_000); + expect(vi.getTimerCount()).toBe(0); + }); + + it('does not let a reaped child reset its replacement', async () => { + const { ensureEmbeddingSidecar, reapEmbeddingSidecar, sidecarEmbedBatch } = + await import('../../src/core/embeddings/embedding-sidecar-client.js'); + await ensureEmbeddingSidecar(); + const first = children[0]; + reapEmbeddingSidecar(); + await ensureEmbeddingSidecar(); + expect(forkMock).toHaveBeenCalledTimes(2); + first.emit('error', new Error('late error from reaped child')); + first.emit('close', 1, null); + await expect(sidecarEmbedBatch(['still-alive'])).resolves.toHaveLength(1); + expect(forkMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/gitnexus/test/unit/embeddings-install-command.test.ts b/gitnexus/test/unit/embeddings-install-command.test.ts index 625a30fb9..b60348c6c 100644 --- a/gitnexus/test/unit/embeddings-install-command.test.ts +++ b/gitnexus/test/unit/embeddings-install-command.test.ts @@ -9,6 +9,7 @@ * capture logger records, assert on process.exitCode + recoveryHint/msg. */ import { beforeEach, describe, expect, it, vi } from 'vitest'; +import type { LoggerCapture } from '../../src/core/logger.js'; const resolveEmbeddingRuntimeMock = vi.fn<() => { source: string } | null>(); const isPrefixRuntimeLoadableMock = vi.fn(() => true); @@ -23,12 +24,19 @@ vi.mock('../../src/core/embeddings/runtime-install.js', async (importOriginal) = getEmbeddingStackSpecs: () => ({ '@huggingface/transformers': '^4.1.0' }), })); -async function run(options: { cuda?: boolean; force?: boolean } = {}) { +async function withInstallCapture( + options: { cuda?: boolean; force?: boolean } = {}, + assert: (cap: LoggerCapture) => void | Promise, +): Promise { const { _captureLogger } = await import('../../src/core/logger.js'); const cap = _captureLogger(); - const { embeddingsInstallCommand } = await import('../../src/cli/embeddings.js'); - await embeddingsInstallCommand(options); - return cap; + try { + const { embeddingsInstallCommand } = await import('../../src/cli/embeddings.js'); + await embeddingsInstallCommand(options); + await assert(cap); + } finally { + cap.restore(); + } } describe('embeddingsInstallCommand outcomes (#2372)', () => { @@ -40,26 +48,52 @@ describe('embeddingsInstallCommand outcomes (#2372)', () => { process.exitCode = undefined; }); + it('refuses to spawn npm on darwin/x64', async () => { + const orig = { platform: process.platform, arch: process.arch }; + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }); + resolveEmbeddingRuntimeMock.mockReturnValue(null); + try { + await withInstallCapture({}, (cap) => { + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + expect(process.exitCode).toBe(1); + expect(cap.records().some((r) => r.recoveryHint === 'local-embedding-unsupported')).toBe( + true, + ); + }); + } finally { + Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true }); + Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true }); + } + }); + + it('package-sourced --force still installs so prefix overrides refresh', async () => { + resolveEmbeddingRuntimeMock.mockReturnValue({ source: 'package' }); + await withInstallCapture({ force: true }, () => { + expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1); + }); + }); + it('already-installed package source without --force: no install, "nothing to do"', async () => { resolveEmbeddingRuntimeMock.mockReturnValue({ source: 'package' }); - const cap = await run(); - expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); - expect( - cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('nothing to do')), - ).toBe(true); - cap.restore(); + await withInstallCapture({}, (cap) => { + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + expect( + cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('nothing to do')), + ).toBe(true); + }); }); it('post-check resolves nothing: exit 1 and the ✗ message', async () => { // First call (pre-check) not package, so it installs; post-check returns null. resolveEmbeddingRuntimeMock.mockReturnValueOnce(null).mockReturnValueOnce(null); - const cap = await run(); - expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1); - expect(process.exitCode).toBe(1); - expect( - cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('does not resolve')), - ).toBe(true); - cap.restore(); + await withInstallCapture({}, (cap) => { + expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1); + expect(process.exitCode).toBe(1); + expect( + cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('does not resolve')), + ).toBe(true); + }); }); it('post-check runtime-prefix + loadable: unqualified ✓, exit unset', async () => { @@ -67,10 +101,12 @@ describe('embeddingsInstallCommand outcomes (#2372)', () => { source: 'runtime-prefix', }); isPrefixRuntimeLoadableMock.mockReturnValue(true); - const cap = await run(); - expect(process.exitCode).toBeUndefined(); - expect(cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('✓'))).toBe(true); - cap.restore(); + await withInstallCapture({}, (cap) => { + expect(process.exitCode).toBeUndefined(); + expect(cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('✓'))).toBe( + true, + ); + }); }); it('post-check runtime-prefix + not loadable: capability warning, no false ✓, exit unset', async () => { @@ -78,17 +114,17 @@ describe('embeddingsInstallCommand outcomes (#2372)', () => { source: 'runtime-prefix', }); isPrefixRuntimeLoadableMock.mockReturnValue(false); - const cap = await run(); - // install itself succeeded, so exit code stays unset... - expect(process.exitCode).toBeUndefined(); - const records = cap.records(); - // ...but the message names the capability requirement, not an unqualified ✓. - expect( - records.some((r) => typeof r.msg === 'string' && r.msg.includes('module.registerHooks')), - ).toBe(true); - expect(records.some((r) => typeof r.msg === 'string' && r.msg.includes('is ready'))).toBe( - false, - ); - cap.restore(); + await withInstallCapture({}, (cap) => { + // install itself succeeded, so exit code stays unset... + expect(process.exitCode).toBeUndefined(); + const records = cap.records(); + // ...but the message names the capability requirement, not an unqualified ✓. + expect( + records.some((r) => typeof r.msg === 'string' && r.msg.includes('module.registerHooks')), + ).toBe(true); + expect(records.some((r) => typeof r.msg === 'string' && r.msg.includes('is ready'))).toBe( + false, + ); + }); }); }); diff --git a/gitnexus/test/unit/embeddings-sync-command.test.ts b/gitnexus/test/unit/embeddings-sync-command.test.ts index 7140c53c5..01afa40c3 100644 --- a/gitnexus/test/unit/embeddings-sync-command.test.ts +++ b/gitnexus/test/unit/embeddings-sync-command.test.ts @@ -21,6 +21,10 @@ const { fetchExistingEmbeddingHashesMock, runEmbeddingPipelineMock, resolveEmbeddingIdentityMock, + installEmbeddingRuntimeMock, + resolveEmbeddingRuntimeMock, + isPrefixRuntimeLoadableMock, + reapEmbeddingSidecarMock, } = vi.hoisted(() => ({ acquireIndexLockMock: vi.fn(), releaseMock: vi.fn(), @@ -34,6 +38,10 @@ const { fetchExistingEmbeddingHashesMock: vi.fn(), runEmbeddingPipelineMock: vi.fn(), resolveEmbeddingIdentityMock: vi.fn(), + installEmbeddingRuntimeMock: vi.fn(), + resolveEmbeddingRuntimeMock: vi.fn(), + isPrefixRuntimeLoadableMock: vi.fn(), + reapEmbeddingSidecarMock: vi.fn(), })); vi.mock('../../src/storage/git.js', () => ({ @@ -67,6 +75,17 @@ vi.mock('../../src/core/embeddings/embedding-identity.js', () => ({ resolveEmbeddingIdentity: () => resolveEmbeddingIdentityMock(), })); +vi.mock('../../src/core/embeddings/runtime-install.js', async (importOriginal) => ({ + ...(await importOriginal()), + installEmbeddingRuntime: (...args: unknown[]) => installEmbeddingRuntimeMock(...args), + resolveEmbeddingRuntime: () => resolveEmbeddingRuntimeMock(), + isPrefixRuntimeLoadable: () => isPrefixRuntimeLoadableMock(), +})); + +vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ + reapEmbeddingSidecar: () => reapEmbeddingSidecarMock(), +})); + const IDENTITY = { model: 'test-model', dimensions: 768, provider: 'local' } as const; const BASE_META = { @@ -96,6 +115,8 @@ async function run(inputPath = '/tmp/emb-sync-repo') { describe('embeddingsSyncCommand writer safety (#3065)', () => { const tmpDirs: string[] = []; + const originalEmbeddingUrl = process.env.GITNEXUS_EMBEDDING_URL; + const originalEmbeddingModel = process.env.GITNEXUS_EMBEDDING_MODEL; async function store(kind: 'file' | 'missing' | 'dir' = 'file') { const dir = await mkdtemp(path.join(tmpdir(), 'emb-sync-')); @@ -126,9 +147,19 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { failedNodeIds: [], }); resolveEmbeddingIdentityMock.mockReset().mockReturnValue({ ...IDENTITY }); + installEmbeddingRuntimeMock.mockReset().mockResolvedValue(undefined); + resolveEmbeddingRuntimeMock.mockReset().mockReturnValue({ source: 'package' }); + isPrefixRuntimeLoadableMock.mockReset().mockReturnValue(true); + reapEmbeddingSidecarMock.mockReset(); + delete process.env.GITNEXUS_EMBEDDING_URL; + delete process.env.GITNEXUS_EMBEDDING_MODEL; }); afterEach(async () => { + if (originalEmbeddingUrl === undefined) delete process.env.GITNEXUS_EMBEDDING_URL; + else process.env.GITNEXUS_EMBEDDING_URL = originalEmbeddingUrl; + if (originalEmbeddingModel === undefined) delete process.env.GITNEXUS_EMBEDDING_MODEL; + else process.env.GITNEXUS_EMBEDDING_MODEL = originalEmbeddingModel; await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); @@ -354,6 +385,61 @@ describe('embeddingsSyncCommand writer safety (#3065)', () => { expect(releaseMock).toHaveBeenCalled(); }); + it('keeps the pipeline error when sidecar reap also throws', async () => { + await store(); + runEmbeddingPipelineMock.mockRejectedValue(new Error('pipeline boom')); + reapEmbeddingSidecarMock.mockImplementation(() => { + throw new Error('reap boom'); + }); + + await expect(run()).rejects.toThrow('pipeline boom'); + expect(releaseMock).toHaveBeenCalled(); + }); + + it('does not spawn npm on darwin/x64', async () => { + await store(); + resolveEmbeddingRuntimeMock.mockReturnValue(null); + const orig = { platform: process.platform, arch: process.arch }; + Object.defineProperty(process, 'platform', { value: 'darwin', configurable: true }); + Object.defineProperty(process, 'arch', { value: 'x64', configurable: true }); + try { + await expect(run()).rejects.toThrow(/macOS Intel/); + expect(installEmbeddingRuntimeMock).not.toHaveBeenCalled(); + } finally { + Object.defineProperty(process, 'platform', { value: orig.platform, configurable: true }); + Object.defineProperty(process, 'arch', { value: orig.arch, configurable: true }); + } + }); + + it('auto-heals a missing stack without requesting CUDA binaries', async () => { + await store(); + resolveEmbeddingRuntimeMock.mockReturnValue(null); + + await run(); + + expect(installEmbeddingRuntimeMock).toHaveBeenCalledTimes(1); + expect(installEmbeddingRuntimeMock.mock.calls[0]?.[0]).toEqual({}); + expect(installEmbeddingRuntimeMock.mock.calls[0]?.[0]).not.toMatchObject({ cuda: true }); + }); + + it('wraps a failed auto-install with the missing-stack recovery path', async () => { + await store(); + resolveEmbeddingRuntimeMock.mockReturnValue(null); + installEmbeddingRuntimeMock.mockRejectedValue(new Error('npm install timed out')); + + await expect(run()).rejects.toThrow( + /Could not install the embedding runtime[\s\S]*gitnexus embeddings install/, + ); + expect(initLbugMock).not.toHaveBeenCalled(); + }); + + it('does not statically import the embedding pipeline', async () => { + const { readFileSync } = await import('node:fs'); + const src = readFileSync(new URL('../../src/cli/embeddings-sync.ts', import.meta.url), 'utf8'); + expect(src).not.toMatch(/^import .*embedding-pipeline/m); + expect(src).toContain("await import('../core/embeddings/embedding-pipeline.js')"); + }); + it('loads existing hashes without materializing cached vectors', async () => { await store(); const hashes = new Map([ diff --git a/gitnexus/test/unit/fts-artifact-coverage.test.ts b/gitnexus/test/unit/fts-artifact-coverage.test.ts index 35f252f20..a415ff858 100644 --- a/gitnexus/test/unit/fts-artifact-coverage.test.ts +++ b/gitnexus/test/unit/fts-artifact-coverage.test.ts @@ -102,7 +102,9 @@ describe('real repo pairing (guards against a silent core bump)', () => { ) as { coreVersion: string; extensionVersion: string }; expect(pkg.dependencies['@ladybugdb/core']).toBe(manifest.coreVersion); expect(manifest.extensionVersion).toMatch(/^\d+\.\d+\.\d+$/); - expect(pkg.files).toContain('vendor'); + expect(filesCoverFtsArtifacts(pkg.files)).toBe(true); + expect(pkg.files).toContain('vendor/**/prebuilds/**'); + expect(pkg.files).not.toContain('vendor'); }); }); diff --git a/gitnexus/test/unit/hf-env.test.ts b/gitnexus/test/unit/hf-env.test.ts index 5999b34a2..8a55fcbb4 100644 --- a/gitnexus/test/unit/hf-env.test.ts +++ b/gitnexus/test/unit/hf-env.test.ts @@ -326,6 +326,16 @@ describe('withHfDownloadRetry env overrides', () => { expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS); }); + it('HF_MAX_ATTEMPTS=0.5 floors below 1 and falls back to the built-in default', async () => { + process.env.HF_MAX_ATTEMPTS = '0.5'; + const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); + await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( + 'fetch failed', + ); + expect(fn).toHaveBeenCalledTimes(HF_MAX_ATTEMPTS); + }); + it('HF_MAX_ATTEMPTS=0 falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = '0'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); diff --git a/gitnexus/test/unit/http-embedder.test.ts b/gitnexus/test/unit/http-embedder.test.ts index 4a03ee2d2..01772dc6b 100644 --- a/gitnexus/test/unit/http-embedder.test.ts +++ b/gitnexus/test/unit/http-embedder.test.ts @@ -61,8 +61,13 @@ describe('HTTP embedding backend', () => { expect(getEmbeddingDims()).toBe(384); }); - it('returns false before initialization', () => { - expect(isEmbedderReady()).toBe(false); + it('is ready from HTTP mode or a ready local runtime, not resolution alone', async () => { + const { isHttpMode } = await import('../../src/core/embeddings/http-client.js'); + const { assessLocalEmbeddingRuntime } = + await import('../../src/core/embeddings/runtime-support.js'); + expect(isEmbedderReady()).toBe( + isHttpMode() || assessLocalEmbeddingRuntime().status === 'ready', + ); }); it('returns true when HTTP environment variables are set', async () => { diff --git a/gitnexus/test/unit/hybrid-search.test.ts b/gitnexus/test/unit/hybrid-search.test.ts index 813f78aa7..771296bc1 100644 --- a/gitnexus/test/unit/hybrid-search.test.ts +++ b/gitnexus/test/unit/hybrid-search.test.ts @@ -199,4 +199,20 @@ describe('hybridSearch — FTS failure fallback (#1489)', () => { expect(results[0].filePath).toBe('src/fts-hit.ts'); expect(results[0].sources).toEqual(['bm25']); }); + + it('keeps BM25 results when semantic search throws', async () => { + const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js'); + vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ + results: [{ filePath: 'src/fts-hit.ts', score: 5, rank: 1 }], + ftsAvailable: true, + }); + + const mockExecuteQuery = vi.fn().mockResolvedValue([]); + const mockSemanticSearch = vi.fn().mockRejectedValue(new Error('sidecar aborted')); + + const results = await hybridSearch('test query', 10, mockExecuteQuery, mockSemanticSearch); + expect(results).toHaveLength(1); + expect(results[0].filePath).toBe('src/fts-hit.ts'); + expect(results[0].sources).toEqual(['bm25']); + }); }); diff --git a/gitnexus/test/unit/local-backend-disconnect-reap.test.ts b/gitnexus/test/unit/local-backend-disconnect-reap.test.ts new file mode 100644 index 000000000..afa4c2099 --- /dev/null +++ b/gitnexus/test/unit/local-backend-disconnect-reap.test.ts @@ -0,0 +1,31 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { closeLbugMock, reapEmbeddingSidecarMock } = vi.hoisted(() => ({ + closeLbugMock: vi.fn(), + reapEmbeddingSidecarMock: vi.fn(), +})); + +vi.mock('../../src/core/lbug/pool-adapter.js', async (importOriginal) => ({ + ...(await importOriginal()), + closeLbug: (...args: unknown[]) => closeLbugMock(...args), +})); + +vi.mock('../../src/core/embeddings/embedding-sidecar-client.js', () => ({ + reapEmbeddingSidecar: () => reapEmbeddingSidecarMock(), +})); + +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; + +describe('LocalBackend.disconnect sidecar reap', () => { + beforeEach(() => { + closeLbugMock.mockReset(); + reapEmbeddingSidecarMock.mockReset(); + }); + + it('reaps the sidecar when closeLbug rejects', async () => { + closeLbugMock.mockRejectedValue(new Error('ladybug close failed')); + const backend = new LocalBackend(); + await expect(backend.disconnect()).rejects.toThrow('ladybug close failed'); + expect(reapEmbeddingSidecarMock).toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/test/unit/local-backend-semantic-warn.test.ts b/gitnexus/test/unit/local-backend-semantic-warn.test.ts index c3aaffff6..9683b7709 100644 --- a/gitnexus/test/unit/local-backend-semantic-warn.test.ts +++ b/gitnexus/test/unit/local-backend-semantic-warn.test.ts @@ -1,13 +1,16 @@ /** - * Tests that MCP semantic search surfaces a pruned/unloadable optional embedding - * stack once instead of silently degrading to BM25 (#2372) — the silent- - * degradation mode #2370 exists to fix. executeQuery is mocked to report a - * populated embedding table so execution reaches the embedder import, which is - * mocked to throw the missing-stack message. + * Tests that MCP semantic search surfaces a missing local embedding stack once + * instead of silently degrading to BM25 (#2372) — the silent-degradation mode + * #2370 exists to fix. executeQuery is mocked to report a populated embedding + * table so execution reaches the embedder import, which is mocked to throw the + * missing-stack message (R20 copy: default install no longer ships the stack). */ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { _captureLogger, type LoggerCapture } from '../../src/core/logger.js'; -import { localEmbeddingStackMissingMessage } from '../../src/core/embeddings/runtime-support.js'; +import { + LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD, + localEmbeddingStackMissingMessage, +} from '../../src/core/embeddings/runtime-support.js'; const executeQueryMock = vi.fn(); const embedQueryMock = vi.fn(); @@ -24,10 +27,15 @@ vi.mock('../../src/mcp/core/embedder.js', () => ({ import { LocalBackend } from '../../src/mcp/local/local-backend.js'; interface SemanticSearchable { - semanticSearch(repo: { lbugPath: string }, query: string, limit: number): Promise; + semanticSearch( + repo: { lbugPath: string }, + query: string, + limit: number, + degraded?: { reason?: string }, + ): Promise; } -const callSemanticSearch = (b: LocalBackend): Promise => - (b as unknown as SemanticSearchable).semanticSearch({ lbugPath: '/tmp/x' }, 'q', 5); +const callSemanticSearch = (b: LocalBackend, degraded?: { reason?: string }): Promise => + (b as unknown as SemanticSearchable).semanticSearch({ lbugPath: '/tmp/x' }, 'q', 5, degraded); const stackWarns = (cap: LoggerCapture): number => cap @@ -36,7 +44,7 @@ const stackWarns = (cap: LoggerCapture): number => (r) => typeof r.msg === 'string' && r.msg.includes('query:vector') && - r.msg.includes('optional embedding stack'), + r.msg.includes('local embedding stack is not installed'), ).length; describe('LocalBackend.semanticSearch — missing-stack warning (#2372)', () => { @@ -49,10 +57,30 @@ describe('LocalBackend.semanticSearch — missing-stack warning (#2372)', () => embedQueryMock.mockRejectedValue(new Error(localEmbeddingStackMissingMessage())); const backend = new LocalBackend(); const cap = _captureLogger(); + const first = { reason: undefined as string | undefined }; + const second = { reason: undefined as string | undefined }; try { - expect(await callSemanticSearch(backend)).toEqual([]); - expect(await callSemanticSearch(backend)).toEqual([]); + expect(await callSemanticSearch(backend, first)).toEqual([]); + expect(await callSemanticSearch(backend, second)).toEqual([]); expect(stackWarns(cap)).toBe(1); // once per LocalBackend instance + expect(first.reason).toContain('local embedding stack is not installed'); + expect(second.reason).toContain('local embedding stack is not installed'); + } finally { + cap.restore(); + } + }); + + it('stashes sidecar-abort text for query() warnings', async () => { + embedQueryMock.mockRejectedValue(new Error(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD)); + const backend = new LocalBackend(); + const cap = _captureLogger(); + const degraded = { reason: undefined as string | undefined }; + try { + expect(await callSemanticSearch(backend, degraded)).toEqual([]); + expect(degraded.reason).toBe(LOCAL_EMBEDDING_SIDECAR_ABORT_LEAD); + expect( + cap.records().some((r) => typeof r.msg === 'string' && r.msg.includes('sidecar aborted')), + ).toBe(true); } finally { cap.restore(); } diff --git a/gitnexus/test/unit/semantic-search-ready.test.ts b/gitnexus/test/unit/semantic-search-ready.test.ts new file mode 100644 index 000000000..0c2ad3f17 --- /dev/null +++ b/gitnexus/test/unit/semantic-search-ready.test.ts @@ -0,0 +1,58 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const { embedTextMock, isEmbedderReadyMock, loadVectorExtensionMock } = vi.hoisted(() => ({ + embedTextMock: vi.fn(), + isEmbedderReadyMock: vi.fn(), + loadVectorExtensionMock: vi.fn(async () => false), +})); + +vi.mock('../../src/core/embeddings/embedder.js', () => ({ + initEmbedder: vi.fn(), + embedBatch: vi.fn(), + embedText: (...args: unknown[]) => embedTextMock(...args), + embeddingToArray: (embedding: Float32Array) => Array.from(embedding), + isEmbedderReady: () => isEmbedderReadyMock(), +})); + +vi.mock('../../src/core/lbug/lbug-adapter.js', () => ({ + loadVectorExtension: (...args: unknown[]) => loadVectorExtensionMock(...args), + createVectorIndex: vi.fn(), +})); + +describe('semanticSearch ready-check (KTD11)', () => { + beforeEach(() => { + embedTextMock.mockReset(); + isEmbedderReadyMock.mockReset(); + loadVectorExtensionMock.mockReset().mockResolvedValue(false); + }); + + it('does not throw when the stack is resolvable and no in-process singleton exists', async () => { + isEmbedderReadyMock.mockReturnValue(true); + embedTextMock.mockResolvedValue(new Float32Array(384)); + const executeQuery = vi.fn(async (cypher: string) => { + if (cypher.includes('RETURN 1 AS ok')) return [{ ok: 1 }]; + if (cypher.includes('count(e) AS cnt')) return [{ cnt: 2 }]; + return []; + }); + + const { semanticSearch } = await import('../../src/core/embeddings/embedding-pipeline.js'); + await expect(semanticSearch(executeQuery, 'find auth', 5)).resolves.toEqual([]); + expect(embedTextMock).toHaveBeenCalledTimes(1); + expect(embedTextMock.mock.calls[0]?.[0]).toBe('find auth'); + }); + + it('does not request a query vector when the embedding table is empty', async () => { + isEmbedderReadyMock.mockReturnValue(true); + embedTextMock.mockResolvedValue(new Float32Array(384)); + const executeQuery = vi.fn(async (cypher: string) => { + if (cypher.includes('RETURN 1 AS ok')) return []; + if (cypher.includes('count(e) AS cnt')) return [{ cnt: 0 }]; + return []; + }); + + const { semanticSearch } = await import('../../src/core/embeddings/embedding-pipeline.js'); + await expect(semanticSearch(executeQuery, 'find auth', 5)).resolves.toEqual([]); + expect(embedTextMock).not.toHaveBeenCalled(); + expect(loadVectorExtensionMock).not.toHaveBeenCalled(); + }); +});