Merge pull request #300 from jim80net/fix/onnxruntime-cuda-provider

gitnexus/package-lock.json

@@ -24,6 +24,7 @@
         "ignore": "^7.0.5",
         "lru-cache": "^11.0.0",
         "mnemonist": "^0.39.0",
+        "onnxruntime-node": "^1.24.0",
         "pandemonium": "^2.4.0",
         "tree-sitter": "^0.21.0",
         "tree-sitter-c": "^0.21.0",
@@ -2403,6 +2404,15 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/adm-zip": {
+      "version": "0.5.16",
+      "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz",
+      "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=12.0"
+      }
+    },
     "node_modules/ajv": {
       "version": "8.17.1",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz",
@@ -4040,15 +4050,15 @@
       }
     },
     "node_modules/onnxruntime-common": {
-      "version": "1.21.0",
-      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.21.0.tgz",
-      "integrity": "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ==",
+      "version": "1.24.3",
+      "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz",
+      "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==",
       "license": "MIT"
     },
     "node_modules/onnxruntime-node": {
-      "version": "1.21.0",
-      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.21.0.tgz",
-      "integrity": "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw==",
+      "version": "1.24.3",
+      "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz",
+      "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==",
       "hasInstallScript": true,
       "license": "MIT",
       "os": [
@@ -4057,55 +4067,9 @@
         "linux"
       ],
       "dependencies": {
+        "adm-zip": "^0.5.16",
         "global-agent": "^3.0.0",
-        "onnxruntime-common": "1.21.0",
-        "tar": "^7.0.1"
-      }
-    },
-    "node_modules/onnxruntime-node/node_modules/chownr": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz",
-      "integrity": "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==",
-      "license": "BlueOak-1.0.0",
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/onnxruntime-node/node_modules/minizlib": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz",
-      "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==",
-      "license": "MIT",
-      "dependencies": {
-        "minipass": "^7.1.2"
-      },
-      "engines": {
-        "node": ">= 18"
-      }
-    },
-    "node_modules/onnxruntime-node/node_modules/tar": {
-      "version": "7.5.7",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.7.tgz",
-      "integrity": "sha512-fov56fJiRuThVFXD6o6/Q354S7pnWMJIVlDBYijsTNx6jKSE4pvrDTs6lUnmGvNyfJwFQQwWy3owKz1ucIhveQ==",
-      "license": "BlueOak-1.0.0",
-      "dependencies": {
-        "@isaacs/fs-minipass": "^4.0.0",
-        "chownr": "^3.0.0",
-        "minipass": "^7.1.2",
-        "minizlib": "^3.1.0",
-        "yallist": "^5.0.0"
-      },
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/onnxruntime-node/node_modules/yallist": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-5.0.0.tgz",
-      "integrity": "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==",
-      "license": "BlueOak-1.0.0",
-      "engines": {
-        "node": ">=18"
+        "onnxruntime-common": "1.24.3"
       }
     },
     "node_modules/onnxruntime-web": {

gitnexus/package.json

@@ -51,6 +51,7 @@
   },
   "dependencies": {
     "@huggingface/transformers": "^3.0.0",
+    "onnxruntime-node": "^1.24.0",
     "@modelcontextprotocol/sdk": "^1.0.0",
     "cli-progress": "^3.12.0",
     "commander": "^12.0.0",
@@ -94,6 +95,11 @@
     "typescript": "^5.4.5",
     "vitest": "^4.0.18"
   },
+  "overrides": {
+    "@huggingface/transformers": {
+      "onnxruntime-node": "$onnxruntime-node"
+    }
+  },
   "engines": {
     "node": ">=18.0.0"
   }

gitnexus/src/core/embeddings/embedder.ts

@@ -17,18 +17,53 @@ if (!process.env.ORT_LOG_LEVEL) {
 import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
 import { existsSync } from 'fs';
 import { execFileSync } from 'child_process';
-import { join } from 'path';
+import { join, dirname } from 'path';
+import { createRequire } from 'module';
 import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
 
+/**
+ * Check whether the onnxruntime-node package that @huggingface/transformers
+ * will actually load at runtime ships the CUDA execution provider.
+ *
+ * Critical: we resolve from transformers' own module scope, NOT from ours.
+ * npm may install two copies — a top-level 1.24.x (our dep) and a nested
+ * 1.21.0 (transformers' pinned dep). The guard must inspect whichever copy
+ * transformers.js will dlopen, otherwise the check is meaningless.
+ */
+function hasOrtCudaProvider(): boolean {
+  try {
+    const require = createRequire(import.meta.url);
+    // Resolve from @huggingface/transformers' scope so we find the same
+    // onnxruntime-node binary that transformers.js will use at runtime
+    const transformersDir = dirname(require.resolve('@huggingface/transformers/package.json'));
+    const ortRequire = createRequire(join(transformersDir, 'package.json'));
+    const ortPath = dirname(ortRequire.resolve('onnxruntime-node/package.json'));
+    // ORT 1.24.x only ships CUDA binaries for linux/x64 (downloaded from NuGet
+    // at postinstall). arm64 will correctly return false here until ORT adds support.
+    const arch = process.arch;
+    return existsSync(join(ortPath, 'bin', 'napi-v6', 'linux', arch, 'libonnxruntime_providers_cuda.so'));
+  } catch {
+    return false;
+  }
+}
+
 /**
  * Check whether CUDA libraries are actually available on this system.
  * ONNX Runtime's native layer crashes (uncatchable) if we attempt CUDA
  * without the required shared libraries, so we probe first.
  *
- * Checks the dynamic linker cache (ldconfig) which covers all architectures
- * and install paths, then falls back to CUDA_PATH / LD_LIBRARY_PATH env vars.
+ * Checks both:
+ * 1. That system CUDA libraries (libcublasLt) are present
+ * 2. That onnxruntime-node ships the CUDA execution provider binary
+ *
+ * Both conditions must be true — system CUDA libs alone are not enough
+ * if onnxruntime-node is a CPU-only build (versions < 1.24.0).
  */
 function isCudaAvailable(): boolean {
+  // First, verify onnxruntime-node has the CUDA provider binary.
+  // Without this, requesting CUDA causes an uncatchable native crash.
+  if (!hasOrtCudaProvider()) return false;
+
   // Primary: query the dynamic linker cache — covers all architectures,
   // distro layouts, and custom install paths registered with ldconfig
   try {