fix(embeddings): bridge HF_ENDPOINT env var to transformers.js env.remoteHost (#1205) (#1252)

gitnexus/src/core/embeddings/embedder.ts

@@ -15,7 +15,6 @@ if (!process.env.ORT_LOG_LEVEL) {
 }
 
 import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
-import os from 'os';
 import { existsSync } from 'fs';
 import { execFileSync } from 'child_process';
 import { join, dirname } from 'path';
@@ -23,6 +22,7 @@ import { createRequire } from 'module';
 import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
 import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
 import { resolveEmbeddingConfig } from './config.js';
+import { applyHfEnvOverrides } from './hf-env.js';
 
 /**
  * Check whether the onnxruntime-node package that @huggingface/transformers
@@ -158,11 +158,11 @@ export const initEmbedder = async (
     try {
       // Configure transformers.js environment
       env.allowLocalModels = false;
-      // Default cache to user-writable location. transformers.js defaults to
-      // ./node_modules/.cache inside its own install dir, which is unwritable
-      // when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
-      // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
-      env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+      // Bridge user-controlled env vars to transformers.js: HF_HOME →
+      // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
+      // applyHfEnvOverrides so the MCP embedder entry point behaves
+      // identically.
+      applyHfEnvOverrides(env);
 
       const isDev = process.env.NODE_ENV === 'development';
       if (isDev) {

gitnexus/src/core/embeddings/hf-env.ts

@@ -0,0 +1,62 @@
+import os from 'node:os';
+import { join } from 'node:path';
+
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Minimal subset of `@huggingface/transformers`' `env` object that gitnexus
+ * mutates. Defining a local structural type keeps this helper free of a
+ * transitive dependency on transformers' generated `.d.ts` while still
+ * giving full type-checking on the two fields we actually touch.
+ */
+export interface HfEnvSubset {
+  cacheDir: string;
+  remoteHost: string;
+}
+
+/**
+ * @internal Exported only for unit tests and the two embedder entry points
+ * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
+ * public package API.
+ *
+ * Apply user-controlled HuggingFace environment overrides to the
+ * `@huggingface/transformers` `env` object. Centralises the two env-var
+ * bridges so every gitnexus embedder entry point (the analyze pipeline
+ * and the MCP server) behaves identically.
+ *
+ * - **`HF_HOME`** → `env.cacheDir` (default: `~/.cache/huggingface`).
+ *   transformers.js otherwise defaults to `./node_modules/.cache` inside
+ *   its own install dir, which is unwritable when gitnexus is installed
+ *   globally (e.g. `/usr/lib/node_modules/`).
+ *
+ * - **`HF_ENDPOINT`** → `env.remoteHost` (#1205). transformers.js does
+ *   not read `HF_ENDPOINT` on its own — it reads `env.remoteHost` —
+ *   even though `HF_ENDPOINT` is the standard env var the upstream
+ *   `huggingface_hub` Python client and the official HF mirror docs
+ *   tell users to set. Bridging the two unblocks `--embeddings` for
+ *   users behind networks where `huggingface.co` is unreachable
+ *   (corporate proxies, the GFW, air-gapped mirrors). The trailing
+ *   slash is normalised because transformers.js builds URLs by string
+ *   concatenation and a missing slash silently falls through to its
+ *   default `huggingface.co/...` host.
+ *
+ * Mutation rather than return-and-apply because callers already hold a
+ * reference to the live `env` object imported from
+ * `@huggingface/transformers` — passing the same reference in keeps the
+ * call site a single line at each entry point.
+ */
+export function applyHfEnvOverrides(env: HfEnvSubset): void {
+  env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+  // `.trim()` guards against the common copy-paste failure mode of
+  // `HF_ENDPOINT="  https://hf-mirror.com  "` (leading/trailing whitespace
+  // from shell scripts or docs) — without it, a whitespace-only value
+  // would be truthy and produce an invalid `env.remoteHost = '   /'` that
+  // silently misroutes downloads. Empty string remains falsy in JS so the
+  // truthy guard already handles the unset/empty cases.
+  const endpoint = process.env.HF_ENDPOINT?.trim();
+  if (endpoint) {
+    env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/';
+  }
+}

gitnexus/src/mcp/core/embedder.ts

@@ -6,14 +6,13 @@
  */
 
 import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
-import os from 'os';
-import { join } from 'path';
 import {
   isHttpMode,
   getHttpDimensions,
   httpEmbedQuery,
 } from '../../core/embeddings/http-client.js';
 import { resolveEmbeddingConfig } from '../../core/embeddings/config.js';
+import { applyHfEnvOverrides } from '../../core/embeddings/hf-env.js';
 import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js';
 
 // Model config
@@ -45,11 +44,11 @@ export const initEmbedder = async (): Promise<FeatureExtractionPipeline> => {
   initPromise = (async () => {
     try {
       env.allowLocalModels = false;
-      // Default cache to user-writable location. transformers.js defaults to
-      // ./node_modules/.cache inside its own install dir, which is unwritable
-      // when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
-      // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
-      env.cacheDir = process.env.HF_HOME ?? join(os.homedir(), '.cache', 'huggingface');
+      // Bridge user-controlled env vars to transformers.js: HF_HOME →
+      // env.cacheDir, HF_ENDPOINT → env.remoteHost (#1205). Centralised in
+      // applyHfEnvOverrides so this MCP entry point behaves identically to
+      // the analyze pipeline embedder.
+      applyHfEnvOverrides(env);
       const embeddingConfig = resolveEmbeddingConfig();
 
       console.error('GitNexus: Loading embedding model (first search may take a moment)...');

gitnexus/test/unit/hf-env.test.ts

@@ -0,0 +1,84 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import os from 'node:os';
+import { join } from 'node:path';
+import { applyHfEnvOverrides, type HfEnvSubset } from '../../src/core/embeddings/hf-env.js';
+
+describe('applyHfEnvOverrides', () => {
+  let envStub: HfEnvSubset;
+  // Snapshot the two env vars so tests don't leak state into each other (or
+  // into the rest of the test run). `delete` + restore is the simplest pattern
+  // — vitest doesn't reset `process.env` between tests by default.
+  let originalHfHome: string | undefined;
+  let originalHfEndpoint: string | undefined;
+
+  beforeEach(() => {
+    envStub = { cacheDir: '', remoteHost: '' };
+    originalHfHome = process.env.HF_HOME;
+    originalHfEndpoint = process.env.HF_ENDPOINT;
+    delete process.env.HF_HOME;
+    delete process.env.HF_ENDPOINT;
+  });
+
+  afterEach(() => {
+    if (originalHfHome === undefined) delete process.env.HF_HOME;
+    else process.env.HF_HOME = originalHfHome;
+    if (originalHfEndpoint === undefined) delete process.env.HF_ENDPOINT;
+    else process.env.HF_ENDPOINT = originalHfEndpoint;
+  });
+
+  it('cacheDir defaults to ~/.cache/huggingface when HF_HOME is unset', () => {
+    applyHfEnvOverrides(envStub);
+    expect(envStub.cacheDir).toBe(join(os.homedir(), '.cache', 'huggingface'));
+  });
+
+  it('cacheDir respects HF_HOME when set', () => {
+    process.env.HF_HOME = '/custom/hf/cache';
+    applyHfEnvOverrides(envStub);
+    expect(envStub.cacheDir).toBe('/custom/hf/cache');
+  });
+
+  it('remoteHost is set when HF_ENDPOINT is set, with a trailing slash appended', () => {
+    process.env.HF_ENDPOINT = 'https://hf-mirror.com';
+    applyHfEnvOverrides(envStub);
+    expect(envStub.remoteHost).toBe('https://hf-mirror.com/');
+  });
+
+  it('remoteHost preserves existing trailing slash on HF_ENDPOINT', () => {
+    process.env.HF_ENDPOINT = 'https://hf-mirror.com/';
+    applyHfEnvOverrides(envStub);
+    expect(envStub.remoteHost).toBe('https://hf-mirror.com/');
+  });
+
+  it('remoteHost is left untouched when HF_ENDPOINT is unset', () => {
+    // Pre-populate to a sentinel so we can prove the function does NOT
+    // overwrite remoteHost when no env var is set. Without this guard a
+    // future refactor that always assigns `env.remoteHost = ...` would
+    // silently break consumers that have already configured it elsewhere.
+    envStub.remoteHost = 'pre-existing-do-not-touch';
+    applyHfEnvOverrides(envStub);
+    expect(envStub.remoteHost).toBe('pre-existing-do-not-touch');
+  });
+
+  it('remoteHost is left untouched when HF_ENDPOINT is whitespace-only', () => {
+    // Common copy-paste failure mode for users on restricted networks who
+    // pull `HF_ENDPOINT` values from shell scripts or docs with stray
+    // whitespace. The `.trim()` + truthiness guard ensures this is treated
+    // as "unset" rather than as an invalid host like `'   /'` that would
+    // silently misroute model downloads. Pinned by the @claude review on
+    // PR #1252.
+    process.env.HF_ENDPOINT = '   ';
+    envStub.remoteHost = 'sentinel';
+    applyHfEnvOverrides(envStub);
+    expect(envStub.remoteHost).toBe('sentinel');
+  });
+
+  it('remoteHost trims surrounding whitespace from HF_ENDPOINT', () => {
+    // Compatible mirror of the previous test for the case where the env
+    // var is non-empty AFTER trimming. Without `.trim()`, the bogus
+    // leading/trailing space would survive into the URL and break
+    // downloads.
+    process.env.HF_ENDPOINT = '  https://hf-mirror.com  ';
+    applyHfEnvOverrides(envStub);
+    expect(envStub.remoteHost).toBe('https://hf-mirror.com/');
+  });
+});