Merge branch 'main' into dependabot/npm_and_yarn/gitnexus/ladybugdb/core-0.18.2

gitnexus-web/package-lock.json

@@ -7894,9 +7894,9 @@
       }
     },
     "node_modules/tar": {
-      "version": "7.5.16",
-      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.16.tgz",
-      "integrity": "sha512-56adEpPMouktRlBLXiaYFFzZ/3+JXa8P9n7WbR+ibIjtviN55mEaOkiysCnPnWm+7kkui1Dn8J9l+g6zV8731w==",
+      "version": "7.5.20",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.20.tgz",
+      "integrity": "sha512-9FcyK4PA6+WbzlTM9WhQm6vB5W7cP7dUiPsv1g7YDwEQnQ1CGpK3MGlKk/ITVWMk05kHZuBhmVhiv8LZoy/PFQ==",
       "dev": true,
       "license": "BlueOak-1.0.0",
       "dependencies": {

gitnexus/README.md

@@ -284,6 +284,7 @@ Set these env vars to use a remote OpenAI-compatible `/v1/embeddings` endpoint i
 export GITNEXUS_EMBEDDING_URL=http://your-server:8080/v1
 export GITNEXUS_EMBEDDING_MODEL=BAAI/bge-large-en-v1.5
 export GITNEXUS_EMBEDDING_DIMS=1024          # optional, default 384
+export GITNEXUS_EMBEDDING_REQUEST_DIMS=omit  # optional: omit "dimensions", or an integer to override it
 export GITNEXUS_EMBEDDING_API_KEY=your-key   # optional, default: "unused"
 export GITNEXUS_EMBEDDING_MAX_ATTEMPTS=3     # optional, total attempts (1-20)
 export GITNEXUS_EMBEDDING_RETRY_CAP_MS=5000  # optional, maximum retry delay
@@ -291,6 +292,15 @@ export GITNEXUS_EMBEDDING_MIN_INTERVAL_MS=0  # optional, minimum request spacing
 gitnexus analyze . --embeddings
 ```
 
+`GITNEXUS_EMBEDDING_REQUEST_DIMS` controls only the `dimensions` field sent in
+the request body, independently of `GITNEXUS_EMBEDDING_DIMS` (which still
+validates the returned vector's length):
+
+- `omit` (or `none`, `off`, `false`, `0`) — do not send `dimensions` at all, for
+  strict backends that return the right vector size but reject the field.
+- a positive integer — send that value instead of `GITNEXUS_EMBEDDING_DIMS`.
+- unset — send `GITNEXUS_EMBEDDING_DIMS` (the previous behavior).
+
 Works with Infinity, vLLM, TEI, llama.cpp, Ollama, LM Studio, or OpenAI. Retry and pacing settings are provider-neutral; provider-specific limits should be supplied through configuration. When unset, local embeddings are used unchanged.
 
 ## Multi-Repo Support

gitnexus/package-lock.json

@@ -24,7 +24,7 @@
         "graphology-indices": "^0.17.0",
         "graphology-utils": "^2.3.0",
         "ignore": "^7.0.5",
-        "js-yaml": "^5.0.0",
+        "js-yaml": "^4.1.1",
         "jsonc-parser": "^3.3.1",
         "mnemonist": "^0.40.3",
         "node-addon-api": "^8.0.0",
@@ -3589,9 +3589,9 @@
       "license": "MIT"
     },
     "node_modules/js-yaml": {
-      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-5.0.0.tgz",
-      "integrity": "sha512-GSvaPUbk1U+FMZ7rJzF+F8e5YVtu7KnD40et/5rBXXRBv2jCO9L3qCewvIDDdudC0QycTFlf6EAA+h3kxBsuUw==",
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.0.tgz",
+      "integrity": "sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==",
       "funding": [
         {
           "type": "github",
@@ -3607,7 +3607,7 @@
         "argparse": "^2.0.1"
       },
       "bin": {
-        "js-yaml": "bin/js-yaml.mjs"
+        "js-yaml": "bin/js-yaml.js"
       }
     },
     "node_modules/jsesc": {

gitnexus/package.json

@@ -70,7 +70,7 @@
     "graphology-indices": "^0.17.0",
     "graphology-utils": "^2.3.0",
     "ignore": "^7.0.5",
-    "js-yaml": "^5.0.0",
+    "js-yaml": "^4.1.1",
     "jsonc-parser": "^3.3.1",
     "mnemonist": "^0.40.3",
     "node-addon-api": "^8.0.0",

gitnexus/src/core/embeddings/http-client.ts

@@ -29,6 +29,7 @@ interface HttpConfig {
   maxAttempts: number;
   retryCapMs: number;
   minIntervalMs: number;
+  requestDimensions?: number;
 }
 
 export interface EmbeddingRequestOptions {
@@ -106,20 +107,26 @@ const paceHttpRequest = async (minIntervalMs: number, signal?: AbortSignal): Pro
 };
 
 /**
- * Stable lead of the {@link readConfig} malformed-`GITNEXUS_EMBEDDING_DIMS`
- * error. `readConfig` throws a plain `Error` (not an {@link HttpEmbeddingError})
- * because this is a *config* mistake, not an endpoint failure — so the CLI
- * recognizes it by this lead ({@link isHttpEmbeddingDimsError}) and prints a
- * clean config message instead of a raw stack dump. See #2385.
+ * Stable lead of a {@link readConfig} malformed dims-env error. `readConfig`
+ * throws a plain `Error` (not an {@link HttpEmbeddingError}) for a malformed
+ * `GITNEXUS_EMBEDDING_DIMS` or `GITNEXUS_EMBEDDING_REQUEST_DIMS` because it's a
+ * *config* mistake, not an endpoint failure — so the CLI recognizes it by this
+ * lead ({@link isHttpEmbeddingDimsError}) and prints a clean config message
+ * instead of a raw stack dump. Each var names itself so the message points the
+ * operator at the variable they actually set, not a sibling. See #2385.
  */
-const EMBEDDING_DIMS_ENV_ERROR_LEAD = 'GITNEXUS_EMBEDDING_DIMS must be a positive integer';
+const dimsEnvErrorLead = (name: string): string => `${name} must be a positive integer`;
+const EMBEDDING_DIMS_ENV_ERROR_LEAD = dimsEnvErrorLead('GITNEXUS_EMBEDDING_DIMS');
+const EMBEDDING_REQUEST_DIMS_ENV_ERROR_LEAD = dimsEnvErrorLead('GITNEXUS_EMBEDDING_REQUEST_DIMS');
 
 /**
- * @internal Exported for the CLI analyze error handler. True when `message` is
- * the {@link readConfig} malformed-DIMS config error (a plain `Error`).
+ * @internal Exported for the CLI analyze error handler. True when `message` is a
+ * {@link readConfig} malformed dims-env config error (a plain `Error`) — for
+ * either `GITNEXUS_EMBEDDING_DIMS` or `GITNEXUS_EMBEDDING_REQUEST_DIMS`.
  */
 export const isHttpEmbeddingDimsError = (message: string): boolean =>
-  message.includes(EMBEDDING_DIMS_ENV_ERROR_LEAD);
+  message.includes(EMBEDDING_DIMS_ENV_ERROR_LEAD) ||
+  message.includes(EMBEDDING_REQUEST_DIMS_ENV_ERROR_LEAD);
 
 /**
  * Build config from the current process.env snapshot.
@@ -147,6 +154,23 @@ const readConfig = (): HttpConfig | null => {
     dimensions = parsed;
   }
 
+  const rawRequestDims = process.env.GITNEXUS_EMBEDDING_REQUEST_DIMS?.trim();
+  let requestDimensions = dimensions;
+  if (rawRequestDims) {
+    if (/^(omit|none|off|false|0)$/i.test(rawRequestDims)) {
+      requestDimensions = undefined;
+    } else {
+      if (!/^\d+$/.test(rawRequestDims)) {
+        throw new Error(`${EMBEDDING_REQUEST_DIMS_ENV_ERROR_LEAD}, got "${rawRequestDims}"`);
+      }
+      const parsed = parseInt(rawRequestDims, 10);
+      if (parsed <= 0) {
+        throw new Error(`${EMBEDDING_REQUEST_DIMS_ENV_ERROR_LEAD}, got "${rawRequestDims}"`);
+      }
+      requestDimensions = parsed;
+    }
+  }
+
   return {
     baseUrl: baseUrl.replace(/\/+$/, ''),
     model,
@@ -163,6 +187,7 @@ const readConfig = (): HttpConfig | null => {
       300_000,
     ),
     minIntervalMs: parseNonNegativeIntegerEnv('GITNEXUS_EMBEDDING_MIN_INTERVAL_MS', 0, 300_000),
+    requestDimensions,
   };
 };
 
@@ -283,9 +308,9 @@ const isEmbeddingItem = (item: unknown): item is EmbeddingItem =>
  *   the `dimensions` field in the request body. Endpoints that implement
  *   Matryoshka truncation (OpenAI text-embedding-3-*, Cohere embed-v3,
  *   Voyage) return a truncated vector at that size; endpoints that do not
- *   recognise the field may ignore it or return 400. Leave
- *   `GITNEXUS_EMBEDDING_DIMS` unset for strict backends that reject
- *   unknown fields.
+ *   recognise the field may ignore it or return 400. Set
+ *   `GITNEXUS_EMBEDDING_REQUEST_DIMS=omit` for strict backends while keeping
+ *   `GITNEXUS_EMBEDDING_DIMS` set to the returned vector size.
  */
 const httpEmbedBatch = async (
   url: string,
@@ -434,7 +459,7 @@ export const httpEmbed = async (
       config.model,
       config.apiKey,
       batchIndex,
-      config.dimensions,
+      config.requestDimensions,
       requestOptions,
       config.maxAttempts,
       config.retryCapMs,
@@ -491,7 +516,7 @@ export const httpEmbedQuery = async (
     config.model,
     config.apiKey,
     0,
-    config.dimensions,
+    config.requestDimensions,
     requestOptions,
     config.maxAttempts,
     config.retryCapMs,

gitnexus/src/core/ingestion/pipeline-phases/processes.ts

@@ -25,6 +25,17 @@ export interface ProcessesOutput {
   processResult: ProcessDetectionResult;
 }
 
+/**
+ * Compute the dynamic max-processes budget from the symbol count.
+ *
+ * Scales proportionally (symbolCount / 10) with a floor of 20.
+ * Prior to #2198 this was capped at 300 via `Math.min(300, …)`,
+ * silently truncating process detection on large repositories.
+ */
+export function computeDynamicMaxProcesses(symbolCount: number): number {
+  return Math.max(20, Math.round(symbolCount / 10));
+}
+
 export const processesPhase: PipelinePhase<ProcessesOutput> = {
   name: 'processes',
   // `structure` supplies `totalFiles` (progress counter) without the spurious
@@ -53,7 +64,7 @@ export const processesPhase: PipelinePhase<ProcessesOutput> = {
     ctx.graph.forEachNode((n) => {
       if (n.label !== 'File') symbolCount++;
     });
-    const dynamicMaxProcesses = Math.max(20, Math.min(300, Math.round(symbolCount / 10)));
+    const dynamicMaxProcesses = computeDynamicMaxProcesses(symbolCount);
 
     const processResult = await processProcesses(
       ctx.graph,

gitnexus/test/unit/http-embedder.test.ts

@@ -9,6 +9,7 @@ const ENV_KEYS = [
   'GITNEXUS_EMBEDDING_MAX_ATTEMPTS',
   'GITNEXUS_EMBEDDING_RETRY_CAP_MS',
   'GITNEXUS_EMBEDDING_MIN_INTERVAL_MS',
+  'GITNEXUS_EMBEDDING_REQUEST_DIMS',
 ] as const;
 
 /** 384d mock vector matching the default schema dimensions. */
@@ -166,6 +167,30 @@ describe('HTTP embedding backend', () => {
       expect(result.length).toBe(1024);
     });
 
+    it('can validate custom dims without forwarding dimensions to strict backends', async () => {
+      process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
+      process.env.GITNEXUS_EMBEDDING_MODEL = 'bge-m3';
+      process.env.GITNEXUS_EMBEDDING_DIMS = '1024';
+      process.env.GITNEXUS_EMBEDDING_REQUEST_DIMS = 'omit';
+
+      const vec1024 = Array.from({ length: 1024 }, (_, i) => i / 1024);
+      vi.stubGlobal(
+        'fetch',
+        vi.fn().mockResolvedValue({
+          ok: true,
+          json: async () => ({ data: [{ embedding: vec1024 }] }),
+        }),
+      );
+
+      const { embedText } = await import('../../src/core/embeddings/embedder.js');
+      const result = await embedText('test text');
+
+      const body = JSON.parse((fetch as any).mock.calls[0][1].body);
+      expect('dimensions' in body).toBe(false);
+      expect(body.model).toBe('bge-m3');
+      expect(result.length).toBe(1024);
+    });
+
     it('forwards dimensions on the single-query path', async () => {
       process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
       process.env.GITNEXUS_EMBEDDING_MODEL = 'text-embedding-3-large';
@@ -188,6 +213,93 @@ describe('HTTP embedding backend', () => {
       expect(result.length).toBe(512);
     });
 
+    it('can omit dimensions on the single-query path while validating custom dims', async () => {
+      process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
+      process.env.GITNEXUS_EMBEDDING_MODEL = 'bge-m3';
+      process.env.GITNEXUS_EMBEDDING_DIMS = '1024';
+      process.env.GITNEXUS_EMBEDDING_REQUEST_DIMS = 'omit';
+
+      const vec1024 = Array.from({ length: 1024 }, (_, i) => i / 1024);
+      vi.stubGlobal(
+        'fetch',
+        vi.fn().mockResolvedValue({
+          ok: true,
+          json: async () => ({ data: [{ embedding: vec1024 }] }),
+        }),
+      );
+
+      const mod = await import('../../src/mcp/core/embedder.js');
+      const result = await mod.embedQuery('query text');
+
+      const body = JSON.parse((fetch as any).mock.calls[0][1].body);
+      expect('dimensions' in body).toBe(false);
+      expect(result.length).toBe(1024);
+    });
+
+    it.each(['none', 'off', 'false', '0'])(
+      'treats GITNEXUS_EMBEDDING_REQUEST_DIMS=%s as omit and drops the request dimensions field',
+      async (alias) => {
+        process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
+        process.env.GITNEXUS_EMBEDDING_MODEL = 'bge-m3';
+        process.env.GITNEXUS_EMBEDDING_DIMS = '1024';
+        process.env.GITNEXUS_EMBEDDING_REQUEST_DIMS = alias;
+
+        const vec1024 = Array.from({ length: 1024 }, (_, i) => i / 1024);
+        vi.stubGlobal(
+          'fetch',
+          vi.fn().mockResolvedValue({
+            ok: true,
+            json: async () => ({ data: [{ embedding: vec1024 }] }),
+          }),
+        );
+
+        const { embedText } = await import('../../src/core/embeddings/embedder.js');
+        const result = await embedText('test text');
+
+        const body = JSON.parse((fetch as any).mock.calls[0][1].body);
+        expect('dimensions' in body).toBe(false);
+        expect(result.length).toBe(1024);
+      },
+    );
+
+    it('sends REQUEST_DIMS as the request dimensions while DIMS validates the response', async () => {
+      process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
+      process.env.GITNEXUS_EMBEDDING_MODEL = 'text-embedding-3-large';
+      process.env.GITNEXUS_EMBEDDING_DIMS = '1024';
+      process.env.GITNEXUS_EMBEDDING_REQUEST_DIMS = '512';
+
+      // Response keeps the DIMS-validated length; only the outgoing request differs.
+      const vec1024 = Array.from({ length: 1024 }, (_, i) => i / 1024);
+      vi.stubGlobal(
+        'fetch',
+        vi.fn().mockResolvedValue({
+          ok: true,
+          json: async () => ({ data: [{ embedding: vec1024 }] }),
+        }),
+      );
+
+      const { embedText } = await import('../../src/core/embeddings/embedder.js');
+      const result = await embedText('test text');
+
+      const body = JSON.parse((fetch as any).mock.calls[0][1].body);
+      expect(body.dimensions).toBe(512);
+      expect(result.length).toBe(1024);
+    });
+
+    it('rejects a malformed GITNEXUS_EMBEDDING_REQUEST_DIMS with an error naming that var', async () => {
+      process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
+      process.env.GITNEXUS_EMBEDDING_MODEL = 'test-model';
+      process.env.GITNEXUS_EMBEDDING_REQUEST_DIMS = 'garbage';
+
+      const { embedText } = await import('../../src/core/embeddings/embedder.js');
+      const { isHttpEmbeddingDimsError } = await import('../../src/core/embeddings/http-client.js');
+      const err = await embedText('test').catch((e: unknown) => e);
+      // Recognizable as a config error so the CLI prints a clean message...
+      expect(isHttpEmbeddingDimsError(String(err))).toBe(true);
+      // ...and it points the operator at the var they set, not GITNEXUS_EMBEDDING_DIMS.
+      expect(String(err)).toContain('GITNEXUS_EMBEDDING_REQUEST_DIMS must be a positive integer');
+    });
+
     it('retries on server error', async () => {
       process.env.GITNEXUS_EMBEDDING_URL = 'http://test:8080/v1';
       process.env.GITNEXUS_EMBEDDING_MODEL = 'test-model';

gitnexus/test/unit/process-processor.test.ts

@@ -3,6 +3,7 @@ import {
   processProcesses,
   type ProcessDetectionConfig,
 } from '../../src/core/ingestion/process-processor.js';
+import { computeDynamicMaxProcesses } from '../../src/core/ingestion/pipeline-phases/processes.js';
 import { createKnowledgeGraph } from '../../src/core/graph/graph.js';
 import type { CommunityMembership } from '../../src/core/ingestion/community-processor.js';
 
@@ -522,4 +523,40 @@ describe('processProcesses', () => {
     expect(result.processes.length).toBeLessThanOrEqual(3);
     expect(result.stats.totalProcesses).toBeLessThanOrEqual(3);
   });
+
+  // Regression for #2198: the processesPhase dynamic sizing used to cap at
+  // Math.min(300, symbolCount/10). On large repos (>3000 symbols) that silently
+  // truncated the process index. The cap was removed by extracting
+  // computeDynamicMaxProcesses() — this test exercises the helper directly
+  // so it fails if someone reintroduces the 300 ceiling.
+  describe('computeDynamicMaxProcesses (#2198)', () => {
+    it('returns at least the floor of 20 for tiny repos', () => {
+      expect(computeDynamicMaxProcesses(0)).toBe(20);
+      expect(computeDynamicMaxProcesses(50)).toBe(20); // 50/10 = 5, floored to 20
+      expect(computeDynamicMaxProcesses(199)).toBe(20); // 199/10 ≈ 20
+    });
+
+    it('scales linearly within the old 0–3000 range', () => {
+      expect(computeDynamicMaxProcesses(500)).toBe(50);
+      expect(computeDynamicMaxProcesses(1000)).toBe(100);
+      expect(computeDynamicMaxProcesses(2999)).toBe(300);
+    });
+
+    it('grows past 300 for large repos — the regression that #2198 fixes', () => {
+      // 3001 symbols → 300 (just at the boundary)
+      expect(computeDynamicMaxProcesses(3001)).toBe(300);
+      // 3100 symbols → 310 — would have been capped to 300 before the fix
+      expect(computeDynamicMaxProcesses(3100)).toBe(310);
+      // 5000 symbols → 500
+      expect(computeDynamicMaxProcesses(5000)).toBe(500);
+      // 28000 symbols (real-world large repo) → 2800
+      expect(computeDynamicMaxProcesses(28000)).toBe(2800);
+    });
+
+    it('does NOT cap at 300 — fails if Math.min(300, ...) is reintroduced', () => {
+      const largeRepo = computeDynamicMaxProcesses(10000);
+      expect(largeRepo).toBe(1000);
+      expect(largeRepo).toBeGreaterThan(300);
+    });
+  });
 });