diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index caf61ab83d..61c255ff4e 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -285,5 +285,5 @@ export const en = { 'help.option.group.contracts.repo': 'Filter by repo', 'help.option.group.contracts.unmatched': 'Show only unmatched contracts', 'help.analyze.environment': - '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', + '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).', } as const; diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 01abeedc1e..d1cda70842 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -265,5 +265,5 @@ export const zhCN = { 'help.option.group.contracts.repo': '按仓库过滤', 'help.option.group.contracts.unmatched': '仅显示未匹配契约', 'help.analyze.environment': - '\n环境变量：\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析（仍读取 .gitnexusignore）\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值（KB）。默认 512，最大 32768。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时（毫秒）。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值（字节，默认 67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1，最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n\n当参数和对应环境变量同时提供时，参数优先。\n\n提示：`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录（#771）。', + '\n环境变量：\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析（仍读取 .gitnexusignore）\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值（KB）。默认 512，最大 32768。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时（毫秒）。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值（字节，默认 67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1，最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离（0 < N <= 2；超出则钳制为 2）。MCP 默认 0.6，其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时，参数优先。\n\n提示：`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录（#771）。', } satisfies EnglishMessages; diff --git a/gitnexus/src/core/embeddings/config.ts b/gitnexus/src/core/embeddings/config.ts index 7f09cf5afa..d80179f4c5 100644 --- a/gitnexus/src/core/embeddings/config.ts +++ b/gitnexus/src/core/embeddings/config.ts @@ -1,6 +1,53 @@ import { defaultEmbeddingThreads } from '../platform/capabilities.js'; +import { logger } from '../logger.js'; import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig } from './types.js'; +export const DEFAULT_VECTOR_MAX_DISTANCE = 0.5; +export const DEFAULT_MCP_VECTOR_MAX_DISTANCE = 0.6; + +/** + * Cosine distance over normalized embeddings is bounded to [0, 2], so any threshold + * above this accepts every row and silently disables the relevance filter. Values + * over the ceiling are clamped to it rather than passed through. + */ +export const VECTOR_MAX_DISTANCE_CEILING = 2; + +const warned = new Set(); + +const warnOnce = (key: string, message: string): void => { + if (warned.has(key)) return; + warned.add(key); + logger.warn(message); +}; + +/** + * Resolve the effective max accepted vector/semantic cosine distance. + * Reads `GITNEXUS_VECTOR_MAX_DISTANCE`. Unset/empty/whitespace → silent fallback. + * Invalid (non-numeric, <= 0, non-finite) → fallback plus a one-time warning. + * Values above the cosine ceiling (2) are clamped to it with a one-time warning. + */ +export const getVectorMaxDistance = (fallback: number = DEFAULT_VECTOR_MAX_DISTANCE): number => { + const raw = process.env.GITNEXUS_VECTOR_MAX_DISTANCE; + if (raw === undefined || raw.trim() === '') return fallback; + + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + warnOnce( + `invalid:${raw}`, + ` GITNEXUS_VECTOR_MAX_DISTANCE must be a positive number in (0, ${VECTOR_MAX_DISTANCE_CEILING}], got "${raw}" — using default ${fallback}`, + ); + return fallback; + } + if (parsed > VECTOR_MAX_DISTANCE_CEILING) { + warnOnce( + `clamp:${raw}`, + ` GITNEXUS_VECTOR_MAX_DISTANCE=${parsed} exceeds the cosine-distance ceiling (${VECTOR_MAX_DISTANCE_CEILING}) — clamping`, + ); + return VECTOR_MAX_DISTANCE_CEILING; + } + return parsed; +}; + const parsePositiveInt = (name: string, value: string | undefined, fallback: number): number => { if (value === undefined) return fallback; const parsed = Number(value); diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index 0405cd47c8..c1732da87e 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -34,7 +34,11 @@ import { STRUCTURAL_LABELS, collectBestChunks, } from './types.js'; -import { resolveEmbeddingConfig } from './config.js'; +import { + DEFAULT_VECTOR_MAX_DISTANCE, + getVectorMaxDistance, + resolveEmbeddingConfig, +} from './config.js'; import { rankExactEmbeddingRows, type ExactEmbeddingRow } from './exact-search.js'; import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, STALE_HASH_SENTINEL } from '../lbug/schema.js'; import { loadVectorExtension, createVectorIndex } from '../lbug/lbug-adapter.js'; @@ -595,7 +599,7 @@ export const semanticSearch = async ( executeQuery: (cypher: string) => Promise, query: string, k: number = 10, - maxDistance: number = 0.5, + maxDistance: number = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE), ): Promise => { if (!isEmbedderReady()) { throw new Error('Embedding model not initialized. Run embedding pipeline first.'); @@ -741,7 +745,7 @@ export const semanticSearchWithContext = async ( k: number = 5, _hops: number = 1, ): Promise => { - const results = await semanticSearch(executeQuery, query, k, 0.5); + const results = await semanticSearch(executeQuery, query, k); return results.map((r) => ({ matchId: r.nodeId, diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index abd00d3927..01f47169ba 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -48,6 +48,10 @@ import { } from '../../core/group/service.js'; import { resolveAtGroupMemberRepoPath } from '../../core/group/resolve-at-member.js'; import { collectBestChunks } from '../../core/embeddings/types.js'; +import { + DEFAULT_MCP_VECTOR_MAX_DISTANCE, + getVectorMaxDistance, +} from '../../core/embeddings/config.js'; import { rankExactEmbeddingRows, type ExactEmbeddingRow, @@ -2127,6 +2131,7 @@ export class LocalBackend { const queryVec = await embedQuery(query); const dims = getEmbeddingDims(); const queryVecStr = `[${queryVec.join(',')}]`; + const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE); let bestChunks = new Map< string, @@ -2140,7 +2145,7 @@ export class LocalBackend { CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit}) YIELD node AS emb, distance WITH emb, distance - WHERE distance < 0.6 + WHERE distance < ${maxDistance} RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex, emb.startLine AS startLine, emb.endLine AS endLine, distance ORDER BY distance @@ -2190,7 +2195,7 @@ export class LocalBackend { embedding: row.embedding ?? row[4] ?? [], })); bestChunks = new Map( - rankExactEmbeddingRows(exactRows, queryVec, limit, 0.6).map((row) => [ + rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [ row.nodeId, { distance: row.distance, diff --git a/gitnexus/test/unit/calltool-dispatch.test.ts b/gitnexus/test/unit/calltool-dispatch.test.ts index 1853f12a0e..f30180dcf5 100644 --- a/gitnexus/test/unit/calltool-dispatch.test.ts +++ b/gitnexus/test/unit/calltool-dispatch.test.ts @@ -465,6 +465,32 @@ describe('LocalBackend.callTool', () => { const queries = (executeQuery as any).mock.calls.map(([, cypher]: [string, string]) => cypher); expect(queries.some((cypher: string) => cypher.includes('QUERY_VECTOR_INDEX'))).toBe(true); + // The configured threshold must reach the WHERE clause (MCP default 0.6), guarding + // against a regression that drops the filter or re-hardcodes a different value. + expect(queries.some((cypher: string) => cypher.includes('distance < 0.6'))).toBe(true); + }); + + it('threads GITNEXUS_VECTOR_MAX_DISTANCE into the vector index WHERE clause', async () => { + platformMocks.isVectorExtensionSupportedByPlatform.mockReturnValue(true); + vi.mocked(executeQuery).mockImplementation(async (_repoId: string, cypher: string) => { + if (cypher.includes('COUNT(*) AS cnt')) return [{ cnt: 1 }]; + return []; + }); + vi.mocked(executeParameterized).mockResolvedValue([]); + + const previous = process.env.GITNEXUS_VECTOR_MAX_DISTANCE; + process.env.GITNEXUS_VECTOR_MAX_DISTANCE = '0.42'; + try { + await backend.callTool('query', { query: 'auth' }); + const queries = vi + .mocked(executeQuery) + .mock.calls.map(([, cypher]: [string, string]) => cypher); + expect(queries.some((cypher: string) => cypher.includes('distance < 0.42'))).toBe(true); + expect(queries.some((cypher: string) => cypher.includes('distance < 0.6'))).toBe(false); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_VECTOR_MAX_DISTANCE; + else process.env.GITNEXUS_VECTOR_MAX_DISTANCE = previous; + } }); it('query tool returns error for empty query', async () => { diff --git a/gitnexus/test/unit/exact-search.test.ts b/gitnexus/test/unit/exact-search.test.ts index 5f6c42c04a..b5b5a9bf36 100644 --- a/gitnexus/test/unit/exact-search.test.ts +++ b/gitnexus/test/unit/exact-search.test.ts @@ -1,6 +1,35 @@ -import { describe, expect, it } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('../../src/core/logger.js', () => ({ + logger: { + warn: vi.fn(), + info: vi.fn(), + error: vi.fn(), + debug: vi.fn(), + trace: vi.fn(), + fatal: vi.fn(), + }, +})); + +import { logger } from '../../src/core/logger.js'; +import { + DEFAULT_VECTOR_MAX_DISTANCE, + getVectorMaxDistance, +} from '../../src/core/embeddings/config.js'; import { rankExactEmbeddingRows } from '../../src/core/embeddings/exact-search.js'; +const withVectorDistanceEnv = (value: string | undefined, run: () => void) => { + const previous = process.env.GITNEXUS_VECTOR_MAX_DISTANCE; + try { + if (value === undefined) delete process.env.GITNEXUS_VECTOR_MAX_DISTANCE; + else process.env.GITNEXUS_VECTOR_MAX_DISTANCE = value; + run(); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_VECTOR_MAX_DISTANCE; + else process.env.GITNEXUS_VECTOR_MAX_DISTANCE = previous; + } +}; + describe('rankExactEmbeddingRows', () => { it('orders rows by cosine distance and applies the limit', () => { const rows = [ @@ -20,4 +49,86 @@ describe('rankExactEmbeddingRows', () => { }, ]); }); + + it('uses a configurable distance threshold for exact-scan fallback', () => { + const rows = [ + { nodeId: 'Function:near', chunkIndex: 0, startLine: 1, endLine: 1, embedding: [1, 0] }, + { nodeId: 'Function:far', chunkIndex: 0, startLine: 1, endLine: 1, embedding: [0, 1] }, + ]; + + withVectorDistanceEnv('1.1', () => { + const ranked = rankExactEmbeddingRows( + rows, + [1, 0], + 10, + getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE), + ); + + expect(ranked.map((row) => row.nodeId)).toEqual(['Function:near', 'Function:far']); + }); + }); +}); + +describe('getVectorMaxDistance', () => { + beforeEach(() => { + vi.mocked(logger.warn).mockClear(); + }); + + it('returns the caller fallback when the env var is unset', () => { + withVectorDistanceEnv(undefined, () => { + expect(getVectorMaxDistance(0.6)).toBe(0.6); + }); + }); + + it('parses a positive numeric env override', () => { + withVectorDistanceEnv('0.82', () => { + expect(getVectorMaxDistance(0.6)).toBe(0.82); + }); + }); + + it('keeps the fallback for invalid values', () => { + for (const value of ['0', '-0.1', 'not-a-number']) { + withVectorDistanceEnv(value, () => { + expect(getVectorMaxDistance(0.6)).toBe(0.6); + }); + } + }); + + it('stays silent for unset, empty, and whitespace values', () => { + for (const value of [undefined, '', ' ']) { + withVectorDistanceEnv(value, () => { + expect(getVectorMaxDistance(0.6)).toBe(0.6); + }); + } + expect(vi.mocked(logger.warn)).not.toHaveBeenCalled(); + }); + + it('falls back and warns once for a non-finite value', () => { + withVectorDistanceEnv('Infinity', () => { + expect(getVectorMaxDistance(0.6)).toBe(0.6); + }); + expect(vi.mocked(logger.warn)).toHaveBeenCalledTimes(1); + }); + + it('clamps values above the cosine ceiling to 2 and warns', () => { + withVectorDistanceEnv('5', () => { + expect(getVectorMaxDistance(0.6)).toBe(2); + }); + expect(vi.mocked(logger.warn)).toHaveBeenCalledTimes(1); + }); + + it('accepts the ceiling value 2 without warning', () => { + withVectorDistanceEnv('2', () => { + expect(getVectorMaxDistance(0.6)).toBe(2); + }); + expect(vi.mocked(logger.warn)).not.toHaveBeenCalled(); + }); + + it('warns only once per offending value across repeated calls', () => { + withVectorDistanceEnv('7', () => { + expect(getVectorMaxDistance(0.6)).toBe(2); + expect(getVectorMaxDistance(0.6)).toBe(2); + }); + expect(vi.mocked(logger.warn)).toHaveBeenCalledTimes(1); + }); });