scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-05 02:43:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

Merge branch 'main' into fix/dart-tree-sitter-napi-and-queries

Browse source
This commit is contained in:
Gergő Magyar 2026-05-09 06:47:24 +01:00 • committed by GitHub
commit d50af1835c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 128 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
gitnexus/package-lock.json generated
View file

@ -4282,15 +4282,15 @@
}
},
"node_modules/onnxruntime-common": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.25.1.tgz",
"integrity": "sha512-kKvYQFdos4LWJqhZ+nmKu3NT8NXzw8I5x9fNUKe1rNKcPfNKnYXUtW7JBpcKFsvLtrJashRgVYSbFap4cHxvNg==",
"version": "1.26.0",
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.26.0.tgz",
"integrity": "sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw==",
"license": "MIT"
},
"node_modules/onnxruntime-node": {
"version": "1.25.1",
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.25.1.tgz",
"integrity": "sha512-N0M58CGTiTsLkPpx9bxmRFi24GT6r67Qei/GrBEIiDyntcYdXU5vQZp112ypydG9vEKRFgbgUYQJnEi+jll8dg==",
"version": "1.26.0",
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.26.0.tgz",
"integrity": "sha512-OHl6PiOEOqxaLHL0N9eFrbzS7IGmu3BtJNH3RTEnRAheCIkfc3gjcjl4sGcjp9C22ZC9YTquDOxSdT/stBQ6BQ==",
"hasInstallScript": true,
"license": "MIT",
"os": [
@ -4301,7 +4301,7 @@
"dependencies": {
"adm-zip": "^0.5.16",
"global-agent": "^4.1.3",
"onnxruntime-common": "1.25.1"
"onnxruntime-common": "1.26.0"
}
},
"node_modules/onnxruntime-web": {

2
gitnexus/src/core/augmentation/engine.ts
View file

@ -104,7 +104,7 @@ export async function augment(pattern: string, cwd?: string): Promise<string> {
}
// Step 1: BM25 search (fast, no embeddings)
const bm25Results = await searchFTSFromLbug(pattern, 10, repoId);
const { results: bm25Results } = await searchFTSFromLbug(pattern, 10, repoId);
if (bm25Results.length === 0) return '';

47
gitnexus/src/core/search/bm25-index.ts
View file

@ -15,9 +15,16 @@ export interface BM25SearchResult {
nodeIds?: string[];
}
export interface FTSSearchResponse {
results: BM25SearchResult[];
/** True when at least one FTS index query succeeded (index exists). */
ftsAvailable: boolean;
}
/**
* Execute a single FTS query via a custom executor (for MCP connection pool).
* Returns the same shape as core queryFTS (from LadybugDB adapter).
* Returns `null` when the query fails (e.g. FTS index does not exist) so the
* caller can distinguish "zero matches" from "index missing".
*/
async function queryFTSViaExecutor(
executor: (cypher: string) => Promise<any[]>,
@ -25,7 +32,7 @@ async function queryFTSViaExecutor(
indexName: string,
query: string,
limit: number,
): Promise<Array<{ filePath: string; score: number; nodeId: string }>> {
): Promise<Array<{ filePath: string; score: number; nodeId: string }> | null> {
// Escape single quotes and backslashes to prevent Cypher injection
const escapedQuery = query.replace(/\\/g, '\\\\').replace(/'/g, "''");
const cypher = `
@ -46,7 +53,7 @@ async function queryFTSViaExecutor(
};
});
} catch {
return [];
return null;
}
}
@ -65,8 +72,9 @@ export const searchFTSFromLbug = async (
query: string,
limit: number = 20,
repoId?: string,
): Promise<BM25SearchResult[]> => {
): Promise<FTSSearchResponse> => {
const resultsByIndex: any[][] = [];
let queriesSucceeded = 0;
if (repoId) {
// Use MCP connection pool via dynamic import
@ -77,15 +85,27 @@ export const searchFTSFromLbug = async (
const executor = (cypher: string) => executeQuery(repoId, cypher);
for (const { table, indexName } of FTS_INDEXES) {
resultsByIndex.push(await queryFTSViaExecutor(executor, table, indexName, query, limit));
const result = await queryFTSViaExecutor(executor, table, indexName, query, limit);
if (result !== null) {
queriesSucceeded++;
resultsByIndex.push(result);
}
}
} else {
// Use core lbug adapter (CLI / pipeline context) — also sequential for safety.
for (const { table, indexName } of FTS_INDEXES) {
resultsByIndex.push(await queryFTS(table, indexName, query, limit, false).catch(() => []));
try {
const result = await queryFTS(table, indexName, query, limit, false);
queriesSucceeded++;
resultsByIndex.push(result);
} catch {
// FTS index may not exist — count as failed
}
}
}
const ftsAvailable = queriesSucceeded > 0;
// Collect all node scores per filePath to track which nodes actually matched
const fileNodeScores = new Map<string, Array<{ score: number; nodeId: string }>>();
@ -116,10 +136,13 @@ export const searchFTSFromLbug = async (
.sort((a, b) => b.score - a.score)
.slice(0, limit);
return sorted.map((r, index) => ({
filePath: r.filePath,
score: r.score,
rank: index + 1,
nodeIds: r.nodeIds,
}));
return {
results: sorted.map((r, index) => ({
filePath: r.filePath,
score: r.score,
rank: index + 1,
nodeIds: r.nodeIds,
})),
ftsAvailable,
};
};

11
gitnexus/src/core/search/hybrid-search.ts
View file

@ -113,12 +113,13 @@ export const mergeWithRRF = (
};
/**
* Check if hybrid search is available
* LadybugDB FTS is always available once the database is initialized.
* Semantic search is optional - hybrid works with just FTS if embeddings aren't ready.
* Check if hybrid search is available.
* FTS indexes may be missing on read-only MCP connections (see #1403);
* callers should inspect `ftsAvailable` from searchFTSFromLbug for
* per-query availability. This helper is a coarse gate only.
*/
export const isHybridSearchReady = (): boolean => {
return true; // FTS is always available via LadybugDB when DB is open
return true; // FTS is attempted on every query; ftsAvailable signals actual availability
};
/**
@ -160,7 +161,7 @@ export const hybridSearch = async (
) => Promise<SemanticSearchResult[]>,
): Promise<HybridSearchResult[]> => {
// Use LadybugDB FTS for always-fresh BM25 results
const bm25Results = await searchFTSFromLbug(query, limit);
const { results: bm25Results } = await searchFTSFromLbug(query, limit);
const semanticResults = await semanticSearch(executeQuery, query, limit);
return mergeWithRRF(bm25Results, semanticResults, limit);
};

9
gitnexus/src/mcp/local/local-backend.ts
View file

@ -979,7 +979,7 @@ export class LocalBackend {
timing,
...(!ftsUsed && {
warning:
'FTS extension unavailable - keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.',
'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.',
}),
};
}
@ -993,9 +993,9 @@ export class LocalBackend {
limit: number,
): Promise<{ results: any[]; ftsUsed: boolean }> {
const { searchFTSFromLbug } = await import('../../core/search/bm25-index.js');
let bm25Results;
let ftsResponse;
try {
bm25Results = await searchFTSFromLbug(query, limit, repo.id);
ftsResponse = await searchFTSFromLbug(query, limit, repo.id);
} catch (err: any) {
logger.error(
{ err: err.message },
@ -1004,7 +1004,8 @@ export class LocalBackend {
return { results: [], ftsUsed: false };
}
const ftsUsed = bm25Results.length === 0 || bm25Results[0]?.ftsUsed !== false;
const bm25Results = ftsResponse.results;
const ftsUsed = ftsResponse.ftsAvailable;
const results: any[] = [];

23
gitnexus/src/server/api.ts
View file

@ -1060,11 +1060,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
const results = await withLbugDb(lbugPath, async () => {
let searchResults: any[];
let ftsAvailable: boolean | undefined;
if (mode === 'semantic') {
const { isEmbedderReady } = await import('../core/embeddings/embedder.js');
if (!isEmbedderReady()) {
return [] as any[];
return { searchResults: [] as any[], ftsAvailable: undefined };
}
const { semanticSearch: semSearch } =
await import('../core/embeddings/embedding-pipeline.js');
@ -1077,8 +1078,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
sources: ['semantic'],
}));
} else if (mode === 'bm25') {
searchResults = await searchFTSFromLbug(query, limit);
searchResults = searchResults.map((r: any, i: number) => ({
const ftsResponse = await searchFTSFromLbug(query, limit);
ftsAvailable = ftsResponse.ftsAvailable;
searchResults = ftsResponse.results.map((r: any, i: number) => ({
...r,
rank: i + 1,
sources: ['bm25'],
@ -1091,11 +1093,13 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
await import('../core/embeddings/embedding-pipeline.js');
searchResults = await hybridSearch(query, limit, executeQuery, semSearch);
} else {
searchResults = await searchFTSFromLbug(query, limit);
const ftsResponse = await searchFTSFromLbug(query, limit);
ftsAvailable = ftsResponse.ftsAvailable;
searchResults = ftsResponse.results;
}
}
if (!enrich) return searchResults;
if (!enrich) return { searchResults, ftsAvailable };
// Server-side enrichment: add connections, cluster, processes per result
// Uses parameterized queries to prevent Cypher injection via nodeId
@ -1177,9 +1181,14 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
}),
);
return enriched;
return { searchResults: enriched, ftsAvailable };
});
res.json({ results });
const response: any = { results: results.searchResults ?? results };
if (results.ftsAvailable === false) {
response.warning =
'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.';
}
res.json(response);
} catch (err: any) {
res.status(500).json({ error: err.message || 'Search failed' });
}

24
gitnexus/test/integration/search-core.test.ts
View file

@ -19,7 +19,7 @@ withTestLbugDB(
(_handle) => {
describe('searchFTSFromLbug — core adapter (no repoId)', () => {
it('returns ranked results for a matching query', async () => {
const results = await searchFTSFromLbug('user authentication', 10);
const { results } = await searchFTSFromLbug('user authentication', 10);
expect(results.length).toBeGreaterThan(0);
@ -40,7 +40,7 @@ withTestLbugDB(
});
it('results are ordered by descending score', async () => {
const results = await searchFTSFromLbug('user authentication', 10);
const { results } = await searchFTSFromLbug('user authentication', 10);
for (let i = 1; i < results.length; i++) {
expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
@ -48,7 +48,7 @@ withTestLbugDB(
});
it('auth-related files rank higher than unrelated files', async () => {
const results = await searchFTSFromLbug('user authentication', 10);
const { results } = await searchFTSFromLbug('user authentication', 10);
const filePaths = results.map((r) => r.filePath);
expect(filePaths).toContain('src/auth.ts');
@ -61,7 +61,7 @@ withTestLbugDB(
});
it('merges scores from multiple node types for the same filePath', async () => {
const results = await searchFTSFromLbug('user authentication', 20);
const { results } = await searchFTSFromLbug('user authentication', 20);
const authResult = results.find((r) => r.filePath === 'src/auth.ts');
expect(authResult).toBeDefined();
@ -73,12 +73,12 @@ withTestLbugDB(
});
it('respects limit parameter', async () => {
const results = await searchFTSFromLbug('user authentication', 2);
const { results } = await searchFTSFromLbug('user authentication', 2);
expect(results.length).toBeLessThanOrEqual(2);
});
it('returns empty array for a non-matching query', async () => {
const results = await searchFTSFromLbug('xyzzyplughtwisty', 10);
const { results } = await searchFTSFromLbug('xyzzyplughtwisty', 10);
expect(results).toEqual([]);
});
});
@ -87,32 +87,32 @@ withTestLbugDB(
describe('unhappy paths', () => {
it('returns empty array for empty query string', async () => {
const results = await searchFTSFromLbug('', 10);
const { results } = await searchFTSFromLbug('', 10);
expect(results).toEqual([]);
});
it('returns empty array for whitespace-only query', async () => {
const results = await searchFTSFromLbug(' ', 10);
const { results } = await searchFTSFromLbug(' ', 10);
expect(results).toEqual([]);
});
it('handles special characters in query gracefully', async () => {
const results = await searchFTSFromLbug('user* OR auth+', 10);
const { results } = await searchFTSFromLbug('user* OR auth+', 10);
expect(Array.isArray(results)).toBe(true);
});
it('handles limit of 0', async () => {
const results = await searchFTSFromLbug('user authentication', 0);
const { results } = await searchFTSFromLbug('user authentication', 0);
expect(results).toEqual([]);
});
it('handles negative limit gracefully', async () => {
const results = await searchFTSFromLbug('user authentication', -1);
const { results } = await searchFTSFromLbug('user authentication', -1);
expect(Array.isArray(results)).toBe(true);
});
it('handles very large limit', async () => {
const results = await searchFTSFromLbug('user authentication', 100000);
const { results } = await searchFTSFromLbug('user authentication', 100000);
expect(results.length).toBeLessThanOrEqual(100000);
expect(results.length).toBeGreaterThan(0);
});

16
gitnexus/test/integration/search-pool.test.ts
View file

@ -19,7 +19,7 @@ withTestLbugDB(
(handle) => {
describe('searchFTSFromLbug — MCP pool adapter (with repoId)', () => {
it('returns ranked results via pool adapter', async () => {
const results = await searchFTSFromLbug('user authentication', 10, handle.repoId);
const { results } = await searchFTSFromLbug('user authentication', 10, handle.repoId);
expect(results.length).toBeGreaterThan(0);
@ -35,7 +35,7 @@ withTestLbugDB(
});
it('results are ordered by descending score via pool adapter', async () => {
const results = await searchFTSFromLbug('user authentication', 10, handle.repoId);
const { results } = await searchFTSFromLbug('user authentication', 10, handle.repoId);
for (let i = 1; i < results.length; i++) {
expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
@ -43,12 +43,12 @@ withTestLbugDB(
});
it('returns empty array for non-matching query via pool adapter', async () => {
const results = await searchFTSFromLbug('xyzzyplughtwisty', 10, handle.repoId);
const { results } = await searchFTSFromLbug('xyzzyplughtwisty', 10, handle.repoId);
expect(results).toEqual([]);
});
it('respects limit parameter via pool adapter', async () => {
const results = await searchFTSFromLbug('user authentication', 1, handle.repoId);
const { results } = await searchFTSFromLbug('user authentication', 1, handle.repoId);
expect(results.length).toBeLessThanOrEqual(1);
});
});
@ -57,22 +57,22 @@ withTestLbugDB(
describe('unhappy paths', () => {
it('returns empty array for empty query via pool', async () => {
const results = await searchFTSFromLbug('', 10, handle.repoId);
const { results } = await searchFTSFromLbug('', 10, handle.repoId);
expect(results).toEqual([]);
});
it('returns empty array for whitespace-only query via pool', async () => {
const results = await searchFTSFromLbug(' ', 10, handle.repoId);
const { results } = await searchFTSFromLbug(' ', 10, handle.repoId);
expect(results).toEqual([]);
});
it('handles special characters in query via pool', async () => {
const results = await searchFTSFromLbug('user* OR auth+', 10, handle.repoId);
const { results } = await searchFTSFromLbug('user* OR auth+', 10, handle.repoId);
expect(Array.isArray(results)).toBe(true);
});
it('handles limit of 0 via pool', async () => {
const results = await searchFTSFromLbug('user authentication', 0, handle.repoId);
const { results } = await searchFTSFromLbug('user authentication', 0, handle.repoId);
expect(results).toEqual([]);
});
});

26
gitnexus/test/unit/bm25-search.test.ts
View file

@ -42,20 +42,24 @@ describe('BM25 search', () => {
});
describe('searchFTSFromLbug', () => {
it('returns empty array when LadybugDB is not initialized', async () => {
// Without LadybugDB init, search should return empty (not crash)
const results = await searchFTSFromLbug('test query');
it('returns empty results when LadybugDB is not initialized', async () => {
// Simulate an uninitialized DB: queryFTS throws instead of returning rows
const { queryFTS } = await import('../../src/core/lbug/lbug-adapter.js');
vi.mocked(queryFTS).mockRejectedValue(new Error('DB not initialized'));
const { results, ftsAvailable } = await searchFTSFromLbug('test query');
expect(Array.isArray(results)).toBe(true);
expect(results).toHaveLength(0);
expect(ftsAvailable).toBe(false);
});
it('handles empty query', async () => {
const results = await searchFTSFromLbug('');
const { results } = await searchFTSFromLbug('');
expect(Array.isArray(results)).toBe(true);
});
it('accepts custom limit parameter', async () => {
const results = await searchFTSFromLbug('test', 5);
const { results } = await searchFTSFromLbug('test', 5);
expect(Array.isArray(results)).toBe(true);
});
});
@ -105,7 +109,7 @@ describe('BM25 search', () => {
.mockResolvedValueOnce([]) // Method
.mockResolvedValueOnce([]); // Interface
const results = await searchFTSFromLbug('queryset');
const { results } = await searchFTSFromLbug('queryset');
expect(results).toHaveLength(1);
expect(results[0].filePath).toBe('src/views.py');
@ -127,7 +131,7 @@ describe('BM25 search', () => {
.mockResolvedValueOnce([]) // Method
.mockResolvedValueOnce([]); // Interface
const results = await searchFTSFromLbug('model');
const { results } = await searchFTSFromLbug('model');
expect(results).toHaveLength(1);
expect(results[0].score).toBe(8); // 5+3
@ -147,7 +151,7 @@ describe('BM25 search', () => {
.mockResolvedValueOnce([]) // Method
.mockResolvedValueOnce([]); // Interface
const results = await searchFTSFromLbug('util');
const { results } = await searchFTSFromLbug('util');
expect(results).toHaveLength(1);
expect(results[0].nodeIds).toEqual([]);
@ -171,7 +175,7 @@ describe('BM25 search', () => {
.mockResolvedValueOnce([]) // Method
.mockResolvedValueOnce([]); // Interface
const results = await searchFTSFromLbug('auth');
const { results } = await searchFTSFromLbug('auth');
expect(results).toHaveLength(1);
// All 3 hits (scores 9+7+4=20) — each from a different table, all top-3
@ -192,7 +196,7 @@ describe('BM25 search', () => {
.mockResolvedValueOnce([]) // Method
.mockResolvedValueOnce([]); // Interface
const results = await searchFTSFromLbug('fn');
const { results } = await searchFTSFromLbug('fn');
expect(results[0].filePath).toBe('src/high.py');
expect(results[1].filePath).toBe('src/low.py');
@ -220,7 +224,7 @@ describe('BM25 search', () => {
return [];
});
const results = await searchFTSFromLbug('login', 5, REPO);
const { results } = await searchFTSFromLbug('login', 5, REPO);
expect(results).toEqual([
{ filePath: 'src/auth.ts', score: 8, rank: 1, nodeIds: ['func:login'] },

23
gitnexus/test/unit/calltool-dispatch.test.ts
View file

@ -62,7 +62,7 @@ vi.mock('../../src/core/platform/capabilities.js', async (importOriginal) => {
// Also mock the search modules to avoid loading onnxruntime
vi.mock('../../src/core/search/bm25-index.js', () => ({
searchFTSFromLbug: vi.fn().mockResolvedValue([]),
searchFTSFromLbug: vi.fn().mockResolvedValue({ results: [], ftsAvailable: true }),
}));
vi.mock('../../src/mcp/core/embedder.js', () => ({
@ -195,6 +195,27 @@ describe('LocalBackend.callTool', () => {
expect(result).toHaveProperty('definitions');
});
it('includes FTS-unavailable warning when ftsAvailable is false (#1403)', async () => {
const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js');
vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ results: [], ftsAvailable: false });
(executeParameterized as any).mockResolvedValue([]);
const result = await backend.callTool('query', { query: 'ProcessActivity' });
expect(result).toHaveProperty('warning');
expect((result as any).warning).toMatch(/gitnexus analyze --force/);
});
it('does not include warning when ftsAvailable is true with zero results', async () => {
const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js');
vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ results: [], ftsAvailable: true });
(executeParameterized as any).mockResolvedValue([]);
const result = await backend.callTool('query', { query: 'nonexistent' });
expect(result).not.toHaveProperty('warning');
});
it('skips vector index query when VECTOR is unsupported by the platform', async () => {
const cap = _captureLogger();
platformMocks.isVectorExtensionSupportedByPlatform.mockReturnValue(false);

2
gitnexus/test/unit/mcp/group-repo-routing.test.ts
View file

@ -32,7 +32,7 @@ vi.mock('../../../src/storage/repo-manager.js', () => ({
}));
vi.mock('../../../src/core/search/bm25-index.js', () => ({
searchFTSFromLbug: vi.fn().mockResolvedValue([]),
searchFTSFromLbug: vi.fn().mockResolvedValue({ results: [], ftsAvailable: true }),
}));
vi.mock('../../../src/mcp/core/embedder.js', () => ({