mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-28 06:22:22 +00:00
gitnexus wal cleanup preventing reindexing issue fixed
This commit is contained in:
abhigyanpatwari
parent
4bc4b68786
commit
0943df9feb
50 changed files with 1855 additions and 1917 deletions
|
|
@ -7,6 +7,7 @@ description: Trace bugs through call chains using knowledge graph
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. gitnexus_search({query}) → Find code related to error
|
||||
2. gitnexus_explore({name, type: "symbol"}) → Get callers and callees
|
||||
3. READ gitnexus://process/{name} → Trace execution flow
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Navigate unfamiliar code using GitNexus knowledge graph
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. READ gitnexus://context → Get codebase overview (~150 tokens)
|
||||
2. READ gitnexus://clusters → See all functional clusters
|
||||
3. READ gitnexus://cluster/{name} → Deep dive on specific cluster
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Analyze blast radius before making code changes
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. gitnexus_impact({target, direction: "upstream"}) → What depends on this
|
||||
2. READ gitnexus://clusters → Check affected areas
|
||||
3. READ gitnexus://processes → Affected execution flows
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Plan safe refactors using blast radius and dependency mapping
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. gitnexus_impact({target, direction: "upstream"}) → Map all dependents
|
||||
2. READ gitnexus://schema → Understand graph structure
|
||||
3. gitnexus_cypher → Find all references
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ This project is indexed by GitNexus, providing AI agents with deep code intellig
|
|||
|
||||
| Metric | Count |
|
||||
|--------|-------|
|
||||
| Files | 150 |
|
||||
| Symbols | 930 |
|
||||
| Relationships | 2411 |
|
||||
| Communities | 280 |
|
||||
| Files | 146 |
|
||||
| Symbols | 907 |
|
||||
| Relationships | 2306 |
|
||||
| Communities | 278 |
|
||||
| Processes | 75 |
|
||||
|
||||
## Quick Start
|
||||
|
|
|
|||
|
|
@ -7,10 +7,10 @@ This project is indexed by GitNexus, providing AI agents with deep code intellig
|
|||
|
||||
| Metric | Count |
|
||||
|--------|-------|
|
||||
| Files | 150 |
|
||||
| Symbols | 930 |
|
||||
| Relationships | 2411 |
|
||||
| Communities | 280 |
|
||||
| Files | 146 |
|
||||
| Symbols | 907 |
|
||||
| Relationships | 2306 |
|
||||
| Communities | 278 |
|
||||
| Processes | 75 |
|
||||
|
||||
## Quick Start
|
||||
|
|
|
|||
|
|
@ -521,7 +521,7 @@ ORDER BY distance
|
|||
| **Visualization** | Sigma.js, Graphology, ForceAtlas2 (WebGL) |
|
||||
| **Parsing** | Tree-sitter WASM (9 languages) |
|
||||
| **Database** | KuzuDB WASM (graph + vector HNSW) |
|
||||
| **Clustering** | Graphology + Leiden (Louvain) |
|
||||
| **Clustering** | Graphology + Leiden algorithm |
|
||||
| **Embeddings** | transformers.js, snowflake-arctic-embed-xs (22M) |
|
||||
| **AI** | LangChain ReAct agent, streaming |
|
||||
| **Concurrency** | Web Workers + Comlink |
|
||||
|
|
@ -537,12 +537,6 @@ ORDER BY distance
|
|||
|
||||
---
|
||||
|
||||
## 📝 License
|
||||
|
||||
MIT License
|
||||
|
||||
---
|
||||
|
||||
## 🙏 Acknowledgments
|
||||
|
||||
- [Tree-sitter](https://tree-sitter.github.io/) - AST parsing
|
||||
|
|
|
|||
2
gitnexus-web/.gitignore
vendored
Normal file
2
gitnexus-web/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
.vercel
|
||||
.env*.local
|
||||
|
|
@ -1,204 +0,0 @@
|
|||
/**
|
||||
* Activity Feed Component
|
||||
*
|
||||
* Shows real-time log of external AI agent tool calls.
|
||||
* Used in RightPanel as an alternative to the Chat tab.
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useRef } from 'react';
|
||||
import { Activity, Search, Database, Terminal, Loader2, CheckCircle, XCircle, Clock, FileText, Zap, Map, Compass } from 'lucide-react';
|
||||
import { getMCPClient, type ActivityEvent } from '../core/mcp/mcp-client';
|
||||
|
||||
// Tool icons
|
||||
const TOOL_ICONS: Record<string, typeof Search> = {
|
||||
context: Zap,
|
||||
search: Search,
|
||||
cypher: Database,
|
||||
grep: Terminal,
|
||||
read: FileText,
|
||||
impact: Activity,
|
||||
overview: Map,
|
||||
explore: Compass,
|
||||
};
|
||||
|
||||
// Tool colors
|
||||
const TOOL_COLORS: Record<string, string> = {
|
||||
context: 'text-amber-400',
|
||||
search: 'text-cyan-400',
|
||||
cypher: 'text-purple-400',
|
||||
grep: 'text-green-400',
|
||||
read: 'text-blue-400',
|
||||
impact: 'text-rose-400',
|
||||
overview: 'text-indigo-400',
|
||||
explore: 'text-teal-400',
|
||||
};
|
||||
|
||||
export function ActivityFeed() {
|
||||
const [events, setEvents] = useState<ActivityEvent[]>([]);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const client = getMCPClient();
|
||||
|
||||
// Subscribe to activity events
|
||||
const unsubscribe = client.onActivity((event) => {
|
||||
setEvents(prev => {
|
||||
// Keep max 100 events
|
||||
const next = [...prev, event];
|
||||
if (next.length > 100) {
|
||||
next.shift();
|
||||
}
|
||||
return next;
|
||||
});
|
||||
});
|
||||
|
||||
// Get existing events
|
||||
setEvents(client.getActivityLog());
|
||||
|
||||
return () => {
|
||||
unsubscribe();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Auto-scroll to bottom
|
||||
useEffect(() => {
|
||||
if (containerRef.current) {
|
||||
containerRef.current.scrollTop = containerRef.current.scrollHeight;
|
||||
}
|
||||
}, [events]);
|
||||
|
||||
const formatTime = (timestamp: number) => {
|
||||
const date = new Date(timestamp);
|
||||
return date.toLocaleTimeString('en-US', {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit',
|
||||
});
|
||||
};
|
||||
|
||||
const formatParams = (params: any): string => {
|
||||
if (!params) return '';
|
||||
// Show first key-value pairs, truncated
|
||||
const entries = Object.entries(params).slice(0, 2);
|
||||
return entries.map(([k, v]) => {
|
||||
const val = typeof v === 'string' ? v.slice(0, 30) : JSON.stringify(v).slice(0, 30);
|
||||
return `${k}: ${val}${val.length >= 30 ? '...' : ''}`;
|
||||
}).join(', ');
|
||||
};
|
||||
|
||||
const formatResult = (event: ActivityEvent): string => {
|
||||
if (event.status === 'running') return 'Running...';
|
||||
if (event.status === 'error') return `Error: ${event.error?.slice(0, 50) || 'Unknown'}`;
|
||||
|
||||
// Format result based on type
|
||||
if (Array.isArray(event.result)) {
|
||||
return `${event.result.length} results`;
|
||||
}
|
||||
if (typeof event.result === 'object' && event.result) {
|
||||
const keys = Object.keys(event.result);
|
||||
if (keys.includes('content')) return `${event.result.content?.length || 0} chars`;
|
||||
if (keys.includes('projectName')) return event.result.projectName;
|
||||
return `{${keys.slice(0, 3).join(', ')}${keys.length > 3 ? '...' : ''}}`;
|
||||
}
|
||||
return String(event.result || 'Done');
|
||||
};
|
||||
|
||||
if (events.length === 0) {
|
||||
return (
|
||||
<div className="flex flex-col items-center justify-center h-full text-center px-8">
|
||||
<div className="w-14 h-14 mb-4 flex items-center justify-center bg-gradient-to-br from-accent to-node-class rounded-xl shadow-glow text-2xl">
|
||||
📡
|
||||
</div>
|
||||
<h3 className="text-base font-medium mb-2">
|
||||
No Agent Activity
|
||||
</h3>
|
||||
<p className="text-sm text-text-secondary leading-relaxed">
|
||||
When external AI agents (Cursor, Claude Code) call GitNexus tools,
|
||||
their activity will appear here in real-time.
|
||||
</p>
|
||||
<p className="text-xs text-text-muted mt-4">
|
||||
Make sure MCP toggle is enabled in the header
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div ref={containerRef} className="flex-1 overflow-y-auto p-4 scrollbar-thin">
|
||||
<div className="space-y-3">
|
||||
{events.map((event) => {
|
||||
const Icon = TOOL_ICONS[event.tool] || Activity;
|
||||
const color = TOOL_COLORS[event.tool] || 'text-text-muted';
|
||||
|
||||
return (
|
||||
<div
|
||||
key={event.id}
|
||||
className="bg-elevated border border-border-subtle rounded-lg p-3 animate-fade-in"
|
||||
>
|
||||
{/* Header row */}
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
{/* Agent color indicator */}
|
||||
{event.agentColor && (
|
||||
<div
|
||||
className="w-2 h-2 rounded-full shrink-0"
|
||||
style={{ backgroundColor: event.agentColor }}
|
||||
title={event.agentName || 'Unknown agent'}
|
||||
/>
|
||||
)}
|
||||
<Icon className={`w-4 h-4 ${color}`} />
|
||||
<span className="font-medium text-sm">{event.tool}</span>
|
||||
{event.agentName && event.agentName !== 'Unknown' && (
|
||||
<span
|
||||
className="text-[10px] px-1.5 py-0.5 rounded-full"
|
||||
style={{
|
||||
backgroundColor: `${event.agentColor}20`,
|
||||
color: event.agentColor
|
||||
}}
|
||||
>
|
||||
{event.agentName}
|
||||
</span>
|
||||
)}
|
||||
<span className="text-xs text-text-muted ml-auto flex items-center gap-1">
|
||||
<Clock className="w-3 h-3" />
|
||||
{formatTime(event.timestamp)}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{/* Params preview */}
|
||||
{event.params && Object.keys(event.params).length > 0 && (
|
||||
<div className="text-xs text-text-secondary font-mono bg-deep rounded px-2 py-1 mb-2 truncate">
|
||||
{formatParams(event.params)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Status/Result */}
|
||||
<div className="flex items-center gap-2 text-xs">
|
||||
{event.status === 'running' && (
|
||||
<>
|
||||
<Loader2 className="w-3 h-3 animate-spin text-accent" />
|
||||
<span className="text-accent">Running...</span>
|
||||
</>
|
||||
)}
|
||||
{event.status === 'complete' && (
|
||||
<>
|
||||
<CheckCircle className="w-3 h-3 text-green-400" />
|
||||
<span className="text-text-secondary">{formatResult(event)}</span>
|
||||
{event.duration && (
|
||||
<span className="text-text-muted ml-auto">{event.duration}ms</span>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{event.status === 'error' && (
|
||||
<>
|
||||
<XCircle className="w-3 h-3 text-rose-400" />
|
||||
<span className="text-rose-300">{formatResult(event)}</span>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -3,8 +3,6 @@ import { useAppState } from '../hooks/useAppState';
|
|||
import { useState, useMemo, useRef, useEffect, useCallback } from 'react';
|
||||
import { GraphNode } from '../core/graph/types';
|
||||
import { EmbeddingStatus } from './EmbeddingStatus';
|
||||
import { MCPToggle } from './MCPToggle';
|
||||
import { buildCodebaseContext } from '../core/llm/context-builder';
|
||||
|
||||
// Color mapping for node types in search results
|
||||
const NODE_TYPE_COLORS: Record<string, string> = {
|
||||
|
|
@ -31,11 +29,6 @@ export const Header = ({ onFocusNode }: HeaderProps) => {
|
|||
isRightPanelOpen,
|
||||
rightPanelTab,
|
||||
setSettingsPanelOpen,
|
||||
runQuery,
|
||||
semanticSearch,
|
||||
setHighlightedNodeIds,
|
||||
fileContents,
|
||||
triggerNodeAnimation,
|
||||
} = useAppState();
|
||||
const [searchQuery, setSearchQuery] = useState('');
|
||||
const [isSearchOpen, setIsSearchOpen] = useState(false);
|
||||
|
|
@ -220,177 +213,6 @@ export const Header = ({ onFocusNode }: HeaderProps) => {
|
|||
{/* Embedding Status */}
|
||||
<EmbeddingStatus />
|
||||
|
||||
{/* MCP Toggle for external AI agents */}
|
||||
<MCPToggle
|
||||
showOnboardingTip={!!graph}
|
||||
onSearch={async (query, limit = 10) => {
|
||||
// Use semantic search from the app
|
||||
const results = await semanticSearch(query, limit);
|
||||
// Trigger pulse animation on search results
|
||||
const nodeIds = results.map((r: any) => r.id).filter(Boolean);
|
||||
if (nodeIds.length > 0) {
|
||||
triggerNodeAnimation(nodeIds, 'pulse');
|
||||
}
|
||||
return results;
|
||||
}}
|
||||
onCypher={async (query) => {
|
||||
// Execute Cypher query
|
||||
const results = await runQuery(query);
|
||||
return results;
|
||||
}}
|
||||
onImpact={async (nodeId: string, hops = 2) => {
|
||||
// Run impact analysis query
|
||||
const query = `
|
||||
MATCH (start)-[*1..${hops}]-(connected)
|
||||
WHERE start.id = '${nodeId}' OR start.name = '${nodeId}'
|
||||
RETURN DISTINCT connected.id AS id, connected.name AS name, labels(connected) AS labels
|
||||
`;
|
||||
const results = await runQuery(query);
|
||||
// Trigger ripple animation on impact results
|
||||
const nodeIds = results.map((r: any) => r.id).filter(Boolean);
|
||||
if (nodeIds.length > 0) {
|
||||
triggerNodeAnimation(nodeIds, 'ripple');
|
||||
}
|
||||
return results;
|
||||
}}
|
||||
onOverview={async () => {
|
||||
// Return codebase overview: clusters + processes
|
||||
const clustersQuery = `
|
||||
MATCH (c:Community)
|
||||
OPTIONAL MATCH (c)<-[:CodeRelation {type: 'MEMBER_OF'}]-(m)
|
||||
RETURN c.id AS id, c.label AS label, c.cohesion AS cohesion, c.description AS description, count(m) AS memberCount
|
||||
ORDER BY memberCount DESC
|
||||
LIMIT 50
|
||||
`;
|
||||
const processesQuery = `
|
||||
MATCH (p:Process)
|
||||
RETURN p.id AS id, p.label AS label, p.processType AS type, p.stepCount AS steps
|
||||
ORDER BY p.stepCount DESC
|
||||
LIMIT 50
|
||||
`;
|
||||
const [clusters, processes] = await Promise.all([
|
||||
runQuery(clustersQuery),
|
||||
runQuery(processesQuery),
|
||||
]);
|
||||
return { clusters, processes };
|
||||
}}
|
||||
onExplore={async (target: string, type?: 'symbol' | 'cluster' | 'process') => {
|
||||
// Explore a specific target
|
||||
if (type === 'cluster' || target.startsWith('comm_')) {
|
||||
const query = `
|
||||
MATCH (c:Community)
|
||||
WHERE c.id = '${target}' OR c.label CONTAINS '${target}'
|
||||
OPTIONAL MATCH (c)<-[:CodeRelation {type: 'MEMBER_OF'}]-(m)
|
||||
RETURN c.id AS id, c.label AS label, c.description AS description, collect(m.name)[0..10] AS members
|
||||
LIMIT 1
|
||||
`;
|
||||
return await runQuery(query);
|
||||
} else if (type === 'process' || target.startsWith('proc_')) {
|
||||
const query = `
|
||||
MATCH (p:Process)
|
||||
WHERE p.id = '${target}' OR p.label CONTAINS '${target}'
|
||||
OPTIONAL MATCH (s)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p)
|
||||
RETURN p.id AS id, p.label AS label, p.stepCount AS steps, collect({name: s.name, step: r.step})[0..20] AS trace
|
||||
LIMIT 1
|
||||
`;
|
||||
return await runQuery(query);
|
||||
} else {
|
||||
// Symbol exploration
|
||||
const query = `
|
||||
MATCH (n)
|
||||
WHERE n.name = '${target}' OR n.id ENDS WITH ':${target}'
|
||||
OPTIONAL MATCH (n)-[:CodeRelation {type: 'MEMBER_OF'}]->(c:Community)
|
||||
OPTIONAL MATCH (n)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process)
|
||||
RETURN n.id AS id, n.name AS name, n.filePath AS filePath, label(n) AS nodeType,
|
||||
c.label AS cluster, collect({process: p.label, step: r.step}) AS processes
|
||||
LIMIT 1
|
||||
`;
|
||||
return await runQuery(query);
|
||||
}
|
||||
}}
|
||||
getContext={async () => {
|
||||
// Build codebase context for external AI agents
|
||||
if (!projectName) return null;
|
||||
const context = await buildCodebaseContext(runQuery, projectName);
|
||||
// Reshape to match MCP CodebaseContext format
|
||||
return {
|
||||
projectName: context.stats.projectName,
|
||||
stats: {
|
||||
fileCount: context.stats.fileCount,
|
||||
functionCount: context.stats.functionCount,
|
||||
classCount: context.stats.classCount,
|
||||
interfaceCount: context.stats.interfaceCount,
|
||||
methodCount: context.stats.methodCount,
|
||||
},
|
||||
hotspots: context.hotspots,
|
||||
folderTree: context.folderTree,
|
||||
};
|
||||
}}
|
||||
onGrep={async (pattern, caseSensitive = false, maxResults = 50) => {
|
||||
// Grep across file contents
|
||||
const results: Array<{ filePath: string; line: string; lineNumber: number; match: string }> = [];
|
||||
const regex = new RegExp(pattern, caseSensitive ? 'g' : 'gi');
|
||||
|
||||
for (const [filePath, content] of fileContents.entries()) {
|
||||
const lines = content.split('\n');
|
||||
for (let i = 0; i < lines.length && results.length < maxResults; i++) {
|
||||
const line = lines[i];
|
||||
const match = line.match(regex);
|
||||
if (match) {
|
||||
results.push({
|
||||
filePath,
|
||||
line: line.trim(),
|
||||
lineNumber: i + 1,
|
||||
match: match[0],
|
||||
});
|
||||
}
|
||||
}
|
||||
if (results.length >= maxResults) break;
|
||||
}
|
||||
return results;
|
||||
}}
|
||||
onRead={async (filePath, startLine, endLine) => {
|
||||
// Read file content
|
||||
let content = fileContents.get(filePath);
|
||||
|
||||
// Try normalized path if not found
|
||||
if (!content) {
|
||||
const normalizedPath = filePath.replace(/\\/g, '/');
|
||||
for (const [path, c] of fileContents.entries()) {
|
||||
if (path.endsWith(normalizedPath) || normalizedPath.endsWith(path)) {
|
||||
content = c;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!content) {
|
||||
return { error: `File not found: ${filePath}` };
|
||||
}
|
||||
|
||||
const lines = content.split('\n');
|
||||
const language = filePath.split('.').pop() || 'text';
|
||||
|
||||
// If line range specified, return only those lines
|
||||
if (startLine !== undefined && endLine !== undefined) {
|
||||
const slice = lines.slice(startLine - 1, endLine);
|
||||
return {
|
||||
filePath,
|
||||
content: slice.join('\n'),
|
||||
language,
|
||||
lines: slice.length,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
filePath,
|
||||
content,
|
||||
language,
|
||||
lines: lines.length,
|
||||
};
|
||||
}}
|
||||
/>
|
||||
|
||||
{/* Icon buttons */}
|
||||
<button
|
||||
onClick={() => setSettingsPanelOpen(true)}
|
||||
|
|
|
|||
|
|
@ -1,140 +0,0 @@
|
|||
import { Brain, Sparkles, X, Settings } from 'lucide-react';
|
||||
|
||||
interface IntelligentClusteringModalProps {
|
||||
isOpen: boolean;
|
||||
onClose: () => void;
|
||||
onEnable: () => void;
|
||||
onConfigure: () => void;
|
||||
}
|
||||
|
||||
export const IntelligentClusteringModal = ({
|
||||
isOpen,
|
||||
onClose,
|
||||
onEnable,
|
||||
onConfigure
|
||||
}: IntelligentClusteringModalProps) => {
|
||||
if (!isOpen) return null;
|
||||
|
||||
// Parent handles all state updates via onEnable/onConfigure/onClose
|
||||
const handleEnable = () => {
|
||||
onEnable();
|
||||
};
|
||||
|
||||
const handleSkip = () => {
|
||||
onClose();
|
||||
};
|
||||
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-50 flex items-center justify-center p-4">
|
||||
{/* Backdrop */}
|
||||
<div
|
||||
className="absolute inset-0 bg-black/60 backdrop-blur-sm"
|
||||
onClick={handleSkip}
|
||||
/>
|
||||
|
||||
{/* Modal Content */}
|
||||
<div className="relative bg-surface border border-border-subtle rounded-2xl shadow-2xl max-w-md w-full overflow-hidden animate-in fade-in zoom-in-95 duration-200">
|
||||
|
||||
{/* Header with cool gradient background */}
|
||||
<div className="bg-gradient-to-br from-accent/20 to-surface p-6 pb-8 border-b border-border-subtle/50 relative overflow-hidden">
|
||||
<div className="absolute top-0 right-0 p-4 opacity-10">
|
||||
<Brain className="w-32 h-32 rotate-12" />
|
||||
</div>
|
||||
|
||||
<button
|
||||
onClick={handleSkip}
|
||||
className="absolute top-4 right-4 p-2 text-text-muted hover:text-text-primary rounded-full hover:bg-black/10 transition-colors"
|
||||
>
|
||||
<X className="w-5 h-5" />
|
||||
</button>
|
||||
|
||||
<div className="flex items-center gap-3 mb-2">
|
||||
<div className="p-3 bg-accent text-white rounded-xl shadow-lg shadow-accent/20">
|
||||
<Sparkles className="w-6 h-6" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2 className="text-xl font-bold text-text-primary mt-4">
|
||||
Upgrade to Intelligent Clustering?
|
||||
</h2>
|
||||
<p className="text-text-secondary mt-1 text-sm leading-relaxed">
|
||||
Your clusters are ready, but they could be smarter! Right now they're just named after folders.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Body */}
|
||||
<div className="p-6 space-y-6">
|
||||
|
||||
<div className="space-y-3">
|
||||
<h3 className="text-sm font-semibold text-text-primary flex items-center gap-2">
|
||||
<Brain className="w-4 h-4 text-accent" />
|
||||
What you get:
|
||||
</h3>
|
||||
<ul className="space-y-2 text-sm text-text-secondary">
|
||||
<li className="flex items-start gap-2">
|
||||
<span className="text-green-400 mt-1">✓</span>
|
||||
Semantic names (e.g., "Auth System" vs "utils")
|
||||
</li>
|
||||
<li className="flex items-start gap-2">
|
||||
<span className="text-green-400 mt-1">✓</span>
|
||||
Search keywords for better agent context
|
||||
</li>
|
||||
<li className="flex items-start gap-2">
|
||||
<span className="text-green-400 mt-1">✓</span>
|
||||
Descriptions of what the code actually does
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
{/* How it works */}
|
||||
<div className="p-4 bg-elevated/50 border border-border-subtle rounded-xl">
|
||||
<div className="flex items-start gap-3">
|
||||
<div className="p-2 bg-blue-500/10 rounded-lg text-blue-400">
|
||||
<Brain className="w-4 h-4" />
|
||||
</div>
|
||||
<div>
|
||||
<h4 className="text-sm font-medium text-text-primary">Uses Your Configured LLM</h4>
|
||||
<p className="text-xs text-text-muted mt-1 leading-relaxed">
|
||||
Runs on your own API key. Very low token usage (~$0.01 for most codebases).
|
||||
<br />
|
||||
<span className="text-amber-400">💡 Tip:</span> Use a cheaper model like GPT-4o-mini in settings!
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
<div className="p-6 pt-2 bg-surface flex flex-col gap-3">
|
||||
<button
|
||||
onClick={handleEnable}
|
||||
className="w-full py-3 px-4 bg-accent text-white font-medium rounded-xl hover:bg-accent-dim shadow-lg shadow-accent/20 transition-all active:scale-[0.98] flex items-center justify-center gap-2"
|
||||
>
|
||||
<Sparkles className="w-4 h-4" />
|
||||
Enable Smart Clustering
|
||||
</button>
|
||||
|
||||
<div className="flex bg-elevated rounded-xl p-1 gap-1">
|
||||
<button
|
||||
onClick={onConfigure}
|
||||
className="flex-1 py-2 px-3 text-sm font-medium text-text-secondary hover:text-text-primary hover:bg-hover rounded-lg transition-colors flex items-center justify-center gap-2"
|
||||
>
|
||||
<Settings className="w-3 h-3" />
|
||||
Configure Model
|
||||
</button>
|
||||
<div className="w-px bg-border-subtle my-2" />
|
||||
<button
|
||||
onClick={handleSkip}
|
||||
className="flex-1 py-2 px-3 text-sm font-medium text-text-muted hover:text-text-primary hover:bg-hover rounded-lg transition-colors"
|
||||
>
|
||||
No thanks (Free)
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
|
@ -1,300 +0,0 @@
|
|||
/**
|
||||
* MCP Toggle Component
|
||||
*
|
||||
* Toggle for enabling MCP exposure to external AI agents (Cursor, Claude, etc.)
|
||||
* Shows MCP config for setup and connection status.
|
||||
*/
|
||||
|
||||
import { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import { Copy, Check, X, Sparkles, Zap, ExternalLink } from 'lucide-react';
|
||||
import { getMCPClient, type CodebaseContext } from '../core/mcp/mcp-client';
|
||||
|
||||
type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'error';
|
||||
|
||||
interface MCPToggleProps {
|
||||
onSearch?: (query: string, limit?: number) => Promise<any>;
|
||||
onCypher?: (query: string) => Promise<any>;
|
||||
onImpact?: (nodeId: string, hops?: number) => Promise<any>;
|
||||
onGrep?: (pattern: string, caseSensitive?: boolean, maxResults?: number) => Promise<any>;
|
||||
onRead?: (filePath: string, startLine?: number, endLine?: number) => Promise<any>;
|
||||
onOverview?: () => Promise<any>;
|
||||
onExplore?: (target: string, type?: 'symbol' | 'cluster' | 'process') => Promise<any>;
|
||||
showOnboardingTip?: boolean;
|
||||
getContext?: () => Promise<CodebaseContext | null>;
|
||||
}
|
||||
|
||||
const MCP_TIP_DISMISSED_KEY = 'gitnexus-mcp-tip-dismissed';
|
||||
|
||||
// MCP config that users copy to their AI agent
|
||||
const MCP_CONFIG = `{
|
||||
"mcpServers": {
|
||||
"gitnexus": {
|
||||
"command": "npx",
|
||||
"args": ["--prefer-online", "-y", "gitnexus-mcp@latest"]
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
export function MCPToggle({
|
||||
onSearch,
|
||||
onCypher,
|
||||
onImpact,
|
||||
onGrep,
|
||||
onRead,
|
||||
onOverview,
|
||||
onExplore,
|
||||
showOnboardingTip = false,
|
||||
getContext,
|
||||
}: MCPToggleProps = {}) {
|
||||
const [status, setStatus] = useState<ConnectionState>('disconnected');
|
||||
const [copied, setCopied] = useState(false);
|
||||
const [showPopup, setShowPopup] = useState(false);
|
||||
const popupRef = useRef<HTMLDivElement | null>(null);
|
||||
const [showTip, setShowTip] = useState(false);
|
||||
|
||||
const isConnected = status === 'connected';
|
||||
const isConnecting = status === 'connecting';
|
||||
|
||||
// Show tip when graph becomes ready
|
||||
useEffect(() => {
|
||||
if (showOnboardingTip) {
|
||||
const dismissed = localStorage.getItem(MCP_TIP_DISMISSED_KEY);
|
||||
if (!dismissed) {
|
||||
const timer = setTimeout(() => setShowTip(true), 1500);
|
||||
return () => clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}, [showOnboardingTip]);
|
||||
|
||||
// Close popup when clicking outside
|
||||
useEffect(() => {
|
||||
if (!showPopup) return;
|
||||
const handleClickOutside = (event: MouseEvent) => {
|
||||
if (popupRef.current && !popupRef.current.contains(event.target as Node)) {
|
||||
setShowPopup(false);
|
||||
}
|
||||
};
|
||||
document.addEventListener('mousedown', handleClickOutside);
|
||||
return () => document.removeEventListener('mousedown', handleClickOutside);
|
||||
}, [showPopup]);
|
||||
|
||||
const dismissTip = () => {
|
||||
setShowTip(false);
|
||||
localStorage.setItem(MCP_TIP_DISMISSED_KEY, 'true');
|
||||
};
|
||||
|
||||
const connect = useCallback(async () => {
|
||||
const client = getMCPClient();
|
||||
setStatus('connecting');
|
||||
setShowTip(false);
|
||||
|
||||
try {
|
||||
await client.connect();
|
||||
|
||||
// Register tool handlers
|
||||
if (onSearch) client.registerHandler('search', async (params) => onSearch(params.query, params.limit));
|
||||
if (onCypher) client.registerHandler('cypher', async (params) => onCypher(params.query));
|
||||
if (onImpact) client.registerHandler('impact', async (params) => onImpact(params.nodeId, params.hops));
|
||||
if (onGrep) client.registerHandler('grep', async (params) => onGrep(params.pattern, params.caseSensitive, params.maxResults));
|
||||
if (onRead) client.registerHandler('read', async (params) => onRead(params.filePath, params.startLine, params.endLine));
|
||||
if (onOverview) client.registerHandler('overview', async () => onOverview());
|
||||
if (onExplore) client.registerHandler('explore', async (params) => onExplore(params.target, params.type));
|
||||
if (getContext) client.registerHandler('context', async () => getContext());
|
||||
|
||||
setStatus('connected');
|
||||
setShowPopup(false);
|
||||
localStorage.setItem(MCP_TIP_DISMISSED_KEY, 'true');
|
||||
|
||||
// Send context after connecting
|
||||
if (getContext) {
|
||||
try {
|
||||
const context = await getContext();
|
||||
if (context) client.sendContext(context);
|
||||
} catch (e) {
|
||||
console.error('[MCP] Failed to send context:', e);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
setStatus('error');
|
||||
}
|
||||
}, [onSearch, onCypher, onImpact, onGrep, onRead, onOverview, onExplore, getContext]);
|
||||
|
||||
const disconnect = useCallback(() => {
|
||||
const client = getMCPClient();
|
||||
client.disconnect();
|
||||
setStatus('disconnected');
|
||||
}, []);
|
||||
|
||||
const toggle = useCallback(() => {
|
||||
if (isConnected) {
|
||||
disconnect();
|
||||
} else if (!isConnecting) {
|
||||
connect();
|
||||
}
|
||||
}, [isConnected, isConnecting, connect, disconnect]);
|
||||
|
||||
const copyConfig = () => {
|
||||
navigator.clipboard.writeText(MCP_CONFIG);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 2000);
|
||||
};
|
||||
|
||||
// Listen for connection changes
|
||||
useEffect(() => {
|
||||
const client = getMCPClient();
|
||||
const unsubscribe = client.onConnectionChange((connected) => {
|
||||
setStatus(connected ? 'connected' : 'disconnected');
|
||||
if (connected) setShowPopup(false);
|
||||
});
|
||||
return () => { unsubscribe(); };
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="relative flex items-center gap-2">
|
||||
{/* MCP Button */}
|
||||
<button
|
||||
onClick={() => setShowPopup(!showPopup)}
|
||||
className={`
|
||||
flex items-center gap-1.5 px-2.5 py-1.5 rounded-lg text-xs font-medium
|
||||
transition-all duration-200
|
||||
${isConnected
|
||||
? 'bg-green-500/15 text-green-400 border border-green-500/30 hover:bg-green-500/25'
|
||||
: 'bg-surface hover:bg-hover text-text-secondary hover:text-text-primary border border-border-subtle'
|
||||
}
|
||||
`}
|
||||
>
|
||||
<Zap className={`w-3.5 h-3.5 ${isConnected ? 'text-green-400' : ''}`} />
|
||||
<span>MCP</span>
|
||||
{isConnected && <span className="w-1.5 h-1.5 rounded-full bg-green-400 animate-pulse" />}
|
||||
</button>
|
||||
|
||||
{/* Popup */}
|
||||
{showPopup && (
|
||||
<div
|
||||
ref={popupRef}
|
||||
className="absolute top-full right-0 mt-2 w-[380px] bg-surface/95 backdrop-blur-xl border border-border-subtle rounded-xl shadow-2xl z-50 overflow-hidden"
|
||||
>
|
||||
{/* Header */}
|
||||
<div className="px-4 py-3 bg-gradient-to-r from-accent/10 to-transparent border-b border-border-subtle">
|
||||
<div className="flex items-center justify-between">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="p-1.5 bg-accent/20 rounded-lg">
|
||||
<Sparkles className="w-4 h-4 text-accent" />
|
||||
</div>
|
||||
<div>
|
||||
<h3 className="text-sm font-semibold text-text-primary">Connect AI Agents</h3>
|
||||
<p className="text-[10px] text-text-muted">Cursor, Claude Code, Antigravity</p>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => setShowPopup(false)}
|
||||
className="p-1 text-text-muted hover:text-text-primary rounded transition-colors"
|
||||
>
|
||||
<X className="w-4 h-4" />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Content */}
|
||||
<div className="p-4 space-y-4">
|
||||
{/* Step 1: Config */}
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className="flex items-center justify-center w-5 h-5 rounded-full bg-accent/20 text-accent text-[10px] font-bold">1</span>
|
||||
<span className="text-xs text-text-secondary">Add to your AI agent's MCP config</span>
|
||||
</div>
|
||||
<div className="relative group">
|
||||
<pre className="p-3 bg-deep rounded-lg text-[11px] font-mono text-text-primary overflow-x-auto border border-border-subtle">
|
||||
{MCP_CONFIG}
|
||||
</pre>
|
||||
<button
|
||||
onClick={copyConfig}
|
||||
className="absolute top-2 right-2 p-1.5 bg-surface/80 hover:bg-hover rounded-md transition-colors opacity-0 group-hover:opacity-100"
|
||||
title="Copy config"
|
||||
>
|
||||
{copied ? (
|
||||
<Check className="w-3.5 h-3.5 text-green-400" />
|
||||
) : (
|
||||
<Copy className="w-3.5 h-3.5 text-text-muted" />
|
||||
)}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Step 2: Connect */}
|
||||
<div>
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<span className="flex items-center justify-center w-5 h-5 rounded-full bg-accent/20 text-accent text-[10px] font-bold">2</span>
|
||||
<span className="text-xs text-text-secondary">Connect browser to daemon</span>
|
||||
</div>
|
||||
<button
|
||||
onClick={toggle}
|
||||
disabled={isConnecting}
|
||||
className={`
|
||||
w-full py-2.5 rounded-lg text-sm font-medium transition-all duration-200
|
||||
${isConnected
|
||||
? 'bg-green-500/15 text-green-400 border border-green-500/30 hover:bg-green-500/25'
|
||||
: isConnecting
|
||||
? 'bg-surface text-text-muted cursor-wait'
|
||||
: 'bg-accent text-white hover:bg-accent-dim'
|
||||
}
|
||||
`}
|
||||
>
|
||||
{isConnected ? '✓ Connected' : isConnecting ? 'Connecting...' : 'Connect'}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Status message */}
|
||||
{status === 'error' && (
|
||||
<p className="text-[11px] text-amber-400 text-center">
|
||||
Daemon not running. Make sure your AI agent has started gitnexus-mcp.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{/* Help link */}
|
||||
<a
|
||||
href="https://github.com/abhigyanpatwari/GitNexus#mcp-integration"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="flex items-center justify-center gap-1.5 text-[11px] text-text-muted hover:text-accent transition-colors"
|
||||
>
|
||||
<span>Learn more</span>
|
||||
<ExternalLink className="w-3 h-3" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Onboarding Tip */}
|
||||
{showTip && !isConnected && !showPopup && (
|
||||
<div className="absolute top-full right-0 mt-3 w-72 p-4 bg-surface/95 backdrop-blur-xl border border-accent/30 rounded-xl shadow-2xl z-50 animate-in fade-in slide-in-from-top-2">
|
||||
<button
|
||||
onClick={dismissTip}
|
||||
className="absolute top-2 right-2 p-1 text-text-muted hover:text-text-primary rounded transition-colors"
|
||||
>
|
||||
<X className="w-3.5 h-3.5" />
|
||||
</button>
|
||||
<div className="flex items-start gap-3">
|
||||
<div className="p-2 bg-accent/20 rounded-lg flex-shrink-0">
|
||||
<Sparkles className="w-4 h-4 text-accent" />
|
||||
</div>
|
||||
<div>
|
||||
<h4 className="text-sm font-semibold text-text-primary mb-1">
|
||||
Connect your AI tools
|
||||
</h4>
|
||||
<p className="text-xs text-text-secondary leading-relaxed mb-3">
|
||||
Let Cursor or Claude access GitNexus code intelligence.
|
||||
</p>
|
||||
<button
|
||||
onClick={() => { dismissTip(); setShowPopup(true); }}
|
||||
className="px-3 py-1.5 text-xs font-medium bg-accent text-white rounded-lg hover:bg-accent-dim transition-colors"
|
||||
>
|
||||
Set up MCP
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,12 +1,11 @@
|
|||
import { useState, useRef, useEffect, useCallback } from 'react';
|
||||
import {
|
||||
Send, Square, Sparkles, User,
|
||||
PanelRightClose, Loader2, AlertTriangle, Activity, GitBranch
|
||||
PanelRightClose, Loader2, AlertTriangle, GitBranch
|
||||
} from 'lucide-react';
|
||||
import { useAppState } from '../hooks/useAppState';
|
||||
import { ToolCallCard } from './ToolCallCard';
|
||||
import { isProviderConfigured } from '../core/llm/settings-service';
|
||||
import { ActivityFeed } from './ActivityFeed';
|
||||
import { MarkdownRenderer } from './MarkdownRenderer';
|
||||
import { ProcessesPanel } from './ProcessesPanel';
|
||||
export const RightPanel = () => {
|
||||
|
|
@ -29,7 +28,7 @@ export const RightPanel = () => {
|
|||
} = useAppState();
|
||||
|
||||
const [chatInput, setChatInput] = useState('');
|
||||
const [activeTab, setActiveTab] = useState<'chat' | 'activity' | 'processes'>('chat');
|
||||
const [activeTab, setActiveTab] = useState<'chat' | 'processes'>('chat');
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
|
|
@ -225,18 +224,6 @@ export const RightPanel = () => {
|
|||
<span>Nexus AI</span>
|
||||
</button>
|
||||
|
||||
{/* Activity Tab */}
|
||||
<button
|
||||
onClick={() => setActiveTab('activity')}
|
||||
className={`flex items-center gap-1.5 px-3 py-1.5 rounded-md text-sm font-medium transition-colors ${activeTab === 'activity'
|
||||
? 'bg-accent/15 text-accent'
|
||||
: 'text-text-muted hover:text-text-primary hover:bg-hover'
|
||||
}`}
|
||||
>
|
||||
<Activity className="w-3.5 h-3.5" />
|
||||
<span>Activity</span>
|
||||
</button>
|
||||
|
||||
{/* Processes Tab */}
|
||||
<button
|
||||
onClick={() => setActiveTab('processes')}
|
||||
|
|
@ -263,13 +250,6 @@ export const RightPanel = () => {
|
|||
</button>
|
||||
</div>
|
||||
|
||||
{/* Activity Feed Tab */}
|
||||
{activeTab === 'activity' && (
|
||||
<div className="flex-1 flex flex-col overflow-hidden">
|
||||
<ActivityFeed />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Processes Tab */}
|
||||
{activeTab === 'processes' && (
|
||||
<div className="flex-1 flex flex-col overflow-hidden">
|
||||
|
|
|
|||
|
|
@ -18,12 +18,17 @@ import { NODE_TABLES, NodeTableName } from './schema';
|
|||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Sanitize string to ensure valid UTF-8
|
||||
* Removes or replaces invalid characters that would break CSV parsing
|
||||
* Sanitize string to ensure valid UTF-8 and safe CSV content for KuzuDB
|
||||
* Removes or replaces invalid characters that would break CSV parsing.
|
||||
*
|
||||
* Critical: KuzuDB's CSV parser can misinterpret \r\n inside quoted fields.
|
||||
* We normalize all line endings to \n only.
|
||||
*/
|
||||
const sanitizeUTF8 = (str: string): string => {
|
||||
return str
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n \r
|
||||
.replace(/\r\n/g, '\n') // Normalize Windows line endings first
|
||||
.replace(/\r/g, '\n') // Normalize remaining \r to \n
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n
|
||||
.replace(/[\uD800-\uDFFF]/g, '') // Remove surrogate pairs (invalid standalone)
|
||||
.replace(/[\uFFFE\uFFFF]/g, ''); // Remove BOM and special chars
|
||||
};
|
||||
|
|
|
|||
|
|
@ -113,70 +113,61 @@ export const loadGraphToKuzu = async (
|
|||
}
|
||||
|
||||
// 5. INSERT relations one by one (COPY doesn't work with multi-pair REL tables)
|
||||
// Parse CSV format: "from","to","type",confidence,"reason"
|
||||
// Build a set of valid table names for fast lookup
|
||||
const validTables = new Set<string>(NODE_TABLES as readonly string[]);
|
||||
|
||||
const getNodeLabel = (nodeId: string): string => {
|
||||
if (nodeId.startsWith('comm_')) return 'Community';
|
||||
if (nodeId.startsWith('proc_')) return 'Process';
|
||||
return nodeId.split(':')[0];
|
||||
};
|
||||
|
||||
// All multi-language tables are created with backticks - must always reference them with backticks
|
||||
const escapeLabel = (label: string): string => {
|
||||
return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label;
|
||||
};
|
||||
|
||||
let insertedRels = 0;
|
||||
let skippedRels = 0;
|
||||
const skippedRelStats = new Map<string, number>();
|
||||
for (const line of relLines) {
|
||||
try {
|
||||
// Parse CSV - handle quoted fields and numeric confidence
|
||||
// Parse CSV - handle quoted fields and numeric confidence
|
||||
// Format: "from","to","type",confidence,"reason",step
|
||||
// Note: step is unquoted numeric
|
||||
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/);
|
||||
if (!match) continue;
|
||||
|
||||
const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match;
|
||||
|
||||
const fromLabel = getNodeLabel(fromId);
|
||||
const toLabel = getNodeLabel(toId);
|
||||
|
||||
// Skip relationships where either node's label doesn't have a table in KuzuDB
|
||||
// Querying a non-existent table causes a fatal native crash
|
||||
if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
|
||||
skippedRels++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const confidence = parseFloat(confidenceStr) || 1.0;
|
||||
const step = parseInt(stepStr) || 0;
|
||||
|
||||
// Extract labels from node IDs
|
||||
// Community nodes have IDs like "comm_14" (no colon)
|
||||
// Other nodes have IDs like "Label:path:name"
|
||||
const getNodeLabel = (nodeId: string): string => {
|
||||
if (nodeId.startsWith('comm_')) {
|
||||
return 'Community';
|
||||
}
|
||||
if (nodeId.startsWith('proc_')) {
|
||||
return 'Process';
|
||||
}
|
||||
return nodeId.split(':')[0];
|
||||
};
|
||||
|
||||
// Reserved Cypher keywords need backtick escaping
|
||||
const RESERVED_LABELS = ['Macro', 'Enum', 'Union', 'Const', 'Module', 'Struct'];
|
||||
const escapeLabel = (label: string): string => {
|
||||
return RESERVED_LABELS.includes(label) ? `\`${label}\`` : label;
|
||||
};
|
||||
|
||||
const fromLabel = escapeLabel(getNodeLabel(fromId));
|
||||
const toLabel = escapeLabel(getNodeLabel(toId));
|
||||
|
||||
// INSERT with explicit node matching (including confidence and reason)
|
||||
const insertQuery = `
|
||||
MATCH (a:${fromLabel} {id: '${fromId.replace(/'/g, "''")}'}),
|
||||
(b:${toLabel} {id: '${toId.replace(/'/g, "''")}'})
|
||||
MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}'}),
|
||||
(b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}'})
|
||||
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b)
|
||||
`;
|
||||
await conn.query(insertQuery);
|
||||
insertedRels++;
|
||||
} catch (err) {
|
||||
// Skip failed insertions (nodes might not exist, or relation pair not allowed by schema)
|
||||
skippedRels++;
|
||||
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)"/);
|
||||
if (match) {
|
||||
const [, fromId, toId, relType] = match;
|
||||
const getNodeLabel = (nodeId: string): string => {
|
||||
if (nodeId.startsWith('comm_')) return 'Community';
|
||||
if (nodeId.startsWith('proc_')) return 'Process';
|
||||
return nodeId.split(':')[0];
|
||||
};
|
||||
const fromLabel = getNodeLabel(fromId);
|
||||
const toLabel = getNodeLabel(toId);
|
||||
const key = `${relType}:${fromLabel}->` + toLabel;
|
||||
skippedRelStats.set(key, (skippedRelStats.get(key) || 0) + 1);
|
||||
|
||||
// Log each skipped relation
|
||||
if (import.meta.env.DEV) {
|
||||
console.warn(`⚠️ Skipped: ${key} | "${fromId}" → "${toId}" | ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
|
@ -222,25 +213,41 @@ export const loadGraphToKuzu = async (
|
|||
}
|
||||
};
|
||||
|
||||
// KuzuDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes).
|
||||
// Source code content is full of backslashes which confuse the auto-detection.
|
||||
// We MUST explicitly set ESCAPE='"' and disable auto_detect.
|
||||
const COPY_CSV_OPTS = `(HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
|
||||
|
||||
// Multi-language table names created with backticks in CODE_ELEMENT_BASE
|
||||
const BACKTICK_TABLES = new Set([
|
||||
'Struct', 'Enum', 'Macro', 'Typedef', 'Union', 'Namespace', 'Trait', 'Impl',
|
||||
'TypeAlias', 'Const', 'Static', 'Property', 'Record', 'Delegate', 'Annotation',
|
||||
'Constructor', 'Template', 'Module',
|
||||
]);
|
||||
|
||||
const escapeTableName = (table: string): string => {
|
||||
return BACKTICK_TABLES.has(table) ? `\`${table}\`` : table;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get the COPY query for a node table with correct column mapping
|
||||
*/
|
||||
const getCopyQuery = (table: NodeTableName, path: string): string => {
|
||||
// File and Folder have different columns than code elements
|
||||
const t = escapeTableName(table);
|
||||
if (table === 'File') {
|
||||
return `COPY File(id, name, filePath, content) FROM "${path}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, name, filePath, content) FROM "${path}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Folder') {
|
||||
return `COPY Folder(id, name, filePath) FROM "${path}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, name, filePath) FROM "${path}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Community') {
|
||||
return `COPY Community(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${path}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${path}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Process') {
|
||||
return `COPY Process(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${path}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${path}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
// All code element tables: Function, Class, Interface, Method, CodeElement
|
||||
return `COPY ${table}(id, name, filePath, startLine, endLine, isExported, content) FROM "${path}" (HEADER=true, PARALLEL=false)`;
|
||||
// Code element tables (Function, Class, Interface, Method, CodeElement, and multi-language)
|
||||
return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content) FROM "${path}" ${COPY_CSV_OPTS}`;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -201,20 +201,20 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM File TO \`Struct\`,
|
||||
FROM File TO \`Enum\`,
|
||||
FROM File TO \`Macro\`,
|
||||
FROM File TO Typedef,
|
||||
FROM File TO \`Typedef\`,
|
||||
FROM File TO \`Union\`,
|
||||
FROM File TO Namespace,
|
||||
FROM File TO Trait,
|
||||
FROM File TO Impl,
|
||||
FROM File TO TypeAlias,
|
||||
FROM File TO \`Namespace\`,
|
||||
FROM File TO \`Trait\`,
|
||||
FROM File TO \`Impl\`,
|
||||
FROM File TO \`TypeAlias\`,
|
||||
FROM File TO \`Const\`,
|
||||
FROM File TO Static,
|
||||
FROM File TO Property,
|
||||
FROM File TO Record,
|
||||
FROM File TO Delegate,
|
||||
FROM File TO Annotation,
|
||||
FROM File TO Constructor,
|
||||
FROM File TO Template,
|
||||
FROM File TO \`Static\`,
|
||||
FROM File TO \`Property\`,
|
||||
FROM File TO \`Record\`,
|
||||
FROM File TO \`Delegate\`,
|
||||
FROM File TO \`Annotation\`,
|
||||
FROM File TO \`Constructor\`,
|
||||
FROM File TO \`Template\`,
|
||||
FROM File TO \`Module\`,
|
||||
FROM Folder TO Folder,
|
||||
FROM Folder TO File,
|
||||
|
|
@ -224,48 +224,48 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM Function TO Community,
|
||||
FROM Function TO \`Macro\`,
|
||||
FROM Function TO \`Struct\`,
|
||||
FROM Function TO Template,
|
||||
FROM Function TO \`Template\`,
|
||||
FROM Function TO \`Enum\`,
|
||||
FROM Function TO Namespace,
|
||||
FROM Function TO TypeAlias,
|
||||
FROM Function TO \`Namespace\`,
|
||||
FROM Function TO \`TypeAlias\`,
|
||||
FROM Function TO \`Module\`,
|
||||
FROM Function TO Impl,
|
||||
FROM Function TO \`Impl\`,
|
||||
FROM Function TO Interface,
|
||||
FROM Function TO Constructor,
|
||||
FROM Function TO \`Constructor\`,
|
||||
FROM Class TO Method,
|
||||
FROM Class TO Function,
|
||||
FROM Class TO Class,
|
||||
FROM Class TO Interface,
|
||||
FROM Class TO Community,
|
||||
FROM Class TO Template,
|
||||
FROM Class TO TypeAlias,
|
||||
FROM Class TO \`Template\`,
|
||||
FROM Class TO \`TypeAlias\`,
|
||||
FROM Class TO \`Struct\`,
|
||||
FROM Class TO \`Enum\`,
|
||||
FROM Class TO Constructor,
|
||||
FROM Class TO \`Constructor\`,
|
||||
FROM Method TO Function,
|
||||
FROM Method TO Method,
|
||||
FROM Method TO Class,
|
||||
FROM Method TO Community,
|
||||
FROM Method TO Template,
|
||||
FROM Method TO \`Template\`,
|
||||
FROM Method TO \`Struct\`,
|
||||
FROM Method TO TypeAlias,
|
||||
FROM Method TO \`TypeAlias\`,
|
||||
FROM Method TO \`Enum\`,
|
||||
FROM Method TO \`Macro\`,
|
||||
FROM Method TO Namespace,
|
||||
FROM Method TO \`Namespace\`,
|
||||
FROM Method TO \`Module\`,
|
||||
FROM Method TO Impl,
|
||||
FROM Method TO \`Impl\`,
|
||||
FROM Method TO Interface,
|
||||
FROM Method TO Constructor,
|
||||
FROM Template TO Template,
|
||||
FROM Template TO Function,
|
||||
FROM Template TO Method,
|
||||
FROM Template TO Class,
|
||||
FROM Template TO \`Struct\`,
|
||||
FROM Template TO TypeAlias,
|
||||
FROM Template TO \`Enum\`,
|
||||
FROM Template TO \`Macro\`,
|
||||
FROM Template TO Interface,
|
||||
FROM Template TO Constructor,
|
||||
FROM Method TO \`Constructor\`,
|
||||
FROM \`Template\` TO \`Template\`,
|
||||
FROM \`Template\` TO Function,
|
||||
FROM \`Template\` TO Method,
|
||||
FROM \`Template\` TO Class,
|
||||
FROM \`Template\` TO \`Struct\`,
|
||||
FROM \`Template\` TO \`TypeAlias\`,
|
||||
FROM \`Template\` TO \`Enum\`,
|
||||
FROM \`Template\` TO \`Macro\`,
|
||||
FROM \`Template\` TO Interface,
|
||||
FROM \`Template\` TO \`Constructor\`,
|
||||
FROM \`Module\` TO \`Module\`,
|
||||
FROM CodeElement TO Community,
|
||||
FROM Interface TO Community,
|
||||
|
|
@ -273,11 +273,11 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM Interface TO Method,
|
||||
FROM Interface TO Class,
|
||||
FROM Interface TO Interface,
|
||||
FROM Interface TO TypeAlias,
|
||||
FROM Interface TO \`TypeAlias\`,
|
||||
FROM Interface TO \`Struct\`,
|
||||
FROM Interface TO Constructor,
|
||||
FROM Interface TO \`Constructor\`,
|
||||
FROM \`Struct\` TO Community,
|
||||
FROM \`Struct\` TO Trait,
|
||||
FROM \`Struct\` TO \`Trait\`,
|
||||
FROM \`Struct\` TO Function,
|
||||
FROM \`Struct\` TO Method,
|
||||
FROM \`Enum\` TO Community,
|
||||
|
|
@ -286,56 +286,56 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM \`Macro\` TO Method,
|
||||
FROM \`Module\` TO Function,
|
||||
FROM \`Module\` TO Method,
|
||||
FROM Typedef TO Community,
|
||||
FROM \`Typedef\` TO Community,
|
||||
FROM \`Union\` TO Community,
|
||||
FROM Namespace TO Community,
|
||||
FROM Trait TO Community,
|
||||
FROM Impl TO Community,
|
||||
FROM Impl TO Trait,
|
||||
FROM TypeAlias TO Community,
|
||||
FROM \`Namespace\` TO Community,
|
||||
FROM \`Trait\` TO Community,
|
||||
FROM \`Impl\` TO Community,
|
||||
FROM \`Impl\` TO \`Trait\`,
|
||||
FROM \`TypeAlias\` TO Community,
|
||||
FROM \`Const\` TO Community,
|
||||
FROM Static TO Community,
|
||||
FROM Property TO Community,
|
||||
FROM Record TO Community,
|
||||
FROM Delegate TO Community,
|
||||
FROM Annotation TO Community,
|
||||
FROM Constructor TO Community,
|
||||
FROM Constructor TO Interface,
|
||||
FROM Constructor TO Class,
|
||||
FROM Constructor TO Method,
|
||||
FROM Constructor TO Function,
|
||||
FROM Constructor TO Constructor,
|
||||
FROM Constructor TO \`Struct\`,
|
||||
FROM Constructor TO \`Macro\`,
|
||||
FROM Constructor TO Template,
|
||||
FROM Constructor TO TypeAlias,
|
||||
FROM Constructor TO \`Enum\`,
|
||||
FROM Constructor TO Impl,
|
||||
FROM Constructor TO Namespace,
|
||||
FROM Template TO Community,
|
||||
FROM \`Static\` TO Community,
|
||||
FROM \`Property\` TO Community,
|
||||
FROM \`Record\` TO Community,
|
||||
FROM \`Delegate\` TO Community,
|
||||
FROM \`Annotation\` TO Community,
|
||||
FROM \`Constructor\` TO Community,
|
||||
FROM \`Constructor\` TO Interface,
|
||||
FROM \`Constructor\` TO Class,
|
||||
FROM \`Constructor\` TO Method,
|
||||
FROM \`Constructor\` TO Function,
|
||||
FROM \`Constructor\` TO \`Constructor\`,
|
||||
FROM \`Constructor\` TO \`Struct\`,
|
||||
FROM \`Constructor\` TO \`Macro\`,
|
||||
FROM \`Constructor\` TO \`Template\`,
|
||||
FROM \`Constructor\` TO \`TypeAlias\`,
|
||||
FROM \`Constructor\` TO \`Enum\`,
|
||||
FROM \`Constructor\` TO \`Impl\`,
|
||||
FROM \`Constructor\` TO \`Namespace\`,
|
||||
FROM \`Template\` TO Community,
|
||||
FROM \`Module\` TO Community,
|
||||
FROM Function TO Process,
|
||||
FROM Method TO Process,
|
||||
FROM Class TO Process,
|
||||
FROM Interface TO Process,
|
||||
FROM \`Struct\` TO Process,
|
||||
FROM Constructor TO Process,
|
||||
FROM \`Constructor\` TO Process,
|
||||
FROM \`Module\` TO Process,
|
||||
FROM \`Macro\` TO Process,
|
||||
FROM Impl TO Process,
|
||||
FROM Typedef TO Process,
|
||||
FROM TypeAlias TO Process,
|
||||
FROM \`Impl\` TO Process,
|
||||
FROM \`Typedef\` TO Process,
|
||||
FROM \`TypeAlias\` TO Process,
|
||||
FROM \`Enum\` TO Process,
|
||||
FROM \`Union\` TO Process,
|
||||
FROM Namespace TO Process,
|
||||
FROM Trait TO Process,
|
||||
FROM \`Namespace\` TO Process,
|
||||
FROM \`Trait\` TO Process,
|
||||
FROM \`Const\` TO Process,
|
||||
FROM Static TO Process,
|
||||
FROM Property TO Process,
|
||||
FROM Record TO Process,
|
||||
FROM Delegate TO Process,
|
||||
FROM Annotation TO Process,
|
||||
FROM Template TO Process,
|
||||
FROM \`Static\` TO Process,
|
||||
FROM \`Property\` TO Process,
|
||||
FROM \`Record\` TO Process,
|
||||
FROM \`Delegate\` TO Process,
|
||||
FROM \`Annotation\` TO Process,
|
||||
FROM \`Template\` TO Process,
|
||||
FROM CodeElement TO Process,
|
||||
type STRING,
|
||||
confidence DOUBLE,
|
||||
|
|
|
|||
|
|
@ -1,341 +0,0 @@
|
|||
/**
|
||||
* MCP Browser Client
|
||||
*
|
||||
* WebSocket client that connects to the gitnexus-mcp daemon.
|
||||
* - Sends codebase context (stats, hotspots, folder tree) on connect
|
||||
* - Receives tool calls from external AI agents and executes them
|
||||
* - Emits activity events for real-time monitoring
|
||||
*/
|
||||
|
||||
/**
|
||||
* Agent color mapping for multi-agent support
|
||||
*/
|
||||
export const AGENT_COLORS: Record<string, string> = {
|
||||
'cursor': '#a855f7', // Purple
|
||||
'claude': '#3b82f6', // Blue
|
||||
'claude-code': '#3b82f6', // Blue
|
||||
'windsurf': '#22c55e', // Green
|
||||
'codeium': '#22c55e', // Green (Windsurf)
|
||||
'unknown': '#6b7280', // Gray
|
||||
};
|
||||
|
||||
export function getAgentColor(agentName: string): string {
|
||||
const normalized = agentName.toLowerCase().trim();
|
||||
for (const [key, color] of Object.entries(AGENT_COLORS)) {
|
||||
if (normalized.includes(key)) {
|
||||
return color;
|
||||
}
|
||||
}
|
||||
return AGENT_COLORS.unknown;
|
||||
}
|
||||
|
||||
export interface ConnectedAgent {
|
||||
name: string;
|
||||
color: string;
|
||||
}
|
||||
|
||||
export interface MCPMessage {
|
||||
id: string;
|
||||
type?: 'context' | 'tool_call' | 'tool_result' | 'agent_info';
|
||||
method?: string;
|
||||
params?: Record<string, any>;
|
||||
result?: any;
|
||||
error?: { message: string };
|
||||
agentName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Codebase context to send to daemon
|
||||
*/
|
||||
export interface CodebaseContext {
|
||||
projectName: string;
|
||||
stats: {
|
||||
fileCount: number;
|
||||
functionCount: number;
|
||||
classCount: number;
|
||||
interfaceCount: number;
|
||||
methodCount: number;
|
||||
};
|
||||
hotspots: Array<{
|
||||
name: string;
|
||||
type: string;
|
||||
filePath: string;
|
||||
connections: number;
|
||||
}>;
|
||||
folderTree: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Activity event for real-time monitoring
|
||||
*/
|
||||
export interface ActivityEvent {
|
||||
id: string;
|
||||
tool: string;
|
||||
params: Record<string, any>;
|
||||
status: 'running' | 'complete' | 'error';
|
||||
result?: any;
|
||||
error?: string;
|
||||
timestamp: number;
|
||||
duration?: number;
|
||||
agentName?: string;
|
||||
agentColor?: string;
|
||||
}
|
||||
|
||||
type ToolHandler = (params: Record<string, any>) => Promise<any>;
|
||||
type ActivityListener = (event: ActivityEvent) => void;
|
||||
|
||||
export class MCPBrowserClient {
|
||||
private ws: WebSocket | null = null;
|
||||
private handlers: Map<string, ToolHandler> = new Map();
|
||||
private connectionListeners: Set<(connected: boolean) => void> = new Set();
|
||||
private activityListeners: Set<ActivityListener> = new Set();
|
||||
private activityLog: ActivityEvent[] = [];
|
||||
private pendingContext: CodebaseContext | null = null;
|
||||
private _connectedAgent: ConnectedAgent | null = null;
|
||||
|
||||
constructor(private port = 54319) {}
|
||||
|
||||
/**
|
||||
* Connect to the MCP daemon
|
||||
*/
|
||||
async connect(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
this.ws = new WebSocket(`ws://localhost:${this.port}`);
|
||||
|
||||
this.ws.onopen = () => {
|
||||
console.log('[MCP] Connected to daemon');
|
||||
this.notifyConnectionListeners(true);
|
||||
|
||||
// Send pending context if available
|
||||
if (this.pendingContext) {
|
||||
this.sendContext(this.pendingContext);
|
||||
}
|
||||
|
||||
resolve();
|
||||
};
|
||||
|
||||
this.ws.onerror = () => {
|
||||
this.notifyConnectionListeners(false);
|
||||
reject(new Error('Failed to connect to MCP bridge'));
|
||||
};
|
||||
|
||||
this.ws.onmessage = (event) => {
|
||||
try {
|
||||
const msg: MCPMessage = JSON.parse(event.data);
|
||||
this.handleMessage(msg);
|
||||
} catch (error) {
|
||||
console.error('[MCP] Failed to parse message:', error);
|
||||
}
|
||||
};
|
||||
|
||||
this.ws.onclose = () => {
|
||||
this.ws = null;
|
||||
this.notifyConnectionListeners(false);
|
||||
};
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send codebase context to daemon
|
||||
* Call this whenever context changes (new repo loaded, etc.)
|
||||
*/
|
||||
sendContext(context: CodebaseContext) {
|
||||
this.pendingContext = context;
|
||||
|
||||
if (this.ws?.readyState === WebSocket.OPEN) {
|
||||
const msg = {
|
||||
id: `ctx_${Date.now()}`,
|
||||
type: 'context',
|
||||
params: context,
|
||||
};
|
||||
this.ws.send(JSON.stringify(msg));
|
||||
console.log('[MCP] Sent context:', context.projectName);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle incoming messages from daemon
|
||||
*/
|
||||
private async handleMessage(msg: MCPMessage) {
|
||||
// Handle agent info updates
|
||||
if (msg.type === 'agent_info' && msg.agentName) {
|
||||
this._connectedAgent = {
|
||||
name: msg.agentName,
|
||||
color: getAgentColor(msg.agentName),
|
||||
};
|
||||
console.log('[MCP] Agent connected:', this._connectedAgent);
|
||||
return;
|
||||
}
|
||||
|
||||
// This is a tool call request from an external agent
|
||||
if (msg.method && msg.id) {
|
||||
const handler = this.handlers.get(msg.method);
|
||||
const startTime = Date.now();
|
||||
|
||||
// Get agent info from message or use connected agent
|
||||
const agentName = msg.agentName || this._connectedAgent?.name || 'Unknown';
|
||||
const agentColor = getAgentColor(agentName);
|
||||
|
||||
// Create activity event with agent info
|
||||
const activityEvent: ActivityEvent = {
|
||||
id: msg.id,
|
||||
tool: msg.method,
|
||||
params: msg.params || {},
|
||||
status: 'running',
|
||||
timestamp: startTime,
|
||||
agentName,
|
||||
agentColor,
|
||||
};
|
||||
this.logActivity(activityEvent);
|
||||
|
||||
if (handler) {
|
||||
try {
|
||||
const result = await handler(msg.params || {});
|
||||
this.send({ id: msg.id, result });
|
||||
|
||||
// Update activity with success
|
||||
this.updateActivity(msg.id, {
|
||||
status: 'complete',
|
||||
result,
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : 'Unknown error';
|
||||
this.send({ id: msg.id, error: { message } });
|
||||
|
||||
// Update activity with error
|
||||
this.updateActivity(msg.id, {
|
||||
status: 'error',
|
||||
error: message,
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
this.send({
|
||||
id: msg.id,
|
||||
error: { message: `Unknown tool: ${msg.method}` }
|
||||
});
|
||||
|
||||
// Update activity with error
|
||||
this.updateActivity(msg.id, {
|
||||
status: 'error',
|
||||
error: `Unknown tool: ${msg.method}`,
|
||||
duration: Date.now() - startTime,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an activity event
|
||||
*/
|
||||
private logActivity(event: ActivityEvent) {
|
||||
this.activityLog.push(event);
|
||||
// Keep max 100 events
|
||||
if (this.activityLog.length > 100) {
|
||||
this.activityLog.shift();
|
||||
}
|
||||
this.notifyActivityListeners(event);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update an existing activity event
|
||||
*/
|
||||
private updateActivity(id: string, updates: Partial<ActivityEvent>) {
|
||||
const event = this.activityLog.find(e => e.id === id);
|
||||
if (event) {
|
||||
Object.assign(event, updates);
|
||||
this.notifyActivityListeners(event);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a message to the daemon
|
||||
*/
|
||||
private send(msg: MCPMessage) {
|
||||
if (this.ws?.readyState === WebSocket.OPEN) {
|
||||
this.ws.send(JSON.stringify(msg));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a handler for a tool
|
||||
*/
|
||||
registerHandler(method: string, handler: ToolHandler) {
|
||||
this.handlers.set(method, handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen for connection state changes
|
||||
*/
|
||||
onConnectionChange(listener: (connected: boolean) => void) {
|
||||
this.connectionListeners.add(listener);
|
||||
return () => this.connectionListeners.delete(listener);
|
||||
}
|
||||
|
||||
private notifyConnectionListeners(connected: boolean) {
|
||||
this.connectionListeners.forEach(listener => listener(connected));
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen for activity events
|
||||
*/
|
||||
onActivity(listener: ActivityListener) {
|
||||
this.activityListeners.add(listener);
|
||||
return () => this.activityListeners.delete(listener);
|
||||
}
|
||||
|
||||
private notifyActivityListeners(event: ActivityEvent) {
|
||||
this.activityListeners.forEach(listener => listener(event));
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the activity log
|
||||
*/
|
||||
getActivityLog(): ActivityEvent[] {
|
||||
return [...this.activityLog];
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear the activity log
|
||||
*/
|
||||
clearActivityLog() {
|
||||
this.activityLog = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if connected
|
||||
*/
|
||||
get isConnected(): boolean {
|
||||
return this.ws?.readyState === WebSocket.OPEN;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the connected agent info
|
||||
*/
|
||||
get connectedAgent(): ConnectedAgent | null {
|
||||
return this._connectedAgent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnect from daemon
|
||||
*/
|
||||
disconnect() {
|
||||
this.ws?.close();
|
||||
this.ws = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Singleton instance
|
||||
let mcpClientInstance: MCPBrowserClient | null = null;
|
||||
|
||||
export function getMCPClient(): MCPBrowserClient {
|
||||
if (!mcpClientInstance) {
|
||||
mcpClientInstance = new MCPBrowserClient();
|
||||
}
|
||||
return mcpClientInstance;
|
||||
}
|
||||
103
gitnexus/package-lock.json
generated
103
gitnexus/package-lock.json
generated
|
|
@ -7,7 +7,7 @@
|
|||
"": {
|
||||
"name": "gitnexus",
|
||||
"version": "0.1.0",
|
||||
"license": "MIT",
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.0.0",
|
||||
"@modelcontextprotocol/sdk": "^1.0.0",
|
||||
|
|
@ -16,23 +16,25 @@
|
|||
"express": "^4.19.2",
|
||||
"glob": "^11.0.0",
|
||||
"graphology": "^0.25.4",
|
||||
"graphology-communities-louvain": "^2.0.1",
|
||||
"graphology-indices": "^0.17.0",
|
||||
"graphology-utils": "^2.3.0",
|
||||
"kuzu": "^0.11.3",
|
||||
"lru-cache": "^11.0.0",
|
||||
"minisearch": "^7.2.0",
|
||||
"mnemonist": "^0.39.0",
|
||||
"ora": "^8.0.0",
|
||||
"pandemonium": "^2.4.0",
|
||||
"tree-sitter": "^0.21.0",
|
||||
"tree-sitter-c": "^0.21.0",
|
||||
"tree-sitter-c-sharp": "^0.21.0",
|
||||
"tree-sitter-cpp": "^0.22.0",
|
||||
"tree-sitter-go": "^0.21.0",
|
||||
"tree-sitter-java": "^0.20.0",
|
||||
"tree-sitter-java": "^0.21.0",
|
||||
"tree-sitter-javascript": "^0.21.0",
|
||||
"tree-sitter-python": "^0.21.0",
|
||||
"tree-sitter-rust": "^0.21.0",
|
||||
"tree-sitter-typescript": "^0.21.0",
|
||||
"uuid": "^13.0.0",
|
||||
"ws": "^8.16.0"
|
||||
"typescript": "^5.4.5",
|
||||
"uuid": "^13.0.0"
|
||||
},
|
||||
"bin": {
|
||||
"gitnexus": "dist/cli/index.js"
|
||||
|
|
@ -42,9 +44,7 @@
|
|||
"@types/express": "^4.17.21",
|
||||
"@types/node": "^20.0.0",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"@types/ws": "^8.5.10",
|
||||
"tsx": "^4.0.0",
|
||||
"typescript": "^5.4.5"
|
||||
"tsx": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
|
|
@ -1572,16 +1572,6 @@
|
|||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/ws": {
|
||||
"version": "8.18.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz",
|
||||
"integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/accepts": {
|
||||
"version": "1.3.8",
|
||||
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
|
||||
|
|
@ -2849,21 +2839,6 @@
|
|||
"graphology-types": ">=0.24.0"
|
||||
}
|
||||
},
|
||||
"node_modules/graphology-communities-louvain": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/graphology-communities-louvain/-/graphology-communities-louvain-2.0.2.tgz",
|
||||
"integrity": "sha512-zt+2hHVPYxjEquyecxWXoUoIuN/UvYzsvI7boDdMNz0rRvpESQ7+e+Ejv6wK7AThycbZXuQ6DkG8NPMCq6XwoA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"graphology-indices": "^0.17.0",
|
||||
"graphology-utils": "^2.4.4",
|
||||
"mnemonist": "^0.39.0",
|
||||
"pandemonium": "^2.4.1"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"graphology-types": ">=0.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/graphology-indices": {
|
||||
"version": "0.17.0",
|
||||
"resolved": "https://registry.npmjs.org/graphology-indices/-/graphology-indices-0.17.0.tgz",
|
||||
|
|
@ -3308,12 +3283,6 @@
|
|||
"node": ">=16 || 14 >=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/minisearch": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz",
|
||||
"integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/minizlib": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz",
|
||||
|
|
@ -3366,12 +3335,6 @@
|
|||
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/nan": {
|
||||
"version": "2.25.0",
|
||||
"resolved": "https://registry.npmjs.org/nan/-/nan-2.25.0.tgz",
|
||||
"integrity": "sha512-0M90Ag7Xn5KMLLZ7zliPWP3rT90P6PN+IzVFS0VqmnPktBk3700xUVv8Ikm9EUaUE5SDWdp/BIxdENzVznpm1g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/negotiator": {
|
||||
"version": "0.6.3",
|
||||
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz",
|
||||
|
|
@ -4500,13 +4463,31 @@
|
|||
}
|
||||
},
|
||||
"node_modules/tree-sitter-java": {
|
||||
"version": "0.20.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.20.2.tgz",
|
||||
"integrity": "sha512-jc6RCnM+JE2ns1AkpErOp2Dp1jOADPbljsrWup0Vj2qTmG8KGYMSTD7HcrVRyZUC6pRLFySPMOh8x7Dn12aynw==",
|
||||
"version": "0.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.21.0.tgz",
|
||||
"integrity": "sha512-CKJiTo1uc3SUsgEcaZgufGx8my6dzihy8JR/JsJH40Tj3uSe2/eFLk+0q+fpbosGAyY4YiXJtEoFB2O4bS2yOw==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"nan": "^2.14.1"
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"tree-sitter": "^0.21.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"tree_sitter": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-java/node_modules/node-addon-api": {
|
||||
"version": "8.5.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.5.0.tgz",
|
||||
"integrity": "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-javascript": {
|
||||
|
|
@ -4680,7 +4661,6 @@
|
|||
"version": "5.9.3",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
|
||||
"integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
|
|
@ -4919,27 +4899,6 @@
|
|||
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.19.0",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz",
|
||||
"integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/y18n": {
|
||||
"version": "5.0.8",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@
|
|||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"skills"
|
||||
"skills",
|
||||
"vendor"
|
||||
],
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
|
|
@ -25,17 +26,19 @@
|
|||
"express": "^4.19.2",
|
||||
"glob": "^11.0.0",
|
||||
"graphology": "^0.25.4",
|
||||
"graphology-communities-louvain": "^2.0.1",
|
||||
"graphology-indices": "^0.17.0",
|
||||
"graphology-utils": "^2.3.0",
|
||||
"kuzu": "^0.11.3",
|
||||
"lru-cache": "^11.0.0",
|
||||
"minisearch": "^7.2.0",
|
||||
"mnemonist": "^0.39.0",
|
||||
"ora": "^8.0.0",
|
||||
"pandemonium": "^2.4.0",
|
||||
"tree-sitter": "^0.21.0",
|
||||
"tree-sitter-c": "^0.21.0",
|
||||
"tree-sitter-c-sharp": "^0.21.0",
|
||||
"tree-sitter-cpp": "^0.22.0",
|
||||
"tree-sitter-go": "^0.21.0",
|
||||
"tree-sitter-java": "^0.20.0",
|
||||
"tree-sitter-java": "^0.21.0",
|
||||
"tree-sitter-javascript": "^0.21.0",
|
||||
"tree-sitter-python": "^0.21.0",
|
||||
"tree-sitter-rust": "^0.21.0",
|
||||
|
|
@ -53,4 +56,4 @@
|
|||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Trace bugs through call chains using knowledge graph
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. gitnexus_search({query}) → Find code related to error
|
||||
2. gitnexus_explore({name, type: "symbol"}) → Get callers and callees
|
||||
3. READ gitnexus://process/{name} → Trace execution flow
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Navigate unfamiliar code using GitNexus knowledge graph
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. READ gitnexus://context → Get codebase overview (~150 tokens)
|
||||
2. READ gitnexus://clusters → See all functional clusters
|
||||
3. READ gitnexus://cluster/{name} → Deep dive on specific cluster
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Analyze blast radius before making code changes
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. gitnexus_impact({target, direction: "upstream"}) → What depends on this
|
||||
2. READ gitnexus://clusters → Check affected areas
|
||||
3. READ gitnexus://processes → Affected execution flows
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ description: Plan safe refactors using blast radius and dependency mapping
|
|||
|
||||
## Quick Start
|
||||
```
|
||||
0. If "Index is stale" → gitnexus_analyze({})
|
||||
1. gitnexus_impact({target, direction: "upstream"}) → Map all dependents
|
||||
2. READ gitnexus://schema → Understand graph structure
|
||||
3. gitnexus_cypher → Find all references
|
||||
|
|
|
|||
|
|
@ -1,13 +0,0 @@
|
|||
/**
|
||||
* FTS Live Update Test File
|
||||
*
|
||||
* This file contains the unique identifier: GITNEXUS_FTS_LIVETEST_2026_ALPHA
|
||||
*
|
||||
* If you can find this file via MCP search after the watcher picks up the commit,
|
||||
* then the KuzuDB FTS live update system is working correctly!
|
||||
*/
|
||||
|
||||
export function ftsLiveTestFunction() {
|
||||
// GITNEXUS_FTS_LIVETEST_2026_ALPHA marker for search
|
||||
return "Live FTS test successful!";
|
||||
}
|
||||
|
|
@ -7,8 +7,7 @@
|
|||
import path from 'path';
|
||||
import ora from 'ora';
|
||||
import { runPipelineFromRepo } from '../core/ingestion/pipeline.js';
|
||||
import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu } from '../core/kuzu/kuzu-adapter.js';
|
||||
import { buildBM25Index, exportBM25Index } from '../core/search/bm25-index.js';
|
||||
import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex } from '../core/kuzu/kuzu-adapter.js';
|
||||
import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js';
|
||||
import { getStoragePaths, saveMeta, loadMeta, addToGitignore } from '../storage/repo-manager.js';
|
||||
import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js';
|
||||
|
|
@ -45,7 +44,7 @@ export const analyzeCommand = async (
|
|||
return;
|
||||
}
|
||||
|
||||
const { storagePath, kuzuPath, bm25Path } = getStoragePaths(repoPath);
|
||||
const { storagePath, kuzuPath } = getStoragePaths(repoPath);
|
||||
const currentCommit = getCurrentCommit(repoPath);
|
||||
const existingMeta = await loadMeta(storagePath);
|
||||
|
||||
|
|
@ -62,14 +61,33 @@ export const analyzeCommand = async (
|
|||
});
|
||||
|
||||
// Load graph into KuzuDB
|
||||
// Always start fresh - remove existing kuzu DB to avoid stale/corrupt data
|
||||
spinner.text = 'Loading graph into KuzuDB...';
|
||||
await closeKuzu();
|
||||
|
||||
// Kuzu 0.11 stores databases as: <name> (main file) + <name>.wal (WAL file)
|
||||
// BOTH must be deleted or kuzu will find the orphaned WAL and corrupt the database
|
||||
const fsClean = await import('fs/promises');
|
||||
const kuzuFiles = [kuzuPath, `${kuzuPath}.wal`, `${kuzuPath}.lock`];
|
||||
for (const f of kuzuFiles) {
|
||||
try { await fsClean.rm(f, { recursive: true, force: true }); } catch { /* may not exist */ }
|
||||
}
|
||||
|
||||
await initKuzu(kuzuPath);
|
||||
await loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath);
|
||||
|
||||
// Build BM25 search index
|
||||
spinner.text = 'Building BM25 index...';
|
||||
buildBM25Index(pipelineResult.fileContents);
|
||||
await exportBM25Index(bm25Path);
|
||||
// Create FTS indexes for keyword search
|
||||
// Indexes searchable content on: File, Function, Class, Method
|
||||
spinner.text = 'Creating FTS indexes...';
|
||||
try {
|
||||
await createFTSIndex('File', 'file_fts', ['name', 'content']);
|
||||
await createFTSIndex('Function', 'function_fts', ['name', 'content']);
|
||||
await createFTSIndex('Class', 'class_fts', ['name', 'content']);
|
||||
await createFTSIndex('Method', 'method_fts', ['name', 'content']);
|
||||
} catch (e: any) {
|
||||
// FTS index creation may fail if tables are empty (no data for that type)
|
||||
console.error('Note: Some FTS indexes may not have been created:', e.message);
|
||||
}
|
||||
|
||||
// Generate embeddings
|
||||
if (!options?.skipEmbeddings) {
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ import { listCommand } from './list.js';
|
|||
import { statusCommand } from './status.js';
|
||||
import { mcpCommand } from './mcp.js';
|
||||
import { cleanCommand } from './clean.js';
|
||||
|
||||
const program = new Command();
|
||||
|
||||
program
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ export const mcpCommand = async () => {
|
|||
const local = new LocalBackend();
|
||||
await local.init(candidate);
|
||||
console.error(`GitNexus: Found index at ${repo.storagePath}`);
|
||||
|
||||
// Start MCP server (no watcher daemon - staleness hints guide LLM to call analyze)
|
||||
await startMCPServer(local);
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,37 @@ export const createKnowledgeGraph = (): KnowledgeGraph => {
|
|||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Remove a single node and all relationships involving it
|
||||
*/
|
||||
const removeNode = (nodeId: string): boolean => {
|
||||
if (!nodeMap.has(nodeId)) return false;
|
||||
|
||||
nodeMap.delete(nodeId);
|
||||
|
||||
// Remove all relationships involving this node
|
||||
for (const [relId, rel] of relationshipMap) {
|
||||
if (rel.sourceId === nodeId || rel.targetId === nodeId) {
|
||||
relationshipMap.delete(relId);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Remove all nodes (and their relationships) belonging to a file
|
||||
*/
|
||||
const removeNodesByFile = (filePath: string): number => {
|
||||
let removed = 0;
|
||||
for (const [nodeId, node] of nodeMap) {
|
||||
if (node.properties?.filePath === filePath) {
|
||||
removeNode(nodeId);
|
||||
removed++;
|
||||
}
|
||||
}
|
||||
return removed;
|
||||
};
|
||||
|
||||
return{
|
||||
get nodes(){
|
||||
return Array.from(nodeMap.values())
|
||||
|
|
@ -36,6 +67,8 @@ export const createKnowledgeGraph = (): KnowledgeGraph => {
|
|||
|
||||
addNode,
|
||||
addRelationship,
|
||||
removeNode,
|
||||
removeNodesByFile,
|
||||
|
||||
};
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -83,4 +83,6 @@ export interface KnowledgeGraph {
|
|||
relationshipCount: number,
|
||||
addNode: (node: GraphNode) => void,
|
||||
addRelationship: (relationship: GraphRelationship) => void,
|
||||
removeNode: (nodeId: string) => boolean,
|
||||
removeNodesByFile: (filePath: string) => number,
|
||||
}
|
||||
|
|
@ -6,7 +6,7 @@ import Parser from 'tree-sitter';
|
|||
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import { getLanguageFromFilename } from './utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
|
||||
/**
|
||||
* Node types that represent function/method definitions across languages.
|
||||
|
|
@ -140,6 +140,7 @@ export const processCalls = async (
|
|||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
onProgress?.(i + 1, files.length);
|
||||
if (i % 20 === 0) await yieldToEventLoop();
|
||||
|
||||
// 1. Check language support first
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
|
|
|
|||
|
|
@ -1,19 +1,29 @@
|
|||
/**
|
||||
* Community Detection Processor
|
||||
*
|
||||
* Uses the Leiden algorithm (via graphology-communities-louvain) to detect
|
||||
* Uses the Leiden algorithm (via graphology-communities-leiden) to detect
|
||||
* communities/clusters in the code graph based on CALLS relationships.
|
||||
*
|
||||
* Communities represent groups of code that work together frequently,
|
||||
* helping agents navigate the codebase by functional area rather than file structure.
|
||||
*/
|
||||
|
||||
// NOTE: graphology + louvain typings are a bit inconsistent across versions.
|
||||
// Keep these as `any` to avoid blocking the CLI build.
|
||||
// NOTE: The Leiden algorithm source is vendored from graphology's repo
|
||||
// (src/communities-leiden) because it was never published to npm.
|
||||
// We use createRequire to load the CommonJS vendored files in ESM context.
|
||||
import Graph from 'graphology';
|
||||
import louvain from 'graphology-communities-louvain';
|
||||
import { createRequire } from 'node:module';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { KnowledgeGraph, NodeLabel } from '../graph/types.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
// Navigate to package root (works from both src/ and dist/)
|
||||
const leidenPath = resolve(__dirname, '..', '..', '..', 'vendor', 'leiden', 'index.cjs');
|
||||
const _require = createRequire(import.meta.url);
|
||||
const leiden = _require(leidenPath);
|
||||
|
||||
// ============================================================================
|
||||
// TYPES
|
||||
// ============================================================================
|
||||
|
|
@ -95,8 +105,8 @@ export const processCommunities = async (
|
|||
|
||||
onProgress?.(`Running Leiden algorithm on ${graph.order} nodes...`, 30);
|
||||
|
||||
// Step 2: Run Leiden (via Louvain implementation with refinement)
|
||||
const details = (louvain as any).detailed(graph, {
|
||||
// Step 2: Run Leiden algorithm for community detection
|
||||
const details = (leiden as any).detailed(graph, {
|
||||
resolution: 1.0, // Default resolution, can be tuned
|
||||
randomWalk: true,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ import Parser from 'tree-sitter';
|
|||
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import { getLanguageFromFilename } from './utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
|
||||
export const processHeritage = async (
|
||||
graph: KnowledgeGraph,
|
||||
|
|
@ -27,6 +27,7 @@ export const processHeritage = async (
|
|||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
onProgress?.(i + 1, files.length);
|
||||
if (i % 20 === 0) await yieldToEventLoop();
|
||||
|
||||
// 1. Check language support
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import Parser from 'tree-sitter';
|
|||
import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js';
|
||||
import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
||||
import { generateId } from '../../lib/utils.js';
|
||||
import { getLanguageFromFilename } from './utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
|
||||
const isDev = process.env.NODE_ENV !== 'production';
|
||||
|
||||
|
|
@ -127,6 +127,7 @@ export const processImports = async (
|
|||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
onProgress?.(i + 1, files.length);
|
||||
if (i % 20 === 0) await yieldToEventLoop();
|
||||
|
||||
// 1. Check language support first
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { LANGUAGE_QUERIES } from './tree-sitter-queries.js';
|
|||
import { generateId } from '../../lib/utils.js';
|
||||
import { SymbolTable } from './symbol-table.js';
|
||||
import { ASTCache } from './ast-cache.js';
|
||||
import { getLanguageFromFilename } from './utils.js';
|
||||
import { getLanguageFromFilename, yieldToEventLoop } from './utils.js';
|
||||
|
||||
export type FileProgressCallback = (current: number, total: number, filePath: string) => void;
|
||||
|
||||
|
|
@ -128,6 +128,9 @@ export const processParsing = async (
|
|||
// Report progress for each file
|
||||
onFileProgress?.(i + 1, total, file.path);
|
||||
|
||||
// Yield to event loop periodically so spinner can update
|
||||
if (i % 20 === 0) await yieldToEventLoop();
|
||||
|
||||
const language = getLanguageFromFilename(file.path);
|
||||
|
||||
if (!language) continue;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,11 @@
|
|||
import { SupportedLanguages } from '../../config/supported-languages.js';
|
||||
|
||||
/**
|
||||
* Yield control to the event loop so spinners/progress can render.
|
||||
* Call periodically in hot loops to prevent UI freezes.
|
||||
*/
|
||||
export const yieldToEventLoop = (): Promise<void> => new Promise(resolve => setImmediate(resolve));
|
||||
|
||||
/**
|
||||
* Map file extension to SupportedLanguage enum
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -18,12 +18,17 @@ import { NODE_TABLES, NodeTableName } from './schema.js';
|
|||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Sanitize string to ensure valid UTF-8
|
||||
* Removes or replaces invalid characters that would break CSV parsing
|
||||
* Sanitize string to ensure valid UTF-8 and safe CSV content for KuzuDB
|
||||
* Removes or replaces invalid characters that would break CSV parsing.
|
||||
*
|
||||
* Critical: KuzuDB's native CSV parser on Windows can misinterpret \r\n
|
||||
* inside quoted fields. We normalize all line endings to \n only.
|
||||
*/
|
||||
const sanitizeUTF8 = (str: string): string => {
|
||||
return str
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n \r
|
||||
.replace(/\r\n/g, '\n') // Normalize Windows line endings first
|
||||
.replace(/\r/g, '\n') // Normalize remaining \r to \n
|
||||
.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n
|
||||
.replace(/[\uD800-\uDFFF]/g, '') // Remove surrogate pairs (invalid standalone)
|
||||
.replace(/[\uFFFE\uFFFF]/g, ''); // Remove BOM and special chars
|
||||
};
|
||||
|
|
@ -175,14 +180,14 @@ const generateFolderCSV = (nodes: GraphNode[]): string => {
|
|||
|
||||
/**
|
||||
* Generate CSV for code element nodes (Function, Class, Interface, Method, CodeElement)
|
||||
* Headers: id,name,filePath,startLine,endLine,content
|
||||
* Headers: id,name,filePath,startLine,endLine,isExported,content
|
||||
*/
|
||||
const generateCodeElementCSV = (
|
||||
nodes: GraphNode[],
|
||||
label: NodeLabel,
|
||||
fileContents: Map<string, string>
|
||||
): string => {
|
||||
const headers = ['id', 'name', 'filePath', 'startLine', 'endLine', 'content'];
|
||||
const headers = ['id', 'name', 'filePath', 'startLine', 'endLine', 'isExported', 'content'];
|
||||
const rows: string[] = [headers.join(',')];
|
||||
|
||||
for (const node of nodes) {
|
||||
|
|
@ -194,6 +199,7 @@ const generateCodeElementCSV = (
|
|||
escapeCSVField(node.properties.filePath || ''),
|
||||
escapeCSVNumber(node.properties.startLine, -1),
|
||||
escapeCSVNumber(node.properties.endLine, -1),
|
||||
node.properties.isExported ? 'true' : 'false',
|
||||
escapeCSVField(content),
|
||||
].join(','));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,21 +19,24 @@ const normalizeCopyPath = (filePath: string): string => filePath.replace(/\\/g,
|
|||
export const initKuzu = async (dbPath: string) => {
|
||||
if (conn) return { db, conn };
|
||||
|
||||
// kuzu v0.11 expects the database path to NOT exist (it will create it)
|
||||
// or to be an existing valid kuzu database
|
||||
// If an empty directory exists from a previous clean, remove it
|
||||
// kuzu v0.11 stores the database as a single file (not a directory).
|
||||
// If the path already exists, it must be a valid kuzu database file.
|
||||
// Remove stale empty directories or files from older versions.
|
||||
try {
|
||||
const stat = await fs.stat(dbPath);
|
||||
if (stat.isDirectory()) {
|
||||
// Check if it's an empty directory
|
||||
// Old-style directory database or empty leftover - remove it
|
||||
const files = await fs.readdir(dbPath);
|
||||
if (files.length === 0) {
|
||||
// Empty directory - remove it so kuzu can create fresh
|
||||
await fs.rmdir(dbPath);
|
||||
} else {
|
||||
// Non-empty directory from older kuzu version - remove entire directory
|
||||
await fs.rm(dbPath, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
// If it's a file, assume it's an existing kuzu database - kuzu will open it
|
||||
} catch {
|
||||
// Path doesn't exist, which is what kuzu v0.11 wants for a new database
|
||||
// Path doesn't exist, which is what kuzu wants for a new database
|
||||
}
|
||||
|
||||
// Ensure parent directory exists
|
||||
|
|
@ -46,8 +49,12 @@ export const initKuzu = async (dbPath: string) => {
|
|||
for (const schemaQuery of SCHEMA_QUERIES) {
|
||||
try {
|
||||
await conn.query(schemaQuery);
|
||||
} catch {
|
||||
// Schema may already exist
|
||||
} catch (err) {
|
||||
// Only ignore "already exists" errors - log everything else
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (!msg.includes('already exists')) {
|
||||
console.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -78,10 +85,50 @@ export const loadGraphToKuzu = async (
|
|||
const relLines = csvData.relCSV.split('\n').slice(1).filter(line => line.trim());
|
||||
|
||||
for (const { table, path: filePath } of nodeFiles) {
|
||||
const copyQuery = getCopyQuery(table, normalizeCopyPath(filePath));
|
||||
await conn.query(copyQuery);
|
||||
const normalizedPath = normalizeCopyPath(filePath);
|
||||
const copyQuery = getCopyQuery(table, normalizedPath);
|
||||
|
||||
// Log CSV stats for diagnostics
|
||||
const csvContent = await fs.readFile(filePath, 'utf-8');
|
||||
const csvLines = csvContent.split('\n').length;
|
||||
console.log(` COPY ${table}: ${csvLines - 1} rows`);
|
||||
|
||||
try {
|
||||
await conn.query(copyQuery);
|
||||
} catch (err) {
|
||||
const errMsg = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`⚠️ COPY failed for ${table} (${csvLines - 1} rows): ${errMsg}`);
|
||||
|
||||
// Retry with IGNORE_ERRORS=true to skip malformed rows
|
||||
console.log(` Retrying ${table} with IGNORE_ERRORS=true...`);
|
||||
try {
|
||||
const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)');
|
||||
await conn.query(retryQuery);
|
||||
console.log(` ✅ ${table} loaded with IGNORE_ERRORS (some rows may have been skipped)`);
|
||||
} catch (retryErr) {
|
||||
const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
|
||||
console.error(`❌ COPY failed for ${table} even with IGNORE_ERRORS: ${retryMsg}`);
|
||||
throw retryErr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log('✅ All COPY commands succeeded. Starting relationship insertion...');
|
||||
|
||||
// Build a set of valid table names for fast lookup
|
||||
const validTables = new Set<string>(NODE_TABLES as readonly string[]);
|
||||
|
||||
const getNodeLabel = (nodeId: string): string => {
|
||||
if (nodeId.startsWith('comm_')) return 'Community';
|
||||
if (nodeId.startsWith('proc_')) return 'Process';
|
||||
return nodeId.split(':')[0];
|
||||
};
|
||||
|
||||
// All multi-language tables are created with backticks - must always reference them with backticks
|
||||
const escapeLabel = (label: string): string => {
|
||||
return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label;
|
||||
};
|
||||
|
||||
let insertedRels = 0;
|
||||
let skippedRels = 0;
|
||||
for (const line of relLines) {
|
||||
|
|
@ -89,26 +136,24 @@ export const loadGraphToKuzu = async (
|
|||
const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/);
|
||||
if (!match) continue;
|
||||
const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match;
|
||||
|
||||
const fromLabel = getNodeLabel(fromId);
|
||||
const toLabel = getNodeLabel(toId);
|
||||
|
||||
// Skip relationships where either node's label doesn't have a table in KuzuDB
|
||||
// (e.g. Variable, Import, Type nodes that aren't in the schema)
|
||||
// Querying a non-existent table causes a fatal native crash
|
||||
if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
|
||||
skippedRels++;
|
||||
continue;
|
||||
}
|
||||
|
||||
const confidence = parseFloat(confidenceStr) || 1.0;
|
||||
const step = parseInt(stepStr) || 0;
|
||||
|
||||
const getNodeLabel = (nodeId: string): string => {
|
||||
if (nodeId.startsWith('comm_')) return 'Community';
|
||||
if (nodeId.startsWith('proc_')) return 'Process';
|
||||
return nodeId.split(':')[0];
|
||||
};
|
||||
|
||||
const RESERVED_LABELS = ['Macro', 'Enum', 'Union', 'Const', 'Module', 'Struct'];
|
||||
const escapeLabel = (label: string): string => {
|
||||
return RESERVED_LABELS.includes(label) ? `\`${label}\`` : label;
|
||||
};
|
||||
|
||||
const fromLabel = escapeLabel(getNodeLabel(fromId));
|
||||
const toLabel = escapeLabel(getNodeLabel(toId));
|
||||
|
||||
const insertQuery = `
|
||||
MATCH (a:${fromLabel} {id: '${fromId.replace(/'/g, "''")}' }),
|
||||
(b:${toLabel} {id: '${toId.replace(/'/g, "''")}' })
|
||||
MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}' }),
|
||||
(b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}' })
|
||||
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b)
|
||||
`;
|
||||
await conn.query(insertQuery);
|
||||
|
|
@ -137,20 +182,157 @@ export const loadGraphToKuzu = async (
|
|||
return { success: true, insertedRels, skippedRels };
|
||||
};
|
||||
|
||||
// KuzuDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes).
|
||||
// Source code content is full of backslashes which confuse the auto-detection.
|
||||
// We MUST explicitly set ESCAPE='"' to use RFC 4180 escaping, and disable auto_detect to prevent
|
||||
// KuzuDB from overriding our settings based on sample rows.
|
||||
const COPY_CSV_OPTS = `(HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
|
||||
|
||||
// Multi-language table names that were created with backticks in CODE_ELEMENT_BASE
|
||||
// and must always be referenced with backticks in queries
|
||||
const BACKTICK_TABLES = new Set([
|
||||
'Struct', 'Enum', 'Macro', 'Typedef', 'Union', 'Namespace', 'Trait', 'Impl',
|
||||
'TypeAlias', 'Const', 'Static', 'Property', 'Record', 'Delegate', 'Annotation',
|
||||
'Constructor', 'Template', 'Module',
|
||||
]);
|
||||
|
||||
const escapeTableName = (table: string): string => {
|
||||
return BACKTICK_TABLES.has(table) ? `\`${table}\`` : table;
|
||||
};
|
||||
|
||||
const getCopyQuery = (table: NodeTableName, filePath: string): string => {
|
||||
const t = escapeTableName(table);
|
||||
if (table === 'File') {
|
||||
return `COPY File(id, name, filePath, content) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, name, filePath, content) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Folder') {
|
||||
return `COPY Folder(id, name, filePath) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, name, filePath) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Community') {
|
||||
return `COPY Community(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Process') {
|
||||
return `COPY Process(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
|
||||
return `COPY ${t}(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
return `COPY ${table}(id, name, filePath, startLine, endLine, content) FROM "${filePath}" (HEADER=true, PARALLEL=false)`;
|
||||
// Code element tables (Function, Class, Interface, Method, CodeElement, and multi-language)
|
||||
return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Insert a single node to KuzuDB
|
||||
* @param label - Node type (File, Function, Class, etc.)
|
||||
* @param properties - Node properties
|
||||
* @param dbPath - Path to KuzuDB database (optional if already initialized)
|
||||
*/
|
||||
export const insertNodeToKuzu = async (
|
||||
label: string,
|
||||
properties: Record<string, any>,
|
||||
dbPath?: string
|
||||
): Promise<boolean> => {
|
||||
// Use provided dbPath or fall back to module-level db
|
||||
const targetDbPath = dbPath || (db ? undefined : null);
|
||||
if (!targetDbPath && !db) {
|
||||
throw new Error('KuzuDB not initialized. Provide dbPath or call initKuzu first.');
|
||||
}
|
||||
|
||||
try {
|
||||
const escapeValue = (v: any): string => {
|
||||
if (v === null || v === undefined) return 'NULL';
|
||||
if (typeof v === 'number') return String(v);
|
||||
// Escape backslashes first (for Windows paths), then single quotes
|
||||
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`;
|
||||
};
|
||||
|
||||
// Build INSERT query based on node type
|
||||
let query: string;
|
||||
|
||||
if (label === 'File') {
|
||||
query = `CREATE (n:File {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, content: ${escapeValue(properties.content || '')}})`;
|
||||
} else if (label === 'Folder') {
|
||||
query = `CREATE (n:Folder {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}})`;
|
||||
} else {
|
||||
// Function, Class, Method, Interface, etc. - standard code element schema
|
||||
query = `CREATE (n:${label} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, content: ${escapeValue(properties.content || '')}})`;
|
||||
}
|
||||
|
||||
// Use per-query connection if dbPath provided (avoids lock conflicts)
|
||||
if (targetDbPath) {
|
||||
const tempDb = new kuzu.Database(targetDbPath);
|
||||
const tempConn = new kuzu.Connection(tempDb);
|
||||
try {
|
||||
await tempConn.query(query);
|
||||
return true;
|
||||
} finally {
|
||||
try { await tempConn.close(); } catch {}
|
||||
try { await tempDb.close(); } catch {}
|
||||
}
|
||||
} else if (conn) {
|
||||
// Use existing persistent connection (when called from analyze)
|
||||
await conn.query(query);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (e: any) {
|
||||
// Node may already exist or other error
|
||||
console.error(`Failed to insert ${label} node:`, e.message);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Batch insert multiple nodes to KuzuDB using a single connection
|
||||
* @param nodes - Array of {label, properties} to insert
|
||||
* @param dbPath - Path to KuzuDB database
|
||||
* @returns Object with success count and error count
|
||||
*/
|
||||
export const batchInsertNodesToKuzu = async (
|
||||
nodes: Array<{ label: string; properties: Record<string, any> }>,
|
||||
dbPath: string
|
||||
): Promise<{ inserted: number; failed: number }> => {
|
||||
if (nodes.length === 0) return { inserted: 0, failed: 0 };
|
||||
|
||||
const escapeValue = (v: any): string => {
|
||||
if (v === null || v === undefined) return 'NULL';
|
||||
if (typeof v === 'number') return String(v);
|
||||
// Escape backslashes first (for Windows paths), then single quotes
|
||||
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`;
|
||||
};
|
||||
|
||||
// Open a single connection for all inserts
|
||||
const tempDb = new kuzu.Database(dbPath);
|
||||
const tempConn = new kuzu.Connection(tempDb);
|
||||
|
||||
let inserted = 0;
|
||||
let failed = 0;
|
||||
|
||||
try {
|
||||
for (const { label, properties } of nodes) {
|
||||
try {
|
||||
let query: string;
|
||||
|
||||
// Use MERGE instead of CREATE for upsert behavior (handles duplicates gracefully)
|
||||
if (label === 'File') {
|
||||
query = `MERGE (n:File {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.content = ${escapeValue(properties.content || '')}`;
|
||||
} else if (label === 'Folder') {
|
||||
query = `MERGE (n:Folder {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}`;
|
||||
} else {
|
||||
query = `MERGE (n:${label} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.content = ${escapeValue(properties.content || '')}`;
|
||||
}
|
||||
|
||||
await tempConn.query(query);
|
||||
inserted++;
|
||||
} catch (e: any) {
|
||||
// Don't console.error here - it corrupts MCP JSON-RPC on stderr
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { await tempConn.close(); } catch {}
|
||||
try { await tempDb.close(); } catch {}
|
||||
}
|
||||
|
||||
return { inserted, failed };
|
||||
};
|
||||
|
||||
export const executeQuery = async (cypher: string): Promise<any[]> => {
|
||||
|
|
@ -244,7 +426,199 @@ export const closeKuzu = async (): Promise<void> => {
|
|||
|
||||
export const isKuzuReady = (): boolean => conn !== null && db !== null;
|
||||
|
||||
/**
|
||||
* Delete all nodes (and their relationships) for a specific file from KuzuDB
|
||||
* @param filePath - The file path to delete nodes for
|
||||
* @param dbPath - Optional path to KuzuDB for per-query connection
|
||||
* @returns Object with counts of deleted nodes
|
||||
*/
|
||||
export const deleteNodesForFile = async (filePath: string, dbPath?: string): Promise<{ deletedNodes: number }> => {
|
||||
const usePerQuery = !!dbPath;
|
||||
|
||||
// Set up connection (either use existing or create per-query)
|
||||
let tempDb: kuzu.Database | null = null;
|
||||
let tempConn: kuzu.Connection | null = null;
|
||||
let targetConn: kuzu.Connection | null = conn;
|
||||
|
||||
if (usePerQuery) {
|
||||
tempDb = new kuzu.Database(dbPath);
|
||||
tempConn = new kuzu.Connection(tempDb);
|
||||
targetConn = tempConn;
|
||||
} else if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Provide dbPath or call initKuzu first.');
|
||||
}
|
||||
|
||||
try {
|
||||
let deletedNodes = 0;
|
||||
const escapedPath = filePath.replace(/'/g, "''");
|
||||
|
||||
// Delete nodes from each table that has filePath
|
||||
// DETACH DELETE removes the node and all its relationships
|
||||
for (const tableName of NODE_TABLES) {
|
||||
// Skip tables that don't have filePath (Community, Process)
|
||||
if (tableName === 'Community' || tableName === 'Process') continue;
|
||||
|
||||
try {
|
||||
// First count how many we'll delete
|
||||
const countResult = await targetConn!.query(
|
||||
`MATCH (n:${tableName}) WHERE n.filePath = '${escapedPath}' RETURN count(n) AS cnt`
|
||||
);
|
||||
const result = Array.isArray(countResult) ? countResult[0] : countResult;
|
||||
const rows = await result.getAll();
|
||||
const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0);
|
||||
|
||||
if (count > 0) {
|
||||
// Delete nodes (and implicitly their relationships via DETACH)
|
||||
await targetConn!.query(
|
||||
`MATCH (n:${tableName}) WHERE n.filePath = '${escapedPath}' DETACH DELETE n`
|
||||
);
|
||||
deletedNodes += count;
|
||||
}
|
||||
} catch (e) {
|
||||
// Some tables may not support this query, skip
|
||||
}
|
||||
}
|
||||
|
||||
// Also delete any embeddings for nodes in this file
|
||||
try {
|
||||
await targetConn!.query(
|
||||
`MATCH (e:${EMBEDDING_TABLE_NAME}) WHERE e.nodeId STARTS WITH '${escapedPath}' DELETE e`
|
||||
);
|
||||
} catch {
|
||||
// Embedding table may not exist or nodeId format may differ
|
||||
}
|
||||
|
||||
return { deletedNodes };
|
||||
} finally {
|
||||
// Close per-query connection if used
|
||||
if (tempConn) {
|
||||
try { await tempConn.close(); } catch {}
|
||||
}
|
||||
if (tempDb) {
|
||||
try { await tempDb.close(); } catch {}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const getEmbeddingTableName = (): string => EMBEDDING_TABLE_NAME;
|
||||
|
||||
// ============================================================================
|
||||
// Full-Text Search (FTS) Functions
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Load the FTS extension (required before using FTS functions)
|
||||
*/
|
||||
export const loadFTSExtension = async (): Promise<void> => {
|
||||
if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Call initKuzu first.');
|
||||
}
|
||||
try {
|
||||
await conn.query('INSTALL fts');
|
||||
await conn.query('LOAD EXTENSION fts');
|
||||
} catch {
|
||||
// Extension may already be loaded
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a full-text search index on a table
|
||||
* @param tableName - The node table name (e.g., 'File', 'CodeSymbol')
|
||||
* @param indexName - Name for the FTS index
|
||||
* @param properties - List of properties to index (e.g., ['name', 'code'])
|
||||
* @param stemmer - Stemming algorithm (default: 'porter')
|
||||
*/
|
||||
export const createFTSIndex = async (
|
||||
tableName: string,
|
||||
indexName: string,
|
||||
properties: string[],
|
||||
stemmer: string = 'porter'
|
||||
): Promise<void> => {
|
||||
if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Call initKuzu first.');
|
||||
}
|
||||
|
||||
await loadFTSExtension();
|
||||
|
||||
const propList = properties.map(p => `'${p}'`).join(', ');
|
||||
const query = `CALL CREATE_FTS_INDEX('${tableName}', '${indexName}', [${propList}], stemmer := '${stemmer}')`;
|
||||
|
||||
try {
|
||||
await conn.query(query);
|
||||
} catch (e: any) {
|
||||
// Index may already exist
|
||||
if (!e.message?.includes('already exists')) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Query a full-text search index
|
||||
* @param tableName - The node table name
|
||||
* @param indexName - FTS index name
|
||||
* @param query - Search query string
|
||||
* @param limit - Maximum results
|
||||
* @param conjunctive - If true, all terms must match (AND); if false, any term matches (OR)
|
||||
* @returns Array of { node properties, score }
|
||||
*/
|
||||
export const queryFTS = async (
|
||||
tableName: string,
|
||||
indexName: string,
|
||||
query: string,
|
||||
limit: number = 20,
|
||||
conjunctive: boolean = false
|
||||
): Promise<Array<{ nodeId: string; name: string; filePath: string; score: number; [key: string]: any }>> => {
|
||||
if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Call initKuzu first.');
|
||||
}
|
||||
|
||||
// Escape single quotes in query
|
||||
const escapedQuery = query.replace(/'/g, "''");
|
||||
|
||||
const cypher = `
|
||||
CALL QUERY_FTS_INDEX('${tableName}', '${indexName}', '${escapedQuery}', conjunctive := ${conjunctive})
|
||||
RETURN node, score
|
||||
ORDER BY score DESC
|
||||
LIMIT ${limit}
|
||||
`;
|
||||
|
||||
try {
|
||||
const queryResult = await conn.query(cypher);
|
||||
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
||||
const rows = await result.getAll();
|
||||
|
||||
return rows.map((row: any) => {
|
||||
const node = row.node || row[0] || {};
|
||||
const score = row.score ?? row[1] ?? 0;
|
||||
return {
|
||||
nodeId: node.nodeId || node.id || '',
|
||||
name: node.name || '',
|
||||
filePath: node.filePath || '',
|
||||
score: typeof score === 'number' ? score : parseFloat(score) || 0,
|
||||
...node,
|
||||
};
|
||||
});
|
||||
} catch (e: any) {
|
||||
// Return empty if index doesn't exist yet
|
||||
if (e.message?.includes('does not exist')) {
|
||||
return [];
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Drop an FTS index
|
||||
*/
|
||||
export const dropFTSIndex = async (tableName: string, indexName: string): Promise<void> => {
|
||||
if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Call initKuzu first.');
|
||||
}
|
||||
|
||||
try {
|
||||
await conn.query(`CALL DROP_FTS_INDEX('${tableName}', '${indexName}')`);
|
||||
} catch {
|
||||
// Index may not exist
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ CREATE NODE TABLE Function (
|
|||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
|
@ -73,6 +74,7 @@ CREATE NODE TABLE Class (
|
|||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
|
@ -84,6 +86,7 @@ CREATE NODE TABLE Interface (
|
|||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
|
@ -95,6 +98,7 @@ CREATE NODE TABLE Method (
|
|||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
|
@ -106,6 +110,7 @@ CREATE NODE TABLE CodeElement (
|
|||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
isExported BOOLEAN,
|
||||
content STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
|
@ -196,20 +201,20 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM File TO \`Struct\`,
|
||||
FROM File TO \`Enum\`,
|
||||
FROM File TO \`Macro\`,
|
||||
FROM File TO Typedef,
|
||||
FROM File TO \`Typedef\`,
|
||||
FROM File TO \`Union\`,
|
||||
FROM File TO Namespace,
|
||||
FROM File TO Trait,
|
||||
FROM File TO Impl,
|
||||
FROM File TO TypeAlias,
|
||||
FROM File TO \`Namespace\`,
|
||||
FROM File TO \`Trait\`,
|
||||
FROM File TO \`Impl\`,
|
||||
FROM File TO \`TypeAlias\`,
|
||||
FROM File TO \`Const\`,
|
||||
FROM File TO Static,
|
||||
FROM File TO Property,
|
||||
FROM File TO Record,
|
||||
FROM File TO Delegate,
|
||||
FROM File TO Annotation,
|
||||
FROM File TO Constructor,
|
||||
FROM File TO Template,
|
||||
FROM File TO \`Static\`,
|
||||
FROM File TO \`Property\`,
|
||||
FROM File TO \`Record\`,
|
||||
FROM File TO \`Delegate\`,
|
||||
FROM File TO \`Annotation\`,
|
||||
FROM File TO \`Constructor\`,
|
||||
FROM File TO \`Template\`,
|
||||
FROM File TO \`Module\`,
|
||||
FROM Folder TO Folder,
|
||||
FROM Folder TO File,
|
||||
|
|
@ -219,48 +224,48 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM Function TO Community,
|
||||
FROM Function TO \`Macro\`,
|
||||
FROM Function TO \`Struct\`,
|
||||
FROM Function TO Template,
|
||||
FROM Function TO \`Template\`,
|
||||
FROM Function TO \`Enum\`,
|
||||
FROM Function TO Namespace,
|
||||
FROM Function TO TypeAlias,
|
||||
FROM Function TO \`Namespace\`,
|
||||
FROM Function TO \`TypeAlias\`,
|
||||
FROM Function TO \`Module\`,
|
||||
FROM Function TO Impl,
|
||||
FROM Function TO \`Impl\`,
|
||||
FROM Function TO Interface,
|
||||
FROM Function TO Constructor,
|
||||
FROM Function TO \`Constructor\`,
|
||||
FROM Class TO Method,
|
||||
FROM Class TO Function,
|
||||
FROM Class TO Class,
|
||||
FROM Class TO Interface,
|
||||
FROM Class TO Community,
|
||||
FROM Class TO Template,
|
||||
FROM Class TO TypeAlias,
|
||||
FROM Class TO \`Template\`,
|
||||
FROM Class TO \`TypeAlias\`,
|
||||
FROM Class TO \`Struct\`,
|
||||
FROM Class TO \`Enum\`,
|
||||
FROM Class TO Constructor,
|
||||
FROM Class TO \`Constructor\`,
|
||||
FROM Method TO Function,
|
||||
FROM Method TO Method,
|
||||
FROM Method TO Class,
|
||||
FROM Method TO Community,
|
||||
FROM Method TO Template,
|
||||
FROM Method TO \`Template\`,
|
||||
FROM Method TO \`Struct\`,
|
||||
FROM Method TO TypeAlias,
|
||||
FROM Method TO \`TypeAlias\`,
|
||||
FROM Method TO \`Enum\`,
|
||||
FROM Method TO \`Macro\`,
|
||||
FROM Method TO Namespace,
|
||||
FROM Method TO \`Namespace\`,
|
||||
FROM Method TO \`Module\`,
|
||||
FROM Method TO Impl,
|
||||
FROM Method TO \`Impl\`,
|
||||
FROM Method TO Interface,
|
||||
FROM Method TO Constructor,
|
||||
FROM Template TO Template,
|
||||
FROM Template TO Function,
|
||||
FROM Template TO Method,
|
||||
FROM Template TO Class,
|
||||
FROM Template TO \`Struct\`,
|
||||
FROM Template TO TypeAlias,
|
||||
FROM Template TO \`Enum\`,
|
||||
FROM Template TO \`Macro\`,
|
||||
FROM Template TO Interface,
|
||||
FROM Template TO Constructor,
|
||||
FROM Method TO \`Constructor\`,
|
||||
FROM \`Template\` TO \`Template\`,
|
||||
FROM \`Template\` TO Function,
|
||||
FROM \`Template\` TO Method,
|
||||
FROM \`Template\` TO Class,
|
||||
FROM \`Template\` TO \`Struct\`,
|
||||
FROM \`Template\` TO \`TypeAlias\`,
|
||||
FROM \`Template\` TO \`Enum\`,
|
||||
FROM \`Template\` TO \`Macro\`,
|
||||
FROM \`Template\` TO Interface,
|
||||
FROM \`Template\` TO \`Constructor\`,
|
||||
FROM \`Module\` TO \`Module\`,
|
||||
FROM CodeElement TO Community,
|
||||
FROM Interface TO Community,
|
||||
|
|
@ -268,11 +273,11 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM Interface TO Method,
|
||||
FROM Interface TO Class,
|
||||
FROM Interface TO Interface,
|
||||
FROM Interface TO TypeAlias,
|
||||
FROM Interface TO \`TypeAlias\`,
|
||||
FROM Interface TO \`Struct\`,
|
||||
FROM Interface TO Constructor,
|
||||
FROM Interface TO \`Constructor\`,
|
||||
FROM \`Struct\` TO Community,
|
||||
FROM \`Struct\` TO Trait,
|
||||
FROM \`Struct\` TO \`Trait\`,
|
||||
FROM \`Struct\` TO Function,
|
||||
FROM \`Struct\` TO Method,
|
||||
FROM \`Enum\` TO Community,
|
||||
|
|
@ -281,56 +286,56 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
|
|||
FROM \`Macro\` TO Method,
|
||||
FROM \`Module\` TO Function,
|
||||
FROM \`Module\` TO Method,
|
||||
FROM Typedef TO Community,
|
||||
FROM \`Typedef\` TO Community,
|
||||
FROM \`Union\` TO Community,
|
||||
FROM Namespace TO Community,
|
||||
FROM Trait TO Community,
|
||||
FROM Impl TO Community,
|
||||
FROM Impl TO Trait,
|
||||
FROM TypeAlias TO Community,
|
||||
FROM \`Namespace\` TO Community,
|
||||
FROM \`Trait\` TO Community,
|
||||
FROM \`Impl\` TO Community,
|
||||
FROM \`Impl\` TO \`Trait\`,
|
||||
FROM \`TypeAlias\` TO Community,
|
||||
FROM \`Const\` TO Community,
|
||||
FROM Static TO Community,
|
||||
FROM Property TO Community,
|
||||
FROM Record TO Community,
|
||||
FROM Delegate TO Community,
|
||||
FROM Annotation TO Community,
|
||||
FROM Constructor TO Community,
|
||||
FROM Constructor TO Interface,
|
||||
FROM Constructor TO Class,
|
||||
FROM Constructor TO Method,
|
||||
FROM Constructor TO Function,
|
||||
FROM Constructor TO Constructor,
|
||||
FROM Constructor TO \`Struct\`,
|
||||
FROM Constructor TO \`Macro\`,
|
||||
FROM Constructor TO Template,
|
||||
FROM Constructor TO TypeAlias,
|
||||
FROM Constructor TO \`Enum\`,
|
||||
FROM Constructor TO Impl,
|
||||
FROM Constructor TO Namespace,
|
||||
FROM Template TO Community,
|
||||
FROM \`Static\` TO Community,
|
||||
FROM \`Property\` TO Community,
|
||||
FROM \`Record\` TO Community,
|
||||
FROM \`Delegate\` TO Community,
|
||||
FROM \`Annotation\` TO Community,
|
||||
FROM \`Constructor\` TO Community,
|
||||
FROM \`Constructor\` TO Interface,
|
||||
FROM \`Constructor\` TO Class,
|
||||
FROM \`Constructor\` TO Method,
|
||||
FROM \`Constructor\` TO Function,
|
||||
FROM \`Constructor\` TO \`Constructor\`,
|
||||
FROM \`Constructor\` TO \`Struct\`,
|
||||
FROM \`Constructor\` TO \`Macro\`,
|
||||
FROM \`Constructor\` TO \`Template\`,
|
||||
FROM \`Constructor\` TO \`TypeAlias\`,
|
||||
FROM \`Constructor\` TO \`Enum\`,
|
||||
FROM \`Constructor\` TO \`Impl\`,
|
||||
FROM \`Constructor\` TO \`Namespace\`,
|
||||
FROM \`Template\` TO Community,
|
||||
FROM \`Module\` TO Community,
|
||||
FROM Function TO Process,
|
||||
FROM Method TO Process,
|
||||
FROM Class TO Process,
|
||||
FROM Interface TO Process,
|
||||
FROM \`Struct\` TO Process,
|
||||
FROM Constructor TO Process,
|
||||
FROM \`Constructor\` TO Process,
|
||||
FROM \`Module\` TO Process,
|
||||
FROM \`Macro\` TO Process,
|
||||
FROM Impl TO Process,
|
||||
FROM Typedef TO Process,
|
||||
FROM TypeAlias TO Process,
|
||||
FROM \`Impl\` TO Process,
|
||||
FROM \`Typedef\` TO Process,
|
||||
FROM \`TypeAlias\` TO Process,
|
||||
FROM \`Enum\` TO Process,
|
||||
FROM \`Union\` TO Process,
|
||||
FROM Namespace TO Process,
|
||||
FROM Trait TO Process,
|
||||
FROM \`Namespace\` TO Process,
|
||||
FROM \`Trait\` TO Process,
|
||||
FROM \`Const\` TO Process,
|
||||
FROM Static TO Process,
|
||||
FROM Property TO Process,
|
||||
FROM Record TO Process,
|
||||
FROM Delegate TO Process,
|
||||
FROM Annotation TO Process,
|
||||
FROM Template TO Process,
|
||||
FROM \`Static\` TO Process,
|
||||
FROM \`Property\` TO Process,
|
||||
FROM \`Record\` TO Process,
|
||||
FROM \`Delegate\` TO Process,
|
||||
FROM \`Annotation\` TO Process,
|
||||
FROM \`Template\` TO Process,
|
||||
FROM CodeElement TO Process,
|
||||
type STRING,
|
||||
confidence DOUBLE,
|
||||
|
|
|
|||
|
|
@ -1,18 +1,11 @@
|
|||
/**
|
||||
* BM25 Full-Text Search Index
|
||||
* Full-Text Search via KuzuDB FTS
|
||||
*
|
||||
* Uses MiniSearch for fast keyword-based search with BM25 ranking.
|
||||
* Complements semantic search - BM25 finds exact terms, semantic finds concepts.
|
||||
* Uses KuzuDB's built-in full-text search indexes for keyword-based search.
|
||||
* Always reads from the database (no cached state to drift).
|
||||
*/
|
||||
|
||||
import MiniSearch from 'minisearch';
|
||||
import fs from 'fs/promises';
|
||||
|
||||
export interface BM25Document {
|
||||
id: string; // File path
|
||||
content: string; // File content
|
||||
name: string; // File name (boosted in search)
|
||||
}
|
||||
import { queryFTS } from '../kuzu/kuzu-adapter.js';
|
||||
|
||||
export interface BM25SearchResult {
|
||||
filePath: string;
|
||||
|
|
@ -21,183 +14,51 @@ export interface BM25SearchResult {
|
|||
}
|
||||
|
||||
/**
|
||||
* BM25 Index singleton
|
||||
* Stores the MiniSearch instance and provides search methods
|
||||
*/
|
||||
let searchIndex: MiniSearch<BM25Document> | null = null;
|
||||
let indexedDocCount = 0;
|
||||
|
||||
/**
|
||||
* Build the BM25 index from file contents
|
||||
* Should be called after ingestion completes
|
||||
* Search using KuzuDB's built-in FTS (always fresh, reads from disk)
|
||||
*
|
||||
* @param fileContents - Map of file path to content
|
||||
* @returns Number of documents indexed
|
||||
* Queries multiple node tables (File, Function, Class, Method) in parallel
|
||||
* and merges results by filePath, summing scores for the same file.
|
||||
*
|
||||
* @param query - Search query string
|
||||
* @param limit - Maximum results
|
||||
* @returns Ranked search results from FTS indexes
|
||||
*/
|
||||
export const buildBM25Index = (fileContents: Map<string, string>): number => {
|
||||
// Create new MiniSearch instance with BM25-like scoring
|
||||
searchIndex = new MiniSearch<BM25Document>({
|
||||
fields: ['content', 'name'], // Fields to index
|
||||
storeFields: ['id'], // Fields to return in results
|
||||
|
||||
// Tokenizer: split on non-alphanumeric, camelCase, snake_case
|
||||
tokenize: (text: string) => {
|
||||
// Split on whitespace and punctuation
|
||||
const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/);
|
||||
|
||||
// Also split camelCase: "getUserById" -> ["get", "user", "by", "id"]
|
||||
const expanded: string[] = [];
|
||||
for (const token of tokens) {
|
||||
if (token.length === 0) continue;
|
||||
|
||||
// Split camelCase
|
||||
const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' ');
|
||||
expanded.push(...camelParts);
|
||||
|
||||
// Also keep original token for exact matches
|
||||
if (camelParts.length > 1) {
|
||||
expanded.push(token);
|
||||
}
|
||||
export const searchFTSFromKuzu = async (query: string, limit: number = 20): Promise<BM25SearchResult[]> => {
|
||||
// Search multiple tables with searchable content
|
||||
const [fileResults, functionResults, classResults, methodResults] = await Promise.all([
|
||||
queryFTS('File', 'file_fts', query, limit, false).catch(() => []),
|
||||
queryFTS('Function', 'function_fts', query, limit, false).catch(() => []),
|
||||
queryFTS('Class', 'class_fts', query, limit, false).catch(() => []),
|
||||
queryFTS('Method', 'method_fts', query, limit, false).catch(() => []),
|
||||
]);
|
||||
|
||||
// Merge results by filePath, summing scores for same file
|
||||
const merged = new Map<string, { filePath: string; score: number }>();
|
||||
|
||||
const addResults = (results: any[]) => {
|
||||
for (const r of results) {
|
||||
const existing = merged.get(r.filePath);
|
||||
if (existing) {
|
||||
existing.score += r.score;
|
||||
} else {
|
||||
merged.set(r.filePath, { filePath: r.filePath, score: r.score });
|
||||
}
|
||||
|
||||
// Filter out very short tokens and common noise
|
||||
return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t));
|
||||
},
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Index all files
|
||||
const documents: BM25Document[] = [];
|
||||
addResults(fileResults);
|
||||
addResults(functionResults);
|
||||
addResults(classResults);
|
||||
addResults(methodResults);
|
||||
|
||||
for (const [filePath, content] of fileContents.entries()) {
|
||||
// Extract filename from path
|
||||
const name = filePath.split('/').pop() || filePath;
|
||||
|
||||
documents.push({
|
||||
id: filePath,
|
||||
content: content,
|
||||
name: name,
|
||||
});
|
||||
}
|
||||
// Sort by score descending and add rank
|
||||
const sorted = Array.from(merged.values())
|
||||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit);
|
||||
|
||||
// Batch add for efficiency
|
||||
searchIndex.addAll(documents);
|
||||
indexedDocCount = documents.length;
|
||||
|
||||
const isDev = process.env.NODE_ENV !== 'production';
|
||||
if (isDev) {
|
||||
console.log(`📚 BM25 index built: ${indexedDocCount} documents`);
|
||||
}
|
||||
|
||||
return indexedDocCount;
|
||||
};
|
||||
|
||||
/**
|
||||
* Search the BM25 index
|
||||
*
|
||||
* @param query - Search query (keywords)
|
||||
* @param limit - Maximum results to return
|
||||
* @returns Ranked search results with file paths and scores
|
||||
*/
|
||||
export const searchBM25 = (query: string, limit: number = 20): BM25SearchResult[] => {
|
||||
if (!searchIndex) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Search with fuzzy matching and prefix support
|
||||
const results = searchIndex.search(query, {
|
||||
fuzzy: 0.2,
|
||||
prefix: true,
|
||||
boost: { name: 2 }, // Boost file name matches
|
||||
});
|
||||
|
||||
// Limit results and add rank
|
||||
return results.slice(0, limit).map((r, index) => ({
|
||||
filePath: r.id,
|
||||
return sorted.map((r, index) => ({
|
||||
filePath: r.filePath,
|
||||
score: r.score,
|
||||
rank: index + 1,
|
||||
}));
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if the BM25 index is ready
|
||||
*/
|
||||
export const isBM25Ready = (): boolean => {
|
||||
return searchIndex !== null && indexedDocCount > 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get index statistics
|
||||
*/
|
||||
export const getBM25Stats = (): { documentCount: number; termCount: number } => {
|
||||
if (!searchIndex) {
|
||||
return { documentCount: 0, termCount: 0 };
|
||||
}
|
||||
|
||||
return {
|
||||
documentCount: indexedDocCount,
|
||||
termCount: searchIndex.termCount,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Clear the index (for cleanup or re-indexing)
|
||||
*/
|
||||
export const clearBM25Index = (): void => {
|
||||
searchIndex = null;
|
||||
indexedDocCount = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Export the BM25 index to disk
|
||||
*/
|
||||
export const exportBM25Index = async (filePath: string): Promise<void> => {
|
||||
if (!searchIndex) return;
|
||||
const json = JSON.stringify(searchIndex.toJSON());
|
||||
await fs.writeFile(filePath, json, 'utf-8');
|
||||
};
|
||||
|
||||
/**
|
||||
* Load a BM25 index from disk
|
||||
*/
|
||||
export const loadBM25Index = async (filePath: string): Promise<boolean> => {
|
||||
try {
|
||||
const json = await fs.readFile(filePath, 'utf-8');
|
||||
const data = JSON.parse(json);
|
||||
searchIndex = MiniSearch.loadJSON(data, {
|
||||
fields: ['content', 'name'],
|
||||
storeFields: ['id'],
|
||||
tokenize: (text: string) => {
|
||||
const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/);
|
||||
const expanded: string[] = [];
|
||||
for (const token of tokens) {
|
||||
if (token.length === 0) continue;
|
||||
const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' ');
|
||||
expanded.push(...camelParts);
|
||||
if (camelParts.length > 1) {
|
||||
expanded.push(token);
|
||||
}
|
||||
}
|
||||
return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t));
|
||||
},
|
||||
});
|
||||
indexedDocCount = searchIndex.documentCount;
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Common stop words to filter out (too common to be useful)
|
||||
*/
|
||||
const STOP_WORDS = new Set([
|
||||
// JavaScript/TypeScript keywords
|
||||
'const', 'let', 'var', 'function', 'return', 'if', 'else', 'for', 'while',
|
||||
'class', 'new', 'this', 'import', 'export', 'from', 'default', 'async', 'await',
|
||||
'try', 'catch', 'throw', 'typeof', 'instanceof', 'true', 'false', 'null', 'undefined',
|
||||
|
||||
// Common English stop words
|
||||
'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with',
|
||||
'to', 'of', 'it', 'be', 'as', 'by', 'that', 'for', 'are', 'was', 'were',
|
||||
]);
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
* production search systems.
|
||||
*/
|
||||
|
||||
import { searchBM25, isBM25Ready, type BM25SearchResult } from './bm25-index.js';
|
||||
import { searchFTSFromKuzu, type BM25SearchResult } from './bm25-index.js';
|
||||
import type { SemanticSearchResult } from '../embeddings/types.js';
|
||||
|
||||
/**
|
||||
|
|
@ -114,11 +114,11 @@ export const mergeWithRRF = (
|
|||
|
||||
/**
|
||||
* Check if hybrid search is available
|
||||
* Requires BM25 index to be built
|
||||
* Note: Semantic search is optional - hybrid works with just BM25 if embeddings aren't ready
|
||||
* KuzuDB FTS is always available once the database is initialized.
|
||||
* Semantic search is optional - hybrid works with just FTS if embeddings aren't ready.
|
||||
*/
|
||||
export const isHybridSearchReady = (): boolean => {
|
||||
return isBM25Ready();
|
||||
return true; // FTS is always available via KuzuDB when DB is open
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -146,6 +146,7 @@ export const formatHybridResults = (results: HybridSearchResult[]): string => {
|
|||
|
||||
/**
|
||||
* Execute BM25 + semantic search and merge with RRF.
|
||||
* Uses KuzuDB FTS for always-fresh BM25 results (no cached data).
|
||||
* The semanticSearch function is injected to keep this module environment-agnostic.
|
||||
*/
|
||||
export const hybridSearch = async (
|
||||
|
|
@ -154,11 +155,8 @@ export const hybridSearch = async (
|
|||
executeQuery: (cypher: string) => Promise<any[]>,
|
||||
semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number) => Promise<SemanticSearchResult[]>
|
||||
): Promise<HybridSearchResult[]> => {
|
||||
const bm25Results = isBM25Ready() ? searchBM25(query, limit) : [];
|
||||
// Use KuzuDB FTS for always-fresh BM25 results
|
||||
const bm25Results = await searchFTSFromKuzu(query, limit);
|
||||
const semanticResults = await semanticSearch(executeQuery, query, limit);
|
||||
return mergeWithRRF(bm25Results, semanticResults, limit);
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,120 +0,0 @@
|
|||
/**
|
||||
* BM25 Full-Text Search Index (Read-Only)
|
||||
*
|
||||
* Uses MiniSearch for fast keyword-based search with BM25 ranking.
|
||||
* For MCP, we only load and search - not build.
|
||||
*/
|
||||
|
||||
import MiniSearch from 'minisearch';
|
||||
import fs from 'fs/promises';
|
||||
|
||||
export interface BM25Document {
|
||||
id: string; // File path
|
||||
content: string; // File content
|
||||
name: string; // File name (boosted in search)
|
||||
}
|
||||
|
||||
export interface BM25SearchResult {
|
||||
filePath: string;
|
||||
score: number;
|
||||
rank: number;
|
||||
}
|
||||
|
||||
let searchIndex: MiniSearch<BM25Document> | null = null;
|
||||
let indexedDocCount = 0;
|
||||
|
||||
/**
|
||||
* Common stop words to filter out
|
||||
*/
|
||||
const STOP_WORDS = new Set([
|
||||
'const', 'let', 'var', 'function', 'return', 'if', 'else', 'for', 'while',
|
||||
'class', 'new', 'this', 'import', 'export', 'from', 'default', 'async', 'await',
|
||||
'try', 'catch', 'throw', 'typeof', 'instanceof', 'true', 'false', 'null', 'undefined',
|
||||
'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with',
|
||||
'to', 'of', 'it', 'be', 'as', 'by', 'that', 'for', 'are', 'was', 'were',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Tokenizer for BM25 search
|
||||
*/
|
||||
const tokenize = (text: string): string[] => {
|
||||
const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/);
|
||||
const expanded: string[] = [];
|
||||
for (const token of tokens) {
|
||||
if (token.length === 0) continue;
|
||||
const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' ');
|
||||
expanded.push(...camelParts);
|
||||
if (camelParts.length > 1) {
|
||||
expanded.push(token);
|
||||
}
|
||||
}
|
||||
return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t));
|
||||
};
|
||||
|
||||
/**
|
||||
* Load a BM25 index from disk
|
||||
*/
|
||||
export const loadBM25Index = async (filePath: string): Promise<boolean> => {
|
||||
try {
|
||||
const json = await fs.readFile(filePath, 'utf-8');
|
||||
// MiniSearch.loadJSON expects the raw JSON string, not a parsed object
|
||||
searchIndex = MiniSearch.loadJSON(json, {
|
||||
fields: ['content', 'name'],
|
||||
storeFields: ['id'],
|
||||
tokenize,
|
||||
});
|
||||
indexedDocCount = searchIndex.documentCount;
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Search the BM25 index
|
||||
*/
|
||||
export const searchBM25 = (query: string, limit: number = 20): BM25SearchResult[] => {
|
||||
if (!searchIndex) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const results = searchIndex.search(query, {
|
||||
fuzzy: 0.2,
|
||||
prefix: true,
|
||||
boost: { name: 2 },
|
||||
});
|
||||
|
||||
return results.slice(0, limit).map((r, index) => ({
|
||||
filePath: r.id,
|
||||
score: r.score,
|
||||
rank: index + 1,
|
||||
}));
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if the BM25 index is ready
|
||||
*/
|
||||
export const isBM25Ready = (): boolean => {
|
||||
return searchIndex !== null && indexedDocCount > 0;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get index statistics
|
||||
*/
|
||||
export const getBM25Stats = (): { documentCount: number; termCount: number } => {
|
||||
if (!searchIndex) {
|
||||
return { documentCount: 0, termCount: 0 };
|
||||
}
|
||||
return {
|
||||
documentCount: indexedDocCount,
|
||||
termCount: searchIndex.termCount,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Clear the index
|
||||
*/
|
||||
export const clearBM25Index = (): void => {
|
||||
searchIndex = null;
|
||||
indexedDocCount = 0;
|
||||
};
|
||||
|
|
@ -1,30 +1,37 @@
|
|||
/**
|
||||
* KuzuDB Adapter (Read-Only)
|
||||
* KuzuDB Adapter (Persistent Connection)
|
||||
*
|
||||
* Simplified adapter for MCP that only reads from existing .gitnexus/ database.
|
||||
* Holds a single database connection for the lifetime of the MCP session.
|
||||
* This is safe since the watcher has been removed -- only one process
|
||||
* accesses the database at a time.
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import kuzu from 'kuzu';
|
||||
|
||||
let db: kuzu.Database | null = null;
|
||||
let conn: kuzu.Connection | null = null;
|
||||
|
||||
export const initKuzu = async (dbPath: string): Promise<void> => {
|
||||
if (conn) return;
|
||||
/**
|
||||
* Initialize with a persistent connection to the database
|
||||
*/
|
||||
export const initKuzu = async (path: string): Promise<void> => {
|
||||
if (conn) return; // Already initialized
|
||||
|
||||
// Check if database exists
|
||||
try {
|
||||
await fs.stat(dbPath);
|
||||
await fs.stat(path);
|
||||
} catch {
|
||||
throw new Error(`KuzuDB not found at ${dbPath}. Run: gitnexus analyze`);
|
||||
throw new Error(`KuzuDB not found at ${path}. Run: gitnexus analyze`);
|
||||
}
|
||||
|
||||
db = new kuzu.Database(dbPath);
|
||||
db = new kuzu.Database(path);
|
||||
conn = new kuzu.Connection(db);
|
||||
};
|
||||
|
||||
/**
|
||||
* Execute a query using the persistent connection
|
||||
*/
|
||||
export const executeQuery = async (cypher: string): Promise<any[]> => {
|
||||
if (!conn) {
|
||||
throw new Error('KuzuDB not initialized. Call initKuzu first.');
|
||||
|
|
@ -36,19 +43,21 @@ export const executeQuery = async (cypher: string): Promise<any[]> => {
|
|||
return rows;
|
||||
};
|
||||
|
||||
/**
|
||||
* Close the persistent connection
|
||||
*/
|
||||
export const closeKuzu = async (): Promise<void> => {
|
||||
if (conn) {
|
||||
try {
|
||||
await conn.close();
|
||||
} catch {}
|
||||
try { await conn.close(); } catch {}
|
||||
conn = null;
|
||||
}
|
||||
if (db) {
|
||||
try {
|
||||
await db.close();
|
||||
} catch {}
|
||||
try { await db.close(); } catch {}
|
||||
db = null;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if the database connection is active
|
||||
*/
|
||||
export const isKuzuReady = (): boolean => conn !== null && db !== null;
|
||||
|
|
|
|||
|
|
@ -8,8 +8,15 @@
|
|||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import { initKuzu, executeQuery, closeKuzu, isKuzuReady } from '../core/kuzu-adapter.js';
|
||||
import { loadBM25Index, searchBM25, isBM25Ready } from '../core/bm25-index.js';
|
||||
import { embedQuery, getEmbeddingDims, disposeEmbedder } from '../core/embedder.js';
|
||||
import { isGitRepo, getCurrentCommit, getGitRoot } from '../../storage/git.js';
|
||||
import {
|
||||
getStoragePaths as getRepoStoragePaths,
|
||||
saveMeta as saveRepoMeta,
|
||||
loadMeta as loadRepoMeta,
|
||||
addToGitignore,
|
||||
} from '../../storage/repo-manager.js';
|
||||
import { generateAIContextFiles } from '../../cli/ai-context.js';
|
||||
|
||||
export interface RepoMeta {
|
||||
repoPath: string;
|
||||
|
|
@ -28,7 +35,6 @@ export interface IndexedRepo {
|
|||
repoPath: string;
|
||||
storagePath: string;
|
||||
kuzuPath: string;
|
||||
bm25Path: string;
|
||||
metaPath: string;
|
||||
meta: RepoMeta;
|
||||
}
|
||||
|
|
@ -40,7 +46,6 @@ function getStoragePaths(repoPath: string) {
|
|||
return {
|
||||
storagePath,
|
||||
kuzuPath: path.join(storagePath, 'kuzu'),
|
||||
bm25Path: path.join(storagePath, 'bm25.json'),
|
||||
metaPath: path.join(storagePath, 'meta.json'),
|
||||
};
|
||||
}
|
||||
|
|
@ -143,7 +148,6 @@ export class LocalBackend {
|
|||
if (this.initialized || !this.repo) return;
|
||||
|
||||
await initKuzu(this.repo.kuzuPath);
|
||||
await loadBM25Index(this.repo.bm25Path);
|
||||
this.initialized = true;
|
||||
}
|
||||
|
||||
|
|
@ -163,6 +167,10 @@ export class LocalBackend {
|
|||
return this.repo?.storagePath || null;
|
||||
}
|
||||
|
||||
get meta(): any {
|
||||
return this.repo?.meta || null;
|
||||
}
|
||||
|
||||
async callTool(method: string, params: any): Promise<any> {
|
||||
if (!this.repo) {
|
||||
throw new Error('Repository not indexed. Run: gitnexus analyze');
|
||||
|
|
@ -269,12 +277,13 @@ export class LocalBackend {
|
|||
}
|
||||
|
||||
/**
|
||||
* BM25 keyword search helper
|
||||
* BM25 keyword search helper - uses KuzuDB FTS for always-fresh results
|
||||
*/
|
||||
private async bm25Search(query: string, limit: number): Promise<any[]> {
|
||||
if (!isBM25Ready()) return [];
|
||||
// Import dynamically to avoid circular dependency
|
||||
const { searchFTSFromKuzu } = await import('../../core/search/bm25-index.js');
|
||||
const bm25Results = await searchFTSFromKuzu(query, limit);
|
||||
|
||||
const bm25Results = searchBM25(query, limit);
|
||||
const results: any[] = [];
|
||||
|
||||
for (const bm25Result of bm25Results) {
|
||||
|
|
@ -667,14 +676,153 @@ export class LocalBackend {
|
|||
};
|
||||
}
|
||||
|
||||
private async analyze(params: { path?: string; force?: boolean }): Promise<any> {
|
||||
const targetPath = params.path ? path.resolve(params.path) : process.cwd();
|
||||
|
||||
return {
|
||||
action: 'analyze',
|
||||
targetPath,
|
||||
message: `To index this repository, run:\n\n cd ${targetPath}\n gitnexus analyze${params.force ? ' --force' : ''}\n\nThis will create a .gitnexus/ folder with the knowledge graph.`,
|
||||
};
|
||||
private async analyze(params: { path?: string; force?: boolean; skipEmbeddings?: boolean }): Promise<any> {
|
||||
// Determine target repo path
|
||||
let repoPath: string;
|
||||
if (params.path) {
|
||||
repoPath = path.resolve(params.path);
|
||||
} else if (this.repo) {
|
||||
repoPath = this.repo.repoPath;
|
||||
} else {
|
||||
const gitRoot = getGitRoot(process.cwd());
|
||||
if (!gitRoot) {
|
||||
return { error: 'Not inside a git repository' };
|
||||
}
|
||||
repoPath = gitRoot;
|
||||
}
|
||||
|
||||
if (!isGitRepo(repoPath)) {
|
||||
return { error: 'Not a git repository' };
|
||||
}
|
||||
|
||||
const { storagePath, kuzuPath } = getRepoStoragePaths(repoPath);
|
||||
const currentCommit = getCurrentCommit(repoPath);
|
||||
const existingMeta = await loadRepoMeta(storagePath);
|
||||
|
||||
// Skip if already indexed at same commit (unless force)
|
||||
if (existingMeta && !params.force && existingMeta.lastCommit === currentCommit) {
|
||||
return { status: 'up_to_date', message: 'Repository already up to date.' };
|
||||
}
|
||||
|
||||
// Close MCP's persistent connection before pipeline takes over
|
||||
await closeKuzu();
|
||||
this.initialized = false;
|
||||
|
||||
try {
|
||||
// Import pipeline modules dynamically to avoid circular deps
|
||||
const { runPipelineFromRepo } = await import('../../core/ingestion/pipeline.js');
|
||||
const coreKuzu = await import('../../core/kuzu/kuzu-adapter.js');
|
||||
|
||||
// Run ingestion pipeline
|
||||
console.error('GitNexus: Running indexing pipeline...');
|
||||
const pipelineResult = await runPipelineFromRepo(repoPath, (progress) => {
|
||||
if (progress.percent % 20 === 0) {
|
||||
console.error(`GitNexus: ${progress.phase} ${progress.percent}%`);
|
||||
}
|
||||
});
|
||||
|
||||
// Load graph into KuzuDB
|
||||
console.error('GitNexus: Loading graph into KuzuDB...');
|
||||
await coreKuzu.initKuzu(kuzuPath);
|
||||
await coreKuzu.loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath);
|
||||
|
||||
// Create FTS indexes
|
||||
console.error('GitNexus: Creating FTS indexes...');
|
||||
try {
|
||||
await coreKuzu.createFTSIndex('File', 'file_fts', ['name', 'content']);
|
||||
await coreKuzu.createFTSIndex('Function', 'function_fts', ['name', 'content']);
|
||||
await coreKuzu.createFTSIndex('Class', 'class_fts', ['name', 'content']);
|
||||
await coreKuzu.createFTSIndex('Method', 'method_fts', ['name', 'content']);
|
||||
} catch (e: any) {
|
||||
console.error('GitNexus: Some FTS indexes may not have been created:', e.message);
|
||||
}
|
||||
|
||||
// Generate embeddings (unless skipped)
|
||||
if (!params.skipEmbeddings) {
|
||||
try {
|
||||
console.error('GitNexus: Generating embeddings...');
|
||||
const { runEmbeddingPipeline } = await import('../../core/embeddings/embedding-pipeline.js');
|
||||
await runEmbeddingPipeline(
|
||||
coreKuzu.executeQuery,
|
||||
coreKuzu.executeWithReusedStatement,
|
||||
(progress) => {
|
||||
if (progress.percent % 25 === 0) {
|
||||
console.error(`GitNexus: Embeddings ${progress.percent}%`);
|
||||
}
|
||||
}
|
||||
);
|
||||
} catch (e: any) {
|
||||
console.error('GitNexus: Embedding generation failed (non-fatal):', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
// Save metadata
|
||||
const stats = await coreKuzu.getKuzuStats();
|
||||
await saveRepoMeta(storagePath, {
|
||||
repoPath,
|
||||
lastCommit: currentCommit,
|
||||
indexedAt: new Date().toISOString(),
|
||||
stats: {
|
||||
files: pipelineResult.fileContents.size,
|
||||
nodes: stats.nodes,
|
||||
edges: stats.edges,
|
||||
communities: pipelineResult.communityResult?.stats.totalCommunities,
|
||||
processes: pipelineResult.processResult?.stats.totalProcesses,
|
||||
},
|
||||
});
|
||||
|
||||
// Add .gitnexus to .gitignore
|
||||
await addToGitignore(repoPath);
|
||||
|
||||
// Generate AI context files
|
||||
const projectName = path.basename(repoPath);
|
||||
await generateAIContextFiles(repoPath, storagePath, projectName, {
|
||||
files: pipelineResult.fileContents.size,
|
||||
nodes: stats.nodes,
|
||||
edges: stats.edges,
|
||||
communities: pipelineResult.communityResult?.stats.totalCommunities,
|
||||
processes: pipelineResult.processResult?.stats.totalProcesses,
|
||||
});
|
||||
|
||||
// Close core kuzu connection (pipeline is done)
|
||||
await coreKuzu.closeKuzu();
|
||||
|
||||
// Re-init MCP state so next tool call reconnects
|
||||
this.repo = await loadRepo(repoPath);
|
||||
if (this.repo) {
|
||||
const repoStats = this.repo.meta.stats || {};
|
||||
this._context = {
|
||||
projectName: path.basename(this.repo.repoPath),
|
||||
stats: {
|
||||
fileCount: repoStats.files || 0,
|
||||
functionCount: repoStats.nodes || 0,
|
||||
classCount: 0,
|
||||
interfaceCount: 0,
|
||||
methodCount: 0,
|
||||
communityCount: repoStats.communities || 0,
|
||||
processCount: repoStats.processes || 0,
|
||||
},
|
||||
hotspots: [],
|
||||
folderTree: '',
|
||||
};
|
||||
}
|
||||
|
||||
console.error('GitNexus: Indexing complete!');
|
||||
return {
|
||||
status: 'success',
|
||||
message: `Repository indexed successfully.`,
|
||||
stats: {
|
||||
files: pipelineResult.fileContents.size,
|
||||
nodes: stats.nodes,
|
||||
edges: stats.edges,
|
||||
communities: pipelineResult.communityResult?.stats.totalCommunities,
|
||||
processes: pipelineResult.processResult?.stats.totalProcesses,
|
||||
},
|
||||
};
|
||||
} catch (e: any) {
|
||||
console.error('GitNexus: Indexing failed:', e.message);
|
||||
return { error: `Indexing failed: ${e.message}` };
|
||||
}
|
||||
}
|
||||
|
||||
async disconnect(): Promise<void> {
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
*/
|
||||
|
||||
import type { LocalBackend } from './local/local-backend.js';
|
||||
import { checkStaleness } from './staleness.js';
|
||||
|
||||
export interface ResourceDefinition {
|
||||
uri: string;
|
||||
|
|
@ -113,28 +114,42 @@ async function getContextResource(backend: LocalBackend): Promise<string> {
|
|||
return 'error: No codebase loaded. Run: gitnexus analyze';
|
||||
}
|
||||
|
||||
// Check staleness
|
||||
const repoPath = backend.repoPath;
|
||||
const lastCommit = backend.meta?.lastCommit || 'HEAD';
|
||||
const staleness = repoPath ? checkStaleness(repoPath, lastCommit) : { isStale: false, commitsBehind: 0 };
|
||||
|
||||
const lines: string[] = [
|
||||
`project: ${context.projectName}`,
|
||||
'stats:',
|
||||
` files: ${context.stats.fileCount}`,
|
||||
` symbols: ${context.stats.functionCount}`,
|
||||
` clusters: ${context.stats.communityCount}`,
|
||||
` processes: ${context.stats.processCount}`,
|
||||
'',
|
||||
'tools_available:',
|
||||
' - search: Hybrid semantic + keyword search',
|
||||
' - explore: Deep dive on symbol/cluster/process',
|
||||
' - impact: Blast radius analysis',
|
||||
' - overview: List all clusters and processes',
|
||||
' - cypher: Raw graph queries',
|
||||
'',
|
||||
'resources_available:',
|
||||
' - gitnexus://clusters: All clusters',
|
||||
' - gitnexus://processes: All processes',
|
||||
' - gitnexus://cluster/{name}: Cluster details',
|
||||
' - gitnexus://process/{name}: Process trace',
|
||||
];
|
||||
|
||||
// Add staleness warning if index is behind
|
||||
if (staleness.isStale && staleness.hint) {
|
||||
lines.push('');
|
||||
lines.push(`staleness: "${staleness.hint}"`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push('stats:');
|
||||
lines.push(` files: ${context.stats.fileCount}`);
|
||||
lines.push(` symbols: ${context.stats.functionCount}`);
|
||||
lines.push(` clusters: ${context.stats.communityCount}`);
|
||||
lines.push(` processes: ${context.stats.processCount}`);
|
||||
lines.push('');
|
||||
lines.push('tools_available:');
|
||||
lines.push(' - search: Hybrid semantic + keyword search');
|
||||
lines.push(' - explore: Deep dive on symbol/cluster/process');
|
||||
lines.push(' - impact: Blast radius analysis');
|
||||
lines.push(' - overview: List all clusters and processes');
|
||||
lines.push(' - cypher: Raw graph queries');
|
||||
lines.push(' - analyze: Re-index to update stale data');
|
||||
lines.push('');
|
||||
lines.push('resources_available:');
|
||||
lines.push(' - gitnexus://clusters: All clusters');
|
||||
lines.push(' - gitnexus://processes: All processes');
|
||||
lines.push(' - gitnexus://cluster/{name}: Cluster details');
|
||||
lines.push(' - gitnexus://process/{name}: Process trace');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
|
|
|
|||
43
gitnexus/src/mcp/staleness.ts
Normal file
43
gitnexus/src/mcp/staleness.ts
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
/**
|
||||
* Staleness Check
|
||||
*
|
||||
* Checks if the GitNexus index is behind the current git HEAD.
|
||||
* Returns a hint for the LLM to call analyze if stale.
|
||||
*/
|
||||
|
||||
import { execSync } from 'child_process';
|
||||
import path from 'path';
|
||||
|
||||
export interface StalenessInfo {
|
||||
isStale: boolean;
|
||||
commitsBehind: number;
|
||||
hint?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check how many commits the index is behind HEAD
|
||||
*/
|
||||
export function checkStaleness(repoPath: string, lastCommit: string): StalenessInfo {
|
||||
try {
|
||||
// Get count of commits between lastCommit and HEAD
|
||||
const result = execSync(
|
||||
`git rev-list --count ${lastCommit}..HEAD`,
|
||||
{ cwd: repoPath, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] }
|
||||
).trim();
|
||||
|
||||
const commitsBehind = parseInt(result, 10) || 0;
|
||||
|
||||
if (commitsBehind > 0) {
|
||||
return {
|
||||
isStale: true,
|
||||
commitsBehind,
|
||||
hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`,
|
||||
};
|
||||
}
|
||||
|
||||
return { isStale: false, commitsBehind: 0 };
|
||||
} catch {
|
||||
// If git command fails, assume not stale (fail open)
|
||||
return { isStale: false, commitsBehind: 0 };
|
||||
}
|
||||
}
|
||||
|
|
@ -24,18 +24,22 @@ export interface ToolDefinition {
|
|||
export const GITNEXUS_TOOLS: ToolDefinition[] = [
|
||||
{
|
||||
name: 'analyze',
|
||||
description: `Index or re-index the current repository.
|
||||
description: `Index or re-index the current repository. Runs the full pipeline in-process.
|
||||
|
||||
Creates .gitnexus/ in repo root with:
|
||||
- Knowledge graph (functions, classes, calls, imports)
|
||||
- BM25 search index
|
||||
- Full-text search indexes
|
||||
- Community detection (Leiden)
|
||||
- Process tracing
|
||||
- Embeddings for semantic search
|
||||
|
||||
Run this when:
|
||||
- First time using GitNexus on a repo
|
||||
- After major code changes
|
||||
- When 'not indexed' error appears`,
|
||||
- When staleness warning appears in gitnexus://context
|
||||
- When 'not indexed' error appears
|
||||
|
||||
Note: This may take 30-120 seconds for large repos.`,
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import { findRepo, loadMeta } from '../storage/repo-manager.js';
|
|||
import { initKuzu, executeQuery } from '../core/kuzu/kuzu-adapter.js';
|
||||
import { NODE_TABLES } from '../core/kuzu/schema.js';
|
||||
import { GraphNode, GraphRelationship } from '../core/graph/types.js';
|
||||
import { loadBM25Index, searchBM25, isBM25Ready } from '../core/search/bm25-index.js';
|
||||
import { searchFTSFromKuzu } from '../core/search/bm25-index.js';
|
||||
import { hybridSearch } from '../core/search/hybrid-search.js';
|
||||
import { semanticSearch } from '../core/embeddings/embedding-pipeline.js';
|
||||
import { isEmbedderReady } from '../core/embeddings/embedder.js';
|
||||
|
|
@ -131,28 +131,19 @@ export const createServer = async (port: number) => {
|
|||
return;
|
||||
}
|
||||
await initKuzu(repo.kuzuPath);
|
||||
await loadBM25Index(repo.bm25Path);
|
||||
|
||||
const query = req.body.query ?? '';
|
||||
const limit = req.body.limit ?? 10;
|
||||
|
||||
if (isBM25Ready() && isEmbedderReady()) {
|
||||
if (isEmbedderReady()) {
|
||||
const results = await hybridSearch(query, limit, executeQuery, semanticSearch);
|
||||
res.json({ results });
|
||||
return;
|
||||
}
|
||||
|
||||
if (isBM25Ready()) {
|
||||
res.json({ results: searchBM25(query, limit) });
|
||||
return;
|
||||
}
|
||||
|
||||
if (isEmbedderReady()) {
|
||||
res.json({ results: await semanticSearch(executeQuery, query, limit) });
|
||||
return;
|
||||
}
|
||||
|
||||
res.json({ results: [] });
|
||||
// FTS-only fallback when embeddings aren't loaded
|
||||
const results = await searchFTSFromKuzu(query, limit);
|
||||
res.json({ results });
|
||||
});
|
||||
|
||||
// Read file
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ export interface RepoMeta {
|
|||
edges?: number;
|
||||
communities?: number;
|
||||
processes?: number;
|
||||
embeddings?: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -24,7 +25,6 @@ export interface IndexedRepo {
|
|||
repoPath: string;
|
||||
storagePath: string;
|
||||
kuzuPath: string;
|
||||
bm25Path: string;
|
||||
metaPath: string;
|
||||
meta: RepoMeta;
|
||||
}
|
||||
|
|
@ -46,7 +46,6 @@ export const getStoragePaths = (repoPath: string) => {
|
|||
return {
|
||||
storagePath,
|
||||
kuzuPath: path.join(storagePath, 'kuzu'),
|
||||
bm25Path: path.join(storagePath, 'bm25.json'),
|
||||
metaPath: path.join(storagePath, 'meta.json'),
|
||||
};
|
||||
};
|
||||
|
|
|
|||
355
gitnexus/vendor/leiden/index.cjs
vendored
Normal file
355
gitnexus/vendor/leiden/index.cjs
vendored
Normal file
|
|
@ -0,0 +1,355 @@
|
|||
/**
|
||||
* Graphology Leiden Algorithm
|
||||
* ============================
|
||||
*
|
||||
* JavaScript implementation of the Leiden community detection
|
||||
* algorithm for graphology.
|
||||
*
|
||||
* Vendored from: https://github.com/graphology/graphology/tree/master/src/communities-leiden
|
||||
* License: MIT
|
||||
*
|
||||
* [Reference]
|
||||
* Traag, V. A., et al. "From Louvain to Leiden: Guaranteeing Well-Connected
|
||||
* Communities". Scientific Reports, vol. 9, no 1, 2019, p. 5233.
|
||||
* https://arxiv.org/abs/1810.08473
|
||||
*/
|
||||
var resolveDefaults = require('graphology-utils/defaults');
|
||||
var isGraph = require('graphology-utils/is-graph');
|
||||
var inferType = require('graphology-utils/infer-type');
|
||||
var SparseMap = require('mnemonist/sparse-map');
|
||||
var SparseQueueSet = require('mnemonist/sparse-queue-set');
|
||||
var createRandomIndex = require('pandemonium/random-index').createRandomIndex;
|
||||
var utils = require('./utils.cjs');
|
||||
|
||||
var indices = require('graphology-indices/louvain');
|
||||
var addWeightToCommunity = utils.addWeightToCommunity;
|
||||
|
||||
var UndirectedLouvainIndex = indices.UndirectedLouvainIndex;
|
||||
|
||||
var UndirectedLeidenAddenda = utils.UndirectedLeidenAddenda;
|
||||
|
||||
var DEFAULTS = {
|
||||
attributes: {
|
||||
community: 'community',
|
||||
weight: 'weight'
|
||||
},
|
||||
randomness: 0.01,
|
||||
randomWalk: true,
|
||||
resolution: 1,
|
||||
rng: Math.random,
|
||||
weighted: false
|
||||
};
|
||||
|
||||
var EPSILON = 1e-10;
|
||||
|
||||
function tieBreaker(
|
||||
bestCommunity,
|
||||
currentCommunity,
|
||||
targetCommunity,
|
||||
delta,
|
||||
bestDelta
|
||||
) {
|
||||
if (Math.abs(delta - bestDelta) < EPSILON) {
|
||||
if (bestCommunity === currentCommunity) {
|
||||
return false;
|
||||
} else {
|
||||
return targetCommunity > bestCommunity;
|
||||
}
|
||||
} else if (delta > bestDelta) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function undirectedLeiden(detailed, graph, options) {
|
||||
var index = new UndirectedLouvainIndex(graph, {
|
||||
attributes: {
|
||||
weight: options.attributes.weight
|
||||
},
|
||||
keepDendrogram: detailed,
|
||||
resolution: options.resolution,
|
||||
weighted: options.weighted
|
||||
});
|
||||
|
||||
var addenda = new UndirectedLeidenAddenda(index, {
|
||||
randomness: options.randomness,
|
||||
rng: options.rng
|
||||
});
|
||||
|
||||
var randomIndex = createRandomIndex(options.rng);
|
||||
|
||||
// Communities
|
||||
var currentCommunity, targetCommunity;
|
||||
var communities = new SparseMap(Float64Array, index.C);
|
||||
|
||||
// Traversal
|
||||
var queue = new SparseQueueSet(index.C),
|
||||
start,
|
||||
end,
|
||||
weight,
|
||||
ci,
|
||||
ri,
|
||||
s,
|
||||
i,
|
||||
j,
|
||||
l;
|
||||
|
||||
// Metrics
|
||||
var degree, targetCommunityDegree;
|
||||
|
||||
// Moves
|
||||
var bestCommunity, bestDelta, deltaIsBetter, delta;
|
||||
|
||||
// Details
|
||||
var deltaComputations = 0,
|
||||
nodesVisited = 0,
|
||||
moves = [],
|
||||
currentMoves;
|
||||
|
||||
while (true) {
|
||||
l = index.C;
|
||||
|
||||
currentMoves = 0;
|
||||
|
||||
// Traversal of the graph
|
||||
ri = options.randomWalk ? randomIndex(l) : 0;
|
||||
|
||||
for (s = 0; s < l; s++, ri++) {
|
||||
i = ri % l;
|
||||
queue.enqueue(i);
|
||||
}
|
||||
|
||||
while (queue.size !== 0) {
|
||||
i = queue.dequeue();
|
||||
nodesVisited++;
|
||||
|
||||
degree = 0;
|
||||
communities.clear();
|
||||
|
||||
currentCommunity = index.belongings[i];
|
||||
|
||||
start = index.starts[i];
|
||||
end = index.starts[i + 1];
|
||||
|
||||
// Traversing neighbors
|
||||
for (; start < end; start++) {
|
||||
j = index.neighborhood[start];
|
||||
weight = index.weights[start];
|
||||
|
||||
targetCommunity = index.belongings[j];
|
||||
|
||||
// Incrementing metrics
|
||||
degree += weight;
|
||||
addWeightToCommunity(communities, targetCommunity, weight);
|
||||
}
|
||||
|
||||
// Finding best community to move to
|
||||
bestDelta = index.fastDeltaWithOwnCommunity(
|
||||
i,
|
||||
degree,
|
||||
communities.get(currentCommunity) || 0,
|
||||
currentCommunity
|
||||
);
|
||||
bestCommunity = currentCommunity;
|
||||
|
||||
for (ci = 0; ci < communities.size; ci++) {
|
||||
targetCommunity = communities.dense[ci];
|
||||
|
||||
if (targetCommunity === currentCommunity) continue;
|
||||
|
||||
targetCommunityDegree = communities.vals[ci];
|
||||
|
||||
deltaComputations++;
|
||||
|
||||
delta = index.fastDelta(
|
||||
i,
|
||||
degree,
|
||||
targetCommunityDegree,
|
||||
targetCommunity
|
||||
);
|
||||
|
||||
deltaIsBetter = tieBreaker(
|
||||
bestCommunity,
|
||||
currentCommunity,
|
||||
targetCommunity,
|
||||
delta,
|
||||
bestDelta
|
||||
);
|
||||
|
||||
if (deltaIsBetter) {
|
||||
bestDelta = delta;
|
||||
bestCommunity = targetCommunity;
|
||||
}
|
||||
}
|
||||
|
||||
if (bestDelta < 0) {
|
||||
bestCommunity = index.isolate(i, degree);
|
||||
|
||||
if (bestCommunity === currentCommunity) continue;
|
||||
} else {
|
||||
if (bestCommunity === currentCommunity) {
|
||||
continue;
|
||||
} else {
|
||||
index.move(i, degree, bestCommunity);
|
||||
}
|
||||
}
|
||||
|
||||
currentMoves++;
|
||||
|
||||
// Adding neighbors from other communities to the queue
|
||||
start = index.starts[i];
|
||||
end = index.starts[i + 1];
|
||||
|
||||
for (; start < end; start++) {
|
||||
j = index.neighborhood[start];
|
||||
targetCommunity = index.belongings[j];
|
||||
|
||||
if (targetCommunity !== bestCommunity) queue.enqueue(j);
|
||||
}
|
||||
}
|
||||
|
||||
moves.push(currentMoves);
|
||||
|
||||
if (currentMoves === 0) {
|
||||
index.zoomOut();
|
||||
break;
|
||||
}
|
||||
|
||||
if (!addenda.onlySingletons()) {
|
||||
// We continue working on the induced graph
|
||||
addenda.zoomOut();
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
var results = {
|
||||
index: index,
|
||||
deltaComputations: deltaComputations,
|
||||
nodesVisited: nodesVisited,
|
||||
moves: moves
|
||||
};
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Function returning the communities mapping of the graph.
|
||||
*
|
||||
* @param {boolean} assign - Assign communities to nodes attributes?
|
||||
* @param {boolean} detailed - Whether to return detailed information.
|
||||
* @param {Graph} graph - Target graph.
|
||||
* @param {object} options - Options:
|
||||
* @param {object} attributes - Attribute names:
|
||||
* @param {string} community - Community node attribute name.
|
||||
* @param {string} weight - Weight edge attribute name.
|
||||
* @param {number} randomness - Randomness parameter.
|
||||
* @param {boolean} randomWalk - Whether to traverse the graph in random order.
|
||||
* @param {number} resolution - Resolution parameter.
|
||||
* @param {function} rng - RNG function to use.
|
||||
* @param {boolean} weighted - Whether to compute the weighted version.
|
||||
* @return {object}
|
||||
*/
|
||||
function leiden(assign, detailed, graph, options) {
|
||||
if (!isGraph(graph))
|
||||
throw new Error(
|
||||
'graphology-communities-leiden: the given graph is not a valid graphology instance.'
|
||||
);
|
||||
|
||||
var type = inferType(graph);
|
||||
|
||||
if (type === 'mixed')
|
||||
throw new Error(
|
||||
'graphology-communities-leiden: cannot run the algorithm on a true mixed graph.'
|
||||
);
|
||||
|
||||
if (type === 'directed')
|
||||
throw new Error(
|
||||
'graphology-communities-leiden: not yet implemented for directed graphs.'
|
||||
);
|
||||
|
||||
// Attributes name
|
||||
options = resolveDefaults(options, DEFAULTS);
|
||||
|
||||
// Empty graph case
|
||||
var c = 0;
|
||||
|
||||
if (graph.size === 0) {
|
||||
if (assign) {
|
||||
graph.forEachNode(function (node) {
|
||||
graph.setNodeAttribute(node, options.attributes.communities, c++);
|
||||
});
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var communities = {};
|
||||
|
||||
graph.forEachNode(function (node) {
|
||||
communities[node] = c++;
|
||||
});
|
||||
|
||||
if (!detailed) return communities;
|
||||
|
||||
return {
|
||||
communities: communities,
|
||||
count: graph.order,
|
||||
deltaComputations: 0,
|
||||
dendrogram: null,
|
||||
level: 0,
|
||||
modularity: NaN,
|
||||
moves: null,
|
||||
nodesVisited: 0,
|
||||
resolution: options.resolution
|
||||
};
|
||||
}
|
||||
|
||||
var fn = undirectedLeiden;
|
||||
|
||||
var results = fn(detailed, graph, options);
|
||||
|
||||
var index = results.index;
|
||||
|
||||
// Standard output
|
||||
if (!detailed) {
|
||||
if (assign) {
|
||||
index.assign(options.attributes.community);
|
||||
return;
|
||||
}
|
||||
|
||||
return index.collect();
|
||||
}
|
||||
|
||||
// Detailed output
|
||||
var output = {
|
||||
count: index.C,
|
||||
deltaComputations: results.deltaComputations,
|
||||
dendrogram: index.dendrogram,
|
||||
level: index.level,
|
||||
modularity: index.modularity(),
|
||||
moves: results.moves,
|
||||
nodesVisited: results.nodesVisited,
|
||||
resolution: options.resolution
|
||||
};
|
||||
|
||||
if (assign) {
|
||||
index.assign(options.attributes.community);
|
||||
return output;
|
||||
}
|
||||
|
||||
output.communities = index.collect();
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
/**
|
||||
* Exporting.
|
||||
*/
|
||||
var fn = leiden.bind(null, false, false);
|
||||
fn.assign = leiden.bind(null, true, false);
|
||||
fn.detailed = leiden.bind(null, false, true);
|
||||
fn.defaults = DEFAULTS;
|
||||
|
||||
module.exports = fn;
|
||||
392
gitnexus/vendor/leiden/utils.cjs
vendored
Normal file
392
gitnexus/vendor/leiden/utils.cjs
vendored
Normal file
|
|
@ -0,0 +1,392 @@
|
|||
/**
|
||||
* Graphology Leiden Utils
|
||||
* ========================
|
||||
*
|
||||
* Miscellaneous utilities used by the Leiden algorithm.
|
||||
*
|
||||
* Vendored from: https://github.com/graphology/graphology/tree/master/src/communities-leiden
|
||||
* License: MIT
|
||||
*/
|
||||
var SparseMap = require('mnemonist/sparse-map');
|
||||
var createRandom = require('pandemonium/random').createRandom;
|
||||
|
||||
function addWeightToCommunity(map, community, weight) {
|
||||
var currentWeight = map.get(community);
|
||||
|
||||
if (typeof currentWeight === 'undefined') currentWeight = 0;
|
||||
|
||||
currentWeight += weight;
|
||||
|
||||
map.set(community, currentWeight);
|
||||
}
|
||||
|
||||
function UndirectedLeidenAddenda(index, options) {
|
||||
options = options || {};
|
||||
|
||||
var rng = options.rng || Math.random;
|
||||
var randomness = 'randomness' in options ? options.randomness : 0.01;
|
||||
|
||||
this.index = index;
|
||||
this.random = createRandom(rng);
|
||||
this.randomness = randomness;
|
||||
this.rng = rng;
|
||||
|
||||
var NodesPointerArray = index.counts.constructor;
|
||||
var WeightsArray = index.weights.constructor;
|
||||
|
||||
var order = index.C;
|
||||
this.resolution = index.resolution;
|
||||
|
||||
// Used to group nodes by communities
|
||||
this.B = index.C;
|
||||
this.C = 0;
|
||||
this.communitiesOffsets = new NodesPointerArray(order);
|
||||
this.nodesSortedByCommunities = new NodesPointerArray(order);
|
||||
this.communitiesBounds = new NodesPointerArray(order + 1);
|
||||
|
||||
// Used to merge nodes subsets
|
||||
this.communityWeights = new WeightsArray(order);
|
||||
this.degrees = new WeightsArray(order);
|
||||
this.nonSingleton = new Uint8Array(order);
|
||||
this.externalEdgeWeightPerCommunity = new WeightsArray(order);
|
||||
this.belongings = new NodesPointerArray(order);
|
||||
this.neighboringCommunities = new SparseMap(WeightsArray, order);
|
||||
this.cumulativeIncrement = new Float64Array(order);
|
||||
this.macroCommunities = null;
|
||||
}
|
||||
|
||||
UndirectedLeidenAddenda.prototype.groupByCommunities = function () {
|
||||
var index = this.index;
|
||||
|
||||
var n, i, c, b, o;
|
||||
|
||||
n = 0;
|
||||
o = 0;
|
||||
|
||||
for (i = 0; i < index.C; i++) {
|
||||
c = index.counts[i];
|
||||
|
||||
if (c !== 0) {
|
||||
this.communitiesBounds[o++] = n;
|
||||
n += c;
|
||||
this.communitiesOffsets[i] = n;
|
||||
}
|
||||
}
|
||||
|
||||
this.communitiesBounds[o] = n;
|
||||
|
||||
o = 0;
|
||||
|
||||
for (i = 0; i < index.C; i++) {
|
||||
b = index.belongings[i];
|
||||
o = --this.communitiesOffsets[b];
|
||||
this.nodesSortedByCommunities[o] = i;
|
||||
}
|
||||
|
||||
this.B = index.C - index.U;
|
||||
this.C = index.C;
|
||||
};
|
||||
|
||||
UndirectedLeidenAddenda.prototype.communities = function () {
|
||||
var communities = new Array(this.B);
|
||||
|
||||
var i, j, community, start, stop;
|
||||
|
||||
for (i = 0; i < this.B; i++) {
|
||||
start = this.communitiesBounds[i];
|
||||
stop = this.communitiesBounds[i + 1];
|
||||
community = [];
|
||||
|
||||
for (j = start; j < stop; j++) {
|
||||
community.push(j);
|
||||
}
|
||||
|
||||
communities[i] = community;
|
||||
}
|
||||
|
||||
return communities;
|
||||
};
|
||||
|
||||
UndirectedLeidenAddenda.prototype.mergeNodesSubset = function (start, stop) {
|
||||
var index = this.index;
|
||||
var currentMacroCommunity =
|
||||
index.belongings[this.nodesSortedByCommunities[start]];
|
||||
var neighboringCommunities = this.neighboringCommunities;
|
||||
|
||||
var totalNodeWeight = 0;
|
||||
|
||||
var i, j, w;
|
||||
var ei, el, et;
|
||||
|
||||
// Initializing singletons
|
||||
for (j = start; j < stop; j++) {
|
||||
i = this.nodesSortedByCommunities[j];
|
||||
|
||||
this.belongings[i] = i;
|
||||
this.nonSingleton[i] = 0;
|
||||
this.degrees[i] = 0;
|
||||
totalNodeWeight += index.loops[i] / 2;
|
||||
|
||||
this.communityWeights[i] = index.loops[i];
|
||||
this.externalEdgeWeightPerCommunity[i] = 0;
|
||||
|
||||
ei = index.starts[i];
|
||||
el = index.starts[i + 1];
|
||||
|
||||
for (; ei < el; ei++) {
|
||||
et = index.neighborhood[ei];
|
||||
w = index.weights[ei];
|
||||
|
||||
this.degrees[i] += w;
|
||||
|
||||
if (index.belongings[et] !== currentMacroCommunity) continue;
|
||||
|
||||
totalNodeWeight += w;
|
||||
this.externalEdgeWeightPerCommunity[i] += w;
|
||||
this.communityWeights[i] += w;
|
||||
}
|
||||
}
|
||||
|
||||
var microDegrees = this.externalEdgeWeightPerCommunity.slice();
|
||||
|
||||
var s, ri, ci;
|
||||
var order = stop - start;
|
||||
|
||||
var degree,
|
||||
bestCommunity,
|
||||
qualityValueIncrement,
|
||||
maxQualityValueIncrement,
|
||||
totalTransformedQualityValueIncrement,
|
||||
targetCommunity,
|
||||
targetCommunityDegree,
|
||||
targetCommunityWeight;
|
||||
|
||||
var r, lo, hi, mid, chosenCommunity;
|
||||
|
||||
ri = this.random(start, stop - 1);
|
||||
|
||||
for (s = start; s < stop; s++, ri++) {
|
||||
j = start + (ri % order);
|
||||
|
||||
i = this.nodesSortedByCommunities[j];
|
||||
|
||||
if (this.nonSingleton[i] === 1) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
this.externalEdgeWeightPerCommunity[i] <
|
||||
this.communityWeights[i] *
|
||||
(totalNodeWeight / 2 - this.communityWeights[i]) *
|
||||
this.resolution
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
this.communityWeights[i] = 0;
|
||||
this.externalEdgeWeightPerCommunity[i] = 0;
|
||||
|
||||
neighboringCommunities.clear();
|
||||
neighboringCommunities.set(i, 0);
|
||||
|
||||
degree = 0;
|
||||
|
||||
ei = index.starts[i];
|
||||
el = index.starts[i + 1];
|
||||
|
||||
for (; ei < el; ei++) {
|
||||
et = index.neighborhood[ei];
|
||||
|
||||
if (index.belongings[et] !== currentMacroCommunity) continue;
|
||||
|
||||
w = index.weights[ei];
|
||||
|
||||
degree += w;
|
||||
|
||||
addWeightToCommunity(neighboringCommunities, this.belongings[et], w);
|
||||
}
|
||||
|
||||
bestCommunity = i;
|
||||
maxQualityValueIncrement = 0;
|
||||
totalTransformedQualityValueIncrement = 0;
|
||||
|
||||
for (ci = 0; ci < neighboringCommunities.size; ci++) {
|
||||
targetCommunity = neighboringCommunities.dense[ci];
|
||||
targetCommunityDegree = neighboringCommunities.vals[ci];
|
||||
targetCommunityWeight = this.communityWeights[targetCommunity];
|
||||
|
||||
if (
|
||||
this.externalEdgeWeightPerCommunity[targetCommunity] >=
|
||||
targetCommunityWeight *
|
||||
(totalNodeWeight / 2 - targetCommunityWeight) *
|
||||
this.resolution
|
||||
) {
|
||||
qualityValueIncrement =
|
||||
targetCommunityDegree -
|
||||
((degree + index.loops[i]) *
|
||||
targetCommunityWeight *
|
||||
this.resolution) /
|
||||
totalNodeWeight;
|
||||
|
||||
if (qualityValueIncrement > maxQualityValueIncrement) {
|
||||
bestCommunity = targetCommunity;
|
||||
maxQualityValueIncrement = qualityValueIncrement;
|
||||
}
|
||||
|
||||
if (qualityValueIncrement >= 0)
|
||||
totalTransformedQualityValueIncrement += Math.exp(
|
||||
qualityValueIncrement / this.randomness
|
||||
);
|
||||
}
|
||||
|
||||
this.cumulativeIncrement[ci] = totalTransformedQualityValueIncrement;
|
||||
}
|
||||
|
||||
if (
|
||||
totalTransformedQualityValueIncrement < Number.MAX_VALUE &&
|
||||
totalTransformedQualityValueIncrement < Infinity
|
||||
) {
|
||||
r = totalTransformedQualityValueIncrement * this.rng();
|
||||
lo = -1;
|
||||
hi = neighboringCommunities.size + 1;
|
||||
|
||||
while (lo < hi - 1) {
|
||||
mid = (lo + hi) >>> 1;
|
||||
|
||||
if (this.cumulativeIncrement[mid] >= r) hi = mid;
|
||||
else lo = mid;
|
||||
}
|
||||
|
||||
chosenCommunity = neighboringCommunities.dense[hi];
|
||||
} else {
|
||||
chosenCommunity = bestCommunity;
|
||||
}
|
||||
|
||||
this.communityWeights[chosenCommunity] += degree + index.loops[i];
|
||||
|
||||
ei = index.starts[i];
|
||||
el = index.starts[i + 1];
|
||||
|
||||
for (; ei < el; ei++) {
|
||||
et = index.neighborhood[ei];
|
||||
|
||||
if (index.belongings[et] !== currentMacroCommunity) continue;
|
||||
|
||||
targetCommunity = this.belongings[et];
|
||||
|
||||
if (targetCommunity === chosenCommunity) {
|
||||
this.externalEdgeWeightPerCommunity[chosenCommunity] -=
|
||||
microDegrees[et];
|
||||
} else {
|
||||
this.externalEdgeWeightPerCommunity[chosenCommunity] +=
|
||||
microDegrees[et];
|
||||
}
|
||||
}
|
||||
|
||||
if (chosenCommunity !== i) {
|
||||
this.belongings[i] = chosenCommunity;
|
||||
this.nonSingleton[chosenCommunity] = 1;
|
||||
this.C--;
|
||||
}
|
||||
}
|
||||
|
||||
var microCommunities = this.neighboringCommunities;
|
||||
microCommunities.clear();
|
||||
|
||||
for (j = start; j < stop; j++) {
|
||||
i = this.nodesSortedByCommunities[j];
|
||||
microCommunities.set(this.belongings[i], 1);
|
||||
}
|
||||
|
||||
return microCommunities.dense.slice(0, microCommunities.size);
|
||||
};
|
||||
|
||||
UndirectedLeidenAddenda.prototype.refinePartition = function () {
|
||||
this.groupByCommunities();
|
||||
|
||||
this.macroCommunities = new Array(this.B);
|
||||
|
||||
var i, start, stop, mapping;
|
||||
|
||||
var bounds = this.communitiesBounds;
|
||||
|
||||
for (i = 0; i < this.B; i++) {
|
||||
start = bounds[i];
|
||||
stop = bounds[i + 1];
|
||||
|
||||
mapping = this.mergeNodesSubset(start, stop);
|
||||
this.macroCommunities[i] = mapping;
|
||||
}
|
||||
};
|
||||
|
||||
UndirectedLeidenAddenda.prototype.split = function () {
|
||||
var index = this.index;
|
||||
var isolates = this.neighboringCommunities;
|
||||
|
||||
isolates.clear();
|
||||
|
||||
var i, community, isolated;
|
||||
|
||||
for (i = 0; i < index.C; i++) {
|
||||
community = this.belongings[i];
|
||||
|
||||
if (i !== community) continue;
|
||||
|
||||
isolated = index.isolate(i, this.degrees[i]);
|
||||
isolates.set(community, isolated);
|
||||
}
|
||||
|
||||
for (i = 0; i < index.C; i++) {
|
||||
community = this.belongings[i];
|
||||
|
||||
if (i === community) continue;
|
||||
|
||||
isolated = isolates.get(community);
|
||||
index.move(i, this.degrees[i], isolated);
|
||||
}
|
||||
|
||||
var j, macro;
|
||||
|
||||
for (i = 0; i < this.macroCommunities.length; i++) {
|
||||
macro = this.macroCommunities[i];
|
||||
|
||||
for (j = 0; j < macro.length; j++) macro[j] = isolates.get(macro[j]);
|
||||
}
|
||||
};
|
||||
|
||||
UndirectedLeidenAddenda.prototype.zoomOut = function () {
|
||||
var index = this.index;
|
||||
this.refinePartition();
|
||||
this.split();
|
||||
|
||||
var newLabels = index.zoomOut();
|
||||
|
||||
var macro, leader, follower;
|
||||
|
||||
var i, j;
|
||||
|
||||
for (i = 0; i < this.macroCommunities.length; i++) {
|
||||
macro = this.macroCommunities[i];
|
||||
leader = newLabels[macro[0]];
|
||||
|
||||
for (j = 1; j < macro.length; j++) {
|
||||
follower = newLabels[macro[j]];
|
||||
index.expensiveMove(follower, leader);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
UndirectedLeidenAddenda.prototype.onlySingletons = function () {
|
||||
var index = this.index;
|
||||
|
||||
var i;
|
||||
|
||||
for (i = 0; i < index.C; i++) {
|
||||
if (index.counts[i] > 1) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
exports.addWeightToCommunity = addWeightToCommunity;
|
||||
exports.UndirectedLeidenAddenda = UndirectedLeidenAddenda;
|
||||
Loading…
Add table
Reference in a new issue