From 0943df9feba462c69b344ce289d8099d41537bad Mon Sep 17 00:00:00 2001 From: abhigyanpatwari Date: Fri, 6 Feb 2026 06:35:15 +0530 Subject: [PATCH] gitnexus wal cleanup preventing reindexing issue fixed --- .claude/skills/gitnexus/debugging/SKILL.md | 1 + .claude/skills/gitnexus/exploring/SKILL.md | 1 + .../skills/gitnexus/impact-analysis/SKILL.md | 1 + .claude/skills/gitnexus/refactoring/SKILL.md | 1 + AGENTS.md | 8 +- CLAUDE.md | 8 +- README.md | 8 +- gitnexus-web/.gitignore | 2 + gitnexus-web/src/components/ActivityFeed.tsx | 204 -------- gitnexus-web/src/components/Header.tsx | 178 ------- .../components/IntelligentClusteringModal.tsx | 140 ------ gitnexus-web/src/components/MCPToggle.tsx | 300 ------------ gitnexus-web/src/components/RightPanel.tsx | 24 +- gitnexus-web/src/core/kuzu/csv-generator.ts | 11 +- gitnexus-web/src/core/kuzu/kuzu-adapter.ts | 93 ++-- gitnexus-web/src/core/kuzu/schema.ts | 150 +++--- gitnexus-web/src/core/mcp/mcp-client.ts | 341 -------------- gitnexus/package-lock.json | 103 ++--- gitnexus/package.json | 13 +- gitnexus/skills/debugging.md | 1 + gitnexus/skills/exploring.md | 1 + gitnexus/skills/impact-analysis.md | 1 + gitnexus/skills/refactoring.md | 1 + gitnexus/src/FTS_LIVE_TEST.ts | 13 - gitnexus/src/cli/analyze.ts | 32 +- gitnexus/src/cli/index.ts | 1 - gitnexus/src/cli/mcp.ts | 2 + gitnexus/src/core/graph/graph.ts | 35 +- gitnexus/src/core/graph/types.ts | 2 + gitnexus/src/core/ingestion/call-processor.ts | 3 +- .../src/core/ingestion/community-processor.ts | 22 +- .../src/core/ingestion/heritage-processor.ts | 3 +- .../src/core/ingestion/import-processor.ts | 3 +- .../src/core/ingestion/parsing-processor.ts | 5 +- gitnexus/src/core/ingestion/utils.ts | 6 + gitnexus/src/core/kuzu/csv-generator.ts | 16 +- gitnexus/src/core/kuzu/kuzu-adapter.ts | 436 ++++++++++++++++-- gitnexus/src/core/kuzu/schema.ts | 155 ++++--- gitnexus/src/core/search/bm25-index.ts | 223 ++------- gitnexus/src/core/search/hybrid-search.ts | 16 +- gitnexus/src/mcp/core/bm25-index.ts | 120 ----- gitnexus/src/mcp/core/kuzu-adapter.ts | 37 +- gitnexus/src/mcp/local/local-backend.ts | 178 ++++++- gitnexus/src/mcp/resources.ts | 51 +- gitnexus/src/mcp/staleness.ts | 43 ++ gitnexus/src/mcp/tools.ts | 10 +- gitnexus/src/server/api.ts | 19 +- gitnexus/src/storage/repo-manager.ts | 3 +- gitnexus/vendor/leiden/index.cjs | 355 ++++++++++++++ gitnexus/vendor/leiden/utils.cjs | 392 ++++++++++++++++ 50 files changed, 1855 insertions(+), 1917 deletions(-) create mode 100644 gitnexus-web/.gitignore delete mode 100644 gitnexus-web/src/components/ActivityFeed.tsx delete mode 100644 gitnexus-web/src/components/IntelligentClusteringModal.tsx delete mode 100644 gitnexus-web/src/components/MCPToggle.tsx delete mode 100644 gitnexus-web/src/core/mcp/mcp-client.ts delete mode 100644 gitnexus/src/FTS_LIVE_TEST.ts delete mode 100644 gitnexus/src/mcp/core/bm25-index.ts create mode 100644 gitnexus/src/mcp/staleness.ts create mode 100644 gitnexus/vendor/leiden/index.cjs create mode 100644 gitnexus/vendor/leiden/utils.cjs diff --git a/.claude/skills/gitnexus/debugging/SKILL.md b/.claude/skills/gitnexus/debugging/SKILL.md index 3d8dab1b78..883ff25b76 100644 --- a/.claude/skills/gitnexus/debugging/SKILL.md +++ b/.claude/skills/gitnexus/debugging/SKILL.md @@ -7,6 +7,7 @@ description: Trace bugs through call chains using knowledge graph ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. gitnexus_search({query}) → Find code related to error 2. gitnexus_explore({name, type: "symbol"}) → Get callers and callees 3. READ gitnexus://process/{name} → Trace execution flow diff --git a/.claude/skills/gitnexus/exploring/SKILL.md b/.claude/skills/gitnexus/exploring/SKILL.md index 2baa7f31ed..52beb6909d 100644 --- a/.claude/skills/gitnexus/exploring/SKILL.md +++ b/.claude/skills/gitnexus/exploring/SKILL.md @@ -7,6 +7,7 @@ description: Navigate unfamiliar code using GitNexus knowledge graph ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. READ gitnexus://context → Get codebase overview (~150 tokens) 2. READ gitnexus://clusters → See all functional clusters 3. READ gitnexus://cluster/{name} → Deep dive on specific cluster diff --git a/.claude/skills/gitnexus/impact-analysis/SKILL.md b/.claude/skills/gitnexus/impact-analysis/SKILL.md index 8f8db00847..dbbffc3124 100644 --- a/.claude/skills/gitnexus/impact-analysis/SKILL.md +++ b/.claude/skills/gitnexus/impact-analysis/SKILL.md @@ -7,6 +7,7 @@ description: Analyze blast radius before making code changes ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. gitnexus_impact({target, direction: "upstream"}) → What depends on this 2. READ gitnexus://clusters → Check affected areas 3. READ gitnexus://processes → Affected execution flows diff --git a/.claude/skills/gitnexus/refactoring/SKILL.md b/.claude/skills/gitnexus/refactoring/SKILL.md index 1513a2ed6c..1ea1c545b8 100644 --- a/.claude/skills/gitnexus/refactoring/SKILL.md +++ b/.claude/skills/gitnexus/refactoring/SKILL.md @@ -7,6 +7,7 @@ description: Plan safe refactors using blast radius and dependency mapping ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. gitnexus_impact({target, direction: "upstream"}) → Map all dependents 2. READ gitnexus://schema → Understand graph structure 3. gitnexus_cypher → Find all references diff --git a/AGENTS.md b/AGENTS.md index 662b1f9298..a4add62be9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,10 +13,10 @@ This project is indexed by GitNexus, providing AI agents with deep code intellig | Metric | Count | |--------|-------| -| Files | 150 | -| Symbols | 930 | -| Relationships | 2411 | -| Communities | 280 | +| Files | 146 | +| Symbols | 907 | +| Relationships | 2306 | +| Communities | 278 | | Processes | 75 | ## Quick Start diff --git a/CLAUDE.md b/CLAUDE.md index 69c4aed5a9..5ca1b0b2d8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,10 +7,10 @@ This project is indexed by GitNexus, providing AI agents with deep code intellig | Metric | Count | |--------|-------| -| Files | 150 | -| Symbols | 930 | -| Relationships | 2411 | -| Communities | 280 | +| Files | 146 | +| Symbols | 907 | +| Relationships | 2306 | +| Communities | 278 | | Processes | 75 | ## Quick Start diff --git a/README.md b/README.md index 6eb5f3a327..89eec7ecd1 100644 --- a/README.md +++ b/README.md @@ -521,7 +521,7 @@ ORDER BY distance | **Visualization** | Sigma.js, Graphology, ForceAtlas2 (WebGL) | | **Parsing** | Tree-sitter WASM (9 languages) | | **Database** | KuzuDB WASM (graph + vector HNSW) | -| **Clustering** | Graphology + Leiden (Louvain) | +| **Clustering** | Graphology + Leiden algorithm | | **Embeddings** | transformers.js, snowflake-arctic-embed-xs (22M) | | **AI** | LangChain ReAct agent, streaming | | **Concurrency** | Web Workers + Comlink | @@ -537,12 +537,6 @@ ORDER BY distance --- -## 📝 License - -MIT License - ---- - ## 🙏 Acknowledgments - [Tree-sitter](https://tree-sitter.github.io/) - AST parsing diff --git a/gitnexus-web/.gitignore b/gitnexus-web/.gitignore new file mode 100644 index 0000000000..c8a733615b --- /dev/null +++ b/gitnexus-web/.gitignore @@ -0,0 +1,2 @@ +.vercel +.env*.local diff --git a/gitnexus-web/src/components/ActivityFeed.tsx b/gitnexus-web/src/components/ActivityFeed.tsx deleted file mode 100644 index a802b859b6..0000000000 --- a/gitnexus-web/src/components/ActivityFeed.tsx +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Activity Feed Component - * - * Shows real-time log of external AI agent tool calls. - * Used in RightPanel as an alternative to the Chat tab. - */ - -import { useState, useEffect, useRef } from 'react'; -import { Activity, Search, Database, Terminal, Loader2, CheckCircle, XCircle, Clock, FileText, Zap, Map, Compass } from 'lucide-react'; -import { getMCPClient, type ActivityEvent } from '../core/mcp/mcp-client'; - -// Tool icons -const TOOL_ICONS: Record = { - context: Zap, - search: Search, - cypher: Database, - grep: Terminal, - read: FileText, - impact: Activity, - overview: Map, - explore: Compass, -}; - -// Tool colors -const TOOL_COLORS: Record = { - context: 'text-amber-400', - search: 'text-cyan-400', - cypher: 'text-purple-400', - grep: 'text-green-400', - read: 'text-blue-400', - impact: 'text-rose-400', - overview: 'text-indigo-400', - explore: 'text-teal-400', -}; - -export function ActivityFeed() { - const [events, setEvents] = useState([]); - const containerRef = useRef(null); - - useEffect(() => { - const client = getMCPClient(); - - // Subscribe to activity events - const unsubscribe = client.onActivity((event) => { - setEvents(prev => { - // Keep max 100 events - const next = [...prev, event]; - if (next.length > 100) { - next.shift(); - } - return next; - }); - }); - - // Get existing events - setEvents(client.getActivityLog()); - - return () => { - unsubscribe(); - }; - }, []); - - // Auto-scroll to bottom - useEffect(() => { - if (containerRef.current) { - containerRef.current.scrollTop = containerRef.current.scrollHeight; - } - }, [events]); - - const formatTime = (timestamp: number) => { - const date = new Date(timestamp); - return date.toLocaleTimeString('en-US', { - hour: '2-digit', - minute: '2-digit', - second: '2-digit', - }); - }; - - const formatParams = (params: any): string => { - if (!params) return ''; - // Show first key-value pairs, truncated - const entries = Object.entries(params).slice(0, 2); - return entries.map(([k, v]) => { - const val = typeof v === 'string' ? v.slice(0, 30) : JSON.stringify(v).slice(0, 30); - return `${k}: ${val}${val.length >= 30 ? '...' : ''}`; - }).join(', '); - }; - - const formatResult = (event: ActivityEvent): string => { - if (event.status === 'running') return 'Running...'; - if (event.status === 'error') return `Error: ${event.error?.slice(0, 50) || 'Unknown'}`; - - // Format result based on type - if (Array.isArray(event.result)) { - return `${event.result.length} results`; - } - if (typeof event.result === 'object' && event.result) { - const keys = Object.keys(event.result); - if (keys.includes('content')) return `${event.result.content?.length || 0} chars`; - if (keys.includes('projectName')) return event.result.projectName; - return `{${keys.slice(0, 3).join(', ')}${keys.length > 3 ? '...' : ''}}`; - } - return String(event.result || 'Done'); - }; - - if (events.length === 0) { - return ( -
-
- 📡 -
-

- No Agent Activity -

-

- When external AI agents (Cursor, Claude Code) call GitNexus tools, - their activity will appear here in real-time. -

-

- Make sure MCP toggle is enabled in the header -

-
- ); - } - - return ( -
-
- {events.map((event) => { - const Icon = TOOL_ICONS[event.tool] || Activity; - const color = TOOL_COLORS[event.tool] || 'text-text-muted'; - - return ( -
- {/* Header row */} -
- {/* Agent color indicator */} - {event.agentColor && ( -
- )} - - {event.tool} - {event.agentName && event.agentName !== 'Unknown' && ( - - {event.agentName} - - )} - - - {formatTime(event.timestamp)} - -
- - {/* Params preview */} - {event.params && Object.keys(event.params).length > 0 && ( -
- {formatParams(event.params)} -
- )} - - {/* Status/Result */} -
- {event.status === 'running' && ( - <> - - Running... - - )} - {event.status === 'complete' && ( - <> - - {formatResult(event)} - {event.duration && ( - {event.duration}ms - )} - - )} - {event.status === 'error' && ( - <> - - {formatResult(event)} - - )} -
-
- ); - })} -
-
- ); -} diff --git a/gitnexus-web/src/components/Header.tsx b/gitnexus-web/src/components/Header.tsx index c1926842aa..2fa3659fb1 100644 --- a/gitnexus-web/src/components/Header.tsx +++ b/gitnexus-web/src/components/Header.tsx @@ -3,8 +3,6 @@ import { useAppState } from '../hooks/useAppState'; import { useState, useMemo, useRef, useEffect, useCallback } from 'react'; import { GraphNode } from '../core/graph/types'; import { EmbeddingStatus } from './EmbeddingStatus'; -import { MCPToggle } from './MCPToggle'; -import { buildCodebaseContext } from '../core/llm/context-builder'; // Color mapping for node types in search results const NODE_TYPE_COLORS: Record = { @@ -31,11 +29,6 @@ export const Header = ({ onFocusNode }: HeaderProps) => { isRightPanelOpen, rightPanelTab, setSettingsPanelOpen, - runQuery, - semanticSearch, - setHighlightedNodeIds, - fileContents, - triggerNodeAnimation, } = useAppState(); const [searchQuery, setSearchQuery] = useState(''); const [isSearchOpen, setIsSearchOpen] = useState(false); @@ -220,177 +213,6 @@ export const Header = ({ onFocusNode }: HeaderProps) => { {/* Embedding Status */} - {/* MCP Toggle for external AI agents */} - { - // Use semantic search from the app - const results = await semanticSearch(query, limit); - // Trigger pulse animation on search results - const nodeIds = results.map((r: any) => r.id).filter(Boolean); - if (nodeIds.length > 0) { - triggerNodeAnimation(nodeIds, 'pulse'); - } - return results; - }} - onCypher={async (query) => { - // Execute Cypher query - const results = await runQuery(query); - return results; - }} - onImpact={async (nodeId: string, hops = 2) => { - // Run impact analysis query - const query = ` - MATCH (start)-[*1..${hops}]-(connected) - WHERE start.id = '${nodeId}' OR start.name = '${nodeId}' - RETURN DISTINCT connected.id AS id, connected.name AS name, labels(connected) AS labels - `; - const results = await runQuery(query); - // Trigger ripple animation on impact results - const nodeIds = results.map((r: any) => r.id).filter(Boolean); - if (nodeIds.length > 0) { - triggerNodeAnimation(nodeIds, 'ripple'); - } - return results; - }} - onOverview={async () => { - // Return codebase overview: clusters + processes - const clustersQuery = ` - MATCH (c:Community) - OPTIONAL MATCH (c)<-[:CodeRelation {type: 'MEMBER_OF'}]-(m) - RETURN c.id AS id, c.label AS label, c.cohesion AS cohesion, c.description AS description, count(m) AS memberCount - ORDER BY memberCount DESC - LIMIT 50 - `; - const processesQuery = ` - MATCH (p:Process) - RETURN p.id AS id, p.label AS label, p.processType AS type, p.stepCount AS steps - ORDER BY p.stepCount DESC - LIMIT 50 - `; - const [clusters, processes] = await Promise.all([ - runQuery(clustersQuery), - runQuery(processesQuery), - ]); - return { clusters, processes }; - }} - onExplore={async (target: string, type?: 'symbol' | 'cluster' | 'process') => { - // Explore a specific target - if (type === 'cluster' || target.startsWith('comm_')) { - const query = ` - MATCH (c:Community) - WHERE c.id = '${target}' OR c.label CONTAINS '${target}' - OPTIONAL MATCH (c)<-[:CodeRelation {type: 'MEMBER_OF'}]-(m) - RETURN c.id AS id, c.label AS label, c.description AS description, collect(m.name)[0..10] AS members - LIMIT 1 - `; - return await runQuery(query); - } else if (type === 'process' || target.startsWith('proc_')) { - const query = ` - MATCH (p:Process) - WHERE p.id = '${target}' OR p.label CONTAINS '${target}' - OPTIONAL MATCH (s)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p) - RETURN p.id AS id, p.label AS label, p.stepCount AS steps, collect({name: s.name, step: r.step})[0..20] AS trace - LIMIT 1 - `; - return await runQuery(query); - } else { - // Symbol exploration - const query = ` - MATCH (n) - WHERE n.name = '${target}' OR n.id ENDS WITH ':${target}' - OPTIONAL MATCH (n)-[:CodeRelation {type: 'MEMBER_OF'}]->(c:Community) - OPTIONAL MATCH (n)-[r:CodeRelation {type: 'STEP_IN_PROCESS'}]->(p:Process) - RETURN n.id AS id, n.name AS name, n.filePath AS filePath, label(n) AS nodeType, - c.label AS cluster, collect({process: p.label, step: r.step}) AS processes - LIMIT 1 - `; - return await runQuery(query); - } - }} - getContext={async () => { - // Build codebase context for external AI agents - if (!projectName) return null; - const context = await buildCodebaseContext(runQuery, projectName); - // Reshape to match MCP CodebaseContext format - return { - projectName: context.stats.projectName, - stats: { - fileCount: context.stats.fileCount, - functionCount: context.stats.functionCount, - classCount: context.stats.classCount, - interfaceCount: context.stats.interfaceCount, - methodCount: context.stats.methodCount, - }, - hotspots: context.hotspots, - folderTree: context.folderTree, - }; - }} - onGrep={async (pattern, caseSensitive = false, maxResults = 50) => { - // Grep across file contents - const results: Array<{ filePath: string; line: string; lineNumber: number; match: string }> = []; - const regex = new RegExp(pattern, caseSensitive ? 'g' : 'gi'); - - for (const [filePath, content] of fileContents.entries()) { - const lines = content.split('\n'); - for (let i = 0; i < lines.length && results.length < maxResults; i++) { - const line = lines[i]; - const match = line.match(regex); - if (match) { - results.push({ - filePath, - line: line.trim(), - lineNumber: i + 1, - match: match[0], - }); - } - } - if (results.length >= maxResults) break; - } - return results; - }} - onRead={async (filePath, startLine, endLine) => { - // Read file content - let content = fileContents.get(filePath); - - // Try normalized path if not found - if (!content) { - const normalizedPath = filePath.replace(/\\/g, '/'); - for (const [path, c] of fileContents.entries()) { - if (path.endsWith(normalizedPath) || normalizedPath.endsWith(path)) { - content = c; - break; - } - } - } - - if (!content) { - return { error: `File not found: ${filePath}` }; - } - - const lines = content.split('\n'); - const language = filePath.split('.').pop() || 'text'; - - // If line range specified, return only those lines - if (startLine !== undefined && endLine !== undefined) { - const slice = lines.slice(startLine - 1, endLine); - return { - filePath, - content: slice.join('\n'), - language, - lines: slice.length, - }; - } - - return { - filePath, - content, - language, - lines: lines.length, - }; - }} - /> - {/* Icon buttons */} - -
-
- -
-
- -

- Upgrade to Intelligent Clustering? -

-

- Your clusters are ready, but they could be smarter! Right now they're just named after folders. -

-
- - {/* Body */} -
- -
-

- - What you get: -

-
    -
  • - ✓ - Semantic names (e.g., "Auth System" vs "utils") -
    • -
  • - ✓ - Search keywords for better agent context -
    • -
  • - ✓ - Descriptions of what the code actually does -
    • -
-
- - {/* How it works */} -
-
-
- -
-
-

Uses Your Configured LLM

-

- Runs on your own API key. Very low token usage (~$0.01 for most codebases). -
- 💡 Tip: Use a cheaper model like GPT-4o-mini in settings! -

-
-
-
- -
- - {/* Actions */} -
- - -
- -
- -
-
- -
- - ); -}; diff --git a/gitnexus-web/src/components/MCPToggle.tsx b/gitnexus-web/src/components/MCPToggle.tsx deleted file mode 100644 index aa05cd04df..0000000000 --- a/gitnexus-web/src/components/MCPToggle.tsx +++ /dev/null @@ -1,300 +0,0 @@ -/** - * MCP Toggle Component - * - * Toggle for enabling MCP exposure to external AI agents (Cursor, Claude, etc.) - * Shows MCP config for setup and connection status. - */ - -import { useState, useEffect, useCallback, useRef } from 'react'; -import { Copy, Check, X, Sparkles, Zap, ExternalLink } from 'lucide-react'; -import { getMCPClient, type CodebaseContext } from '../core/mcp/mcp-client'; - -type ConnectionState = 'disconnected' | 'connecting' | 'connected' | 'error'; - -interface MCPToggleProps { - onSearch?: (query: string, limit?: number) => Promise; - onCypher?: (query: string) => Promise; - onImpact?: (nodeId: string, hops?: number) => Promise; - onGrep?: (pattern: string, caseSensitive?: boolean, maxResults?: number) => Promise; - onRead?: (filePath: string, startLine?: number, endLine?: number) => Promise; - onOverview?: () => Promise; - onExplore?: (target: string, type?: 'symbol' | 'cluster' | 'process') => Promise; - showOnboardingTip?: boolean; - getContext?: () => Promise; -} - -const MCP_TIP_DISMISSED_KEY = 'gitnexus-mcp-tip-dismissed'; - -// MCP config that users copy to their AI agent -const MCP_CONFIG = `{ - "mcpServers": { - "gitnexus": { - "command": "npx", - "args": ["--prefer-online", "-y", "gitnexus-mcp@latest"] - } - } -}`; - -export function MCPToggle({ - onSearch, - onCypher, - onImpact, - onGrep, - onRead, - onOverview, - onExplore, - showOnboardingTip = false, - getContext, -}: MCPToggleProps = {}) { - const [status, setStatus] = useState('disconnected'); - const [copied, setCopied] = useState(false); - const [showPopup, setShowPopup] = useState(false); - const popupRef = useRef(null); - const [showTip, setShowTip] = useState(false); - - const isConnected = status === 'connected'; - const isConnecting = status === 'connecting'; - - // Show tip when graph becomes ready - useEffect(() => { - if (showOnboardingTip) { - const dismissed = localStorage.getItem(MCP_TIP_DISMISSED_KEY); - if (!dismissed) { - const timer = setTimeout(() => setShowTip(true), 1500); - return () => clearTimeout(timer); - } - } - }, [showOnboardingTip]); - - // Close popup when clicking outside - useEffect(() => { - if (!showPopup) return; - const handleClickOutside = (event: MouseEvent) => { - if (popupRef.current && !popupRef.current.contains(event.target as Node)) { - setShowPopup(false); - } - }; - document.addEventListener('mousedown', handleClickOutside); - return () => document.removeEventListener('mousedown', handleClickOutside); - }, [showPopup]); - - const dismissTip = () => { - setShowTip(false); - localStorage.setItem(MCP_TIP_DISMISSED_KEY, 'true'); - }; - - const connect = useCallback(async () => { - const client = getMCPClient(); - setStatus('connecting'); - setShowTip(false); - - try { - await client.connect(); - - // Register tool handlers - if (onSearch) client.registerHandler('search', async (params) => onSearch(params.query, params.limit)); - if (onCypher) client.registerHandler('cypher', async (params) => onCypher(params.query)); - if (onImpact) client.registerHandler('impact', async (params) => onImpact(params.nodeId, params.hops)); - if (onGrep) client.registerHandler('grep', async (params) => onGrep(params.pattern, params.caseSensitive, params.maxResults)); - if (onRead) client.registerHandler('read', async (params) => onRead(params.filePath, params.startLine, params.endLine)); - if (onOverview) client.registerHandler('overview', async () => onOverview()); - if (onExplore) client.registerHandler('explore', async (params) => onExplore(params.target, params.type)); - if (getContext) client.registerHandler('context', async () => getContext()); - - setStatus('connected'); - setShowPopup(false); - localStorage.setItem(MCP_TIP_DISMISSED_KEY, 'true'); - - // Send context after connecting - if (getContext) { - try { - const context = await getContext(); - if (context) client.sendContext(context); - } catch (e) { - console.error('[MCP] Failed to send context:', e); - } - } - } catch { - setStatus('error'); - } - }, [onSearch, onCypher, onImpact, onGrep, onRead, onOverview, onExplore, getContext]); - - const disconnect = useCallback(() => { - const client = getMCPClient(); - client.disconnect(); - setStatus('disconnected'); - }, []); - - const toggle = useCallback(() => { - if (isConnected) { - disconnect(); - } else if (!isConnecting) { - connect(); - } - }, [isConnected, isConnecting, connect, disconnect]); - - const copyConfig = () => { - navigator.clipboard.writeText(MCP_CONFIG); - setCopied(true); - setTimeout(() => setCopied(false), 2000); - }; - - // Listen for connection changes - useEffect(() => { - const client = getMCPClient(); - const unsubscribe = client.onConnectionChange((connected) => { - setStatus(connected ? 'connected' : 'disconnected'); - if (connected) setShowPopup(false); - }); - return () => { unsubscribe(); }; - }, []); - - return ( -
- {/* MCP Button */} - - - {/* Popup */} - {showPopup && ( -
- {/* Header */} -
-
-
-
- -
-
-

Connect AI Agents

-

Cursor, Claude Code, Antigravity

-
-
- -
-
- - {/* Content */} -
- {/* Step 1: Config */} -
-
- 1 - Add to your AI agent's MCP config -
-
- 
-                                    {MCP_CONFIG}
-
- -
-
- - {/* Step 2: Connect */} -
-
- 2 - Connect browser to daemon -
- -
- - {/* Status message */} - {status === 'error' && ( -

- Daemon not running. Make sure your AI agent has started gitnexus-mcp. -

- )} - - {/* Help link */} - - Learn more - - -
-
- )} - - {/* Onboarding Tip */} - {showTip && !isConnected && !showPopup && ( -
- -
-
- -
-
-

- Connect your AI tools -

-

- Let Cursor or Claude access GitNexus code intelligence. -

- -
-
-
- )} -
- ); -} diff --git a/gitnexus-web/src/components/RightPanel.tsx b/gitnexus-web/src/components/RightPanel.tsx index 0c3a1e2b3d..1f27fc9174 100644 --- a/gitnexus-web/src/components/RightPanel.tsx +++ b/gitnexus-web/src/components/RightPanel.tsx @@ -1,12 +1,11 @@ import { useState, useRef, useEffect, useCallback } from 'react'; import { Send, Square, Sparkles, User, - PanelRightClose, Loader2, AlertTriangle, Activity, GitBranch + PanelRightClose, Loader2, AlertTriangle, GitBranch } from 'lucide-react'; import { useAppState } from '../hooks/useAppState'; import { ToolCallCard } from './ToolCallCard'; import { isProviderConfigured } from '../core/llm/settings-service'; -import { ActivityFeed } from './ActivityFeed'; import { MarkdownRenderer } from './MarkdownRenderer'; import { ProcessesPanel } from './ProcessesPanel'; export const RightPanel = () => { @@ -29,7 +28,7 @@ export const RightPanel = () => { } = useAppState(); const [chatInput, setChatInput] = useState(''); - const [activeTab, setActiveTab] = useState<'chat' | 'activity' | 'processes'>('chat'); + const [activeTab, setActiveTab] = useState<'chat' | 'processes'>('chat'); const textareaRef = useRef(null); const messagesEndRef = useRef(null); @@ -225,18 +224,6 @@ export const RightPanel = () => { Nexus AI - {/* Activity Tab */} - - {/* Processes Tab */} - {/* Activity Feed Tab */} - {activeTab === 'activity' && ( -
- -
- )} - {/* Processes Tab */} {activeTab === 'processes' && (
diff --git a/gitnexus-web/src/core/kuzu/csv-generator.ts b/gitnexus-web/src/core/kuzu/csv-generator.ts index a07ea8ccd5..e565399cd5 100644 --- a/gitnexus-web/src/core/kuzu/csv-generator.ts +++ b/gitnexus-web/src/core/kuzu/csv-generator.ts @@ -18,12 +18,17 @@ import { NODE_TABLES, NodeTableName } from './schema'; // ============================================================================ /** - * Sanitize string to ensure valid UTF-8 - * Removes or replaces invalid characters that would break CSV parsing + * Sanitize string to ensure valid UTF-8 and safe CSV content for KuzuDB + * Removes or replaces invalid characters that would break CSV parsing. + * + * Critical: KuzuDB's CSV parser can misinterpret \r\n inside quoted fields. + * We normalize all line endings to \n only. */ const sanitizeUTF8 = (str: string): string => { return str - .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n \r + .replace(/\r\n/g, '\n') // Normalize Windows line endings first + .replace(/\r/g, '\n') // Normalize remaining \r to \n + .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n .replace(/[\uD800-\uDFFF]/g, '') // Remove surrogate pairs (invalid standalone) .replace(/[\uFFFE\uFFFF]/g, ''); // Remove BOM and special chars }; diff --git a/gitnexus-web/src/core/kuzu/kuzu-adapter.ts b/gitnexus-web/src/core/kuzu/kuzu-adapter.ts index 92b8cd5c65..52885b281a 100644 --- a/gitnexus-web/src/core/kuzu/kuzu-adapter.ts +++ b/gitnexus-web/src/core/kuzu/kuzu-adapter.ts @@ -113,70 +113,61 @@ export const loadGraphToKuzu = async ( } // 5. INSERT relations one by one (COPY doesn't work with multi-pair REL tables) - // Parse CSV format: "from","to","type",confidence,"reason" + // Build a set of valid table names for fast lookup + const validTables = new Set(NODE_TABLES as readonly string[]); + + const getNodeLabel = (nodeId: string): string => { + if (nodeId.startsWith('comm_')) return 'Community'; + if (nodeId.startsWith('proc_')) return 'Process'; + return nodeId.split(':')[0]; + }; + + // All multi-language tables are created with backticks - must always reference them with backticks + const escapeLabel = (label: string): string => { + return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label; + }; + let insertedRels = 0; let skippedRels = 0; const skippedRelStats = new Map(); for (const line of relLines) { try { - // Parse CSV - handle quoted fields and numeric confidence - // Parse CSV - handle quoted fields and numeric confidence // Format: "from","to","type",confidence,"reason",step - // Note: step is unquoted numeric const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/); if (!match) continue; const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match; + + const fromLabel = getNodeLabel(fromId); + const toLabel = getNodeLabel(toId); + + // Skip relationships where either node's label doesn't have a table in KuzuDB + // Querying a non-existent table causes a fatal native crash + if (!validTables.has(fromLabel) || !validTables.has(toLabel)) { + skippedRels++; + continue; + } + const confidence = parseFloat(confidenceStr) || 1.0; const step = parseInt(stepStr) || 0; - // Extract labels from node IDs - // Community nodes have IDs like "comm_14" (no colon) - // Other nodes have IDs like "Label:path:name" - const getNodeLabel = (nodeId: string): string => { - if (nodeId.startsWith('comm_')) { - return 'Community'; - } - if (nodeId.startsWith('proc_')) { - return 'Process'; - } - return nodeId.split(':')[0]; - }; - - // Reserved Cypher keywords need backtick escaping - const RESERVED_LABELS = ['Macro', 'Enum', 'Union', 'Const', 'Module', 'Struct']; - const escapeLabel = (label: string): string => { - return RESERVED_LABELS.includes(label) ? `\`${label}\`` : label; - }; - - const fromLabel = escapeLabel(getNodeLabel(fromId)); - const toLabel = escapeLabel(getNodeLabel(toId)); - - // INSERT with explicit node matching (including confidence and reason) const insertQuery = ` - MATCH (a:${fromLabel} {id: '${fromId.replace(/'/g, "''")}'}), - (b:${toLabel} {id: '${toId.replace(/'/g, "''")}'}) + MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}'}), + (b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}'}) CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b) `; await conn.query(insertQuery); insertedRels++; } catch (err) { - // Skip failed insertions (nodes might not exist, or relation pair not allowed by schema) skippedRels++; const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)"/); if (match) { const [, fromId, toId, relType] = match; - const getNodeLabel = (nodeId: string): string => { - if (nodeId.startsWith('comm_')) return 'Community'; - if (nodeId.startsWith('proc_')) return 'Process'; - return nodeId.split(':')[0]; - }; const fromLabel = getNodeLabel(fromId); const toLabel = getNodeLabel(toId); const key = `${relType}:${fromLabel}->` + toLabel; skippedRelStats.set(key, (skippedRelStats.get(key) || 0) + 1); - // Log each skipped relation if (import.meta.env.DEV) { console.warn(`⚠️ Skipped: ${key} | "${fromId}" → "${toId}" | ${err instanceof Error ? err.message : String(err)}`); } @@ -222,25 +213,41 @@ export const loadGraphToKuzu = async ( } }; +// KuzuDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes). +// Source code content is full of backslashes which confuse the auto-detection. +// We MUST explicitly set ESCAPE='"' and disable auto_detect. +const COPY_CSV_OPTS = `(HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`; + +// Multi-language table names created with backticks in CODE_ELEMENT_BASE +const BACKTICK_TABLES = new Set([ + 'Struct', 'Enum', 'Macro', 'Typedef', 'Union', 'Namespace', 'Trait', 'Impl', + 'TypeAlias', 'Const', 'Static', 'Property', 'Record', 'Delegate', 'Annotation', + 'Constructor', 'Template', 'Module', +]); + +const escapeTableName = (table: string): string => { + return BACKTICK_TABLES.has(table) ? `\`${table}\`` : table; +}; + /** * Get the COPY query for a node table with correct column mapping */ const getCopyQuery = (table: NodeTableName, path: string): string => { - // File and Folder have different columns than code elements + const t = escapeTableName(table); if (table === 'File') { - return `COPY File(id, name, filePath, content) FROM "${path}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, name, filePath, content) FROM "${path}" ${COPY_CSV_OPTS}`; } if (table === 'Folder') { - return `COPY Folder(id, name, filePath) FROM "${path}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, name, filePath) FROM "${path}" ${COPY_CSV_OPTS}`; } if (table === 'Community') { - return `COPY Community(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${path}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${path}" ${COPY_CSV_OPTS}`; } if (table === 'Process') { - return `COPY Process(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${path}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${path}" ${COPY_CSV_OPTS}`; } - // All code element tables: Function, Class, Interface, Method, CodeElement - return `COPY ${table}(id, name, filePath, startLine, endLine, isExported, content) FROM "${path}" (HEADER=true, PARALLEL=false)`; + // Code element tables (Function, Class, Interface, Method, CodeElement, and multi-language) + return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content) FROM "${path}" ${COPY_CSV_OPTS}`; }; /** diff --git a/gitnexus-web/src/core/kuzu/schema.ts b/gitnexus-web/src/core/kuzu/schema.ts index ccf4ea5358..d74545c069 100644 --- a/gitnexus-web/src/core/kuzu/schema.ts +++ b/gitnexus-web/src/core/kuzu/schema.ts @@ -201,20 +201,20 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM File TO \`Struct\`, FROM File TO \`Enum\`, FROM File TO \`Macro\`, - FROM File TO Typedef, + FROM File TO \`Typedef\`, FROM File TO \`Union\`, - FROM File TO Namespace, - FROM File TO Trait, - FROM File TO Impl, - FROM File TO TypeAlias, + FROM File TO \`Namespace\`, + FROM File TO \`Trait\`, + FROM File TO \`Impl\`, + FROM File TO \`TypeAlias\`, FROM File TO \`Const\`, - FROM File TO Static, - FROM File TO Property, - FROM File TO Record, - FROM File TO Delegate, - FROM File TO Annotation, - FROM File TO Constructor, - FROM File TO Template, + FROM File TO \`Static\`, + FROM File TO \`Property\`, + FROM File TO \`Record\`, + FROM File TO \`Delegate\`, + FROM File TO \`Annotation\`, + FROM File TO \`Constructor\`, + FROM File TO \`Template\`, FROM File TO \`Module\`, FROM Folder TO Folder, FROM Folder TO File, @@ -224,48 +224,48 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM Function TO Community, FROM Function TO \`Macro\`, FROM Function TO \`Struct\`, - FROM Function TO Template, + FROM Function TO \`Template\`, FROM Function TO \`Enum\`, - FROM Function TO Namespace, - FROM Function TO TypeAlias, + FROM Function TO \`Namespace\`, + FROM Function TO \`TypeAlias\`, FROM Function TO \`Module\`, - FROM Function TO Impl, + FROM Function TO \`Impl\`, FROM Function TO Interface, - FROM Function TO Constructor, + FROM Function TO \`Constructor\`, FROM Class TO Method, FROM Class TO Function, FROM Class TO Class, FROM Class TO Interface, FROM Class TO Community, - FROM Class TO Template, - FROM Class TO TypeAlias, + FROM Class TO \`Template\`, + FROM Class TO \`TypeAlias\`, FROM Class TO \`Struct\`, FROM Class TO \`Enum\`, - FROM Class TO Constructor, + FROM Class TO \`Constructor\`, FROM Method TO Function, FROM Method TO Method, FROM Method TO Class, FROM Method TO Community, - FROM Method TO Template, + FROM Method TO \`Template\`, FROM Method TO \`Struct\`, - FROM Method TO TypeAlias, + FROM Method TO \`TypeAlias\`, FROM Method TO \`Enum\`, FROM Method TO \`Macro\`, - FROM Method TO Namespace, + FROM Method TO \`Namespace\`, FROM Method TO \`Module\`, - FROM Method TO Impl, + FROM Method TO \`Impl\`, FROM Method TO Interface, - FROM Method TO Constructor, - FROM Template TO Template, - FROM Template TO Function, - FROM Template TO Method, - FROM Template TO Class, - FROM Template TO \`Struct\`, - FROM Template TO TypeAlias, - FROM Template TO \`Enum\`, - FROM Template TO \`Macro\`, - FROM Template TO Interface, - FROM Template TO Constructor, + FROM Method TO \`Constructor\`, + FROM \`Template\` TO \`Template\`, + FROM \`Template\` TO Function, + FROM \`Template\` TO Method, + FROM \`Template\` TO Class, + FROM \`Template\` TO \`Struct\`, + FROM \`Template\` TO \`TypeAlias\`, + FROM \`Template\` TO \`Enum\`, + FROM \`Template\` TO \`Macro\`, + FROM \`Template\` TO Interface, + FROM \`Template\` TO \`Constructor\`, FROM \`Module\` TO \`Module\`, FROM CodeElement TO Community, FROM Interface TO Community, @@ -273,11 +273,11 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM Interface TO Method, FROM Interface TO Class, FROM Interface TO Interface, - FROM Interface TO TypeAlias, + FROM Interface TO \`TypeAlias\`, FROM Interface TO \`Struct\`, - FROM Interface TO Constructor, + FROM Interface TO \`Constructor\`, FROM \`Struct\` TO Community, - FROM \`Struct\` TO Trait, + FROM \`Struct\` TO \`Trait\`, FROM \`Struct\` TO Function, FROM \`Struct\` TO Method, FROM \`Enum\` TO Community, @@ -286,56 +286,56 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM \`Macro\` TO Method, FROM \`Module\` TO Function, FROM \`Module\` TO Method, - FROM Typedef TO Community, + FROM \`Typedef\` TO Community, FROM \`Union\` TO Community, - FROM Namespace TO Community, - FROM Trait TO Community, - FROM Impl TO Community, - FROM Impl TO Trait, - FROM TypeAlias TO Community, + FROM \`Namespace\` TO Community, + FROM \`Trait\` TO Community, + FROM \`Impl\` TO Community, + FROM \`Impl\` TO \`Trait\`, + FROM \`TypeAlias\` TO Community, FROM \`Const\` TO Community, - FROM Static TO Community, - FROM Property TO Community, - FROM Record TO Community, - FROM Delegate TO Community, - FROM Annotation TO Community, - FROM Constructor TO Community, - FROM Constructor TO Interface, - FROM Constructor TO Class, - FROM Constructor TO Method, - FROM Constructor TO Function, - FROM Constructor TO Constructor, - FROM Constructor TO \`Struct\`, - FROM Constructor TO \`Macro\`, - FROM Constructor TO Template, - FROM Constructor TO TypeAlias, - FROM Constructor TO \`Enum\`, - FROM Constructor TO Impl, - FROM Constructor TO Namespace, - FROM Template TO Community, + FROM \`Static\` TO Community, + FROM \`Property\` TO Community, + FROM \`Record\` TO Community, + FROM \`Delegate\` TO Community, + FROM \`Annotation\` TO Community, + FROM \`Constructor\` TO Community, + FROM \`Constructor\` TO Interface, + FROM \`Constructor\` TO Class, + FROM \`Constructor\` TO Method, + FROM \`Constructor\` TO Function, + FROM \`Constructor\` TO \`Constructor\`, + FROM \`Constructor\` TO \`Struct\`, + FROM \`Constructor\` TO \`Macro\`, + FROM \`Constructor\` TO \`Template\`, + FROM \`Constructor\` TO \`TypeAlias\`, + FROM \`Constructor\` TO \`Enum\`, + FROM \`Constructor\` TO \`Impl\`, + FROM \`Constructor\` TO \`Namespace\`, + FROM \`Template\` TO Community, FROM \`Module\` TO Community, FROM Function TO Process, FROM Method TO Process, FROM Class TO Process, FROM Interface TO Process, FROM \`Struct\` TO Process, - FROM Constructor TO Process, + FROM \`Constructor\` TO Process, FROM \`Module\` TO Process, FROM \`Macro\` TO Process, - FROM Impl TO Process, - FROM Typedef TO Process, - FROM TypeAlias TO Process, + FROM \`Impl\` TO Process, + FROM \`Typedef\` TO Process, + FROM \`TypeAlias\` TO Process, FROM \`Enum\` TO Process, FROM \`Union\` TO Process, - FROM Namespace TO Process, - FROM Trait TO Process, + FROM \`Namespace\` TO Process, + FROM \`Trait\` TO Process, FROM \`Const\` TO Process, - FROM Static TO Process, - FROM Property TO Process, - FROM Record TO Process, - FROM Delegate TO Process, - FROM Annotation TO Process, - FROM Template TO Process, + FROM \`Static\` TO Process, + FROM \`Property\` TO Process, + FROM \`Record\` TO Process, + FROM \`Delegate\` TO Process, + FROM \`Annotation\` TO Process, + FROM \`Template\` TO Process, FROM CodeElement TO Process, type STRING, confidence DOUBLE, diff --git a/gitnexus-web/src/core/mcp/mcp-client.ts b/gitnexus-web/src/core/mcp/mcp-client.ts deleted file mode 100644 index a546331f9b..0000000000 --- a/gitnexus-web/src/core/mcp/mcp-client.ts +++ /dev/null @@ -1,341 +0,0 @@ -/** - * MCP Browser Client - * - * WebSocket client that connects to the gitnexus-mcp daemon. - * - Sends codebase context (stats, hotspots, folder tree) on connect - * - Receives tool calls from external AI agents and executes them - * - Emits activity events for real-time monitoring - */ - -/** - * Agent color mapping for multi-agent support - */ -export const AGENT_COLORS: Record = { - 'cursor': '#a855f7', // Purple - 'claude': '#3b82f6', // Blue - 'claude-code': '#3b82f6', // Blue - 'windsurf': '#22c55e', // Green - 'codeium': '#22c55e', // Green (Windsurf) - 'unknown': '#6b7280', // Gray -}; - -export function getAgentColor(agentName: string): string { - const normalized = agentName.toLowerCase().trim(); - for (const [key, color] of Object.entries(AGENT_COLORS)) { - if (normalized.includes(key)) { - return color; - } - } - return AGENT_COLORS.unknown; -} - -export interface ConnectedAgent { - name: string; - color: string; -} - -export interface MCPMessage { - id: string; - type?: 'context' | 'tool_call' | 'tool_result' | 'agent_info'; - method?: string; - params?: Record; - result?: any; - error?: { message: string }; - agentName?: string; -} - -/** - * Codebase context to send to daemon - */ -export interface CodebaseContext { - projectName: string; - stats: { - fileCount: number; - functionCount: number; - classCount: number; - interfaceCount: number; - methodCount: number; - }; - hotspots: Array<{ - name: string; - type: string; - filePath: string; - connections: number; - }>; - folderTree: string; -} - -/** - * Activity event for real-time monitoring - */ -export interface ActivityEvent { - id: string; - tool: string; - params: Record; - status: 'running' | 'complete' | 'error'; - result?: any; - error?: string; - timestamp: number; - duration?: number; - agentName?: string; - agentColor?: string; -} - -type ToolHandler = (params: Record) => Promise; -type ActivityListener = (event: ActivityEvent) => void; - -export class MCPBrowserClient { - private ws: WebSocket | null = null; - private handlers: Map = new Map(); - private connectionListeners: Set<(connected: boolean) => void> = new Set(); - private activityListeners: Set = new Set(); - private activityLog: ActivityEvent[] = []; - private pendingContext: CodebaseContext | null = null; - private _connectedAgent: ConnectedAgent | null = null; - - constructor(private port = 54319) {} - - /** - * Connect to the MCP daemon - */ - async connect(): Promise { - return new Promise((resolve, reject) => { - try { - this.ws = new WebSocket(`ws://localhost:${this.port}`); - - this.ws.onopen = () => { - console.log('[MCP] Connected to daemon'); - this.notifyConnectionListeners(true); - - // Send pending context if available - if (this.pendingContext) { - this.sendContext(this.pendingContext); - } - - resolve(); - }; - - this.ws.onerror = () => { - this.notifyConnectionListeners(false); - reject(new Error('Failed to connect to MCP bridge')); - }; - - this.ws.onmessage = (event) => { - try { - const msg: MCPMessage = JSON.parse(event.data); - this.handleMessage(msg); - } catch (error) { - console.error('[MCP] Failed to parse message:', error); - } - }; - - this.ws.onclose = () => { - this.ws = null; - this.notifyConnectionListeners(false); - }; - } catch (error) { - reject(error); - } - }); - } - - /** - * Send codebase context to daemon - * Call this whenever context changes (new repo loaded, etc.) - */ - sendContext(context: CodebaseContext) { - this.pendingContext = context; - - if (this.ws?.readyState === WebSocket.OPEN) { - const msg = { - id: `ctx_${Date.now()}`, - type: 'context', - params: context, - }; - this.ws.send(JSON.stringify(msg)); - console.log('[MCP] Sent context:', context.projectName); - } - } - - /** - * Handle incoming messages from daemon - */ - private async handleMessage(msg: MCPMessage) { - // Handle agent info updates - if (msg.type === 'agent_info' && msg.agentName) { - this._connectedAgent = { - name: msg.agentName, - color: getAgentColor(msg.agentName), - }; - console.log('[MCP] Agent connected:', this._connectedAgent); - return; - } - - // This is a tool call request from an external agent - if (msg.method && msg.id) { - const handler = this.handlers.get(msg.method); - const startTime = Date.now(); - - // Get agent info from message or use connected agent - const agentName = msg.agentName || this._connectedAgent?.name || 'Unknown'; - const agentColor = getAgentColor(agentName); - - // Create activity event with agent info - const activityEvent: ActivityEvent = { - id: msg.id, - tool: msg.method, - params: msg.params || {}, - status: 'running', - timestamp: startTime, - agentName, - agentColor, - }; - this.logActivity(activityEvent); - - if (handler) { - try { - const result = await handler(msg.params || {}); - this.send({ id: msg.id, result }); - - // Update activity with success - this.updateActivity(msg.id, { - status: 'complete', - result, - duration: Date.now() - startTime, - }); - } catch (error) { - const message = error instanceof Error ? error.message : 'Unknown error'; - this.send({ id: msg.id, error: { message } }); - - // Update activity with error - this.updateActivity(msg.id, { - status: 'error', - error: message, - duration: Date.now() - startTime, - }); - } - } else { - this.send({ - id: msg.id, - error: { message: `Unknown tool: ${msg.method}` } - }); - - // Update activity with error - this.updateActivity(msg.id, { - status: 'error', - error: `Unknown tool: ${msg.method}`, - duration: Date.now() - startTime, - }); - } - } - } - - /** - * Log an activity event - */ - private logActivity(event: ActivityEvent) { - this.activityLog.push(event); - // Keep max 100 events - if (this.activityLog.length > 100) { - this.activityLog.shift(); - } - this.notifyActivityListeners(event); - } - - /** - * Update an existing activity event - */ - private updateActivity(id: string, updates: Partial) { - const event = this.activityLog.find(e => e.id === id); - if (event) { - Object.assign(event, updates); - this.notifyActivityListeners(event); - } - } - - /** - * Send a message to the daemon - */ - private send(msg: MCPMessage) { - if (this.ws?.readyState === WebSocket.OPEN) { - this.ws.send(JSON.stringify(msg)); - } - } - - /** - * Register a handler for a tool - */ - registerHandler(method: string, handler: ToolHandler) { - this.handlers.set(method, handler); - } - - /** - * Listen for connection state changes - */ - onConnectionChange(listener: (connected: boolean) => void) { - this.connectionListeners.add(listener); - return () => this.connectionListeners.delete(listener); - } - - private notifyConnectionListeners(connected: boolean) { - this.connectionListeners.forEach(listener => listener(connected)); - } - - /** - * Listen for activity events - */ - onActivity(listener: ActivityListener) { - this.activityListeners.add(listener); - return () => this.activityListeners.delete(listener); - } - - private notifyActivityListeners(event: ActivityEvent) { - this.activityListeners.forEach(listener => listener(event)); - } - - /** - * Get the activity log - */ - getActivityLog(): ActivityEvent[] { - return [...this.activityLog]; - } - - /** - * Clear the activity log - */ - clearActivityLog() { - this.activityLog = []; - } - - /** - * Check if connected - */ - get isConnected(): boolean { - return this.ws?.readyState === WebSocket.OPEN; - } - - /** - * Get the connected agent info - */ - get connectedAgent(): ConnectedAgent | null { - return this._connectedAgent; - } - - /** - * Disconnect from daemon - */ - disconnect() { - this.ws?.close(); - this.ws = null; - } -} - -// Singleton instance -let mcpClientInstance: MCPBrowserClient | null = null; - -export function getMCPClient(): MCPBrowserClient { - if (!mcpClientInstance) { - mcpClientInstance = new MCPBrowserClient(); - } - return mcpClientInstance; -} diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 38102fa8a2..c2d95b25e2 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "gitnexus", "version": "0.1.0", - "license": "MIT", + "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { "@huggingface/transformers": "^3.0.0", "@modelcontextprotocol/sdk": "^1.0.0", @@ -16,23 +16,25 @@ "express": "^4.19.2", "glob": "^11.0.0", "graphology": "^0.25.4", - "graphology-communities-louvain": "^2.0.1", + "graphology-indices": "^0.17.0", + "graphology-utils": "^2.3.0", "kuzu": "^0.11.3", "lru-cache": "^11.0.0", - "minisearch": "^7.2.0", + "mnemonist": "^0.39.0", "ora": "^8.0.0", + "pandemonium": "^2.4.0", "tree-sitter": "^0.21.0", "tree-sitter-c": "^0.21.0", "tree-sitter-c-sharp": "^0.21.0", "tree-sitter-cpp": "^0.22.0", "tree-sitter-go": "^0.21.0", - "tree-sitter-java": "^0.20.0", + "tree-sitter-java": "^0.21.0", "tree-sitter-javascript": "^0.21.0", "tree-sitter-python": "^0.21.0", "tree-sitter-rust": "^0.21.0", "tree-sitter-typescript": "^0.21.0", - "uuid": "^13.0.0", - "ws": "^8.16.0" + "typescript": "^5.4.5", + "uuid": "^13.0.0" }, "bin": { "gitnexus": "dist/cli/index.js" @@ -42,9 +44,7 @@ "@types/express": "^4.17.21", "@types/node": "^20.0.0", "@types/uuid": "^10.0.0", - "@types/ws": "^8.5.10", - "tsx": "^4.0.0", - "typescript": "^5.4.5" + "tsx": "^4.0.0" }, "engines": { "node": ">=18.0.0" @@ -1572,16 +1572,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/ws": { - "version": "8.18.1", - "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.18.1.tgz", - "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -2849,21 +2839,6 @@ "graphology-types": ">=0.24.0" } }, - "node_modules/graphology-communities-louvain": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/graphology-communities-louvain/-/graphology-communities-louvain-2.0.2.tgz", - "integrity": "sha512-zt+2hHVPYxjEquyecxWXoUoIuN/UvYzsvI7boDdMNz0rRvpESQ7+e+Ejv6wK7AThycbZXuQ6DkG8NPMCq6XwoA==", - "license": "MIT", - "dependencies": { - "graphology-indices": "^0.17.0", - "graphology-utils": "^2.4.4", - "mnemonist": "^0.39.0", - "pandemonium": "^2.4.1" - }, - "peerDependencies": { - "graphology-types": ">=0.19.0" - } - }, "node_modules/graphology-indices": { "version": "0.17.0", "resolved": "https://registry.npmjs.org/graphology-indices/-/graphology-indices-0.17.0.tgz", @@ -3308,12 +3283,6 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/minisearch": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/minisearch/-/minisearch-7.2.0.tgz", - "integrity": "sha512-dqT2XBYUOZOiC5t2HRnwADjhNS2cecp9u+TJRiJ1Qp/f5qjkeT5APcGPjHw+bz89Ms8Jp+cG4AlE+QZ/QnDglg==", - "license": "MIT" - }, "node_modules/minizlib": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", @@ -3366,12 +3335,6 @@ "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", "license": "MIT" }, - "node_modules/nan": { - "version": "2.25.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.25.0.tgz", - "integrity": "sha512-0M90Ag7Xn5KMLLZ7zliPWP3rT90P6PN+IzVFS0VqmnPktBk3700xUVv8Ikm9EUaUE5SDWdp/BIxdENzVznpm1g==", - "license": "MIT" - }, "node_modules/negotiator": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", @@ -4500,13 +4463,31 @@ } }, "node_modules/tree-sitter-java": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.20.2.tgz", - "integrity": "sha512-jc6RCnM+JE2ns1AkpErOp2Dp1jOADPbljsrWup0Vj2qTmG8KGYMSTD7HcrVRyZUC6pRLFySPMOh8x7Dn12aynw==", + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.21.0.tgz", + "integrity": "sha512-CKJiTo1uc3SUsgEcaZgufGx8my6dzihy8JR/JsJH40Tj3uSe2/eFLk+0q+fpbosGAyY4YiXJtEoFB2O4bS2yOw==", "hasInstallScript": true, "license": "MIT", "dependencies": { - "nan": "^2.14.1" + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0" + }, + "peerDependencies": { + "tree-sitter": "^0.21.0" + }, + "peerDependenciesMeta": { + "tree_sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-java/node_modules/node-addon-api": { + "version": "8.5.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.5.0.tgz", + "integrity": "sha512-/bRZty2mXUIFY/xU5HLvveNHlswNJej+RnxBjOMkidWfwZzgTbPG1E3K5TOxRLOR+5hX7bSofy8yf1hZevMS8A==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" } }, "node_modules/tree-sitter-javascript": { @@ -4680,7 +4661,6 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -4919,27 +4899,6 @@ "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", "license": "ISC" }, - "node_modules/ws": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.19.0.tgz", - "integrity": "sha512-blAT2mjOEIi0ZzruJfIhb3nps74PRWTCz1IjglWEEpQl5XS/UNama6u2/rjFkDDouqr4L67ry+1aGIALViWjDg==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 5595b2c17b..6afd6147c8 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -10,7 +10,8 @@ }, "files": [ "dist", - "skills" + "skills", + "vendor" ], "scripts": { "build": "tsc", @@ -25,17 +26,19 @@ "express": "^4.19.2", "glob": "^11.0.0", "graphology": "^0.25.4", - "graphology-communities-louvain": "^2.0.1", + "graphology-indices": "^0.17.0", + "graphology-utils": "^2.3.0", "kuzu": "^0.11.3", "lru-cache": "^11.0.0", - "minisearch": "^7.2.0", + "mnemonist": "^0.39.0", "ora": "^8.0.0", + "pandemonium": "^2.4.0", "tree-sitter": "^0.21.0", "tree-sitter-c": "^0.21.0", "tree-sitter-c-sharp": "^0.21.0", "tree-sitter-cpp": "^0.22.0", "tree-sitter-go": "^0.21.0", - "tree-sitter-java": "^0.20.0", + "tree-sitter-java": "^0.21.0", "tree-sitter-javascript": "^0.21.0", "tree-sitter-python": "^0.21.0", "tree-sitter-rust": "^0.21.0", @@ -53,4 +56,4 @@ "engines": { "node": ">=18.0.0" } -} \ No newline at end of file +} diff --git a/gitnexus/skills/debugging.md b/gitnexus/skills/debugging.md index 3d8dab1b78..883ff25b76 100644 --- a/gitnexus/skills/debugging.md +++ b/gitnexus/skills/debugging.md @@ -7,6 +7,7 @@ description: Trace bugs through call chains using knowledge graph ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. gitnexus_search({query}) → Find code related to error 2. gitnexus_explore({name, type: "symbol"}) → Get callers and callees 3. READ gitnexus://process/{name} → Trace execution flow diff --git a/gitnexus/skills/exploring.md b/gitnexus/skills/exploring.md index 2baa7f31ed..52beb6909d 100644 --- a/gitnexus/skills/exploring.md +++ b/gitnexus/skills/exploring.md @@ -7,6 +7,7 @@ description: Navigate unfamiliar code using GitNexus knowledge graph ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. READ gitnexus://context → Get codebase overview (~150 tokens) 2. READ gitnexus://clusters → See all functional clusters 3. READ gitnexus://cluster/{name} → Deep dive on specific cluster diff --git a/gitnexus/skills/impact-analysis.md b/gitnexus/skills/impact-analysis.md index 8f8db00847..dbbffc3124 100644 --- a/gitnexus/skills/impact-analysis.md +++ b/gitnexus/skills/impact-analysis.md @@ -7,6 +7,7 @@ description: Analyze blast radius before making code changes ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. gitnexus_impact({target, direction: "upstream"}) → What depends on this 2. READ gitnexus://clusters → Check affected areas 3. READ gitnexus://processes → Affected execution flows diff --git a/gitnexus/skills/refactoring.md b/gitnexus/skills/refactoring.md index 1513a2ed6c..1ea1c545b8 100644 --- a/gitnexus/skills/refactoring.md +++ b/gitnexus/skills/refactoring.md @@ -7,6 +7,7 @@ description: Plan safe refactors using blast radius and dependency mapping ## Quick Start ``` +0. If "Index is stale" → gitnexus_analyze({}) 1. gitnexus_impact({target, direction: "upstream"}) → Map all dependents 2. READ gitnexus://schema → Understand graph structure 3. gitnexus_cypher → Find all references diff --git a/gitnexus/src/FTS_LIVE_TEST.ts b/gitnexus/src/FTS_LIVE_TEST.ts deleted file mode 100644 index 2044777613..0000000000 --- a/gitnexus/src/FTS_LIVE_TEST.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * FTS Live Update Test File - * - * This file contains the unique identifier: GITNEXUS_FTS_LIVETEST_2026_ALPHA - * - * If you can find this file via MCP search after the watcher picks up the commit, - * then the KuzuDB FTS live update system is working correctly! - */ - -export function ftsLiveTestFunction() { - // GITNEXUS_FTS_LIVETEST_2026_ALPHA marker for search - return "Live FTS test successful!"; -} diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 3126931908..8f4532e588 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -7,8 +7,7 @@ import path from 'path'; import ora from 'ora'; import { runPipelineFromRepo } from '../core/ingestion/pipeline.js'; -import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu } from '../core/kuzu/kuzu-adapter.js'; -import { buildBM25Index, exportBM25Index } from '../core/search/bm25-index.js'; +import { initKuzu, loadGraphToKuzu, getKuzuStats, executeQuery, executeWithReusedStatement, closeKuzu, createFTSIndex } from '../core/kuzu/kuzu-adapter.js'; import { runEmbeddingPipeline } from '../core/embeddings/embedding-pipeline.js'; import { getStoragePaths, saveMeta, loadMeta, addToGitignore } from '../storage/repo-manager.js'; import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js'; @@ -45,7 +44,7 @@ export const analyzeCommand = async ( return; } - const { storagePath, kuzuPath, bm25Path } = getStoragePaths(repoPath); + const { storagePath, kuzuPath } = getStoragePaths(repoPath); const currentCommit = getCurrentCommit(repoPath); const existingMeta = await loadMeta(storagePath); @@ -62,14 +61,33 @@ export const analyzeCommand = async ( }); // Load graph into KuzuDB + // Always start fresh - remove existing kuzu DB to avoid stale/corrupt data spinner.text = 'Loading graph into KuzuDB...'; + await closeKuzu(); + + // Kuzu 0.11 stores databases as: (main file) + .wal (WAL file) + // BOTH must be deleted or kuzu will find the orphaned WAL and corrupt the database + const fsClean = await import('fs/promises'); + const kuzuFiles = [kuzuPath, `${kuzuPath}.wal`, `${kuzuPath}.lock`]; + for (const f of kuzuFiles) { + try { await fsClean.rm(f, { recursive: true, force: true }); } catch { /* may not exist */ } + } + await initKuzu(kuzuPath); await loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath); - // Build BM25 search index - spinner.text = 'Building BM25 index...'; - buildBM25Index(pipelineResult.fileContents); - await exportBM25Index(bm25Path); + // Create FTS indexes for keyword search + // Indexes searchable content on: File, Function, Class, Method + spinner.text = 'Creating FTS indexes...'; + try { + await createFTSIndex('File', 'file_fts', ['name', 'content']); + await createFTSIndex('Function', 'function_fts', ['name', 'content']); + await createFTSIndex('Class', 'class_fts', ['name', 'content']); + await createFTSIndex('Method', 'method_fts', ['name', 'content']); + } catch (e: any) { + // FTS index creation may fail if tables are empty (no data for that type) + console.error('Note: Some FTS indexes may not have been created:', e.message); + } // Generate embeddings if (!options?.skipEmbeddings) { diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 8e354ba50a..7d094dd852 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -6,7 +6,6 @@ import { listCommand } from './list.js'; import { statusCommand } from './status.js'; import { mcpCommand } from './mcp.js'; import { cleanCommand } from './clean.js'; - const program = new Command(); program diff --git a/gitnexus/src/cli/mcp.ts b/gitnexus/src/cli/mcp.ts index 43d7458dcc..36c85b4d29 100644 --- a/gitnexus/src/cli/mcp.ts +++ b/gitnexus/src/cli/mcp.ts @@ -61,6 +61,8 @@ export const mcpCommand = async () => { const local = new LocalBackend(); await local.init(candidate); console.error(`GitNexus: Found index at ${repo.storagePath}`); + + // Start MCP server (no watcher daemon - staleness hints guide LLM to call analyze) await startMCPServer(local); return; } diff --git a/gitnexus/src/core/graph/graph.ts b/gitnexus/src/core/graph/graph.ts index 695daf2bf6..20643d8a2c 100644 --- a/gitnexus/src/core/graph/graph.ts +++ b/gitnexus/src/core/graph/graph.ts @@ -16,6 +16,37 @@ export const createKnowledgeGraph = (): KnowledgeGraph => { } }; + /** + * Remove a single node and all relationships involving it + */ + const removeNode = (nodeId: string): boolean => { + if (!nodeMap.has(nodeId)) return false; + + nodeMap.delete(nodeId); + + // Remove all relationships involving this node + for (const [relId, rel] of relationshipMap) { + if (rel.sourceId === nodeId || rel.targetId === nodeId) { + relationshipMap.delete(relId); + } + } + return true; + }; + + /** + * Remove all nodes (and their relationships) belonging to a file + */ + const removeNodesByFile = (filePath: string): number => { + let removed = 0; + for (const [nodeId, node] of nodeMap) { + if (node.properties?.filePath === filePath) { + removeNode(nodeId); + removed++; + } + } + return removed; + }; + return{ get nodes(){ return Array.from(nodeMap.values()) @@ -36,6 +67,8 @@ export const createKnowledgeGraph = (): KnowledgeGraph => { addNode, addRelationship, + removeNode, + removeNodesByFile, }; -}; \ No newline at end of file +}; diff --git a/gitnexus/src/core/graph/types.ts b/gitnexus/src/core/graph/types.ts index 7bc9a5a95f..ee37d94ea4 100644 --- a/gitnexus/src/core/graph/types.ts +++ b/gitnexus/src/core/graph/types.ts @@ -83,4 +83,6 @@ export interface KnowledgeGraph { relationshipCount: number, addNode: (node: GraphNode) => void, addRelationship: (relationship: GraphRelationship) => void, + removeNode: (nodeId: string) => boolean, + removeNodesByFile: (filePath: string) => number, } \ No newline at end of file diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 895e55352d..bb42502698 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -6,7 +6,7 @@ import Parser from 'tree-sitter'; import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename } from './utils.js'; +import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; /** * Node types that represent function/method definitions across languages. @@ -140,6 +140,7 @@ export const processCalls = async ( for (let i = 0; i < files.length; i++) { const file = files[i]; onProgress?.(i + 1, files.length); + if (i % 20 === 0) await yieldToEventLoop(); // 1. Check language support first const language = getLanguageFromFilename(file.path); diff --git a/gitnexus/src/core/ingestion/community-processor.ts b/gitnexus/src/core/ingestion/community-processor.ts index 42194c076c..4628df4098 100644 --- a/gitnexus/src/core/ingestion/community-processor.ts +++ b/gitnexus/src/core/ingestion/community-processor.ts @@ -1,19 +1,29 @@ /** * Community Detection Processor * - * Uses the Leiden algorithm (via graphology-communities-louvain) to detect + * Uses the Leiden algorithm (via graphology-communities-leiden) to detect * communities/clusters in the code graph based on CALLS relationships. * * Communities represent groups of code that work together frequently, * helping agents navigate the codebase by functional area rather than file structure. */ -// NOTE: graphology + louvain typings are a bit inconsistent across versions. -// Keep these as `any` to avoid blocking the CLI build. +// NOTE: The Leiden algorithm source is vendored from graphology's repo +// (src/communities-leiden) because it was never published to npm. +// We use createRequire to load the CommonJS vendored files in ESM context. import Graph from 'graphology'; -import louvain from 'graphology-communities-louvain'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; import { KnowledgeGraph, NodeLabel } from '../graph/types.js'; +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +// Navigate to package root (works from both src/ and dist/) +const leidenPath = resolve(__dirname, '..', '..', '..', 'vendor', 'leiden', 'index.cjs'); +const _require = createRequire(import.meta.url); +const leiden = _require(leidenPath); + // ============================================================================ // TYPES // ============================================================================ @@ -95,8 +105,8 @@ export const processCommunities = async ( onProgress?.(`Running Leiden algorithm on ${graph.order} nodes...`, 30); - // Step 2: Run Leiden (via Louvain implementation with refinement) - const details = (louvain as any).detailed(graph, { + // Step 2: Run Leiden algorithm for community detection + const details = (leiden as any).detailed(graph, { resolution: 1.0, // Default resolution, can be tuned randomWalk: true, }); diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts index f0143a77d0..0ee2a21b64 100644 --- a/gitnexus/src/core/ingestion/heritage-processor.ts +++ b/gitnexus/src/core/ingestion/heritage-processor.ts @@ -13,7 +13,7 @@ import Parser from 'tree-sitter'; import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename } from './utils.js'; +import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; export const processHeritage = async ( graph: KnowledgeGraph, @@ -27,6 +27,7 @@ export const processHeritage = async ( for (let i = 0; i < files.length; i++) { const file = files[i]; onProgress?.(i + 1, files.length); + if (i % 20 === 0) await yieldToEventLoop(); // 1. Check language support const language = getLanguageFromFilename(file.path); diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts index 1977ca2906..d8d97da99d 100644 --- a/gitnexus/src/core/ingestion/import-processor.ts +++ b/gitnexus/src/core/ingestion/import-processor.ts @@ -4,7 +4,7 @@ import Parser from 'tree-sitter'; import { loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename } from './utils.js'; +import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; const isDev = process.env.NODE_ENV !== 'production'; @@ -127,6 +127,7 @@ export const processImports = async ( for (let i = 0; i < files.length; i++) { const file = files[i]; onProgress?.(i + 1, files.length); + if (i % 20 === 0) await yieldToEventLoop(); // 1. Check language support first const language = getLanguageFromFilename(file.path); diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index cca7098ea0..a4777562dd 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -5,7 +5,7 @@ import { LANGUAGE_QUERIES } from './tree-sitter-queries.js'; import { generateId } from '../../lib/utils.js'; import { SymbolTable } from './symbol-table.js'; import { ASTCache } from './ast-cache.js'; -import { getLanguageFromFilename } from './utils.js'; +import { getLanguageFromFilename, yieldToEventLoop } from './utils.js'; export type FileProgressCallback = (current: number, total: number, filePath: string) => void; @@ -128,6 +128,9 @@ export const processParsing = async ( // Report progress for each file onFileProgress?.(i + 1, total, file.path); + // Yield to event loop periodically so spinner can update + if (i % 20 === 0) await yieldToEventLoop(); + const language = getLanguageFromFilename(file.path); if (!language) continue; diff --git a/gitnexus/src/core/ingestion/utils.ts b/gitnexus/src/core/ingestion/utils.ts index 5ac12a8bec..620fac7acd 100644 --- a/gitnexus/src/core/ingestion/utils.ts +++ b/gitnexus/src/core/ingestion/utils.ts @@ -1,5 +1,11 @@ import { SupportedLanguages } from '../../config/supported-languages.js'; +/** + * Yield control to the event loop so spinners/progress can render. + * Call periodically in hot loops to prevent UI freezes. + */ +export const yieldToEventLoop = (): Promise => new Promise(resolve => setImmediate(resolve)); + /** * Map file extension to SupportedLanguage enum */ diff --git a/gitnexus/src/core/kuzu/csv-generator.ts b/gitnexus/src/core/kuzu/csv-generator.ts index b23471deb9..2663095709 100644 --- a/gitnexus/src/core/kuzu/csv-generator.ts +++ b/gitnexus/src/core/kuzu/csv-generator.ts @@ -18,12 +18,17 @@ import { NODE_TABLES, NodeTableName } from './schema.js'; // ============================================================================ /** - * Sanitize string to ensure valid UTF-8 - * Removes or replaces invalid characters that would break CSV parsing + * Sanitize string to ensure valid UTF-8 and safe CSV content for KuzuDB + * Removes or replaces invalid characters that would break CSV parsing. + * + * Critical: KuzuDB's native CSV parser on Windows can misinterpret \r\n + * inside quoted fields. We normalize all line endings to \n only. */ const sanitizeUTF8 = (str: string): string => { return str - .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n \r + .replace(/\r\n/g, '\n') // Normalize Windows line endings first + .replace(/\r/g, '\n') // Normalize remaining \r to \n + .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control chars except \t \n .replace(/[\uD800-\uDFFF]/g, '') // Remove surrogate pairs (invalid standalone) .replace(/[\uFFFE\uFFFF]/g, ''); // Remove BOM and special chars }; @@ -175,14 +180,14 @@ const generateFolderCSV = (nodes: GraphNode[]): string => { /** * Generate CSV for code element nodes (Function, Class, Interface, Method, CodeElement) - * Headers: id,name,filePath,startLine,endLine,content + * Headers: id,name,filePath,startLine,endLine,isExported,content */ const generateCodeElementCSV = ( nodes: GraphNode[], label: NodeLabel, fileContents: Map ): string => { - const headers = ['id', 'name', 'filePath', 'startLine', 'endLine', 'content']; + const headers = ['id', 'name', 'filePath', 'startLine', 'endLine', 'isExported', 'content']; const rows: string[] = [headers.join(',')]; for (const node of nodes) { @@ -194,6 +199,7 @@ const generateCodeElementCSV = ( escapeCSVField(node.properties.filePath || ''), escapeCSVNumber(node.properties.startLine, -1), escapeCSVNumber(node.properties.endLine, -1), + node.properties.isExported ? 'true' : 'false', escapeCSVField(content), ].join(',')); } diff --git a/gitnexus/src/core/kuzu/kuzu-adapter.ts b/gitnexus/src/core/kuzu/kuzu-adapter.ts index 3681b440c1..d99ac5cee8 100644 --- a/gitnexus/src/core/kuzu/kuzu-adapter.ts +++ b/gitnexus/src/core/kuzu/kuzu-adapter.ts @@ -19,21 +19,24 @@ const normalizeCopyPath = (filePath: string): string => filePath.replace(/\\/g, export const initKuzu = async (dbPath: string) => { if (conn) return { db, conn }; - // kuzu v0.11 expects the database path to NOT exist (it will create it) - // or to be an existing valid kuzu database - // If an empty directory exists from a previous clean, remove it + // kuzu v0.11 stores the database as a single file (not a directory). + // If the path already exists, it must be a valid kuzu database file. + // Remove stale empty directories or files from older versions. try { const stat = await fs.stat(dbPath); if (stat.isDirectory()) { - // Check if it's an empty directory + // Old-style directory database or empty leftover - remove it const files = await fs.readdir(dbPath); if (files.length === 0) { - // Empty directory - remove it so kuzu can create fresh await fs.rmdir(dbPath); + } else { + // Non-empty directory from older kuzu version - remove entire directory + await fs.rm(dbPath, { recursive: true, force: true }); } } + // If it's a file, assume it's an existing kuzu database - kuzu will open it } catch { - // Path doesn't exist, which is what kuzu v0.11 wants for a new database + // Path doesn't exist, which is what kuzu wants for a new database } // Ensure parent directory exists @@ -46,8 +49,12 @@ export const initKuzu = async (dbPath: string) => { for (const schemaQuery of SCHEMA_QUERIES) { try { await conn.query(schemaQuery); - } catch { - // Schema may already exist + } catch (err) { + // Only ignore "already exists" errors - log everything else + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes('already exists')) { + console.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`); + } } } @@ -78,10 +85,50 @@ export const loadGraphToKuzu = async ( const relLines = csvData.relCSV.split('\n').slice(1).filter(line => line.trim()); for (const { table, path: filePath } of nodeFiles) { - const copyQuery = getCopyQuery(table, normalizeCopyPath(filePath)); - await conn.query(copyQuery); + const normalizedPath = normalizeCopyPath(filePath); + const copyQuery = getCopyQuery(table, normalizedPath); + + // Log CSV stats for diagnostics + const csvContent = await fs.readFile(filePath, 'utf-8'); + const csvLines = csvContent.split('\n').length; + console.log(` COPY ${table}: ${csvLines - 1} rows`); + + try { + await conn.query(copyQuery); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + console.warn(`⚠️ COPY failed for ${table} (${csvLines - 1} rows): ${errMsg}`); + + // Retry with IGNORE_ERRORS=true to skip malformed rows + console.log(` Retrying ${table} with IGNORE_ERRORS=true...`); + try { + const retryQuery = copyQuery.replace('auto_detect=false)', 'auto_detect=false, IGNORE_ERRORS=true)'); + await conn.query(retryQuery); + console.log(` ✅ ${table} loaded with IGNORE_ERRORS (some rows may have been skipped)`); + } catch (retryErr) { + const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr); + console.error(`❌ COPY failed for ${table} even with IGNORE_ERRORS: ${retryMsg}`); + throw retryErr; + } + } } + console.log('✅ All COPY commands succeeded. Starting relationship insertion...'); + + // Build a set of valid table names for fast lookup + const validTables = new Set(NODE_TABLES as readonly string[]); + + const getNodeLabel = (nodeId: string): string => { + if (nodeId.startsWith('comm_')) return 'Community'; + if (nodeId.startsWith('proc_')) return 'Process'; + return nodeId.split(':')[0]; + }; + + // All multi-language tables are created with backticks - must always reference them with backticks + const escapeLabel = (label: string): string => { + return BACKTICK_TABLES.has(label) ? `\`${label}\`` : label; + }; + let insertedRels = 0; let skippedRels = 0; for (const line of relLines) { @@ -89,26 +136,24 @@ export const loadGraphToKuzu = async ( const match = line.match(/"([^"]*)","([^"]*)","([^"]*)",([0-9.]+),"([^"]*)",([0-9-]+)/); if (!match) continue; const [, fromId, toId, relType, confidenceStr, reason, stepStr] = match; + + const fromLabel = getNodeLabel(fromId); + const toLabel = getNodeLabel(toId); + + // Skip relationships where either node's label doesn't have a table in KuzuDB + // (e.g. Variable, Import, Type nodes that aren't in the schema) + // Querying a non-existent table causes a fatal native crash + if (!validTables.has(fromLabel) || !validTables.has(toLabel)) { + skippedRels++; + continue; + } + const confidence = parseFloat(confidenceStr) || 1.0; const step = parseInt(stepStr) || 0; - const getNodeLabel = (nodeId: string): string => { - if (nodeId.startsWith('comm_')) return 'Community'; - if (nodeId.startsWith('proc_')) return 'Process'; - return nodeId.split(':')[0]; - }; - - const RESERVED_LABELS = ['Macro', 'Enum', 'Union', 'Const', 'Module', 'Struct']; - const escapeLabel = (label: string): string => { - return RESERVED_LABELS.includes(label) ? `\`${label}\`` : label; - }; - - const fromLabel = escapeLabel(getNodeLabel(fromId)); - const toLabel = escapeLabel(getNodeLabel(toId)); - const insertQuery = ` - MATCH (a:${fromLabel} {id: '${fromId.replace(/'/g, "''")}' }), - (b:${toLabel} {id: '${toId.replace(/'/g, "''")}' }) + MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId.replace(/'/g, "''")}' }), + (b:${escapeLabel(toLabel)} {id: '${toId.replace(/'/g, "''")}' }) CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason.replace(/'/g, "''")}', step: ${step}}]->(b) `; await conn.query(insertQuery); @@ -137,20 +182,157 @@ export const loadGraphToKuzu = async ( return { success: true, insertedRels, skippedRels }; }; +// KuzuDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes). +// Source code content is full of backslashes which confuse the auto-detection. +// We MUST explicitly set ESCAPE='"' to use RFC 4180 escaping, and disable auto_detect to prevent +// KuzuDB from overriding our settings based on sample rows. +const COPY_CSV_OPTS = `(HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`; + +// Multi-language table names that were created with backticks in CODE_ELEMENT_BASE +// and must always be referenced with backticks in queries +const BACKTICK_TABLES = new Set([ + 'Struct', 'Enum', 'Macro', 'Typedef', 'Union', 'Namespace', 'Trait', 'Impl', + 'TypeAlias', 'Const', 'Static', 'Property', 'Record', 'Delegate', 'Annotation', + 'Constructor', 'Template', 'Module', +]); + +const escapeTableName = (table: string): string => { + return BACKTICK_TABLES.has(table) ? `\`${table}\`` : table; +}; + const getCopyQuery = (table: NodeTableName, filePath: string): string => { + const t = escapeTableName(table); if (table === 'File') { - return `COPY File(id, name, filePath, content) FROM "${filePath}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, name, filePath, content) FROM "${filePath}" ${COPY_CSV_OPTS}`; } if (table === 'Folder') { - return `COPY Folder(id, name, filePath) FROM "${filePath}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, name, filePath) FROM "${filePath}" ${COPY_CSV_OPTS}`; } if (table === 'Community') { - return `COPY Community(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${filePath}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, label, heuristicLabel, keywords, description, enrichedBy, cohesion, symbolCount) FROM "${filePath}" ${COPY_CSV_OPTS}`; } if (table === 'Process') { - return `COPY Process(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${filePath}" (HEADER=true, PARALLEL=false)`; + return `COPY ${t}(id, label, heuristicLabel, processType, stepCount, communities, entryPointId, terminalId) FROM "${filePath}" ${COPY_CSV_OPTS}`; } - return `COPY ${table}(id, name, filePath, startLine, endLine, content) FROM "${filePath}" (HEADER=true, PARALLEL=false)`; + // Code element tables (Function, Class, Interface, Method, CodeElement, and multi-language) + return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content) FROM "${filePath}" ${COPY_CSV_OPTS}`; +}; + +/** + * Insert a single node to KuzuDB + * @param label - Node type (File, Function, Class, etc.) + * @param properties - Node properties + * @param dbPath - Path to KuzuDB database (optional if already initialized) + */ +export const insertNodeToKuzu = async ( + label: string, + properties: Record, + dbPath?: string +): Promise => { + // Use provided dbPath or fall back to module-level db + const targetDbPath = dbPath || (db ? undefined : null); + if (!targetDbPath && !db) { + throw new Error('KuzuDB not initialized. Provide dbPath or call initKuzu first.'); + } + + try { + const escapeValue = (v: any): string => { + if (v === null || v === undefined) return 'NULL'; + if (typeof v === 'number') return String(v); + // Escape backslashes first (for Windows paths), then single quotes + return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`; + }; + + // Build INSERT query based on node type + let query: string; + + if (label === 'File') { + query = `CREATE (n:File {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, content: ${escapeValue(properties.content || '')}})`; + } else if (label === 'Folder') { + query = `CREATE (n:Folder {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}})`; + } else { + // Function, Class, Method, Interface, etc. - standard code element schema + query = `CREATE (n:${label} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, content: ${escapeValue(properties.content || '')}})`; + } + + // Use per-query connection if dbPath provided (avoids lock conflicts) + if (targetDbPath) { + const tempDb = new kuzu.Database(targetDbPath); + const tempConn = new kuzu.Connection(tempDb); + try { + await tempConn.query(query); + return true; + } finally { + try { await tempConn.close(); } catch {} + try { await tempDb.close(); } catch {} + } + } else if (conn) { + // Use existing persistent connection (when called from analyze) + await conn.query(query); + return true; + } + + return false; + } catch (e: any) { + // Node may already exist or other error + console.error(`Failed to insert ${label} node:`, e.message); + return false; + } +}; + +/** + * Batch insert multiple nodes to KuzuDB using a single connection + * @param nodes - Array of {label, properties} to insert + * @param dbPath - Path to KuzuDB database + * @returns Object with success count and error count + */ +export const batchInsertNodesToKuzu = async ( + nodes: Array<{ label: string; properties: Record }>, + dbPath: string +): Promise<{ inserted: number; failed: number }> => { + if (nodes.length === 0) return { inserted: 0, failed: 0 }; + + const escapeValue = (v: any): string => { + if (v === null || v === undefined) return 'NULL'; + if (typeof v === 'number') return String(v); + // Escape backslashes first (for Windows paths), then single quotes + return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`; + }; + + // Open a single connection for all inserts + const tempDb = new kuzu.Database(dbPath); + const tempConn = new kuzu.Connection(tempDb); + + let inserted = 0; + let failed = 0; + + try { + for (const { label, properties } of nodes) { + try { + let query: string; + + // Use MERGE instead of CREATE for upsert behavior (handles duplicates gracefully) + if (label === 'File') { + query = `MERGE (n:File {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.content = ${escapeValue(properties.content || '')}`; + } else if (label === 'Folder') { + query = `MERGE (n:Folder {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}`; + } else { + query = `MERGE (n:${label} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.content = ${escapeValue(properties.content || '')}`; + } + + await tempConn.query(query); + inserted++; + } catch (e: any) { + // Don't console.error here - it corrupts MCP JSON-RPC on stderr + failed++; + } + } + } finally { + try { await tempConn.close(); } catch {} + try { await tempDb.close(); } catch {} + } + + return { inserted, failed }; }; export const executeQuery = async (cypher: string): Promise => { @@ -244,7 +426,199 @@ export const closeKuzu = async (): Promise => { export const isKuzuReady = (): boolean => conn !== null && db !== null; +/** + * Delete all nodes (and their relationships) for a specific file from KuzuDB + * @param filePath - The file path to delete nodes for + * @param dbPath - Optional path to KuzuDB for per-query connection + * @returns Object with counts of deleted nodes + */ +export const deleteNodesForFile = async (filePath: string, dbPath?: string): Promise<{ deletedNodes: number }> => { + const usePerQuery = !!dbPath; + + // Set up connection (either use existing or create per-query) + let tempDb: kuzu.Database | null = null; + let tempConn: kuzu.Connection | null = null; + let targetConn: kuzu.Connection | null = conn; + + if (usePerQuery) { + tempDb = new kuzu.Database(dbPath); + tempConn = new kuzu.Connection(tempDb); + targetConn = tempConn; + } else if (!conn) { + throw new Error('KuzuDB not initialized. Provide dbPath or call initKuzu first.'); + } + + try { + let deletedNodes = 0; + const escapedPath = filePath.replace(/'/g, "''"); + + // Delete nodes from each table that has filePath + // DETACH DELETE removes the node and all its relationships + for (const tableName of NODE_TABLES) { + // Skip tables that don't have filePath (Community, Process) + if (tableName === 'Community' || tableName === 'Process') continue; + + try { + // First count how many we'll delete + const countResult = await targetConn!.query( + `MATCH (n:${tableName}) WHERE n.filePath = '${escapedPath}' RETURN count(n) AS cnt` + ); + const result = Array.isArray(countResult) ? countResult[0] : countResult; + const rows = await result.getAll(); + const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0); + + if (count > 0) { + // Delete nodes (and implicitly their relationships via DETACH) + await targetConn!.query( + `MATCH (n:${tableName}) WHERE n.filePath = '${escapedPath}' DETACH DELETE n` + ); + deletedNodes += count; + } + } catch (e) { + // Some tables may not support this query, skip + } + } + + // Also delete any embeddings for nodes in this file + try { + await targetConn!.query( + `MATCH (e:${EMBEDDING_TABLE_NAME}) WHERE e.nodeId STARTS WITH '${escapedPath}' DELETE e` + ); + } catch { + // Embedding table may not exist or nodeId format may differ + } + + return { deletedNodes }; + } finally { + // Close per-query connection if used + if (tempConn) { + try { await tempConn.close(); } catch {} + } + if (tempDb) { + try { await tempDb.close(); } catch {} + } + } +}; + export const getEmbeddingTableName = (): string => EMBEDDING_TABLE_NAME; +// ============================================================================ +// Full-Text Search (FTS) Functions +// ============================================================================ +/** + * Load the FTS extension (required before using FTS functions) + */ +export const loadFTSExtension = async (): Promise => { + if (!conn) { + throw new Error('KuzuDB not initialized. Call initKuzu first.'); + } + try { + await conn.query('INSTALL fts'); + await conn.query('LOAD EXTENSION fts'); + } catch { + // Extension may already be loaded + } +}; +/** + * Create a full-text search index on a table + * @param tableName - The node table name (e.g., 'File', 'CodeSymbol') + * @param indexName - Name for the FTS index + * @param properties - List of properties to index (e.g., ['name', 'code']) + * @param stemmer - Stemming algorithm (default: 'porter') + */ +export const createFTSIndex = async ( + tableName: string, + indexName: string, + properties: string[], + stemmer: string = 'porter' +): Promise => { + if (!conn) { + throw new Error('KuzuDB not initialized. Call initKuzu first.'); + } + + await loadFTSExtension(); + + const propList = properties.map(p => `'${p}'`).join(', '); + const query = `CALL CREATE_FTS_INDEX('${tableName}', '${indexName}', [${propList}], stemmer := '${stemmer}')`; + + try { + await conn.query(query); + } catch (e: any) { + // Index may already exist + if (!e.message?.includes('already exists')) { + throw e; + } + } +}; + +/** + * Query a full-text search index + * @param tableName - The node table name + * @param indexName - FTS index name + * @param query - Search query string + * @param limit - Maximum results + * @param conjunctive - If true, all terms must match (AND); if false, any term matches (OR) + * @returns Array of { node properties, score } + */ +export const queryFTS = async ( + tableName: string, + indexName: string, + query: string, + limit: number = 20, + conjunctive: boolean = false +): Promise> => { + if (!conn) { + throw new Error('KuzuDB not initialized. Call initKuzu first.'); + } + + // Escape single quotes in query + const escapedQuery = query.replace(/'/g, "''"); + + const cypher = ` + CALL QUERY_FTS_INDEX('${tableName}', '${indexName}', '${escapedQuery}', conjunctive := ${conjunctive}) + RETURN node, score + ORDER BY score DESC + LIMIT ${limit} + `; + + try { + const queryResult = await conn.query(cypher); + const result = Array.isArray(queryResult) ? queryResult[0] : queryResult; + const rows = await result.getAll(); + + return rows.map((row: any) => { + const node = row.node || row[0] || {}; + const score = row.score ?? row[1] ?? 0; + return { + nodeId: node.nodeId || node.id || '', + name: node.name || '', + filePath: node.filePath || '', + score: typeof score === 'number' ? score : parseFloat(score) || 0, + ...node, + }; + }); + } catch (e: any) { + // Return empty if index doesn't exist yet + if (e.message?.includes('does not exist')) { + return []; + } + throw e; + } +}; + +/** + * Drop an FTS index + */ +export const dropFTSIndex = async (tableName: string, indexName: string): Promise => { + if (!conn) { + throw new Error('KuzuDB not initialized. Call initKuzu first.'); + } + + try { + await conn.query(`CALL DROP_FTS_INDEX('${tableName}', '${indexName}')`); + } catch { + // Index may not exist + } +}; diff --git a/gitnexus/src/core/kuzu/schema.ts b/gitnexus/src/core/kuzu/schema.ts index 6c20b4bd56..d74545c069 100644 --- a/gitnexus/src/core/kuzu/schema.ts +++ b/gitnexus/src/core/kuzu/schema.ts @@ -62,6 +62,7 @@ CREATE NODE TABLE Function ( filePath STRING, startLine INT64, endLine INT64, + isExported BOOLEAN, content STRING, PRIMARY KEY (id) )`; @@ -73,6 +74,7 @@ CREATE NODE TABLE Class ( filePath STRING, startLine INT64, endLine INT64, + isExported BOOLEAN, content STRING, PRIMARY KEY (id) )`; @@ -84,6 +86,7 @@ CREATE NODE TABLE Interface ( filePath STRING, startLine INT64, endLine INT64, + isExported BOOLEAN, content STRING, PRIMARY KEY (id) )`; @@ -95,6 +98,7 @@ CREATE NODE TABLE Method ( filePath STRING, startLine INT64, endLine INT64, + isExported BOOLEAN, content STRING, PRIMARY KEY (id) )`; @@ -106,6 +110,7 @@ CREATE NODE TABLE CodeElement ( filePath STRING, startLine INT64, endLine INT64, + isExported BOOLEAN, content STRING, PRIMARY KEY (id) )`; @@ -196,20 +201,20 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM File TO \`Struct\`, FROM File TO \`Enum\`, FROM File TO \`Macro\`, - FROM File TO Typedef, + FROM File TO \`Typedef\`, FROM File TO \`Union\`, - FROM File TO Namespace, - FROM File TO Trait, - FROM File TO Impl, - FROM File TO TypeAlias, + FROM File TO \`Namespace\`, + FROM File TO \`Trait\`, + FROM File TO \`Impl\`, + FROM File TO \`TypeAlias\`, FROM File TO \`Const\`, - FROM File TO Static, - FROM File TO Property, - FROM File TO Record, - FROM File TO Delegate, - FROM File TO Annotation, - FROM File TO Constructor, - FROM File TO Template, + FROM File TO \`Static\`, + FROM File TO \`Property\`, + FROM File TO \`Record\`, + FROM File TO \`Delegate\`, + FROM File TO \`Annotation\`, + FROM File TO \`Constructor\`, + FROM File TO \`Template\`, FROM File TO \`Module\`, FROM Folder TO Folder, FROM Folder TO File, @@ -219,48 +224,48 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM Function TO Community, FROM Function TO \`Macro\`, FROM Function TO \`Struct\`, - FROM Function TO Template, + FROM Function TO \`Template\`, FROM Function TO \`Enum\`, - FROM Function TO Namespace, - FROM Function TO TypeAlias, + FROM Function TO \`Namespace\`, + FROM Function TO \`TypeAlias\`, FROM Function TO \`Module\`, - FROM Function TO Impl, + FROM Function TO \`Impl\`, FROM Function TO Interface, - FROM Function TO Constructor, + FROM Function TO \`Constructor\`, FROM Class TO Method, FROM Class TO Function, FROM Class TO Class, FROM Class TO Interface, FROM Class TO Community, - FROM Class TO Template, - FROM Class TO TypeAlias, + FROM Class TO \`Template\`, + FROM Class TO \`TypeAlias\`, FROM Class TO \`Struct\`, FROM Class TO \`Enum\`, - FROM Class TO Constructor, + FROM Class TO \`Constructor\`, FROM Method TO Function, FROM Method TO Method, FROM Method TO Class, FROM Method TO Community, - FROM Method TO Template, + FROM Method TO \`Template\`, FROM Method TO \`Struct\`, - FROM Method TO TypeAlias, + FROM Method TO \`TypeAlias\`, FROM Method TO \`Enum\`, FROM Method TO \`Macro\`, - FROM Method TO Namespace, + FROM Method TO \`Namespace\`, FROM Method TO \`Module\`, - FROM Method TO Impl, + FROM Method TO \`Impl\`, FROM Method TO Interface, - FROM Method TO Constructor, - FROM Template TO Template, - FROM Template TO Function, - FROM Template TO Method, - FROM Template TO Class, - FROM Template TO \`Struct\`, - FROM Template TO TypeAlias, - FROM Template TO \`Enum\`, - FROM Template TO \`Macro\`, - FROM Template TO Interface, - FROM Template TO Constructor, + FROM Method TO \`Constructor\`, + FROM \`Template\` TO \`Template\`, + FROM \`Template\` TO Function, + FROM \`Template\` TO Method, + FROM \`Template\` TO Class, + FROM \`Template\` TO \`Struct\`, + FROM \`Template\` TO \`TypeAlias\`, + FROM \`Template\` TO \`Enum\`, + FROM \`Template\` TO \`Macro\`, + FROM \`Template\` TO Interface, + FROM \`Template\` TO \`Constructor\`, FROM \`Module\` TO \`Module\`, FROM CodeElement TO Community, FROM Interface TO Community, @@ -268,11 +273,11 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM Interface TO Method, FROM Interface TO Class, FROM Interface TO Interface, - FROM Interface TO TypeAlias, + FROM Interface TO \`TypeAlias\`, FROM Interface TO \`Struct\`, - FROM Interface TO Constructor, + FROM Interface TO \`Constructor\`, FROM \`Struct\` TO Community, - FROM \`Struct\` TO Trait, + FROM \`Struct\` TO \`Trait\`, FROM \`Struct\` TO Function, FROM \`Struct\` TO Method, FROM \`Enum\` TO Community, @@ -281,56 +286,56 @@ CREATE REL TABLE ${REL_TABLE_NAME} ( FROM \`Macro\` TO Method, FROM \`Module\` TO Function, FROM \`Module\` TO Method, - FROM Typedef TO Community, + FROM \`Typedef\` TO Community, FROM \`Union\` TO Community, - FROM Namespace TO Community, - FROM Trait TO Community, - FROM Impl TO Community, - FROM Impl TO Trait, - FROM TypeAlias TO Community, + FROM \`Namespace\` TO Community, + FROM \`Trait\` TO Community, + FROM \`Impl\` TO Community, + FROM \`Impl\` TO \`Trait\`, + FROM \`TypeAlias\` TO Community, FROM \`Const\` TO Community, - FROM Static TO Community, - FROM Property TO Community, - FROM Record TO Community, - FROM Delegate TO Community, - FROM Annotation TO Community, - FROM Constructor TO Community, - FROM Constructor TO Interface, - FROM Constructor TO Class, - FROM Constructor TO Method, - FROM Constructor TO Function, - FROM Constructor TO Constructor, - FROM Constructor TO \`Struct\`, - FROM Constructor TO \`Macro\`, - FROM Constructor TO Template, - FROM Constructor TO TypeAlias, - FROM Constructor TO \`Enum\`, - FROM Constructor TO Impl, - FROM Constructor TO Namespace, - FROM Template TO Community, + FROM \`Static\` TO Community, + FROM \`Property\` TO Community, + FROM \`Record\` TO Community, + FROM \`Delegate\` TO Community, + FROM \`Annotation\` TO Community, + FROM \`Constructor\` TO Community, + FROM \`Constructor\` TO Interface, + FROM \`Constructor\` TO Class, + FROM \`Constructor\` TO Method, + FROM \`Constructor\` TO Function, + FROM \`Constructor\` TO \`Constructor\`, + FROM \`Constructor\` TO \`Struct\`, + FROM \`Constructor\` TO \`Macro\`, + FROM \`Constructor\` TO \`Template\`, + FROM \`Constructor\` TO \`TypeAlias\`, + FROM \`Constructor\` TO \`Enum\`, + FROM \`Constructor\` TO \`Impl\`, + FROM \`Constructor\` TO \`Namespace\`, + FROM \`Template\` TO Community, FROM \`Module\` TO Community, FROM Function TO Process, FROM Method TO Process, FROM Class TO Process, FROM Interface TO Process, FROM \`Struct\` TO Process, - FROM Constructor TO Process, + FROM \`Constructor\` TO Process, FROM \`Module\` TO Process, FROM \`Macro\` TO Process, - FROM Impl TO Process, - FROM Typedef TO Process, - FROM TypeAlias TO Process, + FROM \`Impl\` TO Process, + FROM \`Typedef\` TO Process, + FROM \`TypeAlias\` TO Process, FROM \`Enum\` TO Process, FROM \`Union\` TO Process, - FROM Namespace TO Process, - FROM Trait TO Process, + FROM \`Namespace\` TO Process, + FROM \`Trait\` TO Process, FROM \`Const\` TO Process, - FROM Static TO Process, - FROM Property TO Process, - FROM Record TO Process, - FROM Delegate TO Process, - FROM Annotation TO Process, - FROM Template TO Process, + FROM \`Static\` TO Process, + FROM \`Property\` TO Process, + FROM \`Record\` TO Process, + FROM \`Delegate\` TO Process, + FROM \`Annotation\` TO Process, + FROM \`Template\` TO Process, FROM CodeElement TO Process, type STRING, confidence DOUBLE, diff --git a/gitnexus/src/core/search/bm25-index.ts b/gitnexus/src/core/search/bm25-index.ts index 5a64f3c120..bad4128c39 100644 --- a/gitnexus/src/core/search/bm25-index.ts +++ b/gitnexus/src/core/search/bm25-index.ts @@ -1,18 +1,11 @@ /** - * BM25 Full-Text Search Index + * Full-Text Search via KuzuDB FTS * - * Uses MiniSearch for fast keyword-based search with BM25 ranking. - * Complements semantic search - BM25 finds exact terms, semantic finds concepts. + * Uses KuzuDB's built-in full-text search indexes for keyword-based search. + * Always reads from the database (no cached state to drift). */ -import MiniSearch from 'minisearch'; -import fs from 'fs/promises'; - -export interface BM25Document { - id: string; // File path - content: string; // File content - name: string; // File name (boosted in search) -} +import { queryFTS } from '../kuzu/kuzu-adapter.js'; export interface BM25SearchResult { filePath: string; @@ -21,183 +14,51 @@ export interface BM25SearchResult { } /** - * BM25 Index singleton - * Stores the MiniSearch instance and provides search methods - */ -let searchIndex: MiniSearch | null = null; -let indexedDocCount = 0; - -/** - * Build the BM25 index from file contents - * Should be called after ingestion completes + * Search using KuzuDB's built-in FTS (always fresh, reads from disk) * - * @param fileContents - Map of file path to content - * @returns Number of documents indexed + * Queries multiple node tables (File, Function, Class, Method) in parallel + * and merges results by filePath, summing scores for the same file. + * + * @param query - Search query string + * @param limit - Maximum results + * @returns Ranked search results from FTS indexes */ -export const buildBM25Index = (fileContents: Map): number => { - // Create new MiniSearch instance with BM25-like scoring - searchIndex = new MiniSearch({ - fields: ['content', 'name'], // Fields to index - storeFields: ['id'], // Fields to return in results - - // Tokenizer: split on non-alphanumeric, camelCase, snake_case - tokenize: (text: string) => { - // Split on whitespace and punctuation - const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/); - - // Also split camelCase: "getUserById" -> ["get", "user", "by", "id"] - const expanded: string[] = []; - for (const token of tokens) { - if (token.length === 0) continue; - - // Split camelCase - const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' '); - expanded.push(...camelParts); - - // Also keep original token for exact matches - if (camelParts.length > 1) { - expanded.push(token); - } +export const searchFTSFromKuzu = async (query: string, limit: number = 20): Promise => { + // Search multiple tables with searchable content + const [fileResults, functionResults, classResults, methodResults] = await Promise.all([ + queryFTS('File', 'file_fts', query, limit, false).catch(() => []), + queryFTS('Function', 'function_fts', query, limit, false).catch(() => []), + queryFTS('Class', 'class_fts', query, limit, false).catch(() => []), + queryFTS('Method', 'method_fts', query, limit, false).catch(() => []), + ]); + + // Merge results by filePath, summing scores for same file + const merged = new Map(); + + const addResults = (results: any[]) => { + for (const r of results) { + const existing = merged.get(r.filePath); + if (existing) { + existing.score += r.score; + } else { + merged.set(r.filePath, { filePath: r.filePath, score: r.score }); } - - // Filter out very short tokens and common noise - return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t)); - }, - }); + } + }; - // Index all files - const documents: BM25Document[] = []; + addResults(fileResults); + addResults(functionResults); + addResults(classResults); + addResults(methodResults); - for (const [filePath, content] of fileContents.entries()) { - // Extract filename from path - const name = filePath.split('/').pop() || filePath; - - documents.push({ - id: filePath, - content: content, - name: name, - }); - } + // Sort by score descending and add rank + const sorted = Array.from(merged.values()) + .sort((a, b) => b.score - a.score) + .slice(0, limit); - // Batch add for efficiency - searchIndex.addAll(documents); - indexedDocCount = documents.length; - - const isDev = process.env.NODE_ENV !== 'production'; - if (isDev) { - console.log(`📚 BM25 index built: ${indexedDocCount} documents`); - } - - return indexedDocCount; -}; - -/** - * Search the BM25 index - * - * @param query - Search query (keywords) - * @param limit - Maximum results to return - * @returns Ranked search results with file paths and scores - */ -export const searchBM25 = (query: string, limit: number = 20): BM25SearchResult[] => { - if (!searchIndex) { - return []; - } - - // Search with fuzzy matching and prefix support - const results = searchIndex.search(query, { - fuzzy: 0.2, - prefix: true, - boost: { name: 2 }, // Boost file name matches - }); - - // Limit results and add rank - return results.slice(0, limit).map((r, index) => ({ - filePath: r.id, + return sorted.map((r, index) => ({ + filePath: r.filePath, score: r.score, rank: index + 1, })); }; - -/** - * Check if the BM25 index is ready - */ -export const isBM25Ready = (): boolean => { - return searchIndex !== null && indexedDocCount > 0; -}; - -/** - * Get index statistics - */ -export const getBM25Stats = (): { documentCount: number; termCount: number } => { - if (!searchIndex) { - return { documentCount: 0, termCount: 0 }; - } - - return { - documentCount: indexedDocCount, - termCount: searchIndex.termCount, - }; -}; - -/** - * Clear the index (for cleanup or re-indexing) - */ -export const clearBM25Index = (): void => { - searchIndex = null; - indexedDocCount = 0; -}; - -/** - * Export the BM25 index to disk - */ -export const exportBM25Index = async (filePath: string): Promise => { - if (!searchIndex) return; - const json = JSON.stringify(searchIndex.toJSON()); - await fs.writeFile(filePath, json, 'utf-8'); -}; - -/** - * Load a BM25 index from disk - */ -export const loadBM25Index = async (filePath: string): Promise => { - try { - const json = await fs.readFile(filePath, 'utf-8'); - const data = JSON.parse(json); - searchIndex = MiniSearch.loadJSON(data, { - fields: ['content', 'name'], - storeFields: ['id'], - tokenize: (text: string) => { - const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/); - const expanded: string[] = []; - for (const token of tokens) { - if (token.length === 0) continue; - const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' '); - expanded.push(...camelParts); - if (camelParts.length > 1) { - expanded.push(token); - } - } - return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t)); - }, - }); - indexedDocCount = searchIndex.documentCount; - return true; - } catch { - return false; - } -}; - -/** - * Common stop words to filter out (too common to be useful) - */ -const STOP_WORDS = new Set([ - // JavaScript/TypeScript keywords - 'const', 'let', 'var', 'function', 'return', 'if', 'else', 'for', 'while', - 'class', 'new', 'this', 'import', 'export', 'from', 'default', 'async', 'await', - 'try', 'catch', 'throw', 'typeof', 'instanceof', 'true', 'false', 'null', 'undefined', - - // Common English stop words - 'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with', - 'to', 'of', 'it', 'be', 'as', 'by', 'that', 'for', 'are', 'was', 'were', -]); - diff --git a/gitnexus/src/core/search/hybrid-search.ts b/gitnexus/src/core/search/hybrid-search.ts index 4af6f37001..16d3bc42de 100644 --- a/gitnexus/src/core/search/hybrid-search.ts +++ b/gitnexus/src/core/search/hybrid-search.ts @@ -8,7 +8,7 @@ * production search systems. */ -import { searchBM25, isBM25Ready, type BM25SearchResult } from './bm25-index.js'; +import { searchFTSFromKuzu, type BM25SearchResult } from './bm25-index.js'; import type { SemanticSearchResult } from '../embeddings/types.js'; /** @@ -114,11 +114,11 @@ export const mergeWithRRF = ( /** * Check if hybrid search is available - * Requires BM25 index to be built - * Note: Semantic search is optional - hybrid works with just BM25 if embeddings aren't ready + * KuzuDB FTS is always available once the database is initialized. + * Semantic search is optional - hybrid works with just FTS if embeddings aren't ready. */ export const isHybridSearchReady = (): boolean => { - return isBM25Ready(); + return true; // FTS is always available via KuzuDB when DB is open }; /** @@ -146,6 +146,7 @@ export const formatHybridResults = (results: HybridSearchResult[]): string => { /** * Execute BM25 + semantic search and merge with RRF. + * Uses KuzuDB FTS for always-fresh BM25 results (no cached data). * The semanticSearch function is injected to keep this module environment-agnostic. */ export const hybridSearch = async ( @@ -154,11 +155,8 @@ export const hybridSearch = async ( executeQuery: (cypher: string) => Promise, semanticSearch: (executeQuery: (cypher: string) => Promise, query: string, k?: number) => Promise ): Promise => { - const bm25Results = isBM25Ready() ? searchBM25(query, limit) : []; + // Use KuzuDB FTS for always-fresh BM25 results + const bm25Results = await searchFTSFromKuzu(query, limit); const semanticResults = await semanticSearch(executeQuery, query, limit); return mergeWithRRF(bm25Results, semanticResults, limit); }; - - - - diff --git a/gitnexus/src/mcp/core/bm25-index.ts b/gitnexus/src/mcp/core/bm25-index.ts deleted file mode 100644 index b13d6aebd8..0000000000 --- a/gitnexus/src/mcp/core/bm25-index.ts +++ /dev/null @@ -1,120 +0,0 @@ -/** - * BM25 Full-Text Search Index (Read-Only) - * - * Uses MiniSearch for fast keyword-based search with BM25 ranking. - * For MCP, we only load and search - not build. - */ - -import MiniSearch from 'minisearch'; -import fs from 'fs/promises'; - -export interface BM25Document { - id: string; // File path - content: string; // File content - name: string; // File name (boosted in search) -} - -export interface BM25SearchResult { - filePath: string; - score: number; - rank: number; -} - -let searchIndex: MiniSearch | null = null; -let indexedDocCount = 0; - -/** - * Common stop words to filter out - */ -const STOP_WORDS = new Set([ - 'const', 'let', 'var', 'function', 'return', 'if', 'else', 'for', 'while', - 'class', 'new', 'this', 'import', 'export', 'from', 'default', 'async', 'await', - 'try', 'catch', 'throw', 'typeof', 'instanceof', 'true', 'false', 'null', 'undefined', - 'the', 'is', 'at', 'which', 'on', 'a', 'an', 'and', 'or', 'but', 'in', 'with', - 'to', 'of', 'it', 'be', 'as', 'by', 'that', 'for', 'are', 'was', 'were', -]); - -/** - * Tokenizer for BM25 search - */ -const tokenize = (text: string): string[] => { - const tokens = text.toLowerCase().split(/[\s\-_./\\(){}[\]<>:;,!?'"]+/); - const expanded: string[] = []; - for (const token of tokens) { - if (token.length === 0) continue; - const camelParts = token.replace(/([a-z])([A-Z])/g, '$1 $2').toLowerCase().split(' '); - expanded.push(...camelParts); - if (camelParts.length > 1) { - expanded.push(token); - } - } - return expanded.filter(t => t.length > 1 && !STOP_WORDS.has(t)); -}; - -/** - * Load a BM25 index from disk - */ -export const loadBM25Index = async (filePath: string): Promise => { - try { - const json = await fs.readFile(filePath, 'utf-8'); - // MiniSearch.loadJSON expects the raw JSON string, not a parsed object - searchIndex = MiniSearch.loadJSON(json, { - fields: ['content', 'name'], - storeFields: ['id'], - tokenize, - }); - indexedDocCount = searchIndex.documentCount; - return true; - } catch { - return false; - } -}; - -/** - * Search the BM25 index - */ -export const searchBM25 = (query: string, limit: number = 20): BM25SearchResult[] => { - if (!searchIndex) { - return []; - } - - const results = searchIndex.search(query, { - fuzzy: 0.2, - prefix: true, - boost: { name: 2 }, - }); - - return results.slice(0, limit).map((r, index) => ({ - filePath: r.id, - score: r.score, - rank: index + 1, - })); -}; - -/** - * Check if the BM25 index is ready - */ -export const isBM25Ready = (): boolean => { - return searchIndex !== null && indexedDocCount > 0; -}; - -/** - * Get index statistics - */ -export const getBM25Stats = (): { documentCount: number; termCount: number } => { - if (!searchIndex) { - return { documentCount: 0, termCount: 0 }; - } - return { - documentCount: indexedDocCount, - termCount: searchIndex.termCount, - }; -}; - -/** - * Clear the index - */ -export const clearBM25Index = (): void => { - searchIndex = null; - indexedDocCount = 0; -}; diff --git a/gitnexus/src/mcp/core/kuzu-adapter.ts b/gitnexus/src/mcp/core/kuzu-adapter.ts index 80a063b056..3ef8f89c4f 100644 --- a/gitnexus/src/mcp/core/kuzu-adapter.ts +++ b/gitnexus/src/mcp/core/kuzu-adapter.ts @@ -1,30 +1,37 @@ /** - * KuzuDB Adapter (Read-Only) + * KuzuDB Adapter (Persistent Connection) * - * Simplified adapter for MCP that only reads from existing .gitnexus/ database. + * Holds a single database connection for the lifetime of the MCP session. + * This is safe since the watcher has been removed -- only one process + * accesses the database at a time. */ import fs from 'fs/promises'; -import path from 'path'; import kuzu from 'kuzu'; let db: kuzu.Database | null = null; let conn: kuzu.Connection | null = null; -export const initKuzu = async (dbPath: string): Promise => { - if (conn) return; +/** + * Initialize with a persistent connection to the database + */ +export const initKuzu = async (path: string): Promise => { + if (conn) return; // Already initialized // Check if database exists try { - await fs.stat(dbPath); + await fs.stat(path); } catch { - throw new Error(`KuzuDB not found at ${dbPath}. Run: gitnexus analyze`); + throw new Error(`KuzuDB not found at ${path}. Run: gitnexus analyze`); } - db = new kuzu.Database(dbPath); + db = new kuzu.Database(path); conn = new kuzu.Connection(db); }; +/** + * Execute a query using the persistent connection + */ export const executeQuery = async (cypher: string): Promise => { if (!conn) { throw new Error('KuzuDB not initialized. Call initKuzu first.'); @@ -36,19 +43,21 @@ export const executeQuery = async (cypher: string): Promise => { return rows; }; +/** + * Close the persistent connection + */ export const closeKuzu = async (): Promise => { if (conn) { - try { - await conn.close(); - } catch {} + try { await conn.close(); } catch {} conn = null; } if (db) { - try { - await db.close(); - } catch {} + try { await db.close(); } catch {} db = null; } }; +/** + * Check if the database connection is active + */ export const isKuzuReady = (): boolean => conn !== null && db !== null; diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 43b2dad1b4..bb16d311b4 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -8,8 +8,15 @@ import fs from 'fs/promises'; import path from 'path'; import { initKuzu, executeQuery, closeKuzu, isKuzuReady } from '../core/kuzu-adapter.js'; -import { loadBM25Index, searchBM25, isBM25Ready } from '../core/bm25-index.js'; import { embedQuery, getEmbeddingDims, disposeEmbedder } from '../core/embedder.js'; +import { isGitRepo, getCurrentCommit, getGitRoot } from '../../storage/git.js'; +import { + getStoragePaths as getRepoStoragePaths, + saveMeta as saveRepoMeta, + loadMeta as loadRepoMeta, + addToGitignore, +} from '../../storage/repo-manager.js'; +import { generateAIContextFiles } from '../../cli/ai-context.js'; export interface RepoMeta { repoPath: string; @@ -28,7 +35,6 @@ export interface IndexedRepo { repoPath: string; storagePath: string; kuzuPath: string; - bm25Path: string; metaPath: string; meta: RepoMeta; } @@ -40,7 +46,6 @@ function getStoragePaths(repoPath: string) { return { storagePath, kuzuPath: path.join(storagePath, 'kuzu'), - bm25Path: path.join(storagePath, 'bm25.json'), metaPath: path.join(storagePath, 'meta.json'), }; } @@ -143,7 +148,6 @@ export class LocalBackend { if (this.initialized || !this.repo) return; await initKuzu(this.repo.kuzuPath); - await loadBM25Index(this.repo.bm25Path); this.initialized = true; } @@ -163,6 +167,10 @@ export class LocalBackend { return this.repo?.storagePath || null; } + get meta(): any { + return this.repo?.meta || null; + } + async callTool(method: string, params: any): Promise { if (!this.repo) { throw new Error('Repository not indexed. Run: gitnexus analyze'); @@ -269,12 +277,13 @@ export class LocalBackend { } /** - * BM25 keyword search helper + * BM25 keyword search helper - uses KuzuDB FTS for always-fresh results */ private async bm25Search(query: string, limit: number): Promise { - if (!isBM25Ready()) return []; + // Import dynamically to avoid circular dependency + const { searchFTSFromKuzu } = await import('../../core/search/bm25-index.js'); + const bm25Results = await searchFTSFromKuzu(query, limit); - const bm25Results = searchBM25(query, limit); const results: any[] = []; for (const bm25Result of bm25Results) { @@ -667,14 +676,153 @@ export class LocalBackend { }; } - private async analyze(params: { path?: string; force?: boolean }): Promise { - const targetPath = params.path ? path.resolve(params.path) : process.cwd(); - - return { - action: 'analyze', - targetPath, - message: `To index this repository, run:\n\n cd ${targetPath}\n gitnexus analyze${params.force ? ' --force' : ''}\n\nThis will create a .gitnexus/ folder with the knowledge graph.`, - }; + private async analyze(params: { path?: string; force?: boolean; skipEmbeddings?: boolean }): Promise { + // Determine target repo path + let repoPath: string; + if (params.path) { + repoPath = path.resolve(params.path); + } else if (this.repo) { + repoPath = this.repo.repoPath; + } else { + const gitRoot = getGitRoot(process.cwd()); + if (!gitRoot) { + return { error: 'Not inside a git repository' }; + } + repoPath = gitRoot; + } + + if (!isGitRepo(repoPath)) { + return { error: 'Not a git repository' }; + } + + const { storagePath, kuzuPath } = getRepoStoragePaths(repoPath); + const currentCommit = getCurrentCommit(repoPath); + const existingMeta = await loadRepoMeta(storagePath); + + // Skip if already indexed at same commit (unless force) + if (existingMeta && !params.force && existingMeta.lastCommit === currentCommit) { + return { status: 'up_to_date', message: 'Repository already up to date.' }; + } + + // Close MCP's persistent connection before pipeline takes over + await closeKuzu(); + this.initialized = false; + + try { + // Import pipeline modules dynamically to avoid circular deps + const { runPipelineFromRepo } = await import('../../core/ingestion/pipeline.js'); + const coreKuzu = await import('../../core/kuzu/kuzu-adapter.js'); + + // Run ingestion pipeline + console.error('GitNexus: Running indexing pipeline...'); + const pipelineResult = await runPipelineFromRepo(repoPath, (progress) => { + if (progress.percent % 20 === 0) { + console.error(`GitNexus: ${progress.phase} ${progress.percent}%`); + } + }); + + // Load graph into KuzuDB + console.error('GitNexus: Loading graph into KuzuDB...'); + await coreKuzu.initKuzu(kuzuPath); + await coreKuzu.loadGraphToKuzu(pipelineResult.graph, pipelineResult.fileContents, storagePath); + + // Create FTS indexes + console.error('GitNexus: Creating FTS indexes...'); + try { + await coreKuzu.createFTSIndex('File', 'file_fts', ['name', 'content']); + await coreKuzu.createFTSIndex('Function', 'function_fts', ['name', 'content']); + await coreKuzu.createFTSIndex('Class', 'class_fts', ['name', 'content']); + await coreKuzu.createFTSIndex('Method', 'method_fts', ['name', 'content']); + } catch (e: any) { + console.error('GitNexus: Some FTS indexes may not have been created:', e.message); + } + + // Generate embeddings (unless skipped) + if (!params.skipEmbeddings) { + try { + console.error('GitNexus: Generating embeddings...'); + const { runEmbeddingPipeline } = await import('../../core/embeddings/embedding-pipeline.js'); + await runEmbeddingPipeline( + coreKuzu.executeQuery, + coreKuzu.executeWithReusedStatement, + (progress) => { + if (progress.percent % 25 === 0) { + console.error(`GitNexus: Embeddings ${progress.percent}%`); + } + } + ); + } catch (e: any) { + console.error('GitNexus: Embedding generation failed (non-fatal):', e.message); + } + } + + // Save metadata + const stats = await coreKuzu.getKuzuStats(); + await saveRepoMeta(storagePath, { + repoPath, + lastCommit: currentCommit, + indexedAt: new Date().toISOString(), + stats: { + files: pipelineResult.fileContents.size, + nodes: stats.nodes, + edges: stats.edges, + communities: pipelineResult.communityResult?.stats.totalCommunities, + processes: pipelineResult.processResult?.stats.totalProcesses, + }, + }); + + // Add .gitnexus to .gitignore + await addToGitignore(repoPath); + + // Generate AI context files + const projectName = path.basename(repoPath); + await generateAIContextFiles(repoPath, storagePath, projectName, { + files: pipelineResult.fileContents.size, + nodes: stats.nodes, + edges: stats.edges, + communities: pipelineResult.communityResult?.stats.totalCommunities, + processes: pipelineResult.processResult?.stats.totalProcesses, + }); + + // Close core kuzu connection (pipeline is done) + await coreKuzu.closeKuzu(); + + // Re-init MCP state so next tool call reconnects + this.repo = await loadRepo(repoPath); + if (this.repo) { + const repoStats = this.repo.meta.stats || {}; + this._context = { + projectName: path.basename(this.repo.repoPath), + stats: { + fileCount: repoStats.files || 0, + functionCount: repoStats.nodes || 0, + classCount: 0, + interfaceCount: 0, + methodCount: 0, + communityCount: repoStats.communities || 0, + processCount: repoStats.processes || 0, + }, + hotspots: [], + folderTree: '', + }; + } + + console.error('GitNexus: Indexing complete!'); + return { + status: 'success', + message: `Repository indexed successfully.`, + stats: { + files: pipelineResult.fileContents.size, + nodes: stats.nodes, + edges: stats.edges, + communities: pipelineResult.communityResult?.stats.totalCommunities, + processes: pipelineResult.processResult?.stats.totalProcesses, + }, + }; + } catch (e: any) { + console.error('GitNexus: Indexing failed:', e.message); + return { error: `Indexing failed: ${e.message}` }; + } } async disconnect(): Promise { diff --git a/gitnexus/src/mcp/resources.ts b/gitnexus/src/mcp/resources.ts index d184bd46a4..1707cb893b 100644 --- a/gitnexus/src/mcp/resources.ts +++ b/gitnexus/src/mcp/resources.ts @@ -6,6 +6,7 @@ */ import type { LocalBackend } from './local/local-backend.js'; +import { checkStaleness } from './staleness.js'; export interface ResourceDefinition { uri: string; @@ -113,28 +114,42 @@ async function getContextResource(backend: LocalBackend): Promise { return 'error: No codebase loaded. Run: gitnexus analyze'; } + // Check staleness + const repoPath = backend.repoPath; + const lastCommit = backend.meta?.lastCommit || 'HEAD'; + const staleness = repoPath ? checkStaleness(repoPath, lastCommit) : { isStale: false, commitsBehind: 0 }; + const lines: string[] = [ `project: ${context.projectName}`, - 'stats:', - ` files: ${context.stats.fileCount}`, - ` symbols: ${context.stats.functionCount}`, - ` clusters: ${context.stats.communityCount}`, - ` processes: ${context.stats.processCount}`, - '', - 'tools_available:', - ' - search: Hybrid semantic + keyword search', - ' - explore: Deep dive on symbol/cluster/process', - ' - impact: Blast radius analysis', - ' - overview: List all clusters and processes', - ' - cypher: Raw graph queries', - '', - 'resources_available:', - ' - gitnexus://clusters: All clusters', - ' - gitnexus://processes: All processes', - ' - gitnexus://cluster/{name}: Cluster details', - ' - gitnexus://process/{name}: Process trace', ]; + // Add staleness warning if index is behind + if (staleness.isStale && staleness.hint) { + lines.push(''); + lines.push(`staleness: "${staleness.hint}"`); + } + + lines.push(''); + lines.push('stats:'); + lines.push(` files: ${context.stats.fileCount}`); + lines.push(` symbols: ${context.stats.functionCount}`); + lines.push(` clusters: ${context.stats.communityCount}`); + lines.push(` processes: ${context.stats.processCount}`); + lines.push(''); + lines.push('tools_available:'); + lines.push(' - search: Hybrid semantic + keyword search'); + lines.push(' - explore: Deep dive on symbol/cluster/process'); + lines.push(' - impact: Blast radius analysis'); + lines.push(' - overview: List all clusters and processes'); + lines.push(' - cypher: Raw graph queries'); + lines.push(' - analyze: Re-index to update stale data'); + lines.push(''); + lines.push('resources_available:'); + lines.push(' - gitnexus://clusters: All clusters'); + lines.push(' - gitnexus://processes: All processes'); + lines.push(' - gitnexus://cluster/{name}: Cluster details'); + lines.push(' - gitnexus://process/{name}: Process trace'); + return lines.join('\n'); } diff --git a/gitnexus/src/mcp/staleness.ts b/gitnexus/src/mcp/staleness.ts new file mode 100644 index 0000000000..0b7cf2a15e --- /dev/null +++ b/gitnexus/src/mcp/staleness.ts @@ -0,0 +1,43 @@ +/** + * Staleness Check + * + * Checks if the GitNexus index is behind the current git HEAD. + * Returns a hint for the LLM to call analyze if stale. + */ + +import { execSync } from 'child_process'; +import path from 'path'; + +export interface StalenessInfo { + isStale: boolean; + commitsBehind: number; + hint?: string; +} + +/** + * Check how many commits the index is behind HEAD + */ +export function checkStaleness(repoPath: string, lastCommit: string): StalenessInfo { + try { + // Get count of commits between lastCommit and HEAD + const result = execSync( + `git rev-list --count ${lastCommit}..HEAD`, + { cwd: repoPath, encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] } + ).trim(); + + const commitsBehind = parseInt(result, 10) || 0; + + if (commitsBehind > 0) { + return { + isStale: true, + commitsBehind, + hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`, + }; + } + + return { isStale: false, commitsBehind: 0 }; + } catch { + // If git command fails, assume not stale (fail open) + return { isStale: false, commitsBehind: 0 }; + } +} diff --git a/gitnexus/src/mcp/tools.ts b/gitnexus/src/mcp/tools.ts index 1fd3263304..a6cb5c877d 100644 --- a/gitnexus/src/mcp/tools.ts +++ b/gitnexus/src/mcp/tools.ts @@ -24,18 +24,22 @@ export interface ToolDefinition { export const GITNEXUS_TOOLS: ToolDefinition[] = [ { name: 'analyze', - description: `Index or re-index the current repository. + description: `Index or re-index the current repository. Runs the full pipeline in-process. Creates .gitnexus/ in repo root with: - Knowledge graph (functions, classes, calls, imports) -- BM25 search index +- Full-text search indexes - Community detection (Leiden) - Process tracing +- Embeddings for semantic search Run this when: - First time using GitNexus on a repo - After major code changes -- When 'not indexed' error appears`, +- When staleness warning appears in gitnexus://context +- When 'not indexed' error appears + +Note: This may take 30-120 seconds for large repos.`, inputSchema: { type: 'object', properties: { diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index caa2c4fb85..b5d0d7fc90 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -12,7 +12,7 @@ import { findRepo, loadMeta } from '../storage/repo-manager.js'; import { initKuzu, executeQuery } from '../core/kuzu/kuzu-adapter.js'; import { NODE_TABLES } from '../core/kuzu/schema.js'; import { GraphNode, GraphRelationship } from '../core/graph/types.js'; -import { loadBM25Index, searchBM25, isBM25Ready } from '../core/search/bm25-index.js'; +import { searchFTSFromKuzu } from '../core/search/bm25-index.js'; import { hybridSearch } from '../core/search/hybrid-search.js'; import { semanticSearch } from '../core/embeddings/embedding-pipeline.js'; import { isEmbedderReady } from '../core/embeddings/embedder.js'; @@ -131,28 +131,19 @@ export const createServer = async (port: number) => { return; } await initKuzu(repo.kuzuPath); - await loadBM25Index(repo.bm25Path); const query = req.body.query ?? ''; const limit = req.body.limit ?? 10; - if (isBM25Ready() && isEmbedderReady()) { + if (isEmbedderReady()) { const results = await hybridSearch(query, limit, executeQuery, semanticSearch); res.json({ results }); return; } - if (isBM25Ready()) { - res.json({ results: searchBM25(query, limit) }); - return; - } - - if (isEmbedderReady()) { - res.json({ results: await semanticSearch(executeQuery, query, limit) }); - return; - } - - res.json({ results: [] }); + // FTS-only fallback when embeddings aren't loaded + const results = await searchFTSFromKuzu(query, limit); + res.json({ results }); }); // Read file diff --git a/gitnexus/src/storage/repo-manager.ts b/gitnexus/src/storage/repo-manager.ts index 81b47ce3c3..151dc073b9 100644 --- a/gitnexus/src/storage/repo-manager.ts +++ b/gitnexus/src/storage/repo-manager.ts @@ -17,6 +17,7 @@ export interface RepoMeta { edges?: number; communities?: number; processes?: number; + embeddings?: number; }; } @@ -24,7 +25,6 @@ export interface IndexedRepo { repoPath: string; storagePath: string; kuzuPath: string; - bm25Path: string; metaPath: string; meta: RepoMeta; } @@ -46,7 +46,6 @@ export const getStoragePaths = (repoPath: string) => { return { storagePath, kuzuPath: path.join(storagePath, 'kuzu'), - bm25Path: path.join(storagePath, 'bm25.json'), metaPath: path.join(storagePath, 'meta.json'), }; }; diff --git a/gitnexus/vendor/leiden/index.cjs b/gitnexus/vendor/leiden/index.cjs new file mode 100644 index 0000000000..86dab87382 --- /dev/null +++ b/gitnexus/vendor/leiden/index.cjs @@ -0,0 +1,355 @@ +/** + * Graphology Leiden Algorithm + * ============================ + * + * JavaScript implementation of the Leiden community detection + * algorithm for graphology. + * + * Vendored from: https://github.com/graphology/graphology/tree/master/src/communities-leiden + * License: MIT + * + * [Reference] + * Traag, V. A., et al. "From Louvain to Leiden: Guaranteeing Well-Connected + * Communities". Scientific Reports, vol. 9, no 1, 2019, p. 5233. + * https://arxiv.org/abs/1810.08473 + */ +var resolveDefaults = require('graphology-utils/defaults'); +var isGraph = require('graphology-utils/is-graph'); +var inferType = require('graphology-utils/infer-type'); +var SparseMap = require('mnemonist/sparse-map'); +var SparseQueueSet = require('mnemonist/sparse-queue-set'); +var createRandomIndex = require('pandemonium/random-index').createRandomIndex; +var utils = require('./utils.cjs'); + +var indices = require('graphology-indices/louvain'); +var addWeightToCommunity = utils.addWeightToCommunity; + +var UndirectedLouvainIndex = indices.UndirectedLouvainIndex; + +var UndirectedLeidenAddenda = utils.UndirectedLeidenAddenda; + +var DEFAULTS = { + attributes: { + community: 'community', + weight: 'weight' + }, + randomness: 0.01, + randomWalk: true, + resolution: 1, + rng: Math.random, + weighted: false +}; + +var EPSILON = 1e-10; + +function tieBreaker( + bestCommunity, + currentCommunity, + targetCommunity, + delta, + bestDelta +) { + if (Math.abs(delta - bestDelta) < EPSILON) { + if (bestCommunity === currentCommunity) { + return false; + } else { + return targetCommunity > bestCommunity; + } + } else if (delta > bestDelta) { + return true; + } + + return false; +} + +function undirectedLeiden(detailed, graph, options) { + var index = new UndirectedLouvainIndex(graph, { + attributes: { + weight: options.attributes.weight + }, + keepDendrogram: detailed, + resolution: options.resolution, + weighted: options.weighted + }); + + var addenda = new UndirectedLeidenAddenda(index, { + randomness: options.randomness, + rng: options.rng + }); + + var randomIndex = createRandomIndex(options.rng); + + // Communities + var currentCommunity, targetCommunity; + var communities = new SparseMap(Float64Array, index.C); + + // Traversal + var queue = new SparseQueueSet(index.C), + start, + end, + weight, + ci, + ri, + s, + i, + j, + l; + + // Metrics + var degree, targetCommunityDegree; + + // Moves + var bestCommunity, bestDelta, deltaIsBetter, delta; + + // Details + var deltaComputations = 0, + nodesVisited = 0, + moves = [], + currentMoves; + + while (true) { + l = index.C; + + currentMoves = 0; + + // Traversal of the graph + ri = options.randomWalk ? randomIndex(l) : 0; + + for (s = 0; s < l; s++, ri++) { + i = ri % l; + queue.enqueue(i); + } + + while (queue.size !== 0) { + i = queue.dequeue(); + nodesVisited++; + + degree = 0; + communities.clear(); + + currentCommunity = index.belongings[i]; + + start = index.starts[i]; + end = index.starts[i + 1]; + + // Traversing neighbors + for (; start < end; start++) { + j = index.neighborhood[start]; + weight = index.weights[start]; + + targetCommunity = index.belongings[j]; + + // Incrementing metrics + degree += weight; + addWeightToCommunity(communities, targetCommunity, weight); + } + + // Finding best community to move to + bestDelta = index.fastDeltaWithOwnCommunity( + i, + degree, + communities.get(currentCommunity) || 0, + currentCommunity + ); + bestCommunity = currentCommunity; + + for (ci = 0; ci < communities.size; ci++) { + targetCommunity = communities.dense[ci]; + + if (targetCommunity === currentCommunity) continue; + + targetCommunityDegree = communities.vals[ci]; + + deltaComputations++; + + delta = index.fastDelta( + i, + degree, + targetCommunityDegree, + targetCommunity + ); + + deltaIsBetter = tieBreaker( + bestCommunity, + currentCommunity, + targetCommunity, + delta, + bestDelta + ); + + if (deltaIsBetter) { + bestDelta = delta; + bestCommunity = targetCommunity; + } + } + + if (bestDelta < 0) { + bestCommunity = index.isolate(i, degree); + + if (bestCommunity === currentCommunity) continue; + } else { + if (bestCommunity === currentCommunity) { + continue; + } else { + index.move(i, degree, bestCommunity); + } + } + + currentMoves++; + + // Adding neighbors from other communities to the queue + start = index.starts[i]; + end = index.starts[i + 1]; + + for (; start < end; start++) { + j = index.neighborhood[start]; + targetCommunity = index.belongings[j]; + + if (targetCommunity !== bestCommunity) queue.enqueue(j); + } + } + + moves.push(currentMoves); + + if (currentMoves === 0) { + index.zoomOut(); + break; + } + + if (!addenda.onlySingletons()) { + // We continue working on the induced graph + addenda.zoomOut(); + continue; + } + + break; + } + + var results = { + index: index, + deltaComputations: deltaComputations, + nodesVisited: nodesVisited, + moves: moves + }; + + return results; +} + +/** + * Function returning the communities mapping of the graph. + * + * @param {boolean} assign - Assign communities to nodes attributes? + * @param {boolean} detailed - Whether to return detailed information. + * @param {Graph} graph - Target graph. + * @param {object} options - Options: + * @param {object} attributes - Attribute names: + * @param {string} community - Community node attribute name. + * @param {string} weight - Weight edge attribute name. + * @param {number} randomness - Randomness parameter. + * @param {boolean} randomWalk - Whether to traverse the graph in random order. + * @param {number} resolution - Resolution parameter. + * @param {function} rng - RNG function to use. + * @param {boolean} weighted - Whether to compute the weighted version. + * @return {object} + */ +function leiden(assign, detailed, graph, options) { + if (!isGraph(graph)) + throw new Error( + 'graphology-communities-leiden: the given graph is not a valid graphology instance.' + ); + + var type = inferType(graph); + + if (type === 'mixed') + throw new Error( + 'graphology-communities-leiden: cannot run the algorithm on a true mixed graph.' + ); + + if (type === 'directed') + throw new Error( + 'graphology-communities-leiden: not yet implemented for directed graphs.' + ); + + // Attributes name + options = resolveDefaults(options, DEFAULTS); + + // Empty graph case + var c = 0; + + if (graph.size === 0) { + if (assign) { + graph.forEachNode(function (node) { + graph.setNodeAttribute(node, options.attributes.communities, c++); + }); + + return; + } + + var communities = {}; + + graph.forEachNode(function (node) { + communities[node] = c++; + }); + + if (!detailed) return communities; + + return { + communities: communities, + count: graph.order, + deltaComputations: 0, + dendrogram: null, + level: 0, + modularity: NaN, + moves: null, + nodesVisited: 0, + resolution: options.resolution + }; + } + + var fn = undirectedLeiden; + + var results = fn(detailed, graph, options); + + var index = results.index; + + // Standard output + if (!detailed) { + if (assign) { + index.assign(options.attributes.community); + return; + } + + return index.collect(); + } + + // Detailed output + var output = { + count: index.C, + deltaComputations: results.deltaComputations, + dendrogram: index.dendrogram, + level: index.level, + modularity: index.modularity(), + moves: results.moves, + nodesVisited: results.nodesVisited, + resolution: options.resolution + }; + + if (assign) { + index.assign(options.attributes.community); + return output; + } + + output.communities = index.collect(); + + return output; +} + +/** + * Exporting. + */ +var fn = leiden.bind(null, false, false); +fn.assign = leiden.bind(null, true, false); +fn.detailed = leiden.bind(null, false, true); +fn.defaults = DEFAULTS; + +module.exports = fn; diff --git a/gitnexus/vendor/leiden/utils.cjs b/gitnexus/vendor/leiden/utils.cjs new file mode 100644 index 0000000000..b6c6132da8 --- /dev/null +++ b/gitnexus/vendor/leiden/utils.cjs @@ -0,0 +1,392 @@ +/** + * Graphology Leiden Utils + * ======================== + * + * Miscellaneous utilities used by the Leiden algorithm. + * + * Vendored from: https://github.com/graphology/graphology/tree/master/src/communities-leiden + * License: MIT + */ +var SparseMap = require('mnemonist/sparse-map'); +var createRandom = require('pandemonium/random').createRandom; + +function addWeightToCommunity(map, community, weight) { + var currentWeight = map.get(community); + + if (typeof currentWeight === 'undefined') currentWeight = 0; + + currentWeight += weight; + + map.set(community, currentWeight); +} + +function UndirectedLeidenAddenda(index, options) { + options = options || {}; + + var rng = options.rng || Math.random; + var randomness = 'randomness' in options ? options.randomness : 0.01; + + this.index = index; + this.random = createRandom(rng); + this.randomness = randomness; + this.rng = rng; + + var NodesPointerArray = index.counts.constructor; + var WeightsArray = index.weights.constructor; + + var order = index.C; + this.resolution = index.resolution; + + // Used to group nodes by communities + this.B = index.C; + this.C = 0; + this.communitiesOffsets = new NodesPointerArray(order); + this.nodesSortedByCommunities = new NodesPointerArray(order); + this.communitiesBounds = new NodesPointerArray(order + 1); + + // Used to merge nodes subsets + this.communityWeights = new WeightsArray(order); + this.degrees = new WeightsArray(order); + this.nonSingleton = new Uint8Array(order); + this.externalEdgeWeightPerCommunity = new WeightsArray(order); + this.belongings = new NodesPointerArray(order); + this.neighboringCommunities = new SparseMap(WeightsArray, order); + this.cumulativeIncrement = new Float64Array(order); + this.macroCommunities = null; +} + +UndirectedLeidenAddenda.prototype.groupByCommunities = function () { + var index = this.index; + + var n, i, c, b, o; + + n = 0; + o = 0; + + for (i = 0; i < index.C; i++) { + c = index.counts[i]; + + if (c !== 0) { + this.communitiesBounds[o++] = n; + n += c; + this.communitiesOffsets[i] = n; + } + } + + this.communitiesBounds[o] = n; + + o = 0; + + for (i = 0; i < index.C; i++) { + b = index.belongings[i]; + o = --this.communitiesOffsets[b]; + this.nodesSortedByCommunities[o] = i; + } + + this.B = index.C - index.U; + this.C = index.C; +}; + +UndirectedLeidenAddenda.prototype.communities = function () { + var communities = new Array(this.B); + + var i, j, community, start, stop; + + for (i = 0; i < this.B; i++) { + start = this.communitiesBounds[i]; + stop = this.communitiesBounds[i + 1]; + community = []; + + for (j = start; j < stop; j++) { + community.push(j); + } + + communities[i] = community; + } + + return communities; +}; + +UndirectedLeidenAddenda.prototype.mergeNodesSubset = function (start, stop) { + var index = this.index; + var currentMacroCommunity = + index.belongings[this.nodesSortedByCommunities[start]]; + var neighboringCommunities = this.neighboringCommunities; + + var totalNodeWeight = 0; + + var i, j, w; + var ei, el, et; + + // Initializing singletons + for (j = start; j < stop; j++) { + i = this.nodesSortedByCommunities[j]; + + this.belongings[i] = i; + this.nonSingleton[i] = 0; + this.degrees[i] = 0; + totalNodeWeight += index.loops[i] / 2; + + this.communityWeights[i] = index.loops[i]; + this.externalEdgeWeightPerCommunity[i] = 0; + + ei = index.starts[i]; + el = index.starts[i + 1]; + + for (; ei < el; ei++) { + et = index.neighborhood[ei]; + w = index.weights[ei]; + + this.degrees[i] += w; + + if (index.belongings[et] !== currentMacroCommunity) continue; + + totalNodeWeight += w; + this.externalEdgeWeightPerCommunity[i] += w; + this.communityWeights[i] += w; + } + } + + var microDegrees = this.externalEdgeWeightPerCommunity.slice(); + + var s, ri, ci; + var order = stop - start; + + var degree, + bestCommunity, + qualityValueIncrement, + maxQualityValueIncrement, + totalTransformedQualityValueIncrement, + targetCommunity, + targetCommunityDegree, + targetCommunityWeight; + + var r, lo, hi, mid, chosenCommunity; + + ri = this.random(start, stop - 1); + + for (s = start; s < stop; s++, ri++) { + j = start + (ri % order); + + i = this.nodesSortedByCommunities[j]; + + if (this.nonSingleton[i] === 1) { + continue; + } + + if ( + this.externalEdgeWeightPerCommunity[i] < + this.communityWeights[i] * + (totalNodeWeight / 2 - this.communityWeights[i]) * + this.resolution + ) { + continue; + } + + this.communityWeights[i] = 0; + this.externalEdgeWeightPerCommunity[i] = 0; + + neighboringCommunities.clear(); + neighboringCommunities.set(i, 0); + + degree = 0; + + ei = index.starts[i]; + el = index.starts[i + 1]; + + for (; ei < el; ei++) { + et = index.neighborhood[ei]; + + if (index.belongings[et] !== currentMacroCommunity) continue; + + w = index.weights[ei]; + + degree += w; + + addWeightToCommunity(neighboringCommunities, this.belongings[et], w); + } + + bestCommunity = i; + maxQualityValueIncrement = 0; + totalTransformedQualityValueIncrement = 0; + + for (ci = 0; ci < neighboringCommunities.size; ci++) { + targetCommunity = neighboringCommunities.dense[ci]; + targetCommunityDegree = neighboringCommunities.vals[ci]; + targetCommunityWeight = this.communityWeights[targetCommunity]; + + if ( + this.externalEdgeWeightPerCommunity[targetCommunity] >= + targetCommunityWeight * + (totalNodeWeight / 2 - targetCommunityWeight) * + this.resolution + ) { + qualityValueIncrement = + targetCommunityDegree - + ((degree + index.loops[i]) * + targetCommunityWeight * + this.resolution) / + totalNodeWeight; + + if (qualityValueIncrement > maxQualityValueIncrement) { + bestCommunity = targetCommunity; + maxQualityValueIncrement = qualityValueIncrement; + } + + if (qualityValueIncrement >= 0) + totalTransformedQualityValueIncrement += Math.exp( + qualityValueIncrement / this.randomness + ); + } + + this.cumulativeIncrement[ci] = totalTransformedQualityValueIncrement; + } + + if ( + totalTransformedQualityValueIncrement < Number.MAX_VALUE && + totalTransformedQualityValueIncrement < Infinity + ) { + r = totalTransformedQualityValueIncrement * this.rng(); + lo = -1; + hi = neighboringCommunities.size + 1; + + while (lo < hi - 1) { + mid = (lo + hi) >>> 1; + + if (this.cumulativeIncrement[mid] >= r) hi = mid; + else lo = mid; + } + + chosenCommunity = neighboringCommunities.dense[hi]; + } else { + chosenCommunity = bestCommunity; + } + + this.communityWeights[chosenCommunity] += degree + index.loops[i]; + + ei = index.starts[i]; + el = index.starts[i + 1]; + + for (; ei < el; ei++) { + et = index.neighborhood[ei]; + + if (index.belongings[et] !== currentMacroCommunity) continue; + + targetCommunity = this.belongings[et]; + + if (targetCommunity === chosenCommunity) { + this.externalEdgeWeightPerCommunity[chosenCommunity] -= + microDegrees[et]; + } else { + this.externalEdgeWeightPerCommunity[chosenCommunity] += + microDegrees[et]; + } + } + + if (chosenCommunity !== i) { + this.belongings[i] = chosenCommunity; + this.nonSingleton[chosenCommunity] = 1; + this.C--; + } + } + + var microCommunities = this.neighboringCommunities; + microCommunities.clear(); + + for (j = start; j < stop; j++) { + i = this.nodesSortedByCommunities[j]; + microCommunities.set(this.belongings[i], 1); + } + + return microCommunities.dense.slice(0, microCommunities.size); +}; + +UndirectedLeidenAddenda.prototype.refinePartition = function () { + this.groupByCommunities(); + + this.macroCommunities = new Array(this.B); + + var i, start, stop, mapping; + + var bounds = this.communitiesBounds; + + for (i = 0; i < this.B; i++) { + start = bounds[i]; + stop = bounds[i + 1]; + + mapping = this.mergeNodesSubset(start, stop); + this.macroCommunities[i] = mapping; + } +}; + +UndirectedLeidenAddenda.prototype.split = function () { + var index = this.index; + var isolates = this.neighboringCommunities; + + isolates.clear(); + + var i, community, isolated; + + for (i = 0; i < index.C; i++) { + community = this.belongings[i]; + + if (i !== community) continue; + + isolated = index.isolate(i, this.degrees[i]); + isolates.set(community, isolated); + } + + for (i = 0; i < index.C; i++) { + community = this.belongings[i]; + + if (i === community) continue; + + isolated = isolates.get(community); + index.move(i, this.degrees[i], isolated); + } + + var j, macro; + + for (i = 0; i < this.macroCommunities.length; i++) { + macro = this.macroCommunities[i]; + + for (j = 0; j < macro.length; j++) macro[j] = isolates.get(macro[j]); + } +}; + +UndirectedLeidenAddenda.prototype.zoomOut = function () { + var index = this.index; + this.refinePartition(); + this.split(); + + var newLabels = index.zoomOut(); + + var macro, leader, follower; + + var i, j; + + for (i = 0; i < this.macroCommunities.length; i++) { + macro = this.macroCommunities[i]; + leader = newLabels[macro[0]]; + + for (j = 1; j < macro.length; j++) { + follower = newLabels[macro[j]]; + index.expensiveMove(follower, leader); + } + } +}; + +UndirectedLeidenAddenda.prototype.onlySingletons = function () { + var index = this.index; + + var i; + + for (i = 0; i < index.C; i++) { + if (index.counts[i] > 1) return false; + } + + return true; +}; + +exports.addWeightToCommunity = addWeightToCommunity; +exports.UndirectedLeidenAddenda = UndirectedLeidenAddenda;