From 889beff56fa1c24201718d6ad77afe9cbd2f0cc4 Mon Sep 17 00:00:00 2001 From: abhigyantrumio Date: Wed, 20 Aug 2025 06:12:48 +0530 Subject: [PATCH] structured output fix wip, ChatInterface better UX --- repomix-output.txt | 27658 --------------------- src/ai/react-agent-debug.ts | 47 + src/ai/react-agent.test.ts | 91 +- src/ai/react-agent.ts | 198 +- src/lib/chat-history.ts | 2 +- src/lib/query-cache.ts | 398 + src/ui/components/chat/ChatInterface.tsx | 617 +- 7 files changed, 913 insertions(+), 28098 deletions(-) delete mode 100644 repomix-output.txt create mode 100644 src/ai/react-agent-debug.ts create mode 100644 src/lib/query-cache.ts diff --git a/repomix-output.txt b/repomix-output.txt deleted file mode 100644 index 389be6a22..000000000 --- a/repomix-output.txt +++ /dev/null @@ -1,27658 +0,0 @@ -This file is a merged representation of the entire codebase, combining all repository files into a single document. -Generated by Repomix on: 2025-08-17T22:04:12.377Z - -================================================================ -File Summary -================================================================ - -Purpose: --------- -This file contains a packed representation of the entire repository's contents. -It is designed to be easily consumable by AI systems for analysis, code review, -or other automated processes. - -File Format: ------------- -The content is organized as follows: -1. This summary section -2. Repository information -3. Directory structure -4. Multiple file entries, each consisting of: - a. A separator line (================) - b. The file path (File: path/to/file) - c. Another separator line - d. The full contents of the file - e. A blank line - -Usage Guidelines: ------------------ -- This file should be treated as read-only. Any changes should be made to the - original repository files, not this packed version. -- When processing this file, use the file path to distinguish - between different files in the repository. -- Be aware that this file may contain sensitive information. Handle it with - the same level of security as you would the original repository. - -Notes: ------- -- Some files may have been excluded based on .gitignore rules and Repomix's - configuration. -- Binary files are not included in this packed representation. Please refer to - the Repository Structure section for a complete list of file paths, including - binary files. - -Additional Info: ----------------- - -================================================================ -Directory Structure -================================================================ -.gitignore -Agent.md -ARCHITECTURE_UPGRADE.md -CONVERSION_SUMMARY.md -DEBUG_FEATURES.md -eslint.config.js -FILTERING_FIX.md -GITNEXUS_README.md -index.html -ISOLATED_NODES_FIX.md -jest.config.cjs -KUZUDB_INTEGRATION_GUIDE.md -log.txt -package.json -project_guide.md -PRUNING_IMPLEMENTATION.md -public/vite.svg -public/workers/file-processing-worker.js -public/workers/generic-worker.js -public/workers/tree-sitter-worker.js -quicksort.js -README.md -src/__tests__/error-handler.test.ts -src/__tests__/health-monitor.test.ts -src/__tests__/kuzu.test.ts -src/__tests__/memory-manager.test.ts -src/__tests__/setup.ts -src/__tests__/streaming-processor.test.ts -src/__tests__/utils.test.ts -src/ai/cypher-generator.ts -src/ai/index.ts -src/ai/kuzu-rag-orchestrator.ts -src/ai/langchain-orchestrator.ts -src/ai/llm-service.ts -src/ai/orchestrator.ts -src/ai/prompts/kuzu-performance-prompts.ts -src/App.css -src/App.tsx -src/assets/react.svg -src/config/config.ts -src/config/feature-flags.ts -src/config/language-config.ts -src/core/graph/graph.ts -src/core/graph/kuzu-query-engine.ts -src/core/graph/query-engine.ts -src/core/graph/query.ts -src/core/graph/trie.ts -src/core/graph/types.ts -src/core/ingestion/call-processor.ts -src/core/ingestion/import-processor.ts -src/core/ingestion/parallel-parsing-processor.ts -src/core/ingestion/parallel-pipeline.ts -src/core/ingestion/parsing-processor.ts -src/core/ingestion/pipeline.ts -src/core/ingestion/structure-processor.ts -src/core/ingestion/tree-sitter-queries.ts -src/core/kuzu/kuzu-loader.ts -src/core/tree-sitter/parser-loader.ts -src/index.css -src/lib/error-handler.ts -src/lib/export.test.ts -src/lib/export.ts -src/lib/kuzu-integration.ts -src/lib/kuzu-performance-benchmark.ts -src/lib/kuzu-performance-monitor.ts -src/lib/kuzu-test.ts -src/lib/polyfills.ts -src/lib/preload.ts -src/lib/shared-utils.ts -src/lib/streaming-processor.ts -src/lib/utils.ts -src/lib/validation.ts -src/lib/web-worker-pool.ts -src/lib/worker-pool-test.ts -src/lib/worker-pool.ts -src/lib/workerUtils.ts -src/main.tsx -src/services/github.ts -src/services/health-monitor.ts -src/services/ingestion.service.ts -src/services/kuzu.service.ts -src/services/memory-manager.ts -src/services/zip.ts -src/ui/components/chat/ChatInterface.tsx -src/ui/components/chat/CodeAssistant.tsx -src/ui/components/chat/index.ts -src/ui/components/ErrorBoundary.tsx -src/ui/components/ExportFormatModal.tsx -src/ui/components/graph/GraphExplorer.tsx -src/ui/components/graph/index.ts -src/ui/components/graph/SourceViewer.tsx -src/ui/components/graph/Visualization.tsx -src/ui/components/index.ts -src/ui/index.ts -src/ui/pages/HomePage.tsx -src/ui/pages/index.ts -src/vite-env.d.ts -src/workers/ingestion.worker.ts -STRUCTURE_FIX_SUMMARY.md -tsconfig.app.json -tsconfig.app.tsbuildinfo -tsconfig.json -tsconfig.node.json -tsconfig.node.tsbuildinfo -vite.config.ts -WORKER_POOL_IMPLEMENTATION_SUMMARY.md -WORKER_POOL_IMPLEMENTATION.md - -================================================================ -Files -================================================================ - -================ -File: .gitignore -================ -# Logs -logs -*.log -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* -lerna-debug.log* - -node_modules -dist -dist-ssr -*.local - -# Editor directories and files -.vscode/* -!.vscode/extensions.json -.idea -.DS_Store -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw? - -# AI/Development tool directories -.kilocode/ -.gemini/ -.cursor/ -.clinerules/ - -================ -File: Agent.md -================ -# GitNexus - Complete Agent Documentation - -## ๐ŸŽฏ Project Overview - -**GitNexus** is a client-side, edge-based code knowledge graph generator that transforms any codebase into an interactive knowledge graph with AI-powered Graph RAG capabilities. It runs entirely in the browser with zero server dependencies. - -### Core Mission - -- **Zero-Setup Code Intelligence**: Analyze codebases without servers or configuration -- **Graph RAG-Powered**: Use knowledge graphs for AI-powered code understanding -- **Multi-Language Support**: Currently Python-focused with extensible architecture -- **Browser-Native**: All processing happens client-side using WebAssembly and Web Workers - -### Key Capabilities - -- **GitHub Integration**: Direct repository analysis via GitHub API -- **ZIP Processing**: Local archive analysis with intelligent filtering -- **Interactive Visualization**: Cytoscape.js-powered knowledge graphs -- **AI Chat Interface**: Multi-LLM support (OpenAI, Anthropic, Gemini) -- **Advanced Parsing**: Tree-sitter WASM for accurate AST analysis - -## ๐Ÿ—๏ธ Architecture Deep Dive - -### System Architecture Layers - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ User Interface Layer โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ HomePage โ”‚ โ”‚ Chat UI โ”‚ โ”‚ Graph Explorer โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Service Layer โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ GitHub API โ”‚ โ”‚ ZIP Service โ”‚ โ”‚ Ingestion Service โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Processing Pipeline โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Structure โ”‚ โ”‚ Parsing โ”‚ โ”‚ Call Resolution โ”‚ โ”‚ -โ”‚ โ”‚ Processor โ”‚ โ”‚ Processor โ”‚ โ”‚ 3-Stage Strategy โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Core Engine โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Graph โ”‚ โ”‚ Function โ”‚ โ”‚ Import โ”‚ โ”‚ -โ”‚ โ”‚ Types โ”‚ โ”‚ Registry โ”‚ โ”‚ Resolution โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ (Trie) โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### Technology Stack - -**Frontend Framework**: React 18 + TypeScript + Vite -**Graph Visualization**: Cytoscape.js + d3.js -**Code Parsing**: Tree-sitter WebAssembly -**AI Integration**: LangChain.js with ReAct pattern -**State Management**: React Context + custom hooks -**Build System**: Vite with WASM support - -## ๐Ÿ“Š 4-Pass Processing Pipeline - -### Pass 1: Structure Analysis (`StructureProcessor`) - -**Purpose**: Discover complete repository structure without filtering - -**Key Innovations**: - -- **Complete Discovery**: Finds ALL directories and files (including ignored ones) -- **No Early Filtering**: Preserves complete structure for accurate representation -- **Smart Categorization**: Distinguishes files from directories algorithmically -- **Intermediate Paths**: Automatically discovers missing directory levels - -**Implementation Details**: - -```typescript -// Direct path processing instead of inference -const { directories, files } = this.categorizePaths(allPaths); -``` - -### Pass 2: Code Parsing & Definition Extraction (`ParsingProcessor`) - -**Purpose**: Parse source code and extract definitions while applying intelligent filtering - -**Key Components**: - -- **Tree-sitter Integration**: WASM-based parsing for multiple languages -- **Function Registry Trie**: Optimized data structure for definition lookups -- **Two-Stage Filtering**: - - Stage 1: Prune ignored directories (node_modules, .git, etc.) - - Stage 2: Apply user filters (directory patterns, file extensions) - -**Ignore Patterns**: - -```typescript -IGNORE_PATTERNS = [ - '.git', 'node_modules', '__pycache__', '.venv', 'build', 'dist', - '.vscode', '.idea', 'tmp', 'logs', 'coverage' -] -``` - -### Pass 3: Import Resolution (`ImportProcessor`) - -**Purpose**: Build comprehensive project-wide import map - -**Features**: - -- **Multi-language Support**: Python, JavaScript, TypeScript imports -- **Path Resolution**: Handles relative and absolute imports -- **Alias Tracking**: Maps local names to actual exported functions -- **Validation**: Checks against actual project files - -**Resolution Patterns**: - -- Python: `import`, `from...import` -- JS/TS: `import`, `require()`, `export` -- Path normalization for complex project structures - -### Pass 4: Call Resolution (`CallProcessor`) - -**Purpose**: Resolve function calls using 3-stage strategy - -**3-Stage Resolution Strategy**: - -1. **Exact Match** (High Confidence): Uses import map for direct resolution -2. **Same-Module Match** (High Confidence): Local function calls within files -3. **Heuristic Fallback** (Intelligent): Uses FunctionRegistryTrie with distance scoring - -**Heuristic Algorithm**: - -```typescript -// Distance-based scoring for ambiguous calls -const distance = max(caller_parts, candidate_parts) - common_prefix_length -const score = distance - sibling_bonus -``` - -## ๐Ÿค– AI Integration Architecture - -### ReAct Agent Implementation - -**Pattern**: Reasoning + Acting for complex code queries - -**Agent Components**: - -- **LLM Service**: Multi-provider support (OpenAI, Anthropic, Gemini) -- **Cypher Generator**: Natural language to graph query translation -- **Tool System**: Graph queries, code retrieval, file search -- **Memory Management**: Configurable conversation history - -### Available Tools - -1. **query_graph**: Execute Cypher queries on knowledge graph -2. **get_code**: Retrieve specific code snippets -3. **search_files**: Find files by name or content patterns -4. **get_file_content**: Get complete file contents - -### Debug Mode Features - -- **Reasoning Steps**: Complete ReAct process visualization -- **Cypher Queries**: Generated queries with explanations -- **Configuration**: LLM settings and performance metrics -- **Context Info**: Graph statistics and source attribution - -## ๐Ÿ”ง Service Layer Details - -### GitHub Service (`src/services/github.ts`) - -**Purpose**: GitHub API integration with rate limiting and error handling - -**Key Features**: - -- **Rate Limit Handling**: 5,000 requests/hour with token, 60 without -- **Error Recovery**: Comprehensive error handling with user-friendly messages -- **Authentication**: Personal access token support -- **Content Retrieval**: Efficient file and directory fetching - -**API Methods**: - -```typescript -getRepositoryContents(owner, repo, path) // Directory structure -getFileContent(owner, repo, path) // Individual file content -downloadFileRaw(owner, repo, path) // Raw file download -``` - -### ZIP Service (`src/services/zip.ts`) - -**Purpose**: Local archive processing with complete structure discovery - -**Features**: - -- **Complete Structure**: Extracts all paths regardless of filtering -- **Memory Efficient**: Streaming processing for large archives -- **Path Normalization**: Handles common top-level folder removal -- **Content Mapping**: Efficient Map for file contents - -### Ingestion Service (`src/services/ingestion.service.ts`) - -**Purpose**: Orchestrate the complete ingestion pipeline - -**Orchestration Methods**: - -```typescript -processGitHubRepo(url, options) // GitHub repository processing -processZipFile(file, options) // ZIP archive processing -``` - -## ๐Ÿ“ˆ Data Models & Types - -### Core Graph Types - -```typescript -interface KnowledgeGraph { - nodes: GraphNode[] - relationships: Relationship[] -} - -interface GraphNode { - id: string - label: 'Project' | 'Folder' | 'File' | 'Function' | 'Class' | 'Method' | 'Variable' - properties: Record -} - -interface Relationship { - id: string - type: 'CONTAINS' | 'CALLS' | 'IMPORTS' | 'DECORATES' - source: string - target: string - properties: Record -} -``` - -### Function Registry Trie - -**Purpose**: Optimized function definition lookups - -**Key Features**: - -- **Suffix-based search**: `findEndingWith(name)` for heuristic matching -- **Qualified names**: Full paths like `myProject.services.api.fetchUser` -- **Import distance**: Smart scoring for best match selection - -## ๐ŸŽจ User Interface Architecture - -### Component Structure - -``` -App.tsx -โ”œโ”€โ”€ HomePage.tsx (Main application page) -โ”œโ”€โ”€ GraphExplorer.tsx (Interactive graph visualization) -โ”œโ”€โ”€ ChatInterface.tsx (AI chat with debug mode) -โ”œโ”€โ”€ SourceViewer.tsx (Code display with syntax highlighting) -โ””โ”€โ”€ ErrorBoundary.tsx (Comprehensive error handling) -``` - -### Interactive Features - -- **Graph Navigation**: Node selection, zooming, panning -- **Real-time Progress**: Live updates during processing -- **Split-Panel Layout**: Graph + chat interface -- **Settings Management**: Persistent configuration -- **Export Functionality**: JSON export with metadata - -## ๐Ÿ” Performance Optimization - -### Processing Optimizations - -- **Web Workers**: Background processing to keep UI responsive -- **Intelligent Filtering**: Skip massive directories (node_modules, .git) -- **Batch Processing**: Chunked processing for large repositories -- **Memory Management**: Configurable file limits (default: 500 files) - -### Graph Optimization - -- **Node Limiting**: Smart truncation for large graphs -- **Relationship Pruning**: Focus on high-confidence connections -- **Caching**: AST and processing result caching -- **Lazy Loading**: On-demand content loading - -## ๐Ÿ›ก๏ธ Error Handling & Reliability - -### Error Boundaries - -- **Component-level**: Graceful degradation for UI components -- **Worker-level**: Web Worker error recovery -- **Service-level**: API and processing error handling - -### User Experience - -- **Progress Indicators**: Detailed phase-specific messaging -- **Confirmation Dialogs**: Smart warnings for expensive operations -- **Recovery Options**: Clear guidance for error resolution -- **Debug Information**: Comprehensive logging for troubleshooting - -## ๐Ÿ“Š Development Setup - -### Prerequisites - -- **Node.js 18+** and **npm/yarn** -- **GitHub Token** (optional, increases rate limits) -- **AI API Keys**: OpenAI, Anthropic, or Gemini - -### Installation - -```bash -npm install -npm run dev # Development server on http://localhost:5173 -npm run build # Production build -``` - -### Configuration - -- **GitHub Token**: Settings โ†’ GitHub Token -- **AI Keys**: Settings โ†’ AI Provider Configuration -- **Performance**: Settings โ†’ File limits and filtering - -## ๐ŸŽฏ Usage Patterns - -### GitHub Repository Analysis - -1. **URL Input**: Enter GitHub repository URL -2. **Filtering**: Optional directory and file extension filters -3. **Processing**: 4-pass pipeline with progress tracking -4. **Exploration**: Interactive graph with AI chat - -### ZIP Archive Analysis - -1. **File Upload**: Select local ZIP archive -2. **Configuration**: Set processing limits and filters -3. **Analysis**: Complete repository structure discovery -4. **Results**: Knowledge graph with code intelligence - -### Best Practices - -- **Start Small**: Begin with focused directories -- **Use Filters**: Exclude dependencies and build artifacts -- **Monitor Progress**: Watch console for processing insights -- **Leverage AI**: Use chat interface for code exploration - -## ๐Ÿ”ฎ Future Enhancements - -### Language Support - -- **JavaScript/TypeScript**: Enhanced parsing and analysis -- **Java**: Class and method relationship mapping -- **C++**: Template and inheritance analysis -- **Go**: Package and interface resolution - -### Advanced Features - -- **Code Metrics**: Complexity and quality analysis -- **Security Scanning**: Vulnerability detection -- **Documentation Generation**: Auto-generated docs -- **Team Collaboration**: Shared graph exploration - -### Performance Improvements - -- **Incremental Processing**: Update existing graphs -- **Distributed Processing**: Multiple worker threads -- **Caching Layer**: Persistent processing cache -- **Streaming Analysis**: Real-time code changes - ---- - -## ๐Ÿ“ Quick Reference - -### Key Files - -- **Main Entry**: `src/App.tsx` -- **Pipeline**: `src/core/ingestion/pipeline.ts` -- **Services**: `src/services/` -- **AI Logic**: `src/ai/` -- **UI Components**: `src/ui/components/` - -### Debug Commands - -- **Enable Debug**: Click "๐Ÿ” Debug" in chat interface -- **Check Console**: F12 โ†’ Console for processing logs -- **Diagnose Issues**: Use "๐Ÿฉบ Diagnose" button - -### Common Issues - -- **Rate Limits**: Add GitHub token for higher limits -- **Large Repos**: Adjust file limits in settings -- **Parsing Errors**: Check file syntax and extensions -- **Memory Issues**: Reduce processing scope with filters - -This documentation provides complete context for any agent working on GitNexus, from architecture understanding to implementation details and troubleshooting guidance. - -================ -File: ARCHITECTURE_UPGRADE.md -================ -# ๐Ÿš€ Knowledge Graph Generation Architecture Upgrade - -## Overview - -The knowledge graph generation pipeline has been completely restructured to address two critical weaknesses: - -1. **Inaccurate Call Resolution** - Previous monolithic approach lacked project-wide visibility -2. **Inefficient Definition Lookups** - Simple map-based storage limited advanced resolution heuristics - -## ๐Ÿ—๏ธ New Architecture: 4-Pass Decoupled Pipeline - -### **Pass 1: Structure Analysis** ๐Ÿ“ -- **Processor**: `StructureProcessor` -- **Purpose**: Build project hierarchy (folders, files) -- **Output**: Basic graph structure with CONTAINS relationships - -### **Pass 2: Code Parsing & Definition Extraction** ๐Ÿ” -- **Processor**: `ParsingProcessor` (Enhanced) -- **Purpose**: Parse source code and extract definitions -- **Key Enhancement**: Populates `FunctionRegistryTrie` for efficient lookups -- **Output**: Function/class/method nodes + optimized search structure - -### **Pass 3: Import Resolution** ๐Ÿ”— -- **Processor**: `ImportProcessor` (NEW) -- **Purpose**: Build comprehensive project-wide import map -- **Key Features**: - - Resolves all aliases and relative paths - - Handles Python, JavaScript, TypeScript imports - - Creates accurate IMPORTS relationships -- **Output**: Complete dependency graph + import map - -### **Pass 4: Call Resolution** ๐Ÿ“ž -- **Processor**: `CallProcessor` (Completely Rewritten) -- **Purpose**: Resolve function calls using 3-stage strategy -- **Input**: Import map + Function registry trie -- **Output**: Accurate CALLS relationships - -## ๐Ÿง  Key Innovations - -### 1. FunctionRegistryTrie (`src/core/graph/trie.ts`) - -**Purpose**: Optimized data structure for function definition lookups - -**Key Features**: -- **Suffix-based search**: `findEndingWith(name)` for heuristic matching -- **Qualified names**: Stores full paths like `myProject.services.api.fetchUser` -- **Import distance calculation**: Smart scoring for best match selection - -**Example Usage**: -```typescript -// Find all functions ending with "fetchUser" across the project -const candidates = trie.findEndingWith("fetchUser"); -// Returns: [ -// { qualifiedName: "services.api.fetchUser", filePath: "services/api.py" }, -// { qualifiedName: "utils.cache.fetchUser", filePath: "utils/cache.js" } -// ] -``` - -### 2. ImportProcessor (`src/core/ingestion/import-processor.ts`) - -**Purpose**: Dedicated import resolution with project-wide visibility - -**Key Features**: -- **Language Support**: Python (`import`, `from...import`) and JS/TS (`import`, `require`) -- **Path Resolution**: Handles relative imports (`.`, `..`) and absolute imports -- **Alias Tracking**: Maps local names to actual exported functions -- **Validation**: Checks against actual project files - -**Example Output**: -```typescript -importMap = { - "src/api.js": { - "fetchUser": { - targetFile: "src/services/user.js", - exportedName: "fetchUser", - importType: "named" - } - } -} -``` - -### 3. Advanced Call Resolution Strategy - -**3-Stage Resolution Process**: - -#### Stage 1: Exact Match (High Confidence) -- Uses import map for direct resolution -- Example: `import { fetchUser } from './services'` โ†’ Direct link to `services/fetchUser` - -#### Stage 2: Same-Module Match (High Confidence) -- Checks for function definitions within the same file -- Example: Local function calls within a module - -#### Stage 3: Heuristic Fallback (Intelligent Guessing) -- Uses `FunctionRegistryTrie.findEndingWith()` to find candidates -- Applies **import distance** algorithm for best match -- **Distance Formula**: `max(caller_parts, candidate_parts) - common_prefix_length` -- **Sibling Bonus**: -1 for functions in same parent directory - -**Example Heuristic Resolution**: -``` -Call: fetchUser() in "src/components/UserList.js" -Candidates found: -- src/services/user.js:fetchUser (distance: 2) -- src/utils/api.js:fetchUser (distance: 2) -- src/components/utils.js:fetchUser (distance: 1) โ† SELECTED (sibling bonus) -``` - -## ๐Ÿ“Š Performance & Accuracy Improvements - -### Resolution Statistics -The new CallProcessor provides detailed statistics: -- **Exact matches** (Stage 1): Highest confidence -- **Same-file matches** (Stage 2): High confidence -- **Heuristic matches** (Stage 3): Medium confidence with distance scoring -- **Failed resolutions**: Tracked for debugging - -### Expected Improvements -- **๐ŸŽฏ Higher Accuracy**: Project-wide visibility eliminates cross-file resolution failures -- **โšก Better Performance**: Trie-based lookups vs linear searches -- **๐Ÿ” Smarter Heuristics**: Distance-based scoring for ambiguous cases -- **๐Ÿ“ˆ Detailed Metrics**: Comprehensive resolution statistics - -## ๐Ÿ”ง Technical Implementation Details - -### File Structure -``` -src/core/ -โ”œโ”€โ”€ graph/ -โ”‚ โ””โ”€โ”€ trie.ts # FunctionRegistryTrie implementation -โ”œโ”€โ”€ ingestion/ -โ”‚ โ”œโ”€โ”€ pipeline.ts # Updated 4-pass orchestration -โ”‚ โ”œโ”€โ”€ structure-processor.ts -โ”‚ โ”œโ”€โ”€ parsing-processor.ts # Enhanced with trie population -โ”‚ โ”œโ”€โ”€ import-processor.ts # NEW: Dedicated import resolution -โ”‚ โ””โ”€โ”€ call-processor.ts # Completely rewritten -``` - -### Integration Points -1. **ParsingProcessor** populates the trie during definition extraction -2. **ImportProcessor** builds the complete import map -3. **CallProcessor** uses both trie and import map for resolution -4. **Pipeline** orchestrates the sequence with proper data flow - -### Browser Compatibility -- Custom path utilities replace Node.js `path` module -- All processors work in browser environment -- Maintains existing WASM tree-sitter integration - -## ๐Ÿš€ Usage - -The new architecture is fully integrated into the existing pipeline. No changes required for: -- UI components -- Worker integration -- Export functionality -- Statistics display - -The system automatically uses the new 4-pass architecture for all repository processing. - -## ๐ŸŽฏ Results - -This architecture upgrade transforms the knowledge graph generation from a basic parser into an intelligent code analysis system capable of: - -- **Accurate cross-file call resolution** -- **Smart import dependency tracking** -- **Heuristic-based intelligent guessing** -- **Comprehensive project-wide visibility** -- **Detailed resolution analytics** - -The result is a significantly more accurate and comprehensive knowledge graph that truly represents the structure and relationships within a codebase. - -================ -File: CONVERSION_SUMMARY.md -================ -# Deno to Node.js Conversion Summary - -## Overview -Successfully converted the GitNexus repository from Deno to Node.js while maintaining all functionality. - -## Changes Made - -### 1. Configuration Files -- **Removed**: `deno.json`, `deno.lock` -- **Updated**: `package.json` with all dependencies from `deno.json` - - Added all npm dependencies: jszip, axios, cytoscape, web-tree-sitter, langchain packages, etc. - - Updated version to 1.0.0 - - Kept existing build scripts (Vite-based) - -### 2. Import Statements -- **Removed**: All `npm:` prefixes from import statements -- **Removed**: All `@ts-expect-error` comments related to npm: imports -- **Files affected**: 13+ TypeScript files across the codebase - -### 3. Dependencies Successfully Converted -- `react` & `react-dom` (already present) -- `jszip` for ZIP file processing -- `axios` for HTTP requests -- `cytoscape` & `cytoscape-dagre` for graph visualization -- `web-tree-sitter` for code parsing -- `comlink` for web workers -- `@langchain/*` packages for AI functionality -- `zod` for schema validation - -### 4. Build System -- **Unchanged**: Vite configuration remains the same -- **Unchanged**: TypeScript configuration -- **Working**: Development server starts successfully on port 5173 -- **Note**: Some TypeScript errors remain but don't prevent the dev server from running - -## Current Status -โœ… **Development server running** - The application starts and runs on Node.js -โœ… **All dependencies installed** - npm install completed successfully -โœ… **Import statements fixed** - All Deno-style imports converted to Node.js style -โš ๏ธ **TypeScript errors** - Some type errors remain but don't block functionality - -## Next Steps (Optional) -The conversion is complete and functional, but to achieve a clean build: -1. Fix TypeScript errors in langchain imports -2. Update type definitions for cytoscape -3. Fix unused variable warnings -4. Address JSZip type compatibility issues - -## Files Modified -- `package.json` - Added all dependencies -- 13+ TypeScript files - Removed npm: prefixes and Deno comments -- Removed `deno.json` and `deno.lock` - -The repository is now fully converted to Node.js and ready for development! - -================ -File: DEBUG_FEATURES.md -================ -# ๐Ÿ” Debug Mode Features - -The GitNexus chat interface now includes a comprehensive debug mode that shows the internal workings of the Graph RAG agent. - -## ๐Ÿ“ Markdown Formatting - -**NEW**: The chat interface now supports full markdown formatting for better readability! - -### Supported Markdown Features: -- **Headers** (# ## ###) for organizing information -- **Bold** and *italic* text for emphasis -- `Inline code` for function names and file paths -- Code blocks with syntax highlighting for multiple languages -- Bullet points and numbered lists -- Tables for structured data -- Blockquotes for important notes -- Links (automatically open in new tabs) - -### Enhanced Debug Display: -- **Reasoning observations** are now rendered with markdown -- **Query explanations** support formatted text -- **Tool outputs** preserve formatting and structure -- **Code snippets** get proper syntax highlighting - -## How to Use Debug Mode - -1. **Toggle Debug Mode**: Click the `๐Ÿ” Debug` button in the chat interface header -2. **Ask Questions**: When debug mode is enabled, all assistant responses will include detailed debug information -3. **Explore Tabs**: The debug panel includes multiple tabs showing different aspects of the processing - -## Debug Panel Tabs - -### ๐Ÿง  Reasoning Steps -Shows the complete ReAct (Reasoning + Acting) process: -- **Step-by-step thinking**: See how the agent reasons about your question -- **Actions taken**: View which tools the agent decides to use -- **Tool inputs**: See the exact parameters passed to each tool -- **Observations**: View the results returned by each tool -- **Success/failure status**: Monitor tool execution success - -### ๐Ÿ” Cypher Queries -Displays generated graph queries: -- **Generated Cypher**: See the exact database queries created -- **Query explanations**: Understand why each query was generated -- **Confidence scores**: View how confident the system is in each query -- **Syntax highlighting**: Cypher queries displayed with proper formatting - -### โš™๏ธ Configuration -Shows system configuration: -- **LLM Settings**: Provider, model, temperature, max tokens -- **RAG Options**: Reasoning steps, strict mode, temperature -- **Performance Metrics**: Execution time, confidence scores - -### ๐Ÿ“Š Context Info -Displays knowledge graph statistics: -- **Graph Nodes**: Total number of code entities in the graph -- **Files Indexed**: Number of source files processed -- **Sources Used**: Files referenced in the current response -- **Referenced Sources**: List of specific files used for the answer - -## What You Can Learn - -### Understanding Agent Behavior -- See how the agent breaks down complex questions -- Understand the reasoning process step-by-step -- Monitor which tools are used and why - -### Query Optimization -- View generated Cypher queries to understand graph traversal -- See query confidence scores to assess reliability -- Learn about query patterns for different question types - -### Performance Analysis -- Monitor execution times for different operations -- Understand the relationship between question complexity and processing time -- Identify bottlenecks in the reasoning process - -### Context Awareness -- See how much of your codebase is being used -- Understand which files are most relevant to your questions -- Monitor the scope of graph traversal - -## Debug Mode Benefits - -1. **Transparency**: Complete visibility into AI decision-making -2. **Learning**: Understand how Graph RAG works internally -3. **Debugging**: Identify issues with queries or reasoning -4. **Optimization**: Fine-tune your questions for better results -5. **Trust**: Build confidence through explainable AI - -## Example Debug Output - -When you ask "How many functions are in this project?", debug mode shows: - -**Reasoning Steps:** -1. **Thought**: "I need to count all functions in the project using a graph query" -2. **Action**: query_graph -3. **Input**: "Count all functions in the project" -4. **Observation**: Generated Cypher query and results - -**Generated Query:** -```cypher -MATCH (f:Function) RETURN COUNT(f) -``` - -**Configuration:** -- Model: gpt-4o-mini -- Temperature: 0.1 -- Execution Time: 1,234ms - -This level of detail helps you understand exactly how your question was processed and answered. - -================ -File: eslint.config.js -================ -import js from '@eslint/js' -import globals from 'globals' -import reactHooks from 'eslint-plugin-react-hooks' -import reactRefresh from 'eslint-plugin-react-refresh' -import tseslint from 'typescript-eslint' - -export default tseslint.config( - { ignores: ['dist'] }, - { - extends: [js.configs.recommended, ...tseslint.configs.recommended], - files: ['**/*.{ts,tsx}'], - languageOptions: { - ecmaVersion: 2020, - globals: globals.browser, - }, - plugins: { - 'react-hooks': reactHooks, - 'react-refresh': reactRefresh, - }, - rules: { - ...reactHooks.configs.recommended.rules, - 'react-refresh/only-export-components': [ - 'warn', - { allowConstantExport: true }, - ], - }, - }, -) - -================ -File: FILTERING_FIX.md -================ -# ๐Ÿ”ง Directory Filtering Fix - .venv and Ignored Directories Hidden - -## ๐Ÿšจ **Issue Identified and Fixed** - -**Problem**: `.venv` and other ignored directories were still appearing in the Knowledge Graph despite filtering implementation. - -**Root Cause**: The two-stage filtering was only filtering **file parsing**, but ignored directory **nodes** were still being created and displayed in the KG. - -## โœ… **Solution Implemented** - -### **Enhanced StructureProcessor** - -#### **Directory Hiding Logic** -```typescript -// Added to StructureProcessor -private shouldHideDirectory(dirPath: string): boolean { - const pathSegments = dirPath.split('/'); - - // Check if any segment matches ignore patterns - const hasIgnoredSegment = pathSegments.some(segment => - StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - return hasIgnoredSegment || this.matchesAdditionalPatterns(dirPath); -} -``` - -#### **Filtered Node Creation** -```typescript -// Filter directories before creating nodes -const visibleDirectories = directories.filter(dir => !this.shouldHideDirectory(dir)); -const hiddenDirectoriesCount = directories.length - visibleDirectories.length; - -console.log(`StructureProcessor: Hiding ${hiddenDirectoriesCount} ignored directories from KG`); - -// Create nodes only for visible directories -const directoryNodes = this.createDirectoryNodes(visibleDirectories); -``` - -#### **Smart Relationship Handling** -```typescript -// Handle files in hidden directories by connecting to nearest visible parent -private findVisibleParent(path: string, projectId: string): string { - if (path === '') return projectId; - - const parentPath = this.getParentPath(path); - const parentId = this.nodeIdMap.get(parentPath); - - if (parentId) { - return parentId; // Found visible parent - } - - // Recursively look for visible parent - return this.findVisibleParent(parentPath, projectId); -} -``` - -## ๐ŸŽฏ **What's Now Hidden from KG** - -### **Directories Completely Hidden** -- โœ… `.venv`, `venv`, `env`, `virtualenv` (Python virtual environments) -- โœ… `node_modules`, `bower_components` (Package dependencies) -- โœ… `.git`, `.svn`, `.hg` (Version control) -- โœ… `build`, `dist`, `out`, `target` (Build outputs) -- โœ… `.vs`, `.vscode`, `.idea` (IDE directories) -- โœ… `__pycache__`, `.pytest_cache` (Python cache) -- โœ… `coverage`, `.coverage` (Test coverage) -- โœ… `.cache`, `.next`, `.nuxt` (Framework cache) -- โœ… `tmp`, `temp`, `logs` (Temporary directories) - -### **Special Handling** -- โœ… `.github` directory **remains visible** (important for workflows) -- โœ… Files in hidden directories connect to nearest visible parent -- โœ… Complete structure discovery still happens (for performance benefits) - -## ๐Ÿ“Š **Before vs After** - -### **Before (The Problem)** -``` -Knowledge Graph showing: -โ”œโ”€โ”€ src/ โœ… Visible -โ”œโ”€โ”€ tests/ โœ… Visible -โ”œโ”€โ”€ .venv/ โŒ Unwanted visibility -โ”œโ”€โ”€ node_modules/ โŒ Unwanted visibility -โ”œโ”€โ”€ __pycache__/ โŒ Unwanted visibility -โ””โ”€โ”€ package.json โœ… Visible -``` - -### **After (Fixed)** -``` -Knowledge Graph showing: -โ”œโ”€โ”€ src/ โœ… Visible -โ”œโ”€โ”€ tests/ โœ… Visible -โ”œโ”€โ”€ .github/ โœ… Visible (important) -โ””โ”€โ”€ package.json โœ… Visible - -Hidden from view: -- .venv/ (and all contents) -- node_modules/ (and all contents) -- __pycache__/ (and all contents) -``` - -## ๐ŸŽฏ **Technical Implementation** - -### **Two-Level Filtering** -1. **StructureProcessor**: Hides directory **nodes** from KG -2. **ParsingProcessor**: Skips **file parsing** in ignored directories - -### **Performance Benefits Maintained** -- โœ… **Complete Discovery**: Still discovers all paths for performance optimization -- โœ… **Smart Filtering**: Skips expensive parsing operations -- โœ… **Clean Visualization**: Users see only relevant directories -- โœ… **Accurate Relationships**: Files connect to appropriate visible parents - -### **Logging Enhanced** -``` -StructureProcessor: Found 1,247 directories and 892 files -StructureProcessor: Hiding 156 ignored directories from KG -StructureProcessor: Created 983 nodes total (156 directories hidden) -``` - -## ๐Ÿš€ **Result** - -**Perfect Fix!** Now: - -1. **โœ… .venv is Hidden**: No longer appears in Knowledge Graph -2. **โœ… Clean Visualization**: Only relevant directories shown -3. **โœ… Performance Maintained**: Still skip expensive parsing operations -4. **โœ… Accurate Structure**: Files properly connected to visible parents -5. **โœ… Comprehensive Coverage**: All common ignored directories hidden - -The directory filtering is now **working correctly** and `.venv` (along with other ignored directories) will no longer clutter the Knowledge Graph! ๐ŸŽ‰ - -================ -File: GITNEXUS_README.md -================ -# ๐Ÿ” CodeNexus - Edge Knowledge Graph Creator with Graph RAG - -**Transform any codebase into an interactive knowledge graph in your browser. No servers, no setup - just instant Graph RAG-powered code intelligence.** - -CodeNexus is a client-side knowledge graph creator that runs entirely in your browser. Drop in a GitHub repo or ZIP file, and get an interactive knowledge graph with AI-powered chat interface. Perfect for code exploration, documentation, and understanding complex codebases through Graph RAG (Retrieval-Augmented Generation). - -## โœจ Features - -### ๐Ÿ“Š **Code Analysis & Visualization** -- **GitHub Integration**: Analyze any public GitHub repository directly from URL -- **ZIP File Support**: Upload and analyze local code archives -- **Interactive Knowledge Graph**: Visualize code structure with Cytoscape.js -- **Multi-language Support**: Currently optimized for Python with extensible architecture -- **Smart Filtering**: Directory and file pattern filters to focus analysis scope -- **Performance Optimization**: Configurable file limits with confirmation dialogs for large repositories - -### ๐Ÿค– **AI-Powered Chat Interface** -- **Multiple LLM Providers**: OpenAI, Anthropic (Claude), Google Gemini -- **ReAct Agent Pattern**: Uses proper LangChain ReAct implementation for reasoning -- **Tool-Augmented Responses**: Graph queries, code retrieval, file search -- **Context-Aware**: Maintains conversation history with configurable memory - -### ๐Ÿ”ง **Advanced Processing Pipeline** -- **3-Pass Ingestion Strategy**: - 1. **Structure Analysis**: Project hierarchy and file organization - 2. **Code Parsing**: AST-based extraction using Tree-sitter - 3. **Call Resolution**: Function/method call relationship mapping -- **Web Worker Processing**: Non-blocking UI with progress tracking -- **Intelligent Caching**: AST and processing result optimization -- **Error Resilience**: Comprehensive error boundaries and recovery mechanisms - -### ๐ŸŽจ **Modern UI/UX** -- **Responsive Design**: Adaptive layout for different screen sizes -- **Real-time Progress**: Live updates during repository processing -- **Interactive Graph**: Node selection, zooming, panning -- **Split-Panel Layout**: Graph visualization + AI chat interface -- **Settings Management**: Persistent configuration for API keys and preferences -- **Export Functionality**: Download knowledge graphs as JSON with metadata -- **Performance Controls**: File limits, filtering, and optimization settings - -### ๐Ÿ›ก๏ธ **Reliability & Performance** -- **Error Boundaries**: Graceful error handling with user-friendly recovery options -- **Performance Monitoring**: Real-time processing statistics and export size calculation -- **Memory Management**: Efficient handling of large repositories with configurable limits -- **Progress Tracking**: Detailed progress indicators with phase-specific messaging -- **Confirmation Dialogs**: Smart warnings for potentially expensive operations - -## ๐Ÿ—๏ธ Architecture - -### **Frontend Stack** -- **React 18** with TypeScript -- **Vite** for fast development and building -- **Cytoscape.js** for graph visualization -- **Custom CSS** with modern design patterns -- **Error Boundaries** for robust error handling - -### **Processing Engine** -- **Deno Runtime** for TypeScript execution -- **Tree-sitter WASM** for syntax parsing -- **Web Workers** for background processing -- **Comlink** for worker communication - -### **AI Integration** -- **LangChain.js** with proper ReAct agent implementation -- **Multiple LLM Support**: OpenAI, Anthropic, Gemini -- **Tool-based Architecture**: Graph queries, code retrieval, file search -- **Cypher Query Generation**: Natural language to graph queries - -### **Services Layer** -``` -src/ -โ”œโ”€โ”€ services/ # External API integrations -โ”‚ โ”œโ”€โ”€ github.ts # GitHub REST API client -โ”‚ โ””โ”€โ”€ zip.ts # ZIP file processing -โ”œโ”€โ”€ core/ # Core processing logic -โ”‚ โ”œโ”€โ”€ graph/ # Knowledge graph types -โ”‚ โ”œโ”€โ”€ ingestion/ # 3-pass processing pipeline -โ”‚ โ””โ”€โ”€ tree-sitter/ # Syntax parsing infrastructure -โ”œโ”€โ”€ ai/ # AI and RAG components -โ”‚ โ”œโ”€โ”€ llm-service.ts # Multi-provider LLM client -โ”‚ โ”œโ”€โ”€ cypher-generator.ts # NL to Cypher translation -โ”‚ โ”œโ”€โ”€ orchestrator.ts # Custom ReAct implementation -โ”‚ โ””โ”€โ”€ langchain-orchestrator.ts # Standard LangChain ReAct -โ”œโ”€โ”€ workers/ # Web Worker implementations -โ”œโ”€โ”€ ui/ # React components and pages -โ”‚ โ”œโ”€โ”€ components/ # Reusable UI components -โ”‚ โ”‚ โ”œโ”€โ”€ ErrorBoundary.tsx # Error handling component -โ”‚ โ”‚ โ”œโ”€โ”€ graph/ # Graph visualization components -โ”‚ โ”‚ โ””โ”€โ”€ chat/ # Chat interface components -โ”‚ โ””โ”€โ”€ pages/ # Application pages -โ”œโ”€โ”€ lib/ # Shared utilities -โ”‚ โ””โ”€โ”€ export.ts # Graph export functionality -โ””โ”€โ”€ App.tsx # Main application entry point -``` - -## ๐Ÿš€ Getting Started - -### Prerequisites -- **Node.js 18+** and **npm/yarn** -- **Deno 1.40+** for development -- **API Keys** for AI features (OpenAI, Anthropic, or Gemini) - -### Installation - -1. **Clone the repository** - ```bash - git clone - cd gitnexus - ``` - -2. **Install dependencies** - ```bash - npm install - ``` - -3. **Start development server** - ```bash - npm run dev - ``` - -4. **Open in browser** - ``` - http://localhost:5173 - ``` - -### Configuration - -1. **GitHub Token (Optional)** - - Increases rate limit from 60 to 5,000 requests/hour - - Generate at: https://github.com/settings/tokens - - Requires no special permissions for public repos - -2. **AI API Keys** - - **OpenAI**: Get from https://platform.openai.com/api-keys - - **Anthropic**: Get from https://console.anthropic.com/ - - **Gemini**: Get from https://makersuite.google.com/app/apikey - -3. **Performance Settings** - - **File Limit**: Configure maximum files to process (default: 500) - - **Directory Filters**: Focus on specific directories (e.g., "src", "lib") - - **File Patterns**: Filter by file types (e.g., "*.py", "*.js", "*.ts") - -## ๐Ÿ’ก Usage - -### Analyzing a Repository - -1. **GitHub Repository** - ``` - 1. Enter GitHub URL: https://github.com/owner/repo - 2. Optional: Set directory/file filters to focus analysis - 3. Click "Analyze" - 4. For large repos: Confirm processing or adjust filters - 5. Wait for processing (structure โ†’ parsing โ†’ call resolution) - 6. Explore the interactive graph - ``` - -2. **ZIP File Upload** - ``` - 1. Click "Choose File" and select a .zip file - 2. Optional: Configure filters before processing - 3. Click "Analyze" - 4. Processing will extract and analyze text files - 5. Explore results in the graph visualization - ``` - -### Performance Optimization - -1. **Directory Filtering** - ``` - - Enter directory names: "src", "lib", "components" - - Focuses analysis on specific parts of the codebase - - Reduces processing time and memory usage - ``` - -2. **File Pattern Filtering** - ``` - - Use patterns: "*.py", "*.js", "*.ts" - - Supports wildcards: "test*.py", "*util*" - - Comma-separated: "*.py,*.js,*.ts" - ``` - -3. **File Limits** - ``` - - Default limit: 500 files - - Configurable in settings (50-2000 files) - - Large repositories show confirmation dialog - - Automatic truncation to limit if confirmed - ``` - -### Using the AI Chat - -1. **Configure API Key** - ``` - 1. Click the โš™๏ธ settings button - 2. Choose your preferred LLM provider - 3. Enter your API key - 4. Select model (e.g., gpt-4o-mini, claude-3-haiku) - ``` - -2. **Ask Questions** - ``` - - "What functions are in the main.py file?" - - "Show me all classes that inherit from BaseClass" - - "How does the authentication system work?" - - "Find all functions that call the database" - ``` - -### Exporting Data - -1. **Export Knowledge Graph** - ``` - 1. Click the ๐Ÿ“ฅ Export button after processing - 2. Downloads JSON file with graph data and metadata - 3. Includes processing statistics and timestamps - 4. File size shown in UI before export - ``` - -2. **Export Format** - ```json - { - "metadata": { - "exportedAt": "2024-01-01T12:00:00.000Z", - "version": "1.0.0", - "nodeCount": 150, - "relationshipCount": 200, - "fileCount": 25, - "processingDuration": 5000 - }, - "graph": { - "nodes": [...], - "relationships": [...] - }, - "fileContents": {...} - } - ``` - -### Graph Interaction - -- **Node Selection**: Click any node to highlight and view details -- **Zoom & Pan**: Mouse wheel to zoom, drag to pan -- **Node Types**: Different colors/shapes for files, functions, classes, etc. -- **Relationships**: Arrows show CONTAINS, CALLS, INHERITS relationships - -## ๐Ÿ”ง Development - -### Project Structure -``` -GitNexus/ -โ”œโ”€โ”€ src/ -โ”‚ โ”œโ”€โ”€ services/ # External integrations -โ”‚ โ”œโ”€โ”€ core/ # Processing pipeline -โ”‚ โ”œโ”€โ”€ ai/ # AI and RAG systems -โ”‚ โ”œโ”€โ”€ workers/ # Web Workers -โ”‚ โ”œโ”€โ”€ ui/ # React components -โ”‚ โ”‚ โ”œโ”€โ”€ components/ # Reusable components -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ ErrorBoundary.tsx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ graph/ # Graph components -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ chat/ # Chat components -โ”‚ โ”‚ โ””โ”€โ”€ pages/ # Application pages -โ”‚ โ”œโ”€โ”€ lib/ # Utilities -โ”‚ โ”‚ โ””โ”€โ”€ export.ts # Export functionality -โ”‚ โ””โ”€โ”€ App.tsx # Main application -โ”œโ”€โ”€ public/ -โ”‚ โ””โ”€โ”€ wasm/ # Tree-sitter WASM files -โ”œโ”€โ”€ package.json -โ”œโ”€โ”€ vite.config.ts -โ””โ”€โ”€ tsconfig.json -``` - -### Key Components - -#### **Error Handling** -```typescript -// ErrorBoundary component with recovery options - { - console.error('Application error:', error); - }} -> - - -``` - -#### **Performance Optimization** -```typescript -// File filtering and limits -const filterFiles = (files: any[]) => { - return files - .filter(file => matchesDirectoryFilter(file)) - .filter(file => matchesPatternFilter(file)) - .slice(0, maxFiles); -}; -``` - -#### **Export Functionality** -```typescript -// Export with metadata -exportAndDownloadGraph(graph, { - projectName: 'my-project', - includeMetadata: true, - prettyPrint: true -}, fileContents, { duration: 5000 }); -``` - -#### **Processing Pipeline** -```typescript -// 3-pass ingestion strategy with progress tracking -const pipeline = new GraphPipeline(); -const result = await pipeline.run({ - projectRoot: '/', - projectName: 'MyProject', - filePaths: ['src/main.py', 'src/utils.py'], - fileContents: new Map([ - ['src/main.py', 'def main(): pass'], - ['src/utils.py', 'def helper(): pass'] - ]) -}); -``` - -#### **AI Integration** -```typescript -// LangChain ReAct agent with error handling -const orchestrator = new LangChainRAGOrchestrator(llmService, cypherGenerator); -await orchestrator.setContext({ graph, fileContents }, llmConfig); -const response = await orchestrator.answerQuestion("How does auth work?"); -``` - -#### **Graph Visualization** -```typescript -// Interactive graph component with error boundaries - - setSelectedNode(nodeId)} - /> - -``` - -### Adding New Features - -1. **New Language Support** - ```typescript - // Add parser in core/tree-sitter/ - export const loadJavaScriptParser = async () => { - // Load JS Tree-sitter grammar - }; - ``` - -2. **Custom AI Tools** - ```typescript - // Add tools in ai/langchain-orchestrator.ts - const customTool = tool( - async (input: { query: string }) => { - // Tool implementation - }, - { - name: "custom_tool", - description: "Custom functionality", - schema: z.object({ query: z.string() }) - } - ); - ``` - -3. **Export Formats** - ```typescript - // Add new export formats in lib/export.ts - export function exportToCSV(graph: KnowledgeGraph): string { - // CSV export implementation - } - ``` - -## ๐Ÿงช Testing & Quality Assurance - -### Error Handling -- **Error Boundaries**: Catch and display JavaScript errors gracefully -- **User Recovery**: Allow users to reset component state after errors -- **Detailed Logging**: Console logging for debugging and error reporting -- **Fallback UI**: User-friendly error messages with recovery options - -### Performance Testing -1. **Large Repository Handling** - - Test with repositories containing 1000+ files - - Verify confirmation dialogs for file limits - - Monitor memory usage during processing - - Test filtering effectiveness - -2. **UI Responsiveness** - - Ensure non-blocking processing with Web Workers - - Verify progress indicators update correctly - - Test error recovery mechanisms - - Validate export functionality with large graphs - -3. **Error Scenarios** - - Network failures during GitHub API calls - - Corrupted ZIP files - - Invalid API keys - - Memory exhaustion scenarios - -### Manual Testing Checklist -- [ ] GitHub repository analysis with various sizes -- [ ] ZIP file upload and extraction -- [ ] Directory and file pattern filtering -- [ ] Large repository confirmation dialog -- [ ] Export functionality with different options -- [ ] Error boundary activation and recovery -- [ ] API key validation for all providers -- [ ] Settings persistence across sessions -- [ ] Graph visualization interactions -- [ ] Chat interface with different LLM providers - -## ๐Ÿš€ Deployment - -### Production Build -```bash -npm run build -npm run preview -``` - -### Environment Variables -```env -# Optional: Pre-configure API keys -VITE_OPENAI_API_KEY=sk-... -VITE_ANTHROPIC_API_KEY=sk-ant-... -VITE_GEMINI_API_KEY=... - -# Performance settings -VITE_DEFAULT_MAX_FILES=500 -VITE_ENABLE_DEBUG_LOGGING=false -``` - -### Docker Deployment -```dockerfile -FROM node:18-alpine -WORKDIR /app -COPY package*.json ./ -RUN npm ci --only=production -COPY . . -RUN npm run build -EXPOSE 3000 -CMD ["npm", "run", "preview", "--", "--host", "0.0.0.0"] -``` - -### Performance Monitoring -```javascript -// Add performance monitoring -const observer = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - if (entry.entryType === 'measure') { - console.log(`${entry.name}: ${entry.duration}ms`); - } - } -}); -observer.observe({ entryTypes: ['measure'] }); -``` - -## ๐Ÿ”’ Security & Privacy - -- **Client-Side Processing**: All analysis happens in your browser -- **API Keys**: Stored locally, never transmitted to our servers -- **GitHub Access**: Uses public API, respects repository permissions -- **Data Privacy**: No code or analysis results are stored remotely -- **Error Logging**: Sensitive data excluded from error reports -- **Export Security**: User-controlled data export with no server interaction - -## ๐Ÿค Contributing - -### Development Setup -1. Fork the repository -2. Create feature branch: `git checkout -b feature/amazing-feature` -3. Make changes and test thoroughly -4. Run the testing checklist above -5. Commit: `git commit -m 'Add amazing feature'` -6. Push: `git push origin feature/amazing-feature` -7. Open a Pull Request - -### Code Style -- **TypeScript**: Strict mode enabled -- **ESLint**: Follow configured rules -- **Prettier**: Auto-formatting -- **Comments**: Minimal, only when necessary -- **Error Handling**: Comprehensive error boundaries and recovery -- **Performance**: Consider memory usage and processing time - -### Testing Guidelines -- Test error scenarios and edge cases -- Verify performance with large datasets -- Ensure graceful degradation -- Test all export functionality -- Validate error boundary behavior - -## ๐Ÿ“š Technical Details - -### Knowledge Graph Schema -```typescript -interface KnowledgeGraph { - nodes: GraphNode[]; // Code entities - relationships: GraphRelationship[]; // Connections -} - -// Node types: Project, Folder, File, Module, Class, Function, Method, Variable -// Relationship types: CONTAINS, CALLS, INHERITS, OVERRIDES, IMPORTS -``` - -### Export Format -```typescript -interface ExportedGraph { - metadata: { - exportedAt: string; - version: string; - nodeCount: number; - relationshipCount: number; - fileCount?: number; - processingDuration?: number; - }; - graph: KnowledgeGraph; - fileContents?: Record; -} -``` - -### Error Boundary Implementation -- **Component-Level**: Individual components wrapped for isolation -- **Application-Level**: Top-level boundary for catastrophic failures -- **Recovery Options**: Reset state, reload page, or continue with fallback -- **Error Reporting**: Detailed technical information for developers - -### Performance Optimizations -- **Web Workers**: Non-blocking processing -- **AST Caching**: Reuse parsed syntax trees -- **Progressive Loading**: Stream results as available -- **Memory Management**: Efficient data structures -- **File Filtering**: Reduce processing scope -- **Confirmation Dialogs**: Prevent accidental expensive operations - -### ReAct Agent Implementation -- **Standard LangChain**: Uses `createReactAgent` from `@langchain/langgraph/prebuilt` -- **Custom Implementation**: Manual ReAct loop for educational purposes -- **Tools**: Graph queries, code retrieval, file search -- **Memory**: Conversation persistence with thread management -- **Error Recovery**: Graceful handling of API failures - -## ๐Ÿ“„ License - -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. - -## ๐Ÿ™ Acknowledgments - -- **Tree-sitter**: Syntax parsing infrastructure -- **LangChain.js**: AI agent framework -- **Cytoscape.js**: Graph visualization -- **React**: UI framework with error boundaries -- **Vite**: Build tool and dev server - ---- - -**CodeNexus** - Edge Knowledge Graph Creator with instant Graph RAG. Zero setup, maximum insight. ๐Ÿš€ - -*Browser-native code intelligence that runs anywhere, anytime - no servers required.* - -================ -File: index.html -================ - - - - - - - GitNexus - Code Knowledge Graph Explorer - - - -
- - - - -================ -File: ISOLATED_NODES_FIX.md -================ -# ๐Ÿ”ง Fixing Isolated Nodes in Graph Visualization - -## Problem Description -You're seeing nodes "flying away" with no connections in the graph visualization. This indicates **isolated nodes** - nodes that have no relationships to other nodes in the graph. - -## Root Causes - -### 1. **File Parsing Failures** (Most Common) -- Files fail to parse during **Pass 2** of ingestion -- File nodes get created but no functions/classes are extracted -- Results in isolated file nodes - -### 2. **Unsupported File Types** -- Files with extensions not recognized by the parser -- Configuration files, documentation, etc. without code content - -### 3. **Syntax Errors** -- Malformed code that the AST parser can't understand -- Missing imports or exports -- Language-specific syntax issues - -### 4. **Import Resolution Failures** -- **Pass 3** fails to resolve import relationships -- Files exist but aren't connected via imports - -### 5. **Call Resolution Failures** -- **Pass 4** fails to find function calls between files -- Functions exist but no call relationships are created - -## ๐Ÿ› ๏ธ How to Diagnose - -### Step 1: Use the New Diagnostic Tool -1. Load your repository in GitNexus -2. Click the **๐Ÿฉบ Diagnose** button in the chat interface -3. Check the statistics and follow the suggested steps - -### Step 2: Check Browser Console -1. Open Developer Tools (F12) -2. Look for console warnings during ingestion: - - `โš ๏ธ Found X isolated nodes` - - `โš ๏ธ Found X files without definitions` - - `Source files without definitions: [...]` - -### Step 3: Review Console Logs -Look for these specific log messages: -``` -๐Ÿ“ Pass 1: Analyzing project structure... -๐Ÿ” Pass 2: Parsing code and extracting definitions... -๐Ÿ”— Pass 3: Resolving imports and building dependency map... -๐Ÿ“ž Pass 4: Resolving function calls with 3-stage strategy... -``` - -## ๐Ÿ” Diagnostic Information - -The enhanced pipeline now shows: -- **Node counts by type** (Project, Folder, File, Function, Class, etc.) -- **Relationship counts by type** (CONTAINS, CALLS, IMPORTS, etc.) -- **Isolated nodes** with examples -- **Files without definitions** -- **Graph integrity issues** - -## โœ… Recent Fixes Applied - -### **1. Reduced Console Noise (Fixed)** -- **Issue**: Thousands of "Failed to resolve call" messages for Python built-ins like `int`, `str`, `len`, etc. -- **Fix**: Added `shouldIgnoreCall()` method to filter out Python built-in functions and standard library calls -- **Result**: Console output is now much cleaner and shows only relevant failures - -### **2. Improved Python Import Resolution (Fixed)** -- **Issue**: Python imports weren't being resolved correctly, causing "No import relationships found" -- **Fix**: Enhanced `resolveModulePath()` with better pattern matching for complex project structures -- **Features Added**: - - Multiple resolution patterns for Python modules - - Partial path matching for complex project structures - - Better handling of package imports - - Enhanced debugging with import resolution statistics - -### **3. Enhanced Diagnostic Reporting (Added)** -- **New**: Comprehensive graph integrity validation -- **New**: Import resolution success rate reporting -- **New**: Detailed breakdown of isolated nodes by type -- **New**: ๐Ÿฉบ Diagnose button in chat interface - -## ๐Ÿš€ Solutions - -### For File Parsing Issues: -1. **Check file extensions**: Ensure files are `.js`, `.ts`, `.jsx`, `.tsx`, `.py`, etc. -2. **Verify syntax**: Make sure code files have valid syntax -3. **Check file size**: Very large files might timeout during parsing -4. **Review file content**: Empty files or files with only comments won't generate nodes - -### For Import Issues (Now Improved): -1. **Check import syntax**: Ensure proper `import`/`export` or `require()` statements -2. **Verify file paths**: Relative imports should resolve correctly -3. **Check module resolution**: External libraries might not be resolved -4. **Monitor import resolution rate**: Should be >50% for healthy projects - -### For Call Issues: -1. **Function calls**: Ensure functions are actually called between files -2. **Method calls**: Class methods should be invoked -3. **Export/import**: Functions need to be properly exported and imported - -## ๐Ÿ”ง Quick Fixes - -### 1. **Filter Out Non-Code Files** -Use GitNexus filtering options to exclude: -- Documentation files (`.md`, `.txt`) -- Configuration files (`.json`, `.yaml`, `.xml`) -- Asset files (`.png`, `.jpg`, `.css`) - -### 2. **Focus on Core Directories** -- Include only `src/`, `lib/`, `app/` directories -- Exclude `node_modules/`, `.git/`, `dist/`, `build/` - -### 3. **Check File Limits** -- Large repositories might hit processing limits -- Consider processing smaller subsets first - -## ๐Ÿ“Š Expected Results After Fixes - -A healthy graph should show: -- **Project** โ†’ **Folders** โ†’ **Files** (CONTAINS relationships) -- **Files** โ†’ **Functions/Classes** (CONTAINS relationships) -- **Files** โ†’ **Files** (IMPORTS relationships) - **Now working better** -- **Functions** โ†’ **Functions** (CALLS relationships) - **Cleaner console output** - -### **Typical Success Rates**: -- **Import Resolution**: 40-70% (up from 0%) -- **Call Resolution**: 20-30% (excluding built-ins) -- **File Parsing**: 80-95% for source files - -## ๐Ÿ†˜ Still Having Issues? - -1. **Try the diagnostic tool**: Click ๐Ÿฉบ Diagnose button -2. **Check console output**: Look for specific error patterns -3. **Share diagnostic info**: Copy the improved console logs -4. **Test incrementally**: Try with smaller subsets of files - -## ๐Ÿ“ˆ What You Should See Now - -After the fixes, your console output should show: -``` -ImportProcessor: Found 45 imports, resolved 28 (62.2%) -CallProcessor: Success rate: 45.9% (excluding built-ins) -๐Ÿ“Š Graph Statistics: -Relationships by type: {CONTAINS: 395, DECORATES: 48, IMPORTS: 13, CALLS: 47} -โœ… Graph integrity validation passed -``` - -### **Latest Improvements (v2)**: -- **Expanded built-ins filtering**: Now ignores 100+ Python string methods, math functions, and third-party library calls -- **Better diagnostics**: Identifies source files with zero function calls (potential parsing issues) -- **Pattern-based filtering**: Automatically ignores dunder methods (`__init__`, `__str__`) and private methods - -### **Expected Results After All Fixes**: -- **Console noise reduction**: 95%+ reduction in irrelevant error messages -- **Import relationships**: 10-20+ IMPORTS relationships created -- **Call resolution success**: 40-60% (realistic for complex codebases) -- **Failed calls**: Only legitimate issues (domain-specific functions, missing imports) - -Instead of thousands of failed call resolutions, you'll see much cleaner output focused on actual issues that need attention! - -## ๐ŸŽฏ **FINAL STATUS - Issue Resolved!** - -### **๐Ÿ” MAJOR DISCOVERY - Python Call Extraction Bug Found!** - -**Latest diagnostic output revealed a critical issue:** -``` -๐Ÿ“Š Debug: config.py has 38 call nodes, 0 definitions -๐Ÿ“Š Debug: assessment_db.py has 120 call nodes, 4 definitions -๐Ÿ“Š Debug: sonar_analyzer.py has 30 call nodes, 2 definitions -``` - -**This shows that:** -- โœ… **AST parsing works perfectly** - files have hundreds of call nodes -- โŒ **Call extraction is broken** - 0 function calls extracted from files with 120+ call nodes -- ๐Ÿ”ง **Root cause identified** - Python call extraction logic needs fixes - -### **๐Ÿ› ๏ธ Latest Fix Applied** -- **Enhanced Python call extraction** with better node type handling -- **Comprehensive debugging** to identify what's being filtered vs extracted -- **Improved function name extraction** for complex Python call patterns -- **Added support for** subscript calls, nested calls, and more node types - -### **โœ… Latest Results (Your Console Output)** -``` -โœ… Parsing Success: 38 successful, 0 failed (100%) -โœ… Import Resolution: 102 imports found, 61 resolved (59.8%) -โœ… Call Resolution: 129 calls processed, 56.6% success rate -โœ… Graph Health: 367 nodes, 506 relationships -โœ… Console Cleanliness: Only 56 legitimate failures (93% noise reduction) -``` - -### **๐Ÿ” Enhanced Diagnostics Added** -- **Zero-call file detection**: Identifies source files with parsing issues -- **AST node counting**: Shows `call` nodes vs definitions for debugging -- **Suspicious file flagging**: Highlights files with definitions but no calls -- **Comprehensive built-ins filtering**: 100+ Python functions ignored - -### **๐Ÿ“Š Your Graph is Now Healthy!** - -**Before the fixes:** -- โŒ 814+ failed call messages (noise) -- โŒ No import relationships -- โŒ Isolated nodes everywhere -- โŒ Unreadable console output - -**After the fixes:** -- โœ… **56.6% call resolution success** (excellent!) -- โœ… **59.8% import resolution success** (great!) -- โœ… **13 IMPORTS relationships** created -- โœ… **47 CALLS relationships** created -- โœ… **Clean, readable diagnostics** - -### **๐Ÿ”ฌ Remaining Issues Are Expected** - -#### **1. Zero-Call Files (Normal)** -Files showing "No function calls found" are often: -- **Model/config files**: Only contain class definitions -- **Pure data files**: Constants, configurations -- **Interface files**: Abstract base classes -- **Files with only imports**: Router configurations - -#### **2. Failed Calls (Legitimate)** -The remaining 56 failed calls are **appropriate failures**: -- **External libraries**: LangGraph, FastAPI, Azure OpenAI -- **Domain-specific**: Business logic libraries (`ruleset`, `assert_fact`) -- **Custom models**: Application-specific classes - -### **๐ŸŽ‰ Problem Solved!** - -Your **"nodes flying away with no connections"** issue is **fully resolved**: - -1. โœ… **Files connect properly** via CONTAINS relationships -2. โœ… **Import relationships work** (13 created) -3. โœ… **Function calls connect** (47 relationships) -4. โœ… **Console is clean** and diagnostic -5. โœ… **Success rates are realistic** for complex codebases - -### **๐Ÿš€ What You Should See Now** - -When you reload your repository, expect: -- **Significantly fewer isolated nodes** -- **Connected file clusters** via imports -- **Function-to-function connections** within files -- **Clean console output** focusing on real issues -- **Better graph connectivity** overall - -The isolated nodes that remain will be: -- **Configuration files** (expected) -- **Documentation files** (expected) -- **Empty or comment-only files** (expected) -- **External library references** (expected) - -## **๐Ÿ† Mission Accomplished!** - -Your graph now has proper connectivity with realistic success rates. The diagnostic tools will help you identify any remaining issues that need attention. The isolated nodes problem is **solved**! ๐ŸŽฏโœจ - -## ๐ŸŽ›๏ธ **NEW FEATURE: Hide External Libraries Toggle** - -### **โœจ What's New** -Added a **"Hide external libraries"** toggle in the graph visualization that lets you: -- โœ… **Clean view**: Hide isolated external library nodes for cleaner visualization -- โœ… **Full view**: Show all nodes including external dependencies for complete context -- โœ… **Smart filtering**: Automatically identifies external library patterns -- โœ… **Live counter**: Shows how many external nodes are hidden/visible - -### **๐ŸŽฏ How It Works** -The toggle uses intelligent filtering to identify external library nodes: -- **Isolated nodes**: Nodes with no relationships (not connected to your code) -- **External patterns**: Recognizes common library functions like: - - `when_all`, `ruleset` (durable rules) - - `APIRouter`, `FastAPI` (FastAPI framework) - - `StateGraph`, `AsyncAzureOpenAI` (AI libraries) - - CamelCase patterns (often external classes) - -### **๐Ÿš€ When to Use Each Mode** - -#### **Hide External Libraries (Clean View)** -**Best for:** -- ๐Ÿ“Š **Architecture review** - Focus on your internal code structure -- ๐Ÿ” **Code navigation** - See relationships between your functions/classes -- ๐Ÿ“ˆ **Presentations** - Clean, professional visualization -- ๐ŸŽฏ **Debugging** - Trace internal call paths without distractions - -#### **Show External Libraries (Full View)** -**Best for:** -- ๐Ÿ”— **Dependency analysis** - See what external libraries you use -- ๐Ÿ—๏ธ **System design** - Understand integration points -- ๐Ÿ“‹ **Documentation** - Complete picture of your tech stack -- ๐Ÿ”ง **Troubleshooting** - Identify external dependency issues - -### **๐Ÿ’ก Pro Tips** -- **Default state**: External libraries are **visible by default** for complete context -- **Toggle anytime**: Switch between views without reloading the graph -- **Persistent**: Your preference is remembered during the session -- **Smart counting**: See exactly how many external nodes are being hidden - -This gives you the **best of both worlds** - clean focused views when you need them, and complete architectural context when you want it! - -Instead of thousands of failed call resolutions, you'll see much cleaner output focused on actual issues that need attention! - -================ -File: jest.config.cjs -================ -/** - * Jest configuration for GitNexus - */ - -module.exports = { - preset: 'ts-jest', - testEnvironment: 'node', - roots: ['/src'], - testMatch: [ - '**/__tests__/**/*.test.ts', - '**/?(*.)+(spec|test).ts' - ], - transform: { - '^.+\\.ts$': 'ts-jest', - }, - collectCoverageFrom: [ - 'src/**/*.ts', - '!src/**/*.d.ts', - '!src/__tests__/**', - '!src/**/*.test.ts', - ], - coverageDirectory: 'coverage', - coverageReporters: [ - 'text', - 'lcov', - 'html' - ], - coverageThreshold: { - global: { - branches: 70, - functions: 70, - lines: 70, - statements: 70 - } - }, - setupFilesAfterEnv: ['/src/__tests__/setup.ts'], - moduleNameMapping: { - '^@/(.*)$': '/src/$1' - }, - testTimeout: 10000, - verbose: true -}; - -================ -File: KUZUDB_INTEGRATION_GUIDE.md -================ -# KuzuDB Integration Guide - -## ๐ŸŽฏ Current Status: **READY FOR USE** - -### โœ… What's Ready: -- **KuzuDB Core**: Fully implemented with official `kuzu-wasm` npm package -- **Database Operations**: Create, open, close, query with Cypher -- **Schema Management**: Node/relationship table creation -- **Data Operations**: Insert nodes and relationships -- **Persistent Storage**: IDBFS for browser persistence -- **AI Chat Integration**: KuzuDB-enhanced RAG orchestrator -- **Performance Monitoring**: Comprehensive performance tracking -- **Testing Infrastructure**: Complete test suite - -### ๐Ÿ”ง What's Missing: -- **UI Integration**: Need to connect to the chat interface -- **Automatic Graph Import**: Need to trigger when knowledge graph is built - ---- - -## ๐Ÿš€ Quick Start - -### 1. Test KuzuDB Integration -```typescript -// From browser console -await window.testKuzuDB(); -await window.testKuzuDBPerformance(); -``` - -### 2. Use KuzuDB in Code -```typescript -import { initKuzuDB } from './src/core/kuzu/kuzu-loader.js'; - -// Initialize KuzuDB -const kuzuInstance = await initKuzuDB(); - -// Create persistent database -await kuzuInstance.createDatabase('/database'); - -// Create schema -await kuzuInstance.createNodeTable('Function', { - id: 'STRING', - name: 'STRING', - filePath: 'STRING' -}); - -// Insert data -await kuzuInstance.insertNode('Function', { - id: 'func1', - name: 'processData', - filePath: '/src/main.ts' -}); - -// Query with Cypher -const result = await kuzuInstance.executeQuery( - 'MATCH (f:Function) RETURN f.name, f.filePath' -); -``` - ---- - -## ๐Ÿค– AI Chat Integration - -### Using KuzuDB with AI Chat: -```typescript -import { KuzuRAGOrchestrator } from './src/ai/kuzu-rag-orchestrator.js'; - -// Initialize the orchestrator -const orchestrator = new KuzuRAGOrchestrator(llmService, cypherGenerator); -await orchestrator.initialize(); - -// Set context with knowledge graph -await orchestrator.setContext({ - graph: knowledgeGraph, - fileContents: fileContentsMap -}); - -// Ask questions - AI will use KuzuDB for queries -const response = await orchestrator.answerQuestion( - "What functions call the validateInput function?", - llmConfig, - { useKuzuDB: true } -); -``` - -### Example AI Queries: -- "Show me all functions in the authentication module" -- "What files import the utils module?" -- "Find all functions that call the database service" -- "Which classes extend the BaseController?" - ---- - -## ๐Ÿ“Š Performance Monitoring - -### Enable Performance Monitoring: -```typescript -import { setFeatureFlag } from './src/config/feature-flags.js'; - -// Enable performance monitoring -setFeatureFlag('enableKuzuDBPerformanceMonitoring', true); -``` - -### Monitor Performance: -```typescript -import { - startKuzuOperation, - endKuzuOperation, - getKuzuPerformanceReport, - logKuzuPerformanceSummary -} from './src/lib/kuzu-performance-monitor.js'; - -// Start monitoring an operation -const opId = startKuzuOperation('database_query', { query: 'MATCH (n) RETURN n' }); - -// ... perform operation ... - -// End monitoring -endKuzuOperation(opId, true); - -// Get performance report -const report = getKuzuPerformanceReport(); -console.log(report); - -// Log summary -logKuzuPerformanceSummary(); -``` - -### Performance Metrics Available: -- **Query Execution Time**: How long each Cypher query takes -- **Database Operations**: Create, open, close, insert operations -- **Success/Failure Rates**: Track operation success -- **Operation Types**: Group by query type, schema operations, etc. -- **Historical Data**: Last 1000 operations with timestamps - ---- - -## ๐Ÿ”ง Configuration - -### Feature Flags: -```typescript -import { featureFlags } from './src/config/feature-flags.js'; - -// Enable all KuzuDB features -featureFlags.enableKuzuDB(); - -// Or configure individually -featureFlags.setFlags({ - enableKuzuDB: true, - enableKuzuDBPersistence: true, - enableKuzuDBPerformanceMonitoring: true -}); -``` - -### Vite Configuration: -Already configured in `vite.config.ts`: -- WASM support enabled -- KuzuDB package optimized -- Proper bundling configuration - ---- - -## ๐Ÿงช Testing - -### Run Integration Tests: -```typescript -import { testKuzuDB, testKuzuDBPerformance } from './src/lib/kuzu-integration.js'; - -// Test basic functionality -await testKuzuDB(); - -// Test performance -await testKuzuDBPerformance(); -``` - -### Test GitNexus Schema: -```typescript -import { testGitNexusSchema } from './src/lib/kuzu-test.js'; - -// Test with GitNexus-like data -await testGitNexusSchema(); -``` - ---- - -## ๐Ÿ”„ Integration Steps - -### 1. Connect to Chat Interface: -```typescript -// In your chat component -import { KuzuRAGOrchestrator } from './src/ai/kuzu-rag-orchestrator.js'; - -// Initialize when knowledge graph is built -const orchestrator = new KuzuRAGOrchestrator(llmService, cypherGenerator); -await orchestrator.initialize(); - -// Set context when graph is available -await orchestrator.setContext({ - graph: knowledgeGraph, - fileContents: fileContentsMap -}); - -// Use for chat responses -const response = await orchestrator.answerQuestion(userQuestion, llmConfig); -``` - -### 2. Automatic Graph Import: -```typescript -// When knowledge graph is built -import { KuzuQueryEngine } from './src/core/graph/kuzu-query-engine.js'; - -const queryEngine = new KuzuQueryEngine(); -await queryEngine.initialize(); -await queryEngine.importGraph(knowledgeGraph); -``` - ---- - -## ๐Ÿ“ˆ Performance Benefits - -### Expected Improvements: -- **Query Speed**: 5-10x faster than in-memory queries -- **Memory Efficiency**: Persistent storage reduces memory usage -- **Scalability**: Handles larger knowledge graphs -- **Persistence**: Data survives page refreshes - -### Monitoring Results: -```bash -๐Ÿ“Š KuzuDB Performance Summary: - Total Operations: 150 - Success Rate: 98.7% - Average Duration: 12.34ms - Min Duration: 2.15ms - Max Duration: 45.67ms - Total Duration: 1851.00ms - -Operations by Type: - database_query: 100 ops, 8.45ms avg - schema_creation: 20 ops, 15.23ms avg - data_insertion: 30 ops, 18.67ms avg -``` - ---- - -## ๐Ÿšจ Troubleshooting - -### Common Issues: - -1. **KuzuDB not loading**: - ```typescript - // Check if WASM is supported - if (typeof WebAssembly === 'undefined') { - console.error('WebAssembly not supported'); - } - ``` - -2. **Performance monitoring not working**: - ```typescript - // Enable monitoring - setFeatureFlag('enableKuzuDBPerformanceMonitoring', true); - ``` - -3. **Database persistence issues**: - ```typescript - // Check browser storage - console.log('IndexedDB available:', 'indexedDB' in window); - ``` - -### Debug Mode: -```typescript -import { featureFlags } from './src/config/feature-flags.js'; - -// Enable debug mode -featureFlags.enableDebugMode(); -``` - ---- - -## ๐ŸŽ‰ Ready to Use! - -The KuzuDB integration is **fully functional** and ready for production use. The AI chat can now query the knowledge graph using actual Cypher queries with significant performance improvements. - -**Next Steps:** -1. Connect the `KuzuRAGOrchestrator` to your chat interface -2. Test with real knowledge graph data -3. Monitor performance and optimize as needed -4. Enjoy faster, more powerful code analysis! ๐Ÿš€ - -================ -File: log.txt -================ -Starting ZIP processing... trumio-cortex-core.zip -zip.ts:64 Starting complete ZIP extraction of: trumio-cortex-core.zip (27513697 bytes) -zip.ts:118 Skipping large file: trumio-cortex-core/.git/objects/pack/pack-4d5e84ce4d4c68c3dcdaf68f49c25a0693180f62.pack (27326542 bytes) -(anonymous) @ zip.ts:118 -forEach @ jszip.js?v=7c98553f:349 -extractCompleteStructure @ zip.ts:102 -await in extractCompleteStructure -processZipFile @ ingestion.service.ts:94 -handleFileUpload @ HomePage.tsx:118 -callCallback2 @ chunk-YZVM2MHU.js?v=5b2c6c96:3674 -invokeGuardedCallbackDev @ chunk-YZVM2MHU.js?v=5b2c6c96:3699 -invokeGuardedCallback @ chunk-YZVM2MHU.js?v=5b2c6c96:3733 -invokeGuardedCallbackAndCatchFirstError @ chunk-YZVM2MHU.js?v=5b2c6c96:3736 -executeDispatch @ chunk-YZVM2MHU.js?v=5b2c6c96:7014 -processDispatchQueueItemsInOrder @ chunk-YZVM2MHU.js?v=5b2c6c96:7034 -processDispatchQueue @ chunk-YZVM2MHU.js?v=5b2c6c96:7043 -dispatchEventsForPlugins @ chunk-YZVM2MHU.js?v=5b2c6c96:7051 -(anonymous) @ chunk-YZVM2MHU.js?v=5b2c6c96:7174 -batchedUpdates$1 @ chunk-YZVM2MHU.js?v=5b2c6c96:18913 -batchedUpdates @ chunk-YZVM2MHU.js?v=5b2c6c96:3579 -dispatchEventForPluginEventSystem @ chunk-YZVM2MHU.js?v=5b2c6c96:7173 -dispatchEventWithEnableCapturePhaseSelectiveHydrationWithoutDiscreteEventReplay @ chunk-YZVM2MHU.js?v=5b2c6c96:5478 -dispatchEvent @ chunk-YZVM2MHU.js?v=5b2c6c96:5472 -dispatchDiscreteEvent @ chunk-YZVM2MHU.js?v=5b2c6c96:5449Understand this warning -zip.ts:146 ZIP: Discovered 271 total paths, 109 files with content -zip.ts:147 ZIP: Total extracted size: 401503 bytes -workerUtils.ts:43 Ingestion worker initialized successfully -ingestion.worker.ts?worker_file&type=module:15 IngestionWorker: Starting processing with 271 files -pipeline.ts:19 ๐Ÿš€ Starting 4-pass ingestion for project: trumio-cortex-core -pipeline.ts:20 ๐Ÿ“ Pass 1: Analyzing project structure... -structure-processor.ts:71 StructureProcessor: Processing 271 complete paths -structure-processor.ts:75 StructureProcessor: Found 74 directories and 197 files -structure-processor.ts:79 StructureProcessor: Hiding 33 ignored directories from KG -structure-processor.ts:86 StructureProcessor: Hiding 82 files in ignored directories from KG -structure-processor.ts:227 StructureProcessor: Created 156 CONTAINS relationships -structure-processor.ts:92 StructureProcessor: Created 157 nodes total (115 items hidden) -pipeline.ts:26 ๐Ÿ” Pass 2: Parsing code and extracting definitions... -parsing-processor.ts:40 ParsingProcessor: Processing 271 total paths -parsing-processor.ts:42 Memory status: 0MB used, 0 files cached -parsing-processor.ts:44 ParsingProcessor: After filtering: 32 files to parse -parsing-processor.ts:49 ParsingProcessor: Found 22 source files and 3 config files, processing in batches of 10 -parser-loader.ts:65 Loading TypeScript parser from: /wasm/typescript/tree-sitter-typescript.wasm -parser-loader.ts:68 TypeScript parser loaded successfully -parsing-processor.ts:121 typescript parser loaded successfully. -parser-loader.ts:49 Loading JavaScript parser from: /wasm/javascript/tree-sitter-javascript.wasm -parser-loader.ts:52 JavaScript parser loaded successfully -parsing-processor.ts:121 javascript parser loaded successfully. -parser-loader.ts:33 Loading Python parser from: /wasm/python/tree-sitter-python.wasm -parser-loader.ts:36 Python parser loaded successfully -parsing-processor.ts:121 python parser loaded successfully. -parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/docker-compose.yaml -parseFile @ parsing-processor.ts:203 -(anonymous) @ parsing-processor.ts:112 -await in (anonymous) -processAll @ shared-utils.ts:274 -await in processAll -process @ parsing-processor.ts:122 -await in process -run @ pipeline.ts:49 -await in run -processRepository @ ingestion.worker.ts:68 -callback @ comlink.js?v=4af0e45c:91Understand this warning -parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/manifests/uat/deployment_cortex.yml -parseFile @ parsing-processor.ts:203 -(anonymous) @ parsing-processor.ts:112 -await in (anonymous) -processAll @ shared-utils.ts:274 -await in processAll -process @ parsing-processor.ts:122 -await in process -run @ pipeline.ts:49 -await in run -processRepository @ ingestion.worker.ts:68 -callback @ comlink.js?v=4af0e45c:91Understand this warning -parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/manifests/uat/service_cortex.yml -parseFile @ parsing-processor.ts:203 -(anonymous) @ parsing-processor.ts:112 -await in (anonymous) -processAll @ shared-utils.ts:274 -await in processAll -process @ parsing-processor.ts:122 -await in process -run @ pipeline.ts:49 -await in run -processRepository @ ingestion.worker.ts:68 -callback @ comlink.js?v=4af0e45c:91Understand this warning -parsing-processor.ts:72 ParsingProcessor: Successfully processed 25 files -pipeline.ts:36 ๐Ÿ”— Pass 3: Resolving imports and building dependency map... -import-processor.ts:42 ImportProcessor: Starting import resolution... -import-processor.ts:52 ImportProcessor: Completed import resolution -import-processor.ts:53 ImportProcessor: Found 160 imports, resolved 90 (56.3%) -import-processor.ts:54 ImportProcessor: Built import map for 22 files -pipeline.ts:38 ๐Ÿ“ž Pass 4: Resolving function calls with 3-stage strategy... -call-processor.ts:33 CallProcessor: Starting call resolution with 3-stage strategy... -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in check_prompt.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: AsyncIOMotorClient in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: limit in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: sort in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: find in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: close in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: run in check_prompt.py -call-processor.ts:65 CallProcessor: Found 1 function calls in trumio-cortex-core/check_prompt.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: AsyncIOMotorClient in debug_channel_messages.py -2call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: find_one in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: now in debug_channel_messages.py -call-processor.ts:65 CallProcessor: Found 5 function calls in trumio-cortex-core/debug_channel_messages.py -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:24 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:31 -call-processor.ts:86 โŒ Failed to resolve call: timedelta in trumio-cortex-core/debug_channel_messages.py:49 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:51 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in mcp_auth_service.py -call-processor.ts:65 CallProcessor: Found 14 function calls in trumio-cortex-core/src/python/common_services/mcp_auth_service.py -call-processor.ts:86 โŒ Failed to resolve call: HTTPBaseModel in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:32 -call-processor.ts:86 โŒ Failed to resolve call: hexdigest in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:99 -call-processor.ts:86 โŒ Failed to resolve call: new in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:99 -call-processor.ts:86 โŒ Failed to resolve call: compare_digest in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:130 -call-processor.ts:86 โŒ Failed to resolve call: body in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:227 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in openai_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in openai_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ValueError in openai_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: OpenAI in openai_service.py -call-processor.ts:65 CallProcessor: Found 12 function calls in trumio-cortex-core/src/python/common_services/openai_service.py -call-processor.ts:86 โŒ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:30 -call-processor.ts:86 โŒ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:31 -call-processor.ts:86 โŒ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:32 -call-processor.ts:86 โŒ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:33 -call-processor.ts:86 โŒ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:89 -call-processor.ts:86 โŒ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:90 -call-processor.ts:86 โŒ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:91 -call-processor.ts:86 โŒ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:92 -call-processor.ts:86 โŒ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:122 -call-processor.ts:86 โŒ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:123 -call-processor.ts:86 โŒ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:124 -call-processor.ts:86 โŒ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:125 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: isinstance in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: TypeError in prompt_registry_service.py -2call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ValueError in prompt_registry_service.py -2call-processor.ts:553 ๐Ÿ” Filtered out: now in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: insert_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: str in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: limit in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: sort in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: error in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ValueError in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: now in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: update_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ObjectId in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: delete_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ObjectId in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: limit in prompt_registry_service.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/common_services/prompt_registry_service.py -call-processor.ts:58 ๐Ÿ“Š Debug: prompt_registry_service.py has 37 call nodes, 7 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: append in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: HTTPException in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: str in chat.py -call-processor.ts:65 CallProcessor: Found 7 function calls in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:15 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:28 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:29 -11call-processor.ts:553 ๐Ÿ” Filtered out: Field in chat.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/demo_app/app/schemas/chat.py -call-processor.ts:58 ๐Ÿ“Š Debug: chat.py has 11 call nodes, 4 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: basicConfig in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: StreamHandler in start-service.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in start-service.py -call-processor.ts:65 CallProcessor: Found 15 function calls in trumio-cortex-core/src/python/demo_app/start-service.py -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:26 -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:27 -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:28 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:37 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:40 -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:43 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:59 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:62 -call-processor.ts:86 โŒ Failed to resolve call: get_motor_client in trumio-cortex-core/src/python/demo_app/start-service.py:77 -call-processor.ts:86 โŒ Failed to resolve call: compile_graphs in trumio-cortex-core/src/python/demo_app/start-service.py:87 -call-processor.ts:86 โŒ Failed to resolve call: close_motor_client in trumio-cortex-core/src/python/demo_app/start-service.py:99 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:132 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:135 -call-processor.ts:86 โŒ Failed to resolve call: RedirectResponse in trumio-cortex-core/src/python/demo_app/start-service.py:141 -call-processor.ts:86 โŒ Failed to resolve call: get_swagger_ui_html in trumio-cortex-core/src/python/demo_app/start-service.py:145 -call-processor.ts:553 ๐Ÿ” Filtered out: append in interactions_endpoint.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in interactions_endpoint.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in interactions_endpoint.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in interactions_endpoint.py -call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:14 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:16 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:18 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:19 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:20 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:21 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:22 -call-processor.ts:86 โŒ Failed to resolve call: fetch_and_process_project_chats_for_period in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:32 -call-processor.ts:553 ๐Ÿ” Filtered out: append in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in mcp_auth.py -call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py:14 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py:69 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in config.py -9call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: lower in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -3call-processor.ts:553 ๐Ÿ” Filtered out: bool in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in config.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/config.py -call-processor.ts:58 ๐Ÿ“Š Debug: config.py has 19 call nodes, 0 definitions -10call-processor.ts:553 ๐Ÿ” Filtered out: Field in processed_text_unit.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/schemas/processed_text_unit.py -call-processor.ts:58 ๐Ÿ“Š Debug: processed_text_unit.py has 10 call nodes, 1 definitions -2call-processor.ts:553 ๐Ÿ” Filtered out: Field in watermark.py -call-processor.ts:553 ๐Ÿ” Filtered out: now in watermark.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/schemas/watermark.py -call-processor.ts:58 ๐Ÿ“Š Debug: watermark.py has 3 call nodes, 2 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: append in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in path_utils.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/utils/path_utils.py -call-processor.ts:58 ๐Ÿ“Š Debug: path_utils.py has 9 call nodes, 1 definitions -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: AsyncIOMotorClient in debug_channel_messages.py -3call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: aggregate in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: aggregate in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:65 CallProcessor: Found 6 function calls in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:24 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:54 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:61 -call-processor.ts:86 โŒ Failed to resolve call: timedelta in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:77 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:79 -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: basicConfig in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: StreamHandler in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:65 CallProcessor: Found 5 function calls in trumio-cortex-core/src/python/ms_teams_connector/start-service.py -call-processor.ts:86 โŒ Failed to resolve call: connect_db_client in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:44 -call-processor.ts:86 โŒ Failed to resolve call: disconnect_db_client in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:50 -call-processor.ts:86 โŒ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:79 -call-processor.ts:553 ๐Ÿ” Filtered out: append in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in mcp_auth.py -call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py:14 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py:69 -call-processor.ts:553 ๐Ÿ” Filtered out: append in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: critical in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: error in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: HTTPException in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in quality_analyzer.py -call-processor.ts:65 CallProcessor: Found 23 function calls in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:36 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:38 -call-processor.ts:86 โŒ Failed to resolve call: compile_graphs in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:46 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:74 -call-processor.ts:86 โŒ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:80 -call-processor.ts:86 โŒ Failed to resolve call: calculate_date_range in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:100 -call-processor.ts:86 โŒ Failed to resolve call: invoke_weekly_user_fetch_graph in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:104 -call-processor.ts:86 โŒ Failed to resolve call: run_full_weekly_analysis in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:126 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:148 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:157 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:158 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:159 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:160 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:179 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:187 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:197 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:205 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:206 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:214 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:220 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:228 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:229 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:237 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -10call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: strip in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -5call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: lower in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -6call-processor.ts:553 ๐Ÿ” Filtered out: bool in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in config.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/pr_quality_service/app/config.py -call-processor.ts:58 ๐Ÿ“Š Debug: config.py has 38 call nodes, 0 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: append in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in path_utils.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/pr_quality_service/app/utils/path_utils.py -call-processor.ts:58 ๐Ÿ“Š Debug: path_utils.py has 9 call nodes, 1 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: basicConfig in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: StreamHandler in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:65 CallProcessor: Found 6 function calls in trumio-cortex-core/src/python/pr_quality_service/start-service.py -call-processor.ts:86 โŒ Failed to resolve call: connect_db_client in trumio-cortex-core/src/python/pr_quality_service/start-service.py:45 -call-processor.ts:86 โŒ Failed to resolve call: disconnect_db_client in trumio-cortex-core/src/python/pr_quality_service/start-service.py:54 -call-processor.ts:86 โŒ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/pr_quality_service/start-service.py:97 -call-processor.ts:86 โŒ Failed to resolve call: get_database in trumio-cortex-core/src/python/pr_quality_service/start-service.py:104 -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: append in prompt_test.py -6call-processor.ts:553 ๐Ÿ” Filtered out: print in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: run in prompt_test.py -call-processor.ts:65 CallProcessor: Found 3 function calls in trumio-cortex-core/src/python/pr_quality_service/test/prompt_test.py -call-processor.ts:86 โŒ Failed to resolve call: get_weekly_prompts in trumio-cortex-core/src/python/pr_quality_service/test/prompt_test.py:31 -call-processor.ts:723 ๐Ÿ“Š CallProcessor Resolution Statistics: -call-processor.ts:724 Total calls processed: 121 -call-processor.ts:725 โœ… Exact matches (Stage 1): 11 (9.1%) -call-processor.ts:726 โœ… Same-file matches (Stage 2): 19 (15.7%) -call-processor.ts:727 ๐ŸŽฏ Heuristic matches (Stage 3): 4 (3.3%) -call-processor.ts:728 โŒ Failed resolutions: 87 (71.9%) -call-processor.ts:729 Success rate: 28.1% -pipeline.ts:41 Ingestion complete. Graph contains 417 nodes and 446 relationships. -pipeline.ts:50 ๐Ÿ“Š Graph Statistics: -pipeline.ts:51 Nodes by type: {Project: 1, Folder: 41, File: 140, Import: 160, Function: 52,ย โ€ฆ} -pipeline.ts:52 Relationships by type: {CONTAINS: 156, DEFINES: 235, IMPORTS: 23, CALLS: 32} -pipeline.ts:60 โš ๏ธ Found 3 isolated nodes: -run @ pipeline.ts:60 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:65 Isolated nodes by type: {File: 3} -run @ pipeline.ts:65 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:66 Sample isolated nodes: (3)ย [{โ€ฆ}, {โ€ฆ}, {โ€ฆ}] -run @ pipeline.ts:66 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:82 โš ๏ธ Found 140 files without definitions: -run @ pipeline.ts:82 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:83 Files without content: (5)ย ['trumio-cortex-core/', 'trumio-cortex-core/.github/', 'trumio-cortex-core/.github/workflows/', 'trumio-cortex-core/.github/workflows/cortex-ci-cd.yml', 'trumio-cortex-core/.github/workflows/readme.md'] -run @ pipeline.ts:83 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:94 ๐Ÿ” Validating graph integrity... -pipeline.ts:130 Source files without definitions: (3)ย ['trumio-cortex-core/check_prompt.py', 'trumio-cortex-core/debug_channel_messages.py', 'trumio-cortex-core/src/python/common_services/mcp_auth_service.py'] -validateGraphIntegrity @ pipeline.ts:130 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:158 โš ๏ธ Graph integrity issues found: -validateGraphIntegrity @ pipeline.ts:158 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:159 1. 25 files not connected to project structure -(anonymous) @ pipeline.ts:159 -validateGraphIntegrity @ pipeline.ts:159 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:159 2. 59 source files contain no parsed definitions -(anonymous) @ pipeline.ts:159 -validateGraphIntegrity @ pipeline.ts:159 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:159 3. 66 definitions not connected to files -(anonymous) @ pipeline.ts:159 -validateGraphIntegrity @ pipeline.ts:159 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -:5173/src/workers/ingestion.worker.ts?worker_file&type=module:40 IngestionWorker: Processing completed successfully -:5173/src/workers/ingestion.worker.ts?worker_file&type=module:41 Graph contains 417 nodes and 446 relationships -HomePage.tsx:123 ZIP processing completed: {nodeCount: 417, relationshipCount: 446, fileCount: 109} -:5173/src/workers/ingestion.worker.ts?worker_file&type=module:150 Ingestion worker terminated -workerUtils.ts:127 Ingestion worker terminate - -================ -File: package.json -================ -{ - "name": "gitnexus", - "private": true, - "version": "1.0.0", - "type": "module", - "scripts": { - "dev": "vite", - "build": "tsc -b && vite build", - "lint": "eslint .", - "preview": "vite preview", - "test": "jest", - "test:watch": "jest --watch", - "test:coverage": "jest --coverage", - "test:ci": "jest --ci --coverage --watchAll=false" - }, - "dependencies": { - "@langchain/anthropic": "^0.1.21", - "@langchain/core": "^0.3.66", - "@langchain/google-genai": "^0.2.16", - "@langchain/langgraph": "^0.0.26", - "@langchain/openai": "^0.0.28", - "@types/d3": "^7.4.3", - "axios": "^1.6.0", - "comlink": "^4.4.1", - "d3": "^7.9.0", - "jszip": "^3.10.1", - "kuzu-wasm": "^0.11.1", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "react-markdown": "^10.1.0", - "rehype-highlight": "^7.0.2", - "remark-gfm": "^4.0.1", - "uuid": "^11.1.0", - "web-tree-sitter": "^0.20.8", - "zod": "^3.25.76" - }, - "devDependencies": { - "@eslint/js": "^9.11.1", - "@types/jest": "^29.5.12", - "@types/node": "^20.12.12", - "@types/react": "^18.3.10", - "@types/react-dom": "^18.3.0", - "@vitejs/plugin-react": "^4.3.2", - "eslint": "^9.11.1", - "eslint-plugin-react-hooks": "^5.1.0-rc.0", - "eslint-plugin-react-refresh": "^0.4.12", - "globals": "^15.9.0", - "jest": "^29.7.0", - "ts-jest": "^29.1.2", - "typescript": "^5.5.3", - "typescript-eslint": "^8.7.0", - "vite": "^5.4.8" - } -} - -================ -File: project_guide.md -================ -# GitNexus Project Guide - -This document provides a comprehensive technical overview of the GitNexus application, intended to give an LLM full context of the project's purpose, architecture, and implementation details. - -## 1. Project Overview - -GitNexus is a **client-side source code analysis tool** that runs entirely in the browser. It transforms a given codebase (from a public GitHub repository or an uploaded ZIP file) into an interactive **knowledge graph**. - -The primary goal is to allow users to visually explore and understand complex codebases through two main interfaces: -1. **A Graph Visualizer**: Displays the code structure as a network of nodes (files, classes, functions) and relationships (imports, calls, inheritance). -2. **An AI Chat Interface**: A Retrieval-Augmented Generation (RAG) system that allows users to ask natural language questions about the code, which are answered by querying the knowledge graph. - -Because it's fully client-side, no code is ever sent to a server, ensuring privacy and security. - -## 2. Technology Stack - -The project is built with a modern web technology stack: - -- **Frontend Framework**: **React 18** with **TypeScript**. -- **Build Tool**: **Vite** for fast development and optimized builds. -- **Code Parsing**: **Tree-sitter** compiled to **WebAssembly (WASM)**. This allows for fast and accurate Abstract Syntax Tree (AST) parsing directly in the browser. -- **AI & RAG**: **LangChain.js** is used to orchestrate the AI agent, supporting multiple LLM providers (OpenAI, Anthropic, Google Gemini). -- **Concurrency**: **Web Workers** are used to run the entire code ingestion process in a background thread, preventing the UI from freezing. **Comlink** is used for simplifying communication with the worker. -- **Graph Visualization**: The UI likely uses a library like Cytoscape.js or D3.js to render the interactive graph (inferred from dependencies and functionality). -- **Styling**: A custom CSS-in-JS solution implemented directly within the `HomePage.tsx` component. - -## 3. Architecture - -The project follows a clean, modular architecture that separates concerns into distinct layers. - -### `src/ui` - The Frontend Layer -- **Purpose**: Contains all React components, hooks, and pages. -- **Key Files**: - - `pages/HomePage.tsx`: The main component that manages the application's state and orchestrates all user interactions. It handles user input, triggers the ingestion process, and displays the results. - - `components/graph/GraphExplorer.tsx`: The React component responsible for rendering the interactive knowledge graph. - - `components/chat/ChatInterface.tsx`: The component for the AI-powered chat. - - `components/ErrorBoundary.tsx`: A crucial component for catching and gracefully handling runtime errors in the UI. - -### `src/core` - The Core Logic Layer -- **Purpose**: This is the engine of the application where the knowledge graph is built. -- **Key Files**: - - `ingestion/pipeline.ts`: Defines the `GraphPipeline`, which executes the multi-pass ingestion process. - - `ingestion/structure-processor.ts`: **Pass 1**: Analyzes the file and directory structure. - - `ingestion/parsing-processor.ts`: **Pass 2**: Uses Tree-sitter to parse source files into ASTs and extracts definitions (classes, functions, etc.). - - `ingestion/import-processor.ts`: **Pass 3**: Resolves import statements between files. - - `ingestion/call-processor.ts`: **Pass 4**: Resolves function and method calls between definitions. - - `graph/graph.ts`: Defines the `SimpleKnowledgeGraph` class, the core data structure for the graph. - - `tree-sitter/parser-loader.ts`: Manages the loading of the various language-specific Tree-sitter WASM parsers. - -### `src/workers` - The Concurrency Layer -- **Purpose**: Offloads the intensive ingestion process from the main UI thread. -- **Key Files**: - - `ingestion.worker.ts`: The entry point for the Web Worker. It receives file data from the UI, runs the `GraphPipeline`, and posts the resulting knowledge graph back to the main thread. - -### `src/services` - The External Services Layer -- **Purpose**: Handles communication with external sources. -- **Key Files**: - - `ingestion.service.ts`: Acts as a bridge between the UI (`HomePage.tsx`) and the `IngestionWorker`. - - `github.ts`: Contains logic for fetching repository contents from the GitHub API. - - `zip.ts`: Contains logic for reading and extracting files from an uploaded ZIP archive. - -### `src/ai` - The Artificial Intelligence Layer -- **Purpose**: Manages the LLM interactions for the Graph RAG chat. -- **Key Files**: - - `llm-service.ts`: A client that handles communication with the different supported LLM providers (OpenAI, etc.). - - `langchain-orchestrator.ts`: Implements the ReAct agent logic using LangChain, defining the tools the agent can use (e.g., querying the graph). - - `cypher-generator.ts`: Translates natural language questions into Cypher-like queries to be executed against the knowledge graph. - -## 4. Core Concepts & Execution Flow - -### The Knowledge Graph Data Model -The entire application revolves around the `KnowledgeGraph` object defined in `src/core/graph/graph.ts`. It's a simple structure containing two arrays: -- `nodes`: Represent code entities like `File`, `Folder`, `Class`, `Function`, etc. -- `relationships`: Represent the connections between nodes, such as `CONTAINS`, `IMPORTS`, `CALLS`. - -### The Ingestion Pipeline -This is the central process for understanding code. When a user provides a repository, the following happens: -1. **File Gathering**: The `IngestionService` fetches all file paths and their text content, either from GitHub or a ZIP file. -2. **Worker Invocation**: The file data is passed to the `IngestionWorker`. -3. **Pipeline Execution**: The worker runs the `GraphPipeline`, which executes its 4 passes sequentially on the `SimpleKnowledgeGraph` instance: - - **Pass 1 (Structure)**: Creates `Project`, `Folder`, and `File` nodes, and links them with `CONTAINS` relationships. - - **Pass 2 (Parsing)**: Parses the AST of each source file, creating `Function`, `Class`, and other code-level nodes. It links these nodes to their parent `File` node with `DEFINES` relationships. - - **Pass 3 (Imports)**: Analyzes `import` statements and creates `IMPORTS` relationships between code entities. - - **Pass 4 (Calls)**: Analyzes the code to find function and method calls, creating `CALLS` relationships between them. -4. **Return to UI**: The completed graph is returned to the `HomePage.tsx` component, which updates its state and renders the visual graph. - -This multi-pass approach ensures that the graph is built layer by layer, with each pass adding more detail and context. - -## 5. Potential Issues & Areas for Improvement - -This section details findings from a deep analysis of the codebase, highlighting areas for refactoring and potential bugs. - -### 1. State Management in `HomePage.tsx` -- **Issue**: The component uses a single, large `useState` hook to manage the entire application state (`AppState`). Any small update, such as user input in a text field, triggers a re-render of the entire component and all its children. -- **Impact**: This can lead to a sluggish UI and poor performance, especially as the application grows in complexity. -- **Recommendation**: Refactor the state management. Use more granular `useState` hooks for simple, independent state. For state that needs to be shared across many components, consider using React's Context API or a lightweight state management library to prevent unnecessary re-renders. - -### 2. Redundant `generateId` Function -- **Issue**: The file `src/core/ingestion/parsing-processor.ts` contains a local `generateId` function that uses a simple (and potentially collision-prone) hashing algorithm. A more robust, UUID-based `generateId` function already exists in `src/lib/utils.ts`. -- **Impact**: Code duplication and the risk of using an inferior ID generation method, which could lead to node ID collisions in the graph. -- **Recommendation**: Remove the local `generateId` function from `parsing-processor.ts` and update the file to import and use the centralized version from `src/lib/utils.ts`. - -### 3. Inefficient Graph Integrity Checks -- **Issue**: The `validateGraphIntegrity` method in `src/core/ingestion/pipeline.ts` performs several distinct traversals (`filter`, `some`, `find`) over the entire graph to find issues like orphaned nodes or files without definitions. -- **Impact**: On large codebases, this can significantly slow down the final phase of the ingestion process. -- **Recommendation**: Optimize these validation checks. Many of them can be combined into a single pass over the graph's nodes and relationships. Using a `Map` or `Set` for lookups within the pass would be much more performant than repeated array iterations. - -### 4. Hardcoded Filtering Logic -- **Issue**: The `CallProcessor` in `src/core/ingestion/call-processor.ts` defines its own large, hardcoded `Set` of Python built-in functions to ignore during call resolution. This logic is disconnected from the centralized language configurations. -- **Impact**: This makes the list difficult to maintain and extend. It also violates the principle of single-source-of-truth, as language-specific knowledge should be centralized. -- **Recommendation**: Refactor the `CallProcessor` to use the `builtinFunctions` set from the `language-config.ts` file. This centralizes language-specific data and makes the processor more modular. - -### 5. Mock/Incomplete Query Engine -- **Issue**: The `GraphQueryEngine` in `src/core/graph/query-engine.ts` is a mock implementation that uses regular expressions to parse Cypher-like queries. This approach is not robust and only supports a very limited subset of valid queries. -- **Impact**: This is a critical limitation for the AI chat feature. The `CypherGenerator` can produce complex queries that the engine cannot execute, leading to failed tool calls and inaccurate answers from the AI. -- **Recommendation**: This is a major area for future development. The regex-based parser should be replaced with a more robust solution. Options include: - - Implementing a proper parser for a small, well-defined subset of Cypher. - - Integrating a lightweight, in-browser graph database library that supports Cypher queries. - -================ -File: PRUNING_IMPLEMENTATION.md -================ -# ๐ŸŽฏ Two-Stage Filtering Implementation - Complete - -## ๐Ÿš€ **Successfully Implemented!** - -We have successfully implemented the sophisticated two-stage filtering architecture that decouples structural discovery from content analysis. - -## ๐Ÿ—๏ธ **Architecture Overview** - -### **Stage 1: Complete Structural Discovery** -- **โœ… GitHub Service**: Discovers ALL files and directories (including `node_modules`, `.git`, etc.) -- **โœ… ZIP Service**: Extracts ALL paths from archives (complete structure) -- **โœ… StructureProcessor**: Creates nodes for EVERY path discovered -- **โœ… Result**: Knowledge graph contains complete, accurate repository structure - -### **Stage 2: Intelligent Pruning Before Parsing** -- **โœ… ParsingProcessor**: Applies sophisticated filtering before content analysis -- **โœ… Ignore Patterns**: Comprehensive list of directories to skip during parsing -- **โœ… User Filters**: Directory and extension filters still work as before -- **โœ… Result**: Only relevant files get their content parsed and analyzed - -## ๐ŸŽฏ **Implementation Details** - -### **Enhanced ParsingProcessor** - -#### **Comprehensive Ignore Patterns** -```typescript -private static readonly IGNORE_PATTERNS = new Set([ - // Version Control - '.git', '.svn', '.hg', - - // Package Managers & Dependencies - 'node_modules', 'bower_components', 'vendor', 'deps', - - // Python Virtual Environments & Cache - 'venv', 'env', '.venv', 'virtualenv', '__pycache__', - - // Build & Distribution - 'build', 'dist', 'out', 'target', 'bin', 'obj', - - // IDE & Editor Directories - '.vs', '.vscode', '.idea', '.eclipse', - - // Temporary & Logs - 'tmp', 'temp', 'logs', 'log', - - // Coverage & Testing - 'coverage', '.coverage', 'htmlcov', - - // Cache Directories - '.cache', '.next', '.nuxt' -]); -``` - -#### **Two-Stage Filtering Logic** -```typescript -private applyFiltering(allPaths: string[], fileContents: Map, options?: FilterOptions): string[] { - // STAGE 1: Prune ignored directories - let filesToProcess = this.pruneIgnoredPaths(allPaths.filter(path => fileContents.has(path))); - - // STAGE 2: Apply user filters - if (options?.directoryFilter) { /* existing user filter logic */ } - if (options?.fileExtensions) { /* existing user filter logic */ } - - return filesToProcess; -} -``` - -#### **Intelligent Pruning** -```typescript -private pruneIgnoredPaths(filePaths: string[]): string[] { - return filePaths.filter(path => { - const pathSegments = path.split('/'); - - // Check if any segment matches ignore patterns - const hasIgnoredSegment = pathSegments.some(segment => - ParsingProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - return !hasIgnoredSegment && !this.matchesIgnorePatterns(path); - }); -} -``` - -### **Complete Structure Discovery** - -#### **GitHub Service Enhancement** -- **Removed**: `shouldSkipDirectory()` checks in `collectPathsAndContent()` -- **Result**: Discovers ALL directories, including `node_modules`, `.git`, etc. - -#### **ZIP Service Enhancement** -- **Removed**: `shouldSkipDirectory()` checks in `extractCompleteStructure()` -- **Result**: Extracts ALL paths from ZIP archives - -## ๐Ÿ“Š **Before vs After** - -| Aspect | โŒ **Before** | โœ… **After** | -|--------|---------------|--------------| -| **Structure Discovery** | Filtered early, missed directories | Complete discovery of all paths | -| **node_modules Visibility** | Missing from KG | Visible as folder node | -| **Content Parsing** | Parsed everything discovered | Intelligently skips ignored directories | -| **Performance** | Slow (parsed dependencies) | Fast (skips massive directories) | -| **KG Accuracy** | Incomplete structure | Perfect mirror of repository | -| **User Experience** | Cluttered with dependencies | Clean, focused on source code | - -## ๐ŸŽฏ **Benefits Achieved** - -### **1. Complete Accurate Structure** -``` -โœ… Repository Structure in KG: -โ”œโ”€โ”€ src/ (visible, parsed) -โ”œโ”€โ”€ tests/ (visible, parsed) -โ”œโ”€โ”€ node_modules/ (visible, NOT parsed) ๐ŸŽฏ -โ”œโ”€โ”€ .git/ (visible, NOT parsed) ๐ŸŽฏ -โ”œโ”€โ”€ dist/ (visible, NOT parsed) ๐ŸŽฏ -โ””โ”€โ”€ package.json (visible, parsed) -``` - -### **2. Performance Improvements** -- **โšก Skip Massive Directories**: No parsing of `node_modules` (thousands of files) -- **โšก Faster Processing**: Focus on actual source code -- **โšก Smaller Graphs**: Fewer definition nodes to render -- **โšก Better Memory Usage**: Avoid loading massive dependency files - -### **3. Professional User Experience** -- **๐Ÿ“Š Accurate Representation**: Users see complete project structure -- **๐ŸŽฏ Clean Analysis**: Focus on relevant code, not dependencies -- **๐Ÿ” Better Navigation**: Easy to distinguish project code from dependencies -- **๐Ÿ“ˆ Trust**: KG accurately mirrors their actual repository - -## ๐Ÿ” **Technical Highlights** - -### **Sophisticated Pattern Matching** -- **Directory Segments**: Checks each path segment against ignore patterns -- **Pattern-Based**: Handles `.egg-info`, `site-packages`, etc. -- **Hidden Directories**: Smart handling of `.github` (keep) vs `.vscode` (ignore) - -### **Logging & Visibility** -``` -ParsingProcessor: Starting with 1,247 files with content -ParsingProcessor: After pruning ignored directories: 1,247 -> 89 files -ParsingProcessor: Directory filter applied: 89 -> 45 files -``` - -### **Browser Compatibility** -- **โœ… No Node.js Dependencies**: Pure browser implementation -- **โœ… Memory Efficient**: Batched processing with size limits -- **โœ… Performance Optimized**: Skip expensive operations on ignored files - -## ๐Ÿš€ **Deployment Status** - -- **โœ… Build Success**: All TypeScript compilation passes -- **โœ… Architecture Complete**: Two-stage filtering fully implemented -- **โœ… Backward Compatible**: Existing functionality preserved -- **โœ… Production Ready**: Ready for real-world repository analysis - -## ๐ŸŽ‰ **Result** - -**Perfect Implementation!** We now have: - -1. **Complete Structure Discovery**: Every directory appears in the KG -2. **Intelligent Content Filtering**: Skip parsing massive dependency directories -3. **Optimal Performance**: Fast processing focused on relevant code -4. **Professional UX**: Clean, accurate knowledge graphs - -The two-stage filtering architecture is **successfully implemented** and ready for production! ๐Ÿš€ - -================ -File: public/vite.svg -================ - - -================ -File: public/workers/file-processing-worker.js -================ -/** - * File Processing Web Worker - * Handles parallel file processing tasks - */ - -// Import tree-sitter worker functionality -import './tree-sitter-worker.js'; - -// File processing task handlers -const fileProcessors = { - // Parse file with tree-sitter - parseFile: async (input) => { - const { filePath, content } = input; - - // Use the tree-sitter worker functionality - return await parseFile(filePath, content); - }, - - // Analyze file structure - analyzeStructure: async (input) => { - const { filePath, content } = input; - - const analysis = { - filePath, - size: content.length, - lines: content.split('\n').length, - characters: content.length, - words: content.split(/\s+/).filter(word => word.length > 0).length, - language: detectLanguage(filePath), - hasContent: content.trim().length > 0, - structure: { - imports: [], - exports: [], - functions: [], - classes: [], - variables: [] - } - }; - - // Extract basic structure information - const lines = content.split('\n'); - - for (let i = 0; i < lines.length; i++) { - const line = lines[i].trim(); - const lineNumber = i + 1; - - // Detect imports - if (line.startsWith('import ') || line.startsWith('from ')) { - analysis.structure.imports.push({ - line: lineNumber, - content: line - }); - } - - // Detect exports - if (line.startsWith('export ')) { - analysis.structure.exports.push({ - line: lineNumber, - content: line - }); - } - - // Detect function declarations - if (line.match(/^(function|const|let|var)\s+\w+\s*[=\(]/)) { - analysis.structure.functions.push({ - line: lineNumber, - content: line - }); - } - - // Detect class declarations - if (line.match(/^class\s+\w+/)) { - analysis.structure.classes.push({ - line: lineNumber, - content: line - }); - } - - // Detect variable declarations - if (line.match(/^(const|let|var)\s+\w+\s*=/)) { - analysis.structure.variables.push({ - line: lineNumber, - content: line - }); - } - } - - return analysis; - }, - - // Extract dependencies - extractDependencies: async (input) => { - const { filePath, content } = input; - - const dependencies = { - filePath, - imports: [], - requires: [], - dynamicImports: [] - }; - - const lines = content.split('\n'); - - for (let i = 0; i < lines.length; i++) { - const line = lines[i].trim(); - const lineNumber = i + 1; - - // ES6 imports - const importMatch = line.match(/import\s+(?:(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)\s+from\s+)?['"`]([^'"`]+)['"`]/); - if (importMatch) { - dependencies.imports.push({ - module: importMatch[1], - line: lineNumber, - fullLine: line - }); - } - - // CommonJS requires - const requireMatch = line.match(/require\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/); - if (requireMatch) { - dependencies.requires.push({ - module: requireMatch[1], - line: lineNumber, - fullLine: line - }); - } - - // Dynamic imports - const dynamicImportMatch = line.match(/import\s*\(\s*['"`]([^'"`]+)['"`]\s*\)/); - if (dynamicImportMatch) { - dependencies.dynamicImports.push({ - module: dynamicImportMatch[1], - line: lineNumber, - fullLine: line - }); - } - } - - return dependencies; - }, - - // Extract function calls - extractFunctionCalls: async (input) => { - const { filePath, content } = input; - - const functionCalls = { - filePath, - calls: [] - }; - - // Simple regex-based function call detection - // This is a simplified version - in practice, you'd use AST parsing - const functionCallRegex = /(\w+)\s*\(/g; - let match; - - while ((match = functionCallRegex.exec(content)) !== null) { - const functionName = match[1]; - const lineNumber = content.substring(0, match.index).split('\n').length; - - // Skip common keywords that might match the pattern - const keywords = ['if', 'for', 'while', 'switch', 'catch', 'typeof', 'instanceof']; - if (!keywords.includes(functionName)) { - functionCalls.calls.push({ - functionName, - line: lineNumber, - index: match.index - }); - } - } - - return functionCalls; - }, - - // Analyze code complexity - analyzeComplexity: async (input) => { - const { filePath, content } = input; - - const complexity = { - filePath, - cyclomaticComplexity: 0, - nestingDepth: 0, - maxLineLength: 0, - averageLineLength: 0, - commentRatio: 0 - }; - - const lines = content.split('\n'); - let totalLength = 0; - let commentLines = 0; - let currentNesting = 0; - let maxNesting = 0; - - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - const trimmedLine = line.trim(); - - // Calculate line length - const lineLength = line.length; - totalLength += lineLength; - complexity.maxLineLength = Math.max(complexity.maxLineLength, lineLength); - - // Count comment lines - if (trimmedLine.startsWith('//') || trimmedLine.startsWith('/*') || trimmedLine.startsWith('*')) { - commentLines++; - } - - // Calculate nesting depth - if (trimmedLine.includes('{')) { - currentNesting++; - maxNesting = Math.max(maxNesting, currentNesting); - } - if (trimmedLine.includes('}')) { - currentNesting = Math.max(0, currentNesting - 1); - } - - // Calculate cyclomatic complexity (simplified) - const complexityKeywords = ['if', 'else', 'for', 'while', 'case', 'catch', '&&', '||', '?']; - for (const keyword of complexityKeywords) { - if (line.includes(keyword)) { - complexity.cyclomaticComplexity++; - } - } - } - - complexity.averageLineLength = lines.length > 0 ? totalLength / lines.length : 0; - complexity.commentRatio = lines.length > 0 ? commentLines / lines.length : 0; - complexity.nestingDepth = maxNesting; - - return complexity; - } -}; - -// Detect language from file path -function detectLanguage(filePath) { - const ext = filePath.split('.').pop()?.toLowerCase(); - - const languageMap = { - 'js': 'javascript', - 'jsx': 'javascript', - 'ts': 'typescript', - 'tsx': 'typescript', - 'py': 'python', - 'java': 'java', - 'cpp': 'cpp', - 'c': 'c', - 'cs': 'csharp', - 'php': 'php', - 'rb': 'ruby', - 'go': 'go', - 'rs': 'rust', - 'json': 'json', - 'yaml': 'yaml', - 'yml': 'yaml', - 'md': 'markdown', - 'html': 'html', - 'css': 'css', - 'scss': 'scss', - 'sass': 'sass' - }; - - return languageMap[ext] || 'unknown'; -} - -// Main message handler -self.onmessage = async function(event) { - const { taskId, input } = event.data; - - try { - const { processorType, ...processorInput } = input; - - // Get the appropriate processor - const processor = fileProcessors[processorType]; - if (!processor) { - throw new Error(`Unknown processor type: ${processorType}`); - } - - // Execute the processor - const result = await processor(processorInput); - - // Send result back to main thread - self.postMessage({ - taskId, - result - }); - - } catch (error) { - // Send error back to main thread - self.postMessage({ - taskId, - error: error.message || 'Unknown error in file processing worker' - }); - } -}; - -// Handle worker errors -self.onerror = function(error) { - console.error('Worker: Unhandled error:', error); - self.postMessage({ - taskId: 'error', - error: error.message || 'Unhandled worker error' - }); -}; - -// Handle unhandled promise rejections -self.onunhandledrejection = function(event) { - console.error('Worker: Unhandled promise rejection:', event.reason); - self.postMessage({ - taskId: 'error', - error: event.reason?.message || 'Unhandled promise rejection' - }); -}; - -console.log('Worker: File processing worker script loaded'); - -================ -File: public/workers/generic-worker.js -================ -/** - * Generic Web Worker - * Handles various processing tasks that can be offloaded to workers - */ - -// Task handlers -const taskHandlers = { - // Text processing tasks - textAnalysis: async (input) => { - const { text, analysisType } = input; - - switch (analysisType) { - case 'wordCount': - return { - wordCount: text.split(/\s+/).filter(word => word.length > 0).length, - charCount: text.length, - lineCount: text.split('\n').length - }; - - case 'identifierExtraction': - const identifiers = text.match(/[a-zA-Z_][a-zA-Z0-9_]*/g) || []; - return { - identifiers: [...new Set(identifiers)], - count: identifiers.length - }; - - case 'importExtraction': - const importRegex = /import\s+(?:(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)\s+from\s+)?['"`]([^'"`]+)['"`]/g; - const imports = []; - let match; - while ((match = importRegex.exec(text)) !== null) { - imports.push(match[1]); - } - return { imports: [...new Set(imports)] }; - - default: - throw new Error(`Unknown analysis type: ${analysisType}`); - } - }, - - // File processing tasks - fileAnalysis: async (input) => { - const { filePath, content, analysisType } = input; - - switch (analysisType) { - case 'basic': - return { - filePath, - size: content.length, - lines: content.split('\n').length, - hasContent: content.trim().length > 0 - }; - - case 'language': - const ext = filePath.split('.').pop()?.toLowerCase(); - const languageMap = { - 'js': 'javascript', - 'jsx': 'javascript', - 'ts': 'typescript', - 'tsx': 'typescript', - 'py': 'python', - 'java': 'java', - 'cpp': 'cpp', - 'c': 'c', - 'cs': 'csharp', - 'php': 'php', - 'rb': 'ruby', - 'go': 'go', - 'rs': 'rust' - }; - return { - filePath, - language: languageMap[ext] || 'unknown', - extension: ext - }; - - default: - throw new Error(`Unknown file analysis type: ${analysisType}`); - } - }, - - // Data processing tasks - dataProcessing: async (input) => { - const { data, operation } = input; - - switch (operation) { - case 'deduplicate': - return [...new Set(data)]; - - case 'filter': - const { predicate } = input; - // Note: This is a simplified version - in practice, you'd need to serialize the predicate - return data.filter(item => { - try { - // Simple filtering - in real implementation, you'd need proper serialization - return item && item.length > 0; - } catch { - return false; - } - }); - - case 'transform': - const { transformType } = input; - switch (transformType) { - case 'uppercase': - return data.map(item => item.toUpperCase()); - case 'lowercase': - return data.map(item => item.toLowerCase()); - case 'trim': - return data.map(item => item.trim()); - default: - throw new Error(`Unknown transform type: ${transformType}`); - } - - default: - throw new Error(`Unknown data operation: ${operation}`); - } - }, - - // Pattern matching tasks - patternMatching: async (input) => { - const { text, patterns } = input; - const results = []; - - for (const pattern of patterns) { - try { - const regex = new RegExp(pattern, 'g'); - const matches = []; - let match; - - while ((match = regex.exec(text)) !== null) { - matches.push({ - match: match[0], - index: match.index, - groups: match.slice(1) - }); - } - - results.push({ - pattern, - matches, - count: matches.length - }); - } catch (error) { - results.push({ - pattern, - error: error.message, - count: 0 - }); - } - } - - return results; - }, - - // Statistical analysis tasks - statisticalAnalysis: async (input) => { - const { data, analysisType } = input; - - switch (analysisType) { - case 'basic': - const numbers = data.filter(item => typeof item === 'number' && !isNaN(item)); - if (numbers.length === 0) { - return { error: 'No valid numbers found' }; - } - - const sum = numbers.reduce((acc, val) => acc + val, 0); - const mean = sum / numbers.length; - const sorted = numbers.sort((a, b) => a - b); - const median = sorted.length % 2 === 0 - ? (sorted[sorted.length / 2 - 1] + sorted[sorted.length / 2]) / 2 - : sorted[Math.floor(sorted.length / 2)]; - - return { - count: numbers.length, - sum, - mean, - median, - min: sorted[0], - max: sorted[sorted.length - 1] - }; - - case 'frequency': - const frequency = {}; - for (const item of data) { - const key = String(item); - frequency[key] = (frequency[key] || 0) + 1; - } - return frequency; - - default: - throw new Error(`Unknown statistical analysis type: ${analysisType}`); - } - } -}; - -// Main message handler -self.onmessage = async function(event) { - const { taskId, input } = event.data; - - try { - const { taskType, ...taskInput } = input; - - // Get the appropriate task handler - const handler = taskHandlers[taskType]; - if (!handler) { - throw new Error(`Unknown task type: ${taskType}`); - } - - // Execute the task - const result = await handler(taskInput); - - // Send result back to main thread - self.postMessage({ - taskId, - result - }); - - } catch (error) { - // Send error back to main thread - self.postMessage({ - taskId, - error: error.message || 'Unknown error in generic worker' - }); - } -}; - -// Handle worker errors -self.onerror = function(error) { - console.error('Worker: Unhandled error:', error); - self.postMessage({ - taskId: 'error', - error: error.message || 'Unhandled worker error' - }); -}; - -// Handle unhandled promise rejections -self.onunhandledrejection = function(event) { - console.error('Worker: Unhandled promise rejection:', event.reason); - self.postMessage({ - taskId: 'error', - error: event.reason?.message || 'Unhandled promise rejection' - }); -}; - -console.log('Worker: Generic worker script loaded'); - -================ -File: public/workers/tree-sitter-worker.js -================ -/** - * Tree-sitter Web Worker - * Handles parallel parsing of source code files - */ - -// Import tree-sitter and language parsers -import Parser from 'web-tree-sitter'; - -// Initialize tree-sitter -let parser = null; -let languageParsers = new Map(); - -// Initialize the worker -async function initializeWorker() { - try { - // Initialize tree-sitter - await Parser.init(); - parser = new Parser(); - - // Load language parsers - const languageLoaders = { - typescript: async () => { - const language = await Parser.Language.load('/wasm/typescript/tree-sitter-typescript.wasm'); - return language; - }, - javascript: async () => { - const language = await Parser.Language.load('/wasm/javascript/tree-sitter-javascript.wasm'); - return language; - }, - python: async () => { - const language = await Parser.Language.load('/wasm/python/tree-sitter-python.wasm'); - return language; - } - }; - - for (const [lang, loader] of Object.entries(languageLoaders)) { - try { - const languageParser = await loader(); - languageParsers.set(lang, languageParser); - console.log(`Worker: ${lang} parser loaded successfully`); - } catch (error) { - console.error(`Worker: Failed to load ${lang} parser:`, error); - } - } - - console.log('Worker: Tree-sitter worker initialized successfully'); - return true; - } catch (error) { - console.error('Worker: Failed to initialize tree-sitter worker:', error); - return false; - } -} - -// Detect language from file path -function detectLanguage(filePath) { - const ext = filePath.split('.').pop()?.toLowerCase(); - - switch (ext) { - case 'ts': - case 'tsx': - return 'typescript'; - case 'js': - case 'jsx': - return 'javascript'; - case 'py': - return 'python'; - default: - return 'javascript'; // Default fallback - } -} - -// Extract definitions from AST -function extractDefinitions(tree, filePath) { - const definitions = []; - const language = detectLanguage(filePath); - - // Get queries for the language - const queries = getQueriesForLanguage(language); - if (!queries) return definitions; - - // Execute queries to find definitions - for (const [queryName, queryString] of Object.entries(queries)) { - try { - const query = parser.getLanguage().query(queryString); - const matches = query.matches(tree.rootNode); - - for (const match of matches) { - const definition = processMatch(match, filePath, queryName); - if (definition) { - definitions.push(definition); - } - } - } catch (error) { - console.warn(`Worker: Error executing query ${queryName}:`, error); - } - } - - return definitions; -} - -// Get queries for specific language -function getQueriesForLanguage(language) { - const queries = { - typescript: { - function_declaration: ` - (function_declaration - name: (identifier) @function.name - parameters: (formal_parameters) @function.parameters - body: (statement_block) @function.body - ) - `, - class_declaration: ` - (class_declaration - name: (identifier) @class.name - body: (class_body) @class.body - ) - `, - method_definition: ` - (method_definition - name: (property_identifier) @method.name - parameters: (formal_parameters) @method.parameters - body: (statement_block) @method.body - ) - `, - import_statement: ` - (import_statement - source: (string) @import.source - ) - ` - }, - javascript: { - function_declaration: ` - (function_declaration - name: (identifier) @function.name - parameters: (formal_parameters) @function.parameters - body: (statement_block) @function.body - ) - `, - arrow_function: ` - (arrow_function - parameters: (formal_parameters) @function.parameters - body: (statement_block) @function.body - ) - `, - class_declaration: ` - (class_declaration - name: (identifier) @class.name - body: (class_body) @class.body - ) - ` - }, - python: { - function_definition: ` - (function_definition - name: (identifier) @function.name - parameters: (parameters) @function.parameters - body: (block) @function.body - ) - `, - class_definition: ` - (class_definition - name: (identifier) @class.name - body: (block) @class.body - ) - `, - import_statement: ` - (import_statement - name: (dotted_name) @import.name - ) - ` - } - }; - - return queries[language] || null; -} - -// Process a query match into a definition -function processMatch(match, filePath, queryType) { - try { - const captures = match.captures; - const definition = { - type: queryType, - filePath, - startLine: match.node.startPosition.row, - endLine: match.node.endPosition.row, - startColumn: match.node.startPosition.column, - endColumn: match.node.endPosition.column - }; - - // Extract specific information based on query type - for (const capture of captures) { - const { name, node } = capture; - - if (name.includes('name')) { - definition.name = node.text; - } else if (name.includes('parameters')) { - definition.parameters = node.text; - } else if (name.includes('source')) { - definition.importSource = node.text.replace(/['"]/g, ''); - } - } - - return definition; - } catch (error) { - console.warn('Worker: Error processing match:', error); - return null; - } -} - -// Parse a single file -async function parseFile(filePath, content) { - try { - const language = detectLanguage(filePath); - const languageParser = languageParsers.get(language); - - if (!languageParser) { - throw new Error(`No parser available for language: ${language}`); - } - - // Set the language - parser.setLanguage(languageParser); - - // Parse the content - const tree = parser.parse(content); - - // Extract definitions - const definitions = extractDefinitions(tree, filePath); - - return { - filePath, - definitions, - ast: { - tree: { - rootNode: { - startPosition: tree.rootNode.startPosition, - endPosition: tree.rootNode.endPosition, - type: tree.rootNode.type, - text: tree.rootNode.text - } - } - } - }; - } catch (error) { - console.error(`Worker: Error parsing file ${filePath}:`, error); - throw error; - } -} - -// Handle messages from main thread -self.onmessage = async function(event) { - const { taskId, input } = event.data; - - try { - // Initialize worker if not already done - if (!parser) { - const initialized = await initializeWorker(); - if (!initialized) { - throw new Error('Failed to initialize tree-sitter worker'); - } - } - - const { filePath, content } = input; - - // Parse the file - const result = await parseFile(filePath, content); - - // Send result back to main thread - self.postMessage({ - taskId, - result - }); - - } catch (error) { - // Send error back to main thread - self.postMessage({ - taskId, - error: error.message || 'Unknown error in tree-sitter worker' - }); - } -}; - -// Handle worker errors -self.onerror = function(error) { - console.error('Worker: Unhandled error:', error); - self.postMessage({ - taskId: 'error', - error: error.message || 'Unhandled worker error' - }); -}; - -console.log('Worker: Tree-sitter worker script loaded'); - -================ -File: quicksort.js -================ -/** - * Quicksort implementation in JavaScript - * Time Complexity: O(n log n) average, O(nยฒ) worst case - * Space Complexity: O(log n) average, O(n) worst case - */ - -function quicksort(arr) { - // Base case: arrays with 0 or 1 element are already sorted - if (arr.length <= 1) { - return arr; - } - - // Choose pivot (middle element to avoid worst case on sorted arrays) - const pivotIndex = Math.floor(arr.length / 2); - const pivot = arr[pivotIndex]; - - // Partition array into elements less than, equal to, and greater than pivot - const left = []; - const equal = []; - const right = []; - - for (let i = 0; i < arr.length; i++) { - if (arr[i] < pivot) { - left.push(arr[i]); - } else if (arr[i] === pivot) { - equal.push(arr[i]); - } else { - right.push(arr[i]); - } - } - - // Recursively sort left and right partitions, then combine - return [...quicksort(left), ...equal, ...quicksort(right)]; -} - -// In-place quicksort implementation (more memory efficient) -function quicksortInPlace(arr, left = 0, right = arr.length - 1) { - if (left < right) { - const pivotIndex = partition(arr, left, right); - quicksortInPlace(arr, left, pivotIndex - 1); - quicksortInPlace(arr, pivotIndex + 1, right); - } - return arr; -} - -function partition(arr, left, right) { - const pivot = arr[right]; - let i = left - 1; - - for (let j = left; j < right; j++) { - if (arr[j] <= pivot) { - i++; - [arr[i], arr[j]] = [arr[j], arr[i]]; - } - } - - [arr[i + 1], arr[right]] = [arr[right], arr[i + 1]]; - return i + 1; -} - -// Example usage and testing -const testArray = [64, 34, 25, 12, 22, 11, 90]; -console.log('Original array:', testArray); -console.log('Sorted array (functional):', quicksort([...testArray])); -console.log('Sorted array (in-place):', quicksortInPlace([...testArray])); - -// Export for use in other modules -module.exports = { quicksort, quicksortInPlace }; - -================ -File: README.md -================ -# GitNexus: Edge-Based Code Knowledge Graph Generator for Deno - Step-by-Step Implementation Guide - -This guide will walk you through building a fully edge-based code knowledge graph generator from scratch using Deno. I'll explain each concept before showing the implementation, so you understand **why** we're doing something, not just **how** to do it. - -## Phase 1: Project Setup & Core Infrastructure - -### Step 1: Project Structure and Tooling Setup - -**Why this matters:** Before writing any code, we need to set up our development environment properly. A well-structured project makes it easier to add features later and keeps everything organized. - -**Key concepts:** - -- We're using Vite (a modern build tool) with React and TypeScript -- We need special configuration for WebAssembly (WASM) files -- A clear directory structure helps us scale to multiple languages later - -**Implementation Steps:** - -1. **Create the base project:** - -```bash -# Create project root -mkdir GitNexus -cd GitNexus - -# Initialize Vite project with React and TypeScript -npm create vite@latest . -- --template react-ts - -# Initialize Deno project -deno init -``` - -2. **Create the application directory structure:** - -```bash -# Create directories for our core components -mkdir -p src/{core,core/tree-sitter,core/graph,core/ingestion,services,ai,ai/agents,ai/prompts,ui,ui/components,ui/components/graph,ui/components/chat,ui/hooks,workers,lib,config,store} -``` - -**Why this structure?** - -- `core/`: Contains the engine that builds the knowledge graph -- `services/`: Handles external interactions (GitHub API, ZIP processing) -- `ai/`: Contains the RAG and chat functionality -- `ui/`: All user interface components -- `workers/`: Web Workers for heavy processing (keeps UI responsive) -- `lib/`: Utility functions used throughout the app - -### Step 2: Configure Build Tools for WASM - -**Why this matters:** WebAssembly (WASM) is how we'll run the Tree-sitter parsers in the browser. We need special configuration to handle these binary files correctly. - -**Key concepts:** - -- WASM files are binary files that run at near-native speed in browsers -- Vite needs special configuration to handle them properly -- We want to avoid inlining large WASM files in our JavaScript bundles - -**Implementation:** - -1. **Update `vite.config.ts`:** - -```typescript -import { defineConfig } from 'vite' -import react from '@vitejs/plugin-react' -export default defineConfig({ - plugins: [react()], - worker: { - format: 'es' - }, - assetsInclude: ['**/*.wasm'], - build: { - target: 'esnext', - assetsInlineLimit: 0 // Don't inline WASM files - } -}) -``` - -**What this does:** - -- `assetsInclude: ['**/*.wasm']` tells Vite to treat WASM files as assets -- `assetsInlineLimit: 0` ensures WASM files aren't inlined into JavaScript (they're too large) -- `worker: { format: 'es' }` configures Web Workers to use ES modules - -2. **Configure TypeScript** with a `tsconfig.json` that has strict settings for better code quality. - -**Why strict settings?** They help catch errors early and make the code more maintainable as the project grows. - -### Step 3: Set Up WASM Parser Infrastructure - -**Why this matters:** Tree-sitter is the engine that parses code into ASTs (Abstract Syntax Trees). We need to get these parsers working in the browser via WASM. - -**Key concepts:** - -- Tree-sitter parsers for different languages are written in C -- We compile them to WASM so they can run in browsers -- We need to load these parsers on demand - -**Implementation:** - -1. **Create a public directory for WASM files:** - -```bash -mkdir -p public/wasm/python -``` - -2. **Download the Tree-sitter Python parser:** - - Get `tree-sitter-python.wasm` from [tree-sitter-python releases](https://github.com/tree-sitter/tree-sitter-python/releases) - - Place it in `public/wasm/python/` - -**Why host WASM files separately?** Browsers can't access the user's file system directly for security reasons. We need to serve the WASM files from a URL. - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase - Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase - Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase - Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -3. **Create a loader for Tree-sitter parsers:** - -```typescript -import WebTreeSitter from 'web-tree-sitter'; -let parserInstance: WebTreeSitter | null = null; -const parserCache = new Map(); - -export async function initTreeSitter() { - if (parserInstance) return parserInstance; - parserInstance = await WebTreeSitter.init(); - return parserInstance; -} - -export async function loadPythonParser(): Promise { - if (parserCache.has('python')) { - return parserCache.get('python')!; - } - const Parser = await initTreeSitter(); - const pythonLang = await Parser.Language.load( - '/wasm/python/tree-sitter-python.wasm' - ); - parserCache.set('python', pythonLang); - return pythonLang; -} -``` - -**How this works:** - -1. `initTreeSitter()` initializes the WebAssembly module once -2. `loadPythonParser()` loads the Python parser from the WASM file -3. We cache parsers to avoid reloading them multiple times - -**Why cache parsers?** Loading WASM files is relatively slow, so we want to do it once and reuse the parsers. - -## Phase 2: Code Acquisition Module - -### Step 4: Implement GitHub API Integration - -**Why this matters:** Users will want to analyze public GitHub repositories, so we need a way to fetch code from GitHub. - -**Key concepts:** - -- GitHub has a REST API for accessing repository contents -- We need to handle rate limits (GitHub limits how many requests you can make) -- We'll let users provide their own API tokens for higher limits - -**Implementation:** - -```typescript -export class GitHubService { - private token: string | null = null; - - setToken(token: string) { - this.token = token; - } - - async getRepoContents(owner: string, repo: string, path = '') { - const headers: HeadersInit = { - 'Accept': 'application/vnd.github.v3+json' - }; - if (this.token) { - headers['Authorization'] = `token ${this.token}`; - } - - const response = await fetch( - `https://api.github.com/repos/${owner}/${repo}/contents/${path}`, - { headers } - ); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status}`); - } - - return response.json(); - } -} -``` - -**How this works:** - -- `getRepoContents()` fetches the directory structure of a repository -- It uses the GitHub API with proper headers -- It handles authentication via a token - -**Important note:** GitHub API has rate limits. For unauthenticated requests, it's about 60 requests/hour. With a token, it's 5,000/hour. - -### Step 5: Implement ZIP Processing - -**Why this matters:** Not all code is on GitHub. Users might want to analyze local code or private repositories by uploading a ZIP file. - -**Key concepts:** - -- JSZip is a library for handling ZIP files in JavaScript -- We need to extract files and their contents from the ZIP -- We'll use a Map to store file paths and contents - -**Implementation:** - -```typescript -import JSZip from 'jszip'; - -export class ZipService { - async processZip(file: File): Promise> { - const zip = await JSZip.loadAsync(file); - const files = new Map(); - - for (const [filePath, zipEntry] of Object.entries(zip.files)) { - if (!zipEntry.dir) { - const content = await zipEntry.async('text'); - files.set(filePath, content); - } - } - - return files; - } -} -``` - -**How this works:** - -1. `JSZip.loadAsync(file)` loads the ZIP file -2. We iterate through all entries in the ZIP -3. For each file (not directory), we extract its content as text -4. We store the file path and content in a Map - -**Why use a Map?** It provides O(1) lookups by file path, which is important when we need to find files during graph construction. - -## Phase 3: Graph Construction Pipeline - -### Step 6: Define Graph Data Structures - -**Why this matters:** Before we can build a graph, we need to define what nodes and relationships look like. - -**Key concepts:** - -- A knowledge graph consists of nodes and relationships -- Nodes represent code elements (functions, classes, etc.) -- Relationships represent connections between elements (calls, contains, etc.) - -**Implementation:** - -```typescript -export type NodeLabel = - | 'Project' - | 'Package' - | 'Module' - | 'Folder' - | 'File' - | 'Class' - | 'Function' - | 'Method' - | 'Variable'; - -export interface GraphNode { - id: string; - label: NodeLabel; - properties: Record; -} - -export type RelationshipType = - | 'CONTAINS' - | 'CALLS' - | 'INHERITS' - | 'OVERRIDES' - | 'IMPORTS'; - -export interface GraphRelationship { - id: string; - type: RelationshipType; - source: string; - target: string; - properties?: Record; -} - -export interface KnowledgeGraph { - nodes: GraphNode[]; - relationships: GraphRelationship[]; -} -``` - -**Why these specific types?** - -- `NodeLabel` defines all possible types of code elements we'll track -- `RelationshipType` defines how code elements connect to each other -- `KnowledgeGraph` is the complete structure we'll build - -**Important relationships:** - -- `CONTAINS`: A folder contains files, a file contains functions -- `CALLS`: A function calls another function -- `IMPORTS`: One module imports from another - -### Step 7: Implement the 3-Pass Ingestion Pipeline - -**Why this matters:** Building a complete knowledge graph requires multiple passes to handle cross-file references properly. - -**Key concepts:** - -- **Pass 1**: Identify the overall structure (folders, modules) -- **Pass 2**: Parse individual files and cache ASTs -- **Pass 3**: Process function calls across files (the hardest part) - -This three-pass approach solves the "island problem" - where functions in different files appear disconnected. - -#### Pass 1: Structure Identification - -```typescript -export class StructureProcessor { - private graph: KnowledgeGraph; - private projectRoot: string; - private projectName: string; - - constructor(graph: KnowledgeGraph, projectRoot: string, projectName: string) { - this.graph = graph; - this.projectRoot = projectRoot; - this.projectName = projectName; - } - - identifyStructure(filePaths: string[]): void { - // Add Project node - this.graph.nodes.push({ - id: `project:${this.projectName}`, - label: 'Project', - properties: { name: this.projectName } - }); - - // Track directory structure - const directories = new Set(); - for (const filePath of filePaths) { - const dirPath = filePath.substring(0, filePath.lastIndexOf('/')); - if (dirPath && !directories.has(dirPath)) { - directories.add(dirPath); - // Create Folder node - this.graph.nodes.push({ - id: `folder:${dirPath}`, - label: 'Folder', - properties: { path: dirPath } - }); - - // Create CONTAINS relationship with parent - if (dirPath.includes('/')) { - const parentPath = dirPath.substring(0, dirPath.lastIndexOf('/')); - this.graph.relationships.push({ - id: `rel:folder:${dirPath}:parent`, - type: 'CONTAINS', - source: `folder:${parentPath}`, - target: `folder:${dirPath}` - }); - } else { - // Root folder connects to project - this.graph.relationships.push({ - id: `rel:folder:${dirPath}:project`, - type: 'CONTAINS', - source: `project:${this.projectName}`, - target: `folder:${dirPath}` - }); - } - } - } - } -} -``` - -**How this works:** - -1. Creates a root Project node -2. Walks through all file paths to identify directories -3. Creates Folder nodes and CONTAINS relationships - -**Why identify structure first?** We need to know the overall organization before parsing individual files. - -#### Pass 2: File Parsing - -```typescript -export class ParsingProcessor { - private graph: KnowledgeGraph; - private astCache = new Map(); - - constructor(graph: KnowledgeGraph) { - this.graph = graph; - } - - async parseFiles(filePaths: string[], fileContents: Map): Promise> { - for (const [filePath, content] of fileContents) { - if (filePath.endsWith('.py')) { - await this.parsePythonFile(filePath, content); - } - } - return this.astCache; - } - - private async parsePythonFile(filePath: string, content: string): Promise { - const parser = await loadPythonParser(); - const tree = parser.parse(content); - // Cache the AST - this.astCache.set(filePath, tree); - // Extract definitions from the AST - this.extractDefinitions(filePath, tree, content); - } - - private extractDefinitions(filePath: string, tree: any, content: string): void { - // Extract modules - this.graph.nodes.push({ - id: `module:${filePath}`, - label: 'Module', - properties: { - path: filePath, - name: filePath.split('/').pop()!.replace('.py', ''), - extension: '.py' - } - }); - - // Extract functions from the AST - const rootNode = tree.rootNode; - const functionDefs = rootNode.descendantsOfType('function_definition'); - for (const funcNode of functionDefs) { - const nameNode = funcNode.childForFieldName('name'); - const name = nameNode ? nameNode.text : 'unknown'; - - // Calculate position - const startLine = funcNode.startPosition.row + 1; - - // Create function node - this.graph.nodes.push({ - id: `function:${filePath}:${name}`, - label: 'Function', - properties: { - name, - qualified_name: `${this.getModuleName(filePath)}.${name}`, - path: filePath, - start_line: startLine - } - }); - - // Create CONTAINS relationship with module - this.graph.relationships.push({ - id: `rel:function:${filePath}:${name}:module`, - type: 'CONTAINS', - source: `module:${filePath}`, - target: `function:${filePath}:${name}` - }); - } - } -} -``` - -**How this works:** - -1. Parses each file with the appropriate Tree-sitter parser -2. Caches the AST for later use -3. Extracts definitions (functions, classes) from the AST -4. Creates nodes and relationships in the graph - -**Why cache ASTs?** We need them in Pass 3 to resolve cross-file function calls. - -#### Pass 3: Call Resolution - -```typescript -export class CallProcessor { - private graph: KnowledgeGraph; - private astCache: Map; - private projectRoot: string; - private projectName: string; - - constructor( - graph: KnowledgeGraph, - astCache: Map, - projectRoot: string, - projectName: string - ) { - this.graph = graph; - this.astCache = astCache; - this.projectRoot = projectRoot; - this.projectName = projectName; - } - - processCalls(): void { - for (const [filePath, tree] of this.astCache) { - if (filePath.endsWith('.py')) { - this.processPythonCalls(filePath, tree); - } - } - } - - private processPythonCalls(filePath: string, tree: any): void { - const rootNode = tree.rootNode; - // Find all call expressions - const callExpressions = rootNode.descendantsOfType('call'); - for (const callNode of callExpressions) { - const functionNameNode = callNode.childForFieldName('function'); - if (!functionNameNode) continue; - - // Handle different types of function references - let targetFunctionName = ''; - if (functionNameNode.type === 'identifier') { - targetFunctionName = functionNameNode.text; - } else if (functionNameNode.type === 'attribute') { - // Handle method calls like obj.method() - const attrNode = functionNameNode; - const objectNode = attrNode.childForFieldName('object'); - const attrNameNode = attrNode.childForFieldName('attribute'); - if (objectNode && attrNameNode) { - const objectName = objectNode.text; - const methodName = attrNameNode.text; - targetFunctionName = `${objectName}.${methodName}`; - } - } - - if (!targetFunctionName) continue; - - // Try to resolve the target function - const targetNode = this.resolveTargetFunction(targetFunctionName, filePath); - if (targetNode) { - // Create CALLS relationship - const callerId = this.getCallerId(callNode, filePath); - this.graph.relationships.push({ - id: `rel:call:${callerId}:${targetNode.id}`, - type: 'CALLS', - source: callerId, - target: targetNode.id - }); - } - } - } - - private resolveTargetFunction(targetName: string, currentFilePath: string): { id: string; type: string } | null { - // 1. Check if it's a built-in function - if (this.isBuiltInFunction(targetName)) { - return { - id: `builtin:${targetName}`, - type: 'builtin' - }; - } - - // 2. Check if it's an imported function - const importInfo = this.findImportForFunction(targetName, currentFilePath); - if (importInfo) { - const targetId = `function:${importInfo.sourceFile}:${importInfo.targetName}`; - return { - id: targetId, - type: 'imported' - }; - } - - // 3. Check if it's defined in the current file - for (const node of this.graph.nodes) { - if (node.label === 'Function' && - node.properties.name === targetName && - node.properties.path === currentFilePath) { - return { - id: node.id, - type: 'local' - }; - } - } - - return null; - } -} -``` - -**How this works:** - -1. Finds all function calls in the AST -2. Determines what function is being called -3. Resolves the target function across files using imports -4. Creates CALLS relationships in the graph - -**Why is this the hardest part?** Resolving cross-file references requires understanding: - -- How imports work in the language -- How to map a simple name to a fully qualified name -- Handling edge cases like aliases (`import helper as h`) - -### Step 8: Implement Web Workers for Performance - -**Why this matters:** Parsing code and building graphs can be CPU-intensive. Web Workers keep the UI responsive. - -**Key concepts:** - -- Web Workers run JavaScript in background threads -- They can't access the DOM directly -- We use Comlink to simplify communication - -**Implementation:** - -```typescript -// src/workers/ingestion.worker.ts -import { expose } from 'comlink'; -import { GraphPipeline } from '../core/ingestion/pipeline'; - -class IngestionWorker { - async processRepository( - projectRoot: string, - projectName: string, - filePaths: string[], - fileContents: Record - ) { - const pipeline = new GraphPipeline(projectRoot, projectName); - return pipeline.run(filePaths, new Map(Object.entries(fileContents))); - } -} - -expose(new IngestionWorker()); -``` - -**How this works:** - -1. The worker runs the heavy processing in a background thread -2. We expose methods via Comlink to call them from the main thread -3. The main thread can call these methods without blocking the UI - -**Why use Web Workers?** Without them, large repositories would freeze the browser tab while processing. - -## Phase 4: Graph Visualization - -### Step 9: Implement Graph Visualization Components - -**Why this matters:** A knowledge graph is useless if users can't see and interact with it. - -**Key concepts:** - -- Cytoscape.js is a powerful graph visualization library -- We need to convert our graph data to Cytoscape's format -- Users need controls to filter and navigate the graph - -**Implementation:** - -```tsx -import React, { useEffect, useRef } from 'react'; -import cytoscape from 'cytoscape'; -import dagre from 'cytoscape-dagre'; -import { KnowledgeGraph } from '@/core/graph/types'; - -cytoscape.use(dagre); - -interface GraphVisualizationProps { - graph: KnowledgeGraph; - onNodeClick?: (nodeId: string) => void; - filter?: (node: any) => boolean; -} - -export const GraphVisualization: React.FC = ({ - graph, - onNodeClick, - filter -}) => { - const containerRef = useRef(null); - const cyRef = useRef(null); - - useEffect(() => { - if (!containerRef.current) return; - - // Clean up previous instance - if (cyRef.current) { - cyRef.current.destroy(); - } - - // Convert our graph to Cytoscape format - const cyElements = convertToCytoscapeElements(graph, filter); - - const cy = cytoscape({ - container: containerRef.current, - elements: cyElements, - style: [ - { - selector: 'node', - style: { - 'label': 'data(label)', - 'width': 'mapData(size, 0, 100, 20, 80)', - 'height': 'mapData(size, 0, 100, 20, 80)', - 'background-color': 'data(color)', - 'text-valign': 'center', - 'text-halign': 'center', - 'font-size': '8px' - } - }, - { - selector: 'edge', - style: { - 'width': 2, - 'line-color': '#ccc', - 'target-arrow-color': '#ccc', - 'target-arrow-shape': 'triangle' - } - } - ], - layout: { - name: 'dagre', - rankDir: 'TB', - padding: 20 - } - }); - - // Add interactions - cy.on('tap', 'node', (event) => { - const node = event.target; - const nodeId = node.data('id'); - if (onNodeClick) { - onNodeClick(nodeId); - } - }); - - cyRef.current = cy; - - return () => { - if (cyRef.current) { - cyRef.current.destroy(); - cyRef.current = null; - } - }; - }, [graph, filter]); - - return ( -
- ); -}; - -function convertToCytoscapeElements( - graph: KnowledgeGraph, - filter?: (node: any) => boolean -) { - const elements: any[] = []; - - // Add nodes - for (const node of graph.nodes) { - if (filter && !filter(node)) continue; - elements.push({ - data: { - id: node.id, - label: getNodeLabel(node), - type: node.label, - color: getNodeColor(node.label), - size: getNodeSize(node) - } - }); - } - - // Add edges - for (const rel of graph.relationships) { - elements.push({ - data: { - id: rel.id, - source: rel.source, - target: rel.target, - label: rel.type - } - }); - } - - return elements; -} -``` - -**How this works:** - -1. Converts our graph data to Cytoscape's format -2. Sets up visual styles based on node type -3. Applies a hierarchical layout (dagre) -4. Adds interaction handlers for node clicks - -**Why use Cytoscape.js?** It's specifically designed for graph visualization with: - -- Multiple layout algorithms -- Good performance for medium-sized graphs -- Extensive customization options - -### Step 10: Create Source Code Viewer - -**Why this matters:** Seeing the graph isn't enough - users need to see the actual code behind the nodes. - -**Implementation:** - -```tsx -import React, { useState, useEffect } from 'react'; -import { KnowledgeGraph } from '@/core/graph/types'; - -interface SourceViewerProps { - graph: KnowledgeGraph; - selectedNodeId: string | null; -} - -export const SourceViewer: React.FC = ({ graph, selectedNodeId }) => { - const [sourceCode, setSourceCode] = useState(''); - const [fileName, setFileName] = useState(''); - const [lineNumber, setLineNumber] = useState(null); - - useEffect(() => { - if (!selectedNodeId) { - setSourceCode(''); - setFileName(''); - setLineNumber(null); - return; - } - - // Find the node in the graph - const node = graph.nodes.find(n => n.id === selectedNodeId); - if (!node) return; - - // For functions, get the source code - if (node.label === 'Function' || node.label === 'Method') { - const filePath = node.properties.path; - const startLine = node.properties.start_line; - - // In a real implementation, you'd have the source code available - setFileName(filePath); - setLineNumber(startLine); - setSourceCode(`# Source code for ${node.properties.qualified_name} -# Line ${startLine} and following...`); - } - }, [graph, selectedNodeId]); - - if (!selectedNodeId || !sourceCode) { - return ( -
-

Select a node to view source code

-
- ); - } - - return ( -
-
- {fileName} - {lineNumber && ( - Line {lineNumber} - )} -
-
- 
{sourceCode}
-
-
- ); -}; -``` - -**How this works:** - -1. When a node is selected, it finds the corresponding code element -2. It displays the source code with line numbers -3. It highlights the relevant part of the code - -**Why is this important?** It bridges the gap between the abstract graph and the concrete code, helping users understand what they're seeing. - -## Phase 5: RAG Chat Interface - -### Step 11: Implement LLM Service - -**Why this matters:** The chat interface needs to connect to LLMs (Large Language Models) to translate natural language to graph queries. - -**Key concepts:** - -- We'll support multiple LLM providers (OpenAI, Anthropic, Gemini) -- Users provide their own API keys (privacy-focused) -- We need a consistent interface for different providers - -**Implementation:** - -```typescript -import { ChatOpenAI } from 'langchain/chat_models/openai'; -import { ChatAnthropic } from 'langchain/chat_models/anthropic'; -import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; - -export type LLMProvider = 'openai' | 'anthropic' | 'gemini'; - -export interface LLMConfig { - provider: LLMProvider; - apiKey: string; - model?: string; -} - -export class LLMService { - private config: LLMConfig; - - constructor(config: LLMConfig) { - this.config = config; - } - - getChatModel() { - switch (this.config.provider) { - case 'openai': - return new ChatOpenAI({ - apiKey: this.config.apiKey, - modelName: this.config.model || 'gpt-4-turbo', - temperature: 0 - }); - case 'anthropic': - return new ChatAnthropic({ - apiKey: this.config.apiKey, - modelName: this.config.model || 'claude-3-sonnet-20240229', - temperature: 0 - }); - case 'gemini': - return new ChatGoogleGenerativeAI({ - apiKey: this.config.apiKey, - modelName: this.config.model || 'gemini-1.5-pro-latest', - temperature: 0 - }); - default: - throw new Error(`Unsupported LLM provider: ${this.config.provider}`); - } - } -} -``` - -**How this works:** - -1. The service takes an LLM configuration (provider, API key, model) -2. It returns a consistent chat model interface regardless of provider -3. It handles provider-specific initialization - -**Why support multiple providers?** Different users have different preferences and API key availability. - -### Step 12: Implement Cypher Generator - -**Why this matters:** The core of the RAG system - translating natural language questions to graph queries. - -**Key concepts:** - -- We use a system prompt to instruct the LLM -- The prompt includes our graph schema -- We clean the response to get a valid Cypher query - -**Implementation:** - -```typescript -import { BaseChatModel } from 'langchain/chat_models/base'; -import { CYPHER_SYSTEM_PROMPT } from '../prompts/cypher'; - -export class CypherGenerator { - private llm: BaseChatModel; - - constructor(llm: BaseChatModel) { - this.llm = llm; - } - - async generate(naturalLanguageQuery: string): Promise { - const response = await this.llm.call([ - { role: 'system', content: CYPHER_SYSTEM_PROMPT }, - { role: 'user', content: naturalLanguageQuery } - ]); - - return this.cleanResponse(response.content); - } - - private cleanResponse(response: string): string { - // Remove markdown code blocks - let cleaned = response.replace(/```cypher/g, '').replace(/```/g, ''); - // Ensure it ends with a semicolon - if (!cleaned.trim().endsWith(';')) { - cleaned = cleaned.trim() + ';'; - } - return cleaned; - } -} -``` - -**How this works:** - -1. It sends the natural language query with a system prompt to the LLM -2. The system prompt teaches the LLM about our graph structure -3. It cleans the response to extract a valid Cypher query - -**Why is the system prompt important?** It provides the LLM with the context it needs to generate correct queries. Without it, the LLM wouldn't know about our graph schema. - -### Step 13: Implement RAG Orchestrator - -**Why this matters:** This is the "brain" of the system that coordinates the query process. - -**Key concepts:** - -- It follows a ReAct (Reason + Act) pattern -- It plans steps, uses tools, observes results, and responds -- It prevents hallucination by sticking to tool results - -**Implementation:** - -```typescript -import { BaseChatModel } from 'langchain/chat_models/base'; -import { RAG_ORCHESTRATOR_SYSTEM_PROMPT } from '../prompts/rag-orchestrator'; - -export class RAGOrchestrator { - private llm: BaseChatModel; - - constructor(llm: BaseChatModel) { - this.llm = llm; - } - - async query( - userQuery: string, - queryGraph: (cypher: string) => Promise, - retrieveCode: (nodeId: string) => Promise - ) { - // Start with the system prompt - let conversation = [ - { role: 'system', content: RAG_ORCHESTRATOR_SYSTEM_PROMPT } - ]; - - // Add the user's question - conversation.push({ role: 'user', content: userQuery }); - - // Simple ReAct loop - for (let i = 0; i < 5; i++) { // Max 5 steps - const response = await this.llm.call(conversation); - const responseContent = response.content; - - // Check if the response contains a tool call - if (responseContent.includes('Action: query_graph')) { - const match = responseContent.match(/Action Input: (.*)/); - if (match) { - const cypherQuery = match[1].trim(); - - // Execute the query - const queryResults = await queryGraph(cypherQuery); - - // Add the observation to the conversation - conversation.push({ - role: 'assistant', - content: responseContent - }); - - conversation.push({ - role: 'system', - content: `Observation: ${JSON.stringify(queryResults)}` - }); - - // If we have results, we might be done - if (queryResults.length > 0) { - break; - } - } - } - else if (responseContent.includes('Action: retrieve_code')) { - // Similar handling for code retrieval - } - else { - // This appears to be the final answer - return responseContent; - } - } - - // If we got here without a final answer, generate one - conversation.push({ - role: 'user', - content: 'Please provide your final answer based on the information gathered.' - }); - - const finalResponse = await this.llm.call(conversation); - return finalResponse.content; - } -} -``` - -**How this works:** - -1. It starts with a system prompt that defines the rules -2. It sends the user's query to the LLM -3. The LLM responds with either: - - A tool call (query_graph or retrieve_code) - - A final answer -4. If it's a tool call, it executes the tool and adds the result to the conversation -5. It repeats until it gets a final answer or hits the step limit - -**Why the step limit?** To prevent infinite loops if the LLM gets stuck. - -## Phase 6: Main Application Integration - -### Step 14: Create Main Application Component - -**Why this matters:** This brings all the pieces together into a cohesive UI. - -**Implementation:** - -```tsx -import React, { useState, useRef } from 'react'; -import { GraphVisualization } from '@/ui/components/graph/Visualization'; -import { GraphControls } from '@/ui/components/graph/Controls'; -import { SourceViewer } from '@/ui/components/graph/SourceViewer'; -import { ChatInterface } from '@/ui/components/chat/ChatInterface'; -import { KnowledgeGraph } from '@/core/graph/types'; -import { GitHubService } from '@/services/github'; -import { ZipService } from '@/services/zip'; -import { ingestionWorkerApi } from '@/lib/workerUtils'; - -export const HomePage = () => { - const [graph, setGraph] = useState(null); - const [selectedNodeId, setSelectedNodeId] = useState(null); - const [isLoading, setIsLoading] = useState(false); - const [error, setError] = useState(null); - const [repoUrl, setRepoUrl] = useState(''); - const fileInputRef = useRef(null); - - const handleRepoSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - if (!repoUrl.trim() || isLoading) return; - - setIsLoading(true); - setError(null); - - try { - // Parse the GitHub URL - const urlMatch = repoUrl.match(/github\.com\/([^/]+)\/([^/]+)/); - if (!urlMatch) { - throw new Error('Invalid GitHub repository URL'); - } - - const owner = urlMatch[1]; - const repo = urlMatch[2].replace(/\.git$/, ''); - - // Fetch repository contents - const githubService = new GitHubService(); - const contents = await githubService.getRepoContents(owner, repo); - - // Filter for Python files - const pythonFiles = contents - .filter((item: any) => item.type === 'file' && item.name.endsWith('.py')) - .map((item: any) => item.path); - - // Fetch file contents - const fileContents: Record = {}; - for (const filePath of pythonFiles) { - fileContents[filePath] = await githubService.getFileContent(owner, repo, filePath); - } - - // Process the repository - const projectName = `${owner}/${repo}`; - const processedGraph = await ingestionWorkerApi.processRepository( - repoUrl, - projectName, - pythonFiles, - fileContents - ); - - setGraph(processedGraph); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to process repository'); - console.error('Processing error:', err); - } finally { - setIsLoading(false); - } - }; - - const handleQuery = async (query: string): Promise => { - if (!graph) { - throw new Error('No graph available'); - } - - // In a real implementation, this would use the RAG orchestrator - return `I found information related to "${query}" in the codebase.`; - }; - - return ( -
- {/* Header with repository input */} -
-
-

GitNexus

- -
-
-
- setRepoUrl(e.target.value)} - placeholder="https://github.com/owner/repo.git" - className="flex-1 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500" - /> - -
-
- -
- or - - -
-
-
-
- -
- {/* Graph Visualization Pane */} -
-
- {graph ? ( - <> - {}} - onLayoutChange={() => {}} - /> -
- -
- - ) : ( -
- {isLoading ? ( -
-
-

Processing repository...

-
- ) : ( -

Enter a repository URL or upload a ZIP to get started

- )} -
- )} -
-
- - {/* Right Panel */} -
- {/* Source Viewer */} -
-
-

Source Code

-
- {graph ? ( - - ) : ( -
-

Select a node to view source

-
- )} -
- - {/* Chat Interface */} -
-
-

Ask About Code

-
- {graph ? ( - - ) : ( -
-

Process a repository to ask questions about the code

-
- )} -
-
-
-
- ); -}; -``` - -**How this works:** - -1. The header has inputs for GitHub URLs and ZIP uploads -2. The main area has two panes: - - Left: Graph visualization - - Right: Source viewer and chat interface -3. When a repository is processed, the graph is displayed -4. Users can click nodes to see source code and ask questions - -**Why this layout?** It provides a cohesive experience where users can: - -- See the big picture (graph) -- Drill down to specific code (source viewer) -- Ask questions about what they're seeing (chat) - -## Final Steps: Testing and Optimization - -### Step 15: Add Error Boundaries - -**Why this matters:** Inevitably, something will go wrong. We want to handle errors gracefully. - -**Implementation:** - -```tsx -import React, { Component, ErrorInfo, ReactNode } from 'react'; - -interface Props { - children: ReactNode; -} - -interface State { - hasError: boolean; - error: Error | null; -} - -export class ErrorBoundary extends Component { - public state: State = { - hasError: false, - error: null - }; - - public static getDerivedStateFromError(error: Error): State { - return { hasError: true, error }; - } - - public componentDidCatch(error: Error, errorInfo: ErrorInfo) { - console.error("Uncaught error:", error, errorInfo); - } - - public render() { - if (this.state.hasError) { - return ( -
-

Something went wrong

-

{this.state.error?.message}

- -
- ); - } - - return this.props.children; - } -} -``` - -**How this works:** - -- It catches JavaScript errors in child components -- It displays a friendly error message instead of a blank screen -- It allows users to try again without losing their work - -**Why use error boundaries?** They prevent a single error from breaking the entire application. - -### Step 16: Implement Performance Optimizations - -**Why this matters:** Large repositories can be slow to process. We need to keep the UI responsive. - -**Key optimizations:** - -1. **Web Workers**: Already implemented for graph processing -2. **Progress Indicators**: Show users what's happening -3. **File Filtering**: Only process relevant files -4. **Lazy Loading**: Load components as needed - -**Implementation:** - -```tsx -// Add to your GitHub processing function -const MAX_FILES = 500; // Limit for free tier -if (pythonFiles.length > MAX_FILES) { - // Offer to filter by directory or file pattern - const shouldFilter = window.confirm( - `Repository has ${pythonFiles.length} Python files (max ${MAX_FILES}). ` + - `Would you like to filter by directory or file pattern?` - ); - if (shouldFilter) { - const filterPattern = prompt( - "Enter a directory path or file pattern to filter (e.g., 'src/', '*.py')", - "src/" - ); - if (filterPattern) { - const filteredFiles = pythonFiles.filter(file => - file.includes(filterPattern) || file.endsWith(filterPattern) - ); - pythonFiles = filteredFiles; - } - } -} -``` - -**Why limit file processing?** Processing too many files can: - -- Freeze the browser tab -- Exceed GitHub API rate limits -- Use excessive memory - -### Step 17: Add Export Functionality - -**Why this matters:** Users might want to save or share their generated graphs. - -**Implementation:** - -```tsx -export function exportGraphToJson(graph: KnowledgeGraph): string { - return JSON.stringify(graph, null, 2); -} - -export function downloadGraph(graph: KnowledgeGraph, filename: string = 'gitnexus-graph.json') { - const json = exportGraphToJson(graph); - const blob = new Blob([json], { type: 'application/json' }); - const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = filename; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - URL.revokeObjectURL(url); -} -``` - -**How this works:** - -1. Converts the graph to JSON -2. Creates a downloadable file -3. Triggers a download - -**Why include export?** It allows users to: - -- Save their work for later -- Share graphs with teammates -- Use the data in other tools - -## Conclusion - -This implementation guide has walked you through building a complete edge-based code knowledge graph generator using Deno. By following these steps, you'll create a privacy-focused tool that runs entirely in the user's browser. - -**Key advantages of this approach:** - -- **Zero server costs**: All processing happens in the user's browser -- **Strong privacy**: Code never leaves the user's machine -- **Modular architecture**: Easy to add more languages later -- **Clear separation of concerns**: Makes the codebase maintainable -- **Deno compatibility**: Modern runtime with built-in TypeScript support - -Remember to start small (Python support only) and iterate, adding more features and language support as you validate the core functionality. The most important part is getting the graph construction pipeline working correctly - everything else builds on that foundation. - -================ -File: src/__tests__/error-handler.test.ts -================ -import { describe, it, expect, beforeEach, jest } from '@jest/globals'; -import { - GitNexusError, - ValidationError, - NetworkError, - MemoryError, - ErrorRecoveryService, - createSafeAsync, - createSafe -} from '../lib/error-handler'; - -describe('Error Classes', () => { - it('should create a GitNexusError with correct properties', () => { - const error = new GitNexusError('Test', 'TEST_CODE', false, { a: 1 }); - expect(error).toBeInstanceOf(Error); - expect(error.message).toBe('Test'); - expect(error.code).toBe('TEST_CODE'); - expect(error.isRecoverable).toBe(false); - expect(error.context).toEqual({ a: 1 }); - }); -}); - -describe('ErrorRecoveryService', () => { - let service: ErrorRecoveryService; - - beforeEach(() => { - (ErrorRecoveryService as any).instance = undefined; - service = ErrorRecoveryService.getInstance(); - }); - - it('should get a singleton instance', () => { - const instance2 = ErrorRecoveryService.getInstance(); - expect(service).toBe(instance2); - }); - - describe('executeWithRetry', () => { - it('should return value on success', async () => { - const op = jest.fn<() => Promise>().mockResolvedValue('ok'); - const result = await service.executeWithRetry(op, 'test'); - expect(result).toBe('ok'); - }); - - it('should retry on failure', async () => { - const op = jest.fn<() => Promise>() - .mockRejectedValueOnce(new NetworkError('failed')) - .mockResolvedValue('ok'); - await service.executeWithRetry(op, 'test', { initialDelay: 1 }); - expect(op).toHaveBeenCalledTimes(2); - }); - }); - - describe('executeWithFallback', () => { - it('should return primary result', async () => { - const primary = jest.fn<() => Promise>().mockResolvedValue('primary'); - const fallback = jest.fn<() => Promise>().mockResolvedValue('fallback'); - const result = await service.executeWithFallback(primary, fallback, 'test'); - expect(result).toBe('primary'); - }); - - it('should use fallback on failure', async () => { - const primary = jest.fn<() => Promise>().mockRejectedValue(new Error('failed')); - const fallback = jest.fn<() => Promise>().mockResolvedValue('fallback'); - const result = await service.executeWithFallback(primary, fallback, 'test'); - expect(result).toBe('fallback'); - }); - }); -}); - -describe('createSafeAsync', () => { - it('should wrap an async function', async () => { - const fn = createSafeAsync(async (a: number) => a * 2); - const result = await fn(2); - expect(result).toBe(4); - }); -}); - -describe('createSafe', () => { - it('should wrap a sync function', () => { - const fn = createSafe((a: number) => a * 2); - const result = fn(2); - expect(result).toBe(4); - }); -}); - -================ -File: src/__tests__/health-monitor.test.ts -================ -import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals'; -import { HealthMonitor } from '../services/health-monitor'; - -describe('HealthMonitor', () => { - let monitor: HealthMonitor; - - beforeEach(() => { - (HealthMonitor as any).instance = undefined; - monitor = HealthMonitor.getInstance(); - }); - - afterEach(() => { - monitor.stop(); - }); - - it('should return a singleton instance', () => { - const instance1 = HealthMonitor.getInstance(); - const instance2 = HealthMonitor.getInstance(); - expect(instance1).toBe(instance2); - }); - - it('should record a metric', () => { - monitor.recordMetric('test_metric', 10); - const metrics = monitor.getMetrics('test_metric'); - expect(metrics).toHaveLength(1); - expect(metrics[0].value).toBe(10); - }); - - it('should record file processing', () => { - monitor.recordFileProcessing('test.js', true, 100); - const status = monitor.getHealthStatus(); - expect(status.processing.totalFiles).toBe(1); - expect(status.processing.successfulFiles).toBe(1); - }); - - it('should increment and decrement connections', () => { - monitor.incrementConnections(); - let status = monitor.getHealthStatus(); - expect(status.system.activeConnections).toBe(1); - - monitor.decrementConnections(); - status = monitor.getHealthStatus(); - expect(status.system.activeConnections).toBe(0); - }); - - it('should get health status', () => { - const status = monitor.getHealthStatus(); - expect(status).toBeDefined(); - expect(status.uptime).toBeGreaterThanOrEqual(0); - }); -}); - -================ -File: src/__tests__/kuzu.test.ts -================ -import { initKuzuDB, getKuzuDBInstance, resetKuzuDB } from '../core/kuzu/kuzu-loader.js'; -import { KuzuQueryEngine } from '../core/graph/kuzu-query-engine.js'; -import { KuzuPerformanceBenchmark } from '../lib/kuzu-performance-benchmark.js'; -import { kuzuPerformanceMonitor } from '../lib/kuzu-performance-monitor.js'; -import { isKuzuDBEnabled, setFeatureFlag } from '../config/feature-flags.js'; -import type { KnowledgeGraph } from '../core/graph/types.js'; - -// Mock KuzuDB for testing -jest.mock('kuzu-wasm', () => ({ - __esModule: true, - default: jest.fn(() => Promise.resolve({ - Database: jest.fn(), - Connection: jest.fn(), - FS: { - mkdir: jest.fn(), - mountIdbfs: jest.fn(), - syncfs: jest.fn(), - unmount: jest.fn() - }, - setWorkerPath: jest.fn() - })) -})); - -describe('KuzuDB Integration Tests', () => { - beforeEach(() => { - resetKuzuDB(); - kuzuPerformanceMonitor.clearMetrics(); - }); - - afterEach(() => { - resetKuzuDB(); - }); - - describe('KuzuDB Loader', () => { - test('should initialize KuzuDB successfully', async () => { - const instance = await initKuzuDB(); - expect(instance).toBeDefined(); - expect(typeof instance.createDatabase).toBe('function'); - expect(typeof instance.executeQuery).toBe('function'); - }); - - test('should create and manage database operations', async () => { - const instance = await initKuzuDB(); - - await instance.createDatabase('/test-db'); - await instance.createNodeTable('TestNode', { - id: 'STRING', - name: 'STRING', - value: 'INT64' - }); - - await instance.insertNode('TestNode', { - id: 'test1', - name: 'Test Node', - value: 42 - }); - - const result = await instance.executeQuery('MATCH (n:TestNode) RETURN n.name, n.value'); - expect(result.results).toBeDefined(); - - await instance.closeDatabase(); - }); - - test('should handle database errors gracefully', async () => { - const instance = await initKuzuDB(); - - // Test invalid query - await expect(instance.executeQuery('INVALID QUERY')).rejects.toThrow(); - }); - }); - - describe('KuzuQueryEngine', () => { - let queryEngine: KuzuQueryEngine; - - beforeEach(async () => { - queryEngine = new KuzuQueryEngine(); - await queryEngine.initialize(); - }); - - afterEach(async () => { - await queryEngine.close(); - }); - - test('should initialize query engine', () => { - expect(queryEngine.isReady()).toBe(true); - }); - - test('should import graph data', async () => { - const mockGraph: KnowledgeGraph = { - nodes: [ - { - id: 'func1', - label: 'Function', - properties: { name: 'testFunction', filePath: '/test.ts' } - } - ], - relationships: [] - }; - - await queryEngine.importGraph(mockGraph); - expect(queryEngine.isReady()).toBe(true); - }); - - test('should execute Cypher queries', async () => { - const mockGraph: KnowledgeGraph = { - nodes: [ - { - id: 'func1', - label: 'Function', - properties: { name: 'testFunction', filePath: '/test.ts' } - } - ], - relationships: [] - }; - - await queryEngine.importGraph(mockGraph); - - const result = await queryEngine.executeQuery('MATCH (f:Function) RETURN f.name'); - expect(result.nodes).toBeDefined(); - expect(result.executionTime).toBeGreaterThan(0); - }); - }); - - describe('Performance Monitoring', () => { - test('should track performance metrics', () => { - const opId = kuzuPerformanceMonitor.startOperation('test_operation'); - expect(opId).toBeDefined(); - - kuzuPerformanceMonitor.endOperation(opId, true); - - const report = kuzuPerformanceMonitor.getReport(); - expect(report.totalOperations).toBe(1); - expect(report.successfulOperations).toBe(1); - }); - - test('should generate performance reports', () => { - // Add some test metrics - kuzuPerformanceMonitor.addMetric({ - operation: 'test_query', - duration: 100, - timestamp: Date.now(), - success: true - }); - - const report = kuzuPerformanceMonitor.getReport(); - expect(report.totalOperations).toBe(1); - expect(report.averageDuration).toBe(100); - }); - }); - - describe('Performance Benchmarking', () => { - let benchmark: KuzuPerformanceBenchmark; - - beforeEach(() => { - benchmark = new KuzuPerformanceBenchmark(); - }); - - test('should initialize benchmark with test queries', () => { - expect(benchmark).toBeDefined(); - }); - - test('should run performance comparison', async () => { - const mockGraph: KnowledgeGraph = { - nodes: [ - { - id: 'func1', - label: 'Function', - properties: { name: 'testFunction', filePath: '/test.ts' } - } - ], - relationships: [] - }; - - const result = await benchmark.runBenchmark(mockGraph); - expect(result.kuzuDBResults).toBeDefined(); - expect(result.inMemoryResults).toBeDefined(); - expect(result.improvement).toBeDefined(); - }); - }); - - describe('Feature Flags', () => { - test('should enable/disable KuzuDB features', () => { - const initialState = isKuzuDBEnabled(); - - setFeatureFlag('enableKuzuDB', false); - expect(isKuzuDBEnabled()).toBe(false); - - setFeatureFlag('enableKuzuDB', true); - expect(isKuzuDBEnabled()).toBe(true); - }); - }); - - describe('Error Handling', () => { - test('should handle initialization failures gracefully', async () => { - // Mock failure - jest.doMock('kuzu-wasm', () => ({ - __esModule: true, - default: jest.fn(() => Promise.reject(new Error('WASM load failed'))) - })); - - await expect(initKuzuDB()).rejects.toThrow('KuzuDB initialization failed'); - }); - - test('should provide fallback mechanisms', async () => { - // Test that the system can work without KuzuDB - setFeatureFlag('enableKuzuDB', false); - - // Should not throw when KuzuDB is disabled - expect(() => isKuzuDBEnabled()).not.toThrow(); - }); - }); - - describe('Integration Scenarios', () => { - test('should handle complete workflow', async () => { - // Initialize KuzuDB - const instance = await initKuzuDB(); - - // Create database and schema - await instance.createDatabase('/integration-test'); - await instance.createNodeTable('Function', { - id: 'STRING', - name: 'STRING', - filePath: 'STRING' - }); - - // Insert test data - await instance.insertNode('Function', { - id: 'func1', - name: 'testFunction', - filePath: '/test.ts' - }); - - // Execute query - const result = await instance.executeQuery('MATCH (f:Function) RETURN f.name'); - expect(result.results).toBeDefined(); - - // Clean up - await instance.closeDatabase(); - }); - - test('should handle performance monitoring in workflow', async () => { - const opId = kuzuPerformanceMonitor.startOperation('integration_test'); - - const instance = await initKuzuDB(); - await instance.createDatabase('/perf-test'); - - kuzuPerformanceMonitor.endOperation(opId, true); - - const report = kuzuPerformanceMonitor.getReport(); - expect(report.successfulOperations).toBe(1); - - await instance.closeDatabase(); - }); - }); -}); - -================ -File: src/__tests__/memory-manager.test.ts -================ -import { describe, it, expect, beforeEach, afterEach, jest } from '@jest/globals'; -import { MemoryManager } from '../services/memory-manager'; - -describe('MemoryManager', () => { - let memoryManager: MemoryManager; - - beforeEach(() => { - (MemoryManager as any).instance = undefined; - memoryManager = MemoryManager.getInstance(); - }); - - afterEach(() => { - memoryManager.destroy(); - }); - - it('should return a singleton instance', () => { - const instance1 = MemoryManager.getInstance(); - const instance2 = MemoryManager.getInstance(); - expect(instance1).toBe(instance2); - }); - - it('should cache a file', () => { - const result = memoryManager.cacheFile('test.js', 'console.log("hello")'); - expect(result).toBe(true); - expect(memoryManager.getStats().fileCount).toBe(1); - }); - - it('should not cache a file that is too large', () => { - memoryManager.configure({ maxFileSizeMB: 1 }); - const largeContent = 'a'.repeat(2 * 1024 * 1024); - const result = memoryManager.cacheFile('large.js', largeContent); - expect(result).toBe(false); - expect(memoryManager.getStats().fileCount).toBe(0); - }); - - it('should retrieve a cached file', () => { - memoryManager.cacheFile('test.js', 'console.log("hello")'); - const content = memoryManager.getFile('test.js'); - expect(content).toBe('console.log("hello")'); - }); - - it('should return null for a non-cached file', () => { - const content = memoryManager.getFile('non-existent.js'); - expect(content).toBeNull(); - }); - - it('should remove a file from the cache', () => { - memoryManager.cacheFile('test.js', 'console.log("hello")'); - memoryManager.removeFile('test.js'); - const content = memoryManager.getFile('test.js'); - expect(content).toBeNull(); - }); - - it('should clear the cache', () => { - memoryManager.cacheFile('test1.js', 'content1'); - memoryManager.cacheFile('test2.js', 'content2'); - memoryManager.clearCache(); - expect(memoryManager.getStats().fileCount).toBe(0); - }); - - it('should perform cleanup when memory is high', () => { - memoryManager.configure({ cleanupThresholdMB: 1, maxMemoryMB: 2 }); - memoryManager.cacheFile('test1.js', 'a'.repeat(1.5 * 1024 * 1024)); - memoryManager.forceCleanup(); - expect(memoryManager.getStats().fileCount).toBe(0); - }); -}); - -================ -File: src/__tests__/setup.ts -================ -/** - * Test setup file for Jest - */ - -import { jest } from '@jest/globals'; - -// Global test setup -beforeAll(() => { - // Suppress console output during tests - jest.spyOn(console, 'log').mockImplementation(() => {}); - jest.spyOn(console, 'warn').mockImplementation(() => {}); - jest.spyOn(console, 'error').mockImplementation(() => {}); - jest.spyOn(console, 'info').mockImplementation(() => {}); -}); - -afterAll(() => { - // Restore console methods - jest.restoreAllMocks(); -}); - -// Reset singletons between tests -beforeEach(() => { - // Reset singleton instances - const singletons = [ - 'ConfigService', - 'MemoryManager', - 'ErrorRecoveryService', - 'HealthMonitor', - 'StreamingProcessor' - ]; - - singletons.forEach(singleton => { - const modulePath = `../${singleton.toLowerCase().replace(/([A-Z])/g, '-$1').substring(1)}`; - try { - const module = require(modulePath); - if (module[singleton]) { - module[singleton].instance = undefined; - } - } catch (e) { - // Module might not exist, skip - } - }); -}); - -// Mock window for Node.js environment -if (typeof window === 'undefined') { - (global as any).window = { - setInterval: setInterval, - clearInterval: clearInterval - }; -} - -// Mock Node.js built-ins -import { TextEncoder, TextDecoder } from 'util'; - -global.TextEncoder = TextEncoder; -global.TextDecoder = TextDecoder as any; - -// Mock crypto for Node.js environment -if (!global.crypto) { - (global as any).crypto = { - randomUUID: () => 'test-uuid-' + Math.random().toString(36).substring(2, 15) - }; -} - -// Mock performance for Node.js environment -if (!global.performance) { - (global as any).performance = { - now: () => Date.now(), - timeOrigin: Date.now() - }; -} - -// Environment variable setup -process.env.NODE_ENV = 'test'; -process.env.MEMORY_MAX_MB = '512'; -process.env.PROCESSING_MAX_RETRIES = '3'; -process.env.MONITORING_INTERVAL = '1000'; - -// Cleanup function -export const cleanup = () => { - // Clean up any test artifacts - jest.clearAllTimers(); - jest.clearAllMocks(); -}; - -================ -File: src/__tests__/streaming-processor.test.ts -================ -import { describe, it, expect, beforeEach, jest } from '@jest/globals'; -import { StreamingProcessor } from '../lib/streaming-processor'; -import { Readable } from 'stream'; - -describe('StreamingProcessor', () => { - let processor: StreamingProcessor; - - beforeEach(() => { - (StreamingProcessor as any).instance = undefined; - processor = StreamingProcessor.getInstance(); - }); - - it('should get a singleton instance', () => { - const instance2 = StreamingProcessor.getInstance(); - expect(processor).toBe(instance2); - }); - - describe('processFile', () => { - it('should process a file and return data', async () => { - const mockReadStream = new Readable(); - mockReadStream.push('line 1\n'); - mockReadStream.push('line 2\n'); - mockReadStream.push(null); - - jest.spyOn(require('fs'), 'createReadStream').mockReturnValue(mockReadStream); - jest.spyOn(require('fs/promises'), 'stat').mockResolvedValue({ size: 100 }); - - const result = await processor.processFile('test.txt', async (line) => line); - expect(result.data).toEqual(['line 1', 'line 2']); - }); - }); - - describe('processFilesParallel', () => { - it('should process files in parallel', async () => { - const fileProcessor = jest.fn<() => Promise>().mockResolvedValue('processed'); - const results = await processor.processFilesParallel(['a.txt', 'b.txt'], fileProcessor); - expect(results.length).toBe(2); - expect(fileProcessor).toHaveBeenCalledTimes(2); - }); - }); - - describe('processInBatches', () => { - it('should process data in batches', async () => { - const batchProcessor = jest.fn<() => Promise>().mockResolvedValue(['processed']); - const results = await processor.processInBatches([1, 2, 3, 4, 5], batchProcessor, 2); - expect(results.length).toBe(3); - expect(batchProcessor).toHaveBeenCalledTimes(3); - }); - }); -}); - -================ -File: src/__tests__/utils.test.ts -================ -import { describe, it, expect, beforeEach, jest } from '@jest/globals'; -import { ValidationService } from '../lib/validation'; -import { ConfigService } from '../config/config'; - -describe('ValidationService', () => { - it('should validate a valid node', () => { - const validNode = { - id: 'test-id', - label: 'Function', - properties: { - name: 'testFunction', - path: '/test/path.js', - type: 'function', - startLine: 10, - endLine: 20 - } - }; - const result = ValidationService.validateNode(validNode); - expect(result.id).toBe('test-id'); - }); - - it('should throw for an invalid node', () => { - const invalidNode = { id: '', label: 'Invalid', properties: {} }; - expect(() => ValidationService.validateNode(invalidNode)).toThrow(); - }); -}); - -describe('ConfigService', () => { - let configService: ConfigService; - - beforeEach(() => { - (ConfigService as any).instance = undefined; - configService = ConfigService.getInstance(); - }); - - it('should load default config', () => { - const config = configService.getConfiguration(); - expect(config.memory.maxMemoryMB).toBe(512); - }); - - it('should update config', () => { - configService.updateConfig({ memory: { maxMemoryMB: 1024 } } as any); - const config = configService.getConfiguration(); - expect(config.memory.maxMemoryMB).toBe(1024); - }); -}); - -================ -File: src/ai/cypher-generator.ts -================ -import { HumanMessage, SystemMessage } from '@langchain/core/messages'; -import type { LLMService, LLMConfig } from './llm-service.ts'; -import type { KnowledgeGraph } from '../core/graph/types.ts'; - -export interface CypherQuery { - cypher: string; - explanation: string; - confidence: number; - warnings?: string[]; -} - -export interface CypherGenerationOptions { - maxRetries?: number; - includeExamples?: boolean; - strictMode?: boolean; -} - -export class CypherGenerator { - private llmService: LLMService; - private graphSchema: string = ''; - - // Common Cypher patterns and examples - private static readonly CYPHER_EXAMPLES = [ - { - question: "What functions are in the main.py file?", - cypher: "MATCH (f:File {name: 'main.py'})-[:CONTAINS]->(func:Function) RETURN func.name, func.startLine" - }, - { - question: "Which functions call the authenticate function?", - cypher: "MATCH (caller)-[:CALLS]->(target:Function {name: 'authenticate'}) RETURN caller.name, caller.filePath" - }, - { - question: "Show me all classes in the project", - cypher: "MATCH (c:Class) RETURN c.name, c.filePath" - }, - { - question: "What classes inherit from BaseService?", - cypher: "MATCH (child:Class)-[:INHERITS]->(parent:Class {name: 'BaseService'}) RETURN child.name, child.filePath" - }, - { - question: "Find all methods in the UserService class", - cypher: "MATCH (c:Class {name: 'UserService'})-[:CONTAINS]->(m:Method) RETURN m.name, m.startLine" - }, - { - question: "Which methods override the save method?", - cypher: "MATCH (child:Method)-[:OVERRIDES]->(parent:Method {name: 'save'}) RETURN child.name, child.parentClass" - }, - { - question: "Show all interfaces and the classes that implement them", - cypher: "MATCH (c:Class)-[:IMPLEMENTS]->(i:Interface) RETURN i.name, c.name" - }, - { - question: "Find functions decorated with @app.route", - cypher: "MATCH (d:Decorator {name: 'app.route'})-[:DECORATES]->(f:Function) RETURN f.name, f.filePath" - }, - { - question: "What files import the requests module?", - cypher: "MATCH (f:File)-[:IMPORTS]->(target) WHERE target.name CONTAINS 'requests' RETURN f.name" - }, - { - question: "Show the call chain from main to database functions", - cypher: "MATCH (main:Function {name: 'main'})-[:CALLS*1..3]->(db:Function) WHERE db.name CONTAINS 'db' OR db.name CONTAINS 'database' RETURN main.name, db.name" - }, - { - question: "Find all functions containing 'user' in their name", - cypher: "MATCH (f:Function) WHERE f.name CONTAINS 'user' RETURN f.name, f.filePath" - }, - { - question: "What functions are called through a chain of 2-4 calls from the main function?", - cypher: "MATCH (main:Function {name: 'main'})-[:CALLS*2..4]->(target:Function) RETURN main.name, target.name" - }, - { - question: "How many classes are in each file?", - cypher: "MATCH (f:File)-[:CONTAINS]->(c:Class) RETURN f.name, COUNT(c)" - }, - { - question: "Count all functions in the project", - cypher: "MATCH (f:Function) RETURN COUNT(f)" - }, - { - question: "List all function names in alphabetical order", - cypher: "MATCH (f:Function) RETURN COLLECT(f.name)" - }, - { - question: "Find files that contain both classes and functions", - cypher: "MATCH (f:File)-[:CONTAINS]->(c:Class) WHERE EXISTS((f)-[:CONTAINS]->(:Function)) RETURN f.name" - }, - { - question: "Show methods that start with 'get'", - cypher: "MATCH (m:Method) WHERE m.name CONTAINS 'get' RETURN m.name, m.filePath" - }, - { - question: "Find all indirect dependencies (functions that call functions that call a target)", - cypher: "MATCH (caller:Function)-[:CALLS*2..2]->(target:Function {name: 'database_query'}) RETURN caller.name, target.name" - } - ]; - - constructor(llmService: LLMService) { - this.llmService = llmService; - } - - /** - * Update the graph schema for better query generation - */ - public updateSchema(graph: KnowledgeGraph): void { - this.graphSchema = this.generateSchemaDescription(graph); - } - - /** - * Generate a Cypher query from natural language - */ - public async generateQuery( - question: string, - llmConfig: LLMConfig, - options: CypherGenerationOptions = {} - ): Promise { - const { maxRetries = 2, includeExamples = true, strictMode = false } = options; - - let lastError: string | null = null; - - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - const systemPrompt = this.buildSystemPrompt(includeExamples, strictMode, lastError); - const userPrompt = this.buildUserPrompt(question); - - const messages = [ - new SystemMessage(systemPrompt), - new HumanMessage(userPrompt) - ]; - - const response = await this.llmService.chat(llmConfig, messages); - const result = this.parseResponse(String(response.content || '')); - - // Validate the generated query - const validation = this.validateQuery(result.cypher); - if (!validation.isValid) { - lastError = validation.error!; - if (attempt < maxRetries) { - console.warn(`Query validation failed (attempt ${attempt + 1}): ${validation.error}`); - continue; - } - } - - return { - ...result, - warnings: validation.warnings - }; - - } catch (error) { - lastError = error instanceof Error ? error.message : 'Unknown error'; - if (attempt < maxRetries) { - console.warn(`Query generation failed (attempt ${attempt + 1}): ${lastError}`); - continue; - } - - throw new Error(`Failed to generate Cypher query after ${maxRetries + 1} attempts: ${lastError}`); - } - } - - throw new Error('Unexpected error in query generation'); - } - - /** - * Build the system prompt with schema and examples - */ - private buildSystemPrompt(includeExamples: boolean, strictMode: boolean, lastError?: string | null): string { - let prompt = `You are a Cypher query expert for a code knowledge graph using KuzuDB (a high-performance graph database). Your task is to convert natural language questions into valid Cypher queries optimized for KuzuDB. - -GRAPH SCHEMA: -${this.graphSchema} - -NODE TYPES: -- Project: Root project node -- Folder: Directory containers -- File: Source code files -- Module: Python modules (.py files) -- Class: Class definitions -- Function: Function definitions -- Method: Class method definitions -- Variable: Variable declarations - -RELATIONSHIP TYPES: -- CONTAINS: Hierarchical containment (Project->Folder, Folder->File, File->Function, etc.) -- CALLS: Function/method calls between code entities -- INHERITS: Class inheritance relationships -- IMPORTS: Module import relationships -- OVERRIDES: Method override relationships -- IMPLEMENTS: Interface implementation -- DECORATES: Decorator relationships - -KUZUDB OPTIMIZATION GUIDELINES: - -1. PERFORMANCE: KuzuDB excels at complex graph traversals and pattern matching - - Use variable-length paths (*1..5) for call chains and dependency analysis - - Leverage WHERE clauses for efficient filtering - - Use aggregation functions (COUNT, COLLECT) for statistics - -2. QUERY PATTERNS SUPPORTED: - - SIMPLE MATCH: Find nodes by label and properties - Pattern: MATCH (n:Label {property: 'value'}) RETURN n.property - - WHERE CLAUSE: Filter nodes with complex conditions - Pattern: MATCH (n:Label) WHERE n.property CONTAINS 'text' RETURN n.property - - RELATIONSHIP TRAVERSAL: Follow direct relationships - Pattern: MATCH (a)-[:RELATIONSHIP]->(b:Label) RETURN a.name, b.name - - VARIABLE-LENGTH PATHS: Multi-hop relationships (KuzuDB strength) - Pattern: MATCH (a)-[:RELATIONSHIP*1..3]->(b) RETURN a.name, b.name - - AGGREGATION: Count, collect, or summarize data - Pattern: MATCH (n:Label) RETURN COUNT(n) - Pattern: MATCH (n:Label) RETURN COLLECT(n.name) - -3. KUZUDB-SPECIFIC FEATURES: - - Use variable-length paths for dependency analysis: (start)-[:CALLS*1..5]->(end) - - Leverage pattern matching for complex relationships - - Use aggregation for performance statistics - - Optimize for read-heavy workloads - -QUERY SELECTION GUIDELINES: - -- Use SIMPLE MATCH for direct property lookups -- Use WHERE for text search, pattern matching, or complex conditions -- Use RELATIONSHIP TRAVERSAL for direct connections -- Use VARIABLE-LENGTH PATHS for call chains, dependency analysis (KuzuDB excels here) -- Use AGGREGATION for counting, statistics, or collecting lists - -IMPORTANT RULES: -1. Always use MATCH patterns to find nodes -2. Use WHERE clauses for filtering by text content or complex conditions -3. Node properties include: name, filePath, startLine, endLine, type, qualifiedName -4. Return meaningful information, not just node IDs -5. Use case-insensitive matching: WHERE toLower(n.name) CONTAINS toLower('search') -6. Prefer specific node types over generic matches -7. Always return results in a readable format -8. Use variable-length paths (*1..5) for call chains and dependency analysis - KuzuDB handles these efficiently -9. Use aggregation functions (COUNT, COLLECT) for statistics and summaries -10. Limit variable-length path depth to avoid performance issues (max *1..5) -11. Leverage KuzuDB's strength in complex graph traversals for dependency analysis`; - - if (includeExamples) { - prompt += `\n\nEXAMPLES:`; - for (const example of CypherGenerator.CYPHER_EXAMPLES) { - prompt += `\nQ: "${example.question}"\nA: ${example.cypher}\n`; - } - } - - if (strictMode) { - prompt += `\n\nSTRICT MODE: Only generate queries that exactly match the schema. Do not make assumptions about node properties that aren't explicitly defined.`; - } - - if (lastError) { - prompt += `\n\nPREVIOUS ERROR: The last query attempt failed with: "${lastError}". Please fix this issue in your new query.`; - } - - prompt += `\n\nRESPONSE FORMAT: -Provide your response in this exact JSON format: -{ - "cypher": "your cypher query here", - "explanation": "brief explanation of what the query does and why this pattern was chosen", - "confidence": 0.85 -} - -The confidence should be a number between 0 and 1 indicating how confident you are in the query.`; - - return prompt; - } - - /** - * Build the user prompt with the question - */ - private buildUserPrompt(question: string): string { - return `Please convert this question to a Cypher query: "${question}"`; - } - - /** - * Parse the LLM response to extract Cypher query - */ - private parseResponse(response: string): { cypher: string; explanation: string; confidence: number } { - try { - // Try to extract JSON from the response - const jsonMatch = response.match(/\{[\s\S]*\}/); - if (jsonMatch) { - const parsed = JSON.parse(jsonMatch[0]); - return { - cypher: parsed.cypher || '', - explanation: parsed.explanation || '', - confidence: parsed.confidence || 0.5 - }; - } - - // Fallback: try to extract Cypher from code blocks - const cypherMatch = response.match(/```(?:cypher)?\s*(.*?)\s*```/s); - if (cypherMatch) { - return { - cypher: cypherMatch[1].trim(), - explanation: 'Generated Cypher query', - confidence: 0.7 - }; - } - - // Last resort: use the entire response as cypher - return { - cypher: response.trim(), - explanation: 'Raw LLM response', - confidence: 0.3 - }; - - } catch (error) { - throw new Error(`Failed to parse LLM response: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Validate the generated Cypher query - */ - private validateQuery(cypher: string): { isValid: boolean; error?: string; warnings?: string[] } { - const warnings: string[] = []; - - if (!cypher || cypher.trim().length === 0) { - return { isValid: false, error: 'Empty query generated' }; - } - - // Basic syntax checks - const upperCypher = cypher.toUpperCase(); - - // Must have MATCH or CREATE or other valid starting keywords - if (!upperCypher.match(/^\s*(MATCH|CREATE|MERGE|WITH|RETURN|CALL|SHOW)/)) { - return { isValid: false, error: 'Query must start with a valid Cypher keyword (MATCH, CREATE, etc.)' }; - } - - // Check for balanced parentheses - const openParens = (cypher.match(/\(/g) || []).length; - const closeParens = (cypher.match(/\)/g) || []).length; - if (openParens !== closeParens) { - return { isValid: false, error: 'Unbalanced parentheses in query' }; - } - - // Check for balanced brackets - const openBrackets = (cypher.match(/\[/g) || []).length; - const closeBrackets = (cypher.match(/\]/g) || []).length; - if (openBrackets !== closeBrackets) { - return { isValid: false, error: 'Unbalanced brackets in query' }; - } - - // Check for balanced braces - const openBraces = (cypher.match(/\{/g) || []).length; - const closeBraces = (cypher.match(/\}/g) || []).length; - if (openBraces !== closeBraces) { - return { isValid: false, error: 'Unbalanced braces in query' }; - } - - // Warn about potentially expensive operations - if (upperCypher.includes('MATCH ()') || upperCypher.includes('MATCH (*)')) { - warnings.push('Query matches all nodes - this could be expensive'); - } - - if (!upperCypher.includes('RETURN') && !upperCypher.includes('DELETE') && !upperCypher.includes('SET')) { - warnings.push('Query does not return results'); - } - - return { isValid: true, warnings }; - } - - /** - * Generate a schema description from the knowledge graph - */ - private generateSchemaDescription(graph: KnowledgeGraph): string { - const nodeTypes = new Set(); - const relationshipTypes = new Set(); - const nodeProperties = new Map>(); - - // Analyze nodes - graph.nodes.forEach(node => { - nodeTypes.add(node.label); - - if (!nodeProperties.has(node.label)) { - nodeProperties.set(node.label, new Set()); - } - - Object.keys(node.properties).forEach(prop => { - nodeProperties.get(node.label)!.add(prop); - }); - }); - - // Analyze relationships - graph.relationships.forEach(rel => { - relationshipTypes.add(rel.type); - }); - - let schema = `NODES (${graph.nodes.length} total):\n`; - for (const nodeType of Array.from(nodeTypes).sort()) { - const props = nodeProperties.get(nodeType); - const propList = props ? Array.from(props).sort().join(', ') : 'none'; - const count = graph.nodes.filter(n => n.label === nodeType).length; - schema += `- ${nodeType} (${count}): ${propList}\n`; - } - - schema += `\nRELATIONSHIPS (${graph.relationships.length} total):\n`; - for (const relType of Array.from(relationshipTypes).sort()) { - const count = graph.relationships.filter(r => r.type === relType).length; - schema += `- ${relType} (${count})\n`; - } - - return schema; - } - - /** - * Get the current schema description - */ - public getSchema(): string { - return this.graphSchema; - } - - /** - * Clean and format a Cypher query - */ - public cleanQuery(cypher: string): string { - return cypher - .trim() - .replace(/\s+/g, ' ') - .replace(/\s*([(),\[\]{}])\s*/g, '$1') - .replace(/\s*([=<>!]+)\s*/g, ' $1 ') - .replace(/\s+/g, ' ') - .trim(); - } -} - -================ -File: src/ai/index.ts -================ -export * from './llm-service.js'; -export * from './cypher-generator.js'; -export * from './orchestrator.js'; -export * from './langchain-orchestrator.js'; -export * from './kuzu-rag-orchestrator.js'; - -// Export KuzuDB performance prompts -export * from './prompts/kuzu-performance-prompts.js'; - -================ -File: src/ai/kuzu-rag-orchestrator.ts -================ -import { HumanMessage, SystemMessage, AIMessage } from '@langchain/core/messages'; -import type { LLMService, LLMConfig } from './llm-service.ts'; -import type { CypherGenerator, CypherQuery } from './cypher-generator.ts'; -import type { KnowledgeGraph } from '../core/graph/types.ts'; -import type { GraphNode } from '../core/graph/types.ts'; -import { KuzuQueryEngine, type KuzuQueryResponse } from '../core/graph/kuzu-query-engine.js'; -import { isKuzuDBEnabled } from '../config/feature-flags.js'; - -export interface KuzuRAGContext { - graph: KnowledgeGraph; - fileContents: Map; - kuzuQueryEngine: KuzuQueryEngine; -} - -export interface KuzuToolResult { - toolName: string; - input: string; - output: string; - success: boolean; - error?: string; - executionTime?: number; - resultCount?: number; -} - -export interface KuzuReasoningStep { - step: number; - thought: string; - action: string; - actionInput: string; - observation: string; - toolResult?: KuzuToolResult; - cypherQuery?: string; -} - -export interface KuzuRAGResponse { - answer: string; - reasoning: KuzuReasoningStep[]; - cypherQueries: CypherQuery[]; - confidence: number; - sources: string[]; - performance: { - totalExecutionTime: number; - queryExecutionTimes: number[]; - kuzuQueryCount: number; - }; -} - -export interface KuzuRAGOptions { - maxReasoningSteps?: number; - includeReasoning?: boolean; - strictMode?: boolean; - temperature?: number; - useKuzuDB?: boolean; - queryTimeout?: number; - maxResults?: number; -} - -export class KuzuRAGOrchestrator { - private llmService: LLMService; - private cypherGenerator: CypherGenerator; - private context: KuzuRAGContext | null = null; - private kuzuQueryEngine: KuzuQueryEngine; - - constructor(llmService: LLMService, cypherGenerator: CypherGenerator) { - this.llmService = llmService; - this.cypherGenerator = cypherGenerator; - this.kuzuQueryEngine = new KuzuQueryEngine(); - } - - /** - * Initialize the orchestrator with KuzuDB - */ - async initialize(): Promise { - if (isKuzuDBEnabled()) { - await this.kuzuQueryEngine.initialize(); - } - } - - /** - * Set the current context (graph, file contents, and KuzuDB query engine) - */ - public async setContext(context: Omit): Promise { - this.context = { - ...context, - kuzuQueryEngine: this.kuzuQueryEngine - }; - - this.cypherGenerator.updateSchema(context.graph); - - // Import graph into KuzuDB for faster queries if enabled - if (isKuzuDBEnabled() && this.kuzuQueryEngine.isReady()) { - await this.kuzuQueryEngine.importGraph(context.graph); - } - } - - /** - * Answer a question using ReAct pattern with KuzuDB integration - */ - public async answerQuestion( - question: string, - llmConfig: LLMConfig, - options: KuzuRAGOptions = {} - ): Promise { - if (!this.context) { - throw new Error('Context not set. Call setContext() first.'); - } - - const { - maxReasoningSteps = 5, - includeReasoning = true, - strictMode = false, - temperature = 0.1, - useKuzuDB = isKuzuDBEnabled(), - queryTimeout = 30000, - maxResults = 100 - } = options; - - const reasoning: KuzuReasoningStep[] = []; - const cypherQueries: CypherQuery[] = []; - const sources: string[] = []; - const queryExecutionTimes: number[] = []; - let kuzuQueryCount = 0; - - const reasoningConfig: LLMConfig = { - ...llmConfig, - temperature: temperature - }; - - let currentStep = 1; - let finalAnswer = ''; - let confidence = 0.5; - const startTime = performance.now(); - - try { - const systemPrompt = this.buildKuzuReActSystemPrompt(strictMode, useKuzuDB); - const conversation = [new SystemMessage(systemPrompt)]; - - // Add the user question - conversation.push(new HumanMessage(question)); - - while (currentStep <= maxReasoningSteps) { - // Get LLM response with reasoning - const response = await this.llmService.chat(reasoningConfig, conversation); - const content = response.content; - - // Parse the response for thought, action, and observation - const parsed = this.parseReActResponse(content); - - if (!parsed) { - // If we can't parse the response, assume it's the final answer - finalAnswer = content; - break; - } - - const { thought, action, actionInput } = parsed; - - // Execute the action - let observation = ''; - let toolResult: KuzuToolResult | undefined; - - try { - if (action === 'query_graph') { - const queryStartTime = performance.now(); - - if (useKuzuDB && this.kuzuQueryEngine.isReady()) { - // Use KuzuDB for faster query execution - const kuzuResult = await this.kuzuQueryEngine.executeQuery(actionInput, { - timeout: queryTimeout, - maxResults, - includeExecutionTime: true - }); - - observation = this.formatKuzuQueryResult(kuzuResult); - toolResult = { - toolName: 'kuzu_query_graph', - input: actionInput, - output: observation, - success: true, - executionTime: kuzuResult.executionTime, - resultCount: kuzuResult.resultCount - }; - - queryExecutionTimes.push(kuzuResult.executionTime); - kuzuQueryCount++; - - } else { - // Fallback to in-memory graph query - const result = await this.executeGraphQuery(actionInput); - observation = JSON.stringify(result, null, 2); - toolResult = { - toolName: 'query_graph', - input: actionInput, - output: observation, - success: true - }; - } - - // Generate Cypher query for logging - const cypherQuery = await this.cypherGenerator.generateQuery(actionInput, reasoningConfig, { - includeExamples: false, - strictMode: strictMode - }); - - if (cypherQuery) { - cypherQueries.push(cypherQuery); - } - - } else if (action === 'get_code') { - const result = await this.getCodeSnippet(actionInput); - observation = result; - toolResult = { - toolName: 'get_code', - input: actionInput, - output: observation, - success: true - }; - - } else if (action === 'search_files') { - const result = await this.searchFiles(actionInput); - observation = JSON.stringify(result, null, 2); - toolResult = { - toolName: 'search_files', - input: actionInput, - output: observation, - success: true - }; - - } else if (action === 'final_answer') { - finalAnswer = actionInput; - break; - } - - } catch (error) { - observation = `Error executing ${action}: ${error instanceof Error ? error.message : 'Unknown error'}`; - toolResult = { - toolName: action, - input: actionInput, - output: observation, - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }; - } - - // Add reasoning step - reasoning.push({ - step: currentStep, - thought, - action, - actionInput, - observation, - toolResult, - cypherQuery: action === 'query_graph' ? actionInput : undefined - }); - - // Add to conversation for next iteration - conversation.push(new AIMessage(content)); - conversation.push(new HumanMessage(`Observation: ${observation}\n\nWhat should I do next?`)); - - currentStep++; - } - - const totalExecutionTime = performance.now() - startTime; - - return { - answer: finalAnswer, - reasoning: includeReasoning ? reasoning : [], - cypherQueries, - confidence, - sources, - performance: { - totalExecutionTime, - queryExecutionTimes, - kuzuQueryCount - } - }; - - } catch (error) { - console.error('KuzuRAG orchestrator error:', error); - throw new Error(`RAG orchestration failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Build system prompt for KuzuDB-enhanced ReAct - */ - private buildKuzuReActSystemPrompt(strictMode: boolean, useKuzuDB: boolean): string { - const basePrompt = `You are an AI assistant that helps analyze codebases using a knowledge graph powered by KuzuDB (a high-performance graph database). You have access to sophisticated graph querying capabilities for fast and accurate code analysis. - -Available tools: -1. query_graph - Execute Cypher queries on the knowledge graph (${useKuzuDB ? 'using KuzuDB for enhanced performance' : 'using in-memory graph'}) -2. get_code - Retrieve specific code snippets -3. search_files - Find files by name or content patterns -4. final_answer - Provide the final answer - -${useKuzuDB ? ` -KUZUDB CAPABILITIES: -- High-performance graph queries with execution time tracking -- Complex dependency analysis and call chain traversal -- Persistent storage across sessions -- Optimized for large-scale codebases -- Real-time performance monitoring - -PERFORMANCE FEATURES: -- Query execution time is automatically tracked -- Results include performance metrics -- Database operations are optimized for speed -- Support for complex graph traversals` : 'Using in-memory graph for queries.'} - -${strictMode ? 'STRICT MODE: Only use exact matches and precise queries.' : 'FLEXIBLE MODE: Use heuristic matching when exact matches fail.'} - -QUERY OPTIMIZATION GUIDELINES: -- Use specific node types (Function, Class, Method) for better performance -- Leverage variable-length paths for dependency analysis -- Use aggregation functions for statistics and summaries -- Prefer complex graph traversals over simple lookups -- Take advantage of KuzuDB's strength in pattern matching - -Always follow this format: -Thought: I need to think about what information I need -Action: tool_name -Action Input: the input to the tool -Observation: the result of the action -... (repeat if needed) -Thought: I now have enough information to answer -Action: final_answer -Action Input: the final answer to the user's question - -When using query_graph, focus on: -- Complex dependency analysis -- Call chain traversal -- Pattern matching across the codebase -- Statistical analysis of code structure -- Relationship exploration between code entities`; - - return basePrompt; - } - - /** - * Parse ReAct response format - */ - private parseReActResponse(content: string): { thought: string; action: string; actionInput: string } | null { - const thoughtMatch = content.match(/Thought:\s*(.+?)(?=\nAction:)/s); - const actionMatch = content.match(/Action:\s*(.+?)(?=\nAction Input:)/); - const actionInputMatch = content.match(/Action Input:\s*(.+?)(?=\nObservation:|\nThought:|$)/s); - - if (thoughtMatch && actionMatch && actionInputMatch) { - return { - thought: thoughtMatch[1].trim(), - action: actionMatch[1].trim(), - actionInput: actionInputMatch[1].trim() - }; - } - - return null; - } - - /** - * Format KuzuDB query result for observation - */ - private formatKuzuQueryResult(result: KuzuQueryResponse): string { - const summary = `Found ${result.nodes.length} nodes and ${result.relationships.length} relationships (execution time: ${result.executionTime.toFixed(2)}ms)`; - - if (result.nodes.length === 0 && result.relationships.length === 0) { - return `${summary}. No results found.`; - } - - const nodeSummary = result.nodes.length > 0 - ? `\nNodes: ${result.nodes.slice(0, 5).map(n => `${n.label}:${n.properties.name || n.id}`).join(', ')}${result.nodes.length > 5 ? '...' : ''}` - : ''; - - const relSummary = result.relationships.length > 0 - ? `\nRelationships: ${result.relationships.slice(0, 5).map(r => `${r.type}:${r.source}->${r.target}`).join(', ')}${result.relationships.length > 5 ? '...' : ''}` - : ''; - - return `${summary}${nodeSummary}${relSummary}`; - } - - /** - * Execute graph query (fallback to in-memory) - */ - private async executeGraphQuery(query: string): Promise { - if (!this.context) { - throw new Error('Context not set'); - } - - // Simple in-memory query execution as fallback - // This would be replaced with actual graph query logic - return { results: [], count: 0 }; - } - - /** - * Get code snippet - */ - private async getCodeSnippet(filePath: string): Promise { - if (!this.context) { - throw new Error('Context not set'); - } - - const content = this.context.fileContents.get(filePath); - return content || 'File not found'; - } - - /** - * Search files - */ - private async searchFiles(pattern: string): Promise { - if (!this.context) { - throw new Error('Context not set'); - } - - const matchingFiles: string[] = []; - for (const [filePath, content] of this.context.fileContents.entries()) { - if (filePath.includes(pattern) || content.includes(pattern)) { - matchingFiles.push(filePath); - } - } - - return matchingFiles.slice(0, 10); // Limit results - } - - /** - * Get performance statistics - */ - async getPerformanceStats(): Promise { - if (!this.kuzuQueryEngine.isReady()) { - return { status: 'KuzuDB not initialized' }; - } - - const dbStats = await this.kuzuQueryEngine.getDatabaseStats(); - return { - kuzuDBStatus: 'ready', - databaseStats: dbStats, - queryEngineReady: this.kuzuQueryEngine.isReady() - }; - } - - /** - * Close the orchestrator - */ - async close(): Promise { - await this.kuzuQueryEngine.close(); - } -} - -================ -File: src/ai/langchain-orchestrator.ts -================ -import { createReactAgent } from '@langchain/langgraph/prebuilt'; -import { MemorySaver } from '@langchain/langgraph'; -import { DynamicStructuredTool } from '@langchain/core/tools'; -import { z } from 'zod'; -import { SystemMessage } from '@langchain/core/messages'; -import type { LLMService, LLMConfig } from './llm-service.ts'; -import type { CypherGenerator } from './cypher-generator.ts'; -import type { KnowledgeGraph } from '../core/graph/types.ts'; -import type { GraphNode } from '../core/graph/types.ts'; - -export interface LangChainRAGContext { - graph: KnowledgeGraph; - fileContents: Map; -} - -export interface ToolCall { - tool: string; - input: Record; - output: string; -} - -export interface LangChainRAGResponse { - answer: string; - sources: string[]; - confidence: number; - toolCalls: ToolCall[]; -} - -export interface LangChainRAGOptions { - maxIterations?: number; - temperature?: number; - enableMemory?: boolean; - threadId?: string; -} - -type AgentType = ReturnType; -type MemoryType = InstanceType; - -export class LangChainRAGOrchestrator { - private llmService: LLMService; - private cypherGenerator: CypherGenerator; - private context: LangChainRAGContext | null = null; - private agent: AgentType | null = null; - private memory: MemoryType | null = null; - - constructor(llmService: LLMService, cypherGenerator: CypherGenerator) { - this.llmService = llmService; - this.cypherGenerator = cypherGenerator; - this.memory = new MemorySaver(); - } - - /** - * Set the current context and initialize the agent - */ - public async setContext(context: LangChainRAGContext, llmConfig: LLMConfig): Promise { - this.context = context; - this.cypherGenerator.updateSchema(context.graph); - - // Create LangChain-compliant tools - const tools = this.createTools(llmConfig); - - // Get the LLM from our service - const llm = this.llmService.getChatModel(llmConfig); - - // Create system message for ReAct behavior - const systemMessage = this.buildSystemMessage(); - - // Create the ReAct agent using LangGraph - const graph = createReactAgent({ - llm: llm as any, - tools: tools as any, - // Optional system message - messageModifier: systemMessage as any - }); - - this.agent = graph; - } - - /** - * Answer a question using the LangChain ReAct agent - */ - public async answerQuestion( - question: string, - options: LangChainRAGOptions = {} - ): Promise { - if (!this.agent || !this.context) { - throw new Error('Agent not initialized. Call setContext() first.'); - } - - const { - maxIterations = 10, - enableMemory = false, - threadId = 'default' - } = options; - - try { - const config = enableMemory - ? { - configurable: { thread_id: threadId }, - recursionLimit: maxIterations - } - : { recursionLimit: maxIterations }; - - // Stream the agent execution - const stream = await this.agent.stream( - { messages: [{ role: "user", content: question }] }, - config - ); - - let finalAnswer = ''; - const toolCalls: ToolCall[] = []; - const sources: string[] = []; - - // Process the stream - for await (const chunk of stream) { - if (chunk.agent) { - finalAnswer = chunk.agent.messages[chunk.agent.messages.length - 1].content; - } - - if (chunk.tools) { - const toolMessage = chunk.tools.messages[chunk.tools.messages.length - 1]; - if (toolMessage.tool_calls) { - toolMessage.tool_calls.forEach((toolCall: { name: string; args: Record }) => { - toolCalls.push({ - tool: toolCall.name, - input: toolCall.args, - output: toolMessage.content || '' - }); - }); - } - } - } - - // Calculate confidence based on successful tool usage - const confidence = this.calculateConfidence(toolCalls); - - return { - answer: finalAnswer || 'I was unable to find a complete answer to your question.', - sources: Array.from(new Set(sources)), - confidence, - toolCalls - }; - - } catch (error) { - throw new Error(`LangChain RAG orchestration failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Create LangChain-compliant tools - */ - private createTools(llmConfig: LLMConfig) { - const queryGraphTool = new DynamicStructuredTool({ - name: "query_graph", - description: "Query the code knowledge graph using natural language. Use this to find information about code structure, relationships, functions, classes, etc.", - schema: z.object({ - question: z.string().describe("Natural language question about the codebase") - }), - func: async (input: { question: string }) => { - try { - const cypherQuery = await this.cypherGenerator.generateQuery(input.question, llmConfig); - const mockResults = await this.executeGraphQuery(cypherQuery.cypher); - - return `Query: ${cypherQuery.cypher}\n\nResults:\n${mockResults}\n\nExplanation: ${cypherQuery.explanation}`; - } catch (error) { - return `Error generating graph query: ${error instanceof Error ? error.message : 'Unknown error'}`; - } - } - }); - - const getCodeContentTool = new DynamicStructuredTool({ - name: "get_code_content", - description: "Retrieve the source code content of a specific file. Use this when you need to examine the actual code implementation.", - schema: z.object({ - filePath: z.string().describe("The path to the file whose content you want to retrieve") - }), - func: async (input: { filePath: string }) => { - if (!this.context) { - return 'No context available'; - } - - const content = this.context.fileContents.get(input.filePath); - if (!content) { - // Try to find similar file paths - const similarFiles = Array.from(this.context.fileContents.keys()) - .filter(path => path.includes(input.filePath) || input.filePath.includes(path)) - .slice(0, 3); - - if (similarFiles.length > 0) { - return `File not found. Similar files available: ${similarFiles.join(', ')}`; - } - return 'File not found'; - } - - return `File: ${input.filePath}\n\n${content}`; - } - }); - - const searchFilesTool = new DynamicStructuredTool({ - name: "search_files", - description: "Search for files matching a pattern or containing specific text. Use this to find relevant files in the codebase.", - schema: z.object({ - pattern: z.string().describe("Search pattern or text to look for in file names or content") - }), - func: async (input: { pattern: string }) => { - if (!this.context) { - return 'No context available'; - } - - const matchingFiles: string[] = []; - const lowerPattern = input.pattern.toLowerCase(); - - // Search in file paths - for (const filePath of this.context.fileContents.keys()) { - if (filePath.toLowerCase().includes(lowerPattern)) { - matchingFiles.push(filePath); - } - } - - // Search in file contents - for (const [filePath, content] of this.context.fileContents.entries()) { - if (!matchingFiles.includes(filePath) && - content.toLowerCase().includes(lowerPattern)) { - matchingFiles.push(filePath); - } - } - - return matchingFiles.length > 0 - ? `Found ${matchingFiles.length} files:\n${matchingFiles.slice(0, 10).join('\n')}${matchingFiles.length > 10 ? '\n... and more' : ''}` - : 'No files found matching the pattern'; - } - }); - - return [queryGraphTool, getCodeContentTool, searchFilesTool]; - } - - /** - * Build system message for ReAct behavior - */ - private buildSystemMessage(): SystemMessage { - const systemPrompt = `You are an expert code analyst that helps users understand codebases by using available tools. - -You have access to the following tools: -1. query_graph: Query the code knowledge graph using natural language -2. get_code_content: Retrieve the source code content of a specific file -3. search_files: Search for files matching a pattern or containing specific text - -IMPORTANT GUIDELINES: -- Always think step by step about what information you need -- Use tools to gather factual information before providing answers -- Base your responses ONLY on information retrieved from tools -- If you cannot find information, say so explicitly -- Provide helpful, accurate answers about the codebase structure and functionality -- When referencing code, always cite the specific files you examined - -Your goal is to provide accurate, evidence-based answers about the codebase using the available tools.`; - - return new SystemMessage(systemPrompt); - } - - /** - * Execute graph query using the real GraphQueryEngine - */ - private async executeGraphQuery(cypher: string): Promise { - if (!this.context) return 'No context available'; - - try { - // Import and use the real GraphQueryEngine - const { GraphQueryEngine } = await import('../core/graph/query-engine.ts'); - const queryEngine = new GraphQueryEngine(this.context.graph); - - // Execute the actual Cypher query - const queryResult = queryEngine.executeQuery(cypher, { limit: 10 }); - - // Format the results for the LLM - if (queryResult.data.length > 0) { - return queryResult.data.map((row, index) => { - const entries = Object.entries(row); - if (entries.length === 0) return `Result ${index + 1}: (no data)`; - - return entries.map(([key, value]) => { - if (typeof value === 'object' && value !== null && 'properties' in value) { - // This is a node object - const node = value as GraphNode; - return `${key}: ${node.label} "${node.properties.name || node.properties.filePath || node.id}"`; - } - return `${key}: ${value}`; - }).join(', '); - }).join('\n'); - } else { - return 'No results found'; - } - } catch (error) { - console.error('Graph query execution failed:', error); - return `Query execution failed: ${error instanceof Error ? error.message : 'Unknown error'}`; - } - } - - /** - * Calculate confidence based on tool usage - */ - private calculateConfidence(toolCalls: ToolCall[]): number { - if (toolCalls.length === 0) return 0.3; - - const successfulCalls = toolCalls.filter(call => - !call.output.includes('Error') && - !call.output.includes('not found') && - call.output.length > 10 - ).length; - - const baseConfidence = 0.5; - const toolBonus = (successfulCalls / toolCalls.length) * 0.4; - - return Math.min(0.95, Math.max(0.1, baseConfidence + toolBonus)); - } - - /** - * Get current context information - */ - public getContextInfo(): { nodeCount: number; fileCount: number; hasContext: boolean; hasAgent: boolean } { - return { - nodeCount: this.context?.graph.nodes.length || 0, - fileCount: this.context?.fileContents.size || 0, - hasContext: !!this.context, - hasAgent: !!this.agent - }; - } - - /** - * Clear memory for a specific thread - */ - public async clearMemory(threadId: string): Promise { - if (this.memory) { - // Note: MemorySaver doesn't have a direct clear method in the current API - // This would need to be implemented based on the specific memory backend - console.log(`Memory clearing not implemented for thread: ${threadId}`); - } - } -} - -================ -File: src/ai/llm-service.ts -================ -import { ChatOpenAI } from '@langchain/openai'; -import { AzureChatOpenAI } from '@langchain/openai'; -import { ChatAnthropic } from '@langchain/anthropic'; -import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; -import type { BaseMessage } from '@langchain/core/messages'; -import type { BaseChatModel } from '@langchain/core/language_models/chat_models'; -import { HumanMessage } from '@langchain/core/messages'; - -export type LLMProvider = 'openai' | 'azure-openai' | 'anthropic' | 'gemini'; - -export interface LLMConfig { - provider: LLMProvider; - apiKey: string; - model?: string; - temperature?: number; - maxTokens?: number; - maxRetries?: number; - // Azure OpenAI specific fields - azureOpenAIEndpoint?: string; - azureOpenAIApiVersion?: string; - azureOpenAIDeploymentName?: string; -} - -export interface ChatResponse { - content: string; - usage?: { - promptTokens: number; - completionTokens: number; - totalTokens: number; - }; - model?: string; - finishReason?: string; -} - -export class LLMService { - private models: Map = new Map(); - private defaultConfig: Partial = { - temperature: 0.1, - maxTokens: 4000, - maxRetries: 3, - azureOpenAIApiVersion: '2024-02-01' // Default Azure OpenAI API version - }; - - // Default models for each provider - private static readonly DEFAULT_MODELS: Record = { - openai: 'gpt-4o-mini', - 'azure-openai': 'gpt-4o-mini', - anthropic: 'claude-3-haiku-20240307', - gemini: 'gemini-2.5-flash' // Use the latest 2.5 Flash model as default (2025) - }; - - constructor() {} - - /** - * Initialize or get a chat model for the specified provider - */ - public getChatModel(config: LLMConfig): any { - const cacheKey = this.getCacheKey(config); - - if (this.models.has(cacheKey)) { - return this.models.get(cacheKey)!; - } - - const model = this.createChatModel(config); - this.models.set(cacheKey, model); - return model; - } - - /** - * Send a chat message and get a response - */ - public async chat( - config: LLMConfig, - messages: BaseMessage[], - options?: { stream?: boolean } - ): Promise { - try { - const model = this.getChatModel(config); - - if (options?.stream) { - // For streaming, we'd need to handle this differently - // For now, we'll just use regular invoke - console.warn('Streaming not implemented yet, falling back to regular invoke'); - } - - const response = await model.invoke(messages); - - return { - content: response.content as string, - usage: response.response_metadata?.usage ? { - promptTokens: response.response_metadata.usage.prompt_tokens || 0, - completionTokens: response.response_metadata.usage.completion_tokens || 0, - totalTokens: response.response_metadata.usage.total_tokens || 0 - } : undefined, - model: config.model || LLMService.DEFAULT_MODELS[config.provider], - finishReason: response.response_metadata?.finish_reason - }; - } catch (error) { - throw new Error(`LLM chat failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Validate API key format for different providers - */ - public validateApiKey(provider: LLMProvider, apiKey: string): boolean { - if (!apiKey || apiKey.trim().length === 0) { - return false; - } - - switch (provider) { - case 'openai': - return apiKey.startsWith('sk-') && apiKey.length > 20; - case 'azure-openai': - // Azure OpenAI keys are typically 32 characters long and don't have a specific prefix - return apiKey.length >= 20; // More flexible validation for Azure keys - case 'anthropic': - return apiKey.startsWith('sk-ant-') && apiKey.length > 20; - case 'gemini': - return apiKey.length > 20; // Google API keys don't have a consistent prefix - default: - return false; - } - } - - /** - * Validate Azure OpenAI configuration - */ - public validateAzureOpenAIConfig(config: LLMConfig): { valid: boolean; error?: string } { - if (config.provider !== 'azure-openai') { - return { valid: false, error: 'Provider must be azure-openai' }; - } - - if (!config.azureOpenAIEndpoint) { - return { valid: false, error: 'Azure OpenAI endpoint is required' }; - } - - if (!config.azureOpenAIEndpoint.includes('openai.azure.com')) { - return { valid: false, error: 'Invalid Azure OpenAI endpoint format. Should contain "openai.azure.com"' }; - } - - if (!config.azureOpenAIDeploymentName) { - return { valid: false, error: 'Azure OpenAI deployment name is required' }; - } - - return { valid: true }; - } - - /** - * Get available models for a provider - */ - public getAvailableModels(provider: LLMProvider): string[] { - switch (provider) { - case 'openai': - return [ - 'gpt-4o', - 'gpt-4o-mini', - 'gpt-4-turbo', - 'gpt-4', - 'gpt-3.5-turbo' - ]; - case 'azure-openai': - return [ - 'gpt-4o', - 'gpt-4o-mini', - 'gpt-4.1-mini-v2', // Common deployment name - 'gpt-4-turbo', - 'gpt-4', - 'gpt-35-turbo', // Note: Azure uses gpt-35-turbo instead of gpt-3.5-turbo - 'gpt-4-32k' - ]; - case 'anthropic': - return [ - 'claude-3-5-sonnet-20241022', - 'claude-3-5-haiku-20241022', - 'claude-3-opus-20240229', - 'claude-3-sonnet-20240229', - 'claude-3-haiku-20240307' - ]; - case 'gemini': - return [ - 'gemini-2.5-flash', // Latest and fastest (2025) - NEW DEFAULT - 'gemini-2.5-pro', // Latest pro model (2025) - PREMIUM - 'gemini-1.5-flash', // Most stable and widely available - 'gemini-1.5-pro', // Stable pro model - 'gemini-1.0-pro', // Legacy but very stable - 'gemini-1.5-flash-8b', // Smaller, efficient version - 'gemini-2.0-flash', // Newer model (may not be available to all users) - 'gemini-2.0-flash-lite' // Lightweight version - ]; - default: - return []; - } - } - - /** - * Get provider display name - */ - public getProviderDisplayName(provider: LLMProvider): string { - switch (provider) { - case 'openai': - return 'OpenAI'; - case 'azure-openai': - return 'Azure OpenAI'; - case 'anthropic': - return 'Anthropic'; - case 'gemini': - return 'Google Gemini'; - default: - return provider; - } - } - - /** - * Test connection with the provider - */ - public async testConnection(config: LLMConfig): Promise<{ success: boolean; error?: string }> { - try { - // Validate Azure OpenAI config if needed - if (config.provider === 'azure-openai') { - const validation = this.validateAzureOpenAIConfig(config); - if (!validation.valid) { - return { - success: false, - error: validation.error - }; - } - } - - const model = this.createChatModel(config); - - // Send a simple test message - const testMessages = [ - new HumanMessage("Test connection") - ]; - - await model.invoke(testMessages); - return { success: true }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Connection test failed' - }; - } - } - - /** - * Clear cached models (useful for updating API keys) - */ - public clearCache(): void { - this.models.clear(); - } - - /** - * Create a chat model instance based on the provider - */ - private createChatModel(config: LLMConfig): any { - const mergedConfig = { ...this.defaultConfig, ...config }; - const model = mergedConfig.model || LLMService.DEFAULT_MODELS[config.provider]; - - switch (config.provider) { - case 'openai': - return new ChatOpenAI({ - apiKey: config.apiKey, - model, - temperature: mergedConfig.temperature, - maxTokens: mergedConfig.maxTokens, - maxRetries: mergedConfig.maxRetries, - timeout: 30000 - }); - - case 'azure-openai': - return new AzureChatOpenAI({ - azureOpenAIApiKey: config.apiKey, - model: config.azureOpenAIDeploymentName, // Use deployment name as model - temperature: mergedConfig.temperature, - maxTokens: mergedConfig.maxTokens, - maxRetries: mergedConfig.maxRetries, - timeout: 30000, - azureOpenAIApiInstanceName: config.azureOpenAIEndpoint?.replace('https://', '').split('.')[0], - azureOpenAIApiVersion: config.azureOpenAIApiVersion, - azureOpenAIApiDeploymentName: config.azureOpenAIDeploymentName - }); - - case 'anthropic': - return new ChatAnthropic({ - model: config.model || 'claude-3-sonnet-20240229', - anthropicApiKey: config.apiKey, - maxTokens: config.maxTokens || 4096, - temperature: config.temperature || 0.7 - }); - - case 'gemini': - return new ChatGoogleGenerativeAI({ - apiKey: config.apiKey, - model, - temperature: mergedConfig.temperature, - maxOutputTokens: mergedConfig.maxTokens, - maxRetries: mergedConfig.maxRetries - }); - - default: - throw new Error(`Unsupported provider: ${config.provider}`); - } - } - - /** - * Generate cache key for model instances - */ - private getCacheKey(config: LLMConfig): string { - const model = config.model || LLMService.DEFAULT_MODELS[config.provider]; - let baseKey = `${config.provider}:${model}:${config.apiKey.slice(-8)}:${config.temperature}:${config.maxTokens}`; - - // Add Azure OpenAI specific fields to cache key - if (config.provider === 'azure-openai') { - baseKey += `:${config.azureOpenAIEndpoint}:${config.azureOpenAIDeploymentName}:${config.azureOpenAIApiVersion}`; - } - - return baseKey; - } -} - -================ -File: src/ai/orchestrator.ts -================ -import { HumanMessage, SystemMessage, AIMessage } from '@langchain/core/messages'; -import type { LLMService, LLMConfig } from './llm-service.ts'; -import type { CypherGenerator, CypherQuery } from './cypher-generator.ts'; -import type { KnowledgeGraph } from '../core/graph/types.ts'; -import type { GraphNode } from '../core/graph/types.ts'; - -export interface RAGContext { - graph: KnowledgeGraph; - fileContents: Map; -} - -export interface ToolResult { - toolName: string; - input: string; - output: string; - success: boolean; - error?: string; -} - -export interface ReasoningStep { - step: number; - thought: string; - action: string; - actionInput: string; - observation: string; - toolResult?: ToolResult; -} - -export interface RAGResponse { - answer: string; - reasoning: ReasoningStep[]; - cypherQueries: CypherQuery[]; - confidence: number; - sources: string[]; -} - -export interface RAGOptions { - maxReasoningSteps?: number; - includeReasoning?: boolean; - strictMode?: boolean; - temperature?: number; -} - -export class RAGOrchestrator { - private llmService: LLMService; - private cypherGenerator: CypherGenerator; - private context: RAGContext | null = null; - - constructor(llmService: LLMService, cypherGenerator: CypherGenerator) { - this.llmService = llmService; - this.cypherGenerator = cypherGenerator; - } - - /** - * Set the current context (graph and file contents) - */ - public setContext(context: RAGContext): void { - this.context = context; - this.cypherGenerator.updateSchema(context.graph); - } - - /** - * Answer a question using ReAct pattern - */ - public async answerQuestion( - question: string, - llmConfig: LLMConfig, - options: RAGOptions = {} - ): Promise { - if (!this.context) { - throw new Error('Context not set. Call setContext() first.'); - } - - const { - maxReasoningSteps = 5, - includeReasoning = true, - strictMode = false, - temperature = 0.1 - } = options; - - const reasoning: ReasoningStep[] = []; - const cypherQueries: CypherQuery[] = []; - const sources: string[] = []; - - // Enhanced LLM config for reasoning - const reasoningConfig: LLMConfig = { - ...llmConfig, - temperature: temperature - }; - - let currentStep = 1; - let finalAnswer = ''; - let confidence = 0.5; - - try { - // Initial system prompt for ReAct - const systemPrompt = this.buildReActSystemPrompt(strictMode); - const conversation = [new SystemMessage(systemPrompt)]; - - // Add the user question - conversation.push(new HumanMessage(`Question: ${question}`)); - - while (currentStep <= maxReasoningSteps) { - // Get reasoning from LLM - const response = await this.llmService.chat(reasoningConfig, conversation); - const reasoning_step = this.parseReasoningStep(String(response.content || ''), currentStep); - - reasoning.push(reasoning_step); - - // Check if we have a final answer - if (reasoning_step.action.toLowerCase().includes('final_answer')) { - finalAnswer = reasoning_step.actionInput; - confidence = this.calculateConfidence(reasoning, cypherQueries); - break; - } - - // Execute the action - let toolResult: ToolResult | null = null; - - try { - if (reasoning_step.action.toLowerCase().includes('query_graph')) { - toolResult = await this.executeGraphQuery(reasoning_step.actionInput, reasoningConfig); - if (toolResult.success) { - cypherQueries.push(...this.extractCypherQueries(toolResult)); - } - } else if (reasoning_step.action.toLowerCase().includes('get_code')) { - toolResult = await this.getCodeContent(reasoning_step.actionInput); - if (toolResult.success) { - sources.push(...this.extractSources(toolResult)); - } - } else if (reasoning_step.action.toLowerCase().includes('search_files')) { - toolResult = await this.searchFiles(reasoning_step.actionInput); - } else { - toolResult = { - toolName: 'unknown', - input: reasoning_step.actionInput, - output: 'Unknown action type', - success: false, - error: `Unknown action: ${reasoning_step.action}` - }; - } - } catch (error) { - toolResult = { - toolName: reasoning_step.action, - input: reasoning_step.actionInput, - output: '', - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }; - } - - // Update the reasoning step with tool result - reasoning_step.observation = toolResult.output; - reasoning_step.toolResult = toolResult; - - // Add the tool result to conversation - conversation.push(new AIMessage(String(response.content || ''))); - conversation.push(new HumanMessage(`Observation: ${toolResult.output}`)); - - currentStep++; - } - - // If we didn't get a final answer, generate one based on the reasoning - if (!finalAnswer && reasoning.length > 0) { - const summaryPrompt = this.buildSummaryPrompt(question, reasoning); - conversation.push(new HumanMessage(summaryPrompt)); - - const summaryResponse = await this.llmService.chat(reasoningConfig, conversation); - finalAnswer = String(summaryResponse.content || ''); - confidence = Math.max(0.3, confidence - 0.2); // Lower confidence for incomplete reasoning - } - - return { - answer: finalAnswer || 'I was unable to find a complete answer to your question.', - reasoning: includeReasoning ? reasoning : [], - cypherQueries, - confidence, - sources: Array.from(new Set(sources)) // Remove duplicates - }; - - } catch (error) { - throw new Error(`RAG orchestration failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Build the ReAct system prompt - */ - private buildReActSystemPrompt(strictMode: boolean): string { - const prompt = `You are an expert code analyst using a ReAct (Reasoning + Acting) approach to answer questions about a codebase. - -You have access to the following tools: -1. query_graph(question): Query the code knowledge graph using natural language -2. get_code(file_path): Retrieve the source code content of a specific file -3. search_files(pattern): Search for files matching a pattern or containing specific text - -IMPORTANT INSTRUCTIONS: -- Always think step by step using the format: Thought: [your reasoning] -- Then specify an action using: Action: [tool_name] -- Provide the input using: Action Input: [input for the tool] -- After receiving an observation, continue reasoning or provide a final answer -- Use Final Answer: [your answer] when you have sufficient information -- Base your answers ONLY on the information retrieved from tools -- Do not make assumptions or hallucinate information -- If you cannot find information, say so explicitly - -RESPONSE FORMAT: -Thought: [Your reasoning about what to do next] -Action: [query_graph, get_code, search_files, or Final Answer] -Action Input: [The input for the action] - -After receiving an Observation, continue with: -Thought: [Your analysis of the observation] -Action: [Next action or Final Answer] -Action Input: [Input for next action or your final answer] - -FINAL ANSWER FORMATTING: -When providing your final answer, use markdown formatting for better readability: -- Use **bold** for important terms and concepts -- Use \`inline code\` for function names, file names, and code snippets -- Use code blocks with language specification for longer code examples -- Use bullet points or numbered lists for structured information -- Use headers (##, ###) to organize complex answers -- Use tables when presenting structured data - -Example final answer format: -## Summary -The codebase contains **15 functions** across **3 files**. - -### Key Functions: -- \`authenticate()\` - Handles user authentication -- \`process_data()\` - Main data processing logic -- \`save_results()\` - Saves processed data to database - -${strictMode ? '\nSTRICT MODE: Only use information explicitly found in the tools. Do not infer or assume anything.' : ''} - -Remember: Your goal is to provide accurate, evidence-based, and well-formatted answers about the codebase.`; - - return prompt; - } - - /** - * Parse a reasoning step from LLM response - */ - private parseReasoningStep(response: string, stepNumber: number): ReasoningStep { - // Ensure response is a string - const responseText = typeof response === 'string' ? response : String(response || ''); - - const thoughtMatch = responseText.match(/Thought:\s*(.*?)(?=\n|Action:|$)/s); - const actionMatch = responseText.match(/Action:\s*(.*?)(?=\n|Action Input:|$)/s); - const inputMatch = responseText.match(/Action Input:\s*(.*?)(?=\n|$)/s); - - return { - step: stepNumber, - thought: thoughtMatch?.[1]?.trim() || 'No thought provided', - action: actionMatch?.[1]?.trim() || 'unknown', - actionInput: inputMatch?.[1]?.trim() || '', - observation: '' // Will be filled after tool execution - }; - } - - /** - * Execute a graph query using KuzuDB when available, fallback to in-memory - */ - private async executeGraphQuery(question: string, llmConfig: LLMConfig): Promise { - try { - const cypherQuery = await this.cypherGenerator.generateQuery(question, llmConfig); - - // Try to use KuzuDB if available - try { - const { isKuzuDBEnabled } = await import('../config/feature-flags.js'); - if (isKuzuDBEnabled()) { - const { KuzuQueryEngine } = await import('../core/graph/kuzu-query-engine.js'); - const kuzuQueryEngine = new KuzuQueryEngine(); - - if (kuzuQueryEngine.isReady()) { - const kuzuResult = await kuzuQueryEngine.executeQuery(cypherQuery.cypher, { - timeout: 30000, - maxResults: 100, - includeExecutionTime: true - }); - - const formattedResults = this.formatKuzuResults(kuzuResult); - - return { - toolName: 'kuzu_query_graph', - input: question, - output: `Query: ${cypherQuery.cypher}\n\nResults (${kuzuResult.nodes.length} nodes, ${kuzuResult.relationships.length} relationships, execution time: ${kuzuResult.executionTime.toFixed(2)}ms):\n${formattedResults}\n\nExplanation: ${cypherQuery.explanation}`, - success: true - }; - } - } - } catch (kuzuError) { - console.warn('KuzuDB query failed, falling back to in-memory:', kuzuError); - } - - // Fallback to in-memory GraphQueryEngine - const { GraphQueryEngine } = await import('../core/graph/query-engine.ts'); - const queryEngine = new GraphQueryEngine(this.context!.graph); - - // Execute the actual Cypher query - const queryResult = queryEngine.executeQuery(cypherQuery.cypher, { limit: 10 }); - - // Format the results for the LLM - let formattedResults = ''; - if (queryResult.data.length > 0) { - formattedResults = queryResult.data.map((row, index) => { - const entries = Object.entries(row); - if (entries.length === 0) return `Result ${index + 1}: (no data)`; - - return entries.map(([key, value]) => { - if (typeof value === 'object' && value !== null && 'properties' in value) { - // This is a node object - const node = value as GraphNode; - return `${key}: ${node.label} "${node.properties.name || node.properties.filePath || node.id}"`; - } - return `${key}: ${value}`; - }).join(', '); - }).join('\n'); - } else { - formattedResults = 'No results found'; - } - - return { - toolName: 'query_graph', - input: question, - output: `Query: ${cypherQuery.cypher}\n\nResults (${queryResult.data.length} found):\n${formattedResults}\n\nExplanation: ${cypherQuery.explanation}`, - success: true - }; - } catch (error) { - return { - toolName: 'query_graph', - input: question, - output: '', - success: false, - error: error instanceof Error ? error.message : 'Query execution failed' - }; - } - } - - /** - * Format KuzuDB query results for display - */ - private formatKuzuResults(result: { nodes: Array<{ label: string; properties: Record; id: string }>; relationships: Array<{ type: string; source: string; target: string }> }): string { - const nodeResults = result.nodes.slice(0, 10).map((node, index) => - `${index + 1}. ${node.label}: "${(node.properties.name as string) || node.id}"` - ).join('\n'); - - const relResults = result.relationships.slice(0, 10).map((rel, index) => - `${index + 1}. ${rel.type}: ${rel.source} -> ${rel.target}` - ).join('\n'); - - let output = ''; - if (result.nodes.length > 0) { - output += `Nodes (${result.nodes.length}):\n${nodeResults}`; - if (result.nodes.length > 10) output += '\n... (more nodes)'; - } - - if (result.relationships.length > 0) { - if (output) output += '\n\n'; - output += `Relationships (${result.relationships.length}):\n${relResults}`; - if (result.relationships.length > 10) output += '\n... (more relationships)'; - } - - return output || 'No results found'; - } - - /** - * Get code content from a file - */ - private async getCodeContent(filePath: string): Promise { - if (!this.context) { - return { - toolName: 'get_code', - input: filePath, - output: '', - success: false, - error: 'No context available' - }; - } - - const content = this.context.fileContents.get(filePath); - if (!content) { - // Try to find similar file paths - const similarFiles = Array.from(this.context.fileContents.keys()) - .filter(path => path.includes(filePath) || filePath.includes(path)) - .slice(0, 3); - - if (similarFiles.length > 0) { - return { - toolName: 'get_code', - input: filePath, - output: `File not found. Similar files available: ${similarFiles.join(', ')}`, - success: false, - error: 'File not found' - }; - } - - return { - toolName: 'get_code', - input: filePath, - output: 'File not found', - success: false, - error: 'File not found' - }; - } - - return { - toolName: 'get_code', - input: filePath, - output: `File: ${filePath}\n\n${content}`, - success: true - }; - } - - /** - * Search for files matching a pattern - */ - private async searchFiles(pattern: string): Promise { - if (!this.context) { - return { - toolName: 'search_files', - input: pattern, - output: '', - success: false, - error: 'No context available' - }; - } - - const matchingFiles: string[] = []; - const lowerPattern = pattern.toLowerCase(); - - // Search in file paths - for (const filePath of this.context.fileContents.keys()) { - if (filePath.toLowerCase().includes(lowerPattern)) { - matchingFiles.push(filePath); - } - } - - // Search in file contents - for (const [filePath, content] of this.context.fileContents.entries()) { - if (!matchingFiles.includes(filePath) && - content.toLowerCase().includes(lowerPattern)) { - matchingFiles.push(filePath); - } - } - - return { - toolName: 'search_files', - input: pattern, - output: matchingFiles.length > 0 - ? `Found ${matchingFiles.length} files:\n${matchingFiles.slice(0, 10).join('\n')}${matchingFiles.length > 10 ? '\n... and more' : ''}` - : 'No files found matching the pattern', - success: true - }; - } - - /** - * Extract Cypher queries from tool results - */ - private extractCypherQueries(toolResult: ToolResult): CypherQuery[] { - const queries: CypherQuery[] = []; - const queryMatch = toolResult.output.match(/Query: (.*?)(?=\n|$)/); - - if (queryMatch) { - queries.push({ - cypher: queryMatch[1], - explanation: 'Generated during reasoning', - confidence: 0.8 - }); - } - - return queries; - } - - /** - * Extract sources from tool results - */ - private extractSources(toolResult: ToolResult): string[] { - const sources: string[] = []; - const fileMatch = toolResult.output.match(/File: (.*?)(?=\n|$)/); - - if (fileMatch) { - sources.push(fileMatch[1]); - } - - return sources; - } - - /** - * Calculate confidence based on reasoning quality - */ - private calculateConfidence(reasoning: ReasoningStep[], queries: CypherQuery[]): number { - let confidence = 0.5; - - // Boost confidence for successful tool usage - const successfulSteps = reasoning.filter(step => step.toolResult?.success).length; - confidence += (successfulSteps / reasoning.length) * 0.3; - - // Boost confidence for high-quality queries - const avgQueryConfidence = queries.length > 0 - ? queries.reduce((sum, q) => sum + q.confidence, 0) / queries.length - : 0.5; - confidence += avgQueryConfidence * 0.2; - - // Cap at reasonable bounds - return Math.min(0.95, Math.max(0.1, confidence)); - } - - /** - * Build summary prompt for incomplete reasoning - */ - private buildSummaryPrompt(question: string, reasoning: ReasoningStep[]): string { - const observations = reasoning - .map(step => `Step ${step.step}: ${step.observation}`) - .join('\n'); - - return `Based on the following observations from your reasoning process, please provide a final answer to the question: "${question}" - -Observations: -${observations} - -Final Answer:`; - } - - /** - * Get current context information - */ - public getContextInfo(): { nodeCount: number; fileCount: number; hasContext: boolean } { - if (!this.context) { - return { nodeCount: 0, fileCount: 0, hasContext: false }; - } - - return { - nodeCount: this.context.graph.nodes.length, - fileCount: this.context.fileContents.size, - hasContext: true - }; - } - - /** - * Test the orchestrator with a simple query (for debugging) - */ - public async testOrchestrator(llmConfig: LLMConfig): Promise<{ success: boolean; error?: string; response?: RAGResponse }> { - if (!this.context) { - return { success: false, error: 'No context set' }; - } - - try { - const testQuestion = "How many functions are in this project?"; - const response = await this.answerQuestion(testQuestion, llmConfig, { - maxReasoningSteps: 2, - includeReasoning: true - }); - - return { success: true, response }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }; - } - } -} - -================ -File: src/ai/prompts/kuzu-performance-prompts.ts -================ -/** - * KuzuDB Performance-Optimized Prompts - * Specialized prompts for leveraging KuzuDB's strengths - */ - -export const KUZU_PERFORMANCE_PROMPTS = { - /** - * System prompt for performance-focused queries - */ - PERFORMANCE_SYSTEM: `You are a Cypher query expert specializing in high-performance graph database queries using KuzuDB. Your goal is to generate optimized queries that leverage KuzuDB's strengths. - -KUZUDB OPTIMIZATION PRINCIPLES: -1. COMPLEX TRAVERSALS: KuzuDB excels at variable-length path queries -2. PATTERN MATCHING: Use sophisticated WHERE clauses for filtering -3. AGGREGATION: Leverage COUNT, COLLECT, and other aggregation functions -4. INDEXING: Prefer queries that can use node property indexes -5. BATCHING: Structure queries to minimize round trips - -PERFORMANCE PATTERNS: -- Use (start)-[:RELATIONSHIP*1..5]->(end) for dependency chains -- Leverage WHERE clauses with CONTAINS for text search -- Use aggregation for statistics: COUNT, COLLECT, AVG -- Prefer specific node types over generic matches -- Use LIMIT clauses to control result size - -QUERY TYPES TO OPTIMIZE FOR: -1. Dependency Analysis: Find all functions that depend on a target -2. Call Chain Traversal: Follow function call chains -3. Pattern Matching: Find code patterns across the codebase -4. Statistical Analysis: Count and summarize code metrics -5. Relationship Exploration: Discover connections between entities - -Always consider execution time and result relevance when generating queries.`, - - /** - * Prompt for complex dependency analysis - */ - DEPENDENCY_ANALYSIS: `Generate a Cypher query for dependency analysis that leverages KuzuDB's strength in variable-length path traversal. - -Focus on: -- Finding all dependencies of a specific function/class -- Identifying call chains and dependency trees -- Discovering indirect dependencies (2+ hops away) -- Analyzing dependency depth and complexity - -Use patterns like: -- (start)-[:CALLS*1..5]->(end) for call chains -- (start)-[:IMPORTS*1..3]->(end) for import dependencies -- WHERE clauses to filter by specific criteria -- Aggregation to summarize dependency statistics`, - - /** - * Prompt for performance monitoring queries - */ - PERFORMANCE_MONITORING: `Generate Cypher queries for monitoring and analyzing codebase performance metrics. - -Focus on: -- Counting entities by type (functions, classes, methods) -- Analyzing code complexity through relationship density -- Identifying performance bottlenecks in call chains -- Measuring code coupling and cohesion - -Use aggregation functions: -- COUNT() for entity counting -- COLLECT() for gathering lists -- AVG() for average metrics -- MAX()/MIN() for range analysis - -Structure queries to provide actionable performance insights.`, - - /** - * Prompt for code pattern discovery - */ - PATTERN_DISCOVERY: `Generate Cypher queries for discovering code patterns and architectural insights. - -Focus on: -- Finding similar code structures -- Identifying design patterns -- Discovering architectural relationships -- Analyzing code organization - -Use patterns like: -- Pattern matching with WHERE clauses -- Relationship traversal for structural analysis -- Aggregation for pattern frequency -- Variable-length paths for complex relationships - -Aim to reveal hidden patterns and architectural insights.` -}; - -/** - * Performance-focused query examples - */ -export const PERFORMANCE_QUERY_EXAMPLES = [ - { - question: "Find all functions that are called through a chain of 3-5 function calls from the main function", - cypher: "MATCH (main:Function {name: 'main'})-[:CALLS*3..5]->(target:Function) RETURN main.name, target.name, target.filePath", - explanation: "Uses variable-length path to find functions 3-5 calls away from main" - }, - { - question: "Count how many functions each class contains and show the most complex classes", - cypher: "MATCH (c:Class)-[:CONTAINS]->(f:Function) RETURN c.name, COUNT(f) as functionCount ORDER BY functionCount DESC LIMIT 10", - explanation: "Uses aggregation to count functions per class and orders by complexity" - }, - { - question: "Find all functions that are called by more than 5 other functions", - cypher: "MATCH (caller:Function)-[:CALLS]->(target:Function) WITH target, COUNT(caller) as callCount WHERE callCount > 5 RETURN target.name, callCount ORDER BY callCount DESC", - explanation: "Uses aggregation to find frequently called functions" - }, - { - question: "Show the dependency chain from authentication functions to database functions", - cypher: "MATCH (auth:Function)-[:CALLS*1..5]->(db:Function) WHERE auth.name CONTAINS 'auth' AND db.name CONTAINS 'db' RETURN auth.name, db.name", - explanation: "Uses variable-length path to trace authentication to database calls" - }, - { - question: "Find all classes that implement more than 2 interfaces", - cypher: "MATCH (c:Class)-[:IMPLEMENTS]->(i:Interface) WITH c, COUNT(i) as interfaceCount WHERE interfaceCount > 2 RETURN c.name, interfaceCount", - explanation: "Uses aggregation to find classes with multiple interface implementations" - } -]; - -/** - * Performance monitoring query templates - */ -export const PERFORMANCE_TEMPLATES = { - // Code complexity analysis - COMPLEXITY_ANALYSIS: ` - MATCH (f:Function) - OPTIONAL MATCH (f)-[:CALLS]->(called:Function) - WITH f, COUNT(called) as outgoingCalls - OPTIONAL MATCH (caller:Function)-[:CALLS]->(f) - WITH f, outgoingCalls, COUNT(caller) as incomingCalls - RETURN f.name, f.filePath, outgoingCalls, incomingCalls, (outgoingCalls + incomingCalls) as totalComplexity - ORDER BY totalComplexity DESC - LIMIT 20 - `, - - // Dependency depth analysis - DEPENDENCY_DEPTH: ` - MATCH (start:Function {name: $functionName})-[:CALLS*1..10]->(target:Function) - WITH target, LENGTH(shortestPath((start)-[:CALLS*]->(target))) as depth - RETURN target.name, target.filePath, depth - ORDER BY depth - `, - - // Code coupling analysis - COUPLING_ANALYSIS: ` - MATCH (f1:Function)-[:CALLS]->(f2:Function) - WHERE f1.filePath <> f2.filePath - WITH f1.filePath as file1, f2.filePath as file2, COUNT(*) as coupling - WHERE coupling > 5 - RETURN file1, file2, coupling - ORDER BY coupling DESC - `, - - // Architecture pattern detection - PATTERN_DETECTION: ` - MATCH (c:Class)-[:CONTAINS]->(m:Method) - WHERE m.name CONTAINS 'get' OR m.name CONTAINS 'set' - WITH c, COUNT(m) as accessorCount - WHERE accessorCount > 3 - RETURN c.name, c.filePath, accessorCount - ORDER BY accessorCount DESC - ` -}; - -================ -File: src/App.css -================ -#root { - max-width: 1280px; - margin: 0 auto; - padding: 2rem; - text-align: center; -} - -.logo { - height: 6em; - padding: 1.5em; - will-change: filter; - transition: filter 300ms; -} -.logo:hover { - filter: drop-shadow(0 0 2em #646cffaa); -} -.logo.react:hover { - filter: drop-shadow(0 0 2em #61dafbaa); -} - -@keyframes logo-spin { - from { - transform: rotate(0deg); - } - to { - transform: rotate(360deg); - } -} - -@media (prefers-reduced-motion: no-preference) { - a:nth-of-type(2) .logo { - animation: logo-spin infinite 20s linear; - } -} - -.card { - padding: 2em; -} - -.read-the-docs { - color: #888; -} - -================ -File: src/App.tsx -================ -import React from 'react'; -import { HomePage, ErrorBoundary } from './ui/index.ts'; - -const App: React.FC = () => { - return ( - - - - ); -}; - -export default App; - -================ -File: src/assets/react.svg -================ - - -================ -File: src/config/config.ts -================ -/** - * Centralized Configuration System - * Provides type-safe configuration with environment validation and defaults - */ - -import { z } from 'zod'; - -// Configuration schemas with validation -const MemoryConfigSchema = z.object({ - maxMemoryMB: z.number().min(100).max(2048).default(512), - cleanupThresholdMB: z.number().min(50).max(1024).default(400), - gcIntervalMs: z.number().min(5000).max(60000).default(30000), - maxFileSizeMB: z.number().min(1).max(50).default(10), - maxFilesInMemory: z.number().min(100).max(10000).default(1000) -}); - -const GitHubConfigSchema = z.object({ - apiUrl: z.string().url().default('https://api.github.com'), - token: z.string().optional(), - rateLimit: z.object({ - maxRequests: z.number().min(1).max(5000).default(60), - windowMs: z.number().min(1000).max(3600000).default(60000) - }), - retry: z.object({ - maxRetries: z.number().min(0).max(5).default(3), - backoffMs: z.number().min(100).max(10000).default(1000) - }) -}); - -const LLMConfigSchema = z.object({ - providers: z.object({ - openai: z.object({ - apiKey: z.string().optional(), - model: z.string().default('gpt-4'), - maxTokens: z.number().min(100).max(10000).default(2000), - temperature: z.number().min(0).max(2).default(0.7) - }).optional(), - azure: z.object({ - apiKey: z.string().optional(), - endpoint: z.string().url().optional(), - deployment: z.string().optional(), - maxTokens: z.number().min(100).max(10000).default(2000), - temperature: z.number().min(0).max(2).default(0.7) - }).optional(), - anthropic: z.object({ - apiKey: z.string().optional(), - model: z.string().default('claude-3-sonnet-20240229'), - maxTokens: z.number().min(100).max(10000).default(2000), - temperature: z.number().min(0).max(2).default(0.7) - }).optional(), - gemini: z.object({ - apiKey: z.string().optional(), - model: z.string().default('gemini-pro'), - maxTokens: z.number().min(100).max(10000).default(2000), - temperature: z.number().min(0).max(2).default(0.7) - }).optional() - }), - defaultProvider: z.enum(['openai', 'azure', 'anthropic', 'gemini']).default('openai') -}); - -const ProcessingConfigSchema = z.object({ - batchSize: z.number().min(1).max(100).default(10), - maxConcurrentRequests: z.number().min(1).max(50).default(5), - timeoutMs: z.number().min(1000).max(300000).default(30000), - retry: z.object({ - maxRetries: z.number().min(0).max(5).default(3), - backoffMs: z.number().min(100).max(10000).default(1000) - }), - fileExtensions: z.array(z.string()).default([ - '.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', - '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.dart', - '.json', '.yaml', '.yml', '.xml', '.toml', '.ini', '.cfg', '.properties' - ]) -}); - -const LoggingConfigSchema = z.object({ - level: z.enum(['debug', 'info', 'warn', 'error']).default('info'), - enableMetrics: z.boolean().default(true), - enablePerformanceTracking: z.boolean().default(true), - maxLogEntries: z.number().min(100).max(10000).default(1000) -}); - -// Main configuration schema -const AppConfigSchema = z.object({ - memory: MemoryConfigSchema, - github: GitHubConfigSchema, - llm: LLMConfigSchema, - processing: ProcessingConfigSchema, - logging: LoggingConfigSchema, - environment: z.enum(['development', 'staging', 'production']).default('development') -}); - -export type AppConfig = z.infer; -export type MemoryConfig = z.infer; -export type GitHubConfig = z.infer; -export type LLMConfig = z.infer; -export type ProcessingConfig = z.infer; -export type LoggingConfig = z.infer; - -/** - * Configuration service with environment validation - */ -export class ConfigService { - private static instance: ConfigService; - private config: AppConfig; - private validationErrors: string[] = []; - - private constructor() { - this.config = this.loadConfiguration(); - this.validateEnvironment(); - } - - public static getInstance(): ConfigService { - if (!ConfigService.instance) { - ConfigService.instance = new ConfigService(); - } - return ConfigService.instance; - } - - /** - * Load configuration from environment variables and defaults - */ - private loadConfiguration(): AppConfig { - try { - const config: AppConfig = { - memory: { - maxMemoryMB: this.getEnvNumber('MEMORY_MAX_MB', 512), - cleanupThresholdMB: this.getEnvNumber('MEMORY_CLEANUP_THRESHOLD_MB', 400), - gcIntervalMs: this.getEnvNumber('MEMORY_GC_INTERVAL_MS', 30000), - maxFileSizeMB: this.getEnvNumber('MEMORY_MAX_FILE_SIZE_MB', 10), - maxFilesInMemory: this.getEnvNumber('MEMORY_MAX_FILES', 1000) - }, - github: { - apiUrl: this.getEnvString('GITHUB_API_URL', 'https://api.github.com') ?? 'https://api.github.com', - token: this.getEnvString('GITHUB_TOKEN'), - rateLimit: { - maxRequests: this.getEnvNumber('GITHUB_RATE_LIMIT_MAX', 60), - windowMs: this.getEnvNumber('GITHUB_RATE_LIMIT_WINDOW_MS', 60000) - }, - retry: { - maxRetries: this.getEnvNumber('GITHUB_RETRY_MAX', 3), - backoffMs: this.getEnvNumber('GITHUB_RETRY_BACKOFF_MS', 1000) - } - }, - llm: { - providers: { - openai: this.getLLMProviderConfig('OPENAI'), - azure: this.getLLMProviderConfig('AZURE'), - anthropic: this.getLLMProviderConfig('ANTHROPIC'), - gemini: this.getLLMProviderConfig('GEMINI') - }, - defaultProvider: (this.getEnvString('LLM_DEFAULT_PROVIDER', 'openai') as 'openai' | 'azure' | 'anthropic' | 'gemini') ?? 'openai' - }, - processing: { - batchSize: this.getEnvNumber('PROCESSING_BATCH_SIZE', 10), - maxConcurrentRequests: this.getEnvNumber('PROCESSING_MAX_CONCURRENT', 5), - timeoutMs: this.getEnvNumber('PROCESSING_TIMEOUT_MS', 30000), - retry: { - maxRetries: this.getEnvNumber('PROCESSING_RETRY_MAX', 3), - backoffMs: this.getEnvNumber('PROCESSING_RETRY_BACKOFF_MS', 1000) - }, - fileExtensions: this.getEnvArray('PROCESSING_FILE_EXTENSIONS', [ - '.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', - '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.dart', - '.json', '.yaml', '.yml', '.xml', '.toml', '.ini', '.cfg', '.properties' - ]) - }, - logging: { - level: (this.getEnvString('LOG_LEVEL', 'info') as 'debug' | 'info' | 'warn' | 'error') ?? 'info', - enableMetrics: this.getEnvBoolean('LOG_ENABLE_METRICS', true), - enablePerformanceTracking: this.getEnvBoolean('LOG_ENABLE_PERFORMANCE', true), - maxLogEntries: this.getEnvNumber('LOG_MAX_ENTRIES', 1000) - }, - environment: (this.getEnvString('NODE_ENV', 'development') as 'development' | 'staging' | 'production') ?? 'development' - }; - - // Validate with Zod - const result = AppConfigSchema.safeParse(config); - if (!result.success) { - this.validationErrors = result.error.errors.map(e => `${e.path.join('.')}: ${e.message}`); - console.warn('Configuration validation errors:', this.validationErrors); - } - - return result.success ? result.data : AppConfigSchema.parse({ - memory: {}, - github: { rateLimit: {}, retry: {} }, - llm: { providers: {} }, - processing: { retry: {} }, - logging: {}, - }); - } catch (error) { - console.error('Failed to load configuration:', error); - return AppConfigSchema.parse({}); - } - } - - private getLLMProviderConfig(prefix: string) { - const apiKey = this.getEnvString(`${prefix}_API_KEY`); - if (!apiKey) return undefined; - - return { - apiKey, - model: this.getEnvString(`${prefix}_MODEL`) ?? 'gpt-4', - maxTokens: this.getEnvNumber(`${prefix}_MAX_TOKENS`, 2000), - temperature: this.getEnvNumber(`${prefix}_TEMPERATURE`, 0.7) - }; - } - - private getEnvString(key: string, defaultValue?: string): string | undefined { - return typeof process !== 'undefined' ? process.env[key] : defaultValue; - } - - private getEnvNumber(key: string, defaultValue: number): number { - const value = this.getEnvString(key); - return value ? parseInt(value, 10) || defaultValue : defaultValue; - } - - private getEnvBoolean(key: string, defaultValue: boolean): boolean { - const value = this.getEnvString(key); - return value ? value.toLowerCase() === 'true' : defaultValue; - } - - private getEnvArray(key: string, defaultValue: string[]): string[] { - const value = this.getEnvString(key); - return value ? value.split(',').map(s => s.trim()) : defaultValue; - } - - /** - * Validate environment configuration - */ - private validateEnvironment(): void { - const warnings: string[] = []; - - // Check for required API keys - if (!this.config.github.token) { - warnings.push('GitHub token not provided - rate limits will be lower'); - } - - const hasAnyLLMProvider = Object.values(this.config.llm.providers) - .some(provider => provider?.apiKey); - - if (!hasAnyLLMProvider) { - warnings.push('No LLM provider API keys configured - AI features disabled'); - } - - // Check memory limits - if (this.config.memory.maxMemoryMB < 256) { - warnings.push('Memory limit is very low - may cause performance issues'); - } - - if (warnings.length > 0) { - console.warn('Configuration warnings:', warnings); - } - } - - /** - * Get configuration values - */ - public getConfiguration(): AppConfig { - return this.config; - } - - public get memory(): MemoryConfig { - return this.config.memory; - } - - public get github(): GitHubConfig { - return this.config.github; - } - - public get llm(): LLMConfig { - return this.config.llm; - } - - public get processing(): ProcessingConfig { - return this.config.processing; - } - - public get logging(): LoggingConfig { - return this.config.logging; - } - - public get environment(): string { - return this.config.environment; - } - - public get isDevelopment(): boolean { - return this.config.environment === 'development'; - } - - public get isProduction(): boolean { - return this.config.environment === 'production'; - } - - /** - * Get validation errors - */ - public getValidationErrors(): string[] { - return [...this.validationErrors]; - } - - /** - * Update configuration at runtime - */ - public updateConfig(updates: Partial): void { - try { - const newConfig = { ...this.config, ...updates }; - const result = AppConfigSchema.safeParse(newConfig); - if (result.success) { - this.config = result.data; - } else { - throw new Error(`Invalid configuration: ${result.error.errors.map(e => e.message).join(', ')}`); - } - } catch (error) { - console.error('Failed to update configuration:', error); - throw error; - } - } -} - -// Export singleton instance -export const config = ConfigService.getInstance(); - -================ -File: src/config/feature-flags.ts -================ -/** - * Feature Flags Configuration - * Controls experimental features and integrations - */ - -export interface FeatureFlags { - // KuzuDB Integration - enableKuzuDB: boolean; - enableKuzuDBPersistence: boolean; - enableKuzuDBPerformanceMonitoring: boolean; - - // AI Features - enableAdvancedRAG: boolean; - enableReActReasoning: boolean; - enableMultiLLM: boolean; - - // Performance Features - enableWebWorkers: boolean; - enableBatchProcessing: boolean; - enableCaching: boolean; - enableWorkerPool: boolean; - enableParallelParsing: boolean; - enableParallelProcessing: boolean; - - // Debug Features - enableDebugMode: boolean; - enablePerformanceLogging: boolean; - enableQueryLogging: boolean; -} - -export const DEFAULT_FEATURE_FLAGS: FeatureFlags = { - // KuzuDB Integration - Now enabled with npm package - enableKuzuDB: true, - enableKuzuDBPersistence: true, - enableKuzuDBPerformanceMonitoring: true, - - // AI Features - enableAdvancedRAG: true, - enableReActReasoning: true, - enableMultiLLM: true, - - // Performance Features - enableWebWorkers: true, - enableBatchProcessing: true, - enableCaching: true, - enableWorkerPool: true, - enableParallelParsing: true, - enableParallelProcessing: true, - - // Debug Features - enableDebugMode: false, - enablePerformanceLogging: false, - enableQueryLogging: false -}; - -class FeatureFlagManager { - private flags: FeatureFlags; - private listeners: Set<(flags: FeatureFlags) => void> = new Set(); - - constructor() { - this.flags = this.loadFlags(); - } - - /** - * Load feature flags from localStorage or use defaults - */ - private loadFlags(): FeatureFlags { - try { - const stored = localStorage.getItem('gitnexus_feature_flags'); - if (stored) { - const parsed = JSON.parse(stored); - return { ...DEFAULT_FEATURE_FLAGS, ...parsed }; - } - } catch (error) { - console.warn('Failed to load feature flags from localStorage:', error); - } - - return { ...DEFAULT_FEATURE_FLAGS }; - } - - /** - * Save feature flags to localStorage - */ - private saveFlags(): void { - try { - localStorage.setItem('gitnexus_feature_flags', JSON.stringify(this.flags)); - } catch (error) { - console.warn('Failed to save feature flags to localStorage:', error); - } - } - - /** - * Get all feature flags - */ - getFlags(): FeatureFlags { - return { ...this.flags }; - } - - /** - * Get a specific feature flag - */ - getFlag(key: K): FeatureFlags[K] { - return this.flags[key]; - } - - /** - * Set a feature flag - */ - setFlag(key: K, value: FeatureFlags[K]): void { - this.flags[key] = value; - this.saveFlags(); - this.notifyListeners(); - } - - /** - * Set multiple feature flags - */ - setFlags(updates: Partial): void { - this.flags = { ...this.flags, ...updates }; - this.saveFlags(); - this.notifyListeners(); - } - - /** - * Reset all feature flags to defaults - */ - resetFlags(): void { - this.flags = { ...DEFAULT_FEATURE_FLAGS }; - this.saveFlags(); - this.notifyListeners(); - } - - /** - * Enable KuzuDB integration - */ - enableKuzuDB(): void { - this.setFlags({ - enableKuzuDB: true, - enableKuzuDBPersistence: true, - enableKuzuDBPerformanceMonitoring: true - }); - } - - /** - * Disable KuzuDB integration - */ - disableKuzuDB(): void { - this.setFlags({ - enableKuzuDB: false, - enableKuzuDBPersistence: false, - enableKuzuDBPerformanceMonitoring: false - }); - } - - /** - * Enable worker pool and parallel processing - */ - enableWorkerPool(): void { - this.setFlags({ - enableWorkerPool: true, - enableParallelParsing: true, - enableParallelProcessing: true - }); - } - - /** - * Disable worker pool and parallel processing - */ - disableWorkerPool(): void { - this.setFlags({ - enableWorkerPool: false, - enableParallelParsing: false, - enableParallelProcessing: false - }); - } - - /** - * Enable debug mode - */ - enableDebugMode(): void { - this.setFlags({ - enableDebugMode: true, - enablePerformanceLogging: true, - enableQueryLogging: true - }); - } - - /** - * Disable debug mode - */ - disableDebugMode(): void { - this.setFlags({ - enableDebugMode: false, - enablePerformanceLogging: false, - enableQueryLogging: false - }); - } - - /** - * Add a listener for flag changes - */ - addListener(listener: (flags: FeatureFlags) => void): void { - this.listeners.add(listener); - } - - /** - * Remove a listener - */ - removeListener(listener: (flags: FeatureFlags) => void): void { - this.listeners.delete(listener); - } - - /** - * Notify all listeners of flag changes - */ - private notifyListeners(): void { - for (const listener of this.listeners) { - try { - listener(this.getFlags()); - } catch (error) { - console.error('Feature flag listener error:', error); - } - } - } - - /** - * Check if KuzuDB is fully enabled - */ - isKuzuDBEnabled(): boolean { - return this.flags.enableKuzuDB && this.flags.enableKuzuDBPersistence; - } - - /** - * Check if debug mode is enabled - */ - isDebugModeEnabled(): boolean { - return this.flags.enableDebugMode; - } - - /** - * Check if performance monitoring is enabled - */ - isPerformanceMonitoringEnabled(): boolean { - return this.flags.enableKuzuDBPerformanceMonitoring || this.flags.enablePerformanceLogging; - } -} - -// Export singleton instance -export const featureFlags = new FeatureFlagManager(); - -// Export convenience functions -export const getFeatureFlag = (key: K): FeatureFlags[K] => - featureFlags.getFlag(key); - -export const setFeatureFlag = (key: K, value: FeatureFlags[K]): void => - featureFlags.setFlag(key, value); - -export const isKuzuDBEnabled = (): boolean => featureFlags.isKuzuDBEnabled(); -export const isDebugModeEnabled = (): boolean => featureFlags.isDebugModeEnabled(); -export const isPerformanceMonitoringEnabled = (): boolean => featureFlags.isPerformanceMonitoringEnabled(); -export const isWorkerPoolEnabled = (): boolean => featureFlags.getFlag('enableWorkerPool'); -export const isParallelParsingEnabled = (): boolean => featureFlags.getFlag('enableParallelParsing'); -export const isParallelProcessingEnabled = (): boolean => featureFlags.getFlag('enableParallelProcessing'); - -================ -File: src/config/language-config.ts -================ -/** - * Language-specific configuration and built-ins - * Centralized configuration to eliminate hardcoded values - */ - -export interface LanguageConfig { - name: string; - extensions: string[]; - builtinFunctions: Set; - builtinTypes: Set; - commentPatterns: { - singleLine: string[]; - multiLineStart: string[]; - multiLineEnd: string[]; - }; - importPatterns: { - import: RegExp[]; - fromImport: RegExp[]; - require: RegExp[]; - }; - -} - -// Python built-in functions and types -const PYTHON_BUILTINS = new Set([ - // Core built-ins - 'int', 'str', 'float', 'bool', 'list', 'dict', 'set', 'tuple', - 'len', 'range', 'enumerate', 'zip', 'map', 'filter', 'sorted', - 'sum', 'min', 'max', 'abs', 'round', 'all', 'any', 'hasattr', - 'getattr', 'setattr', 'isinstance', 'issubclass', 'type', - 'print', 'input', 'open', 'format', 'join', 'split', 'strip', - 'replace', 'upper', 'lower', 'append', 'extend', 'insert', - 'remove', 'pop', 'clear', 'copy', 'update', 'keys', 'values', - 'items', 'get', 'add', 'discard', 'union', 'intersection', - 'difference', 'locals', 'globals', 'vars', 'dir', 'help', 'id', 'hash', - 'ord', 'chr', 'bin', 'oct', 'hex', 'divmod', 'pow', 'exec', - 'eval', 'compile', 'next', 'iter', 'reversed', 'slice', - - // String methods - 'endswith', 'startswith', 'find', 'rfind', 'index', 'rindex', - 'count', 'encode', 'decode', 'capitalize', 'title', 'swapcase', - 'center', 'ljust', 'rjust', 'zfill', 'expandtabs', 'splitlines', - 'partition', 'rpartition', 'translate', 'maketrans', 'casefold', - 'isalnum', 'isalpha', 'isascii', 'isdecimal', 'isdigit', 'isidentifier', - 'islower', 'isnumeric', 'isprintable', 'isspace', 'istitle', 'isupper', - 'lstrip', 'rstrip', 'removeprefix', 'removesuffix', - - // List/sequence methods - 'sort', 'reverse', 'count', 'index', - - // Dictionary methods - 'setdefault', 'popitem', 'fromkeys', - - // Set methods - 'difference_update', 'intersection_update', 'symmetric_difference', - 'symmetric_difference_update', 'isdisjoint', 'issubset', 'issuperset', - - // Common exceptions - 'ValueError', 'TypeError', 'KeyError', 'IndexError', 'AttributeError', - 'ImportError', 'ModuleNotFoundError', 'FileNotFoundError', - 'ConnectionError', 'HTTPException', 'RuntimeError', 'OSError', - 'Exception', 'BaseException', 'StopIteration', 'GeneratorExit' -]); - -const PYTHON_LIBRARY_FUNCTIONS = new Set([ - // Date/time methods - 'now', 'today', 'fromisoformat', 'isoformat', 'astimezone', - 'strftime', 'strptime', 'timestamp', 'weekday', 'isoweekday', - 'date', 'time', 'timetz', 'utctimetuple', 'timetuple', - - // Random - 'random', 'choice', 'randint', 'shuffle', - - // Logging methods - 'debug', 'info', 'warning', 'error', 'critical', 'exception', - 'getLogger', 'basicConfig', 'StreamHandler', - - // Environment - 'load_dotenv', 'getenv', 'dirname', 'abspath', 'join', 'exists', 'run', - - // Database/ORM methods - 'find', 'find_one', 'update_one', 'insert_one', 'delete_one', - 'aggregate', 'bulk_write', 'to_list', 'sort', 'limit', 'close', - 'ObjectId', 'UpdateOne', 'AsyncIOMotorClient', 'command', - - // Pydantic/FastAPI - 'Field', 'validator', 'field_validator', 'model_dump', 'model_dump_json', - 'FastAPI', 'HTTPException', 'add_middleware', 'include_router', - - // Threading/async - 'Lock', 'RLock', 'Semaphore', 'Event', 'Condition', 'Barrier', - 'sleep', 'gather', 'create_task', 'run_until_complete', - - // Collections - 'defaultdict', 'Counter', 'OrderedDict', 'deque', 'namedtuple', - - // Math/statistics - 'mean', 'median', 'mode', 'stdev', 'variance', 'sqrt', 'pow', - 'sin', 'cos', 'tan', 'log', 'exp', 'ceil', 'floor', - - // UUID - 'uuid4', 'uuid1', 'uuid3', 'uuid5', - - // URL/HTTP - 'quote', 'unquote', 'quote_plus', 'unquote_plus', 'urlencode', - - // JSON - 'loads', 'dumps', 'load', 'dump', - - // Regex - 'match', 'search', 'findall', 'finditer', 'sub', 'subn', 'compile', - - // AI/ML libraries - 'AsyncAzureOpenAI', 'AzureOpenAI', 'OpenAI', 'wrap_openai', 'create' -]); - -// JavaScript built-in functions and types -const JAVASCRIPT_BUILTINS = new Set([ - // Global functions - 'parseInt', 'parseFloat', 'isNaN', 'isFinite', 'decodeURI', 'decodeURIComponent', - 'encodeURI', 'encodeURIComponent', 'eval', 'setTimeout', 'setInterval', - 'clearTimeout', 'clearInterval', 'console', 'alert', 'confirm', 'prompt', - - // Object methods - 'toString', 'valueOf', 'hasOwnProperty', 'isPrototypeOf', 'propertyIsEnumerable', - - // Array methods - 'push', 'pop', 'shift', 'unshift', 'slice', 'splice', 'concat', 'join', - 'reverse', 'sort', 'indexOf', 'lastIndexOf', 'forEach', 'map', 'filter', - 'reduce', 'reduceRight', 'every', 'some', 'find', 'findIndex', 'includes', - - // String methods - 'charAt', 'charCodeAt', 'concat', 'indexOf', 'lastIndexOf', 'localeCompare', - 'match', 'replace', 'search', 'slice', 'split', 'substring', 'toLowerCase', - 'toUpperCase', 'trim', 'padStart', 'padEnd', - - // Math - 'abs', 'ceil', 'floor', 'round', 'max', 'min', 'pow', 'sqrt', 'random', - - // Date - 'getTime', 'getFullYear', 'getMonth', 'getDate', 'getDay', 'getHours', - 'getMinutes', 'getSeconds', 'getMilliseconds', 'toISOString', 'toDateString', - - // Promise/async - 'then', 'catch', 'finally', 'resolve', 'reject', 'all', 'race', - - // JSON - 'parse', 'stringify', - - // Types - 'Object', 'Array', 'String', 'Number', 'Boolean', 'Function', 'Date', - 'RegExp', 'Error', 'Promise', 'Map', 'Set', 'WeakMap', 'WeakSet' -]); - -// TypeScript built-ins (extends JavaScript) -const TYPESCRIPT_BUILTINS = new Set([ - ...JAVASCRIPT_BUILTINS, - // TypeScript specific - 'Partial', 'Required', 'Readonly', 'Pick', 'Omit', 'Exclude', 'Extract', - 'Record', 'Parameters', 'ConstructorParameters', 'ReturnType', - 'InstanceType', 'ThisParameterType', 'OmitThisParameter', 'ThisType' -]); - -// Centralized ignore patterns -export const IGNORE_PATTERNS = new Set([ - // Version Control - '.git', '.svn', '.hg', - - // Package Managers & Dependencies - 'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps', - - // Python Virtual Environments & Cache - 'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__', - '.pytest_cache', '.mypy_cache', '.tox', - - // Build & Distribution Directories - 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', - - // IDE & Editor Directories - '.vs', '.vscode', '.idea', '.eclipse', '.settings', - - // Temporary & Log Directories - 'tmp', '.tmp', 'temp', 'logs', 'log', - - // Coverage & Testing - 'coverage', '.coverage', 'htmlcov', '.nyc_output', - - // OS & System - '.DS_Store', 'Thumbs.db', - - // Documentation Build Output - '_site', '.docusaurus', - - // Cache Directories - '.cache', '.parcel-cache', '.next', '.nuxt' -]); - -// Language configurations -export const LANGUAGE_CONFIGS: Record = { - python: { - name: 'Python', - extensions: ['.py', '.pyx', '.pyi'], - builtinFunctions: new Set([...PYTHON_BUILTINS, ...PYTHON_LIBRARY_FUNCTIONS]), - builtinTypes: new Set(['int', 'str', 'float', 'bool', 'list', 'dict', 'set', 'tuple']), - commentPatterns: { - singleLine: ['#'], - multiLineStart: ['"""', "'''"], - multiLineEnd: ['"""', "'''"] - }, - importPatterns: { - import: [/^import\s+(.+)$/], - fromImport: [/^from\s+(.+)\s+import\s+(.+)$/], - require: [] - } - }, - - javascript: { - name: 'JavaScript', - extensions: ['.js', '.mjs', '.cjs', '.jsx'], - builtinFunctions: JAVASCRIPT_BUILTINS, - builtinTypes: new Set(['Object', 'Array', 'String', 'Number', 'Boolean', 'Function']), - commentPatterns: { - singleLine: ['//'], - multiLineStart: ['/*'], - multiLineEnd: ['*/'] - }, - importPatterns: { - import: [/^import\s+.*\s+from\s+['"'](.+)['"]$/], - fromImport: [], - require: [/require\s*\(\s*['"'](.+)['"]\s*\)/] - } - }, - - typescript: { - name: 'TypeScript', - extensions: ['.ts', '.tsx'], - builtinFunctions: TYPESCRIPT_BUILTINS, - builtinTypes: new Set(['Object', 'Array', 'String', 'Number', 'Boolean', 'Function']), - commentPatterns: { - singleLine: ['//'], - multiLineStart: ['/*'], - multiLineEnd: ['*/'] - }, - importPatterns: { - import: [/^import\s+.*\s+from\s+['"'](.+)['"]$/], - fromImport: [], - require: [/require\s*\(\s*['"'](.+)['"]\s*\)/] - } - } -}; - -// Language abstraction interface -export interface ParsedDefinition { - name: string; - type: 'function' | 'class' | 'method' | 'interface' | 'enum' | 'decorator' | 'variable'; - startLine: number; - endLine: number; - parentClass?: string; - decorators?: string[]; - baseClasses?: string[]; - parameters?: string[]; - returnType?: string; -} - -export interface ImportInfo { - localName: string; - importedFrom: string; - exportedName: string; - importType: 'default' | 'named' | 'namespace' | 'dynamic'; -} - -export interface LanguageProcessor { - name: string; - extensions: string[]; - isBuiltinFunction(name: string): boolean; - isBuiltinType(name: string): boolean; - parseDefinitions(content: string, filePath: string): ParsedDefinition[]; - extractImports(content: string): ImportInfo[]; -} - -// Base language processor implementation -export abstract class BaseLanguageProcessor implements LanguageProcessor { - protected config: LanguageConfig; - - constructor(config: LanguageConfig) { - this.config = config; - } - - get name(): string { - return this.config.name; - } - - get extensions(): string[] { - return this.config.extensions; - } - - isBuiltinFunction(name: string): boolean { - return this.config.builtinFunctions.has(name); - } - - isBuiltinType(name: string): boolean { - return this.config.builtinTypes.has(name); - } - - abstract parseDefinitions(content: string, filePath: string): ParsedDefinition[]; - abstract extractImports(content: string): ImportInfo[]; -} - -// Language processor factory -export class LanguageProcessorFactory { - private static processors: Map LanguageProcessor> = new Map(); - - static register(language: string, factory: () => LanguageProcessor): void { - this.processors.set(language, factory); - } - - static create(language: string): LanguageProcessor | null { - const factory = this.processors.get(language); - return factory ? factory() : null; - } - - static getConfig(language: string): LanguageConfig | null { - return LANGUAGE_CONFIGS[language] || null; - } - - static getSupportedLanguages(): string[] { - return Object.keys(LANGUAGE_CONFIGS); - } -} - -// Utility functions for language detection -export const languageDetection = { - detectFromExtension(filePath: string): string | null { - const extension = filePath.substring(filePath.lastIndexOf('.')).toLowerCase(); - - for (const [lang, config] of Object.entries(LANGUAGE_CONFIGS)) { - if (config.extensions.includes(extension)) { - return lang; - } - } - - return null; - }, - - detectFromContent(content: string): string | null { - // Simple content-based detection - if (content.includes('def ') && content.includes('import ')) { - return 'python'; - } - if (content.includes('function ') || content.includes('const ') || content.includes('let ')) { - if (content.includes('interface ') || content.includes(': string')) { - return 'typescript'; - } - return 'javascript'; - } - - return null; - } -}; - -================ -File: src/core/graph/graph.ts -================ -/** - * Graph interfaces and types - */ - -import { GraphNode, GraphRelationship, NodeProperties, RelationshipProperties } from './types.js'; - -export interface KnowledgeGraph { - nodes: GraphNode[]; - relationships: GraphRelationship[]; - addNode(node: GraphNode): void; - addRelationship(relationship: GraphRelationship): void; -} - -export interface GraphProcessor { - process(graph: KnowledgeGraph, input: T): Promise; -} - -// Simple implementation of KnowledgeGraph -export class SimpleKnowledgeGraph implements KnowledgeGraph { - nodes: GraphNode[] = []; - relationships: GraphRelationship[] = []; - - addNode(node: GraphNode): void { - this.nodes.push(node); - } - - addRelationship(relationship: GraphRelationship): void { - this.relationships.push(relationship); - } -} - -================ -File: src/core/graph/kuzu-query-engine.ts -================ -import { KuzuService, type KuzuQueryResult } from '../../services/kuzu.service.js'; -import type { KnowledgeGraph, GraphNode, GraphRelationship } from './types.js'; - -export interface KuzuQueryOptions { - timeout?: number; - maxResults?: number; - includeExecutionTime?: boolean; -} - -export interface KuzuQueryResponse { - nodes: GraphNode[]; - relationships: GraphRelationship[]; - executionTime: number; - resultCount: number; - warnings?: string[]; -} - -export class KuzuQueryEngine { - private kuzuService: KuzuService; - private isInitialized: boolean = false; - - constructor() { - this.kuzuService = new KuzuService(); - } - - /** - * Initialize the query engine - */ - async initialize(): Promise { - if (this.isInitialized) { - return; - } - - await this.kuzuService.initialize(); - this.isInitialized = true; - } - - /** - * Execute a Cypher query and return results in GitNexus format - */ - async executeQuery( - cypherQuery: string, - options: KuzuQueryOptions = {} - ): Promise { - if (!this.isInitialized) { - await this.initialize(); - } - - const startTime = performance.now(); - - try { - // Execute query in KuzuDB - const result = await this.kuzuService.executeQuery(cypherQuery); - - // Convert KuzuDB results to GitNexus format - const { nodes, relationships } = this.convertKuzuResultsToGraph(result.results); - - const executionTime = performance.now() - startTime; - - return { - nodes, - relationships, - executionTime, - resultCount: result.count, - warnings: options.includeExecutionTime ? [`Query executed in ${executionTime.toFixed(2)}ms`] : undefined - }; - - } catch (error) { - console.error('KuzuDB query execution failed:', error); - throw new Error(`Query execution failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Convert KuzuDB query results to GitNexus graph format - */ - private convertKuzuResultsToGraph(kuzuResults: any[]): { - nodes: GraphNode[]; - relationships: GraphRelationship[]; - } { - const nodes: GraphNode[] = []; - const relationships: GraphRelationship[] = []; - const nodeMap = new Map(); - const relMap = new Map(); - - for (const result of kuzuResults) { - // Extract nodes from result - for (const [key, value] of Object.entries(result)) { - if (this.isNodeResult(value)) { - const node = this.convertKuzuNodeToGraphNode(value); - if (!nodeMap.has(node.id)) { - nodeMap.set(node.id, node); - nodes.push(node); - } - } - } - - // Extract relationships from result - for (const [key, value] of Object.entries(result)) { - if (this.isRelationshipResult(value)) { - const rel = this.convertKuzuRelToGraphRel(value); - if (!relMap.has(rel.id)) { - relMap.set(rel.id, rel); - relationships.push(rel); - } - } - } - } - - return { nodes, relationships }; - } - - /** - * Check if a result value represents a node - */ - private isNodeResult(value: any): boolean { - return value && typeof value === 'object' && - (value._label || value.label) && - (value._id || value.id); - } - - /** - * Check if a result value represents a relationship - */ - private isRelationshipResult(value: any): boolean { - return value && typeof value === 'object' && - (value._type || value.type) && - (value._source || value.source) && - (value._target || value.target); - } - - /** - * Convert KuzuDB node format to GitNexus node format - */ - private convertKuzuNodeToGraphNode(kuzuNode: any): GraphNode { - return { - id: kuzuNode._id || kuzuNode.id, - label: (kuzuNode._label || kuzuNode.label) as any, - properties: { - ...kuzuNode, - // Remove internal KuzuDB properties - _id: undefined, - _label: undefined, - id: kuzuNode._id || kuzuNode.id, - label: kuzuNode._label || kuzuNode.label - } - }; - } - - /** - * Convert KuzuDB relationship format to GitNexus relationship format - */ - private convertKuzuRelToGraphRel(kuzuRel: any): GraphRelationship { - return { - id: kuzuRel._id || kuzuRel.id, - type: (kuzuRel._type || kuzuRel.type) as any, - source: kuzuRel._source || kuzuRel.source, - target: kuzuRel._target || kuzuRel.target, - properties: { - ...kuzuRel, - // Remove internal KuzuDB properties - _id: undefined, - _type: undefined, - _source: undefined, - _target: undefined, - id: kuzuRel._id || kuzuRel.id, - type: kuzuRel._type || kuzuRel.type, - source: kuzuRel._source || kuzuRel.source, - target: kuzuRel._target || kuzuRel.target - } - }; - } - - /** - * Execute a simple node query - */ - async queryNodes( - nodeType?: string, - filters?: Record, - options: KuzuQueryOptions = {} - ): Promise { - let cypherQuery = 'MATCH (n'; - - if (nodeType) { - cypherQuery += `:${nodeType}`; - } - - cypherQuery += ')'; - - if (filters && Object.keys(filters).length > 0) { - const filterConditions = Object.entries(filters) - .map(([key, value]) => `n.${key} = '${value}'`) - .join(' AND '); - cypherQuery += ` WHERE ${filterConditions}`; - } - - cypherQuery += ' RETURN n'; - - if (options.maxResults) { - cypherQuery += ` LIMIT ${options.maxResults}`; - } - - const result = await this.executeQuery(cypherQuery, options); - return result.nodes; - } - - /** - * Execute a simple relationship query - */ - async queryRelationships( - relType?: string, - sourceNodeId?: string, - targetNodeId?: string, - options: KuzuQueryOptions = {} - ): Promise { - let cypherQuery = 'MATCH (a)'; - - if (relType) { - cypherQuery += `-[r:${relType}]`; - } else { - cypherQuery += '-[r]'; - } - - cypherQuery += '->(b)'; - - const conditions = []; - if (sourceNodeId) { - conditions.push(`a.id = '${sourceNodeId}'`); - } - if (targetNodeId) { - conditions.push(`b.id = '${targetNodeId}'`); - } - - if (conditions.length > 0) { - cypherQuery += ` WHERE ${conditions.join(' AND ')}`; - } - - cypherQuery += ' RETURN r'; - - if (options.maxResults) { - cypherQuery += ` LIMIT ${options.maxResults}`; - } - - const result = await this.executeQuery(cypherQuery, options); - return result.relationships; - } - - /** - * Get database statistics - */ - async getDatabaseStats(): Promise { - if (!this.isInitialized) { - await this.initialize(); - } - - return await this.kuzuService.getDatabaseStats(); - } - - /** - * Import knowledge graph into KuzuDB - */ - async importGraph(graph: KnowledgeGraph): Promise { - if (!this.isInitialized) { - await this.initialize(); - } - - await this.kuzuService.importKnowledgeGraph(graph); - } - - /** - * Clear the database - */ - async clearDatabase(): Promise { - if (!this.isInitialized) { - await this.initialize(); - } - - await this.kuzuService.clearDatabase(); - } - - /** - * Close the query engine - */ - async close(): Promise { - await this.kuzuService.close(); - this.isInitialized = false; - } - - /** - * Check if the query engine is ready - */ - isReady(): boolean { - return this.isInitialized && this.kuzuService.isReady(); - } -} - -================ -File: src/core/graph/query-engine.ts -================ -import type { KnowledgeGraph, GraphNode, GraphRelationship } from './types.ts'; - -export interface QueryResult { - nodes: GraphNode[]; - relationships: GraphRelationship[]; - data: Record[]; -} - -export interface QueryOptions { - limit?: number; - offset?: number; -} - -export class GraphQueryEngine { - constructor(private graph: KnowledgeGraph) {} - - /** - * Execute a simplified Cypher-like query against the knowledge graph - */ - public executeQuery(cypher: string, options: QueryOptions = {}): QueryResult { - const { limit = 100, offset = 0 } = options; - - try { - const parsedQuery = this.parseCypher(cypher); - - switch (parsedQuery.type) { - case 'MATCH': - return this.executeMatchQuery(parsedQuery, limit, offset); - case 'MATCH_WHERE': - return this.executeWhereQuery(parsedQuery, limit, offset); - case 'MATCH_PATH': - return this.executePathQuery(parsedQuery, limit, offset); - case 'MATCH_AGGREGATION': - return this.executeAggregationQuery(parsedQuery, limit, offset); - case 'MATCH_RELATIONSHIP': - return this.executeRelationshipQuery(parsedQuery, limit, offset); - default: - throw new Error(`Unsupported query type: ${parsedQuery.type}`); - } - } catch (error) { - console.error('Query execution failed:', error); - return { nodes: [], relationships: [], data: [] }; - } - } - - private parseCypher(cypher: string) { - // Enhanced parsing with WHERE clause support - const simpleMatchWithWherePattern = /MATCH\s+\((\w+):(\w+)(?:\s*\{([^}]*)\})?\)\s+WHERE\s+(.+?)\s+RETURN\s+(.+)/i; - const whereMatch = cypher.match(simpleMatchWithWherePattern); - - if (whereMatch) { - const [, variable, label, properties, whereClause, returnClause] = whereMatch; - return { - type: 'MATCH_WHERE', - variable, - label, - properties: this.parseProperties(properties || ''), - whereClause: whereClause.trim(), - returnClause: returnClause.trim() - }; - } - - // Variable-length relationship pattern: MATCH (a)-[:REL*1..3]->(b) - const variableLengthPattern = /MATCH\s+\((\w+)(?::(\w+))?\)-\[:(\w+)\*(\d+)\.\.(\d+)\]->\((\w+)(?::(\w+))?\)\s+RETURN\s+(.+)/i; - const varLenMatch = cypher.match(variableLengthPattern); - - if (varLenMatch) { - const [, sourceVar, sourceLabel, relType, minDepth, maxDepth, targetVar, targetLabel, returnClause] = varLenMatch; - return { - type: 'MATCH_PATH', - sourceVar, - sourceLabel, - relationshipType: relType, - minDepth: parseInt(minDepth), - maxDepth: parseInt(maxDepth), - targetVar, - targetLabel, - returnClause: returnClause.trim() - }; - } - - // Aggregation pattern: MATCH (n:Label) RETURN COUNT(n) - const aggregationPattern = /MATCH\s+\((\w+):(\w+)(?:\s*\{([^}]*)\})?\)\s+RETURN\s+(COUNT|COLLECT|AVG|SUM)\(([^)]+)\)/i; - const aggMatch = cypher.match(aggregationPattern); - - if (aggMatch) { - const [, variable, label, properties, aggFunction, aggTarget] = aggMatch; - return { - type: 'MATCH_AGGREGATION', - variable, - label, - properties: this.parseProperties(properties || ''), - aggregationFunction: aggFunction.toUpperCase(), - aggregationTarget: aggTarget.trim(), - returnClause: `${aggFunction}(${aggTarget})` - }; - } - - // Pattern: MATCH (n:Label {property: 'value'}) RETURN n.property - const simpleMatchPattern = /MATCH\s+\((\w+):(\w+)(?:\s*\{([^}]*)\})?\)\s+RETURN\s+(.+)/i; - const simpleMatch = cypher.match(simpleMatchPattern); - - if (simpleMatch) { - const [, variable, label, properties, returnClause] = simpleMatch; - return { - type: 'MATCH', - variable, - label, - properties: this.parseProperties(properties || ''), - returnClause: returnClause.trim() - }; - } - - // Pattern: MATCH (a)-[:RELATIONSHIP]->(b:Label) RETURN a, b - const relationshipPattern = /MATCH\s+\((\w+)(?::(\w+))?\)-\[:(\w+)\]->\((\w+)(?::(\w+))?\)\s+RETURN\s+(.+)/i; - const relMatch = cypher.match(relationshipPattern); - - if (relMatch) { - const [, sourceVar, sourceLabel, relType, targetVar, targetLabel, returnClause] = relMatch; - return { - type: 'MATCH_RELATIONSHIP', - sourceVar, - sourceLabel, - relationshipType: relType, - targetVar, - targetLabel, - returnClause: returnClause.trim() - }; - } - - throw new Error(`Cannot parse Cypher query: ${cypher}`); - } - - private parseProperties(propString: string): Record { - const props: Record = {}; - if (!propString.trim()) return props; - - // Simple property parsing: name: 'value', type: 'Function' - const matches = propString.match(/(\w+):\s*['"]([^'"]*)['"]/g); - if (matches) { - matches.forEach(match => { - const [, key, value] = match.match(/(\w+):\s*['"]([^'"]*)['"]/!) || []; - if (key && value) { - props[key] = value; - } - }); - } - - return props; - } - - private executeMatchQuery(query: any, limit: number, offset: number): QueryResult { - let matchingNodes = this.graph.nodes.filter(node => { - // Match by label - if (query.label && node.label !== query.label) { - return false; - } - - // Match by properties - for (const [key, value] of Object.entries(query.properties)) { - if (node.properties[key] !== value) { - return false; - } - } - - return true; - }); - - // Apply pagination - matchingNodes = matchingNodes.slice(offset, offset + limit); - - // Process return clause - const data = matchingNodes.map(node => { - const result: Record = {}; - - if (query.returnClause.includes(`${query.variable}.`)) { - // Return specific properties: n.name, n.filePath - const propertyMatches = query.returnClause.match(new RegExp(`${query.variable}\\.(\\w+)`, 'g')); - if (propertyMatches) { - propertyMatches.forEach((match: string) => { - const prop = match.split('.')[1]; - result[prop] = node.properties[prop]; - }); - } - } else if (query.returnClause === query.variable) { - // Return entire node - result.node = node; - } - - return result; - }); - - return { - nodes: matchingNodes, - relationships: [], - data - }; - } - - private executeWhereQuery(query: any, limit: number, offset: number): QueryResult { - let matchingNodes = this.graph.nodes.filter(node => { - if (query.label && node.label !== query.label) return false; - - for (const [key, value] of Object.entries(query.properties)) { - if (node.properties[key] !== value) return false; - } - - return this.evaluateWhereClause(node, query.whereClause, query.variable); - }); - - matchingNodes = matchingNodes.slice(offset, offset + limit); - - const data = matchingNodes.map(node => { - const result: Record = {}; - if (query.returnClause.includes(`${query.variable}.`)) { - const propertyMatches = query.returnClause.match(new RegExp(`${query.variable}\\.(\\w+)`, 'g')); - if (propertyMatches) { - propertyMatches.forEach((match: string) => { - const prop = match.split('.')[1]; - result[prop] = node.properties[prop]; - }); - } - } else if (query.returnClause === query.variable) { - result.node = node; - } - return result; - }); - - return { nodes: matchingNodes, relationships: [], data }; - } - - private evaluateWhereClause(node: GraphNode, whereClause: string, variable: string): boolean { - // Simple WHERE clause evaluation - const containsPattern = new RegExp(`${variable}\\.(\\w+)\\s+CONTAINS\\s+['"]([^'"]*)['"]/i`); - const containsMatch = whereClause.match(containsPattern); - - if (containsMatch) { - const [, property, value] = containsMatch; - const nodeValue = node.properties[property]; - return typeof nodeValue === 'string' && nodeValue.toLowerCase().includes(value.toLowerCase()); - } - - const equalsPattern = new RegExp(`${variable}\\.(\\w+)\\s*=\\s*['"]([^'"]*)['"]/i`); - const equalsMatch = whereClause.match(equalsPattern); - - if (equalsMatch) { - const [, property, value] = equalsMatch; - return node.properties[property] === value; - } - - return true; - } - - private executePathQuery(query: any, limit: number, offset: number): QueryResult { - // Use existing pathsBetween function from query.ts - const sourceNodes = this.graph.nodes.filter(n => - !query.sourceLabel || n.label === query.sourceLabel - ); - const targetNodes = this.graph.nodes.filter(n => - !query.targetLabel || n.label === query.targetLabel - ); - - const allResults: { source: GraphNode; target: GraphNode; path: GraphRelationship[] }[] = []; - - for (const source of sourceNodes.slice(0, 50)) { // Limit source nodes to avoid explosion - for (const target of targetNodes.slice(0, 50)) { - if (source.id === target.id) continue; - - const paths = this.findPaths(source.id, target.id, query.relationshipType, query.minDepth, query.maxDepth); - paths.forEach(path => { - allResults.push({ source, target, path }); - }); - } - } - - const paginatedResults = allResults.slice(offset, offset + limit); - - const data = paginatedResults.map(({ source, target, path }) => { - const result: Record = {}; - if (query.returnClause.includes(query.sourceVar)) { - result[query.sourceVar] = source; - } - if (query.returnClause.includes(query.targetVar)) { - result[query.targetVar] = target; - } - result.pathLength = path.length; - return result; - }); - - return { - nodes: paginatedResults.flatMap(r => [r.source, r.target]), - relationships: paginatedResults.flatMap(r => r.path), - data - }; - } - - private findPaths(sourceId: string, targetId: string, relType: string, minDepth: number, maxDepth: number): GraphRelationship[][] { - const paths: GraphRelationship[][] = []; - const visited = new Set(); - - const dfs = (currentId: string, currentPath: GraphRelationship[], depth: number) => { - if (depth > maxDepth) return; - if (currentId === targetId && depth >= minDepth) { - paths.push([...currentPath]); - return; - } - - visited.add(currentId); - - const outgoingRels = this.graph.relationships.filter(r => - r.source === currentId && r.type === relType && !visited.has(r.target) - ); - - for (const rel of outgoingRels) { - dfs(rel.target, [...currentPath, rel], depth + 1); - } - - visited.delete(currentId); - }; - - dfs(sourceId, [], 0); - return paths.slice(0, 10); // Limit paths to prevent explosion - } - - private executeAggregationQuery(query: any, _limit: number, _offset: number): QueryResult { - let matchingNodes = this.graph.nodes.filter(node => { - if (query.label && node.label !== query.label) return false; - - for (const [key, value] of Object.entries(query.properties)) { - if (node.properties[key] !== value) return false; - } - - return true; - }); - - let aggregatedValue: any; - - switch (query.aggregationFunction) { - case 'COUNT': - aggregatedValue = matchingNodes.length; - break; - case 'COLLECT': - const targetProperty = query.aggregationTarget.includes('.') - ? query.aggregationTarget.split('.')[1] - : 'name'; - aggregatedValue = matchingNodes.map(n => n.properties[targetProperty]).filter(Boolean); - break; - default: - aggregatedValue = matchingNodes.length; - } - - return { - nodes: [], - relationships: [], - data: [{ [query.returnClause]: aggregatedValue }] - }; - } - - private executeRelationshipQuery(query: any, limit: number, offset: number): QueryResult { - const results: { source: GraphNode; target: GraphNode; relationship: GraphRelationship }[] = []; - - // Find all relationships of the specified type - const matchingRels = this.graph.relationships.filter(rel => - rel.type === query.relationshipType - ); - - for (const rel of matchingRels) { - const sourceNode = this.graph.nodes.find(n => n.id === rel.source); - const targetNode = this.graph.nodes.find(n => n.id === rel.target); - - if (!sourceNode || !targetNode) continue; - - // Apply label filters - if (query.sourceLabel && sourceNode.label !== query.sourceLabel) continue; - if (query.targetLabel && targetNode.label !== query.targetLabel) continue; - - results.push({ source: sourceNode, target: targetNode, relationship: rel }); - } - - // Apply pagination - const paginatedResults = results.slice(offset, offset + limit); - - // Process return clause - const data = paginatedResults.map(({ source, target }) => { - const result: Record = {}; - - if (query.returnClause.includes(query.sourceVar)) { - result[query.sourceVar] = source; - } - if (query.returnClause.includes(query.targetVar)) { - result[query.targetVar] = target; - } - - return result; - }); - - return { - nodes: paginatedResults.flatMap(r => [r.source, r.target]), - relationships: paginatedResults.map(r => r.relationship), - data - }; - } - - /** - * Test the query engine with sample queries (for development/debugging) - */ - public testQueries(): { query: string; result: any; success: boolean }[] { - const testCases = [ - "MATCH (f:Function) RETURN COUNT(f)", - "MATCH (c:Class) WHERE c.name CONTAINS 'Service' RETURN c.name", - "MATCH (a:Function)-[:CALLS*1..2]->(b:Function) RETURN a.name, b.name", - "MATCH (f:File)-[:CONTAINS]->(c:Class) RETURN f.name, COUNT(c)", - "MATCH (m:Method) WHERE m.name CONTAINS 'get' RETURN COLLECT(m.name)" - ]; - - return testCases.map(query => { - try { - const result = this.executeQuery(query, { limit: 5 }); - return { query, result, success: true }; - } catch (error) { - return { - query, - result: error instanceof Error ? error.message : 'Unknown error', - success: false - }; - } - }); - } - - /** - * Get query statistics - */ - public getStats(): { nodeCount: number; relationshipCount: number; nodeTypes: string[]; relationshipTypes: string[] } { - const nodeTypes = [...new Set(this.graph.nodes.map(n => n.label))]; - const relationshipTypes = [...new Set(this.graph.relationships.map(r => r.type))]; - - return { - nodeCount: this.graph.nodes.length, - relationshipCount: this.graph.relationships.length, - nodeTypes, - relationshipTypes - }; - } - - /** - * Find nodes by text search - */ - public searchNodes(searchTerm: string, nodeType?: string): GraphNode[] { - const lowerSearch = searchTerm.toLowerCase(); - - return this.graph.nodes.filter(node => { - if (nodeType && node.label !== nodeType) return false; - - // Search in node properties - const searchableText = [ - node.properties.name, - node.properties.filePath, - node.properties.qualifiedName - ].filter(Boolean).join(' ').toLowerCase(); - - return searchableText.includes(lowerSearch); - }); - } - - /** - * Get all relationships for a node - */ - public getNodeRelationships(nodeId: string): { incoming: GraphRelationship[]; outgoing: GraphRelationship[] } { - const incoming = this.graph.relationships.filter(r => r.target === nodeId); - const outgoing = this.graph.relationships.filter(r => r.source === nodeId); - - return { incoming, outgoing }; - } -} - -================ -File: src/core/graph/query.ts -================ -import type { KnowledgeGraph, GraphNode, GraphRelationship } from './types.ts'; - -export type NodeFilter = { - idEquals?: string | string[]; - labelIn?: Array; - nameContains?: string; - pathContains?: string; - props?: Record; -}; - -export type RelFilter = { - typeIn?: Array; - fromIdIn?: string[]; - toIdIn?: string[]; -}; - -export type TraverseOptions = { - depth?: number; - direction?: 'out' | 'in' | 'both'; - relTypeIn?: Array; - limitNodes?: number; -}; - -export type PathOptions = { - relTypeIn?: Array; - maxDepth?: number; - maxPaths?: number; -}; - -export interface GraphIndex { - nodeById: Map; - outAdjacency: Map; - inAdjacency: Map; -} - -export function indexGraph(graph: KnowledgeGraph): GraphIndex { - const nodeById = new Map(); - const outAdjacency = new Map(); - const inAdjacency = new Map(); - - for (const node of graph.nodes) { - nodeById.set(node.id, node); - } - - for (const rel of graph.relationships) { - if (!outAdjacency.has(rel.source)) outAdjacency.set(rel.source, []); - if (!inAdjacency.has(rel.target)) inAdjacency.set(rel.target, []); - outAdjacency.get(rel.source)!.push(rel); - inAdjacency.get(rel.target)!.push(rel); - } - - return { nodeById, outAdjacency, inAdjacency }; -} - -export function queryNodes(graph: KnowledgeGraph, filter: NodeFilter): GraphNode[] { - const idSet = new Set(typeof filter.idEquals === 'string' ? [filter.idEquals] : filter.idEquals || []); - const nameNeedle = filter.nameContains?.toLowerCase(); - const pathNeedle = filter.pathContains?.toLowerCase(); - - return graph.nodes.filter((node) => { - if (idSet.size > 0 && !idSet.has(node.id)) return false; - if (filter.labelIn && filter.labelIn.length > 0 && !filter.labelIn.includes(node.label)) return false; - - if (nameNeedle) { - const name = String(node.properties?.name || '').toLowerCase(); - if (!name.includes(nameNeedle)) return false; - } - - if (pathNeedle) { - const path = String(node.properties?.path || node.properties?.filePath || '').toLowerCase(); - if (!path.includes(pathNeedle)) return false; - } - - if (filter.props) { - for (const [key, expected] of Object.entries(filter.props)) { - if ((node.properties as Record)?.[key] !== expected) return false; - } - } - - return true; - }); -} - -export function queryRelationships(graph: KnowledgeGraph, filter: RelFilter): GraphRelationship[] { - const typeSet = new Set(filter.typeIn || []); - const fromSet = new Set(filter.fromIdIn || []); - const toSet = new Set(filter.toIdIn || []); - - return graph.relationships.filter((rel) => { - if (typeSet.size > 0 && !typeSet.has(rel.type)) return false; - if (fromSet.size > 0 && !fromSet.has(rel.source)) return false; - if (toSet.size > 0 && !toSet.has(rel.target)) return false; - return true; - }); -} - -export function neighborsOf( - graph: KnowledgeGraph, - seedNodeIds: string[], - options: TraverseOptions = {} -): KnowledgeGraph { - const { depth = 1, direction = 'both', relTypeIn, limitNodes = 500 } = options; - const { outAdjacency, inAdjacency, nodeById } = indexGraph(graph); - - const allowedRelTypes = relTypeIn ? new Set(relTypeIn) : null; - const visitedNodes = new Set(seedNodeIds); - const resultNodes = new Set(seedNodeIds); - const resultRels: GraphRelationship[] = []; - - let frontier = [...seedNodeIds]; - let currentDepth = 0; - - while (frontier.length > 0 && currentDepth < depth && resultNodes.size < limitNodes) { - const nextFrontier: string[] = []; - for (const nodeId of frontier) { - if (direction === 'out' || direction === 'both') { - const outRels = outAdjacency.get(nodeId) || []; - for (const rel of outRels) { - if (allowedRelTypes && !allowedRelTypes.has(rel.type)) continue; - resultRels.push(rel); - if (!visitedNodes.has(rel.target)) { - visitedNodes.add(rel.target); - resultNodes.add(rel.target); - nextFrontier.push(rel.target); - if (resultNodes.size >= limitNodes) break; - } - } - } - if (resultNodes.size >= limitNodes) break; - if (direction === 'in' || direction === 'both') { - const inRels = inAdjacency.get(nodeId) || []; - for (const rel of inRels) { - if (allowedRelTypes && !allowedRelTypes.has(rel.type)) continue; - resultRels.push(rel); - if (!visitedNodes.has(rel.source)) { - visitedNodes.add(rel.source); - resultNodes.add(rel.source); - nextFrontier.push(rel.source); - if (resultNodes.size >= limitNodes) break; - } - } - } - if (resultNodes.size >= limitNodes) break; - } - frontier = nextFrontier; - currentDepth++; - } - - const nodes = Array.from(resultNodes).map((id) => nodeById.get(id)!).filter(Boolean); - const relationships = dedupeRelationships(resultRels); - return { nodes, relationships }; -} - -export function pathsBetween( - graph: KnowledgeGraph, - fromId: string, - toId: string, - options: PathOptions = {} -): KnowledgeGraph { - const { maxDepth = 6, relTypeIn, maxPaths = 3 } = options; - const { outAdjacency, nodeById } = indexGraph(graph); - const allowedRelTypes = relTypeIn ? new Set(relTypeIn) : null; - - const queue: Array<{ nodeId: string; path: GraphRelationship[] }> = [{ nodeId: fromId, path: [] }]; - const visited = new Set([fromId]); - const foundPaths: GraphRelationship[][] = []; - - while (queue.length > 0 && foundPaths.length < maxPaths) { - const { nodeId, path } = queue.shift()!; - if (path.length > maxDepth) continue; - if (nodeId === toId) { - foundPaths.push(path); - continue; - } - const outRels = outAdjacency.get(nodeId) || []; - for (const rel of outRels) { - if (allowedRelTypes && !allowedRelTypes.has(rel.type)) continue; - const nextId = rel.target; - if (!visited.has(nextId)) { - visited.add(nextId); - queue.push({ nodeId: nextId, path: [...path, rel] }); - } - } - } - - const relSet = new Set(); - for (const p of foundPaths) for (const r of p) relSet.add(r); - const nodeSet = new Set(); - for (const r of relSet) { - nodeSet.add(r.source); - nodeSet.add(r.target); - } - - const nodes = Array.from(nodeSet).map((id) => nodeById.get(id)!).filter(Boolean); - const relationships = dedupeRelationships(Array.from(relSet)); - return { nodes, relationships }; -} - -export function subgraphFromNodesAndRels(graph: KnowledgeGraph, nodeIds: string[], rels: GraphRelationship[]): KnowledgeGraph { - const nodeIdSet = new Set(nodeIds); - const nodes = graph.nodes.filter((n) => nodeIdSet.has(n.id)); - const relationships = dedupeRelationships(rels); - return { nodes, relationships }; -} - -export function summarizeSubgraph(subgraph: KnowledgeGraph): string { - const counts = subgraph.nodes.reduce>((acc, n) => { - acc[n.label] = (acc[n.label] || 0) + 1; - return acc; - }, {}); - const relCounts = subgraph.relationships.reduce>((acc, r) => { - acc[r.type] = (acc[r.type] || 0) + 1; - return acc; - }, {}); - const parts: string[] = []; - parts.push(`Nodes: ${subgraph.nodes.length} (${Object.entries(counts).map(([k, v]) => `${k}:${v}`).join(', ') || 'none'})`); - parts.push(`Relationships: ${subgraph.relationships.length} (${Object.entries(relCounts).map(([k, v]) => `${k}:${v}`).join(', ') || 'none'})`); - const examples = subgraph.nodes.slice(0, 5).map((n) => `${n.label}:${String(n.properties?.name || n.id)}`); - if (examples.length > 0) parts.push(`Examples: ${examples.join(', ')}`); - return parts.join('\n'); -} - -function dedupeRelationships(rels: GraphRelationship[]): GraphRelationship[] { - const seen = new Set(); - const out: GraphRelationship[] = []; - for (const r of rels) { - const key = `${r.type}|${r.source}|${r.target}`; - if (!seen.has(key)) { - seen.add(key); - out.push(r); - } - } - return out; -} - -================ -File: src/core/graph/trie.ts -================ -interface TrieNode { - children: Map; - definitions: FunctionDefinition[]; - isEndOfWord: boolean; -} - -interface FunctionDefinition { - nodeId: string; - qualifiedName: string; - filePath: string; - functionName: string; - type: 'function' | 'method' | 'class' | 'interface' | 'enum'; - startLine?: number; - endLine?: number; -} - -export class FunctionRegistryTrie { - private root: TrieNode; - private allDefinitions: Map; - // Performance optimization: Index for fast lookups - private functionNameIndex: Map; - private filePathIndex: Map; - - constructor() { - this.root = { - children: new Map(), - definitions: [], - isEndOfWord: false - }; - this.allDefinitions = new Map(); - this.functionNameIndex = new Map(); - this.filePathIndex = new Map(); - } - - /** - * Add a function definition to the trie with optimized indexing - * @param definition The function definition to add - */ - addDefinition(definition: FunctionDefinition): void { - const parts = definition.qualifiedName.split('.'); - let currentNode = this.root; - - // Build the trie path - for (const part of parts) { - if (!currentNode.children.has(part)) { - currentNode.children.set(part, { - children: new Map(), - definitions: [], - isEndOfWord: false - }); - } - currentNode = currentNode.children.get(part)!; - } - - // Mark end of word and store definition - currentNode.isEndOfWord = true; - currentNode.definitions.push(definition); - - // Store in flat map for quick access - this.allDefinitions.set(definition.nodeId, definition); - - // Update indexes for performance - this.updateIndexes(definition); - } - - /** - * Update performance indexes when adding definitions - */ - private updateIndexes(definition: FunctionDefinition): void { - // Function name index - if (!this.functionNameIndex.has(definition.functionName)) { - this.functionNameIndex.set(definition.functionName, []); - } - this.functionNameIndex.get(definition.functionName)!.push(definition); - - // File path index - if (!this.filePathIndex.has(definition.filePath)) { - this.filePathIndex.set(definition.filePath, []); - } - this.filePathIndex.get(definition.filePath)!.push(definition); - } - - /** - * Find all definitions that end with the given name (OPTIMIZED) - * This is the key method for heuristic resolution - * @param name The function name to search for - * @returns Array of matching definitions - */ - findEndingWith(name: string): FunctionDefinition[] { - // Use index for O(1) lookup instead of O(n) tree traversal - return this.functionNameIndex.get(name) || []; - } - - /** - * Get exact definition by qualified name - * @param qualifiedName The full qualified name - * @returns The definition if found - */ - getExactMatch(qualifiedName: string): FunctionDefinition[] { - const parts = qualifiedName.split('.'); - let currentNode = this.root; - - for (const part of parts) { - if (!currentNode.children.has(part)) { - return []; - } - currentNode = currentNode.children.get(part)!; - } - - return currentNode.isEndOfWord ? currentNode.definitions : []; - } - - /** - * Find definitions in the same file (OPTIMIZED) - * @param filePath The file path to search in - * @param functionName The function name to find - * @returns Array of matching definitions in the same file - */ - findInSameFile(filePath: string, functionName: string): FunctionDefinition[] { - // Use file path index for faster lookup - const fileDefinitions = this.filePathIndex.get(filePath) || []; - return fileDefinitions.filter(def => def.functionName === functionName); - } - - /** - * Get all definitions in a specific file (NEW - OPTIMIZED) - * @param filePath The file path to search in - * @returns Array of all definitions in the file - */ - getDefinitionsInFile(filePath: string): FunctionDefinition[] { - return this.filePathIndex.get(filePath) || []; - } - - /** - * Find definitions by type (NEW - OPTIMIZED) - * @param type The definition type to search for - * @returns Array of matching definitions - */ - findByType(type: FunctionDefinition['type']): FunctionDefinition[] { - return Array.from(this.allDefinitions.values()).filter(def => def.type === type); - } - - /** - * Get all definitions for debugging/stats - * @returns All stored definitions - */ - getAllDefinitions(): FunctionDefinition[] { - return Array.from(this.allDefinitions.values()); - } - - /** - * Get statistics about the trie (NEW) - * @returns Trie statistics - */ - getStatistics(): { - totalDefinitions: number; - definitionsByType: Record; - fileCount: number; - uniqueFunctionNames: number; - } { - const definitionsByType: Record = {}; - - for (const definition of this.allDefinitions.values()) { - definitionsByType[definition.type] = (definitionsByType[definition.type] || 0) + 1; - } - - return { - totalDefinitions: this.allDefinitions.size, - definitionsByType, - fileCount: this.filePathIndex.size, - uniqueFunctionNames: this.functionNameIndex.size - }; - } - - /** - * Calculate import distance between two file paths - * Lower score = closer/better match - * @param callerPath Path of the calling file - * @param candidatePath Path of the candidate definition file - * @returns Distance score (lower is better) - */ - static calculateImportDistance(callerPath: string, candidatePath: string): number { - const callerParts = callerPath.split('/').filter(p => p !== ''); - const candidateParts = candidatePath.split('/').filter(p => p !== ''); - - // Find common prefix length - let commonPrefixLength = 0; - const minLength = Math.min(callerParts.length, candidateParts.length); - - for (let i = 0; i < minLength; i++) { - if (callerParts[i] === candidateParts[i]) { - commonPrefixLength++; - } else { - break; - } - } - - // Calculate base distance - const maxLength = Math.max(callerParts.length, candidateParts.length); - let distance = maxLength - commonPrefixLength; - - // Bonus for sibling modules (same parent directory) - if (commonPrefixLength === Math.min(callerParts.length, candidateParts.length) - 1) { - distance -= 1; // Sibling bonus - } - - return distance; - } - - /** - * Clear all data - */ - clear(): void { - this.root = { - children: new Map(), - definitions: [], - isEndOfWord: false - }; - this.allDefinitions.clear(); - this.functionNameIndex.clear(); - this.filePathIndex.clear(); - } - - /** - * Remove definitions from a specific file (NEW - for incremental updates) - * @param filePath The file path to remove definitions for - */ - removeFileDefinitions(filePath: string): void { - const definitions = this.filePathIndex.get(filePath); - if (!definitions) return; - - // Remove from all indexes - for (const definition of definitions) { - this.allDefinitions.delete(definition.nodeId); - - // Remove from function name index - const functionDefs = this.functionNameIndex.get(definition.functionName); - if (functionDefs) { - const index = functionDefs.indexOf(definition); - if (index > -1) { - functionDefs.splice(index, 1); - } - if (functionDefs.length === 0) { - this.functionNameIndex.delete(definition.functionName); - } - } - } - - // Remove from file path index - this.filePathIndex.delete(filePath); - - // TODO: Clean up the trie structure (complex operation, can be done later) - // For now, we'll leave empty nodes in the trie for performance - } -} - -export type { FunctionDefinition }; - -================ -File: src/core/graph/types.ts -================ -export type NodeLabel = - | 'Project' - | 'Package' - | 'Module' - | 'Folder' - | 'File' - | 'Class' - | 'Function' - | 'Method' - | 'Variable' - | 'Interface' - | 'Enum' - | 'Decorator' - | 'Import' - | 'Type' - | 'CodeElement'; - -export interface GraphNode { - id: string; - label: NodeLabel; - properties: NodeProperties; -} - -export type RelationshipType = - | 'CONTAINS' - | 'CALLS' - | 'INHERITS' - | 'OVERRIDES' - | 'IMPORTS' - | 'USES' - | 'DEFINES' - | 'DECORATES' - | 'IMPLEMENTS' - | 'ACCESSES' - | 'EXTENDS' - | 'BELONGS_TO'; - -export interface GraphRelationship { - id: string; - type: RelationshipType; - source: string; - target: string; - properties: RelationshipProperties; -} - -// Type-safe property interfaces -export interface NodeProperties { - // Common properties - name?: string; - path?: string; - filePath?: string; - extension?: string; - language?: string; - size?: number; - - // Project-specific - description?: string; - version?: string; - - // File-specific - definitionCount?: number; - lineCount?: number; - - // Definition-specific - type?: string; - startLine?: number; - endLine?: number; - qualifiedName?: string; - parameters?: string[]; - returnType?: string; - - // Relationship-specific - relationshipType?: string; - [key: string]: string | number | boolean | string[] | undefined; -} - -export interface RelationshipProperties { - // Common properties - strength?: number; - confidence?: number; - - // Import-specific - importType?: 'default' | 'named' | 'namespace'; - alias?: string; - - // Call-specific - callType?: 'function' | 'method' | 'constructor'; - arguments?: string[]; - - // Dependency-specific - dependencyType?: 'direct' | 'transitive' | 'dev'; - version?: string; - - [key: string]: string | number | boolean | string[] | undefined; -} - -export interface KnowledgeGraph { - nodes: GraphNode[]; - relationships: GraphRelationship[]; -} - -================ -File: src/core/ingestion/call-processor.ts -================ -import type { KnowledgeGraph, GraphRelationship } from '../graph/types.ts'; -import type { ParsedAST } from './parsing-processor.ts'; -import type { ImportMap } from './import-processor.ts'; -import { FunctionRegistryTrie } from '../graph/trie.ts'; -import { generateId } from '../../lib/utils.ts'; -import Parser from 'web-tree-sitter'; - -// Simple path utilities for browser compatibility -const pathUtils = { - extname: (filePath: string): string => { - const lastDot = filePath.lastIndexOf('.'); - return lastDot === -1 ? '' : filePath.substring(lastDot); - }, - dirname: (filePath: string): string => { - const lastSlash = filePath.lastIndexOf('/'); - return lastSlash === -1 ? '' : filePath.substring(0, lastSlash); - } -}; - -interface CallInfo { - callerFile: string; - callerFunction?: string; - functionName: string; - startLine: number; - endLine: number; - callType: 'function_call' | 'method_call' | 'constructor_call'; -} - -interface ResolutionResult { - success: boolean; - targetNodeId?: string; - stage: 'exact' | 'same_file' | 'heuristic' | 'failed'; - confidence: 'high' | 'medium' | 'low'; - distance?: number; -} - -export class CallProcessor { - private importMap: ImportMap = {}; - private functionTrie: FunctionRegistryTrie; - private astMap: Map = new Map(); - - // Statistics - private stats = { - totalCalls: 0, - exactMatches: 0, - sameFileMatches: 0, - heuristicMatches: 0, - failed: 0, - callTypes: {} as Record - }; - - constructor(functionTrie: FunctionRegistryTrie) { - this.functionTrie = functionTrie; - } - - /** - * Process function calls using the 3-stage resolution strategy - * This runs AFTER ImportProcessor has built the complete import map - */ - async process( - graph: KnowledgeGraph, - astMap: Map, - importMap: ImportMap - ): Promise { - console.log('CallProcessor: Starting call resolution with 3-stage strategy...'); - - this.importMap = importMap; - this.astMap = astMap; - this.resetStats(); - - // Process calls for each file - for (const [filePath, ast] of astMap) { - if (ast.tree) { - await this.processFileCalls(filePath, ast, graph); - } - } - - this.logStats(); - return graph; - } - - /** - * Process function calls in a single file - */ - private async processFileCalls( - filePath: string, - ast: ParsedAST, - graph: KnowledgeGraph - ): Promise { - const calls = this.extractFunctionCalls(ast.tree!.rootNode, filePath); - - if (calls.length === 0) { - // Only log for source files that should have function calls - if (this.isSourceFile(filePath)) { - console.log(`โš ๏ธ CallProcessor: No function calls found in source file: ${filePath}`); - - // Debug: Check if this file has any 'call' nodes at all - if (filePath.endsWith('.py')) { - const callNodeCount = this.countNodeType(ast.tree!.rootNode, 'call'); - const definitionCount = graph.nodes.filter(n => - (n.label === 'Function' || n.label === 'Class' || n.label === 'Method') && - n.properties.filePath === filePath - ).length; - - console.log(` ๐Ÿ“Š Debug: ${filePath.split('/').pop()} has ${callNodeCount} call nodes, ${definitionCount} definitions`); - - // If we have definitions but no calls, that's suspicious - if (definitionCount > 0 && callNodeCount === 0) { - console.log(` ๐Ÿšจ Suspicious: File has definitions but no call nodes - possible AST parsing issue`); - } - } - } - } else { - console.log(`CallProcessor: Found ${calls.length} function calls in ${filePath}`); - } - - for (const call of calls) { - this.stats.totalCalls++; - this.stats.callTypes[call.callType] = (this.stats.callTypes[call.callType] || 0) + 1; - - const resolution = await this.resolveCall(call); - - if (resolution.success && resolution.targetNodeId) { - this.createCallRelationship(graph, call, resolution.targetNodeId); - - // Update statistics - switch (resolution.stage) { - case 'exact': - this.stats.exactMatches++; - break; - case 'same_file': - this.stats.sameFileMatches++; - break; - case 'heuristic': - this.stats.heuristicMatches++; - break; - } - } else { - this.stats.failed++; - console.log(`โŒ Failed to resolve call: ${call.functionName} in ${call.callerFile}:${call.startLine}`); - } - } - } - - /** - * Count nodes of a specific type in the AST (for debugging) - */ - private countNodeType(node: Parser.SyntaxNode, nodeType: string): number { - let count = 0; - - if (node.type === nodeType) { - count++; - } - - // Recursively count in children - for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); - if (child) { - count += this.countNodeType(child, nodeType); - } - } - - return count; - } - - /** - * 3-Stage Call Resolution Strategy - */ - private async resolveCall(call: CallInfo): Promise { - // Stage 1: Exact Match using ImportMap - const exactResult = this.stageExactMatch(call); - if (exactResult.success) { - return exactResult; - } - - // Stage 2: Same-Module Match - const sameFileResult = this.stageSameFileMatch(call); - if (sameFileResult.success) { - return sameFileResult; - } - - // Stage 3: Heuristic Fallback - const heuristicResult = this.stageHeuristicMatch(call); - return heuristicResult; - } - - /** - * Stage 1: Exact Match using ImportMap (High Confidence) - */ - private stageExactMatch(call: CallInfo): ResolutionResult { - const importInfo = this.importMap[call.callerFile]?.[call.functionName]; - - if (importInfo) { - // We have an import for this function name - const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def => - def.filePath === importInfo.targetFile && - (def.functionName === importInfo.exportedName || - (importInfo.exportedName === 'default' && def.functionName === call.functionName)) - ); - - if (targetDefinitions.length > 0) { - return { - success: true, - targetNodeId: targetDefinitions[0].nodeId, - stage: 'exact', - confidence: 'high' - }; - } - } - - return { success: false, stage: 'exact', confidence: 'high' }; - } - - /** - * Stage 2: Same-Module Match (High Confidence) - */ - private stageSameFileMatch(call: CallInfo): ResolutionResult { - const sameFileDefinitions = this.functionTrie.findInSameFile(call.callerFile, call.functionName); - - if (sameFileDefinitions.length > 0) { - return { - success: true, - targetNodeId: sameFileDefinitions[0].nodeId, - stage: 'same_file', - confidence: 'high' - }; - } - - return { success: false, stage: 'same_file', confidence: 'high' }; - } - - /** - * Stage 3: Heuristic Fallback (Intelligent Guessing) - */ - private stageHeuristicMatch(call: CallInfo): ResolutionResult { - // Use trie to find all functions ending with this name - const candidates = this.functionTrie.findEndingWith(call.functionName); - - if (candidates.length === 0) { - return { success: false, stage: 'heuristic', confidence: 'low' }; - } - - // If only one candidate, use it - if (candidates.length === 1) { - return { - success: true, - targetNodeId: candidates[0].nodeId, - stage: 'heuristic', - confidence: 'medium' - }; - } - - // Multiple candidates - apply smart heuristics - let bestCandidate = candidates[0]; - let bestScore = this.calculateImportDistance(call.callerFile, bestCandidate.filePath); - - for (const candidate of candidates) { - let score = this.calculateImportDistance(call.callerFile, candidate.filePath); - - // Special handling for method calls - if (call.callType === 'method_call' && candidate.type === 'method') { - // Bonus for methods in the same file (likely self/this calls) - if (candidate.filePath === call.callerFile) { - score -= 2; // Strong preference for same-file methods - } - - // Bonus for methods in the same class context - // This would require more context about the calling class - // For now, we give a small bonus to method-to-method calls - score -= 0.5; - } - - // Bonus for function calls to functions (type matching) - if (call.callType === 'function_call' && candidate.type === 'function') { - score -= 0.5; - } - - // Bonus for sibling modules (same parent directory) - if (this.areSiblingModules(call.callerFile, candidate.filePath)) { - score -= 1; - } - - if (score < bestScore) { - bestScore = score; - bestCandidate = candidate; - } - } - - return { - success: true, - targetNodeId: bestCandidate.nodeId, - stage: 'heuristic', - confidence: bestScore <= 1 ? 'medium' : 'low' - }; - } - - /** - * Calculate import distance between two file paths - */ - private calculateImportDistance(callerFile: string, targetFile: string): number { - const callerParts = callerFile.split('/'); - const targetParts = targetFile.split('/'); - - // Find common prefix length - let commonPrefixLength = 0; - const minLength = Math.min(callerParts.length, targetParts.length); - - for (let i = 0; i < minLength; i++) { - if (callerParts[i] === targetParts[i]) { - commonPrefixLength++; - } else { - break; - } - } - - // Distance is max depth minus common prefix - return Math.max(callerParts.length, targetParts.length) - commonPrefixLength; - } - - /** - * Check if two file paths are sibling modules (same parent directory) - */ - private areSiblingModules(file1: string, file2: string): boolean { - const parent1 = pathUtils.dirname(file1); - const parent2 = pathUtils.dirname(file2); - return parent1 === parent2; - } - - /** - * Check if a function call should be ignored (built-ins, standard library, etc.) - */ - private shouldIgnoreCall(functionName: string, filePath: string): boolean { - // Python built-in functions that should be ignored - const pythonBuiltins = new Set([ - 'int', 'str', 'float', 'bool', 'list', 'dict', 'set', 'tuple', - 'len', 'range', 'enumerate', 'zip', 'map', 'filter', 'sorted', - 'sum', 'min', 'max', 'abs', 'round', 'all', 'any', 'hasattr', - 'getattr', 'setattr', 'isinstance', 'issubclass', 'type', - 'print', 'input', 'open', 'format', 'join', 'split', 'strip', - 'replace', 'upper', 'lower', 'append', 'extend', 'insert', - 'remove', 'pop', 'clear', 'copy', 'update', 'keys', 'values', - 'items', 'get', 'add', 'discard', 'union', 'intersection', - 'difference', 'now', 'today', 'fromisoformat', 'isoformat', - 'astimezone', 'random', 'choice', 'randint', 'shuffle', - 'locals', 'globals', 'vars', 'dir', 'help', 'id', 'hash', - 'ord', 'chr', 'bin', 'oct', 'hex', 'divmod', 'pow', 'exec', - 'eval', 'compile', 'next', 'iter', 'reversed', 'slice', - // String methods - 'endswith', 'startswith', 'find', 'rfind', 'index', 'rindex', - 'count', 'encode', 'decode', 'capitalize', 'title', 'swapcase', - 'center', 'ljust', 'rjust', 'zfill', 'expandtabs', 'splitlines', - 'partition', 'rpartition', 'translate', 'maketrans', 'casefold', - 'isalnum', 'isalpha', 'isascii', 'isdecimal', 'isdigit', 'isidentifier', - 'islower', 'isnumeric', 'isprintable', 'isspace', 'istitle', 'isupper', - 'lstrip', 'rstrip', 'removeprefix', 'removesuffix', - // List/sequence methods - 'sort', 'reverse', 'count', 'index', - // Dictionary methods - 'setdefault', 'popitem', 'fromkeys', - // Set methods - 'difference_update', 'intersection_update', 'symmetric_difference', - 'symmetric_difference_update', 'isdisjoint', 'issubset', 'issuperset', - // Date/time methods - 'strftime', 'strptime', 'timestamp', 'weekday', 'isoweekday', - 'date', 'time', 'timetz', 'utctimetuple', 'timetuple', - // Common exceptions - 'ValueError', 'TypeError', 'KeyError', 'IndexError', 'AttributeError', - 'ImportError', 'ModuleNotFoundError', 'FileNotFoundError', - 'ConnectionError', 'HTTPException', 'RuntimeError', 'OSError', - 'Exception', 'BaseException', 'StopIteration', 'GeneratorExit', - // Logging methods - 'debug', 'info', 'warning', 'error', 'critical', 'exception', - // Common library functions - 'getLogger', 'basicConfig', 'StreamHandler', 'load_dotenv', - 'getenv', 'dirname', 'abspath', 'join', 'exists', 'run', - // Database/ORM methods - 'find', 'find_one', 'update_one', 'insert_one', 'delete_one', - 'aggregate', 'bulk_write', 'to_list', 'sort', 'limit', 'close', - // Pydantic/FastAPI - 'Field', 'validator', 'field_validator', 'model_dump', 'model_dump_json', - // Motor/MongoDB - 'ObjectId', 'UpdateOne', 'AsyncIOMotorClient', 'command', - // FastAPI - 'FastAPI', 'HTTPException', 'add_middleware', 'include_router', - // Threading/async - 'Lock', 'RLock', 'Semaphore', 'Event', 'Condition', 'Barrier', - 'sleep', 'gather', 'create_task', 'run_until_complete', - // Collections - 'defaultdict', 'Counter', 'OrderedDict', 'deque', 'namedtuple', - // Math/statistics (numpy, pandas, statistics) - 'mean', 'median', 'mode', 'stdev', 'variance', 'sqrt', 'pow', - 'sin', 'cos', 'tan', 'log', 'exp', 'ceil', 'floor', - // UUID - 'uuid4', 'uuid1', 'uuid3', 'uuid5', - // URL/HTTP - 'quote', 'unquote', 'quote_plus', 'unquote_plus', 'urlencode', - // JSON - 'loads', 'dumps', 'load', 'dump', - // Regex - 'match', 'search', 'findall', 'finditer', 'sub', 'subn', 'compile', - // Azure/OpenAI specific - 'AsyncAzureOpenAI', 'AzureOpenAI', 'OpenAI', 'wrap_openai', 'create' - ]); - - // Check if it's a Python file and the function is a built-in - if (filePath.endsWith('.py') && pythonBuiltins.has(functionName)) { - return true; - } - - // Ignore very short function names (likely built-ins or operators) - if (functionName.length <= 2) { - return true; - } - - // Ignore common method patterns that are likely built-ins - const commonMethodPatterns = [ - /^__\w+__$/, // Dunder methods like __init__, __str__, etc. - /^\w+_$/, // Methods ending with underscore (often private) - ]; - - for (const pattern of commonMethodPatterns) { - if (pattern.test(functionName)) { - return true; - } - } - - return false; - } - - /** - * Extract function calls from AST - */ - private extractFunctionCalls(node: Parser.SyntaxNode, filePath: string): CallInfo[] { - const calls: CallInfo[] = []; - const language = this.detectLanguage(filePath); - - if (language === 'python') { - this.extractPythonCalls(node, filePath, calls); - } else { - this.extractJSCalls(node, filePath, calls); - } - - return calls; - } - - /** - * Extract Python function calls - */ - private extractPythonCalls(node: Parser.SyntaxNode, filePath: string, calls: CallInfo[]): void { - if (node.type === 'call') { - const functionNode = node.childForFieldName('function'); - if (functionNode) { - const functionName = this.extractPythonCallName(functionNode); - - // Debug: Log what we're finding vs filtering - if (functionName) { - const shouldIgnore = this.shouldIgnoreCall(functionName, filePath); - if (shouldIgnore) { - // Only log a few examples to avoid spam - if (calls.length < 3) { - console.log(`๐Ÿ” Filtered out: ${functionName} in ${filePath.split('/').pop()}`); - } - } else { - calls.push({ - callerFile: filePath, - functionName, - startLine: node.startPosition.row + 1, - endLine: node.endPosition.row + 1, - callType: 'function_call' - }); - } - } else { - // Debug: Log when we can't extract function name - if (calls.length < 3) { - console.log(`๐Ÿ” Could not extract function name from: ${functionNode.type} in ${filePath.split('/').pop()}`); - } - } - } - } - - // Recursively process children - for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); - if (child) { - this.extractPythonCalls(child, filePath, calls); - } - } - } - - /** - * Extract JavaScript/TypeScript function calls - */ - private extractJSCalls(node: Parser.SyntaxNode, filePath: string, calls: CallInfo[]): void { - if (node.type === 'call_expression') { - const functionNode = node.childForFieldName('function'); - if (functionNode) { - const functionName = this.extractJSCallName(functionNode); - if (functionName && !this.shouldIgnoreCall(functionName, filePath)) { - calls.push({ - callerFile: filePath, - functionName, - startLine: node.startPosition.row + 1, - endLine: node.endPosition.row + 1, - callType: functionNode.type === 'member_expression' ? 'method_call' : 'function_call' - }); - } - } - } else if (node.type === 'new_expression') { - const constructorNode = node.childForFieldName('constructor'); - if (constructorNode) { - const constructorName = constructorNode.text; - calls.push({ - callerFile: filePath, - functionName: constructorName, - startLine: node.startPosition.row + 1, - endLine: node.endPosition.row + 1, - callType: 'constructor_call' - }); - } - } - - // Recursively process children - for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); - if (child) { - this.extractJSCalls(child, filePath, calls); - } - } - } - - /** - * Extract function name from Python call node - */ - private extractPythonCallName(node: Parser.SyntaxNode): string | null { - if (node.type === 'identifier') { - return node.text; - } else if (node.type === 'attribute') { - // For method calls like obj.method(), we want just 'method' - const attributeNode = node.childForFieldName('attribute'); - return attributeNode ? attributeNode.text : null; - } else if (node.type === 'subscript') { - // For calls like obj[key](), try to get the base object - const valueNode = node.childForFieldName('value'); - if (valueNode) { - return this.extractPythonCallName(valueNode); - } - } else if (node.type === 'call') { - // Nested call - try to get the function being called - const functionNode = node.childForFieldName('function'); - if (functionNode) { - return this.extractPythonCallName(functionNode); - } - } - - // Debug: Log unhandled node types (but limit spam) - if (Math.random() < 0.1) { // Only log 10% of cases to avoid spam - console.log(`๐Ÿ” Unhandled Python call node type: ${node.type} (text: "${node.text}")`); - } - - return null; - } - - /** - * Extract function name from JavaScript call node - */ - private extractJSCallName(node: Parser.SyntaxNode): string | null { - if (node.type === 'identifier') { - return node.text; - } else if (node.type === 'member_expression') { - // For method calls like obj.method(), we want just 'method' - const propertyNode = node.childForFieldName('property'); - return propertyNode ? propertyNode.text : null; - } - return null; - } - - /** - * Create CALLS relationship in the graph - */ - private createCallRelationship( - graph: KnowledgeGraph, - call: CallInfo, - targetNodeId: string - ): void { - // Find the caller node (could be a function, method, or file) - const callerNode = this.findCallerNode(graph, call); - - if (callerNode) { - const relationship: GraphRelationship = { - id: generateId('calls', `${callerNode.id}-calls-${targetNodeId}`), - type: 'CALLS', - source: callerNode.id, - target: targetNodeId, - properties: { - callType: call.callType, - functionName: call.functionName, - startLine: call.startLine, - endLine: call.endLine - } - }; - - // Check if relationship already exists - const existingRel = graph.relationships.find(r => - r.type === 'CALLS' && - r.source === callerNode.id && - r.target === targetNodeId - ); - - if (!existingRel) { - graph.relationships.push(relationship); - } - } - } - - /** - * Find the caller node in the graph - */ - private findCallerNode(graph: KnowledgeGraph, call: CallInfo): any { - // First try to find a function/method that contains this call - const containingFunction = graph.nodes.find(node => - (node.label === 'Function' || node.label === 'Method') && - node.properties.filePath === call.callerFile && - (node.properties.startLine as number) <= call.startLine && - (node.properties.endLine as number) >= call.endLine - ); - - if (containingFunction) { - return containingFunction; - } - - // If no containing function found, try to find a class that contains this call - // This helps with method calls at class level - const containingClass = graph.nodes.find(node => - node.label === 'Class' && - node.properties.filePath === call.callerFile && - (node.properties.startLine as number) <= call.startLine && - (node.properties.endLine as number) >= call.endLine - ); - - if (containingClass) { - return containingClass; - } - - // Fallback to file node - return graph.nodes.find(node => - node.label === 'File' && - node.properties.filePath === call.callerFile - ); - } - - /** - * Detect programming language - */ - private detectLanguage(filePath: string): 'python' | 'javascript' { - const ext = pathUtils.extname(filePath).toLowerCase(); - return ext === '.py' ? 'python' : 'javascript'; - } - - /** - * Reset statistics - */ - private resetStats(): void { - this.stats = { - totalCalls: 0, - exactMatches: 0, - sameFileMatches: 0, - heuristicMatches: 0, - failed: 0, - callTypes: {} - }; - } - - /** - * Log resolution statistics - */ - private logStats(): void { - console.log('๐Ÿ“Š CallProcessor Resolution Statistics:'); - console.log(` Total calls processed: ${this.stats.totalCalls}`); - console.log(` โœ… Exact matches (Stage 1): ${this.stats.exactMatches} (${((this.stats.exactMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`); - console.log(` โœ… Same-file matches (Stage 2): ${this.stats.sameFileMatches} (${((this.stats.sameFileMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`); - console.log(` ๐ŸŽฏ Heuristic matches (Stage 3): ${this.stats.heuristicMatches} (${((this.stats.heuristicMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`); - console.log(` โŒ Failed resolutions: ${this.stats.failed} (${((this.stats.failed / this.stats.totalCalls) * 100).toFixed(1)}%)`); - console.log(` Success rate: ${(((this.stats.totalCalls - this.stats.failed) / this.stats.totalCalls) * 100).toFixed(1)}%`); - } - - /** - * Get resolution statistics - */ - getStats() { - return this.stats; - } - - /** - * Clear all data - */ - clear(): void { - this.importMap = {}; - this.astMap.clear(); - this.resetStats(); - } - - /** - * Check if a file is a source file that should contain function calls - */ - private isSourceFile(filePath: string): boolean { - const sourceExtensions = ['.py', '.js', '.ts', '.jsx', '.tsx']; - const ext = pathUtils.extname(filePath).toLowerCase(); - return sourceExtensions.includes(ext); - } -} - -================ -File: src/core/ingestion/import-processor.ts -================ -import type { KnowledgeGraph, GraphRelationship } from '../graph/types.ts'; -import type { ParsedAST } from './parsing-processor.ts'; -import Parser from 'web-tree-sitter'; - -// Simple path utilities for browser compatibility -const pathUtils = { - extname: (filePath: string): string => { - const lastDot = filePath.lastIndexOf('.'); - return lastDot === -1 ? '' : filePath.substring(lastDot); - }, - dirname: (filePath: string): string => { - const lastSlash = Math.max(filePath.lastIndexOf('/'), filePath.lastIndexOf('\\')); - return lastSlash === -1 ? '.' : filePath.substring(0, lastSlash); - }, - resolve: (basePath: string, relativePath: string): string => { - // Simple relative path resolution - if (relativePath.startsWith('./')) { - return basePath + '/' + relativePath.substring(2); - } else if (relativePath.startsWith('../')) { - const parts = basePath.split('/'); - const relativeParts = relativePath.split('/'); - let upCount = 0; - for (const part of relativeParts) { - if (part === '..') upCount++; - else break; - } - const resultParts = parts.slice(0, -upCount); - const remainingParts = relativeParts.slice(upCount); - return [...resultParts, ...remainingParts].join('/'); - } - return basePath + '/' + relativePath; - }, - join: (...parts: string[]): string => { - return parts.join('/').replace(/\/+/g, '/'); - } -}; - -interface ImportMap { - [importingFile: string]: { - [localName: string]: { - targetFile: string; - exportedName: string; - importType: 'default' | 'named' | 'namespace' | 'dynamic'; - } - } -} - -interface ImportInfo { - importingFile: string; - localName: string; - targetFile: string; - exportedName: string; - importType: 'default' | 'named' | 'namespace' | 'dynamic'; -} - -export class ImportProcessor { - private importMap: ImportMap = {}; - private projectFiles: Set = new Set(); - - /** - * Process all imports after parsing is complete - * @param graph The knowledge graph being built - * @param astMap Map of file paths to their parsed ASTs - * @param fileContents Map of file contents - * @returns Updated graph with import relationships - */ - async process( - graph: KnowledgeGraph, - astMap: Map, - fileContents: Map - ): Promise { - console.log('ImportProcessor: Starting import resolution...'); - - // Build set of all project files for validation - this.projectFiles = new Set(fileContents.keys()); - - // Clear previous import map - this.importMap = {}; - - let totalImportsFound = 0; - let totalImportsResolved = 0; - - // Process imports for each file - for (const [filePath, ast] of astMap) { - const fileImports = await this.processFileImports(filePath, ast, graph); - totalImportsFound += fileImports.found; - totalImportsResolved += fileImports.resolved; - } - - console.log('ImportProcessor: Completed import resolution'); - console.log(`ImportProcessor: Found ${totalImportsFound} imports, resolved ${totalImportsResolved} (${((totalImportsResolved/totalImportsFound)*100).toFixed(1)}%)`); - console.log(`ImportProcessor: Built import map for ${Object.keys(this.importMap).length} files`); - - return graph; - } - - /** - * Process imports for a single file - */ - private async processFileImports( - filePath: string, - ast: ParsedAST, - graph: KnowledgeGraph - ): Promise<{ found: number; resolved: number }> { - if (!ast.tree) return { found: 0, resolved: 0 }; - - const imports = this.extractImports(ast.tree.rootNode, filePath); - - if (imports.length === 0) return { found: 0, resolved: 0 }; - - // Initialize import map for this file - this.importMap[filePath] = {}; - - let found = 0; - let resolved = 0; - - for (const importInfo of imports) { - // Store in import map - this.importMap[filePath][importInfo.localName] = { - targetFile: importInfo.targetFile, - exportedName: importInfo.exportedName, - importType: importInfo.importType - }; - - // Create IMPORTS relationship in graph - this.createImportRelationship(graph, importInfo); - found++; - if (importInfo.targetFile !== importInfo.exportedName) { // Only count as resolved if it's not a default import - resolved++; - } - } - return { found, resolved }; - } - - /** - * Extract import statements from AST - */ - private extractImports(rootNode: Parser.SyntaxNode, filePath: string): ImportInfo[] { - const imports: ImportInfo[] = []; - const language = this.detectLanguage(filePath); - - if (language === 'python') { - this.extractPythonImports(rootNode, filePath, imports); - } else if (language === 'javascript' || language === 'typescript') { - this.extractJSImports(rootNode, filePath, imports); - } - - return imports; - } - - /** - * Extract Python imports - */ - private extractPythonImports( - node: Parser.SyntaxNode, - filePath: string, - imports: ImportInfo[] - ): void { - if (node.type === 'import_statement') { - // Handle: import module - // Handle: import module as alias - const moduleNode = node.childForFieldName('name'); - if (moduleNode) { - const moduleName = moduleNode.text; - const targetFile = this.resolveModulePath(moduleName, filePath, 'python'); - - imports.push({ - importingFile: filePath, - localName: moduleName.split('.').pop() || moduleName, - targetFile, - exportedName: moduleName, - importType: 'namespace' - }); - } - } else if (node.type === 'import_from_statement') { - // Handle: from module import name - // Handle: from module import name as alias - const moduleNode = node.childForFieldName('module_name'); - const namesNode = node.childForFieldName('name'); - - if (moduleNode && namesNode) { - const moduleName = moduleNode.text; - const targetFile = this.resolveModulePath(moduleName, filePath, 'python'); - - // Handle multiple imports: from module import a, b, c - if (namesNode.type === 'import_list') { - for (let i = 0; i < namesNode.childCount; i++) { - const nameNode = namesNode.child(i); - if (nameNode && nameNode.type === 'import_from_statement') { - const importName = nameNode.text; - imports.push({ - importingFile: filePath, - localName: importName, - targetFile, - exportedName: importName, - importType: 'named' - }); - } - } - } else { - const importName = namesNode.text; - imports.push({ - importingFile: filePath, - localName: importName, - targetFile, - exportedName: importName, - importType: 'named' - }); - } - } - } - - // Recursively process children - for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); - if (child) { - this.extractPythonImports(child, filePath, imports); - } - } - } - - /** - * Extract JavaScript/TypeScript imports - */ - private extractJSImports( - node: Parser.SyntaxNode, - filePath: string, - imports: ImportInfo[] - ): void { - if (node.type === 'import_statement') { - const sourceNode = node.childForFieldName('source'); - if (!sourceNode) return; - - const sourcePath = sourceNode.text.replace(/['"]/g, ''); - const targetFile = this.resolveModulePath(sourcePath, filePath, 'javascript'); - - // Handle different import patterns - const importClauseNode = node.childForFieldName('import_clause'); - if (importClauseNode) { - this.processJSImportClause(importClauseNode, filePath, targetFile, imports); - } - } else if (node.type === 'variable_declaration') { - // Handle CommonJS: const x = require('module') - this.processRequireStatement(node, filePath, imports); - } - - // Recursively process children - for (let i = 0; i < node.childCount; i++) { - const child = node.child(i); - if (child) { - this.extractJSImports(child, filePath, imports); - } - } - } - - /** - * Process JS import clause (handles named, default, namespace imports) - */ - private processJSImportClause( - importClauseNode: Parser.SyntaxNode, - filePath: string, - targetFile: string, - imports: ImportInfo[] - ): void { - for (let i = 0; i < importClauseNode.childCount; i++) { - const child = importClauseNode.child(i); - if (!child) continue; - - if (child.type === 'import_specifier') { - // Named import: { name } or { name as alias } - const nameNode = child.childForFieldName('name'); - const aliasNode = child.childForFieldName('alias'); - - if (nameNode) { - const exportedName = nameNode.text; - const localName = aliasNode ? aliasNode.text : exportedName; - - imports.push({ - importingFile: filePath, - localName, - targetFile, - exportedName, - importType: 'named' - }); - } - } else if (child.type === 'namespace_import') { - // Namespace import: * as name - const nameNode = child.childForFieldName('name'); - if (nameNode) { - imports.push({ - importingFile: filePath, - localName: nameNode.text, - targetFile, - exportedName: '*', - importType: 'namespace' - }); - } - } else if (child.type === 'identifier') { - // Default import - imports.push({ - importingFile: filePath, - localName: child.text, - targetFile, - exportedName: 'default', - importType: 'default' - }); - } - } - } - - /** - * Process CommonJS require statements - */ - private processRequireStatement( - node: Parser.SyntaxNode, - filePath: string, - imports: ImportInfo[] - ): void { - // Look for: const x = require('module') - const declaratorNode = node.child(1); // variable_declarator - if (!declaratorNode) return; - - const nameNode = declaratorNode.childForFieldName('name'); - const valueNode = declaratorNode.childForFieldName('value'); - - if (nameNode && valueNode && valueNode.type === 'call_expression') { - const functionNode = valueNode.childForFieldName('function'); - const argumentsNode = valueNode.childForFieldName('arguments'); - - if (functionNode?.text === 'require' && argumentsNode) { - const firstArg = argumentsNode.child(1); // Skip opening paren - if (firstArg && firstArg.type === 'string') { - const modulePath = firstArg.text.replace(/['"]/g, ''); - const targetFile = this.resolveModulePath(modulePath, filePath, 'javascript'); - - imports.push({ - importingFile: filePath, - localName: nameNode.text, - targetFile, - exportedName: 'default', - importType: 'dynamic' - }); - } - } - } - } - - /** - * Resolve module path to actual file path - */ - private resolveModulePath(moduleName: string, importingFile: string, language: 'python' | 'javascript'): string { - // Handle relative imports - if (moduleName.startsWith('.')) { - const importingDir = pathUtils.dirname(importingFile); - const resolvedPath = pathUtils.resolve(importingDir, moduleName); - - // Try different extensions - const extensions = language === 'python' ? ['.py'] : ['.js', '.ts', '.tsx', '.jsx']; - - for (const ext of extensions) { - const candidate = resolvedPath + ext; - if (this.projectFiles.has(candidate)) { - return candidate; - } - } - - // Try index files - for (const ext of extensions) { - const indexCandidate = pathUtils.join(resolvedPath, `index${ext}`); - if (this.projectFiles.has(indexCandidate)) { - return indexCandidate; - } - } - - return resolvedPath; // Return even if not found, for external modules - } - - // Handle absolute/package imports for Python - if (language === 'python') { - // First, try to find files that match the module pattern - const modulePatterns = [ - // Direct module.py - moduleName.replace(/\./g, '/') + '.py', - // Package with __init__.py - moduleName.replace(/\./g, '/') + '/__init__.py', - // Try within the project structure - `src/python/${moduleName.replace(/\./g, '/')}.py`, - `src/python/${moduleName.replace(/\./g, '/')}/__init__.py`, - ]; - - // Also try to match partial paths for complex project structures - for (const filePath of this.projectFiles) { - if (filePath.endsWith('.py')) { - // Check if this file could match the module name - const moduleSegments = moduleName.split('.'); - const pathSegments = filePath.replace('.py', '').split('/'); - - // Try to match the last few segments - if (moduleSegments.length > 0) { - const lastSegment = moduleSegments[moduleSegments.length - 1]; - const fileName = pathSegments[pathSegments.length - 1]; - - // If the last segment matches the filename, this could be it - if (fileName === lastSegment) { - // Check if the path contains the module structure - const modulePathInFile = moduleSegments.slice(0, -1).join('/'); - if (!modulePathInFile || filePath.includes(modulePathInFile)) { - return filePath; - } - } - } - } - } - - // Try the direct patterns - for (const pattern of modulePatterns) { - if (this.projectFiles.has(pattern)) { - return pattern; - } - } - - // For complex module paths, try to find any file that ends with the module name - const lastModuleSegment = moduleName.split('.').pop(); - if (lastModuleSegment) { - for (const filePath of this.projectFiles) { - if (filePath.endsWith(`${lastModuleSegment}.py`)) { - return filePath; - } - } - } - } - - // For external modules or unresolved, return as-is - return moduleName; - } - - /** - * Create IMPORTS relationship in the graph - */ - private createImportRelationship(graph: KnowledgeGraph, importInfo: ImportInfo): void { - // Find source and target nodes - const sourceNode = graph.nodes.find(n => - n.label === 'File' && n.properties.filePath === importInfo.importingFile - ); - - const targetNode = graph.nodes.find(n => - n.label === 'File' && n.properties.filePath === importInfo.targetFile - ); - - if (sourceNode && targetNode && sourceNode.id !== targetNode.id) { - // Check if relationship already exists - const existingRel = graph.relationships.find(r => - r.type === 'IMPORTS' && - r.source === sourceNode.id && - r.target === targetNode.id - ); - - if (!existingRel) { - const relationship: GraphRelationship = { - id: `imports_${sourceNode.id}_${targetNode.id}_${Date.now()}`, - type: 'IMPORTS', - source: sourceNode.id, - target: targetNode.id, - properties: { - importType: importInfo.importType, - localName: importInfo.localName, - exportedName: importInfo.exportedName - } - }; - - graph.relationships.push(relationship); - } - } - } - - /** - * Detect programming language from file extension - */ - private detectLanguage(filePath: string): 'python' | 'javascript' | 'typescript' { - const ext = pathUtils.extname(filePath).toLowerCase(); - - if (ext === '.py') return 'python'; - if (ext === '.ts' || ext === '.tsx') return 'typescript'; - return 'javascript'; // .js, .jsx, or default - } - - /** - * Get the complete import map for use by CallProcessor - */ - getImportMap(): ImportMap { - return this.importMap; - } - - /** - * Get import info for a specific file and local name - */ - getImportInfo(filePath: string, localName: string): ImportMap[string][string] | null { - return this.importMap[filePath]?.[localName] || null; - } - - /** - * Clear all data - */ - clear(): void { - this.importMap = {}; - this.projectFiles.clear(); - } -} - -export type { ImportMap, ImportInfo }; - -================ -File: src/core/ingestion/parallel-parsing-processor.ts -================ -import { GraphNode, GraphRelationship, NodeLabel, RelationshipType } from '../graph/types.js'; -import { MemoryManager } from '../../services/memory-manager.js'; -import { KnowledgeGraph, GraphProcessor } from '../graph/graph.js'; -import { - pathUtils, - OptimizedSet, - DuplicateDetector -} from '../../lib/shared-utils.js'; -import { IGNORE_PATTERNS } from '../../config/language-config.js'; -import { WebWorkerPool, WebWorkerPoolUtils } from '../../lib/web-worker-pool.js'; -import { FunctionRegistryTrie, FunctionDefinition } from '../graph/trie.js'; - -export interface ParsingInput { - filePaths: string[]; - fileContents: Map; - options?: { directoryFilter?: string; fileExtensions?: string }; -} - -export interface ParsedDefinition { - name: string; - type: 'function' | 'class' | 'method' | 'variable' | 'import' | 'interface' | 'type' | 'decorator'; - startLine: number; - endLine?: number; - parameters?: string[] | undefined; - returnType?: string | undefined; - accessibility?: 'public' | 'private' | 'protected'; - isStatic?: boolean | undefined; - isAsync?: boolean | undefined; - parentClass?: string | undefined; - decorators?: string[] | undefined; - extends?: string[] | undefined; - implements?: string[] | undefined; - importPath?: string | undefined; - exportType?: 'named' | 'default' | 'namespace'; - docstring?: string | undefined; -} - -export interface ParsedAST { - tree: any; -} - -export interface ParallelParsingResult { - filePath: string; - definitions: ParsedDefinition[]; - ast: ParsedAST; - success: boolean; - error?: string; -} - -function generateId(prefix: string, identifier: string): string { - let hash = 0; - for (let i = 0; i < identifier.length; i++) { - const char = identifier.charCodeAt(i); - hash = ((hash << 5) - hash) + char; - hash = hash & hash; - } - return `${prefix}-${Math.abs(hash)}-${identifier.replace(/[^a-zA-Z0-9]/g, '_')}`; -} - -export class ParallelParsingProcessor implements GraphProcessor { - private memoryManager: MemoryManager; - private duplicateDetector = new DuplicateDetector((item: string) => item); - private processedFiles = new OptimizedSet(); - private astMap: Map = new Map(); - private functionTrie: FunctionRegistryTrie = new FunctionRegistryTrie(); - private workerPool: WebWorkerPool; - private isInitialized: boolean = false; - - constructor() { - this.memoryManager = MemoryManager.getInstance(); - this.workerPool = WebWorkerPoolUtils.createCPUPool({ - workerScript: '/workers/tree-sitter-worker.js', - name: 'ParallelParsingPool', - timeout: 60000 // 60 seconds for parsing - }); - } - - public getASTMap(): Map { - return this.astMap; - } - - public getFunctionRegistry(): FunctionRegistryTrie { - return this.functionTrie; - } - - /** - * Initialize the worker pool - */ - private async initializeWorkerPool(): Promise { - if (this.isInitialized) return; - - try { - console.log('ParallelParsingProcessor: Initializing worker pool...'); - - // Check if Web Workers are supported - if (!WebWorkerPoolUtils.isSupported()) { - throw new Error('Web Workers are not supported in this environment'); - } - - // Set up worker pool event listeners - this.workerPool.on('workerCreated', (data) => { - console.log(`ParallelParsingProcessor: Worker ${data.workerId} created (${data.totalWorkers} total)`); - }); - - this.workerPool.on('workerError', (data) => { - console.warn(`ParallelParsingProcessor: Worker ${data.workerId} error:`, data.error); - }); - - this.workerPool.on('shutdown', () => { - console.log('ParallelParsingProcessor: Worker pool shutdown'); - }); - - this.isInitialized = true; - console.log('ParallelParsingProcessor: Worker pool initialized successfully'); - } catch (error) { - console.error('ParallelParsingProcessor: Failed to initialize worker pool:', error); - throw error; - } - } - - public async process(graph: KnowledgeGraph, input: ParsingInput): Promise { - const { filePaths, fileContents, options } = input; - - console.log(`ParallelParsingProcessor: Processing ${filePaths.length} total paths with worker pool`); - - const memoryStats = this.memoryManager.getStats(); - console.log(`Memory status: ${memoryStats.usedMemoryMB}MB used, ${memoryStats.fileCount} files cached`); - - // Initialize worker pool - await this.initializeWorkerPool(); - - const filteredFiles = this.applyFiltering(filePaths, fileContents, options); - - console.log(`ParallelParsingProcessor: After filtering: ${filteredFiles.length} files to parse`); - - const sourceFiles = filteredFiles.filter((path: string) => this.isSourceFile(path)); - const configFiles = filteredFiles.filter((path: string) => this.isConfigFile(path)); - const allProcessableFiles = [...sourceFiles, ...configFiles]; - - console.log(`ParallelParsingProcessor: Found ${sourceFiles.length} source files and ${configFiles.length} config files`); - - try { - // Process files in parallel using worker pool - const results = await this.processFilesInParallel(allProcessableFiles, fileContents); - - // Process results and build graph - await this.processResults(results, graph); - - console.log(`ParallelParsingProcessor: Successfully processed ${this.processedFiles.size} files`); - } catch (error) { - console.error('ParallelParsingProcessor: Error during parallel processing:', error); - throw error; - } - } - - /** - * Process files in parallel using worker pool - */ - private async processFilesInParallel( - filePaths: string[], - fileContents: Map - ): Promise { - const startTime = performance.now(); - - // Prepare tasks for worker pool - const tasks = filePaths.map(filePath => ({ - filePath, - content: fileContents.get(filePath) || '' - })); - - console.log(`ParallelParsingProcessor: Starting parallel processing of ${tasks.length} files`); - - try { - // Process with progress tracking - const results = await this.workerPool.executeWithProgress( - tasks, - (completed, total) => { - const progress = ((completed / total) * 100).toFixed(1); - console.log(`ParallelParsingProcessor: Progress: ${progress}% (${completed}/${total})`); - } - ); - - const endTime = performance.now(); - const duration = endTime - startTime; - - console.log(`ParallelParsingProcessor: Parallel processing completed in ${duration.toFixed(2)}ms`); - console.log(`ParallelParsingProcessor: Average time per file: ${(duration / tasks.length).toFixed(2)}ms`); - - // Log worker pool statistics - const stats = this.workerPool.getStats(); - console.log('ParallelParsingProcessor: Worker pool stats:', stats); - - return results; - } catch (error) { - console.error('ParallelParsingProcessor: Error in parallel processing:', error); - throw error; - } - } - - /** - * Process parsing results and build graph - */ - private async processResults(results: ParallelParsingResult[], graph: KnowledgeGraph): Promise { - console.log(`ParallelParsingProcessor: Processing ${results.length} parsing results`); - - let successfulFiles = 0; - let failedFiles = 0; - let totalDefinitions = 0; - - for (const result of results) { - if (result.success) { - successfulFiles++; - - // Store AST - if (result.ast) { - this.astMap.set(result.filePath, result.ast); - } - - // Process definitions - if (result.definitions && result.definitions.length > 0) { - await this.processDefinitions(result.filePath, result.definitions, graph); - totalDefinitions += result.definitions.length; - } - - this.processedFiles.add(result.filePath); - } else { - failedFiles++; - console.warn(`ParallelParsingProcessor: Failed to parse ${result.filePath}: ${result.error}`); - } - } - - console.log(`ParallelParsingProcessor: Processing complete - ${successfulFiles} successful, ${failedFiles} failed`); - console.log(`ParallelParsingProcessor: Total definitions extracted: ${totalDefinitions}`); - } - - /** - * Process definitions and add to graph - */ - private async processDefinitions( - filePath: string, - definitions: ParsedDefinition[], - graph: KnowledgeGraph - ): Promise { - for (const definition of definitions) { - try { - await this.addDefinitionToGraph(filePath, definition, graph); - } catch (error) { - console.warn(`ParallelParsingProcessor: Error processing definition ${definition.name}:`, error); - } - } - } - - /** - * Add a definition to the graph - */ - private async addDefinitionToGraph( - filePath: string, - definition: ParsedDefinition, - graph: KnowledgeGraph - ): Promise { - const nodeId = generateId(definition.type, `${filePath}:${definition.name}`); - - if (this.duplicateDetector.isDuplicate(nodeId)) { - return; - } - - // Create graph node - const node: GraphNode = { - id: nodeId, - label: this.mapDefinitionTypeToNodeLabel(definition.type), - properties: { - name: definition.name, - filePath, - startLine: definition.startLine, - endLine: definition.endLine, - type: definition.type, - parameters: definition.parameters?.join(', '), - returnType: definition.returnType, - accessibility: definition.accessibility, - isStatic: definition.isStatic, - isAsync: definition.isAsync, - parentClass: definition.parentClass, - decorators: definition.decorators?.join(', '), - extends: definition.extends?.join(', '), - implements: definition.implements?.join(', '), - importPath: definition.importPath, - exportType: definition.exportType, - docstring: definition.docstring - } - }; - - graph.addNode(node); - - // Add to function registry if applicable - if (definition.type === 'function' || definition.type === 'method') { - const functionDef: FunctionDefinition = { - name: definition.name, - filePath, - startLine: definition.startLine, - endLine: definition.endLine, - parameters: definition.parameters || [], - returnType: definition.returnType, - isStatic: definition.isStatic, - isAsync: definition.isAsync, - parentClass: definition.parentClass - }; - - this.functionTrie.insert(functionDef); - } - - // Add CONTAINS relationship from file to definition - const fileNodeId = generateId('file', filePath); - const containsRel: GraphRelationship = { - id: generateId('contains', `${fileNodeId}:${nodeId}`), - type: 'CONTAINS', - source: fileNodeId, - target: nodeId, - properties: { - filePath, - definitionType: definition.type - } - }; - - graph.addRelationship(containsRel); - } - - /** - * Map definition type to node label - */ - private mapDefinitionTypeToNodeLabel(type: string): NodeLabel { - switch (type) { - case 'function': - return 'Function'; - case 'class': - return 'Class'; - case 'method': - return 'Method'; - case 'variable': - return 'Variable'; - case 'import': - return 'Import'; - case 'interface': - return 'Interface'; - case 'type': - return 'Type'; - case 'decorator': - return 'Decorator'; - default: - return 'CodeElement'; - } - } - - private applyFiltering( - filePaths: string[], - fileContents: Map, - options?: { directoryFilter?: string; fileExtensions?: string }): string[] { - - let filtered = filePaths; - - if (options?.directoryFilter) { - filtered = filtered.filter(path => path.includes(options.directoryFilter ?? '')); - } - - if (options?.fileExtensions) { - const extensions = options.fileExtensions.split(',').map(ext => ext.trim()).filter(ext => ext.length); - filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext))); - } - - filtered = filtered.filter(path => - ![...IGNORE_PATTERNS].some(pattern => { - if (typeof pattern === 'string') { - return path.includes(pattern); - } - return false; - }) - ); - - filtered = filtered.filter(path => { - const content = fileContents.get(path); - return content && content.trim().length > 0; - }); - - return filtered; - } - - private isSourceFile(filePath: string): boolean { - const sourceExtensions = ['.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', '.cs', '.php', '.rb', '.go', '.rs']; - return sourceExtensions.some(ext => filePath.toLowerCase().endsWith(ext)); - } - - private isConfigFile(filePath: string): boolean { - const configFiles = ['package.json', 'tsconfig.json', 'webpack.config.js', 'vite.config.ts', '.eslintrc.js', '.prettierrc']; - const configExtensions = ['.json', '.yaml', '.yml', '.toml']; - - return configFiles.some(name => filePath.endsWith(name)) || - configExtensions.some(ext => filePath.toLowerCase().endsWith(ext)); - } - - /** - * Shutdown the worker pool - */ - public async shutdown(): Promise { - if (this.workerPool) { - await this.workerPool.shutdown(); - } - } - - /** - * Get worker pool statistics - */ - public getWorkerPoolStats() { - return this.workerPool ? this.workerPool.getStats() : null; - } -} - -================ -File: src/core/ingestion/parallel-pipeline.ts -================ -import { SimpleKnowledgeGraph } from '../graph/graph.js'; -import type { KnowledgeGraph } from '../graph/types.ts'; -import { StructureProcessor } from './structure-processor.ts'; -import { ParallelParsingProcessor } from './parallel-parsing-processor.ts'; -import { ImportProcessor } from './import-processor.ts'; -import { CallProcessor } from './call-processor.ts'; -import { WebWorkerPoolUtils } from '../../lib/web-worker-pool.js'; - -export interface PipelineInput { - projectRoot: string; - projectName: string; - filePaths: string[]; - fileContents: Map; - options?: { - directoryFilter?: string; - fileExtensions?: string; - useParallelProcessing?: boolean; - maxWorkers?: number; - }; -} - -export interface PipelineProgress { - phase: 'structure' | 'parsing' | 'imports' | 'calls'; - message: string; - progress: number; - timestamp: number; -} - -export class ParallelGraphPipeline { - private structureProcessor: StructureProcessor; - private parsingProcessor: ParallelParsingProcessor; - private importProcessor: ImportProcessor; - private callProcessor!: CallProcessor; - private progressCallback?: (progress: PipelineProgress) => void; - - constructor() { - this.structureProcessor = new StructureProcessor(); - this.parsingProcessor = new ParallelParsingProcessor(); - this.importProcessor = new ImportProcessor(); - } - - /** - * Set progress callback - */ - public setProgressCallback(callback: (progress: PipelineProgress) => void): void { - this.progressCallback = callback; - } - - /** - * Update progress - */ - private updateProgress(phase: PipelineProgress['phase'], message: string, progress: number): void { - if (this.progressCallback) { - this.progressCallback({ - phase, - message, - progress: Math.min(progress, 100), - timestamp: Date.now() - }); - } - } - - public async run(input: PipelineInput): Promise { - const { projectRoot, projectName, filePaths, fileContents, options } = input; - - const graph = new SimpleKnowledgeGraph(); - const startTime = performance.now(); - - console.log(`๐Ÿš€ Starting parallel 4-pass ingestion for project: ${projectName}`); - console.log(`๐Ÿ“Š Processing ${filePaths.length} files with ${options?.useParallelProcessing ? 'parallel' : 'sequential'} processing`); - - try { - // Pass 1: Structure Analysis (Sequential - lightweight) - console.log('๐Ÿ“ Pass 1: Analyzing project structure...'); - this.updateProgress('structure', 'Analyzing project structure...', 0); - - await this.structureProcessor.process(graph, { - projectRoot, - projectName, - filePaths - }); - - this.updateProgress('structure', 'Project structure analysis complete', 100); - - // Pass 2: Code Parsing and Definition Extraction (Parallel - CPU intensive) - console.log('๐Ÿ” Pass 2: Parsing code and extracting definitions (parallel)...'); - this.updateProgress('parsing', 'Initializing parallel parsing...', 0); - - await this.parsingProcessor.process(graph, { - filePaths, - fileContents, - options - }); - - this.updateProgress('parsing', 'Parallel parsing complete', 100); - - // Get AST map and function registry from parsing processor - const astMap = this.parsingProcessor.getASTMap(); - const functionTrie = this.parsingProcessor.getFunctionRegistry(); - - this.callProcessor = new CallProcessor(functionTrie); - - // Pass 3: Import Resolution (Sequential - depends on parsing results) - console.log('๐Ÿ”— Pass 3: Resolving imports and building dependency map...'); - this.updateProgress('imports', 'Resolving imports...', 0); - - await this.importProcessor.process(graph, astMap, fileContents); - - this.updateProgress('imports', 'Import resolution complete', 100); - - // Pass 4: Call Resolution (Sequential - depends on import map) - console.log('๐Ÿ“ž Pass 4: Resolving function calls with 3-stage strategy...'); - this.updateProgress('calls', 'Resolving function calls...', 0); - - const importMap = this.importProcessor.getImportMap(); - await this.callProcessor.process(graph, astMap, importMap); - - this.updateProgress('calls', 'Call resolution complete', 100); - - const endTime = performance.now(); - const totalDuration = endTime - startTime; - - console.log(`โœ… Parallel ingestion complete in ${totalDuration.toFixed(2)}ms`); - console.log(`๐Ÿ“Š Graph contains ${graph.nodes.length} nodes and ${graph.relationships.length} relationships`); - - // Log performance statistics - this.logPerformanceStats(graph, totalDuration); - - // Log worker pool statistics if available - const workerStats = this.parsingProcessor.getWorkerPoolStats(); - if (workerStats) { - console.log('๐Ÿ”ง Worker Pool Statistics:', workerStats); - } - - return graph; - - } catch (error) { - console.error('โŒ Error in parallel pipeline:', error); - throw error; - } finally { - // Cleanup worker pools - await this.cleanup(); - } - } - - /** - * Log performance statistics - */ - private logPerformanceStats(graph: KnowledgeGraph, totalDuration: number): void { - // Debug: Show graph structure - const nodesByType = graph.nodes.reduce((acc, node) => { - acc[node.label] = (acc[node.label] || 0) + 1; - return acc; - }, {} as Record); - - const relationshipsByType = graph.relationships.reduce((acc, rel) => { - acc[rel.type] = (acc[rel.type] || 0) + 1; - return acc; - }, {} as Record); - - console.log('๐Ÿ“Š Graph Statistics:'); - console.log('Nodes by type:', nodesByType); - console.log('Relationships by type:', relationshipsByType); - - // Debug: Find isolated nodes (nodes with no relationships) - const connectedNodeIds = new Set(); - graph.relationships.forEach(rel => { - connectedNodeIds.add(rel.source); - connectedNodeIds.add(rel.target); - }); - - const isolatedNodes = graph.nodes.filter(node => !connectedNodeIds.has(node.id)); - if (isolatedNodes.length > 0) { - console.warn(`โš ๏ธ Found ${isolatedNodes.length} isolated nodes:`); - const isolatedByType = isolatedNodes.reduce((acc, node) => { - acc[node.label] = (acc[node.label] || 0) + 1; - return acc; - }, {} as Record); - console.log('Isolated nodes by type:', isolatedByType); - } - - // Performance metrics - const totalNodes = graph.nodes.length; - const totalRelationships = graph.relationships.length; - const processingRate = totalDuration > 0 ? (totalNodes + totalRelationships) / (totalDuration / 1000) : 0; - - console.log('โšก Performance Metrics:'); - console.log(` Total processing time: ${totalDuration.toFixed(2)}ms`); - console.log(` Processing rate: ${processingRate.toFixed(2)} entities/second`); - console.log(` Average time per node: ${totalNodes > 0 ? (totalDuration / totalNodes).toFixed(2) : 0}ms`); - console.log(` Average time per relationship: ${totalRelationships > 0 ? (totalDuration / totalRelationships).toFixed(2) : 0}ms`); - } - - /** - * Cleanup resources - */ - private async cleanup(): Promise { - try { - console.log('๐Ÿงน Cleaning up parallel pipeline resources...'); - - // Shutdown parsing processor (which includes worker pool) - await this.parsingProcessor.shutdown(); - - console.log('โœ… Parallel pipeline cleanup complete'); - } catch (error) { - console.warn('โš ๏ธ Error during cleanup:', error); - } - } - - /** - * Get worker pool statistics - */ - public getWorkerPoolStats() { - return this.parsingProcessor.getWorkerPoolStats(); - } - - /** - * Check if parallel processing is supported - */ - public static isParallelProcessingSupported(): boolean { - return WebWorkerPoolUtils.isSupported(); - } - - /** - * Get optimal worker count for current system - */ - public static getOptimalWorkerCount(): number { - return WebWorkerPoolUtils.getOptimalWorkerCount('cpu'); - } - - /** - * Get hardware concurrency - */ - public static getHardwareConcurrency(): number { - return WebWorkerPoolUtils.getHardwareConcurrency(); - } -} - -================ -File: src/core/ingestion/parsing-processor.ts -================ -import { GraphNode, GraphRelationship, NodeLabel, RelationshipType } from '../graph/types.js'; -import { MemoryManager } from '../../services/memory-manager.js'; -import { KnowledgeGraph, GraphProcessor } from '../graph/graph.js'; -import { - pathUtils, - OptimizedSet, - DuplicateDetector, - BatchProcessor -} from '../../lib/shared-utils.js'; -import { IGNORE_PATTERNS } from '../../config/language-config.js'; -import Parser from 'web-tree-sitter'; -import { TYPESCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries.ts'; -import { initTreeSitter, loadTypeScriptParser, loadPythonParser, loadJavaScriptParser } from '../tree-sitter/parser-loader.js'; -import { FunctionRegistryTrie, FunctionDefinition } from '../graph/trie.js'; - -export interface ParsingInput { - filePaths: string[]; - fileContents: Map; - options?: { directoryFilter?: string; fileExtensions?: string }; -} - -export interface ParsedDefinition { - name: string; - type: 'function' | 'class' | 'method' | 'variable' | 'import' | 'interface' | 'type' | 'decorator'; - startLine: number; - endLine?: number; - parameters?: string[] | undefined; - returnType?: string | undefined; - accessibility?: 'public' | 'private' | 'protected'; - - isStatic?: boolean | undefined; - isAsync?: boolean | undefined; - parentClass?: string | undefined; - decorators?: string[] | undefined; - extends?: string[] | undefined; - implements?: string[] | undefined; - importPath?: string | undefined; - exportType?: 'named' | 'default' | 'namespace'; - docstring?: string | undefined; -} - -export interface ParsedAST { - tree: Parser.Tree; -} - -function generateId(prefix: string, identifier: string): string { - let hash = 0; - for (let i = 0; i < identifier.length; i++) { - const char = identifier.charCodeAt(i); - hash = ((hash << 5) - hash) + char; - hash = hash & hash; - } - return `${prefix}-${Math.abs(hash)}-${identifier.replace(/[^a-zA-Z0-9]/g, '_')}`; -} - -export class ParsingProcessor implements GraphProcessor { - private memoryManager: MemoryManager; - private duplicateDetector = new DuplicateDetector((item: string) => item); - private processedFiles = new OptimizedSet(); - private parser: Parser | null = null; - private languageParsers: Map = new Map(); - private astMap: Map = new Map(); - private functionTrie: FunctionRegistryTrie = new FunctionRegistryTrie(); - - constructor() { - this.memoryManager = MemoryManager.getInstance(); - } - - public getASTMap(): Map { - return this.astMap; - } - - public getFunctionRegistry(): FunctionRegistryTrie { - return this.functionTrie; - } - - public async process(graph: KnowledgeGraph, input: ParsingInput): Promise { - const { filePaths, fileContents, options } = input; - - console.log(`ParsingProcessor: Processing ${filePaths.length} total paths`); - - const memoryStats = this.memoryManager.getStats(); - console.log(`Memory status: ${memoryStats.usedMemoryMB}MB used, ${memoryStats.fileCount} files cached`); - - const filteredFiles = this.applyFiltering(filePaths, fileContents, options); - - console.log(`ParsingProcessor: After filtering: ${filteredFiles.length} files to parse`); - - const BATCH_SIZE = 10; - const sourceFiles = filteredFiles.filter((path: string) => this.isSourceFile(path)); - const configFiles = filteredFiles.filter((path: string) => this.isConfigFile(path)); - const allProcessableFiles = [...sourceFiles, ...configFiles]; - - console.log(`ParsingProcessor: Found ${sourceFiles.length} source files and ${configFiles.length} config files, processing in batches of ${BATCH_SIZE}`); - - try { - await this.initializeParser(); - - const batchProcessor = new BatchProcessor(BATCH_SIZE, async (filePaths: string[]) => { - for (const filePath of filePaths) { - if (this.processedFiles.has(filePath)) { - continue; - } - - const content = fileContents.get(filePath); - if (!content) { - console.warn(`No content found for file: ${filePath}`); - continue; - } - - try { - await this.parseFile(graph, filePath, content); - this.processedFiles.add(filePath); - } catch (error) { - console.error(`Error parsing file ${filePath}:`, error); - } - } - return []; - }); - - - await batchProcessor.processAll(allProcessableFiles); - - console.log(`ParsingProcessor: Successfully processed ${this.processedFiles.size} files`); - } catch (error) { - console.error('Error initializing parser:', error); - } - } - - private applyFiltering( - filePaths: string[], - fileContents: Map, - options?: { directoryFilter?: string; fileExtensions?: string }): string[] { - - let filtered = filePaths; - - if (options?.directoryFilter) { - filtered = filtered.filter(path => path.includes(options.directoryFilter ?? '')); - } - - if (options?.fileExtensions) { - const extensions = options.fileExtensions.split(',').map(ext => ext.trim()).filter(ext => ext.length); - filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext))); - } - - filtered = filtered.filter(path => - ![...IGNORE_PATTERNS].some(pattern => { - if (typeof pattern === 'string') { - return path.includes(pattern); - } - return false; - }) - ); - - filtered = filtered.filter(path => { - const content = fileContents.get(path); - return content && content.trim().length > 0; - }); - - return filtered; - } - - private isSourceFile(filePath: string): boolean { - const sourceExtensions = ['.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', '.cs', '.php', '.rb', '.go', '.rs']; - return sourceExtensions.some(ext => filePath.toLowerCase().endsWith(ext)); - } - - private isConfigFile(filePath: string): boolean { - const configFiles = ['package.json', 'tsconfig.json', 'webpack.config.js', 'vite.config.ts', '.eslintrc.js', '.prettierrc']; - const configExtensions = ['.json', '.yaml', '.yml', '.toml']; - - return configFiles.some(name => filePath.endsWith(name)) || - configExtensions.some(ext => filePath.toLowerCase().endsWith(ext)); - } - - private async initializeParser(): Promise { - if (this.parser) return; - - this.parser = await initTreeSitter(); - - const languageLoaders = { - typescript: loadTypeScriptParser, - javascript: loadJavaScriptParser, - python: loadPythonParser, - }; - - for (const [lang, loader] of Object.entries(languageLoaders)) { - try { - const languageParser = await loader(); - this.languageParsers.set(lang, languageParser); - console.log(`${lang} parser loaded successfully.`); - } catch (error) { - console.error(`Failed to load ${lang} parser:`, error); - } - } - } - - private async parseFile(graph: KnowledgeGraph, filePath: string, content: string): Promise { - const language = this.detectLanguage(filePath); - const langParser = this.languageParsers.get(language); - - if (!langParser || !this.parser) { - console.warn(`No parser available for language: ${language}. Skipping file: ${filePath}`); - await this.parseGenericFile(graph, filePath, content); - return; - } - - this.parser.setLanguage(langParser); - const tree = this.parser.parse(content); - this.astMap.set(filePath, { tree }); - const definitions: ParsedDefinition[] = []; - - const queries = this.getQueriesForLanguage(language); - if (!queries) { - console.warn(`No queries available for language: ${language}.`); - return; - } - - for (const [queryName, queryString] of Object.entries(queries)) { - const query = langParser.query(queryString as string); - const matches = query.matches(tree.rootNode); - - for (const match of matches) { - for (const capture of match.captures) { - const node = capture.node; - const definition = this.extractDefinition(node, queryName, filePath); - if (definition) { - definitions.push(definition); - } - } - } - } - - await this.addDefinitionsToGraph(graph, filePath, definitions); - } - - private extractDefinition(node: Parser.SyntaxNode, queryName: string, filePath: string): ParsedDefinition | null { - const nameNode = node.childForFieldName('name'); - const name = nameNode ? nameNode.text : 'anonymous'; - - return { - name, - type: this.getDefinitionType(queryName), - startLine: node.startPosition.row + 1, - endLine: node.endPosition.row + 1, - filePath, - } as ParsedDefinition; - } - - private getDefinitionType(queryName: string): ParsedDefinition['type'] { - switch (queryName) { - case 'classes': return 'class'; - case 'methods': return 'method'; - case 'functions': - case 'arrowFunctions': return 'function'; - case 'imports': - case 'from_imports': return 'import'; - case 'interfaces': return 'interface'; - case 'types': return 'type'; - case 'decorators': return 'decorator'; - default: return 'variable'; - } - } - - private detectLanguage(filePath: string): string { - const extension = pathUtils.extname(filePath).toLowerCase(); - - switch (extension) { - case '.ts': - case '.tsx': return 'typescript'; - case '.js': - case '.jsx': return 'javascript'; - case '.py': return 'python'; - case '.java': return 'java'; - default: return 'generic'; - } - } - - private getQueriesForLanguage(language: string): Record | null { - switch (language) { - case 'typescript': - case 'javascript': - return TYPESCRIPT_QUERIES; - case 'python': - return PYTHON_QUERIES; - case 'java': - return JAVA_QUERIES; - default: - return null; - } - } - - private async parseGenericFile(graph: KnowledgeGraph, filePath: string, _content: string): Promise { - const fileNode: GraphNode = { - id: generateId('file', filePath), - label: 'File' as NodeLabel, - properties: { - name: pathUtils.getFileName(filePath), - path: filePath, - size: _content.length, - language: this.detectLanguage(filePath) - } - }; - - graph.addNode(fileNode); - } - - private async addDefinitionsToGraph( - graph: KnowledgeGraph, - filePath: string, - definitions: ParsedDefinition[] - ): Promise { - const fileNode: GraphNode = { - id: generateId('file', filePath), - label: 'File' as NodeLabel, - properties: { - name: pathUtils.getFileName(filePath), - path: filePath, - language: this.detectLanguage(filePath) - } - }; - - graph.addNode(fileNode); - - for (const def of definitions) { - const nodeId = generateId(def.type, `${filePath}:${def.name}`); - - if (this.duplicateDetector.checkAndMark(nodeId)) continue; - - const node: GraphNode = { - id: nodeId, - label: this.getNodeLabelForType(def.type), - properties: { - name: def.name, - type: def.type, - startLine: def.startLine, - endLine: def.endLine, - parameters: def.parameters, - returnType: def.returnType, - accessibility: def.accessibility, - isStatic: def.isStatic, - isAsync: def.isAsync, - parentClass: def.parentClass, - decorators: def.decorators, - extends: def.extends, - implements: def.implements, - importPath: def.importPath, - exportType: def.exportType, - docstring: def.docstring, - filePath: filePath - } - }; - - graph.addNode(node); - - if (def.type === 'function' || def.type === 'method' || def.type === 'class' || def.type === 'interface') { - const functionDef: FunctionDefinition = { - nodeId: nodeId, - qualifiedName: `${filePath}:${def.name}`, - filePath: filePath, - functionName: def.name, - type: def.type, - startLine: def.startLine, - endLine: def.endLine, - }; - this.functionTrie.addDefinition(functionDef); - } - - const definesRelationship: GraphRelationship = { - id: generateId('defines', `${fileNode.id}:${node.id}`), - type: 'DEFINES' as RelationshipType, - source: fileNode.id, - target: node.id, - properties: { - filePath: filePath, - line_number: def.startLine - } - }; - - graph.addRelationship(definesRelationship); - - if (def.extends && def.extends.length > 0) { - for (const extend of def.extends) { - const extendsRelationship: GraphRelationship = { - id: generateId('extends', `${node.id}:${extend}`), - type: 'EXTENDS' as RelationshipType, - source: node.id, - target: generateId('class', extend), - properties: {} - }; - - graph.addRelationship(extendsRelationship); - } - } - - if (def.implements && def.implements.length > 0) { - for (const interfaceName of def.implements) { - const implementsRelationship: GraphRelationship = { - id: generateId('implements', `${node.id}:${interfaceName}`), - type: 'IMPLEMENTS' as RelationshipType, - source: node.id, - target: generateId('interface', interfaceName), - properties: {} - }; - - graph.addRelationship(implementsRelationship); - } - } - - if (def.importPath) { - const importRelationship: GraphRelationship = { - id: generateId('imports', `${node.id}:${def.importPath}`), - type: 'IMPORTS' as RelationshipType, - source: node.id, - target: generateId('file', def.importPath), - properties: { - importPath: def.importPath - } - }; - graph.addRelationship(importRelationship); - } - - if (def.parentClass) { - const parentRelationship: GraphRelationship = { - id: generateId('belongs_to', `${node.id}:${def.parentClass}`), - type: 'BELONGS_TO' as RelationshipType, - source: node.id, - target: generateId('class', `${filePath}:${def.parentClass}`), - properties: {} - }; - graph.addRelationship(parentRelationship); - } - } - } - - private getNodeLabelForType(type: string): NodeLabel { - switch (type) { - case 'class': return 'Class' as NodeLabel; - case 'function': return 'Function' as NodeLabel; - case 'method': return 'Method' as NodeLabel; - case 'variable': return 'Variable' as NodeLabel; - case 'import': return 'Import' as NodeLabel; - case 'interface': return 'Interface' as NodeLabel; - case 'type': return 'Type' as NodeLabel; - case 'decorator': return 'Decorator' as NodeLabel; - default: return 'CodeElement' as NodeLabel; - } - } - - public reset(): void { - this.processedFiles.clear(); - this.duplicateDetector.clear(); - this.memoryManager.clearCache(); - } -} - -================ -File: src/core/ingestion/pipeline.ts -================ -import { SimpleKnowledgeGraph } from '../graph/graph.js'; -import type { KnowledgeGraph } from '../graph/types.ts'; -import { StructureProcessor } from './structure-processor.ts'; -import { ParsingProcessor } from './parsing-processor.ts'; -import { ImportProcessor } from './import-processor.ts'; -import { CallProcessor } from './call-processor.ts'; - -export interface PipelineInput { - projectRoot: string; - projectName: string; - filePaths: string[]; - fileContents: Map; - options?: { - directoryFilter?: string; - fileExtensions?: string; - }; -} - -export class GraphPipeline { - private structureProcessor: StructureProcessor; - private parsingProcessor: ParsingProcessor; - private importProcessor: ImportProcessor; - private callProcessor!: CallProcessor; - - constructor() { - this.structureProcessor = new StructureProcessor(); - this.parsingProcessor = new ParsingProcessor(); - this.importProcessor = new ImportProcessor(); - - } - - public async run(input: PipelineInput): Promise { - const { projectRoot, projectName, filePaths, fileContents, options } = input; - - const graph = new SimpleKnowledgeGraph(); - - console.log(`๐Ÿš€ Starting 4-pass ingestion for project: ${projectName}`); - - // Pass 1: Structure Analysis - console.log('๐Ÿ“ Pass 1: Analyzing project structure...'); - await this.structureProcessor.process(graph, { - projectRoot, - projectName, - filePaths - }); - - // Pass 2: Code Parsing and Definition Extraction (populates FunctionRegistryTrie) - console.log('๐Ÿ” Pass 2: Parsing code and extracting definitions...'); - await this.parsingProcessor.process(graph, { - filePaths, - fileContents, - options // Pass filtering options to ParsingProcessor - }); - - // Get AST map and function registry from parsing processor - const astMap = this.parsingProcessor.getASTMap(); - const functionTrie = this.parsingProcessor.getFunctionRegistry(); - - this.callProcessor = new CallProcessor(functionTrie); - - // Pass 3: Import Resolution (builds complete import map) - console.log('๐Ÿ”— Pass 3: Resolving imports and building dependency map...'); - await this.importProcessor.process(graph, astMap, fileContents); - - // Pass 4: Call Resolution (uses import map and function trie) - console.log('๐Ÿ“ž Pass 4: Resolving function calls with 3-stage strategy...'); - const importMap = this.importProcessor.getImportMap(); - await this.callProcessor.process(graph, astMap, importMap); - - console.log(`Ingestion complete. Graph contains ${graph.nodes.length} nodes and ${graph.relationships.length} relationships.`); - - // Debug: Show graph structure - const nodesByType = graph.nodes.reduce((acc, node) => { - acc[node.label] = (acc[node.label] || 0) + 1; - return acc; - }, {} as Record); - - const relationshipsByType = graph.relationships.reduce((acc, rel) => { - acc[rel.type] = (acc[rel.type] || 0) + 1; - return acc; - }, {} as Record); - - console.log('๐Ÿ“Š Graph Statistics:'); - console.log('Nodes by type:', nodesByType); - console.log('Relationships by type:', relationshipsByType); - - // Debug: Find isolated nodes (nodes with no relationships) - const connectedNodeIds = new Set(); - graph.relationships.forEach(rel => { - connectedNodeIds.add(rel.source); - connectedNodeIds.add(rel.target); - }); - - const isolatedNodes = graph.nodes.filter(node => !connectedNodeIds.has(node.id)); - if (isolatedNodes.length > 0) { - console.warn(`โš ๏ธ Found ${isolatedNodes.length} isolated nodes:`); - const isolatedByType = isolatedNodes.reduce((acc, node) => { - acc[node.label] = (acc[node.label] || 0) + 1; - return acc; - }, {} as Record); - console.warn('Isolated nodes by type:', isolatedByType); - - // Show some examples - console.warn('Sample isolated nodes:', isolatedNodes.slice(0, 5).map(n => ({ - type: n.label, - name: n.properties.name || n.properties.filePath || n.id, - properties: Object.keys(n.properties) - }))); - } - - // Debug: Check for files without content - const fileNodes = graph.nodes.filter(n => n.label === 'File'); - const filesWithoutDefinitions = fileNodes.filter(fileNode => { - const hasDefinitions = graph.relationships.some(rel => - rel.source === fileNode.id && - rel.type === 'CONTAINS' && - graph.nodes.some(targetNode => - targetNode.id === rel.target && - ['Function', 'Class', 'Method'].includes(targetNode.label) - ) - ); - return !hasDefinitions; - }); - - if (filesWithoutDefinitions.length > 0) { - console.warn(`โš ๏ธ Found ${filesWithoutDefinitions.length} files without definitions:`); - console.warn('Files without content:', filesWithoutDefinitions.slice(0, 5).map(n => - n.properties.filePath || n.properties.name - )); - } - - // Validate graph integrity - this.validateGraphIntegrity(graph); - - return graph; - } - - /** - * Validate graph integrity and identify potential issues - */ - private validateGraphIntegrity(graph: KnowledgeGraph): void { - console.log('๐Ÿ” Validating graph integrity...'); - - const issues: string[] = []; - - // Check 1: Orphaned relationships (references to non-existent nodes) - const nodeIds = new Set(graph.nodes.map(n => n.id)); - const orphanedRels = graph.relationships.filter(rel => - !nodeIds.has(rel.source) || !nodeIds.has(rel.target) - ); - - if (orphanedRels.length > 0) { - issues.push(`${orphanedRels.length} relationships reference non-existent nodes`); - } - - // Check 2: Files without proper structure connections - const projectNodes = graph.nodes.filter(n => n.label === 'Project'); - const folderNodes = graph.nodes.filter(n => n.label === 'Folder'); - const fileNodes = graph.nodes.filter(n => n.label === 'File'); - - const filesNotConnectedToStructure = fileNodes.filter(fileNode => { - const hasStructuralParent = graph.relationships.some(rel => - rel.target === fileNode.id && - rel.type === 'CONTAINS' && - (projectNodes.some(p => p.id === rel.source) || folderNodes.some(f => f.id === rel.source)) - ); - return !hasStructuralParent; - }); - - if (filesNotConnectedToStructure.length > 0) { - issues.push(`${filesNotConnectedToStructure.length} files not connected to project structure`); - } - - // Check 3: Source files without any definitions - const sourceFileExtensions = ['.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.cs']; - const sourceFiles = fileNodes.filter(fileNode => { - const filePath = fileNode.properties.filePath as string || ''; - return sourceFileExtensions.some(ext => filePath.endsWith(ext)); - }); - - const sourceFilesWithoutDefinitions = sourceFiles.filter(fileNode => { - const hasDefinitions = graph.relationships.some(rel => - rel.source === fileNode.id && - rel.type === 'CONTAINS' && - graph.nodes.some(n => - n.id === rel.target && - ['Function', 'Class', 'Method', 'Variable'].includes(n.label) - ) - ); - return !hasDefinitions; - }); - - if (sourceFilesWithoutDefinitions.length > 0) { - issues.push(`${sourceFilesWithoutDefinitions.length} source files contain no parsed definitions`); - console.warn('Source files without definitions:', - sourceFilesWithoutDefinitions.slice(0, 3).map(n => n.properties.filePath) - ); - } - - // Check 4: Functions/Classes without file parents - const definitionNodes = graph.nodes.filter(n => - ['Function', 'Class', 'Method', 'Variable'].includes(n.label) - ); - - const definitionsWithoutFiles = definitionNodes.filter(defNode => { - const hasFileParent = graph.relationships.some(rel => - rel.target === defNode.id && - rel.type === 'CONTAINS' && - graph.nodes.some(n => n.id === rel.source && n.label === 'File') - ); - return !hasFileParent; - }); - - if (definitionsWithoutFiles.length > 0) { - issues.push(`${definitionsWithoutFiles.length} definitions not connected to files`); - } - - // Check 5: Import/Call relationship issues - const importRels = graph.relationships.filter(r => r.type === 'IMPORTS'); - const callRels = graph.relationships.filter(r => r.type === 'CALLS'); - - if (sourceFiles.length > 1 && importRels.length === 0) { - issues.push('No import relationships found between files'); - } - - if (definitionNodes.length > 1 && callRels.length === 0) { - issues.push('No function call relationships found'); - } - - // Report results - if (issues.length === 0) { - console.log('โœ… Graph integrity validation passed'); - } else { - console.warn('โš ๏ธ Graph integrity issues found:'); - issues.forEach((issue, i) => console.warn(` ${i + 1}. ${issue}`)); - } - } - - public getStats(graph: KnowledgeGraph): { nodeStats: Record; relationshipStats: Record } { - const nodeStats: Record = {}; - const relationshipStats: Record = {}; - - for (const node of graph.nodes) { - nodeStats[node.label] = (nodeStats[node.label] || 0) + 1; - } - - for (const relationship of graph.relationships) { - relationshipStats[relationship.type] = (relationshipStats[relationship.type] || 0) + 1; - } - - return { nodeStats, relationshipStats }; - } - - public getCallStats() { - return this.callProcessor.getStats(); - } -} - -================ -File: src/core/ingestion/structure-processor.ts -================ -import type { KnowledgeGraph, GraphNode, GraphRelationship } from '../graph/types.ts'; -import { generateId } from '../../lib/utils.ts'; - -export interface StructureInput { - projectRoot: string; - projectName: string; - filePaths: string[]; // Now includes ALL paths: files AND directories -} - -export class StructureProcessor { - private nodeIdMap: Map = new Map(); - - // Import ignore patterns from ParsingProcessor - private static readonly IGNORE_PATTERNS = new Set([ - // Version Control - '.git', '.svn', '.hg', - - // Package Managers & Dependencies - 'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps', - - // Python Virtual Environments & Cache - 'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__', - '.pytest_cache', '.mypy_cache', '.tox', - - // Build & Distribution Directories - 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', - - // IDE & Editor Directories - '.vs', '.vscode', '.idea', '.eclipse', '.settings', - - // Temporary & Log Directories - 'tmp', '.tmp', 'temp', 'logs', 'log', - - // Coverage & Testing - 'coverage', '.coverage', 'htmlcov', '.nyc_output', - - // OS & System - '.DS_Store', 'Thumbs.db', - - // Documentation Build Output - '_site', '.docusaurus', - - // Cache Directories - '.cache', '.parcel-cache', '.next', '.nuxt' - ]); - - /** - * Process complete repository structure directly from discovered paths - * This is the new robust approach that doesn't infer structure - */ - public async process(graph: KnowledgeGraph, input: StructureInput): Promise { - const { projectRoot, projectName, filePaths } = input; - - console.log(`StructureProcessor: Processing ${filePaths.length} complete paths`); - - // Create project root node - const projectNode = this.createProjectNode(projectName, projectRoot); - graph.addNode(projectNode); - - // Separate files and directories from the complete path list - const { directories, files } = this.categorizePaths(filePaths); - - console.log(`StructureProcessor: Found ${directories.length} directories and ${files.length} files`); - - // Filter out ignored directories from KG display (but keep for internal structure) - const visibleDirectories = directories.filter(dir => !this.shouldHideDirectory(dir)); - const hiddenDirectoriesCount = directories.length - visibleDirectories.length; - - if (hiddenDirectoriesCount > 0) { - console.log(`StructureProcessor: Hiding ${hiddenDirectoriesCount} ignored directories from KG`); - } - - // Create directory nodes only for visible directories - const directoryNodes = this.createDirectoryNodes(visibleDirectories); - directoryNodes.forEach(node => graph.addNode(node)); - - // Filter out files that are inside ignored directories - const visibleFiles = files.filter(file => !this.shouldHideFile(file)); - const hiddenFilesCount = files.length - visibleFiles.length; - - if (hiddenFilesCount > 0) { - console.log(`StructureProcessor: Hiding ${hiddenFilesCount} files in ignored directories from KG`); - } - - // Create file nodes only for visible files - const fileNodes = this.createFileNodes(visibleFiles); - fileNodes.forEach(node => graph.addNode(node)); - - // Establish CONTAINS relationships for visible structure only - this.createContainsRelationships(graph, projectNode.id, visibleDirectories, visibleFiles); - - const totalHidden = hiddenDirectoriesCount + hiddenFilesCount; - console.log(`StructureProcessor: Created ${graph.nodes.length} nodes total (${totalHidden} items hidden)`); - } - - /** - * Categorize paths into files and directories - * Since we now receive the complete structure, we need to distinguish between them - */ - private categorizePaths(allPaths: string[]): { directories: string[], files: string[] } { - const directories: string[] = []; - const files: string[] = []; - const pathSet = new Set(allPaths); - - for (const path of allPaths) { - // A path is a directory if: - // 1. Other paths exist that start with this path + "/" - // 2. OR it doesn't have a file extension and other paths are nested under it - const isDirectory = allPaths.some(otherPath => - otherPath !== path && otherPath.startsWith(path + '/') - ); - - if (isDirectory) { - directories.push(path); - } else { - // It's a file if it's not identified as a directory - files.push(path); - } - } - - // Also add intermediate directories that might not be explicitly listed - const allIntermediateDirs = new Set(); - for (const path of allPaths) { - const parts = path.split('/'); - for (let i = 1; i < parts.length; i++) { - const intermediatePath = parts.slice(0, i).join('/'); - if (intermediatePath && !pathSet.has(intermediatePath)) { - allIntermediateDirs.add(intermediatePath); - } - } - } - - // Add intermediate directories that weren't explicitly listed - directories.push(...Array.from(allIntermediateDirs)); - - return { - directories: [...new Set(directories)].sort(), // Remove duplicates and sort - files: files.sort() - }; - } - - private createProjectNode(projectName: string, projectRoot: string): GraphNode { - const id = generateId('project', projectName); - this.nodeIdMap.set('', id); // Empty path represents project root - - return { - id, - label: 'Project', - properties: { - name: projectName, - path: projectRoot, - createdAt: new Date().toISOString() - } - }; - } - - /** - * Create nodes for directories directly from discovered directory paths - */ - private createDirectoryNodes(directoryPaths: string[]): GraphNode[] { - const nodes: GraphNode[] = []; - - for (const dirPath of directoryPaths) { - if (!dirPath) continue; - - const id = generateId('folder', dirPath); - this.nodeIdMap.set(dirPath, id); - - const pathParts = dirPath.split('/'); - const dirName = pathParts[pathParts.length - 1]; - - const node: GraphNode = { - id, - label: 'Folder', - properties: { - name: dirName, - path: dirPath, - fullPath: dirPath, - depth: pathParts.length - } - }; - - nodes.push(node); - } - - return nodes; - } - - /** - * Create nodes for files directly from discovered file paths - */ - private createFileNodes(filePaths: string[]): GraphNode[] { - const nodes: GraphNode[] = []; - - for (const filePath of filePaths) { - if (!filePath) continue; - - const id = generateId('file', filePath); - this.nodeIdMap.set(filePath, id); - - const fileName = filePath.split('/').pop() || filePath; - const extension = this.getFileExtension(fileName); - - const node: GraphNode = { - id, - label: 'File', - properties: { - name: fileName, - path: filePath, - filePath: filePath, // For compatibility with existing code - extension, - // Note: definitionCount will be set later by ParsingProcessor - // language will be determined later by ParsingProcessor - } - }; - - nodes.push(node); - } - - return nodes; - } - - /** - * Create CONTAINS relationships for the complete discovered structure - */ - private createContainsRelationships( - graph: KnowledgeGraph, - projectId: string, - directories: string[], - files: string[] - ): void { - // Create relationships: directories contain subdirectories and files - const allPaths = [...directories, ...files]; - - for (const path of allPaths) { - const parentPath = this.getParentPath(path); - const parentId = parentPath === '' ? projectId : this.nodeIdMap.get(parentPath); - const childId = this.nodeIdMap.get(path); - - // Only create relationships if both parent and child nodes exist in the graph - if (parentId && childId && parentId !== childId) { - const relationship: GraphRelationship = { - id: generateId('contains', `${parentId}-${childId}`), - type: 'CONTAINS', - source: parentId, - target: childId, - properties: {} - }; - - graph.addRelationship(relationship); - } else if (!parentId && parentPath !== '') { - // If parent directory was hidden, connect directly to project or nearest visible parent - const visibleParentId = this.findVisibleParent(parentPath, projectId); - if (visibleParentId && childId && visibleParentId !== childId) { - const relationship: GraphRelationship = { - id: generateId('contains', `${visibleParentId}-${childId}`), - type: 'CONTAINS', - source: visibleParentId, - target: childId, - properties: {} - }; - - graph.addRelationship(relationship); - } - } - } - - console.log(`StructureProcessor: Created ${graph.relationships.length} CONTAINS relationships`); - } - - /** - * Find the nearest visible parent directory or project root - */ - private findVisibleParent(path: string, projectId: string): string { - if (path === '') return projectId; - - const parentPath = this.getParentPath(path); - const parentId = this.nodeIdMap.get(parentPath); - - if (parentId) { - return parentId; // Found visible parent - } - - // Recursively look for visible parent - return this.findVisibleParent(parentPath, projectId); - } - - private getParentPath(path: string): string { - if (!path || !path.includes('/')) { - return ''; // Root level - } - - const lastSlashIndex = path.lastIndexOf('/'); - return path.substring(0, lastSlashIndex); - } - - private getFileExtension(fileName: string): string { - const lastDotIndex = fileName.lastIndexOf('.'); - if (lastDotIndex === -1 || lastDotIndex === 0) { - return ''; - } - return fileName.substring(lastDotIndex); - } - - public getNodeId(path: string): string | undefined { - return this.nodeIdMap.get(path); - } - - public clear(): void { - this.nodeIdMap.clear(); - } - - /** - * Check if a directory should be hidden from the KG visualization - * This matches the ignore patterns used in ParsingProcessor - */ - private shouldHideDirectory(dirPath: string): boolean { - const pathSegments = dirPath.split('/'); - - // Check if any segment of the path matches an ignore pattern - const hasIgnoredSegment = pathSegments.some(segment => - StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - if (hasIgnoredSegment) { - return true; - } - - // Additional pattern matching - const lowerPath = dirPath.toLowerCase(); - - // Hide Python egg-info directories - if (lowerPath.includes('.egg-info')) { - return true; - } - - // Hide site-packages directories - if (lowerPath.includes('site-packages')) { - return true; - } - - // Hide most hidden directories (except important ones like .github) - for (const segment of pathSegments) { - if (segment.startsWith('.') && segment !== '.github') { - return true; - } - } - - return false; - } - - /** - * Check if a file should be hidden from the KG visualization - * This matches the ignore patterns used in ParsingProcessor - */ - private shouldHideFile(filePath: string): boolean { - const pathSegments = filePath.split('/'); - - // Check if any segment of the path matches an ignore pattern - const hasIgnoredSegment = pathSegments.some(segment => - StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - if (hasIgnoredSegment) { - return true; - } - - // Additional pattern matching - const lowerPath = filePath.toLowerCase(); - - // Hide Python egg-info directories - if (lowerPath.includes('.egg-info')) { - return true; - } - - // Hide site-packages directories - if (lowerPath.includes('site-packages')) { - return true; - } - - // Hide most hidden directories (except important ones like .github) - for (const segment of pathSegments) { - if (segment.startsWith('.') && segment !== '.github') { - return true; - } - } - - return false; - } -} - -================ -File: src/core/ingestion/tree-sitter-queries.ts -================ -export const TYPESCRIPT_QUERIES = { - imports: ` - (import_statement) @import - `, - classes: ` - (class_declaration) @class - `, - methods: ` - (method_definition) @method - `, - functions: ` - (function_declaration) @function - `, - arrowFunctions: ` - (lexical_declaration - (variable_declarator - name: (identifier) @name - value: (arrow_function))) @arrow_function - `, - interfaces: ` - (interface_declaration) @interface - `, - types: ` - (type_alias_declaration) @type - `, -}; - -export const PYTHON_QUERIES = { - imports: ` - (import_statement) @import - `, - from_imports: ` - (import_from_statement) @from_import - `, - classes: ` - (class_definition) @class - `, - functions: ` - (function_definition) @function - `, - decorators: ` - (decorator) @decorator - `, -}; - -export const JAVA_QUERIES = { - classes: ` - (class_declaration) @class - `, - methods: ` - (method_declaration) @method - `, - interfaces: ` - (interface_declaration) @interface - `, -}; - -================ -File: src/core/kuzu/kuzu-loader.ts -================ -/** - * KuzuDB WASM Loader - * Handles loading and initialization of KuzuDB WebAssembly module using the official npm package - */ - -import kuzu from 'kuzu-wasm'; - -export interface KuzuDBInstance { - // Basic database operations - createDatabase(path: string): Promise; - openDatabase(path: string): Promise; - closeDatabase(): Promise; - - // Query execution - executeQuery(query: string): Promise; - - // Schema operations - createNodeTable(tableName: string, properties: Record): Promise; - createRelTable(tableName: string, properties: Record): Promise; - - // Data operations - insertNode(tableName: string, properties: Record): Promise; - insertRel(tableName: string, sourceId: string, targetId: string, properties: Record): Promise; - - // Utility - getDatabaseInfo(): Promise; - clearDatabase(): Promise; -} - -let kuzuInstance: KuzuDBInstance | null = null; -let kuzuModule: any = null; -let database: any = null; -let connection: any = null; - -export async function initKuzuDB(): Promise { - if (kuzuInstance) { - return kuzuInstance; - } - - try { - console.log('Loading KuzuDB WASM...'); - - // Set worker path for browser environment - if (typeof window !== 'undefined') { - kuzu.setWorkerPath('/node_modules/kuzu-wasm/dist/worker.js'); - } - - // Initialize KuzuDB module - kuzuModule = await kuzu(); - - console.log('KuzuDB WASM loaded successfully'); - - // Create KuzuDB instance wrapper - kuzuInstance = createKuzuDBInstance(kuzuModule); - - return kuzuInstance; - - } catch (error) { - console.error('Failed to initialize KuzuDB:', error); - throw new Error(`KuzuDB initialization failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } -} - -function createKuzuDBInstance(kuzuModule: any): KuzuDBInstance { - return { - async createDatabase(path: string): Promise { - console.log(`Creating database at: ${path}`); - - // Set up IDBFS for persistent storage - if (typeof window !== 'undefined') { - await kuzuModule.FS.mkdir('/database'); - await kuzuModule.FS.mountIdbfs('/database'); - } - - // Create database instance - database = new kuzuModule.Database(path); - connection = new kuzuModule.Connection(database); - }, - - async openDatabase(path: string): Promise { - console.log(`Opening database at: ${path}`); - - if (!database) { - // Set up IDBFS for persistent storage - if (typeof window !== 'undefined') { - await kuzuModule.FS.mkdir('/database'); - await kuzuModule.FS.mountIdbfs('/database'); - await kuzuModule.FS.syncfs(true); - } - - // Open database instance - database = new kuzuModule.Database(path); - connection = new kuzuModule.Connection(database); - } - }, - - async closeDatabase(): Promise { - console.log('Closing database'); - - if (connection) { - await connection.close(); - connection = null; - } - - if (database) { - await database.close(); - database = null; - } - - // Sync filesystem for persistence - if (typeof window !== 'undefined' && kuzuModule.FS) { - await kuzuModule.FS.syncfs(false); - await kuzuModule.FS.unmount('/database'); - } - }, - - async executeQuery(query: string): Promise { - console.log(`Executing query: ${query}`); - - if (!connection) { - throw new Error('Database not initialized. Call createDatabase() or openDatabase() first.'); - } - - const queryResult = await connection.query(query); - const results = await queryResult.getAllObjects(); - const count = results.length; - - await queryResult.close(); - - return { results, count }; - }, - - async createNodeTable(tableName: string, properties: Record): Promise { - console.log(`Creating node table: ${tableName}`, properties); - - const propertyDefs = Object.entries(properties) - .map(([name, type]) => `${name} ${type}`) - .join(', '); - - const query = `CREATE NODE TABLE ${tableName}(${propertyDefs}, PRIMARY KEY (id))`; - await this.executeQuery(query); - }, - - async createRelTable(tableName: string, properties: Record): Promise { - console.log(`Creating relationship table: ${tableName}`, properties); - - const propertyDefs = Object.entries(properties) - .map(([name, type]) => `${name} ${type}`) - .join(', '); - - const query = `CREATE REL TABLE ${tableName}(FROM Node TO Node, ${propertyDefs})`; - await this.executeQuery(query); - }, - - async insertNode(tableName: string, properties: Record): Promise { - console.log(`Inserting node into ${tableName}:`, properties); - - const columns = Object.keys(properties).join(', '); - const values = Object.values(properties) - .map(value => typeof value === 'string' ? `'${value}'` : value) - .join(', '); - - const query = `INSERT INTO ${tableName}(${columns}) VALUES (${values})`; - await this.executeQuery(query); - }, - - async insertRel(tableName: string, sourceId: string, targetId: string, properties: Record): Promise { - console.log(`Inserting relationship into ${tableName}: ${sourceId} -> ${targetId}`, properties); - - const columns = ['FROM', 'TO', ...Object.keys(properties)].join(', '); - const values = [`'${sourceId}'`, `'${targetId}'`, ...Object.values(properties) - .map(value => typeof value === 'string' ? `'${value}'` : value) - ].join(', '); - - const query = `INSERT INTO ${tableName}(${columns}) VALUES (${values})`; - await this.executeQuery(query); - }, - - async getDatabaseInfo(): Promise { - console.log('Getting database info'); - - // Get table information - const tablesQuery = "SHOW TABLES"; - const tablesResult = await this.executeQuery(tablesQuery); - - // Get node count - const nodeCountQuery = "MATCH (n) RETURN COUNT(n) as count"; - const nodeResult = await this.executeQuery(nodeCountQuery); - - // Get relationship count - const relCountQuery = "MATCH ()-[r]->() RETURN COUNT(r) as count"; - const relResult = await this.executeQuery(relCountQuery); - - return { - tables: tablesResult.results, - nodeCount: nodeResult.results[0]?.count || 0, - relCount: relResult.results[0]?.count || 0 - }; - }, - - async clearDatabase(): Promise { - console.log('Clearing database'); - - // Drop all tables - const tablesQuery = "SHOW TABLES"; - const tablesResult = await this.executeQuery(tablesQuery); - - for (const table of tablesResult.results) { - const tableName = table.name; - await this.executeQuery(`DROP TABLE ${tableName}`); - } - } - }; -} - -export function getKuzuDBInstance(): KuzuDBInstance | null { - return kuzuInstance; -} - -export function resetKuzuDB(): void { - kuzuInstance = null; -} - -================ -File: src/core/tree-sitter/parser-loader.ts -================ -// Import tree-sitter explicitly to ensure Vite pre-optimizes it -import Parser from "web-tree-sitter"; - -const getWasmPath = (path: string) => { - const baseUrl = import.meta.env.BASE_URL || '/'; - const finalBaseUrl = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`; - return `${finalBaseUrl}wasm/${path}`; -}; - -let parserInstance: Parser | null = null; -const parserCache = new Map(); - -export async function initTreeSitter(): Promise { - if (parserInstance) return parserInstance; - - try { - // Initialize WebAssembly with proper configuration - await Parser.init({ - locateFile(scriptName: string, scriptDirectory: string) { - // Return the correct path for WASM files - if (scriptName.endsWith('.wasm')) { - return getWasmPath(scriptName); - } - return scriptDirectory + scriptName; - } - }); - parserInstance = new Parser(); - return parserInstance; - } catch (error) { - console.error('Failed to initialize Tree-sitter:', error); - throw new Error(`Tree-sitter initialization failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } -} - -export async function loadPythonParser(): Promise { - if (parserCache.has('python')) { - return parserCache.get('python')!; - } - - try { - // Load Python language from WASM file - const wasmPath = getWasmPath('python/tree-sitter-python.wasm'); - console.log('Loading Python parser from:', wasmPath); - - const pythonLang = await Parser.Language.load(wasmPath); - - parserCache.set('python', pythonLang); - console.log('Python parser loaded successfully'); - return pythonLang; - } catch (error) { - console.error('Failed to load Python parser:', error); - throw new Error(`Python parser loading failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } -} - -export async function loadJavaScriptParser(): Promise { - if (parserCache.has('javascript')) { - return parserCache.get('javascript')!; - } - try { - const wasmPath = getWasmPath('javascript/tree-sitter-javascript.wasm'); - console.log('Loading JavaScript parser from:', wasmPath); - const jsLang = await Parser.Language.load(wasmPath); - parserCache.set('javascript', jsLang); - console.log('JavaScript parser loaded successfully'); - return jsLang; - } catch (error) { - console.error('Failed to load JavaScript parser:', error); - throw new Error(`JavaScript parser loading failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } -} - -export async function loadTypeScriptParser(): Promise { - if (parserCache.has('typescript')) { - return parserCache.get('typescript')!; - } - try { - const wasmPath = getWasmPath('typescript/tree-sitter-typescript.wasm'); - console.log('Loading TypeScript parser from:', wasmPath); - const tsLang = await Parser.Language.load(wasmPath); - parserCache.set('typescript', tsLang); - console.log('TypeScript parser loaded successfully'); - return tsLang; - } catch (error) { - console.error('Failed to load TypeScript parser:', error); - throw new Error(`TypeScript parser loading failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } -} - -export async function loadTsxParser(): Promise { - if (parserCache.has('tsx')) { - return parserCache.get('tsx')!; - } - try { - const wasmPath = getWasmPath('typescript/tree-sitter-tsx.wasm'); - console.log('Loading TSX parser from:', wasmPath); - const tsxLang = await Parser.Language.load(wasmPath); - parserCache.set('tsx', tsxLang); - console.log('TSX parser loaded successfully'); - return tsxLang; - } catch (error) { - console.error('Failed to load TSX parser:', error); - throw new Error(`TSX parser loading failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } -} - -================ -File: src/index.css -================ -:root { - font-family: Inter, system-ui, Avenir, Helvetica, Arial, sans-serif; - line-height: 1.5; - font-weight: 400; - - color-scheme: light dark; - color: rgba(255, 255, 255, 0.87); - background-color: #242424; - - font-synthesis: none; - text-rendering: optimizeLegibility; - -webkit-font-smoothing: antialiased; - -moz-osx-font-smoothing: grayscale; -} - -a { - font-weight: 500; - color: #646cff; - text-decoration: inherit; -} -a:hover { - color: #535bf2; -} - -body { - margin: 0; - display: flex; - place-items: center; - min-width: 320px; - min-height: 100vh; -} - -h1 { - font-size: 3.2em; - line-height: 1.1; -} - -button { - border-radius: 8px; - border: 1px solid transparent; - padding: 0.6em 1.2em; - font-size: 1em; - font-weight: 500; - font-family: inherit; - background-color: #1a1a1a; - cursor: pointer; - transition: border-color 0.25s; -} -button:hover { - border-color: #646cff; -} -button:focus, -button:focus-visible { - outline: 4px auto -webkit-focus-ring-color; -} - -@media (prefers-color-scheme: light) { - :root { - color: #213547; - background-color: #ffffff; - } - a:hover { - color: #747bff; - } - button { - background-color: #f9f9f9; - } -} - -================ -File: src/lib/error-handler.ts -================ -/** - * Comprehensive error handling and recovery mechanisms - */ - -import { ConfigService } from '../config/config'; -import { MemoryManager } from '../services/memory-manager'; - -// Base error class for all application errors -export class GitNexusError extends Error { - public readonly code: string; - public readonly timestamp: Date; - public readonly context?: Record; - public readonly isRecoverable: boolean; - - constructor( - message: string, - code: string, - isRecoverable = true, - context?: Record - ) { - super(message); - this.name = 'GitNexusError'; - this.code = code; - this.timestamp = new Date(); - this.context = context; - this.isRecoverable = isRecoverable; - - if (Error.captureStackTrace) { - Error.captureStackTrace(this, GitNexusError); - } - } -} - -// Specific error types -export class ValidationError extends GitNexusError { - constructor(message: string, context?: Record) { - super(message, 'VALIDATION_ERROR', true, context); - this.name = 'ValidationError'; - } -} - -export class NetworkError extends GitNexusError { - constructor(message: string, context?: Record) { - super(message, 'NETWORK_ERROR', true, context); - this.name = 'NetworkError'; - } -} - -export class MemoryError extends GitNexusError { - constructor(message: string, context?: Record) { - super(message, 'MEMORY_ERROR', true, context); - this.name = 'MemoryError'; - } -} - -export class ParsingError extends GitNexusError { - constructor(message: string, context?: Record) { - super(message, 'PARSING_ERROR', true, context); - this.name = 'ParsingError'; - } -} - -export class ConfigurationError extends GitNexusError { - constructor(message: string, context?: Record) { - super(message, 'CONFIGURATION_ERROR', false, context); - this.name = 'ConfigurationError'; - } -} - -// Retry configuration -interface RetryConfig { - maxAttempts: number; - initialDelay: number; - maxDelay: number; - backoffMultiplier: number; - retryableErrors: string[]; -} - -// Circuit breaker states -enum CircuitBreakerState { - CLOSED = 'CLOSED', - OPEN = 'OPEN', - HALF_OPEN = 'HALF_OPEN' -} - -class CircuitBreaker { - private state = CircuitBreakerState.CLOSED; - private failureCount = 0; - private lastFailureTime: number | null = null; - private readonly failureThreshold: number; - private readonly resetTimeout: number; - - constructor(failureThreshold = 5, resetTimeout = 60000) { - this.failureThreshold = failureThreshold; - this.resetTimeout = resetTimeout; - } - - canExecute(): boolean { - if (this.state === CircuitBreakerState.OPEN) { - if (this.lastFailureTime && Date.now() - this.lastFailureTime > this.resetTimeout) { - this.state = CircuitBreakerState.HALF_OPEN; - return true; - } - return false; - } - return true; - } - - recordSuccess(): void { - this.failureCount = 0; - this.state = CircuitBreakerState.CLOSED; - } - - recordFailure(): void { - this.failureCount++; - this.lastFailureTime = Date.now(); - - if (this.failureCount >= this.failureThreshold) { - this.state = CircuitBreakerState.OPEN; - } - } - - getState(): string { - return this.state; - } -} - -// Error recovery service -export class ErrorRecoveryService { - private static instance: ErrorRecoveryService; - private readonly config: ConfigService; - private readonly memoryManager: MemoryManager; - private readonly circuitBreakers = new Map(); - private readonly retryConfig: RetryConfig; - - private constructor() { - this.config = ConfigService.getInstance(); - this.memoryManager = MemoryManager.getInstance(); - this.retryConfig = this.getRetryConfig(); - } - - static getInstance(): ErrorRecoveryService { - if (!ErrorRecoveryService.instance) { - ErrorRecoveryService.instance = new ErrorRecoveryService(); - } - return ErrorRecoveryService.instance; - } - - private getRetryConfig(): RetryConfig { - const processingConfig = this.config.processing; - return { - maxAttempts: processingConfig.retry.maxRetries, - initialDelay: 1000, - maxDelay: 10000, - backoffMultiplier: 2, - retryableErrors: ['NETWORK_ERROR', 'MEMORY_ERROR', 'VALIDATION_ERROR'] - }; - } - - /** - * Execute an operation with retry logic and circuit breaker - */ - async executeWithRetry( - operation: () => Promise, - operationName: string, - customRetryConfig?: Partial - ): Promise { - const circuitBreaker = this.getCircuitBreaker(operationName); - - if (!circuitBreaker.canExecute()) { - throw new GitNexusError( - `Circuit breaker is open for ${operationName}`, - 'CIRCUIT_BREAKER_OPEN', - false, - { operationName, state: circuitBreaker.getState() } - ); - } - - const retryConfig = { ...this.retryConfig, ...customRetryConfig }; - let lastError: Error | null = null; - - for (let attempt = 1; attempt <= retryConfig.maxAttempts; attempt++) { - try { - // Check memory before operation - await this.memoryManager.forceCleanup(); - - const result = await operation(); - circuitBreaker.recordSuccess(); - return result; - } catch (error) { - lastError = error as Error; - - if (!this.shouldRetry(error, retryConfig)) { - circuitBreaker.recordFailure(); - throw error; - } - - if (attempt < retryConfig.maxAttempts) { - const delay = this.calculateDelay(attempt, retryConfig); - await this.wait(delay); - - console.warn(`Retrying ${operationName} (attempt ${attempt}/${retryConfig.maxAttempts})`); - } - } - } - - circuitBreaker.recordFailure(); - throw lastError; - } - - /** - * Execute an operation with graceful degradation - */ - async executeWithFallback( - primaryOperation: () => Promise, - fallbackOperation: () => Promise, - operationName: string - ): Promise { - try { - return await this.executeWithRetry(primaryOperation, operationName); - } catch (error) { - console.warn(`Primary operation failed for ${operationName}, using fallback`, error); - - try { - return await fallbackOperation(); - } catch (fallbackError) { - throw new GitNexusError( - `Both primary and fallback operations failed for ${operationName}`, - 'FALLBACK_FAILED', - false, - { - operationName, - primaryError: error instanceof Error ? error.message : String(error), - fallbackError: fallbackError instanceof Error ? fallbackError.message : String(fallbackError) - } - ); - } - } - } - - /** - * Handle memory-related errors with recovery - */ - async handleMemoryError(error: Error, context?: Record): Promise { - console.error('Memory error detected, attempting recovery', error, context); - - try { - // Force cleanup - await this.memoryManager.clearCache(); - - // Trigger garbage collection if available - if (global.gc) { - global.gc(); - } - - console.info('Memory recovery completed'); - } catch (recoveryError) { - throw new MemoryError( - 'Failed to recover from memory error', - { - originalError: error.message, - recoveryError: recoveryError instanceof Error ? recoveryError.message : String(recoveryError), - ...context - } - ); - } - } - - /** - * Log error with context for debugging - */ - logError(error: GitNexusError, context?: Record): void { - const errorLog = { - timestamp: error.timestamp.toISOString(), - code: error.code, - message: error.message, - stack: error.stack, - context: { ...error.context, ...context }, - isRecoverable: error.isRecoverable - }; - - // In production, this would send to external logging service - console.error('Application error:', JSON.stringify(errorLog, null, 2)); - } - - private getCircuitBreaker(operationName: string): CircuitBreaker { - if (!this.circuitBreakers.has(operationName)) { - this.circuitBreakers.set(operationName, new CircuitBreaker()); - } - return this.circuitBreakers.get(operationName)!; - } - - private shouldRetry(error: unknown, retryConfig: RetryConfig): boolean { - if (error instanceof GitNexusError) { - return retryConfig.retryableErrors.includes(error.code) && error.isRecoverable; - } - return true; // Retry unknown errors by default - } - - private calculateDelay(attempt: number, retryConfig: RetryConfig): number { - const delay = retryConfig.initialDelay * Math.pow(retryConfig.backoffMultiplier, attempt - 1); - return Math.min(delay, retryConfig.maxDelay); - } - - private wait(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); - } -} - -// Global error handler -export function setupGlobalErrorHandlers(): void { - process.on('uncaughtException', (error: Error) => { - const recoveryService = ErrorRecoveryService.getInstance(); - const gitNexusError = error instanceof GitNexusError - ? error - : new GitNexusError( - 'Uncaught exception', - 'UNCAUGHT_EXCEPTION', - false, - { originalError: error.message, stack: error.stack } - ); - - recoveryService.logError(gitNexusError); - - // Attempt graceful shutdown - process.exit(1); - }); - - process.on('unhandledRejection', (reason: unknown) => { - const recoveryService = ErrorRecoveryService.getInstance(); - const error = reason instanceof Error ? reason : new Error(String(reason)); - const gitNexusError = new GitNexusError( - 'Unhandled promise rejection', - 'UNHANDLED_REJECTION', - false, - { reason: error.message, stack: error.stack } - ); - - recoveryService.logError(gitNexusError); - }); -} - -// Utility functions for common error scenarios -export function createSafeAsync Promise>( - fn: T, - errorHandler?: (error: Error, ...args: Parameters) => Promise -): T { - return (async (...args: Parameters) => { - try { - return await fn(...args); - } catch (error) { - if (errorHandler) { - return await errorHandler(error as Error, ...args); - } - throw error; - } - }) as T; -} - -export function createSafe unknown>( - fn: T, - errorHandler?: (error: Error, ...args: Parameters) => unknown -): T { - return ((...args: Parameters) => { - try { - return fn(...args); - } catch (error) { - if (errorHandler) { - return errorHandler(error as Error, ...args); - } - throw error; - } - }) as T; -} - -================ -File: src/lib/export.test.ts -================ -import { exportGraphToCSV } from './export.js'; -import type { KnowledgeGraph } from '../core/graph/types.js'; - -// Mock knowledge graph for testing -const mockGraph: KnowledgeGraph = { - nodes: [ - { - id: 'func_123', - label: 'Function', - properties: { - name: 'validateInput', - filePath: 'src/utils/validation.ts', - startLine: 15, - endLine: 25, - type: 'function', - language: 'typescript', - qualifiedName: 'utils.validation.validateInput' - } - }, - { - id: 'class_456', - label: 'Class', - properties: { - name: 'UserService', - filePath: 'src/services/user.ts', - startLine: 1, - endLine: 50, - type: 'class', - language: 'typescript', - qualifiedName: 'services.user.UserService' - } - } - ], - relationships: [ - { - id: 'rel_789', - type: 'CALLS', - source: 'func_123', - target: 'func_456', - properties: {} - }, - { - id: 'rel_101', - type: 'CONTAINS', - source: 'class_456', - target: 'func_123', - properties: {} - } - ] -}; - -describe('CSV Export', () => { - test('should generate valid CSV for nodes', () => { - const csvData = exportGraphToCSV(mockGraph); - - expect(csvData.nodes).toContain(':ID,name,filePath,startLine,endLine,type,language,qualifiedName,:LABEL'); - expect(csvData.nodes).toContain('func_123,validateInput,src/utils/validation.ts,15,25,function,typescript,utils.validation.validateInput,Function'); - expect(csvData.nodes).toContain('class_456,UserService,src/services/user.ts,1,50,class,typescript,services.user.UserService,Class'); - }); - - test('should generate valid CSV for relationships', () => { - const csvData = exportGraphToCSV(mockGraph); - - expect(csvData.relationships).toContain(':START_ID,:TYPE,:END_ID,source,target'); - expect(csvData.relationships).toContain('func_123,CALLS,func_456,func_123,func_456'); - expect(csvData.relationships).toContain('class_456,CONTAINS,func_123,class_456,func_123'); - }); - - test('should generate proper filename', () => { - const csvData = exportGraphToCSV(mockGraph, { projectName: 'test-project' }); - - expect(csvData.filename).toMatch(/gitnexus-test-project_\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}/); - }); - - test('should handle empty graph', () => { - const emptyGraph: KnowledgeGraph = { nodes: [], relationships: [] }; - const csvData = exportGraphToCSV(emptyGraph); - - expect(csvData.nodes).toBe(':ID,name,filePath,startLine,endLine,type,language,qualifiedName,:LABEL\n'); - expect(csvData.relationships).toBe(':START_ID,:TYPE,:END_ID,source,target\n'); - }); - - test('should escape CSV values with commas and quotes', () => { - const graphWithSpecialChars: KnowledgeGraph = { - nodes: [ - { - id: 'func_123', - label: 'Function', - properties: { - name: 'validateInput, "special" function', - filePath: 'src/utils/validation.ts', - startLine: 15, - endLine: 25, - type: 'function', - language: 'typescript', - qualifiedName: 'utils.validation.validateInput' - } - } - ], - relationships: [] - }; - - const csvData = exportGraphToCSV(graphWithSpecialChars); - - // Should properly escape the name with quotes and double quotes - expect(csvData.nodes).toContain('func_123,"validateInput, ""special"" function",src/utils/validation.ts,15,25,function,typescript,utils.validation.validateInput,Function'); - }); -}); - -console.log('CSV Export tests completed successfully!'); - -================ -File: src/lib/export.ts -================ -import type { KnowledgeGraph } from '../core/graph/types.ts'; - -export interface ExportOptions { - filename?: string; - includeMetadata?: boolean; - prettyPrint?: boolean; - includeTimestamp?: boolean; -} - -export interface ExportMetadata { - exportedAt: string; - version: string; - nodeCount: number; - relationshipCount: number; - fileCount?: number; - processingDuration?: number; -} - -export interface ExportedGraph { - metadata: ExportMetadata; - graph: KnowledgeGraph; - fileContents?: Record; -} - -export interface CSVExportData { - nodes: string; - relationships: string; - filename: string; -} - -/** - * Export a KnowledgeGraph to JSON format - */ -export function exportGraphToJSON( - graph: KnowledgeGraph, - options: ExportOptions = {}, - fileContents?: Map, - processingStats?: { duration: number } -): string { - const { - includeMetadata = true, - prettyPrint = true, - includeTimestamp = true - } = options; - - let exportData: ExportedGraph | KnowledgeGraph; - - if (includeMetadata) { - const metadata: ExportMetadata = { - exportedAt: includeTimestamp ? new Date().toISOString() : '', - version: '1.0.0', - nodeCount: graph.nodes.length, - relationshipCount: graph.relationships.length, - fileCount: fileContents?.size, - processingDuration: processingStats?.duration - }; - - exportData = { - metadata, - graph, - ...(fileContents && { fileContents: Object.fromEntries(fileContents) }) - }; - } else { - exportData = graph; - } - - return JSON.stringify(exportData, null, prettyPrint ? 2 : 0); -} - -/** - * Export a KnowledgeGraph to CSV format - */ -export function exportGraphToCSV( - graph: KnowledgeGraph, - options: ExportOptions & { projectName?: string } = {} -): CSVExportData { - const { projectName, includeTimestamp = true } = options; - - // Generate CSV content for nodes - const nodesCSV = generateNodesCSV(graph.nodes); - - // Generate CSV content for relationships - const relationshipsCSV = generateRelationshipsCSV(graph.relationships); - - // Generate filename - const filename = generateCSVFilename(projectName, includeTimestamp); - - return { - nodes: nodesCSV, - relationships: relationshipsCSV, - filename - }; -} - -/** - * Generate CSV content for nodes - */ -function generateNodesCSV(nodes: KnowledgeGraph['nodes']): string { - if (nodes.length === 0) { - return ':ID,name,filePath,startLine,endLine,type,language,qualifiedName,:LABEL\n'; - } - - const headers = ':ID,name,filePath,startLine,endLine,type,language,qualifiedName,:LABEL'; - const rows = nodes.map(node => { - const properties = node.properties; - const values = [ - escapeCSVValue(node.id), - escapeCSVValue(properties.name || ''), - escapeCSVValue(properties.filePath || ''), - properties.startLine || '', - properties.endLine || '', - escapeCSVValue(properties.type || ''), - escapeCSVValue(properties.language || ''), - escapeCSVValue(properties.qualifiedName || ''), - node.label - ]; - return values.join(','); - }); - - return [headers, ...rows].join('\n'); -} - -/** - * Generate CSV content for relationships - */ -function generateRelationshipsCSV(relationships: KnowledgeGraph['relationships']): string { - if (relationships.length === 0) { - return ':START_ID,:TYPE,:END_ID,source,target\n'; - } - - const headers = ':START_ID,:TYPE,:END_ID,source,target'; - const rows = relationships.map(rel => { - const values = [ - escapeCSVValue(rel.source), - rel.type, - escapeCSVValue(rel.target), - escapeCSVValue(rel.source), - escapeCSVValue(rel.target) - ]; - return values.join(','); - }); - - return [headers, ...rows].join('\n'); -} - -/** - * Escape CSV values (handle commas, quotes, newlines) - */ -function escapeCSVValue(value: string): string { - if (!value) return ''; - - // If value contains comma, quote, or newline, wrap in quotes and escape internal quotes - if (value.includes(',') || value.includes('"') || value.includes('\n')) { - return `"${value.replace(/"/g, '""')}"`; - } - - return value; -} - -/** - * Generate filename for CSV export - */ -function generateCSVFilename(projectName?: string, includeTimestamp: boolean = true): string { - const baseName = projectName - ? `gitnexus-${projectName.replace(/[^a-zA-Z0-9-_]/g, '-')}` - : 'gitnexus-graph'; - - if (includeTimestamp) { - const timestamp = new Date().toISOString() - .replace(/[:.]/g, '-') - .replace('T', '_') - .split('.')[0]; - return `${baseName}_${timestamp}`; - } - - return baseName; -} - -/** - * Trigger download of a JSON file - */ -export function downloadJSON(content: string, filename: string): void { - try { - // Create blob with JSON content - const blob = new Blob([content], { type: 'application/json' }); - - // Create download URL - const url = URL.createObjectURL(blob); - - // Create temporary download link - const link = document.createElement('a'); - link.href = url; - link.download = filename; - link.style.display = 'none'; - - // Add to DOM, click, and remove - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - - // Clean up URL - URL.revokeObjectURL(url); - } catch (error) { - console.error('Failed to download JSON file:', error); - throw new Error('Failed to download file. Please check your browser permissions.'); - } -} - -/** - * Trigger download of CSV files - */ -export function downloadCSV(content: string, filename: string): void { - try { - // Create blob with CSV content - const blob = new Blob([content], { type: 'text/csv' }); - - // Create download URL - const url = URL.createObjectURL(blob); - - // Create temporary download link - const link = document.createElement('a'); - link.href = url; - link.download = filename; - link.style.display = 'none'; - - // Add to DOM, click, and remove - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - - // Clean up URL - URL.revokeObjectURL(url); - } catch (error) { - console.error('Failed to download CSV file:', error); - throw new Error('Failed to download file. Please check your browser permissions.'); - } -} - -/** - * Generate a default filename for the export - */ -export function generateExportFilename( - projectName?: string, - includeTimestamp: boolean = true -): string { - const baseName = projectName - ? `gitnexus-${projectName.replace(/[^a-zA-Z0-9-_]/g, '-')}` - : 'gitnexus-graph'; - - if (includeTimestamp) { - const timestamp = new Date().toISOString() - .replace(/[:.]/g, '-') - .replace('T', '_') - .split('.')[0]; // Remove milliseconds - return `${baseName}_${timestamp}.json`; - } - - return `${baseName}.json`; -} - -/** - * Export and download a KnowledgeGraph - */ -export function exportAndDownloadGraph( - graph: KnowledgeGraph, - options: ExportOptions & { projectName?: string } = {}, - fileContents?: Map, - processingStats?: { duration: number } -): void { - const { - filename, - projectName, - includeTimestamp = true, - ...exportOptions - } = options; - - try { - // Generate filename if not provided - const finalFilename = filename || generateExportFilename(projectName, includeTimestamp); - - // Export to JSON - const jsonContent = exportGraphToJSON(graph, exportOptions, fileContents, processingStats); - - // Trigger download - downloadJSON(jsonContent, finalFilename); - - console.log(`Successfully exported graph to ${finalFilename}`); - } catch (error) { - console.error('Export failed:', error); - throw error; - } -} - -/** - * Export and download a KnowledgeGraph as CSV - */ -export function exportAndDownloadGraphAsCSV( - graph: KnowledgeGraph, - options: ExportOptions & { projectName?: string } = {} -): void { - try { - // Export to CSV - const csvData = exportGraphToCSV(graph, options); - - // Download nodes CSV - downloadCSV(csvData.nodes, `${csvData.filename}_nodes.csv`); - - // Download relationships CSV - downloadCSV(csvData.relationships, `${csvData.filename}_relationships.csv`); - - console.log(`Successfully exported graph to CSV files: ${csvData.filename}_nodes.csv, ${csvData.filename}_relationships.csv`); - } catch (error) { - console.error('CSV export failed:', error); - throw error; - } -} - -/** - * Calculate export file size (approximate) - */ -export function calculateExportSize( - graph: KnowledgeGraph, - includeFileContents: boolean = false, - fileContents?: Map -): { sizeBytes: number; sizeFormatted: string } { - // Create a sample export to measure size - const sampleExport = exportGraphToJSON( - graph, - { includeMetadata: true, prettyPrint: false }, - includeFileContents ? fileContents : undefined - ); - - const sizeBytes = new Blob([sampleExport]).size; - const sizeFormatted = formatFileSize(sizeBytes); - - return { sizeBytes, sizeFormatted }; -} - -/** - * Format file size in human-readable format - */ -export function formatFileSize(bytes: number): string { - if (bytes === 0) return '0 Bytes'; - - const k = 1024; - const sizes = ['Bytes', 'KB', 'MB', 'GB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - - return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; -} - -/** - * Validate if a graph can be exported - */ -export function validateGraphForExport(graph: KnowledgeGraph): { - isValid: boolean; - errors: string[]; - warnings: string[]; -} { - const errors: string[] = []; - const warnings: string[] = []; - - // Check if graph exists - if (!graph) { - errors.push('Graph is null or undefined'); - return { isValid: false, errors, warnings }; - } - - // Check if graph has nodes - if (!graph.nodes || graph.nodes.length === 0) { - warnings.push('Graph has no nodes'); - } - - // Check if graph has relationships - if (!graph.relationships || graph.relationships.length === 0) { - warnings.push('Graph has no relationships'); - } - - // Check for invalid node IDs - const nodeIds = new Set(graph.nodes.map(n => n.id)); - if (nodeIds.size !== graph.nodes.length) { - errors.push('Graph contains duplicate node IDs'); - } - - // Check for invalid relationships - graph.relationships.forEach((rel, index) => { - if (!nodeIds.has(rel.source)) { - errors.push(`Relationship ${index} has invalid source node ID: ${rel.source}`); - } - if (!nodeIds.has(rel.target)) { - errors.push(`Relationship ${index} has invalid target node ID: ${rel.target}`); - } - }); - - // Check for very large exports - const approximateSize = JSON.stringify(graph).length; - if (approximateSize > 50 * 1024 * 1024) { // 50MB - warnings.push('Export file will be very large (>50MB). Consider filtering the data.'); - } - - return { - isValid: errors.length === 0, - errors, - warnings - }; -} - -/** - * Import a KnowledgeGraph from JSON string - */ -export function importGraphFromJSON(jsonString: string): { - graph: KnowledgeGraph; - metadata?: ExportMetadata; - fileContents?: Map; -} { - try { - const parsed = JSON.parse(jsonString); - - // Check if it's an exported graph with metadata - if (parsed.metadata && parsed.graph) { - const result: { - graph: KnowledgeGraph; - metadata: ExportMetadata; - fileContents?: Map; - } = { - graph: parsed.graph, - metadata: parsed.metadata - }; - - // Convert file contents back to Map if present - if (parsed.fileContents) { - result.fileContents = new Map(Object.entries(parsed.fileContents)); - } - - return result; - } - - // Assume it's a raw graph - return { graph: parsed }; - } catch (error) { - throw new Error(`Failed to import graph: ${error instanceof Error ? error.message : 'Invalid JSON'}`); - } -} - -/** - * Create a filtered export of the graph - */ -export function createFilteredExport( - graph: KnowledgeGraph, - filters: { - nodeTypes?: string[]; - relationshipTypes?: string[]; - filePatterns?: string[]; - maxNodes?: number; - } -): KnowledgeGraph { - const { nodeTypes, relationshipTypes, filePatterns, maxNodes } = filters; - - let filteredNodes = graph.nodes; - let filteredRelationships = graph.relationships; - - // Filter by node types - if (nodeTypes && nodeTypes.length > 0) { - filteredNodes = filteredNodes.filter(node => nodeTypes.includes(node.label)); - } - - // Filter by file patterns - if (filePatterns && filePatterns.length > 0) { - filteredNodes = filteredNodes.filter(node => { - const filePath = node.properties.filePath as string; - if (!filePath) return true; // Keep nodes without file paths - - return filePatterns.some(pattern => { - const regex = new RegExp(pattern.replace(/\*/g, '.*'), 'i'); - return regex.test(filePath); - }); - }); - } - - // Limit number of nodes - if (maxNodes && filteredNodes.length > maxNodes) { - filteredNodes = filteredNodes.slice(0, maxNodes); - } - - // Get filtered node IDs - const filteredNodeIds = new Set(filteredNodes.map(n => n.id)); - - // Filter relationships to only include those between filtered nodes - filteredRelationships = filteredRelationships.filter(rel => - filteredNodeIds.has(rel.source) && filteredNodeIds.has(rel.target) - ); - - // Filter by relationship types - if (relationshipTypes && relationshipTypes.length > 0) { - filteredRelationships = filteredRelationships.filter(rel => - relationshipTypes.includes(rel.type) - ); - } - - return { - nodes: filteredNodes, - relationships: filteredRelationships - }; -} - -================ -File: src/lib/kuzu-integration.ts -================ -import { testKuzuDBIntegration, testGitNexusSchema } from './kuzu-test.js'; - -/** - * Browser-friendly integration test for KuzuDB - * Can be called from browser console: window.testKuzuDB() - */ -export async function testKuzuDB(): Promise { - try { - console.log('๐Ÿš€ Starting KuzuDB Integration Test...'); - - // Test basic functionality - await testKuzuDBIntegration(); - - // Test GitNexus schema - await testGitNexusSchema(); - - console.log('๐ŸŽ‰ All KuzuDB integration tests completed successfully!'); - - } catch (error) { - console.error('โŒ KuzuDB integration test failed:', error); - throw error; - } -} - -/** - * Quick performance test for KuzuDB - */ -export async function testKuzuDBPerformance(): Promise { - try { - console.log('โšก Starting KuzuDB Performance Test...'); - - const { initKuzuDB } = await import('../core/kuzu/kuzu-loader.js'); - const kuzuInstance = await initKuzuDB(); - - // Create database - const startTime = performance.now(); - await kuzuInstance.createDatabase('/perf-test'); - const initTime = performance.now() - startTime; - - // Create schema - const schemaStart = performance.now(); - await kuzuInstance.createNodeTable('PerfNode', { - id: 'STRING', - name: 'STRING', - value: 'INT64', - data: 'STRING' - }); - const schemaTime = performance.now() - schemaStart; - - // Insert bulk data - const insertStart = performance.now(); - const insertCount = 1000; - for (let i = 0; i < insertCount; i++) { - await kuzuInstance.insertNode('PerfNode', { - id: `node${i}`, - name: `Node ${i}`, - value: i, - data: `Data for node ${i}` - }); - } - const insertTime = performance.now() - insertStart; - - // Query performance - const queryStart = performance.now(); - const result = await kuzuInstance.executeQuery('MATCH (n:PerfNode) RETURN COUNT(n) as count'); - const queryTime = performance.now() - queryStart; - - // Close database - await kuzuInstance.closeDatabase(); - - console.log('๐Ÿ“Š KuzuDB Performance Results:'); - console.log(` Initialization: ${initTime.toFixed(2)}ms`); - console.log(` Schema Creation: ${schemaTime.toFixed(2)}ms`); - console.log(` Insert ${insertCount} nodes: ${insertTime.toFixed(2)}ms (${(insertTime / insertCount).toFixed(2)}ms per node)`); - console.log(` Query execution: ${queryTime.toFixed(2)}ms`); - console.log(` Total time: ${(initTime + schemaTime + insertTime + queryTime).toFixed(2)}ms`); - console.log(` Query result: ${result.results[0]?.count || 0} nodes`); - - } catch (error) { - console.error('โŒ KuzuDB performance test failed:', error); - throw error; - } -} - -// Make functions available globally for browser testing -if (typeof window !== 'undefined') { - (window as any).testKuzuDB = testKuzuDB; - (window as any).testKuzuDBPerformance = testKuzuDBPerformance; -} - -================ -File: src/lib/kuzu-performance-benchmark.ts -================ -import { KuzuQueryEngine } from '../core/graph/kuzu-query-engine.js'; -import type { KnowledgeGraph } from '../core/graph/types.js'; - -export interface BenchmarkResult { - testName: string; - executionTime: number; - resultCount: number; - memoryUsage?: number; - success: boolean; - error?: string; -} - -export interface PerformanceComparison { - kuzuDBResults: BenchmarkResult[]; - inMemoryResults: BenchmarkResult[]; - improvement: { - averageSpeedup: number; - totalSpeedup: number; - memoryEfficiency: number; - }; -} - -export class KuzuPerformanceBenchmark { - private kuzuQueryEngine: KuzuQueryEngine; - private testQueries: Array<{ - name: string; - query: string; - description: string; - }>; - - constructor() { - this.kuzuQueryEngine = new KuzuQueryEngine(); - this.testQueries = this.initializeTestQueries(); - } - - /** - * Initialize standard test queries for benchmarking - */ - private initializeTestQueries(): Array<{ - name: string; - query: string; - description: string; - }> { - return [ - { - name: 'Simple Node Query', - query: 'MATCH (n:Function) RETURN n LIMIT 100', - description: 'Basic node retrieval with limit' - }, - { - name: 'Complex Relationship Query', - query: 'MATCH (a:Function)-[:CALLS]->(b:Function) RETURN a.name, b.name LIMIT 50', - description: 'Relationship traversal with property access' - }, - { - name: 'Multi-hop Query', - query: 'MATCH (a:Function)-[:CALLS*1..3]->(b:Function) RETURN a.name, b.name LIMIT 30', - description: 'Variable-length path traversal' - }, - { - name: 'Aggregation Query', - query: 'MATCH (f:Function) RETURN f.filePath, COUNT(f) ORDER BY COUNT(f) DESC LIMIT 10', - description: 'Grouping and aggregation operations' - }, - { - name: 'Filtered Query', - query: 'MATCH (f:Function) WHERE f.name CONTAINS "get" OR f.name CONTAINS "set" RETURN f.name, f.filePath LIMIT 50', - description: 'Complex filtering with string operations' - }, - { - name: 'Join Query', - query: 'MATCH (f:File)-[:CONTAINS]->(func:Function)-[:CALLS]->(target:Function) RETURN f.name, func.name, target.name LIMIT 40', - description: 'Multi-table join operations' - }, - { - name: 'Pattern Matching', - query: 'MATCH (c:Class)-[:CONTAINS]->(m:Method)-[:CALLS]->(f:Function) WHERE c.name CONTAINS "Service" RETURN c.name, m.name, f.name LIMIT 25', - description: 'Complex pattern matching with filters' - }, - { - name: 'Subquery', - query: 'MATCH (f:Function) WHERE f.filePath IN (MATCH (file:File) WHERE file.name CONTAINS "service" RETURN file.path) RETURN f.name LIMIT 30', - description: 'Subquery operations' - } - ]; - } - - /** - * Run comprehensive performance benchmark - */ - async runBenchmark(graph: KnowledgeGraph): Promise { - console.log('๐Ÿš€ Starting KuzuDB Performance Benchmark...'); - - // Initialize KuzuDB - await this.kuzuQueryEngine.initialize(); - await this.kuzuQueryEngine.importGraph(graph); - - const kuzuDBResults: BenchmarkResult[] = []; - const inMemoryResults: BenchmarkResult[] = []; - - // Run KuzuDB benchmarks - console.log('๐Ÿ“Š Running KuzuDB benchmarks...'); - for (const testQuery of this.testQueries) { - const result = await this.benchmarkKuzuDBQuery(testQuery); - kuzuDBResults.push(result); - } - - // Run in-memory benchmarks (simulated) - console.log('๐Ÿ“Š Running in-memory benchmarks...'); - for (const testQuery of this.testQueries) { - const result = await this.benchmarkInMemoryQuery(testQuery, graph); - inMemoryResults.push(result); - } - - // Calculate improvements - const improvement = this.calculateImprovement(kuzuDBResults, inMemoryResults); - - console.log('โœ… Performance benchmark completed'); - this.printBenchmarkResults(kuzuDBResults, inMemoryResults, improvement); - - return { - kuzuDBResults, - inMemoryResults, - improvement - }; - } - - /** - * Benchmark a single KuzuDB query - */ - private async benchmarkKuzuDBQuery(testQuery: { - name: string; - query: string; - description: string; - }): Promise { - const startTime = performance.now(); - const startMemory = performance.memory?.usedJSHeapSize; - - try { - const result = await this.kuzuQueryEngine.executeQuery(testQuery.query, { - includeExecutionTime: true - }); - - const executionTime = performance.now() - startTime; - const endMemory = performance.memory?.usedJSHeapSize; - const memoryUsage = endMemory && startMemory ? endMemory - startMemory : undefined; - - return { - testName: testQuery.name, - executionTime, - resultCount: result.resultCount, - memoryUsage, - success: true - }; - - } catch (error) { - const executionTime = performance.now() - startTime; - - return { - testName: testQuery.name, - executionTime, - resultCount: 0, - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }; - } - } - - /** - * Benchmark a single in-memory query (simulated) - */ - private async benchmarkInMemoryQuery( - testQuery: { name: string; query: string; description: string }, - graph: KnowledgeGraph - ): Promise { - const startTime = performance.now(); - const startMemory = performance.memory?.usedJSHeapSize; - - try { - // Simulate in-memory query execution - // This is a simplified simulation - real implementation would be more complex - await this.simulateInMemoryQuery(testQuery.query, graph); - - const executionTime = performance.now() - startTime; - const endMemory = performance.memory?.usedJSHeapSize; - const memoryUsage = endMemory && startMemory ? endMemory - startMemory : undefined; - - // Simulate result count based on query complexity - const resultCount = this.estimateResultCount(testQuery.query, graph); - - return { - testName: testQuery.name, - executionTime, - resultCount, - memoryUsage, - success: true - }; - - } catch (error) { - const executionTime = performance.now() - startTime; - - return { - testName: testQuery.name, - executionTime, - resultCount: 0, - success: false, - error: error instanceof Error ? error.message : 'Unknown error' - }; - } - } - - /** - * Simulate in-memory query execution - */ - private async simulateInMemoryQuery(query: string, graph: KnowledgeGraph): Promise { - // Simulate processing time based on query complexity - const complexity = this.estimateQueryComplexity(query); - const baseTime = 10; // Base processing time in ms - const complexityMultiplier = Math.max(1, complexity / 10); - - await new Promise(resolve => setTimeout(resolve, baseTime * complexityMultiplier)); - } - - /** - * Estimate query complexity for simulation - */ - private estimateQueryComplexity(query: string): number { - let complexity = 1; - - if (query.includes('*1..3')) complexity += 3; - if (query.includes('COUNT')) complexity += 2; - if (query.includes('WHERE')) complexity += 1; - if (query.includes('ORDER BY')) complexity += 1; - if (query.includes('CONTAINS')) complexity += 1; - if (query.includes('IN (')) complexity += 2; - - return complexity; - } - - /** - * Estimate result count for in-memory simulation - */ - private estimateResultCount(query: string, graph: KnowledgeGraph): number { - // Simple estimation based on query patterns - if (query.includes('LIMIT')) { - const limitMatch = query.match(/LIMIT (\d+)/); - return limitMatch ? parseInt(limitMatch[1]) : 50; - } - - if (query.includes('Function')) return Math.min(graph.nodes.filter(n => n.label === 'Function').length, 100); - if (query.includes('Class')) return Math.min(graph.nodes.filter(n => n.label === 'Class').length, 50); - if (query.includes('File')) return Math.min(graph.nodes.filter(n => n.label === 'File').length, 200); - - return 25; // Default estimate - } - - /** - * Calculate performance improvement metrics - */ - private calculateImprovement( - kuzuDBResults: BenchmarkResult[], - inMemoryResults: BenchmarkResult[] - ): PerformanceComparison['improvement'] { - const successfulKuzu = kuzuDBResults.filter(r => r.success); - const successfulInMemory = inMemoryResults.filter(r => r.success); - - if (successfulKuzu.length === 0 || successfulInMemory.length === 0) { - return { - averageSpeedup: 1, - totalSpeedup: 1, - memoryEfficiency: 1 - }; - } - - // Calculate speedup ratios - const speedups = successfulKuzu.map((kuzuResult, index) => { - const inMemoryResult = successfulInMemory[index]; - if (!inMemoryResult) return 1; - - return inMemoryResult.executionTime / kuzuResult.executionTime; - }); - - const averageSpeedup = speedups.reduce((sum, speedup) => sum + speedup, 0) / speedups.length; - - // Calculate total execution time improvement - const totalKuzuTime = successfulKuzu.reduce((sum, r) => sum + r.executionTime, 0); - const totalInMemoryTime = successfulInMemory.reduce((sum, r) => sum + r.executionTime, 0); - const totalSpeedup = totalInMemoryTime / totalKuzuTime; - - // Calculate memory efficiency - const kuzuMemory = successfulKuzu.reduce((sum, r) => sum + (r.memoryUsage || 0), 0); - const inMemoryMemory = successfulInMemory.reduce((sum, r) => sum + (r.memoryUsage || 0), 0); - const memoryEfficiency = inMemoryMemory > 0 ? kuzuMemory / inMemoryMemory : 1; - - return { - averageSpeedup, - totalSpeedup, - memoryEfficiency - }; - } - - /** - * Print benchmark results to console - */ - private printBenchmarkResults( - kuzuDBResults: BenchmarkResult[], - inMemoryResults: BenchmarkResult[], - improvement: PerformanceComparison['improvement'] - ): void { - console.log('\n๐Ÿ“ˆ Performance Benchmark Results:'); - console.log('====================================='); - - console.log('\n๐Ÿ” Individual Test Results:'); - kuzuDBResults.forEach((kuzuResult, index) => { - const inMemoryResult = inMemoryResults[index]; - const speedup = inMemoryResult && kuzuResult.success && inMemoryResult.success - ? inMemoryResult.executionTime / kuzuResult.executionTime - : 1; - - console.log(`${kuzuResult.testName}:`); - console.log(` KuzuDB: ${kuzuResult.executionTime.toFixed(2)}ms (${kuzuResult.resultCount} results)`); - if (inMemoryResult) { - console.log(` In-Memory: ${inMemoryResult.executionTime.toFixed(2)}ms (${inMemoryResult.resultCount} results)`); - console.log(` Speedup: ${speedup.toFixed(2)}x`); - } - console.log(''); - }); - - console.log('๐Ÿ“Š Overall Performance:'); - console.log(` Average Speedup: ${improvement.averageSpeedup.toFixed(2)}x`); - console.log(` Total Speedup: ${improvement.totalSpeedup.toFixed(2)}x`); - console.log(` Memory Efficiency: ${improvement.memoryEfficiency.toFixed(2)}x`); - - if (improvement.averageSpeedup > 1) { - console.log('โœ… KuzuDB shows performance improvements!'); - } else { - console.log('โš ๏ธ KuzuDB performance needs optimization'); - } - } - - /** - * Generate benchmark report - */ - generateReport(comparison: PerformanceComparison): string { - const { kuzuDBResults, inMemoryResults, improvement } = comparison; - - let report = '# KuzuDB Performance Benchmark Report\n\n'; - - report += '## Summary\n'; - report += `- **Average Speedup**: ${improvement.averageSpeedup.toFixed(2)}x\n`; - report += `- **Total Speedup**: ${improvement.totalSpeedup.toFixed(2)}x\n`; - report += `- **Memory Efficiency**: ${improvement.memoryEfficiency.toFixed(2)}x\n\n`; - - report += '## Detailed Results\n\n'; - report += '| Test | KuzuDB (ms) | In-Memory (ms) | Speedup | Status |\n'; - report += '|------|-------------|----------------|---------|--------|\n'; - - kuzuDBResults.forEach((kuzuResult, index) => { - const inMemoryResult = inMemoryResults[index]; - const speedup = inMemoryResult && kuzuResult.success && inMemoryResult.success - ? inMemoryResult.executionTime / kuzuResult.executionTime - : 1; - - const status = kuzuResult.success ? 'โœ…' : 'โŒ'; - - report += `| ${kuzuResult.testName} | ${kuzuResult.executionTime.toFixed(2)} | ${inMemoryResult?.executionTime.toFixed(2) || 'N/A'} | ${speedup.toFixed(2)}x | ${status} |\n`; - }); - - return report; - } - - /** - * Clean up resources - */ - async cleanup(): Promise { - await this.kuzuQueryEngine.close(); - } -} - -================ -File: src/lib/kuzu-performance-monitor.ts -================ -import { isPerformanceMonitoringEnabled } from '../config/feature-flags.js'; - -export interface PerformanceMetric { - operation: string; - duration: number; - timestamp: number; - success: boolean; - error?: string; - metadata?: Record; -} - -export interface PerformanceReport { - totalOperations: number; - successfulOperations: number; - failedOperations: number; - averageDuration: number; - minDuration: number; - maxDuration: number; - totalDuration: number; - operationsByType: Record; - recentOperations: PerformanceMetric[]; - timestamp: number; -} - -export class KuzuPerformanceMonitor { - private metrics: PerformanceMetric[] = []; - private maxMetrics: number = 1000; - private enabled: boolean = false; - - constructor() { - this.enabled = isPerformanceMonitoringEnabled(); - } - - /** - * Start monitoring a performance metric - */ - startOperation(operation: string, metadata?: Record): string { - if (!this.enabled) return ''; - - const id = `${operation}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - const startTime = performance.now(); - - // Store start time in a weak map or similar for cleanup - (this as any)[`_start_${id}`] = { startTime, operation, metadata }; - - return id; - } - - /** - * End monitoring a performance metric - */ - endOperation(id: string, success: boolean = true, error?: string): void { - if (!this.enabled || !id) return; - - const startData = (this as any)[`_start_${id}`]; - if (!startData) return; - - const endTime = performance.now(); - const duration = endTime - startData.startTime; - - const metric: PerformanceMetric = { - operation: startData.operation, - duration, - timestamp: Date.now(), - success, - error, - metadata: startData.metadata - }; - - this.addMetric(metric); - - // Clean up start data - delete (this as any)[`_start_${id}`]; - } - - /** - * Add a performance metric directly - */ - addMetric(metric: PerformanceMetric): void { - if (!this.enabled) return; - - this.metrics.push(metric); - - // Keep only the most recent metrics - if (this.metrics.length > this.maxMetrics) { - this.metrics = this.metrics.slice(-this.maxMetrics); - } - } - - /** - * Get performance report for the last N operations - */ - getReport(operationCount: number = 100): PerformanceReport { - const recentMetrics = this.metrics.slice(-operationCount); - - if (recentMetrics.length === 0) { - return { - totalOperations: 0, - successfulOperations: 0, - failedOperations: 0, - averageDuration: 0, - minDuration: 0, - maxDuration: 0, - totalDuration: 0, - operationsByType: {}, - recentOperations: [], - timestamp: Date.now() - }; - } - - const successfulOperations = recentMetrics.filter(m => m.success); - const failedOperations = recentMetrics.filter(m => !m.success); - - const durations = recentMetrics.map(m => m.duration); - const totalDuration = durations.reduce((sum, d) => sum + d, 0); - const averageDuration = totalDuration / recentMetrics.length; - const minDuration = Math.min(...durations); - const maxDuration = Math.max(...durations); - - // Group by operation type - const operationsByType: Record = {}; - - recentMetrics.forEach(metric => { - if (!operationsByType[metric.operation]) { - operationsByType[metric.operation] = { count: 0, averageDuration: 0, totalDuration: 0 }; - } - - operationsByType[metric.operation].count++; - operationsByType[metric.operation].totalDuration += metric.duration; - }); - - // Calculate averages for each operation type - Object.keys(operationsByType).forEach(operation => { - const data = operationsByType[operation]; - data.averageDuration = data.totalDuration / data.count; - }); - - return { - totalOperations: recentMetrics.length, - successfulOperations: successfulOperations.length, - failedOperations: failedOperations.length, - averageDuration, - minDuration, - maxDuration, - totalDuration, - operationsByType, - recentOperations: recentMetrics.slice(-20), // Last 20 operations - timestamp: Date.now() - }; - } - - /** - * Get performance report for a specific operation type - */ - getOperationReport(operation: string, operationCount: number = 100): PerformanceReport | null { - const operationMetrics = this.metrics - .filter(m => m.operation === operation) - .slice(-operationCount); - - if (operationMetrics.length === 0) return null; - - const successfulOperations = operationMetrics.filter(m => m.success); - const failedOperations = operationMetrics.filter(m => !m.success); - - const durations = operationMetrics.map(m => m.duration); - const totalDuration = durations.reduce((sum, d) => sum + d, 0); - const averageDuration = totalDuration / operationMetrics.length; - const minDuration = Math.min(...durations); - const maxDuration = Math.max(...durations); - - return { - totalOperations: operationMetrics.length, - successfulOperations: successfulOperations.length, - failedOperations: failedOperations.length, - averageDuration, - minDuration, - maxDuration, - totalDuration, - operationsByType: { - [operation]: { - count: operationMetrics.length, - averageDuration, - totalDuration - } - }, - recentOperations: operationMetrics.slice(-10), - timestamp: Date.now() - }; - } - - /** - * Clear all metrics - */ - clearMetrics(): void { - this.metrics = []; - } - - /** - * Enable or disable monitoring - */ - setEnabled(enabled: boolean): void { - this.enabled = enabled; - } - - /** - * Check if monitoring is enabled - */ - isEnabled(): boolean { - return this.enabled; - } - - /** - * Get metrics count - */ - getMetricsCount(): number { - return this.metrics.length; - } - - /** - * Export metrics for analysis - */ - exportMetrics(): PerformanceMetric[] { - return [...this.metrics]; - } - - /** - * Import metrics from external source - */ - importMetrics(metrics: PerformanceMetric[]): void { - this.metrics = [...this.metrics, ...metrics].slice(-this.maxMetrics); - } - - /** - * Get performance summary for console logging - */ - getSummary(): string { - const report = this.getReport(); - - if (report.totalOperations === 0) { - return 'No performance data available'; - } - - const successRate = ((report.successfulOperations / report.totalOperations) * 100).toFixed(1); - - return ` -๐Ÿ“Š KuzuDB Performance Summary: - Total Operations: ${report.totalOperations} - Success Rate: ${successRate}% - Average Duration: ${report.averageDuration.toFixed(2)}ms - Min Duration: ${report.minDuration.toFixed(2)}ms - Max Duration: ${report.maxDuration.toFixed(2)}ms - Total Duration: ${report.totalDuration.toFixed(2)}ms - -Operations by Type: -${Object.entries(report.operationsByType) - .map(([operation, data]) => - ` ${operation}: ${data.count} ops, ${data.averageDuration.toFixed(2)}ms avg` - ).join('\n')} - `.trim(); - } -} - -// Export singleton instance -export const kuzuPerformanceMonitor = new KuzuPerformanceMonitor(); - -// Export convenience functions -export const startKuzuOperation = (operation: string, metadata?: Record) => - kuzuPerformanceMonitor.startOperation(operation, metadata); - -export const endKuzuOperation = (id: string, success?: boolean, error?: string) => - kuzuPerformanceMonitor.endOperation(id, success, error); - -export const getKuzuPerformanceReport = (operationCount?: number) => - kuzuPerformanceMonitor.getReport(operationCount); - -export const logKuzuPerformanceSummary = () => - console.log(kuzuPerformanceMonitor.getSummary()); - -================ -File: src/lib/kuzu-test.ts -================ -import { initKuzuDB } from '../core/kuzu/kuzu-loader.js'; - -/** - * Simple test to verify KuzuDB integration - */ -export async function testKuzuDBIntegration(): Promise { - try { - console.log('๐Ÿงช Testing KuzuDB Integration...'); - - // Initialize KuzuDB - const kuzuInstance = await initKuzuDB(); - console.log('โœ… KuzuDB initialized successfully'); - - // Create database - await kuzuInstance.createDatabase('/database'); - console.log('โœ… Database created successfully'); - - // Create a simple test table - await kuzuInstance.createNodeTable('TestNode', { - id: 'STRING', - name: 'STRING', - value: 'INT64' - }); - console.log('โœ… Test table created successfully'); - - // Insert test data - await kuzuInstance.insertNode('TestNode', { - id: 'test1', - name: 'Test Node 1', - value: 42 - }); - console.log('โœ… Test data inserted successfully'); - - // Query the data - const result = await kuzuInstance.executeQuery('MATCH (n:TestNode) RETURN n.name, n.value'); - console.log('โœ… Query executed successfully'); - console.log('Query result:', result); - - // Get database info - const dbInfo = await kuzuInstance.getDatabaseInfo(); - console.log('โœ… Database info retrieved'); - console.log('Database info:', dbInfo); - - // Close database - await kuzuInstance.closeDatabase(); - console.log('โœ… Database closed successfully'); - - console.log('๐ŸŽ‰ All KuzuDB tests passed!'); - - } catch (error) { - console.error('โŒ KuzuDB test failed:', error); - throw error; - } -} - -/** - * Test KuzuDB with GitNexus-like schema - */ -export async function testGitNexusSchema(): Promise { - try { - console.log('๐Ÿงช Testing GitNexus Schema with KuzuDB...'); - - const kuzuInstance = await initKuzuDB(); - await kuzuInstance.createDatabase('/gitnexus'); - - // Create GitNexus-like schema - await kuzuInstance.createNodeTable('File', { - id: 'STRING', - name: 'STRING', - path: 'STRING', - language: 'STRING', - size: 'INT64' - }); - - await kuzuInstance.createNodeTable('Function', { - id: 'STRING', - name: 'STRING', - filePath: 'STRING', - startLine: 'INT64', - endLine: 'INT64', - parameters: 'STRING' - }); - - await kuzuInstance.createRelTable('CALLS', { - id: 'STRING', - source: 'STRING', - target: 'STRING', - confidence: 'DOUBLE' - }); - - // Insert test data - await kuzuInstance.insertNode('File', { - id: 'file1', - name: 'main.ts', - path: '/src/main.ts', - language: 'typescript', - size: 1024 - }); - - await kuzuInstance.insertNode('Function', { - id: 'func1', - name: 'processData', - filePath: '/src/main.ts', - startLine: 10, - endLine: 25, - parameters: 'data: string' - }); - - await kuzuInstance.insertNode('Function', { - id: 'func2', - name: 'validateInput', - filePath: '/src/main.ts', - startLine: 30, - endLine: 40, - parameters: 'input: any' - }); - - await kuzuInstance.insertRel('CALLS', 'func1', 'func2', { - id: 'call1', - source: 'func1', - target: 'func2', - confidence: 0.95 - }); - - // Query the knowledge graph - const result = await kuzuInstance.executeQuery(` - MATCH (f:Function)-[:CALLS]->(g:Function) - RETURN f.name as caller, g.name as callee - `); - - console.log('โœ… GitNexus schema test completed'); - console.log('Query result:', result); - - await kuzuInstance.closeDatabase(); - - } catch (error) { - console.error('โŒ GitNexus schema test failed:', error); - throw error; - } -} - -================ -File: src/lib/polyfills.ts -================ -// Browser polyfill for Node.js AsyncLocalStorage -export class AsyncLocalStorage { - private storage = new Map(); - private currentId = 0; - - constructor() {} - - run(store: T, callback: () => R): R { - const id = (++this.currentId).toString(); - this.storage.set(id, store); - try { - return callback(); - } finally { - this.storage.delete(id); - } - } - - getStore(): T | undefined { - // In browser context, we can't truly replicate AsyncLocalStorage - // Return undefined as fallback - return undefined; - } -} - -// Export as both named and default to match different import styles -export { AsyncLocalStorage as default }; - -// Polyfill for global async_hooks if not available -if (typeof globalThis !== 'undefined' && !(globalThis as any).AsyncLocalStorage) { - (globalThis as any).AsyncLocalStorage = AsyncLocalStorage; -} - -================ -File: src/lib/preload.ts -================ -// Preload all dependencies that might be loaded during processing -// This ensures Vite optimizes them during initial build rather than during runtime - -import 'web-tree-sitter'; -import 'comlink'; - -// Import all the processing modules to ensure their dependencies are discovered -import '../core/ingestion/pipeline'; -import '../core/ingestion/parsing-processor'; -import '../core/ingestion/call-processor'; -import '../core/ingestion/structure-processor'; -import '../core/tree-sitter/parser-loader'; - -console.log('Dependencies preloaded to prevent runtime optimization'); - -================ -File: src/lib/shared-utils.ts -================ -/** - * Shared utilities for path handling, deduplication, and common operations - * Consolidates duplicated code across processors - */ - -// Path utilities for browser compatibility -export const pathUtils = { - extname: (filePath: string): string => { - const lastDot = filePath.lastIndexOf('.'); - return lastDot === -1 ? '' : filePath.substring(lastDot); - }, - - dirname: (filePath: string): string => { - const lastSlash = Math.max(filePath.lastIndexOf('/'), filePath.lastIndexOf('\\')); - return lastSlash === -1 ? '.' : filePath.substring(0, lastSlash); - }, - - resolve: (basePath: string, relativePath: string): string => { - // Simple relative path resolution - if (relativePath.startsWith('./')) { - return basePath + '/' + relativePath.substring(2); - } else if (relativePath.startsWith('../')) { - const parts = basePath.split('/'); - const relativeParts = relativePath.split('/'); - let upCount = 0; - for (const part of relativeParts) { - if (part === '..') upCount++; - else break; - } - const resultParts = parts.slice(0, -upCount); - const remainingParts = relativeParts.slice(upCount); - return [...resultParts, ...remainingParts].join('/'); - } - return basePath + '/' + relativePath; - }, - - join: (...parts: string[]): string => { - return parts.join('/').replace(/\/+/g, '/'); - }, - - normalize: (filePath: string): string => { - return filePath.replace(/\\/g, '/'); - }, - - getFileExtension: (filePath: string): string => { - if (!filePath) return ''; - const lastDotIndex = filePath.lastIndexOf('.'); - if (lastDotIndex === -1) { - return ''; - } - return filePath.substring(lastDotIndex); - }, - - getFileName: (filePath: string): string => { - return filePath.split('/').pop() || filePath; - } -}; - -// Performance optimizations using Sets instead of Arrays -export class OptimizedSet { - private items: Set; - private _array: T[] | null = null; - - constructor(items?: Iterable) { - this.items = new Set(items); - } - - add(item: T): this { - this.items.add(item); - this._array = null; // Invalidate cache - return this; - } - - has(item: T): boolean { - return this.items.has(item); - } - - delete(item: T): boolean { - const result = this.items.delete(item); - if (result) { - this._array = null; // Invalidate cache - } - return result; - } - - clear(): void { - this.items.clear(); - this._array = null; - } - - get size(): number { - return this.items.size; - } - - // Convert to array only when needed and cache the result - toArray(): T[] { - if (this._array === null) { - this._array = Array.from(this.items); - } - return this._array; - } - - forEach(callback: (value: T) => void): void { - this.items.forEach(callback); - } - - filter(predicate: (value: T) => boolean): T[] { - return this.toArray().filter(predicate); - } - - map(mapper: (value: T) => U): U[] { - return this.toArray().map(mapper); - } - - some(predicate: (value: T) => boolean): boolean { - for (const item of this.items) { - if (predicate(item)) return true; - } - return false; - } - - every(predicate: (value: T) => boolean): boolean { - for (const item of this.items) { - if (!predicate(item)) return false; - } - return true; - } -} - -// Duplicate detection using Sets for O(1) performance -export class DuplicateDetector { - private seen: Set; - private keyExtractor: (item: T) => string; - - constructor(keyExtractor: (item: T) => string) { - this.seen = new Set(); - this.keyExtractor = keyExtractor; - } - - isDuplicate(item: T): boolean { - const key = this.keyExtractor(item); - return this.seen.has(key); - } - - markAsSeen(item: T): void { - const key = this.keyExtractor(item); - this.seen.add(key); - } - - checkAndMark(item: T): boolean { - const key = this.keyExtractor(item); - if (this.seen.has(key)) { - return true; // Is duplicate - } - this.seen.add(key); - return false; // Not duplicate - } - - clear(): void { - this.seen.clear(); - } - - get size(): number { - return this.seen.size; - } -} - -// Language detection utilities -export const languageUtils = { - getLanguageFromExtension: (extension: string): string => { - switch (extension.toLowerCase()) { - case '.py': - case '.pyx': - case '.pyi': - return 'python'; - case '.js': - case '.mjs': - case '.cjs': - case '.jsx': - return 'javascript'; - case '.ts': - return 'typescript'; - case '.tsx': - return 'tsx'; - case '.java': - return 'java'; - case '.cpp': - case '.cc': - case '.cxx': - return 'cpp'; - case '.c': - return 'c'; - case '.h': - case '.hpp': - return 'header'; - case '.cs': - return 'csharp'; - case '.php': - return 'php'; - case '.rb': - return 'ruby'; - case '.go': - return 'go'; - case '.rs': - return 'rust'; - case '.swift': - return 'swift'; - case '.kt': - return 'kotlin'; - case '.scala': - return 'scala'; - case '.dart': - return 'dart'; - default: - return 'unknown'; - } - }, - - isSourceFile: (filePath: string): boolean => { - if (!filePath) return false; - const fileName = pathUtils.getFileName(filePath); - - // Include special files - if (fileName === '__init__.py') return true; - - const extension = pathUtils.getFileExtension(filePath).toLowerCase(); - const sourceExtensions = new Set([ - '.py', '.js', '.jsx', '.ts', '.tsx', '.java', '.cpp', '.c', '.h', '.hpp', - '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.dart' - ]); - return sourceExtensions.has(extension); - }, - - isConfigFile: (filePath: string): boolean => { - if (!filePath) return false; - const fileName = pathUtils.getFileName(filePath); - - const configFiles = new Set([ - 'package.json', 'tsconfig.json', 'tsconfig.base.json', - 'vite.config.ts', 'vite.config.js', 'webpack.config.js', - '.eslintrc', '.eslintrc.json', '.eslintrc.js', - '.prettierrc', '.prettierrc.json', - 'docker-compose.yml', 'docker-compose.yaml', - 'dockerfile', 'Dockerfile', - '.env', '.env.example', '.env.local', '.env.production', - 'pyproject.toml', 'setup.py', 'requirements.txt', 'poetry.lock', - 'Cargo.toml', 'Cargo.lock', - 'pom.xml', 'build.gradle', 'build.gradle.kts' - ]); - - return configFiles.has(fileName.toLowerCase()); - } -}; - -// Memory-efficient batch processing -export class BatchProcessor { - private batchSize: number; - private processor: (batch: T[]) => Promise; - - constructor(batchSize: number, processor: (batch: T[]) => Promise) { - this.batchSize = batchSize; - this.processor = processor; - } - - async processAll( - items: T[], - progressCallback?: (processed: number, total: number) => void - ): Promise { - const results: R[] = []; - const total = items.length; - - for (let i = 0; i < items.length; i += this.batchSize) { - const batch = items.slice(i, i + this.batchSize); - const batchResults = await this.processor(batch); - results.push(...batchResults); - - if (progressCallback) { - progressCallback(Math.min(i + this.batchSize, total), total); - } - - // Small delay to prevent blocking - if (i + this.batchSize < items.length) { - await new Promise(resolve => setTimeout(resolve, 1)); - } - } - - return results; - } -} - -// String utilities for sanitization and validation -export const stringUtils = { - sanitize: (input: string, maxLength = 1000): string => { - return input - .trim() - .replace(/[<>]/g, '') // Remove potential XSS vectors - .substring(0, maxLength); - }, - - isValidIdentifier: (name: string): boolean => { - return /^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name); - }, - - extractIdentifiers: (text: string): string[] => { - const identifiers = text.match(/[a-zA-Z_][a-zA-Z0-9_]*/g); - return identifiers ? [...new Set(identifiers)] : []; - }, - - levenshteinDistance: (a: string, b: string): number => { - if (a.length === 0) return b.length; - if (b.length === 0) return a.length; - - const matrix: number[][] = []; - - for (let i = 0; i <= b.length; i++) { - matrix[i] = [i]; - } - - for (let j = 0; j <= a.length; j++) { - matrix[0][j] = j; - } - - for (let i = 1; i <= b.length; i++) { - for (let j = 1; j <= a.length; j++) { - if (b.charAt(i - 1) === a.charAt(j - 1)) { - matrix[i][j] = matrix[i - 1][j - 1]; - } else { - matrix[i][j] = Math.min( - matrix[i - 1][j - 1] + 1, // substitution - matrix[i][j - 1] + 1, // insertion - matrix[i - 1][j] + 1 // deletion - ); - } - } - } - - return matrix[b.length][a.length]; - } -}; - -================ -File: src/lib/streaming-processor.ts -================ -/** - * Streaming processor for handling large files with memory constraints - */ - -import { createReadStream } from 'fs'; -import { createInterface } from 'readline'; -import { pipeline } from 'stream/promises'; -import { Transform } from 'stream'; -import { MemoryManager } from '../services/memory-manager'; -import { ErrorRecoveryService } from './error-handler'; - -export interface StreamingOptions { - chunkSize?: number; - maxFileSize?: number; - encoding?: BufferEncoding; - parallelProcessing?: boolean; - maxConcurrency?: number; -} - -export interface ProcessingResult { - data: T[]; - processedLines: number; - skippedLines: number; - errors: ProcessingError[]; - processingTime: number; -} - -export interface ProcessingError { - lineNumber: number; - error: string; - content?: string; -} - -export interface ProgressCallback { - (progress: { - processedLines: number; - totalLines?: number; - percentage?: number; - currentChunk: T[]; - }): void; -} - -export class StreamingProcessor { - private static instance: StreamingProcessor; - private readonly memoryManager: MemoryManager; - private readonly errorRecovery: ErrorRecoveryService; - - private constructor() { - this.memoryManager = MemoryManager.getInstance(); - this.errorRecovery = ErrorRecoveryService.getInstance(); - } - - static getInstance(): StreamingProcessor { - if (!StreamingProcessor.instance) { - StreamingProcessor.instance = new StreamingProcessor(); - } - return StreamingProcessor.instance; - } - - /** - * Process a file line by line using streaming - */ - async processFile( - filePath: string, - lineProcessor: (line: string, lineNumber: number) => Promise, - options: StreamingOptions = {}, - progressCallback?: ProgressCallback - ): Promise> { - const startTime = Date.now(); - const defaultOptions: StreamingOptions = { - chunkSize: 1000, - maxFileSize: 50 * 1024 * 1024, // 50MB - encoding: 'utf8', - parallelProcessing: false, - maxConcurrency: 4 - }; - - const finalOptions = { ...defaultOptions, ...options }; - - return this.errorRecovery.executeWithRetry(async () => { - // Check file size - const fileSize = await this.getFileSize(filePath); - if (fileSize > (finalOptions.maxFileSize ?? 0)) { - throw new Error(`File too large: ${fileSize} bytes (max: ${finalOptions.maxFileSize})`); - } - - // Check memory - await this.memoryManager.forceCleanup(); - - const results: T[] = []; - const errors: ProcessingError[] = []; - let processedLines = 0; - let skippedLines = 0; - - const readStream = createReadStream(filePath, { encoding: finalOptions.encoding }); - const rl = createInterface({ - input: readStream, - crlfDelay: Infinity - }); - - let currentChunk: T[] = []; - let lineNumber = 0; - - try { - for await (const line of rl) { - lineNumber++; - - try { - const result = await lineProcessor(line, lineNumber); - if (result !== null) { - currentChunk.push(result); - results.push(result); - } else { - skippedLines++; - } - - processedLines++; - - // Process chunk if size reached - if (currentChunk.length >= (finalOptions.chunkSize ?? 0)) { - if (progressCallback) { - progressCallback({ - processedLines, - percentage: fileSize ? (processedLines / (fileSize / 100)) * 100 : undefined, - currentChunk - }); - } - - // Clear chunk to manage memory - currentChunk = []; - - // Periodic memory check - if (processedLines % 10000 === 0) { - await this.memoryManager.forceCleanup(); - } - } - } catch (error) { - errors.push({ - lineNumber, - error: error instanceof Error ? error.message : String(error), - content: line.substring(0, 100) // Limit content length - }); - } - } - - // Final progress callback - if (progressCallback && currentChunk.length > 0) { - progressCallback({ - processedLines, - percentage: 100, - currentChunk - }); - } - - } finally { - rl.close(); - readStream.destroy(); - } - - return { - data: results, - processedLines, - skippedLines, - errors, - processingTime: Date.now() - startTime - }; - }, `processFile:${filePath}`); - } - - /** - * Process multiple files in parallel with memory management - */ - async processFilesParallel( - files: string[], - fileProcessor: (filePath: string) => Promise, - maxConcurrency = 4 - ): Promise> { - const semaphore = new Semaphore(maxConcurrency); - const results: Array<{ filePath: string; result: T | null; error?: string }> = []; - - const processFileWithSemaphore = async (filePath: string) => { - await semaphore.acquire(); - try { - const result = await this.errorRecovery.executeWithRetry( - () => fileProcessor(filePath), - `processFile:${filePath}` - ); - results.push({ filePath, result }); - } catch (error) { - results.push({ - filePath, - result: null, - error: error instanceof Error ? error.message : String(error) - }); - } finally { - semaphore.release(); - } - }; - - await Promise.all(files.map(processFileWithSemaphore)); - return results; - } - - /** - * Transform stream for processing large data streams - */ - createTransformStream( - transformer: (chunk: T) => Promise, - options: { parallel?: boolean; maxConcurrency?: number } = {} - ): Transform { - const { parallel = false, maxConcurrency = 4 } = options; - - return new Transform({ - objectMode: true, - async transform(chunk: T, encoding, callback) { - try { - if (parallel) { - const semaphore = new Semaphore(maxConcurrency); - await semaphore.acquire(); - try { - const result = await transformer(chunk); - callback(null, result); - } finally { - semaphore.release(); - } - } else { - const result = await transformer(chunk); - callback(null, result); - } - } catch (error) { - callback(error as Error); - } - } - }); - } - - /** - * Process data in batches with memory management - */ - async processInBatches( - data: T[], - batchProcessor: (batch: T[]) => Promise, - batchSize = 100, - progressCallback?: (progress: { processed: number; total: number; percentage: number }) => void - ): Promise { - const results: R[] = []; - const total = data.length; - - for (let i = 0; i < data.length; i += batchSize) { - const batch = data.slice(i, i + batchSize); - - try { - const batchResults = await this.errorRecovery.executeWithRetry( - () => batchProcessor(batch), - `processBatch:${i}` - ); - - results.push(...batchResults); - - if (progressCallback) { - progressCallback({ - processed: Math.min(i + batchSize, total), - total, - percentage: ((Math.min(i + batchSize, total) / total) * 100) - }); - } - - // Memory management every few batches - if (i % (batchSize * 10) === 0) { - await this.memoryManager.forceCleanup(); - } - } catch (error) { - console.error(`Failed to process batch ${i}-${i + batchSize}:`, error); - // Continue processing other batches - } - } - - return results; - } - - /** - * Get file size in bytes - */ - private async getFileSize(filePath: string): Promise { - const fs = await import('fs/promises'); - const stats = await fs.stat(filePath); - return stats.size; - } - - /** - * Create a memory-efficient pipeline - */ - async createPipeline( - inputStream: NodeJS.ReadableStream, - transforms: Transform[], - outputStream: NodeJS.WritableStream - ): Promise { - const streamArray: (NodeJS.ReadableStream | Transform | NodeJS.WritableStream)[] = [inputStream, ...transforms, outputStream]; - await pipeline(streamArray); - } -} - -// Semaphore for controlling concurrency -class Semaphore { - private permits: number; - private readonly queue: Array<() => void> = []; - - constructor(permits: number) { - this.permits = permits; - } - - async acquire(): Promise { - if (this.permits > 0) { - this.permits--; - return; - } - - return new Promise(resolve => { - this.queue.push(resolve); - }); - } - - release(): void { - this.permits++; - const resolve = this.queue.shift(); - if (resolve) { - resolve(); - } - } -} - -// Caching utilities -export class ProcessingCache { - private cache = new Map(); - private readonly maxSize: number; - private readonly ttl: number; - - constructor(maxSize = 100, ttl = 300000) { // 100 items, 5 minutes TTL - this.maxSize = maxSize; - this.ttl = ttl; - } - - get(key: string): T | null { - const item = this.cache.get(key); - if (!item) return null; - - if (Date.now() - item.timestamp > this.ttl) { - this.cache.delete(key); - return null; - } - - return item.data; - } - - set(key: string, data: T, size = 1): void { - if (this.cache.size >= this.maxSize) { - this.evictOldest(); - } - - this.cache.set(key, { - data, - timestamp: Date.now(), - size - }); - } - - private evictOldest(): void { - let oldestKey: string | null = null; - let oldestTime = Infinity; - - for (const [key, item] of this.cache.entries()) { - if (item.timestamp < oldestTime) { - oldestTime = item.timestamp; - oldestKey = key; - } - } - - if (oldestKey) { - this.cache.delete(oldestKey); - } - } - - clear(): void { - this.cache.clear(); - } - - size(): number { - return this.cache.size; - } -} - -================ -File: src/lib/utils.ts -================ -export function generateId(type: string, identifier: string): string { - // Use cryptographically secure UUID v4 - const uuid = crypto.randomUUID(); - return `${type}_${uuid}`; -} - -/** - * Legacy hash function - deprecated, use generateId instead - * @deprecated Use generateId with UUID - */ -function simpleHash(str: string): string { - let hash = 0; - for (let i = 0; i < str.length; i++) { - const char = str.charCodeAt(i); - hash = ((hash << 5) - hash) + char; - hash = hash & hash; - } - return Math.abs(hash).toString(36); -} - -================ -File: src/lib/validation.ts -================ -/** - * Type validation utilities for ensuring data integrity - */ - -import { z } from 'zod'; -import type { GraphNode, GraphRelationship, NodeProperties, RelationshipProperties } from '../core/graph/types'; - -// Validation schemas for nodes and relationships -const NodePropertiesSchema = z.object({ - name: z.string().optional(), - path: z.string().optional(), - filePath: z.string().optional(), - extension: z.string().optional(), - language: z.string().optional(), - size: z.number().nonnegative().optional(), - description: z.string().optional(), - version: z.string().optional(), - definitionCount: z.number().nonnegative().optional(), - lineCount: z.number().nonnegative().optional(), - type: z.string().optional(), - startLine: z.number().positive().optional(), - endLine: z.number().positive().optional(), - qualifiedName: z.string().optional(), - parameters: z.array(z.string()).optional(), - returnType: z.string().optional(), - relationshipType: z.string().optional() -}).catchall(z.union([z.string(), z.number(), z.boolean(), z.array(z.string()), z.undefined()])); - -const RelationshipPropertiesSchema = z.object({ - strength: z.number().min(0).max(1).optional(), - confidence: z.number().min(0).max(1).optional(), - importType: z.enum(['default', 'named', 'namespace']).optional(), - alias: z.string().optional(), - callType: z.enum(['function', 'method', 'constructor']).optional(), - arguments: z.array(z.string()).optional(), - dependencyType: z.enum(['direct', 'transitive', 'dev']).optional(), - version: z.string().optional() -}).catchall(z.union([z.string(), z.number(), z.boolean(), z.array(z.string()), z.undefined()])); - -const GraphNodeSchema = z.object({ - id: z.string().min(1), - label: z.enum([ - 'Project', 'Package', 'Module', 'Folder', 'File', 'Class', 'Function', - 'Method', 'Variable', 'Interface', 'Enum', 'Decorator' - ]), - properties: NodePropertiesSchema -}); - -const GraphRelationshipSchema = z.object({ - id: z.string().min(1), - type: z.enum([ - 'CONTAINS', 'CALLS', 'INHERITS', 'OVERRIDES', 'IMPORTS', 'USES', - 'DEFINES', 'DECORATES', 'IMPLEMENTS', 'ACCESSES' - ]), - source: z.string().min(1), - target: z.string().min(1), - properties: RelationshipPropertiesSchema -}); - -/** - * Validation utilities for graph data - */ -export class ValidationService { - /** - * Validate a node object - */ - static validateNode(node: unknown): GraphNode { - try { - return GraphNodeSchema.parse(node); - } catch (error) { - throw new Error(`Invalid node: ${error}`); - } - } - - /** - * Validate a relationship object - */ - static validateRelationship(relationship: unknown): GraphRelationship { - try { - return GraphRelationshipSchema.parse(relationship); - } catch (error) { - throw new Error(`Invalid relationship: ${error}`); - } - } - - /** - * Validate node properties - */ - static validateNodeProperties(properties: unknown): NodeProperties { - try { - return NodePropertiesSchema.parse(properties); - } catch (error) { - throw new Error(`Invalid node properties: ${error}`); - } - } - - /** - * Validate relationship properties - */ - static validateRelationshipProperties(properties: unknown): RelationshipProperties { - try { - return RelationshipPropertiesSchema.parse(properties); - } catch (error) { - throw new Error(`Invalid relationship properties: ${error}`); - } - } - - /** - * Validate a complete graph - */ - static validateGraph(graph: { nodes: unknown[]; relationships: unknown[] }): void { - const errors: string[] = []; - - // Validate nodes - const nodeIds = new Set(); - graph.nodes.forEach((node, index) => { - try { - const validated = this.validateNode(node); - if (nodeIds.has(validated.id)) { - errors.push(`Duplicate node ID: ${validated.id} at index ${index}`); - } - nodeIds.add(validated.id); - } catch (error) { - errors.push(`Node at index ${index}: ${error}`); - } - }); - - // Validate relationships - const relationshipIds = new Set(); - graph.relationships.forEach((relationship, index) => { - try { - const validated = this.validateRelationship(relationship); - if (relationshipIds.has(validated.id)) { - errors.push(`Duplicate relationship ID: ${validated.id} at index ${index}`); - } - relationshipIds.add(validated.id); - - // Check if referenced nodes exist - if (!nodeIds.has(validated.source)) { - errors.push(`Relationship at index ${index} references non-existent source: ${validated.source}`); - } - if (!nodeIds.has(validated.target)) { - errors.push(`Relationship at index ${index} references non-existent target: ${validated.target}`); - } - } catch (error) { - errors.push(`Relationship at index ${index}: ${error}`); - } - }); - - if (errors.length > 0) { - throw new Error(`Graph validation failed:\n${errors.join('\n')}`); - } - } - - /** - * Sanitize string inputs - */ - static sanitizeString(input: string): string { - return input - .trim() - .replace(/[<>]/g, '') // Remove potential XSS vectors - .substring(0, 1000); // Limit length - } - - /** - * Validate file path - */ - static validateFilePath(path: string): string { - if (!path || typeof path !== 'string') { - throw new Error('Invalid file path'); - } - - const sanitized = path.trim(); - if (sanitized.length === 0 || sanitized.length > 500) { - throw new Error('File path too long or empty'); - } - - // Check for directory traversal - if (sanitized.includes('..') || sanitized.includes('~')) { - throw new Error('Invalid file path: contains directory traversal'); - } - - return sanitized; - } - - /** - * Validate language identifier - */ - static validateLanguage(language: string): string { - const validLanguages = [ - 'javascript', 'typescript', 'python', 'java', 'cpp', 'c', 'csharp', - 'php', 'ruby', 'go', 'rust', 'swift', 'kotlin', 'scala', 'dart' - ]; - - const normalized = language.toLowerCase(); - if (!validLanguages.includes(normalized)) { - throw new Error(`Invalid language: ${language}`); - } - - return normalized; - } -} - -================ -File: src/lib/web-worker-pool.ts -================ -/** - * Web Worker Pool Manager for parallel processing in browsers - * Manages a pool of Web Workers to process tasks concurrently - */ - -export interface WorkerTask { - id: string; - input: TInput; - resolve: (result: TOutput) => void; - reject: (error: Error) => void; -} - -export interface WorkerPoolOptions { - maxWorkers?: number; - workerScript?: string; - timeout?: number; - name?: string; -} - -export interface WorkerPoolStats { - totalWorkers: number; - availableWorkers: number; - activeTasks: number; - queuedTasks: number; - maxWorkers: number; - memoryUsage?: number; -} - -export class WebWorkerPool { - private workers: Worker[] = []; - private availableWorkers: Worker[] = []; - private taskQueue: WorkerTask[] = []; - private activeTasks: Map> = new Map(); - private maxWorkers: number; - private workerScript: string; - private timeout: number; - private isShuttingDown: boolean = false; - private name: string; - private eventListeners: Map void>> = new Map(); - - constructor(options: WorkerPoolOptions = {}) { - this.maxWorkers = options.maxWorkers || Math.max(2, Math.min(8, navigator.hardwareConcurrency || 4)); - this.workerScript = options.workerScript || '/workers/generic-worker.js'; - this.timeout = options.timeout || 30000; // 30 seconds - this.name = options.name || 'WebWorkerPool'; - } - - /** - * Execute a task using available worker - */ - async execute(input: TInput): Promise { - if (this.isShuttingDown) { - throw new Error('Worker pool is shutting down'); - } - - return new Promise((resolve, reject) => { - const task: WorkerTask = { - id: this.generateTaskId(), - input, - resolve, - reject - }; - - this.taskQueue.push(task as WorkerTask); - this.processQueue(); - }); - } - - /** - * Execute multiple tasks in parallel - */ - async executeAll(inputs: TInput[]): Promise { - const promises = inputs.map(input => this.execute(input)); - return Promise.all(promises); - } - - /** - * Execute tasks with concurrency limit - */ - async executeBatch( - inputs: TInput[], - batchSize: number = this.maxWorkers - ): Promise { - const results: TOutput[] = []; - for (let i = 0; i < inputs.length; i += batchSize) { - const batch = inputs.slice(i, i + batchSize); - const batchResults = await this.executeAll(batch); - results.push(...batchResults); - } - return results; - } - - /** - * Execute tasks with progress callback - */ - async executeWithProgress( - inputs: TInput[], - onProgress?: (completed: number, total: number) => void - ): Promise { - const results: TOutput[] = []; - const total = inputs.length; - let completed = 0; - - const batchSize = Math.min(this.maxWorkers, 10); - - for (let i = 0; i < inputs.length; i += batchSize) { - const batch = inputs.slice(i, i + batchSize); - const batchResults = await this.executeAll(batch); - - results.push(...batchResults); - completed += batch.length; - - if (onProgress) { - onProgress(completed, total); - } - } - - return results; - } - - /** - * Get pool statistics - */ - getStats(): WorkerPoolStats { - return { - totalWorkers: this.workers.length, - availableWorkers: this.availableWorkers.length, - activeTasks: this.activeTasks.size, - queuedTasks: this.taskQueue.length, - maxWorkers: this.maxWorkers, - memoryUsage: undefined - }; - } - - /** - * Shut down the worker pool - */ - async shutdown(): Promise { - this.isShuttingDown = true; - - console.log(`${this.name}: Shutting down worker pool...`); - - // Reject all queued tasks - for (const task of this.taskQueue) { - task.reject(new Error('Worker pool is shutting down')); - } - this.taskQueue.length = 0; - - // Wait for active tasks to complete or timeout - const activeTaskPromises = Array.from(this.activeTasks.values()).map(task => - new Promise((resolve) => { - const originalResolve = task.resolve; - const originalReject = task.reject; - - task.resolve = (result) => { - originalResolve(result); - resolve(); - }; - - task.reject = (error) => { - originalReject(error); - resolve(); - }; - }) - ); - - // Terminate all workers - const terminatePromises = this.workers.map(worker => { - try { - worker.terminate(); - return Promise.resolve(); - } catch (error) { - console.warn(`${this.name}: Error terminating worker:`, error); - return Promise.resolve(); - } - }); - - try { - await Promise.race([ - Promise.all(activeTaskPromises), - new Promise(resolve => setTimeout(resolve, 5000)) // 5 second timeout - ]); - } catch { - // Ignore timeout errors during shutdown - } - - await Promise.all(terminatePromises); - - this.workers.length = 0; - this.availableWorkers.length = 0; - this.activeTasks.clear(); - - this.emit('shutdown'); - console.log(`${this.name}: Worker pool shutdown complete`); - } - - /** - * Add event listener - */ - on(event: string, listener: (data: unknown) => void): void { - if (!this.eventListeners.has(event)) { - this.eventListeners.set(event, new Set()); - } - this.eventListeners.get(event)!.add(listener); - } - - /** - * Remove event listener - */ - off(event: string, listener: (data: unknown) => void): void { - const listeners = this.eventListeners.get(event); - if (listeners) { - listeners.delete(listener); - } - } - - /** - * Emit event - */ - private emit(event: string, data?: unknown): void { - const listeners = this.eventListeners.get(event); - if (listeners) { - for (const listener of listeners) { - try { - listener(data); - } catch (error) { - console.error(`${this.name}: Error in event listener for ${event}:`, error); - } - } - } - } - - private processQueue(): void { - while (this.taskQueue.length > 0 && this.getAvailableWorker()) { - const task = this.taskQueue.shift()!; - const worker = this.getAvailableWorker()!; - - this.assignTaskToWorker(task, worker); - } - } - - private getAvailableWorker(): Worker | null { - if (this.availableWorkers.length > 0) { - return this.availableWorkers.pop()!; - } - - if (this.workers.length < this.maxWorkers) { - return this.createWorker(); - } - - return null; - } - - private createWorker(): Worker { - try { - const worker = new Worker(this.workerScript, { type: 'module' }); - - worker.onerror = (error) => { - this.handleWorkerError(worker, error); - }; - - worker.onmessageerror = (error) => { - console.error(`${this.name}: Worker message error:`, error); - this.handleWorkerError(worker, new Error('Worker message error')); - }; - - this.workers.push(worker); - this.emit('workerCreated', { - workerId: this.workers.length - 1, - totalWorkers: this.workers.length - }); - - return worker; - } catch (error) { - console.error(`${this.name}: Failed to create worker:`, error); - throw new Error(`Failed to create worker: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - private assignTaskToWorker(task: WorkerTask, worker: Worker): void { - this.activeTasks.set(task.id, task); - - const timeoutId = setTimeout(() => { - task.reject(new Error(`Task ${task.id} timed out after ${this.timeout}ms`)); - this.activeTasks.delete(task.id); - this.recycleWorker(worker); - }, this.timeout); - - const messageHandler = (event: MessageEvent) => { - const { taskId, result, error } = event.data; - - if (taskId !== task.id) { - return; // Not our task - } - - clearTimeout(timeoutId); - worker.removeEventListener('message', messageHandler); - worker.removeEventListener('error', errorHandler); - - this.activeTasks.delete(task.id); - - if (error) { - task.reject(new Error(error)); - } else { - task.resolve(result); - } - - this.recycleWorker(worker); - }; - - const errorHandler = (error: ErrorEvent) => { - clearTimeout(timeoutId); - worker.removeEventListener('message', messageHandler); - worker.removeEventListener('error', errorHandler); - - this.activeTasks.delete(task.id); - task.reject(new Error(`Worker error: ${error.message}`)); - this.handleWorkerError(worker, error); - }; - - worker.addEventListener('message', messageHandler); - worker.addEventListener('error', errorHandler); - - // Send task to worker - worker.postMessage({ - taskId: task.id, - input: task.input - }); - } - - private recycleWorker(worker: Worker): void { - if (!this.isShuttingDown && this.workers.includes(worker)) { - this.availableWorkers.push(worker); - this.processQueue(); - } - } - - private handleWorkerError(worker: Worker, error: Error | ErrorEvent): void { - const errorMessage = error instanceof ErrorEvent ? error.message : error.message; - this.emit('workerError', { - workerId: this.workers.indexOf(worker), - error: errorMessage - }); - - // Remove worker from pools - const workerIndex = this.workers.indexOf(worker); - if (workerIndex !== -1) { - this.workers.splice(workerIndex, 1); - } - - const availableIndex = this.availableWorkers.indexOf(worker); - if (availableIndex !== -1) { - this.availableWorkers.splice(availableIndex, 1); - } - - // Try to replace the worker if not shutting down - if (!this.isShuttingDown && this.workers.length < this.maxWorkers) { - this.processQueue(); - } - } - - private generateTaskId(): string { - return `task_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } -} - -/** - * Specialized worker pool for file processing - */ -export class FileProcessingPool extends WebWorkerPool { - private static instance: FileProcessingPool; - - static getInstance(): FileProcessingPool { - if (!FileProcessingPool.instance) { - FileProcessingPool.instance = new FileProcessingPool({ - maxWorkers: Math.max(2, Math.min(6, navigator.hardwareConcurrency || 4)), - workerScript: '/workers/file-processing-worker.js', - timeout: 45000, // 45 seconds for file processing - name: 'FileProcessingPool' - }); - } - return FileProcessingPool.instance; - } - - /** - * Process files in parallel - */ - async processFiles( - filePaths: string[], - processor: (filePath: string) => Promise - ): Promise { - const processingTasks = filePaths.map(filePath => ({ - filePath, - processorFunction: processor.toString() - })); - - return this.executeAll(processingTasks); - } - - /** - * Process files with progress callback - */ - async processFilesWithProgress( - filePaths: string[], - processor: (filePath: string) => Promise, - onProgress?: (completed: number, total: number) => void - ): Promise { - return this.executeWithProgress( - filePaths.map(filePath => ({ filePath, processorFunction: processor.toString() })), - onProgress - ); - } -} - -/** - * Worker pool utilities - */ -export const WebWorkerPoolUtils = { - /** - * Create a specialized worker pool for CPU-intensive tasks - */ - createCPUPool(options: Partial = {}): WebWorkerPool { - return new WebWorkerPool({ - maxWorkers: navigator.hardwareConcurrency || 4, - timeout: 60000, // 1 minute - name: 'CPUPool', - ...options - }); - }, - - /** - * Create a worker pool for I/O operations - */ - createIOPool(options: Partial = {}): WebWorkerPool { - return new WebWorkerPool({ - maxWorkers: Math.min(20, (navigator.hardwareConcurrency || 4) * 4), // More workers for I/O - timeout: 30000, // 30 seconds - name: 'IOPool', - ...options - }); - }, - - /** - * Get optimal worker count for different task types - */ - getOptimalWorkerCount(taskType: 'cpu' | 'io' | 'mixed' = 'mixed'): number { - const cpuCount = navigator.hardwareConcurrency || 4; - - switch (taskType) { - case 'cpu': - return cpuCount; - case 'io': - return Math.min(20, cpuCount * 4); - case 'mixed': - default: - return Math.max(2, Math.min(8, cpuCount)); - } - }, - - /** - * Check if Web Workers are supported - */ - isSupported(): boolean { - return typeof Worker !== 'undefined'; - }, - - /** - * Get hardware concurrency - */ - getHardwareConcurrency(): number { - return navigator.hardwareConcurrency || 4; - } -}; - -================ -File: src/lib/worker-pool-test.ts -================ -/** - * Worker Pool Test Suite - * Tests the Web Worker Pool implementation - */ - -import { WebWorkerPool, WebWorkerPoolUtils } from './web-worker-pool.js'; - -/** - * Test basic worker pool functionality - */ -export async function testWorkerPoolBasic(): Promise { - console.log('๐Ÿงช Testing Basic Worker Pool Functionality...'); - - const workerPool = new WebWorkerPool({ - maxWorkers: 2, - workerScript: '/workers/generic-worker.js', - name: 'TestPool' - }); - - try { - // Test simple text analysis - const textAnalysisTask = { - taskType: 'textAnalysis', - text: 'Hello world! This is a test.', - analysisType: 'wordCount' - }; - - const result = await workerPool.execute(textAnalysisTask); - console.log('โœ… Text analysis result:', result); - - // Test multiple tasks - const tasks = [ - { taskType: 'textAnalysis', text: 'First test', analysisType: 'wordCount' }, - { taskType: 'textAnalysis', text: 'Second test', analysisType: 'wordCount' }, - { taskType: 'textAnalysis', text: 'Third test', analysisType: 'wordCount' } - ]; - - const results = await workerPool.executeAll(tasks); - console.log('โœ… Multiple tasks results:', results); - - // Test batch processing - const batchResults = await workerPool.executeBatch(tasks, 2); - console.log('โœ… Batch processing results:', batchResults); - - // Get statistics - const stats = workerPool.getStats(); - console.log('โœ… Worker pool stats:', stats); - - await workerPool.shutdown(); - console.log('โœ… Basic worker pool test completed successfully'); - - } catch (error) { - console.error('โŒ Basic worker pool test failed:', error); - throw error; - } -} - -/** - * Test file processing worker pool - */ -export async function testFileProcessingPool(): Promise { - console.log('๐Ÿงช Testing File Processing Worker Pool...'); - - const filePool = WebWorkerPoolUtils.createCPUPool({ - workerScript: '/workers/file-processing-worker.js', - name: 'FileProcessingTest' - }); - - try { - // Test file analysis - const fileAnalysisTask = { - processorType: 'analyzeStructure', - filePath: '/src/test.ts', - content: ` - import { Component } from 'react'; - - export class TestComponent extends Component { - render() { - return
Hello World
; - } - } - ` - }; - - const result = await filePool.execute(fileAnalysisTask); - console.log('โœ… File analysis result:', result); - - // Test dependency extraction - const dependencyTask = { - processorType: 'extractDependencies', - filePath: '/src/test.ts', - content: ` - import React from 'react'; - import { useState } from 'react'; - import './styles.css'; - ` - }; - - const dependencyResult = await filePool.execute(dependencyTask); - console.log('โœ… Dependency extraction result:', dependencyResult); - - await filePool.shutdown(); - console.log('โœ… File processing worker pool test completed successfully'); - - } catch (error) { - console.error('โŒ File processing worker pool test failed:', error); - throw error; - } -} - -/** - * Test performance with multiple workers - */ -export async function testWorkerPoolPerformance(): Promise { - console.log('๐Ÿงช Testing Worker Pool Performance...'); - - const workerCount = WebWorkerPoolUtils.getOptimalWorkerCount('cpu'); - console.log(`Using ${workerCount} workers for performance test`); - - const workerPool = new WebWorkerPool({ - maxWorkers: workerCount, - workerScript: '/workers/generic-worker.js', - name: 'PerformanceTest' - }); - - try { - // Create many tasks - const tasks = Array.from({ length: 100 }, (_, i) => ({ - taskType: 'textAnalysis', - text: `This is test text number ${i} for performance testing.`, - analysisType: 'wordCount' - })); - - const startTime = performance.now(); - - const results = await workerPool.executeWithProgress( - tasks, - (completed, total) => { - const progress = ((completed / total) * 100).toFixed(1); - console.log(`Progress: ${progress}% (${completed}/${total})`); - } - ); - - const endTime = performance.now(); - const duration = endTime - startTime; - const avgTimePerTask = duration / tasks.length; - - console.log(`โœ… Performance test completed:`); - console.log(` Total time: ${duration.toFixed(2)}ms`); - console.log(` Tasks processed: ${results.length}`); - console.log(` Average time per task: ${avgTimePerTask.toFixed(2)}ms`); - console.log(` Processing rate: ${(tasks.length / (duration / 1000)).toFixed(2)} tasks/second`); - - // Get final statistics - const stats = workerPool.getStats(); - console.log('โœ… Final worker pool stats:', stats); - - await workerPool.shutdown(); - console.log('โœ… Performance test completed successfully'); - - } catch (error) { - console.error('โŒ Performance test failed:', error); - throw error; - } -} - -/** - * Test error handling - */ -export async function testWorkerPoolErrorHandling(): Promise { - console.log('๐Ÿงช Testing Worker Pool Error Handling...'); - - const workerPool = new WebWorkerPool({ - maxWorkers: 2, - workerScript: '/workers/generic-worker.js', - name: 'ErrorTest' - }); - - try { - // Test invalid task type - const invalidTask = { - taskType: 'invalidTaskType', - data: 'test' - }; - - try { - await workerPool.execute(invalidTask); - console.log('โŒ Expected error was not thrown'); - } catch (error) { - console.log('โœ… Invalid task type error handled correctly:', error.message); - } - - // Test timeout - const timeoutTask = { - taskType: 'textAnalysis', - text: 'This should timeout', - analysisType: 'wordCount' - }; - - // Set a very short timeout for testing - const timeoutWorkerPool = new WebWorkerPool({ - maxWorkers: 1, - workerScript: '/workers/generic-worker.js', - timeout: 100, // 100ms timeout - name: 'TimeoutTest' - }); - - try { - await timeoutWorkerPool.execute(timeoutTask); - console.log('โŒ Expected timeout was not triggered'); - } catch (error) { - console.log('โœ… Timeout error handled correctly:', error.message); - } - - await timeoutWorkerPool.shutdown(); - await workerPool.shutdown(); - console.log('โœ… Error handling test completed successfully'); - - } catch (error) { - console.error('โŒ Error handling test failed:', error); - throw error; - } -} - -/** - * Test worker pool utilities - */ -export function testWorkerPoolUtils(): void { - console.log('๐Ÿงช Testing Worker Pool Utilities...'); - - // Test hardware concurrency - const concurrency = WebWorkerPoolUtils.getHardwareConcurrency(); - console.log(`โœ… Hardware concurrency: ${concurrency}`); - - // Test optimal worker counts - const cpuCount = WebWorkerPoolUtils.getOptimalWorkerCount('cpu'); - const ioCount = WebWorkerPoolUtils.getOptimalWorkerCount('io'); - const mixedCount = WebWorkerPoolUtils.getOptimalWorkerCount('mixed'); - - console.log(`โœ… Optimal worker counts:`); - console.log(` CPU: ${cpuCount}`); - console.log(` I/O: ${ioCount}`); - console.log(` Mixed: ${mixedCount}`); - - // Test Web Worker support - const isSupported = WebWorkerPoolUtils.isSupported(); - console.log(`โœ… Web Workers supported: ${isSupported}`); - - console.log('โœ… Worker pool utilities test completed successfully'); -} - -/** - * Run all worker pool tests - */ -export async function runWorkerPoolTests(): Promise { - console.log('๐Ÿš€ Starting Worker Pool Test Suite...'); - - try { - // Test utilities first - testWorkerPoolUtils(); - - // Test basic functionality - await testWorkerPoolBasic(); - - // Test file processing - await testFileProcessingPool(); - - // Test performance - await testWorkerPoolPerformance(); - - // Test error handling - await testWorkerPoolErrorHandling(); - - console.log('๐ŸŽ‰ All worker pool tests completed successfully!'); - - } catch (error) { - console.error('โŒ Worker pool test suite failed:', error); - throw error; - } -} - -// Make functions available globally for browser testing -if (typeof window !== 'undefined') { - (window as any).testWorkerPoolBasic = testWorkerPoolBasic; - (window as any).testFileProcessingPool = testFileProcessingPool; - (window as any).testWorkerPoolPerformance = testWorkerPoolPerformance; - (window as any).testWorkerPoolErrorHandling = testWorkerPoolErrorHandling; - (window as any).testWorkerPoolUtils = testWorkerPoolUtils; - (window as any).runWorkerPoolTests = runWorkerPoolTests; -} - -================ -File: src/lib/worker-pool.ts -================ -/** - * Worker Pool Manager for parallel processing - * Manages a pool of workers to process tasks concurrently - */ - -import { Worker } from 'worker_threads'; -import { EventEmitter } from 'events'; -import * as path from 'path'; -import * as os from 'os'; - -export interface WorkerTask { - id: string; - input: TInput; - resolve: (result: TOutput) => void; - reject: (error: Error) => void; -} - -export interface WorkerPoolOptions { - maxWorkers?: number; - workerScript?: string; - timeout?: number; -} - -export class WorkerPool extends EventEmitter { - private workers: Worker[] = []; - private availableWorkers: Worker[] = []; - private taskQueue: WorkerTask[] = []; - private activeTasks: Map> = new Map(); - protected maxWorkers: number; - private workerScript: string; - private timeout: number; - private isShuttingDown: boolean = false; - - constructor(options: WorkerPoolOptions = {}) { - super(); - this.maxWorkers = options.maxWorkers || Math.max(2, Math.min(8, os.cpus().length)); - this.workerScript = options.workerScript || path.join(__dirname, 'worker-scripts', 'generic-worker.js'); - this.timeout = options.timeout || 30000; // 30 seconds - } - - /** - * Execute a task using available worker - */ - async execute(input: TInput): Promise { - if (this.isShuttingDown) { - throw new Error('Worker pool is shutting down'); - } - - return new Promise((resolve, reject) => { - const task: WorkerTask = { - id: this.generateTaskId(), - input, - resolve, - reject - }; - - this.taskQueue.push(task as WorkerTask); - this.processQueue(); - }); - } - - /** - * Execute multiple tasks in parallel - */ - async executeAll(inputs: TInput[]): Promise { - const promises = inputs.map(input => this.execute(input)); - return Promise.all(promises); - } - - /** - * Execute tasks with concurrency limit - */ - async executeBatch( - inputs: TInput[], - batchSize: number = this.maxWorkers - ): Promise { - const results: TOutput[] = []; - for (let i = 0; i < inputs.length; i += batchSize) { - const batch = inputs.slice(i, i + batchSize); - const batchResults = await this.executeAll(batch); - results.push(...batchResults); - } - return results; - } - - /** - * Get pool statistics - */ - getStats() { - return { - totalWorkers: this.workers.length, - availableWorkers: this.availableWorkers.length, - activeTasks: this.activeTasks.size, - queuedTasks: this.taskQueue.length, - maxWorkers: this.maxWorkers - }; - } - - /** - * Shut down the worker pool - */ - async shutdown(): Promise { - this.isShuttingDown = true; - - // Reject all queued tasks - for (const task of this.taskQueue) { - task.reject(new Error('Worker pool is shutting down')); - } - this.taskQueue.length = 0; - - // Wait for active tasks to complete or timeout - const activeTaskPromises = Array.from(this.activeTasks.values()).map(task => - new Promise((resolve) => { - const originalResolve = task.resolve; - const originalReject = task.reject; - - task.resolve = (result) => { - originalResolve(result); - resolve(); - }; - - task.reject = (error) => { - originalReject(error); - resolve(); - }; - }) - ); - - // Terminate all workers - const terminatePromises = this.workers.map(worker => worker.terminate()); - - try { - await Promise.race([ - Promise.all(activeTaskPromises), - new Promise(resolve => setTimeout(resolve, 5000)) // 5 second timeout - ]); - } catch { - // Ignore timeout errors during shutdown - } - - await Promise.all(terminatePromises); - - this.workers.length = 0; - this.availableWorkers.length = 0; - this.activeTasks.clear(); - - this.emit('shutdown'); - } - - private processQueue(): void { - while (this.taskQueue.length > 0 && this.getAvailableWorker()) { - const task = this.taskQueue.shift()!; - const worker = this.getAvailableWorker()!; - - this.assignTaskToWorker(task, worker); - } - } - - private getAvailableWorker(): Worker | null { - if (this.availableWorkers.length > 0) { - return this.availableWorkers.pop()!; - } - - if (this.workers.length < this.maxWorkers) { - return this.createWorker(); - } - - return null; - } - - private createWorker(): Worker { - const worker = new Worker(this.workerScript); - - worker.on('error', (error) => { - this.handleWorkerError(worker, error); - }); - - worker.on('exit', (code) => { - this.handleWorkerExit(worker, code); - }); - - this.workers.push(worker); - this.emit('workerCreated', { workerId: worker.threadId, totalWorkers: this.workers.length }); - - return worker; - } - - private assignTaskToWorker(task: WorkerTask, worker: Worker): void { - this.activeTasks.set(task.id, task); - - const timeoutId = setTimeout(() => { - task.reject(new Error(`Task ${task.id} timed out after ${this.timeout}ms`)); - this.activeTasks.delete(task.id); - this.recycleWorker(worker); - }, this.timeout); - - const messageHandler = (result: unknown) => { - clearTimeout(timeoutId); - worker.off('message', messageHandler); - worker.off('error', errorHandler); - - this.activeTasks.delete(task.id); - task.resolve(result); - this.recycleWorker(worker); - }; - - const errorHandler = (error: Error) => { - clearTimeout(timeoutId); - worker.off('message', messageHandler); - worker.off('error', errorHandler); - - this.activeTasks.delete(task.id); - task.reject(error); - this.handleWorkerError(worker, error); - }; - - worker.on('message', messageHandler); - worker.on('error', errorHandler); - worker.postMessage({ taskId: task.id, input: task.input }); - } - - private recycleWorker(worker: Worker): void { - if (!this.isShuttingDown && this.workers.includes(worker)) { - this.availableWorkers.push(worker); - this.processQueue(); - } - } - - private handleWorkerError(worker: Worker, error: Error): void { - this.emit('workerError', { workerId: worker.threadId, error }); - - // Remove worker from pools - const workerIndex = this.workers.indexOf(worker); - if (workerIndex !== -1) { - this.workers.splice(workerIndex, 1); - } - - const availableIndex = this.availableWorkers.indexOf(worker); - if (availableIndex !== -1) { - this.availableWorkers.splice(availableIndex, 1); - } - - // Try to replace the worker if not shutting down - if (!this.isShuttingDown && this.workers.length < this.maxWorkers) { - this.processQueue(); - } - } - - private handleWorkerExit(worker: Worker, code: number): void { - this.emit('workerExit', { workerId: worker.threadId, exitCode: code }); - - // Remove worker from pools - const workerIndex = this.workers.indexOf(worker); - if (workerIndex !== -1) { - this.workers.splice(workerIndex, 1); - } - - const availableIndex = this.availableWorkers.indexOf(worker); - if (availableIndex !== -1) { - this.availableWorkers.splice(availableIndex, 1); - } - } - - private generateTaskId(): string { - return `task_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - } -} - -/** - * Singleton worker pool for file processing - */ -export class FileProcessingPool extends WorkerPool { - private static instance: FileProcessingPool; - - static getInstance(): FileProcessingPool { - if (!FileProcessingPool.instance) { - FileProcessingPool.instance = new FileProcessingPool({ - maxWorkers: Math.max(2, Math.min(6, os.cpus().length - 1)), - workerScript: path.join(__dirname, 'worker-scripts', 'file-processing-worker.js'), - timeout: 45000 // 45 seconds for file processing - }); - } - return FileProcessingPool.instance; - } - - /** - * Process files in parallel - */ - async processFiles( - filePaths: string[], - processor: (filePath: string) => Promise - ): Promise { - const processingTasks = filePaths.map(filePath => ({ - filePath, - processorFunction: processor.toString() - })); - - return this.executeAll(processingTasks); - } - - /** - * Process files with progress callback - */ - async processFilesWithProgress( - filePaths: string[], - processor: (filePath: string) => Promise, - onProgress?: (completed: number, total: number) => void - ): Promise { - const results: TOutput[] = []; - const total = filePaths.length; - let completed = 0; - - const batchSize = Math.min(this.maxWorkers, 10); - - for (let i = 0; i < filePaths.length; i += batchSize) { - const batch = filePaths.slice(i, i + batchSize); - const batchResults = await this.processFiles(batch, processor); - - results.push(...batchResults); - completed += batch.length; - - if (onProgress) { - onProgress(completed, total); - } - } - - return results; - } -} - -/** - * Worker pool utilities - */ -export const WorkerPoolUtils = { - /** - * Create a specialized worker pool for CPU-intensive tasks - */ - createCPUPool(options: Partial = {}): WorkerPool { - return new WorkerPool({ - maxWorkers: os.cpus().length, - timeout: 60000, // 1 minute - ...options - }); - }, - - /** - * Create a worker pool for I/O operations - */ - createIOPool(options: Partial = {}): WorkerPool { - return new WorkerPool({ - maxWorkers: Math.min(20, os.cpus().length * 4), // More workers for I/O - timeout: 30000, // 30 seconds - ...options - }); - }, - - /** - * Get optimal worker count for different task types - */ - getOptimalWorkerCount(taskType: 'cpu' | 'io' | 'mixed' = 'mixed'): number { - const cpuCount = os.cpus().length; - - switch (taskType) { - case 'cpu': - return cpuCount; - case 'io': - return Math.min(20, cpuCount * 4); - case 'mixed': - default: - return Math.max(2, Math.min(8, cpuCount)); - } - } -}; - -================ -File: src/lib/workerUtils.ts -================ -// Worker utilities for handling different runtime environments -import * as Comlink from 'comlink'; -import { IngestionWorker } from '../workers/ingestion.worker.ts'; -import type { IngestionProgress, IngestionResult } from '../workers/ingestion.worker.ts'; -import type { PipelineInput } from '../core/ingestion/pipeline.ts'; - -// Export types for external use -export type { IngestionProgress, IngestionResult }; - -export interface WorkerProxy { - processRepository(input: PipelineInput): Promise; - processFiles(projectName: string, files: { path: string; content: string }[]): Promise; - validateRepository(input: PipelineInput): Promise<{ valid: boolean; errors: string[] }>; - getWorkerInfo(): Promise<{ version: string; capabilities: string[] }>; - setProgressCallback(callback: (progress: IngestionProgress) => void): Promise; - terminate(): Promise; -} - -export class IngestionWorkerManager { - private worker: Worker | null = null; - private workerProxy: WorkerProxy | null = null; - private isInitialized = false; - - public async initialize(): Promise { - if (this.isInitialized) { - return; - } - - try { - // Create the worker - this.worker = new Worker( - new URL('../workers/ingestion.worker.ts', import.meta.url).href, - { - type: 'module', - name: 'ingestion-worker' - } - ); - - // Wrap with Comlink - this.workerProxy = Comlink.wrap(this.worker) as WorkerProxy; - - this.isInitialized = true; - console.log('Ingestion worker initialized successfully'); - - } catch (error) { - throw new Error(`Failed to initialize ingestion worker: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - public async processRepository(input: PipelineInput): Promise { - await this.ensureInitialized(); - - try { - return await this.workerProxy!.processRepository(input); - } catch (error) { - throw new Error(`Worker processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - public async processFiles( - projectName: string, - files: { path: string; content: string }[] - ): Promise { - await this.ensureInitialized(); - - try { - return await this.workerProxy!.processFiles(projectName, files); - } catch (error) { - throw new Error(`Worker file processing failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - public async validateRepository(input: PipelineInput): Promise<{ valid: boolean; errors: string[] }> { - await this.ensureInitialized(); - - try { - return await this.workerProxy!.validateRepository(input); - } catch (error) { - return { - valid: false, - errors: [`Validation failed: ${error instanceof Error ? error.message : 'Unknown error'}`] - }; - } - } - - public async setProgressCallback(callback: (progress: IngestionProgress) => void): Promise { - await this.ensureInitialized(); - - try { - // Wrap callback with Comlink.proxy to allow it to be called from worker - const proxiedCallback = Comlink.proxy(callback); - await this.workerProxy!.setProgressCallback(proxiedCallback); - } catch (error) { - console.warn('Failed to set progress callback:', error); - } - } - - public async getWorkerInfo(): Promise<{ version: string; capabilities: string[] }> { - await this.ensureInitialized(); - - try { - return await this.workerProxy!.getWorkerInfo(); - } catch (error) { - throw new Error(`Failed to get worker info: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - public async terminate(): Promise { - if (!this.isInitialized || !this.worker) { - return; - } - - try { - // Notify worker to cleanup - if (this.workerProxy) { - await this.workerProxy.terminate(); - } - - // Terminate the worker - this.worker.terminate(); - - // Cleanup references - this.worker = null; - this.workerProxy = null; - this.isInitialized = false; - - console.log('Ingestion worker terminated'); - - } catch (error) { - console.warn('Error during worker termination:', error); - - // Force terminate if cleanup fails - if (this.worker) { - this.worker.terminate(); - this.worker = null; - this.workerProxy = null; - this.isInitialized = false; - } - } - } - - public isWorkerReady(): boolean { - return this.isInitialized && this.worker !== null && this.workerProxy !== null; - } - - private async ensureInitialized(): Promise { - if (!this.isInitialized) { - await this.initialize(); - } - } -} - -// Singleton instance for easy access -let workerManager: IngestionWorkerManager | null = null; - -export function getIngestionWorker(): IngestionWorkerManager { - if (!workerManager) { - workerManager = new IngestionWorkerManager(); - } - return workerManager; -} - -export async function createIngestionWorker(): Promise { - const manager = new IngestionWorkerManager(); - await manager.initialize(); - return manager; -} - -// Utility function for processing with automatic cleanup -export async function processWithWorker( - processor: (worker: IngestionWorkerManager) => Promise -): Promise { - const worker = await createIngestionWorker(); - - try { - return await processor(worker); - } finally { - await worker.terminate(); - } -} - -// Error handling utilities -export class WorkerError extends Error { - constructor(message: string, public readonly cause?: Error) { - super(message); - this.name = 'WorkerError'; - } -} - -export function isWorkerSupported(): boolean { - try { - return typeof Worker !== 'undefined'; - } catch { - return false; - } -} - -================ -File: src/main.tsx -================ -import './lib/polyfills.ts'; -import './lib/preload.ts'; -import { createRoot } from 'react-dom/client' -import App from './App.tsx' -import './index.css' - -createRoot(document.getElementById('root')!).render( - -) - -================ -File: src/services/github.ts -================ -import axios, { type AxiosInstance, type AxiosResponse } from 'axios'; - -interface GitHubFile { - name: string; - path: string; - sha: string; - size: number; - url: string; - html_url: string; - git_url: string; - download_url: string | null; - type: 'file' | 'dir'; - content?: string; - encoding?: string; -} - -interface GitHubDirectory { - name: string; - path: string; - sha: string; - size: number; - url: string; - html_url: string; - git_url: string; - download_url: string | null; - type: 'file' | 'dir'; -} - -interface RateLimitInfo { - limit: number; - remaining: number; - reset: number; - used: number; -} - -interface GitHubError { - message: string; - documentation_url?: string; -} - -export interface CompleteRepositoryStructure { - allPaths: string[]; // All file and directory paths - fileContents: Map; // Only files with content -} - -export class GitHubService { - private client: AxiosInstance; - private baseURL = 'https://api.github.com'; - private rateLimitInfo: RateLimitInfo | null = null; - - constructor(token?: string) { - this.client = axios.create({ - baseURL: this.baseURL, - headers: { - 'Accept': 'application/vnd.github.v3+json', - ...(token && { 'Authorization': `Bearer ${token}` }) - }, - timeout: 30000 - }); - - this.setupInterceptors(); - } - - private setupInterceptors(): void { - this.client.interceptors.response.use( - (response: AxiosResponse) => { - this.updateRateLimitInfo(response); - return response; - }, - (error: { response?: AxiosResponse; message: string }) => { - if (error.response) { - this.updateRateLimitInfo(error.response); - - if (error.response.status === 403 && this.isRateLimited()) { - const resetTime = new Date(this.rateLimitInfo!.reset * 1000); - throw new Error(`GitHub API rate limit exceeded. Resets at ${resetTime.toISOString()}`); - } - - if (error.response.status === 401) { - throw new Error('GitHub API authentication failed. Please check your token.'); - } - - if (error.response.status === 404) { - throw new Error('Repository or resource not found.'); - } - - const githubError: GitHubError = error.response.data; - throw new Error(`GitHub API error: ${githubError.message}`); - } - - throw new Error(`Network error: ${error.message}`); - } - ); - } - - private updateRateLimitInfo(response: AxiosResponse): void { - const headers = response.headers; - if (headers['x-ratelimit-limit']) { - this.rateLimitInfo = { - limit: parseInt(headers['x-ratelimit-limit'], 10), - remaining: parseInt(headers['x-ratelimit-remaining'], 10), - reset: parseInt(headers['x-ratelimit-reset'], 10), - used: parseInt(headers['x-ratelimit-used'], 10) - }; - } - } - - private isRateLimited(): boolean { - return this.rateLimitInfo !== null && this.rateLimitInfo.remaining === 0; - } - - public getRateLimitInfo(): RateLimitInfo | null { - return this.rateLimitInfo; - } - - public async checkRateLimit(): Promise { - if (this.isRateLimited()) { - const resetTime = new Date(this.rateLimitInfo!.reset * 1000); - const now = new Date(); - - if (now < resetTime) { - const waitTime = Math.ceil((resetTime.getTime() - now.getTime()) / 1000); - throw new Error(`Rate limit exceeded. Wait ${waitTime} seconds before making another request.`); - } - } - } - - public async getRepositoryContents( - owner: string, - repo: string, - path: string = '' - ): Promise<(GitHubFile | GitHubDirectory)[]> { - await this.checkRateLimit(); - - try { - const response = await this.client.get(`/repos/${owner}/${repo}/contents/${path}`); - - if (!Array.isArray(response.data)) { - throw new Error('Expected directory contents, but received a single file.'); - } - - return response.data as (GitHubFile | GitHubDirectory)[]; - } catch (error) { - if (error instanceof Error) { - throw error; - } - throw new Error('Failed to fetch repository contents'); - } - } - - public async getFileContent( - owner: string, - repo: string, - path: string - ): Promise { - await this.checkRateLimit(); - - try { - const response = await this.client.get(`/repos/${owner}/${repo}/contents/${path}`); - const file = response.data as GitHubFile; - - if (file.type !== 'file') { - throw new Error(`Path ${path} is not a file`); - } - - // If content or encoding is missing, try to download directly - if (!file.content || !file.encoding) { - if (file.download_url) { - console.warn(`File ${path} missing content/encoding, downloading directly`); - return await this.downloadFileRaw(owner, repo, path); - } else { - throw new Error('File content, encoding, and download URL are all missing'); - } - } - - if (file.encoding === 'base64') { - try { - return atob(file.content.replace(/\s/g, '')); - } catch { - throw new Error('Failed to decode base64 content'); - } - } - - return file.content; - } catch (error) { - if (error instanceof Error) { - throw error; - } - throw new Error('Failed to fetch file content'); - } - } - - public async downloadFileRaw( - owner: string, - repo: string, - path: string - ): Promise { - await this.checkRateLimit(); - - try { - const response = await this.client.get(`/repos/${owner}/${repo}/contents/${path}`); - const file = response.data as GitHubFile; - - if (file.type !== 'file' || !file.download_url) { - throw new Error(`Cannot download file: ${path}`); - } - - const downloadResponse = await axios.get(file.download_url, { - timeout: 30000 - }); - - return downloadResponse.data; - } catch (error) { - if (error instanceof Error) { - throw error; - } - throw new Error('Failed to download file'); - } - } - - public async getAllFilesRecursively(owner: string, repo: string, path: string = ''): Promise { - const files: GitHubFile[] = []; - - try { - const contents = await this.getRepositoryContents(owner, repo, path); - - for (const item of contents) { - if (item.type === 'dir') { - // Skip common directories that shouldn't be processed - if (this.shouldSkipDirectory(item.path)) { - console.log(`Skipping directory: ${item.path}`); - continue; - } - - // Recursively get files from subdirectories - const subFiles = await this.getAllFilesRecursively(owner, repo, item.path); - files.push(...subFiles); - } else if (item.type === 'file') { - // Only include files that should be processed - if (this.shouldIncludeFile(item.path)) { - files.push(item); - } else { - console.log(`Skipping file: ${item.path}`); - } - } - } - } catch (error) { - console.error(`Error fetching contents for ${path}:`, error); - } - - return files; - } - - private shouldSkipDirectory(path: string): boolean { - if (!path) return true; // Skip if path is undefined/null - - const skipDirs = [ - // Git version control - '.git', - // JavaScript dependencies (common in full-stack projects) - 'node_modules', - // Python bytecode cache - '__pycache__', - // Python virtual environments - 'venv', - 'env', - '.venv', - 'envs', - 'virtualenv', - // Build, distribution, and temporary directories - 'build', - 'dist', - 'logs', - 'tmp', - '.tmp', - // Additional common directories to skip - 'coverage', - '.coverage', - 'htmlcov', - 'vendor', - 'deps', - '_build', - '.gradle', - 'bin', - 'obj', - '.vs', - '.vscode', - '.idea', - 'temp' - ]; - - // Check each directory component in the path - const pathParts = path.split('/'); - for (const part of pathParts) { - const dirName = part.toLowerCase(); - - // Check for exact matches - if (skipDirs.includes(dirName) || dirName.startsWith('.')) { - return true; - } - - // Check for .egg-info directories - if (dirName.endsWith('.egg-info')) { - return true; - } - } - - // Check for virtual environment patterns anywhere in the path - const fullPathLower = path.toLowerCase(); - const venvPatterns = [ - '/.venv/', - '/venv/', - '/env/', - '/.env/', - '/envs/', - '/virtualenv/', - '/site-packages/', - '/lib/python', - '/lib64/python', - '/scripts/', - '/bin/python' - ]; - - if (venvPatterns.some(pattern => fullPathLower.includes(pattern))) { - return true; - } - - return false; - } - - private shouldIncludeFile(path: string): boolean { - if (!path) return false; // Skip if path is undefined/null - - const fileName = path.split('/').pop() || ''; - - // Skip hidden files except specific config files - if (fileName.startsWith('.') && !fileName.endsWith('.env.example')) { - return false; - } - - // Skip Python-specific file patterns - const skipPatterns = [ - // Python compiled bytecode - /\.pyc$/, - /\.pyo$/, - // Python extension modules (binary) - /\.pyd$/, - /\.so$/, - // Python packages - /\.egg$/, - /\.whl$/, - // Lock files - /\.lock$/, - /poetry\.lock$/, - /Pipfile\.lock$/, - // Editor swap files - /\..*\.swp$/, - /\..*\.swo$/, - // OS metadata files - /^Thumbs\.db$/, - /^\.DS_Store$/, - // General binary and archive files - /\.zip$/, - /\.tar$/, - /\.rar$/, - /\.7z$/, - /\.gz$/, - // Media files - /\.(jpg|jpeg|png|gif|bmp|svg|ico)$/i, - /\.(mp4|avi|mov|wmv|flv|webm)$/i, - /\.(mp3|wav|flac|aac|ogg)$/i, - // Document files - /\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i, - // Other binary files - /\.(exe|dll|dylib)$/i, - // Minified files and source maps - /\.min\.(js|css)$/, - /\.map$/, - // Log and temporary files - /\.log$/, - /\.tmp$/, - /\.cache$/, - /\.pid$/, - /\.seed$/ - ]; - - if (skipPatterns.some(pattern => pattern.test(fileName))) { - return false; - } - - // Include common source and important config files - const extension = '.' + (fileName.split('.').pop() || '').toLowerCase(); - - const includeSourceExts = new Set(['.py', '.js', '.jsx', '.ts', '.tsx']); - if (includeSourceExts.has(extension)) { - return true; - } - - const importantConfigFiles = new Set([ - // Python - 'pyproject.toml', 'setup.py', 'requirements.txt', 'setup.cfg', 'tox.ini', 'pytest.ini', 'pipfile', 'poetry.toml', - '__init__.py', - // JS/TS ecosystem - 'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', - 'tsconfig.json', 'tsconfig.base.json', - 'vite.config.ts', 'vite.config.js', - '.eslintrc', '.eslintrc.json', '.eslintrc.js', '.prettierrc', '.prettierrc.json', - // Docs/licenses - 'readme.md', 'license', 'changelog.md', 'manifest.in' - ]); - if (importantConfigFiles.has(fileName.toLowerCase())) { - return true; - } - - return false; - } - - public async getAllPathsRecursively(owner: string, repo: string, path: string = ''): Promise { - const allPaths: string[] = []; - const fileContents: Map = new Map(); - - try { - const contents = await this.getRepositoryContents(owner, repo, path); - - for (const item of contents) { - const fullPath = item.path; - allPaths.push(fullPath); - - if (item.type === 'file' && this.shouldIncludeFile(fullPath)) { - const content = await this.getFileContent(owner, repo, fullPath); - fileContents.set(fullPath, content); - } - - if (item.type === 'dir') { - // Skip common directories that shouldn't be processed - if (this.shouldSkipDirectory(fullPath)) { - console.log(`Skipping directory: ${fullPath}`); - continue; - } - const subPaths = await this.getAllPathsRecursively(owner, repo, fullPath); - allPaths.push(...subPaths); - } - } - } catch (error) { - console.error(`Error fetching contents for ${path}:`, error); - } - - return allPaths; - } - - /** - * Get complete repository structure including all paths and file contents - * This is the new robust method that discovers structure first, then filters during parsing - */ - public async getCompleteRepositoryStructure(owner: string, repo: string): Promise { - const allPaths: string[] = []; - const fileContents: Map = new Map(); - - await this.collectPathsAndContent(owner, repo, '', allPaths, fileContents); - - console.log(`GitHub: Discovered ${allPaths.length} total paths, ${fileContents.size} files with content`); - - return { - allPaths, - fileContents - }; - } - - private async collectPathsAndContent( - owner: string, - repo: string, - path: string, - allPaths: string[], - fileContents: Map - ): Promise { - try { - const contents = await this.getRepositoryContents(owner, repo, path); - - for (const item of contents) { - const fullPath = item.path; - allPaths.push(fullPath); - - if (item.type === 'file') { - // Always try to get content for files, regardless of filtering - // Filtering will happen later in ParsingProcessor - try { - const content = await this.getFileContent(owner, repo, fullPath); - fileContents.set(fullPath, content); - } catch (error) { - console.warn(`Failed to get content for ${fullPath}:`, error); - } - } else if (item.type === 'dir') { - // REMOVED: shouldSkipDirectory check for complete structure discovery - // All directories are now discovered, filtering happens during parsing - await this.collectPathsAndContent(owner, repo, fullPath, allPaths, fileContents); - } - } - } catch (error) { - console.error(`Error fetching contents for ${path}:`, error); - } - } - - public getAuthenticationStatus(): { authenticated: boolean; rateLimitInfo: RateLimitInfo | null } { - const authHeader = this.client.defaults.headers['Authorization']; - return { - authenticated: !!authHeader, - rateLimitInfo: this.rateLimitInfo - }; - } -} - -================ -File: src/services/health-monitor.ts -================ -/** - * Health monitoring and metrics collection service - */ - -import { performance } from 'perf_hooks'; -import { ConfigService } from '../config/config'; -import { MemoryManager } from './memory-manager.js'; -import { ErrorRecoveryService } from '../lib/error-handler.js'; - -export interface HealthMetrics { - timestamp: Date; - uptime: number; - memory: { - used: number; - total: number; - free: number; - percentage: number; - }; - processing: { - totalFiles: number; - successfulFiles: number; - failedFiles: number; - averageProcessingTime: number; - errors: string[]; - }; - system: { - loadAverage?: number[]; - cpuUsage?: number; - activeConnections: number; - }; -} - -export interface MetricPoint { - timestamp: Date; - value: number; - label?: string; - tags?: Record; -} - -export interface AlertThreshold { - metric: string; - threshold: number; - comparison: 'gt' | 'lt' | 'gte' | 'lte'; - severity: 'low' | 'medium' | 'high' | 'critical'; - message: string; -} - -export class HealthMonitor { - private static instance: HealthMonitor; - private readonly config: ConfigService; - private readonly memoryManager: MemoryManager; - private readonly errorService: ErrorRecoveryService; - private metrics: Map = new Map(); - private alerts: AlertThreshold[] = []; - private startTime: Date; - private processingStats = { - totalFiles: 0, - successfulFiles: 0, - failedFiles: 0, - processingTimes: [] as number[], - errors: [] as string[] - }; - private activeConnections = 0; - private monitoringInterval?: NodeJS.Timeout; - - private constructor() { - this.config = ConfigService.getInstance(); - this.memoryManager = MemoryManager.getInstance(); - this.errorService = ErrorRecoveryService.getInstance(); - this.startTime = new Date(); - this.setupDefaultAlerts(); - } - - static getInstance(): HealthMonitor { - if (!HealthMonitor.instance) { - HealthMonitor.instance = new HealthMonitor(); - } - return HealthMonitor.instance; - } - - /** - * Start health monitoring - */ - start(): void { - const interval = this.config.get('monitoring.interval', 30000); // 30 seconds - - this.monitoringInterval = setInterval(() => { - this.collectMetrics(); - this.checkAlerts(); - }, interval); - - console.info('Health monitoring started'); - } - - /** - * Stop health monitoring - */ - stop(): void { - if (this.monitoringInterval) { - clearInterval(this.monitoringInterval); - this.monitoringInterval = undefined; - } - console.info('Health monitoring stopped'); - } - - /** - * Record a metric value - */ - recordMetric(name: string, value: number, label?: string, tags?: Record): void { - const metricPoint: MetricPoint = { - timestamp: new Date(), - value, - label, - tags - }; - - if (!this.metrics.has(name)) { - this.metrics.set(name, []); - } - - const metricArray = this.metrics.get(name)!; - metricArray.push(metricPoint); - - // Keep only last 1000 points per metric - if (metricArray.length > 1000) { - metricArray.shift(); - } - } - - /** - * Record file processing statistics - */ - recordFileProcessing(filePath: string, success: boolean, processingTime: number, error?: string): void { - this.processingStats.totalFiles++; - - if (success) { - this.processingStats.successfulFiles++; - } else { - this.processingStats.failedFiles++; - if (error) { - this.processingStats.errors.push(error); - } - } - - this.processingStats.processingTimes.push(processingTime); - - // Keep only last 1000 processing times - if (this.processingStats.processingTimes.length > 1000) { - this.processingStats.processingTimes.shift(); - } - - // Record metrics - this.recordMetric('file_processing_time', processingTime, filePath); - this.recordMetric('file_processing_success', success ? 1 : 0, filePath); - } - - /** - * Increment active connections counter - */ - incrementConnections(): void { - this.activeConnections++; - this.recordMetric('active_connections', this.activeConnections); - } - - /** - * Decrement active connections counter - */ - decrementConnections(): void { - this.activeConnections = Math.max(0, this.activeConnections - 1); - this.recordMetric('active_connections', this.activeConnections); - } - - /** - * Get current health status - */ - getHealthStatus(): HealthMetrics { - const memoryUsage = process.memoryUsage(); - const uptime = Date.now() - this.startTime.getTime(); - - return { - timestamp: new Date(), - uptime, - memory: { - used: memoryUsage.heapUsed, - total: memoryUsage.heapTotal, - free: memoryUsage.heapTotal - memoryUsage.heapUsed, - percentage: (memoryUsage.heapUsed / memoryUsage.heapTotal) * 100 - }, - processing: { - totalFiles: this.processingStats.totalFiles, - successfulFiles: this.processingStats.successfulFiles, - failedFiles: this.processingStats.failedFiles, - averageProcessingTime: this.calculateAverageProcessingTime(), - errors: [...new Set(this.processingStats.errors)].slice(-10) // Last 10 unique errors - }, - system: { - loadAverage: this.getLoadAverage(), - cpuUsage: this.getCpuUsage(), - activeConnections: this.activeConnections - } - }; - } - - /** - * Get metrics for a specific name - */ - getMetrics(name: string, timeRange?: { start: Date; end: Date }): MetricPoint[] { - const allMetrics = this.metrics.get(name) || []; - - if (!timeRange) { - return allMetrics; - } - - return allMetrics.filter(metric => - metric.timestamp >= timeRange.start && metric.timestamp <= timeRange.end - ); - } - - /** - * Get aggregated metrics - */ - getAggregatedMetrics(name: string, aggregation: 'sum' | 'avg' | 'min' | 'max' | 'count', timeRange?: { start: Date; end: Date }): number { - const metrics = this.getMetrics(name, timeRange); - - if (metrics.length === 0) return 0; - - const values = metrics.map(m => m.value); - - switch (aggregation) { - case 'sum': - return values.reduce((a, b) => a + b, 0); - case 'avg': - return values.reduce((a, b) => a + b, 0) / values.length; - case 'min': - return Math.min(...values); - case 'max': - return Math.max(...values); - case 'count': - return values.length; - default: - return 0; - } - } - - /** - * Add custom alert threshold - */ - addAlert(alert: AlertThreshold): void { - this.alerts.push(alert); - } - - /** - * Get active alerts - */ - getActiveAlerts(): AlertThreshold[] { - return this.alerts.filter(alert => this.isAlertTriggered(alert)); - } - - /** - * Export health report - */ - exportHealthReport(): string { - const health = this.getHealthStatus(); - const alerts = this.getActiveAlerts(); - - const report = { - generatedAt: new Date().toISOString(), - healthStatus: health, - activeAlerts: alerts, - metricsSummary: this.getMetricsSummary() - }; - - return JSON.stringify(report, null, 2); - } - - /** - * Collect system metrics - */ - private collectMetrics(): void { - const memoryUsage = process.memoryUsage(); - - this.recordMetric('memory_heap_used', memoryUsage.heapUsed); - this.recordMetric('memory_heap_total', memoryUsage.heapTotal); - this.recordMetric('memory_rss', memoryUsage.rss); - this.recordMetric('memory_external', memoryUsage.external); - - this.recordMetric('uptime', Date.now() - this.startTime.getTime()); - this.recordMetric('active_connections', this.activeConnections); - - // Memory manager metrics - const cacheStats = this.memoryManager.getCacheStats(); - this.recordMetric('cache_size', cacheStats.size); - this.recordMetric('cache_hits', cacheStats.hits); - this.recordMetric('cache_misses', cacheStats.misses); - } - - /** - * Check alert thresholds - */ - private checkAlerts(): void { - for (const alert of this.alerts) { - if (this.isAlertTriggered(alert)) { - this.triggerAlert(alert); - } - } - } - - /** - * Check if an alert is triggered - */ - private isAlertTriggered(alert: AlertThreshold): boolean { - const currentValue = this.getCurrentMetricValue(alert.metric); - if (currentValue === null) return false; - - switch (alert.comparison) { - case 'gt': - return currentValue > alert.threshold; - case 'lt': - return currentValue < alert.threshold; - case 'gte': - return currentValue >= alert.threshold; - case 'lte': - return currentValue <= alert.threshold; - default: - return false; - } - } - - /** - * Get current value for a metric - */ - private getCurrentMetricValue(metric: string): number | null { - const health = this.getHealthStatus(); - - switch (metric) { - case 'memory_percentage': - return health.memory.percentage; - case 'processing_errors': - return this.processingStats.errors.length; - case 'active_connections': - return health.system.activeConnections; - case 'uptime': - return health.uptime; - default: - return null; - } - } - - /** - * Trigger an alert - */ - private triggerAlert(alert: AlertThreshold): void { - const message = { - timestamp: new Date().toISOString(), - severity: alert.severity, - message: alert.message, - metric: alert.metric, - threshold: alert.threshold, - currentValue: this.getCurrentMetricValue(alert.metric) - }; - - console.warn(`ALERT [${alert.severity}]: ${alert.message}`, message); - - // In production, this would send to external monitoring service - this.recordMetric('alerts_triggered', 1, alert.message, { severity: alert.severity }); - } - - /** - * Setup default alert thresholds - */ - private setupDefaultAlerts(): void { - this.addAlert({ - metric: 'memory_percentage', - threshold: 85, - comparison: 'gte', - severity: 'high', - message: 'Memory usage is critically high' - }); - - this.addAlert({ - metric: 'processing_errors', - threshold: 10, - comparison: 'gte', - severity: 'medium', - message: 'High number of processing errors detected' - }); - - this.addAlert({ - metric: 'active_connections', - threshold: 100, - comparison: 'gte', - severity: 'medium', - message: 'High number of active connections' - }); - } - - /** - * Calculate average processing time - */ - private calculateAverageProcessingTime(): number { - const times = this.processingStats.processingTimes; - if (times.length === 0) return 0; - return times.reduce((a, b) => a + b, 0) / times.length; - } - - /** - * Get system load average (Node.js specific) - */ - private getLoadAverage(): number[] | undefined { - if (process.platform !== 'win32') { - return require('os').loadavg(); - } - return undefined; - } - - /** - * Get CPU usage (simplified) - */ - private getCpuUsage(): number | undefined { - // This is a simplified version - in production, use proper CPU monitoring - const usage = process.cpuUsage(); - return (usage.user + usage.system) / 1000000; // Convert to milliseconds - } - - /** - * Get metrics summary - */ - private getMetricsSummary(): Record { - const summary: Record = {}; - - for (const [name, metrics] of this.metrics.entries()) { - if (metrics.length > 0) { - const values = metrics.map(m => m.value); - summary[name] = { - count: values.length, - avg: values.reduce((a, b) => a + b, 0) / values.length, - min: Math.min(...values), - max: Math.max(...values), - last: values[values.length - 1] - }; - } - } - - return summary; - } -} - -// Health check endpoint for external monitoring -export class HealthCheckEndpoint { - private healthMonitor: HealthMonitor; - - constructor() { - this.healthMonitor = HealthMonitor.getInstance(); - } - - async handleHealthCheck(): Promise<{ - status: 'healthy' | 'degraded' | 'unhealthy'; - metrics: HealthMetrics; - alerts: AlertThreshold[]; - }> { - const metrics = this.healthMonitor.getHealthStatus(); - const alerts = this.healthMonitor.getActiveAlerts(); - - let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy'; - - if (alerts.some(alert => alert.severity === 'critical')) { - status = 'unhealthy'; - } else if (alerts.length > 0) { - status = 'degraded'; - } - - return { status, metrics, alerts }; - } -} - -================ -File: src/services/ingestion.service.ts -================ -import { GitHubService, type CompleteRepositoryStructure } from './github.ts'; -import { ZipService, type CompleteZipStructure } from './zip.ts'; -import { getIngestionWorker } from '../lib/workerUtils.ts'; -import type { KnowledgeGraph } from '../core/graph/types.ts'; - -export interface IngestionOptions { - directoryFilter?: string; - fileExtensions?: string; - onProgress?: (message: string) => void; -} - -export interface IngestionResult { - graph: KnowledgeGraph; - fileContents: Map; -} - -export class IngestionService { - private githubService: GitHubService; - private zipService: ZipService; - - constructor(githubToken?: string) { - this.githubService = new GitHubService(githubToken); - this.zipService = new ZipService(); - } - - async processGitHubRepo( - githubUrl: string, - options: IngestionOptions = {} - ): Promise { - const { onProgress } = options; - - // Parse GitHub URL - const match = githubUrl.match(/^https:\/\/github\.com\/([^/]+)\/([^/]+)(?:\/.*)?$/); - if (!match) { - throw new Error('Invalid GitHub repository URL'); - } - - const [, owner, repo] = match; - - onProgress?.('Discovering complete repository structure...'); - - // Get complete repository structure (all paths + file contents) - const structure: CompleteRepositoryStructure = await this.githubService.getCompleteRepositoryStructure(owner, repo); - - onProgress?.(`Discovered ${structure.allPaths.length} paths, ${structure.fileContents.size} files. Processing...`); - - // Prepare data for pipeline - const projectName = `${owner}/${repo}`; - const projectRoot = ''; - - // The pipeline now receives ALL paths (files + directories) - // Filtering will happen during parsing, not here - const filePaths = structure.allPaths; - const fileContents = structure.fileContents; - - onProgress?.('Generating knowledge graph...'); - - // Create worker and process - const worker = await getIngestionWorker(); - - try { - const result = await worker.processRepository({ - projectName, - projectRoot, - filePaths, - fileContents: fileContents - }); - - if (!result.success) { - throw new Error(result.error || 'Processing failed'); - } - - return { - graph: result.graph!, - fileContents - }; - } finally { - // Clean up worker - if ('terminate' in worker) { - (worker as { terminate: () => void }).terminate(); - } - } - } - - async processZipFile( - file: File, - options: IngestionOptions = {} - ): Promise { - const { onProgress } = options; - - onProgress?.('Discovering complete ZIP structure...'); - - // Get complete ZIP structure (all paths + file contents) - const structure: CompleteZipStructure = await this.zipService.extractCompleteStructure(file); - - // Normalize ZIP paths to remove common top-level folder - const normalizedStructure = this.normalizeZipPaths(structure); - - onProgress?.(`Discovered ${normalizedStructure.allPaths.length} paths, ${normalizedStructure.fileContents.size} files. Processing...`); - - // Prepare data for pipeline - const projectName = file.name.replace('.zip', ''); - const projectRoot = ''; - - // The pipeline now receives ALL paths (files + directories) - // Filtering will happen during parsing, not here - const filePaths = normalizedStructure.allPaths; - const fileContents = normalizedStructure.fileContents; - - onProgress?.('Generating knowledge graph...'); - - // Create worker and process - const worker = await getIngestionWorker(); - - try { - const result = await worker.processRepository({ - projectName, - projectRoot, - filePaths, - fileContents: fileContents - }); - - if (!result.success) { - throw new Error(result.error || 'Processing failed'); - } - - return { - graph: result.graph!, - fileContents - }; - } finally { - // Clean up worker - if ('terminate' in worker) { - (worker as { terminate: () => void }).terminate(); - } - } - } - - private normalizeZipPaths(structure: CompleteZipStructure): CompleteZipStructure { - const paths = structure.allPaths; - - if (paths.length === 0) { - return structure; - } - - // Find common prefix to remove (usually the top-level folder) - const firstPath = paths[0]; - const pathParts = firstPath.split('/'); - - if (pathParts.length <= 1) { - return structure; // No normalization needed - } - - // Check if all paths start with the same top-level folder - const potentialPrefix = pathParts[0] + '/'; - const allHaveSamePrefix = paths.every(path => path.startsWith(potentialPrefix)); - - if (!allHaveSamePrefix) { - return structure; // No common prefix to remove - } - - console.log(`Normalizing ZIP paths: removing common prefix "${potentialPrefix}"`); - - // Remove the common prefix from all paths - const normalizedPaths = paths.map(path => { - const withoutPrefix = path.substring(potentialPrefix.length); - return withoutPrefix || path; // Keep original if normalization would result in empty string - }).filter(path => path.length > 0); // Remove empty paths - - // Normalize file contents map - const normalizedContents = new Map(); - for (const [originalPath, content] of structure.fileContents) { - const normalizedPath = originalPath.startsWith(potentialPrefix) - ? originalPath.substring(potentialPrefix.length) - : originalPath; - - if (normalizedPath) { - normalizedContents.set(normalizedPath, content); - } - } - - return { - allPaths: normalizedPaths, - fileContents: normalizedContents - }; - } -} - -================ -File: src/services/kuzu.service.ts -================ -import { initKuzuDB, type KuzuDBInstance } from '../core/kuzu/kuzu-loader.js'; -import type { KnowledgeGraph, GraphNode, GraphRelationship, NodeLabel, RelationshipType } from '../core/graph/types.js'; - -export interface KuzuDBSchema { - nodeTables: Map; - relTables: Map; -} - -interface NodeTableSchema { - name: string; - properties: Record; // property name -> type - primaryKey: string; -} - -interface RelTableSchema { - name: string; - sourceTable: string; - targetTable: string; - properties: Record; // property name -> type -} - -export interface KuzuQueryResult { - results: any[]; - count: number; - executionTime: number; -} - -export class KuzuService { - private kuzuInstance: KuzuDBInstance | null = null; - private schema: KuzuDBSchema; - private databasePath: string; - private isInitialized: boolean = false; - - constructor() { - this.schema = { - nodeTables: new Map(), - relTables: new Map() - }; - this.databasePath = '/kuzu/gitnexus.db'; - } - - /** - * Initialize KuzuDB and create schema - */ - async initialize(): Promise { - if (this.isInitialized) { - return; - } - - try { - console.log('Initializing KuzuDB service...'); - - // Initialize KuzuDB WASM - this.kuzuInstance = await initKuzuDB(); - - // Create or open database - await this.kuzuInstance.createDatabase(this.databasePath); - - // Create schema - await this.createSchema(); - - this.isInitialized = true; - console.log('KuzuDB service initialized successfully'); - - } catch (error) { - console.error('Failed to initialize KuzuDB service:', error); - throw error; - } - } - - /** - * Create the database schema for GitNexus knowledge graph - */ - private async createSchema(): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - console.log('Creating KuzuDB schema...'); - - // Create node tables for each node type - const nodeTypes: NodeLabel[] = [ - 'Project', 'Package', 'Module', 'Folder', 'File', - 'Class', 'Function', 'Method', 'Variable', 'Interface', - 'Enum', 'Decorator', 'Import', 'Type', 'CodeElement' - ]; - - for (const nodeType of nodeTypes) { - const tableName = `Node_${nodeType}`; - const properties = { - id: 'STRING', - name: 'STRING', - path: 'STRING', - filePath: 'STRING', - language: 'STRING', - startLine: 'INT64', - endLine: 'INT64', - type: 'STRING', - parameters: 'STRING', // JSON string - returnType: 'STRING', - accessibility: 'STRING', - isStatic: 'BOOL', - isAsync: 'BOOL', - parentClass: 'STRING', - decorators: 'STRING', // JSON string - extends: 'STRING', // JSON string - implements: 'STRING', // JSON string - importPath: 'STRING', - exportType: 'STRING', - docstring: 'STRING', - size: 'INT64', - definitionCount: 'INT64', - lineCount: 'INT64', - qualifiedName: 'STRING' - }; - - this.schema.nodeTables.set(nodeType, { - name: tableName, - properties, - primaryKey: 'id' - }); - - await this.kuzuInstance.createNodeTable(tableName, properties); - } - - // Create relationship tables - const relTypes: RelationshipType[] = [ - 'CONTAINS', 'CALLS', 'INHERITS', 'OVERRIDES', 'IMPORTS', - 'USES', 'DEFINES', 'DECORATES', 'IMPLEMENTS', 'ACCESSES', - 'EXTENDS', 'BELONGS_TO' - ]; - - for (const relType of relTypes) { - const tableName = `Rel_${relType}`; - const properties = { - id: 'STRING', - source: 'STRING', - target: 'STRING', - strength: 'DOUBLE', - confidence: 'DOUBLE', - importType: 'STRING', - alias: 'STRING', - callType: 'STRING', - arguments: 'STRING', // JSON string - dependencyType: 'STRING', - version: 'STRING', - filePath: 'STRING', - line_number: 'INT64' - }; - - this.schema.relTables.set(relType, { - name: tableName, - sourceTable: 'Node_File', // Default, will be updated based on actual relationships - targetTable: 'Node_File', - properties - }); - - await this.kuzuInstance.createRelTable(tableName, properties); - } - - console.log('KuzuDB schema created successfully'); - } - - /** - * Import knowledge graph data into KuzuDB - */ - async importKnowledgeGraph(graph: KnowledgeGraph): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - console.log(`Importing knowledge graph with ${graph.nodes.length} nodes and ${graph.relationships.length} relationships`); - - // Import nodes - for (const node of graph.nodes) { - await this.insertNode(node); - } - - // Import relationships - for (const rel of graph.relationships) { - await this.insertRelationship(rel); - } - - console.log('Knowledge graph imported successfully'); - } - - /** - * Insert a node into KuzuDB - */ - private async insertNode(node: GraphNode): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - const tableName = `Node_${node.label}`; - const properties = { - ...node.properties, - id: node.id - }; - - await this.kuzuInstance.insertNode(tableName, properties); - } - - /** - * Insert a relationship into KuzuDB - */ - private async insertRelationship(rel: GraphRelationship): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - const tableName = `Rel_${rel.type}`; - const properties = { - ...rel.properties, - id: rel.id, - source: rel.source, - target: rel.target - }; - - await this.kuzuInstance.insertRel(tableName, rel.source, rel.target, properties); - } - - /** - * Execute a Cypher query - */ - async executeQuery(query: string): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - const startTime = performance.now(); - - try { - const result = await this.kuzuInstance.executeQuery(query); - const executionTime = performance.now() - startTime; - - return { - results: result.results || [], - count: result.count || 0, - executionTime - }; - } catch (error) { - console.error('KuzuDB query execution failed:', error); - throw new Error(`Query execution failed: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - /** - * Get database statistics - */ - async getDatabaseStats(): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - return await this.kuzuInstance.getDatabaseInfo(); - } - - /** - * Clear all data from the database - */ - async clearDatabase(): Promise { - if (!this.kuzuInstance) { - throw new Error('KuzuDB not initialized'); - } - - await this.kuzuInstance.clearDatabase(); - console.log('KuzuDB database cleared'); - } - - /** - * Close the database connection - */ - async close(): Promise { - if (this.kuzuInstance) { - await this.kuzuInstance.closeDatabase(); - this.isInitialized = false; - console.log('KuzuDB connection closed'); - } - } - - /** - * Get the current schema - */ - getSchema(): KuzuDBSchema { - return this.schema; - } - - /** - * Check if the service is initialized - */ - isReady(): boolean { - return this.isInitialized && this.kuzuInstance !== null; - } -} - -================ -File: src/services/memory-manager.ts -================ -/** - * Memory Management Service - * Provides resource limits, cleanup, and monitoring for the GitNexus application - */ - -export interface MemoryConfig { - maxMemoryMB: number; - cleanupThresholdMB: number; - gcIntervalMs: number; - maxFileSizeMB: number; - maxFilesInMemory: number; -} - -export interface MemoryStats { - usedMemoryMB: number; - totalMemoryMB: number; - fileCount: number; - lastCleanup: Date; - warnings: string[]; -} - -export class MemoryManager { - private static instance: MemoryManager; - private config: MemoryConfig; - private fileCache: Map; - private cleanupTimer: NodeJS.Timeout | null = null; - private memoryWarnings: string[] = []; - - private constructor() { - this.config = { - maxMemoryMB: 512, - cleanupThresholdMB: 400, - gcIntervalMs: 30000, - maxFileSizeMB: 10, - maxFilesInMemory: 1000 - }; - this.fileCache = new Map(); - this.startCleanupTimer(); - } - - public static getInstance(): MemoryManager { - if (!MemoryManager.instance) { - MemoryManager.instance = new MemoryManager(); - } - return MemoryManager.instance; - } - - /** - * Configure memory limits - */ - public configure(config: Partial): void { - this.config = { ...this.config, ...config }; - } - - /** - * Get current memory statistics - */ - public getStats(): MemoryStats { - const totalSize = Array.from(this.fileCache.values()) - .reduce((sum, item) => sum + item.size, 0); - - return { - usedMemoryMB: Math.round(totalSize / (1024 * 1024)), - totalMemoryMB: this.config.maxMemoryMB, - fileCount: this.fileCache.size, - lastCleanup: new Date(), - warnings: [...this.memoryWarnings] - }; - } - - /** - * Cache file content with memory limits - */ - public cacheFile(filePath: string, content: string): boolean { - const size = typeof Blob !== 'undefined' ? new Blob([content]).size : Buffer.byteLength(content, 'utf-8'); - - // Check file size limit - if (size > this.config.maxFileSizeMB * 1024 * 1024) { - this.addWarning(`File too large: ${filePath} (${Math.round(size / (1024 * 1024))}MB)`); - return false; - } - - // Check memory limit - const currentTotal = Array.from(this.fileCache.values()) - .reduce((sum, item) => sum + item.size, 0); - - if (currentTotal + size > this.config.maxMemoryMB * 1024 * 1024) { - this.performCleanup(); - } - - // Check file count limit - if (this.fileCache.size >= this.config.maxFilesInMemory) { - this.evictOldestFiles(); - } - - this.fileCache.set(filePath, { - content, - size, - lastAccess: new Date() - }); - - return true; - } - - /** - * Get cached file content - */ - public getFile(filePath: string): string | null { - const item = this.fileCache.get(filePath); - if (item) { - item.lastAccess = new Date(); - return item.content; - } - return null; - } - - /** - * Remove file from cache - */ - public removeFile(filePath: string): void { - this.fileCache.delete(filePath); - } - - /** - * Clear all cached files - */ - public clearCache(): void { - this.fileCache.clear(); - this.memoryWarnings = []; - } - - /** - * Force cleanup when memory is low - */ - public forceCleanup(): void { - this.performCleanup(); - } - - private performCleanup(): void { - const currentTotal = Array.from(this.fileCache.values()) - .reduce((sum, item) => sum + item.size, 0); - - if (currentTotal > this.config.cleanupThresholdMB * 1024 * 1024) { - // Remove least recently used files - const sorted = Array.from(this.fileCache.entries()) - .sort((a, b) => a[1].lastAccess.getTime() - b[1].lastAccess.getTime()); - - const toRemove = Math.ceil(sorted.length * 0.2); // Remove 20% - for (let i = 0; i < toRemove; i++) { - this.fileCache.delete(sorted[i][0]); - } - - this.addWarning(`Cleanup performed: removed ${toRemove} files`); - } - - // Trigger garbage collection if available - if (typeof globalThis.gc === 'function') { - globalThis.gc(); - } - } - - private evictOldestFiles(): void { - const sorted = Array.from(this.fileCache.entries()) - .sort((a, b) => a[1].lastAccess.getTime() - b[1].lastAccess.getTime()); - - const toRemove = Math.max(1, Math.floor(this.config.maxFilesInMemory * 0.1)); - for (let i = 0; i < toRemove; i++) { - this.fileCache.delete(sorted[i][0]); - } - } - - private startCleanupTimer(): void { - if (this.cleanupTimer) { - clearInterval(this.cleanupTimer); - } - - this.cleanupTimer = setInterval(() => { - this.performCleanup(); - }, this.config.gcIntervalMs); - } - - private addWarning(message: string): void { - this.memoryWarnings.push(message); - if (this.memoryWarnings.length > 10) { - this.memoryWarnings.shift(); - } - - console.warn(`[MemoryManager] ${message}`); - } - - /** - * Stop the cleanup timer - */ - public destroy(): void { - if (this.cleanupTimer) { - clearInterval(this.cleanupTimer); - this.cleanupTimer = null; - } - this.clearCache(); - } -} - -================ -File: src/services/zip.ts -================ -import JSZip from 'jszip'; - -interface JSZipObjectWithData extends JSZip.JSZipObject { - _data?: { - uncompressedSize: number; - }; -} - -interface ZipFileEntry { - path: string; - content: string; - isDirectory: boolean; - size: number; - lastModified: Date; -} - -interface ExtractionOptions { - maxFileSize?: number; - maxTotalSize?: number; - allowedExtensions?: string[]; - excludeDirectories?: boolean; -} - -export interface CompleteZipStructure { - allPaths: string[]; // All file and directory paths - fileContents: Map; // Only files with content -} - -export class ZipService { - private static readonly DEFAULT_MAX_FILE_SIZE = 10 * 1024 * 1024; // 10MB - private static readonly DEFAULT_MAX_TOTAL_SIZE = 100 * 1024 * 1024; // 100MB - private static readonly TEXT_EXTENSIONS = new Set([ - '.js', '.ts', '.tsx', '.jsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', - '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.clj', - '.html', '.htm', '.xml', '.css', '.scss', '.sass', '.less', '.json', - '.yaml', '.yml', '.toml', '.ini', '.cfg', '.conf', '.md', '.txt', - '.sql', '.sh', '.bash', '.zsh', '.fish', '.ps1', '.bat', '.cmd', - '.dockerfile', '.gitignore', '.gitattributes', '.env', '.properties' - ]); - - constructor() {} - - /** - * Extract complete ZIP structure including all paths and file contents - * This is the new robust method that discovers structure first, then filters during parsing - */ - public async extractCompleteStructure( - file: File, - options: ExtractionOptions = {} - ): Promise { - const { - maxFileSize = ZipService.DEFAULT_MAX_FILE_SIZE, - maxTotalSize = ZipService.DEFAULT_MAX_TOTAL_SIZE - } = options; - - if (!file) { - throw new Error('No file provided'); - } - - if (!file.name.toLowerCase().endsWith('.zip')) { - throw new Error('File must be a ZIP archive'); - } - - console.log(`Starting complete ZIP extraction of: ${file.name} (${file.size} bytes)`); - - try { - const zip = await JSZip.loadAsync(file); - const allPaths: string[] = []; - const fileContents: Map = new Map(); - const directories: Set = new Set(); - let totalExtractedSize = 0; - - // First pass: collect all paths and identify directories - zip.forEach((relativePath, zipObject) => { - // Normalize path separators - const normalizedPath = relativePath.replace(/\\/g, '/'); - - // Add all parent directories - const pathParts = normalizedPath.split('/'); - for (let i = 1; i < pathParts.length; i++) { - const dirPath = pathParts.slice(0, i).join('/'); - if (dirPath && !directories.has(dirPath)) { - directories.add(dirPath); - allPaths.push(dirPath); - } - } - - // Add the current path - if (!allPaths.includes(normalizedPath)) { - allPaths.push(normalizedPath); - } - - // If it's a directory entry, mark it - if (zipObject.dir) { - directories.add(normalizedPath.replace(/\/$/, '')); - } - }); - - // Second pass: extract file contents - const filePromises: Promise[] = []; - - zip.forEach((relativePath, zipObject) => { - const normalizedPath = relativePath.replace(/\\/g, '/'); - - // Skip directories and empty paths - if (zipObject.dir || !normalizedPath || normalizedPath.endsWith('/')) { - return; - } - - // REMOVED: shouldSkipDirectory check for complete structure discovery - // All files are now discovered, filtering happens during parsing - - const zipObjectWithData = zipObject as JSZipObjectWithData; - const uncompressedSize = zipObjectWithData._data?.uncompressedSize || 0; - - // Check individual file size - if (uncompressedSize > maxFileSize) { - console.warn(`Skipping large file: ${normalizedPath} (${uncompressedSize} bytes)`); - return; - } - - // Check total extracted size - if (totalExtractedSize + uncompressedSize > maxTotalSize) { - console.warn(`Stopping extraction: total size limit reached (${maxTotalSize} bytes)`); - return; - } - - // Extract file content - const promise = zipObject.async('text') - .then(content => { - if (content.length > 0) { - fileContents.set(normalizedPath, content); - totalExtractedSize += content.length; - } - }) - .catch(error => { - console.warn(`Failed to extract ${normalizedPath}:`, error); - }); - - filePromises.push(promise); - }); - - // Wait for all file extractions to complete - await Promise.all(filePromises); - - console.log(`ZIP: Discovered ${allPaths.length} total paths, ${fileContents.size} files with content`); - console.log(`ZIP: Total extracted size: ${totalExtractedSize} bytes`); - - return { - allPaths: allPaths.sort(), // Sort for consistent ordering - fileContents - }; - - } catch (error) { - console.error('Error extracting ZIP file:', error); - throw new Error(`Failed to extract ZIP file: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - public async extractTextFiles( - file: File, - options: ExtractionOptions = {} - ): Promise> { - const { - maxFileSize = ZipService.DEFAULT_MAX_FILE_SIZE, - maxTotalSize = ZipService.DEFAULT_MAX_TOTAL_SIZE, - allowedExtensions = Array.from(ZipService.TEXT_EXTENSIONS), - excludeDirectories = true - } = options; - - if (!file) { - throw new Error('No file provided'); - } - - if (file.type !== 'application/zip' && !file.name.toLowerCase().endsWith('.zip')) { - throw new Error('File must be a ZIP archive'); - } - - try { - const arrayBuffer = await this.fileToArrayBuffer(file); - const zip = new JSZip(); - const zipContent = await zip.loadAsync(arrayBuffer); - - const extractedFiles = new Map(); - let totalExtractedSize = 0; - - const files = Object.keys(zipContent.files); - - for (const filePath of files) { - const zipFile = zipContent.files[filePath]; - - if (zipFile.dir && excludeDirectories) { - continue; - } - - if (zipFile.dir) { - extractedFiles.set(filePath, ''); - continue; - } - - // Skip directories and files that shouldn't be processed - if (this.shouldSkipPath(filePath)) { - console.log(`Skipping filtered path: ${filePath}`); - continue; - } - - if (!this.isTextFile(filePath, allowedExtensions)) { - continue; - } - - const zipFileWithData = zipFile as JSZipObjectWithData; - if (zipFileWithData._data && zipFileWithData._data.uncompressedSize > maxFileSize) { - console.warn(`Skipping file ${filePath}: exceeds maximum file size (${maxFileSize} bytes)`); - continue; - } - - if (totalExtractedSize + (zipFileWithData._data?.uncompressedSize || 0) > maxTotalSize) { - console.warn(`Stopping extraction: total size would exceed maximum (${maxTotalSize} bytes)`); - break; - } - - try { - const content = await zipFile.async('text'); - - if (content.length > maxFileSize) { - console.warn(`Skipping file ${filePath}: content exceeds maximum file size`); - continue; - } - - extractedFiles.set(filePath, content); - totalExtractedSize += content.length; - - } catch (fileError) { - console.warn(`Failed to extract file ${filePath}:`, fileError); - continue; - } - } - - return extractedFiles; - - } catch (error) { - if (error instanceof Error) { - if (error.message.includes('corrupt') || error.message.includes('invalid')) { - throw new Error('ZIP file appears to be corrupted or invalid'); - } - throw new Error(`Failed to extract ZIP file: ${error.message}`); - } - throw new Error('Unknown error occurred while extracting ZIP file'); - } - } - - public async getZipFileInfo(file: File): Promise { - if (!file) { - throw new Error('No file provided'); - } - - try { - const arrayBuffer = await this.fileToArrayBuffer(file); - const zip = new JSZip(); - const zipContent = await zip.loadAsync(arrayBuffer); - - const fileInfos: ZipFileEntry[] = []; - - for (const [path, zipFile] of Object.entries(zipContent.files)) { - const file = zipFile as { dir: boolean; _data?: { uncompressedSize: number }; date?: Date }; - fileInfos.push({ - path, - content: '', // Don't load content for info request - isDirectory: file.dir, - size: file._data?.uncompressedSize || 0, - lastModified: file.date || new Date() - }); - } - - return fileInfos.sort((a, b) => a.path.localeCompare(b.path)); - - } catch (error) { - if (error instanceof Error) { - throw new Error(`Failed to read ZIP file info: ${error.message}`); - } - throw new Error('Unknown error occurred while reading ZIP file info'); - } - } - - public async extractSpecificFiles( - file: File, - filePaths: string[] - ): Promise> { - if (!file) { - throw new Error('No file provided'); - } - - if (!filePaths || filePaths.length === 0) { - throw new Error('No file paths specified'); - } - - try { - const arrayBuffer = await this.fileToArrayBuffer(file); - const zip = new JSZip(); - const zipContent = await zip.loadAsync(arrayBuffer); - - const extractedFiles = new Map(); - - for (const filePath of filePaths) { - const zipFile = zipContent.files[filePath]; - - if (!zipFile) { - console.warn(`File not found in ZIP: ${filePath}`); - continue; - } - - if (zipFile.dir) { - extractedFiles.set(filePath, ''); - continue; - } - - try { - const content = await zipFile.async('text'); - extractedFiles.set(filePath, content); - } catch (fileError) { - console.warn(`Failed to extract file ${filePath}:`, fileError); - continue; - } - } - - return extractedFiles; - - } catch (error) { - if (error instanceof Error) { - throw new Error(`Failed to extract specific files from ZIP: ${error.message}`); - } - throw new Error('Unknown error occurred while extracting specific files'); - } - } - - public isValidZipFile(file: File): boolean { - if (!file) return false; - - return ( - file.type === 'application/zip' || - file.type === 'application/x-zip-compressed' || - file.name.toLowerCase().endsWith('.zip') - ); - } - - private async fileToArrayBuffer(file: File): Promise { - return new Promise((resolve, reject) => { - const reader = new FileReader(); - - reader.onload = () => { - if (reader.result instanceof ArrayBuffer) { - resolve(reader.result); - } else { - reject(new Error('Failed to read file as ArrayBuffer')); - } - }; - - reader.onerror = () => { - reject(new Error('Error reading file')); - }; - - reader.readAsArrayBuffer(file); - }); - } - - private isTextFile(filePath: string, allowedExtensions: string[]): boolean { - if (!filePath || filePath.endsWith('/')) { - return false; - } - - const extension = filePath.toLowerCase().split('.').pop(); - return extension ? allowedExtensions.includes(`.${extension}`) : false; - } - - private shouldSkipPath(filePath: string): boolean { - // Skip directories that shouldn't be processed - if (this.shouldSkipDirectory(filePath)) { - return true; - } - - // Skip files that shouldn't be processed - if (!this.shouldIncludeFile(filePath)) { - return true; - } - - return false; - } - - private shouldSkipDirectory(path: string): boolean { - if (!path) return true; // Skip if path is undefined/null - - const skipDirs = [ - // Git version control - '.git', - // JavaScript dependencies (common in full-stack projects) - 'node_modules', - // Python bytecode cache - '__pycache__', - // Python virtual environments - 'venv', - 'env', - '.venv', - 'envs', - 'virtualenv', - // Build, distribution, and temporary directories - 'build', - 'dist', - 'logs', - 'tmp', - '.tmp', - // Additional common directories to skip - 'coverage', - '.coverage', - 'htmlcov', - 'vendor', - 'deps', - '_build', - '.gradle', - 'bin', - 'obj', - '.vs', - '.vscode', - '.idea', - 'temp' - ]; - - // Check each directory component in the path - const pathParts = path.split('/'); - for (const part of pathParts) { - const dirName = part.toLowerCase(); - - // Check for exact matches - if (skipDirs.includes(dirName) || dirName.startsWith('.')) { - return true; - } - - // Check for .egg-info directories - if (dirName.endsWith('.egg-info')) { - return true; - } - } - - // Check for virtual environment patterns anywhere in the path - const fullPathLower = path.toLowerCase(); - const venvPatterns = [ - '/.venv/', - '/venv/', - '/env/', - '/.env/', - '/envs/', - '/virtualenv/', - '/site-packages/', - '/lib/python', - '/lib64/python', - '/scripts/', - '/bin/python' - ]; - - if (venvPatterns.some(pattern => fullPathLower.includes(pattern))) { - return true; - } - - return false; - } - - private shouldIncludeFile(path: string): boolean { - if (!path) return false; // Skip if path is undefined/null - - const fileName = path.split('/').pop() || ''; - - // Skip hidden files except specific config files - if (fileName.startsWith('.') && !fileName.endsWith('.env.example')) { - return false; - } - - // Skip common compiled/binary patterns - const skipPatterns = [ - // Python compiled bytecode - /\.pyc$/, - /\.pyo$/, - // Python extension modules (binary) - /\.pyd$/, - /\.so$/, - // Python packages - /\.egg$/, - /\.whl$/, - // Lock files - /\.lock$/, - /poetry\.lock$/, - /Pipfile\.lock$/, - // Editor swap files - /\..*\.swp$/, - /\..*\.swo$/, - // OS metadata files - /^Thumbs\.db$/, - /^\.DS_Store$/, - // General binary and archive files - /\.zip$/, - /\.tar$/, - /\.rar$/, - /\.7z$/, - /\.gz$/, - // Media files - /\.(jpg|jpeg|png|gif|bmp|svg|ico)$/i, - /\.(mp4|avi|mov|wmv|flv|webm)$/i, - /\.(mp3|wav|flac|aac|ogg)$/i, - // Document files - /\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i, - // Other binary files - /\.(exe|dll|dylib)$/i, - // Minified files and source maps - /\.min\.(js|css)$/, - /\.map$/, - // Log and temporary files - /\.log$/, - /\.tmp$/, - /\.cache$/, - /\.pid$/, - /\.seed$/ - ]; - - if (skipPatterns.some(pattern => pattern.test(fileName))) { - return false; - } - - // Include common source and important config files - const extension = '.' + (fileName.split('.').pop() || '').toLowerCase(); - const includeSourceExts = new Set(['.py', '.js', '.jsx', '.ts', '.tsx']); - if (includeSourceExts.has(extension)) { - return true; - } - - const importantConfigFiles = new Set([ - // Python - 'pyproject.toml', 'setup.py', 'requirements.txt', 'setup.cfg', 'tox.ini', 'pytest.ini', 'pipfile', 'poetry.toml', - '__init__.py', - // JS/TS ecosystem - 'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', - 'tsconfig.json', 'tsconfig.base.json', - 'vite.config.ts', 'vite.config.js', - '.eslintrc', '.eslintrc.json', '.eslintrc.js', '.prettierrc', '.prettierrc.json', - // Docs/licenses - 'readme.md', 'license', 'changelog.md', 'manifest.in' - ]); - if (importantConfigFiles.has(fileName.toLowerCase())) { - return true; - } - - return false; - } - - public getDefaultTextExtensions(): string[] { - return Array.from(ZipService.TEXT_EXTENSIONS); - } - - public async validateZipFile(file: File): Promise<{ valid: boolean; error?: string }> { - try { - if (!this.isValidZipFile(file)) { - return { valid: false, error: 'File is not a valid ZIP archive' }; - } - - const arrayBuffer = await this.fileToArrayBuffer(file); - const zip = new JSZip(); - await zip.loadAsync(arrayBuffer); - - return { valid: true }; - - } catch (error) { - return { - valid: false, - error: error instanceof Error ? error.message : 'Unknown validation error' - }; - } - } -} - -================ -File: src/ui/components/chat/ChatInterface.tsx -================ -import React, { useState, useRef, useEffect } from 'react'; -import ReactMarkdown from 'react-markdown'; -import remarkGfm from 'remark-gfm'; -import rehypeHighlight from 'rehype-highlight'; -import type { KnowledgeGraph } from '../../../core/graph/types.ts'; -import { LLMService, type LLMProvider, type LLMConfig } from '../../../ai/llm-service.ts'; -import { CypherGenerator } from '../../../ai/cypher-generator.ts'; -import { RAGOrchestrator, type RAGResponse, type RAGOptions } from '../../../ai/orchestrator.ts'; - -interface ChatMessage { - id: string; - role: 'user' | 'assistant'; - content: string; - timestamp: Date; - metadata?: { - cypherQueries?: Array<{ cypher: string; explanation: string; confidence?: number }>; - sources?: string[]; - confidence?: number; - reasoning?: Array<{ - step: number; - thought: string; - action: string; - actionInput?: string; - observation?: string; - toolResult?: { - toolName: string; - input: string; - output: string; - success: boolean; - error?: string; - }; - }>; - debugInfo?: { - llmConfig: LLMConfig; - ragOptions: RAGOptions; - contextInfo: { - nodeCount: number; - fileCount: number; - hasContext: boolean; - }; - totalExecutionTime?: number; - queryExecutionTimes?: Array<{ query: string; time: number }>; - }; - }; -} - -interface ChatInterfaceProps { - graph: KnowledgeGraph; - fileContents: Map; - className?: string; - style?: React.CSSProperties; -} - -interface LLMSettings { - provider: LLMProvider; - apiKey: string; - model: string; - temperature: number; - maxTokens: number; - // Azure OpenAI specific fields - azureOpenAIEndpoint?: string; - azureOpenAIDeploymentName?: string; - azureOpenAIApiVersion?: string; -} - -const ChatInterface: React.FC = ({ - graph, - fileContents, - className = '', - style = {} -}) => { - const [messages, setMessages] = useState([]); - const [inputValue, setInputValue] = useState(''); - const [isLoading, setIsLoading] = useState(false); - const [showSettings, setShowSettings] = useState(false); - const [showReasoning, setShowReasoning] = useState(false); - const [debugMode, setDebugMode] = useState(false); - - // LLM Configuration - const [llmSettings, setLLMSettings] = useState({ - provider: 'openai', - apiKey: '', - model: 'gpt-4o-mini', - temperature: 0.1, - maxTokens: 4000, - azureOpenAIEndpoint: '', - azureOpenAIDeploymentName: '', - azureOpenAIApiVersion: '2024-02-01' - }); - - // Services - const [llmService] = useState(new LLMService()); - const [cypherGenerator] = useState(new CypherGenerator(llmService)); - const [ragOrchestrator] = useState(new RAGOrchestrator(llmService, cypherGenerator)); - - const messagesEndRef = useRef(null); - const inputRef = useRef(null); - - // Initialize RAG context when graph or fileContents change - useEffect(() => { - ragOrchestrator.setContext({ graph, fileContents }); - }, [graph, fileContents, ragOrchestrator]); - - // Load settings from localStorage on mount - useEffect(() => { - const savedProvider = localStorage.getItem('llm_provider') as LLMProvider; - const savedApiKey = localStorage.getItem('llm_api_key'); - const savedAzureEndpoint = localStorage.getItem('azure_openai_endpoint'); - const savedAzureDeployment = localStorage.getItem('azure_openai_deployment'); - const savedAzureApiVersion = localStorage.getItem('azure_openai_api_version'); - const savedDebugMode = localStorage.getItem('debug_mode') === 'true'; - - if (savedProvider || savedApiKey || savedAzureEndpoint) { - setLLMSettings(prev => ({ - ...prev, - provider: savedProvider || prev.provider, - apiKey: savedApiKey || prev.apiKey, - azureOpenAIEndpoint: savedAzureEndpoint || prev.azureOpenAIEndpoint, - azureOpenAIDeploymentName: savedAzureDeployment || prev.azureOpenAIDeploymentName, - azureOpenAIApiVersion: savedAzureApiVersion || prev.azureOpenAIApiVersion, - // For Azure OpenAI, use deployment name as model, otherwise use default model - model: savedProvider === 'azure-openai' - ? (savedAzureDeployment || 'gpt-4.1-mini-v2') - : (savedProvider ? llmService.getAvailableModels(savedProvider)[0] : prev.model) - })); - } - - setDebugMode(savedDebugMode); - }, [llmService]); - - // Auto-scroll to bottom when new messages arrive - useEffect(() => { - messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); - }, [messages]); - - // Handle form submission - const handleSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - if (!inputValue.trim() || isLoading) return; - - // Validate API key - if (!llmSettings.apiKey.trim()) { - alert('Please configure your API key in settings'); - setShowSettings(true); - return; - } - - if (!llmService.validateApiKey(llmSettings.provider, llmSettings.apiKey)) { - alert('Invalid API key format. Please check your settings.'); - setShowSettings(true); - return; - } - - // Additional validation for Azure OpenAI - if (llmSettings.provider === 'azure-openai') { - if (!llmSettings.azureOpenAIEndpoint?.trim()) { - alert('Please configure your Azure OpenAI endpoint in settings'); - setShowSettings(true); - return; - } - if (!llmSettings.azureOpenAIDeploymentName?.trim()) { - alert('Please configure your Azure OpenAI deployment name in settings'); - setShowSettings(true); - return; - } - } - - const userMessage: ChatMessage = { - id: generateId(), - role: 'user', - content: inputValue.trim(), - timestamp: new Date() - }; - - setMessages(prev => [...prev, userMessage]); - setInputValue(''); - setIsLoading(true); - - try { - const startTime = Date.now(); - - const llmConfig: LLMConfig = { - provider: llmSettings.provider, - apiKey: llmSettings.apiKey, - model: llmSettings.model, - temperature: llmSettings.temperature, - maxTokens: llmSettings.maxTokens, - // Azure OpenAI specific fields - azureOpenAIEndpoint: llmSettings.azureOpenAIEndpoint, - azureOpenAIDeploymentName: llmSettings.azureOpenAIDeploymentName, - azureOpenAIApiVersion: llmSettings.azureOpenAIApiVersion - }; - - const ragOptions: RAGOptions = { - maxReasoningSteps: 5, - includeReasoning: debugMode || showReasoning, // Always include reasoning in debug mode - strictMode: false, - temperature: llmSettings.temperature - }; - - const response: RAGResponse = await ragOrchestrator.answerQuestion( - userMessage.content, - llmConfig, - ragOptions - ); - - const endTime = Date.now(); - const executionTime = endTime - startTime; - - const assistantMessage: ChatMessage = { - id: generateId(), - role: 'assistant', - content: response.answer, - timestamp: new Date(), - metadata: { - cypherQueries: response.cypherQueries.map(q => ({ - cypher: q.cypher, - explanation: q.explanation, - confidence: q.confidence - })), - sources: response.sources, - confidence: response.confidence, - reasoning: (debugMode || showReasoning) ? response.reasoning.map(r => ({ - step: r.step, - thought: r.thought, - action: r.action, - actionInput: r.actionInput, - observation: r.observation, - toolResult: r.toolResult - })) : undefined, - debugInfo: debugMode ? { - llmConfig, - ragOptions, - contextInfo: ragOrchestrator.getContextInfo(), - totalExecutionTime: executionTime, - queryExecutionTimes: response.cypherQueries.map(q => ({ - query: q.cypher, - time: 0 // We'd need to instrument the query engine for this - })) - } : undefined - } - }; - - setMessages(prev => [...prev, assistantMessage]); - - } catch (error) { - const errorMessage: ChatMessage = { - id: generateId(), - role: 'assistant', - content: `I apologize, but I encountered an error while processing your question: ${error instanceof Error ? error.message : 'Unknown error'}`, - timestamp: new Date() - }; - - setMessages(prev => [...prev, errorMessage]); - } finally { - setIsLoading(false); - } - }; - - // Handle key press in textarea - const handleKeyPress = (e: React.KeyboardEvent) => { - if (e.key === 'Enter' && !e.shiftKey) { - e.preventDefault(); - handleSubmit(e as any); - } - }; - - // Clear conversation - const clearConversation = () => { - setMessages([]); - }; - - // Toggle debug mode and save to localStorage - const toggleDebugMode = () => { - const newDebugMode = !debugMode; - setDebugMode(newDebugMode); - localStorage.setItem('debug_mode', newDebugMode.toString()); - }; - - // Graph diagnostics - const runGraphDiagnostics = () => { - const contextInfo = ragOrchestrator.getContextInfo(); - - if (!contextInfo.hasContext) { - alert('No graph loaded yet. Please load a repository first.'); - return; - } - - // Basic statistics - const stats = [ - `๐Ÿ“Š Graph Statistics:`, - `โ€ข Nodes: ${contextInfo.nodeCount}`, - `โ€ข Files: ${contextInfo.fileCount}`, - ``, - `๐Ÿ” Diagnostic Tips:`, - `โ€ข Check browser console for detailed ingestion logs`, - `โ€ข Look for warnings about isolated nodes or parsing failures`, - `โ€ข Verify that source files contain recognizable functions/classes`, - ``, - `If you see isolated nodes:`, - `1. Check if files failed to parse (console warnings)`, - `2. Ensure files contain valid code syntax`, - `3. Check if file extensions are supported (.js, .ts, .py, etc.)`, - `4. Look for import/export syntax errors` - ].join('\n'); - - alert(stats); - - // Also log to console for more details - console.log('๐Ÿ” Graph Diagnostics Requested'); - console.log('Context Info:', contextInfo); - }; - - // Debug Panel Component - const DebugPanel: React.FC<{ message: ChatMessage }> = ({ message }) => { - if (!message.metadata?.debugInfo && !message.metadata?.reasoning && !message.metadata?.cypherQueries) { - return null; - } - - const [activeTab, setActiveTab] = useState<'reasoning' | 'queries' | 'config' | 'context'>('reasoning'); - - return ( -
-
- -
- -
- {/* Reasoning Steps Tab */} - {activeTab === 'reasoning' && message.metadata?.reasoning && ( -
- {message.metadata.reasoning.map((step, index) => ( -
-
-

Step {step.step}

- - {step.action} - -
- -
-
-
Thought:
-

{step.thought}

-
- - {step.actionInput && ( -
-
Action Input:
-

{step.actionInput}

-
- )} - - {step.observation && ( -
-
Observation:
-
- -
-
- )} - - {step.toolResult && ( -
-
Tool Result:
-
-
- Tool: {step.toolResult.toolName} -
-
- Success: - - {step.toolResult.success ? 'Yes' : 'No'} - -
- {step.toolResult.error && ( -
- Error: -

{step.toolResult.error}

-
- )} -
-
- )} -
-
- ))} -
- )} - - {/* Cypher Queries Tab */} - {activeTab === 'queries' && message.metadata?.cypherQueries && ( -
- {message.metadata.cypherQueries.map((query, index) => ( -
-
-

Query {index + 1}

- {query.confidence && ( - 0.8 - ? 'bg-green-100 text-green-800' - : query.confidence > 0.6 - ? 'bg-yellow-100 text-yellow-800' - : 'bg-red-100 text-red-800' - }`}> - Confidence: {(query.confidence * 100).toFixed(0)}% - - )} -
- -
-
-
Cypher Query:
- 
-                        {query.cypher}
-
-
- -
-
Explanation:
-
- -
-
-
-
- ))} -
- )} - - {/* Configuration Tab */} - {activeTab === 'config' && message.metadata?.debugInfo && ( -
-
-

LLM Configuration

-
-
Provider: {message.metadata.debugInfo.llmConfig.provider}
-
Model: {message.metadata.debugInfo.llmConfig.model}
-
Temperature: {message.metadata.debugInfo.llmConfig.temperature}
-
Max Tokens: {message.metadata.debugInfo.llmConfig.maxTokens}
-
-
- -
-

RAG Options

-
-
Max Reasoning Steps: {message.metadata.debugInfo.ragOptions.maxReasoningSteps}
-
Include Reasoning: {message.metadata.debugInfo.ragOptions.includeReasoning ? 'Yes' : 'No'}
-
Strict Mode: {message.metadata.debugInfo.ragOptions.strictMode ? 'Yes' : 'No'}
-
Temperature: {message.metadata.debugInfo.ragOptions.temperature}
-
-
- -
-

Performance

-
-
Total Execution Time: {message.metadata.debugInfo.totalExecutionTime}ms
-
Confidence Score: {message.metadata.confidence ? (message.metadata.confidence * 100).toFixed(1) + '%' : 'N/A'}
-
-
-
- )} - - {/* Context Info Tab */} - {activeTab === 'context' && message.metadata?.debugInfo && ( -
-
-

Knowledge Graph Context

-
-
-
{message.metadata.debugInfo.contextInfo.nodeCount}
-
Graph Nodes
-
-
-
{message.metadata.debugInfo.contextInfo.fileCount}
-
Files Indexed
-
-
-
{message.metadata.sources?.length || 0}
-
Sources Used
-
-
-
- - {message.metadata.sources && message.metadata.sources.length > 0 && ( -
-

Sources Referenced

-
- {message.metadata.sources.map((source, index) => ( -
- - {index + 1} - - {source} -
- ))} -
-
- )} -
- )} -
-
- ); - }; - - // Generate unique ID - const generateId = () => { - return `msg_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; - }; - - // Markdown Content Component - const MarkdownContent: React.FC<{ content: string; role: 'user' | 'assistant' }> = ({ content, role }) => { - if (role === 'user') { - // Don't apply markdown to user messages - return
{content}
; - } - - return ( -
-

{children}

, - h2: ({ children }) =>

{children}

, - h3: ({ children }) =>

{children}

, - p: ({ children }) =>

{children}

, - ul: ({ children }) =>
    {children}
, - ol: ({ children }) =>
    {children}
, - li: ({ children }) =>
  • {children}
    • , - code: ({ children, className, ...props }) => { - const inline = !className?.includes('language-'); - return inline ? ( - - {children} - - ) : ( - 
    -                  
-                    {children}
-                  
-
    - ); - }, - blockquote: ({ children }) => ( -
    - {children} -
    - ), - table: ({ children }) => ( - - {children} -
    - ), - th: ({ children }) => ( - - {children} - - ), - td: ({ children }) => ( - - {children} - - ), - strong: ({ children }) => {children}, - em: ({ children }) => {children}, - a: ({ children, href }) => ( - e.currentTarget.style.textDecoration = 'underline'} - onMouseOut={(e) => e.currentTarget.style.textDecoration = 'none'} - > - {children} - - ) - }} - > - {content} -     -
    - ); - }; - - // Get available models for current provider - const getAvailableModels = () => { - return llmService.getAvailableModels(llmSettings.provider); - }; - - const containerStyle: React.CSSProperties = { - display: 'flex', - flexDirection: 'column', - height: '600px', - border: '1px solid #ddd', - borderRadius: '8px', - backgroundColor: '#fff', - ...style - }; - - const headerStyle: React.CSSProperties = { - padding: '16px', - borderBottom: '1px solid #eee', - backgroundColor: '#f8f9fa', - borderRadius: '8px 8px 0 0', - display: 'flex', - justifyContent: 'space-between', - alignItems: 'center' - }; - - const messagesStyle: React.CSSProperties = { - flex: 1, - overflowY: 'auto', - padding: '16px', - display: 'flex', - flexDirection: 'column', - gap: '16px' - }; - - const inputAreaStyle: React.CSSProperties = { - padding: '16px', - borderTop: '1px solid #eee' - }; - - const messageStyle = (role: 'user' | 'assistant'): React.CSSProperties => ({ - padding: '12px 16px', - borderRadius: '12px', - maxWidth: '80%', - alignSelf: role === 'user' ? 'flex-end' : 'flex-start', - backgroundColor: role === 'user' ? '#007bff' : '#f1f3f4', - color: role === 'user' ? '#fff' : '#333', - wordWrap: 'break-word' - }); - - const buttonStyle: React.CSSProperties = { - padding: '8px 16px', - border: 'none', - borderRadius: '4px', - backgroundColor: '#007bff', - color: '#fff', - cursor: 'pointer', - fontSize: '14px' - }; - - const textareaStyle: React.CSSProperties = { - width: '100%', - minHeight: '60px', - padding: '12px', - border: '1px solid #ddd', - borderRadius: '4px', - resize: 'vertical', - fontSize: '14px', - fontFamily: 'inherit' - }; - - return ( -
    - {/* Header */} -
    -
    - ๐Ÿ’ฌ - Code Assistant - - {llmService.getProviderDisplayName(llmSettings.provider)} - -
    - -
    - - - - - -
    -
    - - {/* Settings Panel */} - {showSettings && ( -
    -

    LLM Configuration

    - -
    -
    - - -
    - -
    - - {llmSettings.provider === 'azure-openai' ? ( - setLLMSettings(prev => ({ ...prev, model: e.target.value }))} - placeholder="gpt-4.1-mini-v2" - style={{ width: '100%', padding: '6px', fontSize: '14px' }} - /> - ) : ( - - )} -
    -
    - -
    - - setLLMSettings(prev => ({ ...prev, apiKey: e.target.value }))} - placeholder={ - llmSettings.provider === 'azure-openai' ? 'Your Azure OpenAI key...' : - llmSettings.provider === 'anthropic' ? 'sk-ant-...' : - llmSettings.provider === 'gemini' ? 'Your Google API key...' : 'sk-...' - } - style={{ width: '100%', padding: '6px', fontSize: '14px' }} - /> -
    - - {/* Azure OpenAI Specific Fields */} - {llmSettings.provider === 'azure-openai' && ( - <> -
    - - setLLMSettings(prev => ({ ...prev, azureOpenAIEndpoint: e.target.value }))} - placeholder="https://your-resource.openai.azure.com" - style={{ width: '100%', padding: '6px', fontSize: '14px' }} - /> -
    - -
    - - setLLMSettings(prev => ({ ...prev, azureOpenAIApiVersion: e.target.value }))} - placeholder="2024-02-01" - style={{ width: '100%', padding: '6px', fontSize: '14px' }} - /> -
    - - )} - -
    -
    - - setLLMSettings(prev => ({ ...prev, temperature: parseFloat(e.target.value) }))} - style={{ width: '100%' }} - /> -
    - -
    - - setLLMSettings(prev => ({ ...prev, maxTokens: parseInt(e.target.value) }))} - style={{ width: '100%', padding: '6px', fontSize: '14px' }} - /> -
    -
    - - {/* Save Button */} -
    - - -
    -
    - )} - - {/* Messages */} -
    - {messages.length === 0 && ( -
    -
    ๐Ÿ’ฌ
    -
    Ask me anything about the codebase!
    -
    - I can help you understand functions, classes, dependencies, and more. -
    -
    - )} - - {messages.map((message) => ( -
    -
    -
    - -
    - - {/* Debug Panel (when debug mode is enabled) */} - {debugMode && message.role === 'assistant' && ( - - )} - - {/* Simple Metadata (when debug mode is disabled) */} - {!debugMode && message.metadata && ( -
    - {message.metadata.confidence && ( -
    - Confidence: {Math.round(message.metadata.confidence * 100)}% -
    - )} - - {message.metadata.sources && message.metadata.sources.length > 0 && ( -
    - Sources: {message.metadata.sources.join(', ')} -
    - )} - - {message.metadata.cypherQueries && message.metadata.cypherQueries.length > 0 && ( -
    - View Queries ({message.metadata.cypherQueries.length}) - {message.metadata.cypherQueries.map((query, index) => ( -
    -
    Query: {query.cypher}
    -
    Explanation: {query.explanation}
    -
    - ))} -
    - )} - - {message.metadata.reasoning && message.metadata.reasoning.length > 0 && ( -
    - View Reasoning ({message.metadata.reasoning.length} steps) - {message.metadata.reasoning.map((step, index) => ( -
    -
    Step {step.step}: {step.thought}
    -
    Action: {step.action}
    -
    - ))} -
    - )} -
    - )} -
    - -
    - {message.timestamp.toLocaleTimeString()} -
    -
    - ))} - - {isLoading && ( -
    -
    -
    - Thinking... -
    -
    - )} - -
    -
    - - {/* Input Area */} -
    -
    -