diff --git a/.env b/.env new file mode 100644 index 000000000..2a4d8805e --- /dev/null +++ b/.env @@ -0,0 +1,95 @@ + +# ======================================== +# ENGINE CONFIGURATION +# ======================================== + +# Default engine to use ('legacy' or 'nextgen') +ENGINE_DEFAULT=nextgen + +# Enable/disable engines +ENGINE_LEGACY_ENABLED=true +ENGINE_NEXTGEN_ENABLED=true + +# Allow fallback between engines if one fails +ENGINE_ALLOW_FALLBACK=true + +# Enable performance monitoring +ENGINE_PERFORMANCE_MONITORING=true + +# Auto engine selection based on repository size (experimental) +ENGINE_AUTO_SELECTION=false + +# Time threshold before falling back to another engine (ms) +ENGINE_FALLBACK_THRESHOLD_MS=10000 + +# ======================================== +# LEGACY ENGINE CONFIGURATION +# ======================================== + +# Memory limits for Legacy engine +ENGINE_LEGACY_MEMORY_LIMIT_MB=512 +ENGINE_LEGACY_GC_INTERVAL_MS=30000 + +# Processing settings for Legacy engine +ENGINE_LEGACY_BATCH_SIZE=10 +ENGINE_LEGACY_TIMEOUT_MS=30000 +ENGINE_LEGACY_USE_WORKERS=true + +# ======================================== +# NEXT-GEN ENGINE CONFIGURATION +# ======================================== + +# KuzuDB settings +ENGINE_NEXTGEN_KUZU_DB_PATH=gitnexus.kuzu +ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=256 +ENGINE_NEXTGEN_KUZU_ENABLE_WAL=true +ENGINE_NEXTGEN_KUZU_ENABLE_COMPRESSION=true + +# Parallel processing settings +ENGINE_NEXTGEN_MAX_WORKERS=4 +ENGINE_NEXTGEN_BATCH_SIZE=20 +ENGINE_NEXTGEN_WORKER_TIMEOUT_MS=60000 +ENGINE_NEXTGEN_ENABLE_PARALLEL_PARSING=true + +# ======================================== +# GENERAL APPLICATION CONFIGURATION +# ======================================== + +# GitHub token for API access (increases rate limits) +GITHUB_TOKEN=your_github_token_here + +# Logging level +LOG_LEVEL=info + +# Enable performance metrics +LOG_ENABLE_METRICS=true +LOG_ENABLE_PERFORMANCE=true + +# Memory configuration +MEMORY_MAX_MB=512 +MEMORY_CLEANUP_THRESHOLD_MB=400 + +# ======================================== +# EXAMPLES AND RECOMMENDED CONFIGURATIONS +# ======================================== + +# For large repositories (prefer Next-Gen engine): +# ENGINE_DEFAULT=nextgen +# ENGINE_NEXTGEN_MAX_WORKERS=8 +# ENGINE_NEXTGEN_BATCH_SIZE=50 +# ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=512 + +# For stable processing (prefer Legacy engine): +# ENGINE_DEFAULT=legacy +# ENGINE_LEGACY_MEMORY_LIMIT_MB=1024 +# ENGINE_LEGACY_BATCH_SIZE=5 + +# For development/testing (enable both with fallback): +# ENGINE_DEFAULT=nextgen +# ENGINE_ALLOW_FALLBACK=true +# ENGINE_PERFORMANCE_MONITORING=true + +# For production (disable experimental features): +# ENGINE_AUTO_SELECTION=false +# ENGINE_NEXTGEN_ENABLE_PARALLEL_PARSING=false +# LOG_LEVEL=warn \ No newline at end of file diff --git a/ARCHITECTURE_UPGRADE.md b/ARCHITECTURE_UPGRADE.md deleted file mode 100644 index 0d211f4d3..000000000 --- a/ARCHITECTURE_UPGRADE.md +++ /dev/null @@ -1,170 +0,0 @@ -# ๐Ÿš€ Knowledge Graph Generation Architecture Upgrade - -## Overview - -The knowledge graph generation pipeline has been completely restructured to address two critical weaknesses: - -1. **Inaccurate Call Resolution** - Previous monolithic approach lacked project-wide visibility -2. **Inefficient Definition Lookups** - Simple map-based storage limited advanced resolution heuristics - -## ๐Ÿ—๏ธ New Architecture: 4-Pass Decoupled Pipeline - -### **Pass 1: Structure Analysis** ๐Ÿ“ -- **Processor**: `StructureProcessor` -- **Purpose**: Build project hierarchy (folders, files) -- **Output**: Basic graph structure with CONTAINS relationships - -### **Pass 2: Code Parsing & Definition Extraction** ๐Ÿ” -- **Processor**: `ParsingProcessor` (Enhanced) -- **Purpose**: Parse source code and extract definitions -- **Key Enhancement**: Populates `FunctionRegistryTrie` for efficient lookups -- **Output**: Function/class/method nodes + optimized search structure - -### **Pass 3: Import Resolution** ๐Ÿ”— -- **Processor**: `ImportProcessor` (NEW) -- **Purpose**: Build comprehensive project-wide import map -- **Key Features**: - - Resolves all aliases and relative paths - - Handles Python, JavaScript, TypeScript imports - - Creates accurate IMPORTS relationships -- **Output**: Complete dependency graph + import map - -### **Pass 4: Call Resolution** ๐Ÿ“ž -- **Processor**: `CallProcessor` (Completely Rewritten) -- **Purpose**: Resolve function calls using 3-stage strategy -- **Input**: Import map + Function registry trie -- **Output**: Accurate CALLS relationships - -## ๐Ÿง  Key Innovations - -### 1. FunctionRegistryTrie (`src/core/graph/trie.ts`) - -**Purpose**: Optimized data structure for function definition lookups - -**Key Features**: -- **Suffix-based search**: `findEndingWith(name)` for heuristic matching -- **Qualified names**: Stores full paths like `myProject.services.api.fetchUser` -- **Import distance calculation**: Smart scoring for best match selection - -**Example Usage**: -```typescript -// Find all functions ending with "fetchUser" across the project -const candidates = trie.findEndingWith("fetchUser"); -// Returns: [ -// { qualifiedName: "services.api.fetchUser", filePath: "services/api.py" }, -// { qualifiedName: "utils.cache.fetchUser", filePath: "utils/cache.js" } -// ] -``` - -### 2. ImportProcessor (`src/core/ingestion/import-processor.ts`) - -**Purpose**: Dedicated import resolution with project-wide visibility - -**Key Features**: -- **Language Support**: Python (`import`, `from...import`) and JS/TS (`import`, `require`) -- **Path Resolution**: Handles relative imports (`.`, `..`) and absolute imports -- **Alias Tracking**: Maps local names to actual exported functions -- **Validation**: Checks against actual project files - -**Example Output**: -```typescript -importMap = { - "src/api.js": { - "fetchUser": { - targetFile: "src/services/user.js", - exportedName: "fetchUser", - importType: "named" - } - } -} -``` - -### 3. Advanced Call Resolution Strategy - -**3-Stage Resolution Process**: - -#### Stage 1: Exact Match (High Confidence) -- Uses import map for direct resolution -- Example: `import { fetchUser } from './services'` โ†’ Direct link to `services/fetchUser` - -#### Stage 2: Same-Module Match (High Confidence) -- Checks for function definitions within the same file -- Example: Local function calls within a module - -#### Stage 3: Heuristic Fallback (Intelligent Guessing) -- Uses `FunctionRegistryTrie.findEndingWith()` to find candidates -- Applies **import distance** algorithm for best match -- **Distance Formula**: `max(caller_parts, candidate_parts) - common_prefix_length` -- **Sibling Bonus**: -1 for functions in same parent directory - -**Example Heuristic Resolution**: -``` -Call: fetchUser() in "src/components/UserList.js" -Candidates found: -- src/services/user.js:fetchUser (distance: 2) -- src/utils/api.js:fetchUser (distance: 2) -- src/components/utils.js:fetchUser (distance: 1) โ† SELECTED (sibling bonus) -``` - -## ๐Ÿ“Š Performance & Accuracy Improvements - -### Resolution Statistics -The new CallProcessor provides detailed statistics: -- **Exact matches** (Stage 1): Highest confidence -- **Same-file matches** (Stage 2): High confidence -- **Heuristic matches** (Stage 3): Medium confidence with distance scoring -- **Failed resolutions**: Tracked for debugging - -### Expected Improvements -- **๐ŸŽฏ Higher Accuracy**: Project-wide visibility eliminates cross-file resolution failures -- **โšก Better Performance**: Trie-based lookups vs linear searches -- **๐Ÿ” Smarter Heuristics**: Distance-based scoring for ambiguous cases -- **๐Ÿ“ˆ Detailed Metrics**: Comprehensive resolution statistics - -## ๐Ÿ”ง Technical Implementation Details - -### File Structure -``` -src/core/ -โ”œโ”€โ”€ graph/ -โ”‚ โ””โ”€โ”€ trie.ts # FunctionRegistryTrie implementation -โ”œโ”€โ”€ ingestion/ -โ”‚ โ”œโ”€โ”€ pipeline.ts # Updated 4-pass orchestration -โ”‚ โ”œโ”€โ”€ structure-processor.ts -โ”‚ โ”œโ”€โ”€ parsing-processor.ts # Enhanced with trie population -โ”‚ โ”œโ”€โ”€ import-processor.ts # NEW: Dedicated import resolution -โ”‚ โ””โ”€โ”€ call-processor.ts # Completely rewritten -``` - -### Integration Points -1. **ParsingProcessor** populates the trie during definition extraction -2. **ImportProcessor** builds the complete import map -3. **CallProcessor** uses both trie and import map for resolution -4. **Pipeline** orchestrates the sequence with proper data flow - -### Browser Compatibility -- Custom path utilities replace Node.js `path` module -- All processors work in browser environment -- Maintains existing WASM tree-sitter integration - -## ๐Ÿš€ Usage - -The new architecture is fully integrated into the existing pipeline. No changes required for: -- UI components -- Worker integration -- Export functionality -- Statistics display - -The system automatically uses the new 4-pass architecture for all repository processing. - -## ๐ŸŽฏ Results - -This architecture upgrade transforms the knowledge graph generation from a basic parser into an intelligent code analysis system capable of: - -- **Accurate cross-file call resolution** -- **Smart import dependency tracking** -- **Heuristic-based intelligent guessing** -- **Comprehensive project-wide visibility** -- **Detailed resolution analytics** - -The result is a significantly more accurate and comprehensive knowledge graph that truly represents the structure and relationships within a codebase. \ No newline at end of file diff --git a/Agent.md b/Agent.md deleted file mode 100644 index e9f460513..000000000 --- a/Agent.md +++ /dev/null @@ -1,387 +0,0 @@ -# GitNexus - Complete Agent Documentation - -## ๐ŸŽฏ Project Overview - -**GitNexus** is a client-side, edge-based code knowledge graph generator that transforms any codebase into an interactive knowledge graph with AI-powered Graph RAG capabilities. It runs entirely in the browser with zero server dependencies. - -### Core Mission - -- **Zero-Setup Code Intelligence**: Analyze codebases without servers or configuration -- **Graph RAG-Powered**: Use knowledge graphs for AI-powered code understanding -- **Multi-Language Support**: Currently Python-focused with extensible architecture -- **Browser-Native**: All processing happens client-side using WebAssembly and Web Workers - -### Key Capabilities - -- **GitHub Integration**: Direct repository analysis via GitHub API -- **ZIP Processing**: Local archive analysis with intelligent filtering -- **Interactive Visualization**: Cytoscape.js-powered knowledge graphs -- **AI Chat Interface**: Multi-LLM support (OpenAI, Anthropic, Gemini) -- **Advanced Parsing**: Tree-sitter WASM for accurate AST analysis - -## ๐Ÿ—๏ธ Architecture Deep Dive - -### System Architecture Layers - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ User Interface Layer โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ HomePage โ”‚ โ”‚ Chat UI โ”‚ โ”‚ Graph Explorer โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Service Layer โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ GitHub API โ”‚ โ”‚ ZIP Service โ”‚ โ”‚ Ingestion Service โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Processing Pipeline โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Structure โ”‚ โ”‚ Parsing โ”‚ โ”‚ Call Resolution โ”‚ โ”‚ -โ”‚ โ”‚ Processor โ”‚ โ”‚ Processor โ”‚ โ”‚ 3-Stage Strategy โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค -โ”‚ Core Engine โ”‚ -โ”‚ โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”‚ -โ”‚ โ”‚ Graph โ”‚ โ”‚ Function โ”‚ โ”‚ Import โ”‚ โ”‚ -โ”‚ โ”‚ Types โ”‚ โ”‚ Registry โ”‚ โ”‚ Resolution โ”‚ โ”‚ -โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ (Trie) โ”‚ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### Technology Stack - -**Frontend Framework**: React 18 + TypeScript + Vite -**Graph Visualization**: Cytoscape.js + d3.js -**Code Parsing**: Tree-sitter WebAssembly -**AI Integration**: LangChain.js with ReAct pattern -**State Management**: React Context + custom hooks -**Build System**: Vite with WASM support - -## ๐Ÿ“Š 4-Pass Processing Pipeline - -### Pass 1: Structure Analysis (`StructureProcessor`) - -**Purpose**: Discover complete repository structure without filtering - -**Key Innovations**: - -- **Complete Discovery**: Finds ALL directories and files (including ignored ones) -- **No Early Filtering**: Preserves complete structure for accurate representation -- **Smart Categorization**: Distinguishes files from directories algorithmically -- **Intermediate Paths**: Automatically discovers missing directory levels - -**Implementation Details**: - -```typescript -// Direct path processing instead of inference -const { directories, files } = this.categorizePaths(allPaths); -``` - -### Pass 2: Code Parsing & Definition Extraction (`ParsingProcessor`) - -**Purpose**: Parse source code and extract definitions while applying intelligent filtering - -**Key Components**: - -- **Tree-sitter Integration**: WASM-based parsing for multiple languages -- **Function Registry Trie**: Optimized data structure for definition lookups -- **Two-Stage Filtering**: - - Stage 1: Prune ignored directories (node_modules, .git, etc.) - - Stage 2: Apply user filters (directory patterns, file extensions) - -**Ignore Patterns**: - -```typescript -IGNORE_PATTERNS = [ - '.git', 'node_modules', '__pycache__', '.venv', 'build', 'dist', - '.vscode', '.idea', 'tmp', 'logs', 'coverage' -] -``` - -### Pass 3: Import Resolution (`ImportProcessor`) - -**Purpose**: Build comprehensive project-wide import map - -**Features**: - -- **Multi-language Support**: Python, JavaScript, TypeScript imports -- **Path Resolution**: Handles relative and absolute imports -- **Alias Tracking**: Maps local names to actual exported functions -- **Validation**: Checks against actual project files - -**Resolution Patterns**: - -- Python: `import`, `from...import` -- JS/TS: `import`, `require()`, `export` -- Path normalization for complex project structures - -### Pass 4: Call Resolution (`CallProcessor`) - -**Purpose**: Resolve function calls using 3-stage strategy - -**3-Stage Resolution Strategy**: - -1. **Exact Match** (High Confidence): Uses import map for direct resolution -2. **Same-Module Match** (High Confidence): Local function calls within files -3. **Heuristic Fallback** (Intelligent): Uses FunctionRegistryTrie with distance scoring - -**Heuristic Algorithm**: - -```typescript -// Distance-based scoring for ambiguous calls -const distance = max(caller_parts, candidate_parts) - common_prefix_length -const score = distance - sibling_bonus -``` - -## ๐Ÿค– AI Integration Architecture - -### ReAct Agent Implementation - -**Pattern**: Reasoning + Acting for complex code queries - -**Agent Components**: - -- **LLM Service**: Multi-provider support (OpenAI, Anthropic, Gemini) -- **Cypher Generator**: Natural language to graph query translation -- **Tool System**: Graph queries, code retrieval, file search -- **Memory Management**: Configurable conversation history - -### Available Tools - -1. **query_graph**: Execute Cypher queries on knowledge graph -2. **get_code**: Retrieve specific code snippets -3. **search_files**: Find files by name or content patterns -4. **get_file_content**: Get complete file contents - -### Debug Mode Features - -- **Reasoning Steps**: Complete ReAct process visualization -- **Cypher Queries**: Generated queries with explanations -- **Configuration**: LLM settings and performance metrics -- **Context Info**: Graph statistics and source attribution - -## ๐Ÿ”ง Service Layer Details - -### GitHub Service (`src/services/github.ts`) - -**Purpose**: GitHub API integration with rate limiting and error handling - -**Key Features**: - -- **Rate Limit Handling**: 5,000 requests/hour with token, 60 without -- **Error Recovery**: Comprehensive error handling with user-friendly messages -- **Authentication**: Personal access token support -- **Content Retrieval**: Efficient file and directory fetching - -**API Methods**: - -```typescript -getRepositoryContents(owner, repo, path) // Directory structure -getFileContent(owner, repo, path) // Individual file content -downloadFileRaw(owner, repo, path) // Raw file download -``` - -### ZIP Service (`src/services/zip.ts`) - -**Purpose**: Local archive processing with complete structure discovery - -**Features**: - -- **Complete Structure**: Extracts all paths regardless of filtering -- **Memory Efficient**: Streaming processing for large archives -- **Path Normalization**: Handles common top-level folder removal -- **Content Mapping**: Efficient Map for file contents - -### Ingestion Service (`src/services/ingestion.service.ts`) - -**Purpose**: Orchestrate the complete ingestion pipeline - -**Orchestration Methods**: - -```typescript -processGitHubRepo(url, options) // GitHub repository processing -processZipFile(file, options) // ZIP archive processing -``` - -## ๐Ÿ“ˆ Data Models & Types - -### Core Graph Types - -```typescript -interface KnowledgeGraph { - nodes: GraphNode[] - relationships: Relationship[] -} - -interface GraphNode { - id: string - label: 'Project' | 'Folder' | 'File' | 'Function' | 'Class' | 'Method' | 'Variable' - properties: Record -} - -interface Relationship { - id: string - type: 'CONTAINS' | 'CALLS' | 'IMPORTS' | 'DECORATES' - source: string - target: string - properties: Record -} -``` - -### Function Registry Trie - -**Purpose**: Optimized function definition lookups - -**Key Features**: - -- **Suffix-based search**: `findEndingWith(name)` for heuristic matching -- **Qualified names**: Full paths like `myProject.services.api.fetchUser` -- **Import distance**: Smart scoring for best match selection - -## ๐ŸŽจ User Interface Architecture - -### Component Structure - -``` -App.tsx -โ”œโ”€โ”€ HomePage.tsx (Main application page) -โ”œโ”€โ”€ GraphExplorer.tsx (Interactive graph visualization) -โ”œโ”€โ”€ ChatInterface.tsx (AI chat with debug mode) -โ”œโ”€โ”€ SourceViewer.tsx (Code display with syntax highlighting) -โ””โ”€โ”€ ErrorBoundary.tsx (Comprehensive error handling) -``` - -### Interactive Features - -- **Graph Navigation**: Node selection, zooming, panning -- **Real-time Progress**: Live updates during processing -- **Split-Panel Layout**: Graph + chat interface -- **Settings Management**: Persistent configuration -- **Export Functionality**: JSON export with metadata - -## ๐Ÿ” Performance Optimization - -### Processing Optimizations - -- **Web Workers**: Background processing to keep UI responsive -- **Intelligent Filtering**: Skip massive directories (node_modules, .git) -- **Batch Processing**: Chunked processing for large repositories -- **Memory Management**: Configurable file limits (default: 500 files) - -### Graph Optimization - -- **Node Limiting**: Smart truncation for large graphs -- **Relationship Pruning**: Focus on high-confidence connections -- **Caching**: AST and processing result caching -- **Lazy Loading**: On-demand content loading - -## ๐Ÿ›ก๏ธ Error Handling & Reliability - -### Error Boundaries - -- **Component-level**: Graceful degradation for UI components -- **Worker-level**: Web Worker error recovery -- **Service-level**: API and processing error handling - -### User Experience - -- **Progress Indicators**: Detailed phase-specific messaging -- **Confirmation Dialogs**: Smart warnings for expensive operations -- **Recovery Options**: Clear guidance for error resolution -- **Debug Information**: Comprehensive logging for troubleshooting - -## ๐Ÿ“Š Development Setup - -### Prerequisites - -- **Node.js 18+** and **npm/yarn** -- **GitHub Token** (optional, increases rate limits) -- **AI API Keys**: OpenAI, Anthropic, or Gemini - -### Installation - -```bash -npm install -npm run dev # Development server on http://localhost:5173 -npm run build # Production build -``` - -### Configuration - -- **GitHub Token**: Settings โ†’ GitHub Token -- **AI Keys**: Settings โ†’ AI Provider Configuration -- **Performance**: Settings โ†’ File limits and filtering - -## ๐ŸŽฏ Usage Patterns - -### GitHub Repository Analysis - -1. **URL Input**: Enter GitHub repository URL -2. **Filtering**: Optional directory and file extension filters -3. **Processing**: 4-pass pipeline with progress tracking -4. **Exploration**: Interactive graph with AI chat - -### ZIP Archive Analysis - -1. **File Upload**: Select local ZIP archive -2. **Configuration**: Set processing limits and filters -3. **Analysis**: Complete repository structure discovery -4. **Results**: Knowledge graph with code intelligence - -### Best Practices - -- **Start Small**: Begin with focused directories -- **Use Filters**: Exclude dependencies and build artifacts -- **Monitor Progress**: Watch console for processing insights -- **Leverage AI**: Use chat interface for code exploration - -## ๐Ÿ”ฎ Future Enhancements - -### Language Support - -- **JavaScript/TypeScript**: Enhanced parsing and analysis -- **Java**: Class and method relationship mapping -- **C++**: Template and inheritance analysis -- **Go**: Package and interface resolution - -### Advanced Features - -- **Code Metrics**: Complexity and quality analysis -- **Security Scanning**: Vulnerability detection -- **Documentation Generation**: Auto-generated docs -- **Team Collaboration**: Shared graph exploration - -### Performance Improvements - -- **Incremental Processing**: Update existing graphs -- **Distributed Processing**: Multiple worker threads -- **Caching Layer**: Persistent processing cache -- **Streaming Analysis**: Real-time code changes - ---- - -## ๐Ÿ“ Quick Reference - -### Key Files - -- **Main Entry**: `src/App.tsx` -- **Pipeline**: `src/core/ingestion/pipeline.ts` -- **Services**: `src/services/` -- **AI Logic**: `src/ai/` -- **UI Components**: `src/ui/components/` - -### Debug Commands - -- **Enable Debug**: Click "๐Ÿ” Debug" in chat interface -- **Check Console**: F12 โ†’ Console for processing logs -- **Diagnose Issues**: Use "๐Ÿฉบ Diagnose" button - -### Common Issues - -- **Rate Limits**: Add GitHub token for higher limits -- **Large Repos**: Adjust file limits in settings -- **Parsing Errors**: Check file syntax and extensions -- **Memory Issues**: Reduce processing scope with filters - -This documentation provides complete context for any agent working on GitNexus, from architecture understanding to implementation details and troubleshooting guidance. diff --git a/CONVERSION_SUMMARY.md b/CONVERSION_SUMMARY.md deleted file mode 100644 index bb58b246e..000000000 --- a/CONVERSION_SUMMARY.md +++ /dev/null @@ -1,54 +0,0 @@ -# Deno to Node.js Conversion Summary - -## Overview -Successfully converted the GitNexus repository from Deno to Node.js while maintaining all functionality. - -## Changes Made - -### 1. Configuration Files -- **Removed**: `deno.json`, `deno.lock` -- **Updated**: `package.json` with all dependencies from `deno.json` - - Added all npm dependencies: jszip, axios, cytoscape, web-tree-sitter, langchain packages, etc. - - Updated version to 1.0.0 - - Kept existing build scripts (Vite-based) - -### 2. Import Statements -- **Removed**: All `npm:` prefixes from import statements -- **Removed**: All `@ts-expect-error` comments related to npm: imports -- **Files affected**: 13+ TypeScript files across the codebase - -### 3. Dependencies Successfully Converted -- `react` & `react-dom` (already present) -- `jszip` for ZIP file processing -- `axios` for HTTP requests -- `cytoscape` & `cytoscape-dagre` for graph visualization -- `web-tree-sitter` for code parsing -- `comlink` for web workers -- `@langchain/*` packages for AI functionality -- `zod` for schema validation - -### 4. Build System -- **Unchanged**: Vite configuration remains the same -- **Unchanged**: TypeScript configuration -- **Working**: Development server starts successfully on port 5173 -- **Note**: Some TypeScript errors remain but don't prevent the dev server from running - -## Current Status -โœ… **Development server running** - The application starts and runs on Node.js -โœ… **All dependencies installed** - npm install completed successfully -โœ… **Import statements fixed** - All Deno-style imports converted to Node.js style -โš ๏ธ **TypeScript errors** - Some type errors remain but don't block functionality - -## Next Steps (Optional) -The conversion is complete and functional, but to achieve a clean build: -1. Fix TypeScript errors in langchain imports -2. Update type definitions for cytoscape -3. Fix unused variable warnings -4. Address JSZip type compatibility issues - -## Files Modified -- `package.json` - Added all dependencies -- 13+ TypeScript files - Removed npm: prefixes and Deno comments -- Removed `deno.json` and `deno.lock` - -The repository is now fully converted to Node.js and ready for development! \ No newline at end of file diff --git a/DEBUG_FEATURES.md b/DEBUG_FEATURES.md deleted file mode 100644 index 85dedccac..000000000 --- a/DEBUG_FEATURES.md +++ /dev/null @@ -1,111 +0,0 @@ -# ๐Ÿ” Debug Mode Features - -The GitNexus chat interface now includes a comprehensive debug mode that shows the internal workings of the Graph RAG agent. - -## ๐Ÿ“ Markdown Formatting - -**NEW**: The chat interface now supports full markdown formatting for better readability! - -### Supported Markdown Features: -- **Headers** (# ## ###) for organizing information -- **Bold** and *italic* text for emphasis -- `Inline code` for function names and file paths -- Code blocks with syntax highlighting for multiple languages -- Bullet points and numbered lists -- Tables for structured data -- Blockquotes for important notes -- Links (automatically open in new tabs) - -### Enhanced Debug Display: -- **Reasoning observations** are now rendered with markdown -- **Query explanations** support formatted text -- **Tool outputs** preserve formatting and structure -- **Code snippets** get proper syntax highlighting - -## How to Use Debug Mode - -1. **Toggle Debug Mode**: Click the `๐Ÿ” Debug` button in the chat interface header -2. **Ask Questions**: When debug mode is enabled, all assistant responses will include detailed debug information -3. **Explore Tabs**: The debug panel includes multiple tabs showing different aspects of the processing - -## Debug Panel Tabs - -### ๐Ÿง  Reasoning Steps -Shows the complete ReAct (Reasoning + Acting) process: -- **Step-by-step thinking**: See how the agent reasons about your question -- **Actions taken**: View which tools the agent decides to use -- **Tool inputs**: See the exact parameters passed to each tool -- **Observations**: View the results returned by each tool -- **Success/failure status**: Monitor tool execution success - -### ๐Ÿ” Cypher Queries -Displays generated graph queries: -- **Generated Cypher**: See the exact database queries created -- **Query explanations**: Understand why each query was generated -- **Confidence scores**: View how confident the system is in each query -- **Syntax highlighting**: Cypher queries displayed with proper formatting - -### โš™๏ธ Configuration -Shows system configuration: -- **LLM Settings**: Provider, model, temperature, max tokens -- **RAG Options**: Reasoning steps, strict mode, temperature -- **Performance Metrics**: Execution time, confidence scores - -### ๐Ÿ“Š Context Info -Displays knowledge graph statistics: -- **Graph Nodes**: Total number of code entities in the graph -- **Files Indexed**: Number of source files processed -- **Sources Used**: Files referenced in the current response -- **Referenced Sources**: List of specific files used for the answer - -## What You Can Learn - -### Understanding Agent Behavior -- See how the agent breaks down complex questions -- Understand the reasoning process step-by-step -- Monitor which tools are used and why - -### Query Optimization -- View generated Cypher queries to understand graph traversal -- See query confidence scores to assess reliability -- Learn about query patterns for different question types - -### Performance Analysis -- Monitor execution times for different operations -- Understand the relationship between question complexity and processing time -- Identify bottlenecks in the reasoning process - -### Context Awareness -- See how much of your codebase is being used -- Understand which files are most relevant to your questions -- Monitor the scope of graph traversal - -## Debug Mode Benefits - -1. **Transparency**: Complete visibility into AI decision-making -2. **Learning**: Understand how Graph RAG works internally -3. **Debugging**: Identify issues with queries or reasoning -4. **Optimization**: Fine-tune your questions for better results -5. **Trust**: Build confidence through explainable AI - -## Example Debug Output - -When you ask "How many functions are in this project?", debug mode shows: - -**Reasoning Steps:** -1. **Thought**: "I need to count all functions in the project using a graph query" -2. **Action**: query_graph -3. **Input**: "Count all functions in the project" -4. **Observation**: Generated Cypher query and results - -**Generated Query:** -```cypher -MATCH (f:Function) RETURN COUNT(f) -``` - -**Configuration:** -- Model: gpt-4o-mini -- Temperature: 0.1 -- Execution Time: 1,234ms - -This level of detail helps you understand exactly how your question was processed and answered. \ No newline at end of file diff --git a/DEPLOYMENT_GUIDE.md b/DEPLOYMENT_GUIDE.md deleted file mode 100644 index 621984fe4..000000000 --- a/DEPLOYMENT_GUIDE.md +++ /dev/null @@ -1,182 +0,0 @@ -# GitNexus Deployment Guide - -## ๐Ÿš€ Deploying to Vercel - -### Quick Start (Recommended) - -1. **Push your code to GitHub** - ```bash - git add . - git commit -m "Prepare for deployment" - git push origin main - ``` - -2. **Deploy via Vercel Dashboard** - - Go to [vercel.com](https://vercel.com) and sign in - - Click "New Project" - - Import your GitHub repository - - Vercel will auto-detect it's a Vite project - - Click "Deploy" - -### Manual Deployment - -1. **Install Vercel CLI** - ```bash - npm install -g vercel - ``` - -2. **Login to Vercel** - ```bash - vercel login - ``` - -3. **Deploy from your project directory** - ```bash - vercel - ``` - -4. **Follow the prompts:** - - Set up and deploy? `Y` - - Which scope? `[Your account]` - - Link to existing project? `N` - - Project name: `gitnexus` (or your preferred name) - - Directory: `./` (current directory) - - Override settings? `N` - -### Environment Variables - -Set these in your Vercel project dashboard under Settings โ†’ Environment Variables: - -```env -# Optional: Pre-configure API keys for users -VITE_OPENAI_API_KEY=sk-... -VITE_ANTHROPIC_API_KEY=sk-ant-... -VITE_GEMINI_API_KEY=... - -# Performance settings -VITE_DEFAULT_MAX_FILES=500 -VITE_ENABLE_DEBUG_LOGGING=false -``` - -### Build Configuration - -The `vercel.json` file is already configured with: - -- **Build Command**: `npm run build` -- **Output Directory**: `dist` -- **Framework**: `vite` -- **WASM Support**: Proper headers for WebAssembly files -- **CORS Headers**: Required for SharedArrayBuffer support - -### Custom Domain (Optional) - -1. Go to your Vercel project dashboard -2. Navigate to Settings โ†’ Domains -3. Add your custom domain -4. Update DNS records as instructed - -### Deployment Verification - -After deployment, verify: - -1. **Homepage loads** - Should show the GitNexus interface -2. **WASM files load** - Check browser console for WASM loading errors -3. **GitHub integration works** - Test repository analysis -4. **AI features work** - Test chat interface (if API keys configured) - -### Troubleshooting - -#### Build Failures - -**Error**: "Module not found" or TypeScript errors -```bash -# Fix locally first -npm run build -# If successful locally, the issue is resolved -``` - -**Error**: WASM files not found -- Ensure `public/wasm/` directory is included in your repository -- Check that WASM files are not gitignored - -#### Runtime Issues - -**Error**: "SharedArrayBuffer not available" -- This is expected in development -- Vercel automatically sets the required CORS headers - -**Error**: "Cross-origin isolation required" -- The `vercel.json` headers should handle this -- If issues persist, check browser console for specific errors - -#### Performance Issues - -**Slow loading**: -- Check bundle size in Vercel dashboard -- Consider code splitting for large dependencies -- Optimize WASM file loading - -### Monitoring & Analytics - -1. **Vercel Analytics** (Optional) - ```bash - npm install @vercel/analytics - ``` - -2. **Add to your app**: - ```typescript - import { Analytics } from '@vercel/analytics/react'; - - function App() { - return ( - <> - {/* Your app content */} - - - ); - } - ``` - -### Continuous Deployment - -Vercel automatically deploys on: -- Push to `main` branch โ†’ Production -- Push to other branches โ†’ Preview deployments -- Pull requests โ†’ Preview deployments - -### Rollback - -If you need to rollback: -1. Go to Vercel dashboard -2. Navigate to Deployments -3. Find the working deployment -4. Click "Promote to Production" - -### Security Considerations - -- API keys in environment variables are secure -- Client-side code is public (as expected for this app) -- No server-side secrets are exposed -- WASM files are served as static assets - -### Cost Optimization - -Vercel Hobby Plan (Free) includes: -- 100GB bandwidth/month -- 100GB storage -- 100GB function execution time -- Custom domains -- Automatic HTTPS - -For higher usage, consider Vercel Pro ($20/month). - -## ๐ŸŽฏ Next Steps - -1. **Deploy your first version** -2. **Test all features thoroughly** -3. **Set up monitoring** (optional) -4. **Configure custom domain** (optional) -5. **Set up environment variables** for AI features - -Your GitNexus application is now ready for production deployment! ๐Ÿš€ - diff --git a/DUAL_TRACK_IMPLEMENTATION_SUMMARY.md b/DUAL_TRACK_IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index a8dadd2e8..000000000 --- a/DUAL_TRACK_IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,330 +0,0 @@ -# GitNexus Dual-Track System Implementation Summary - -## ๐ŸŽฏ Overview - -Successfully implemented a dual-track processing system for GitNexus that provides: - -1. **Complete separation** between Legacy (Sequential + In-Memory) and Next-Gen (Parallel + KuzuDB) engines -2. **Clean UI refactoring** from monolithic 700+ line HomePage to focused components -3. **Robust fallback system** with logging when Next-Gen fails -4. **Feature toggle capability** for easy engine switching -5. **Maintainable architecture** with clear separation of concerns - -## ๐Ÿ“ New Directory Structure - -``` -src/ -โ”œโ”€โ”€ config/ -โ”‚ โ”œโ”€โ”€ feature-flags.ts # โœ… Enhanced with engine switching -โ”‚ โ””โ”€โ”€ ... -โ”œโ”€โ”€ core/ -โ”‚ โ”œโ”€โ”€ engines/ -โ”‚ โ”‚ โ”œโ”€โ”€ engine-interface.ts # โœ… Common engine interface -โ”‚ โ”‚ โ”œโ”€โ”€ legacy/ -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ legacy-engine.ts # โœ… Wrapper for current system -โ”‚ โ”‚ โ””โ”€โ”€ nextgen/ -โ”‚ โ”‚ โ””โ”€โ”€ nextgen-engine.ts # โœ… Wrapper for parallel+kuzu -โ”‚ โ”œโ”€โ”€ orchestration/ -โ”‚ โ”‚ โ””โ”€โ”€ engine-manager.ts # โœ… Engine switching & fallback -โ”‚ โ””โ”€โ”€ validation/ -โ”‚ โ””โ”€โ”€ dual-track-validation.ts # โœ… System validation tests -โ”œโ”€โ”€ services/ -โ”‚ โ”œโ”€โ”€ facade/ -โ”‚ โ”‚ โ””โ”€โ”€ gitnexus-facade.ts # โœ… Simplified API for UI -โ”‚ โ””โ”€โ”€ ... -โ”œโ”€โ”€ ui/ -โ”‚ โ”œโ”€โ”€ components/ -โ”‚ โ”‚ โ”œโ”€โ”€ engine/ -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ EngineSelector.tsx # โœ… Engine switching UI -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ ProcessingStatus.tsx # โœ… Engine-aware status -โ”‚ โ”‚ โ””โ”€โ”€ repository/ -โ”‚ โ”‚ โ””โ”€โ”€ RepositoryInput.tsx # โœ… GitHub/ZIP input -โ”‚ โ”œโ”€โ”€ hooks/ -โ”‚ โ”‚ โ”œโ”€โ”€ useEngine.ts # โœ… Engine management -โ”‚ โ”‚ โ”œโ”€โ”€ useProcessing.ts # โœ… Processing operations -โ”‚ โ”‚ โ”œโ”€โ”€ useSettings.ts # โœ… Settings management -โ”‚ โ”‚ โ””โ”€โ”€ useGitNexus.ts # โœ… Main application hook -โ”‚ โ””โ”€โ”€ pages/ -โ”‚ โ””โ”€โ”€ HomePage/ -โ”‚ โ”œโ”€โ”€ HomePage.tsx # โœ… New lightweight container -โ”‚ โ””โ”€โ”€ index.ts -``` - -## ๐Ÿ”ง Key Components Implemented - -### 1. Enhanced Feature Flags (`src/config/feature-flags.ts`) - -```typescript -interface FeatureFlags { - // NEW: Engine Selection - processingEngine: ProcessingEngineType; - autoFallbackOnError: boolean; - enablePerformanceComparison: boolean; - // ... existing flags -} - -// NEW: Engine switching methods -featureFlagManager.switchToLegacyEngine(); -featureFlagManager.switchToNextGenEngine(); -featureFlagManager.logEngineFallback(error); -``` - -### 2. Engine Interface (`src/core/engines/engine-interface.ts`) - -```typescript -interface ProcessingEngine { - readonly name: string; - readonly type: ProcessingEngineType; - readonly capabilities: string[]; - - process(input: ProcessingInput): Promise; - validate(): Promise; - cleanup(): Promise; - getStatus(): EngineStatus; -} -``` - -### 3. Legacy Engine Wrapper (`src/core/engines/legacy/legacy-engine.ts`) - -- Wraps existing `IngestionService` -- Uses `GraphPipeline` + `SimpleKnowledgeGraph` -- Capabilities: `['sequential-processing', 'in-memory-storage', 'basic-queries']` - -### 4. Next-Gen Engine Wrapper (`src/core/engines/nextgen/nextgen-engine.ts`) - -- Wraps `KuzuIngestionService` -- Uses `KuzuGraphPipeline` + `ParallelProcessing` + `KuzuKnowledgeGraph` -- Capabilities: `['parallel-processing', 'kuzu-db-storage', 'advanced-queries']` - -### 5. Engine Manager (`src/core/orchestration/engine-manager.ts`) - -```typescript -class EngineManager { - async process(input: ProcessingInput): Promise { - try { - // Try selected engine - return await this.processWithEngine(selectedEngine, input); - } catch (error) { - // Auto-fallback if enabled - if (this.config.autoFallback && selectedEngine === 'nextgen') { - featureFlagManager.logEngineFallback(error.message); - return await this.processWithEngine('legacy', input); - } - throw error; - } - } -} -``` - -### 6. GitNexus Facade (`src/services/facade/gitnexus-facade.ts`) - -Simplified API for UI: - -```typescript -class GitNexusFacade { - async processGitHubRepository(url: string): Promise; - async processZipFile(file: File): Promise; - async switchEngine(engine: ProcessingEngineType): Promise; - getCurrentEngine(): EngineInfo; - getAvailableEngines(): EngineInfo[]; -} -``` - -### 7. UI Components - -#### EngineSelector (`src/ui/components/engine/EngineSelector.tsx`) -- Dropdown for engine selection -- Real-time engine status display -- Performance info toggle - -#### ProcessingStatus (`src/ui/components/engine/ProcessingStatus.tsx`) -- Engine-aware progress display -- Fallback notifications with logging -- Success/error states with metrics - -#### RepositoryInput (`src/ui/components/repository/RepositoryInput.tsx`) -- Tabbed interface (GitHub/ZIP) -- Drag-and-drop ZIP support -- Input validation - -### 8. Custom Hooks - -#### useEngine (`src/ui/hooks/useEngine.ts`) -- Engine switching logic -- Status monitoring -- Performance comparison - -#### useProcessing (`src/ui/hooks/useProcessing.ts`) -- GitHub/ZIP processing -- Progress tracking -- Error handling - -#### useGitNexus (`src/ui/hooks/useGitNexus.ts`) -- Main application state -- Combines all functionality -- Clean API for components - -### 9. New HomePage (`src/ui/pages/HomePage/HomePage.tsx`) - -**Before**: 1161 lines, monolithic -**After**: ~200 lines, focused components - -```typescript -const HomePage = () => { - const { state, engine, processing, settings, ... } = useGitNexus(); - - return ( -
- - - - - -
- ); -}; -``` - -## ๐Ÿš€ How Engine Switching Works - -### 1. UI Selection -```typescript - switchEngine(engine)} - options={[ - { value: 'legacy', label: '๐Ÿ”ง Stable (Sequential + In-Memory)' }, - { value: 'nextgen', label: '๐Ÿš€ Advanced (Parallel + KuzuDB)' } - ]} -/> -``` - -### 2. Feature Flag Update -```typescript -// User selects Next-Gen -switchEngine('nextgen') - โ†’ featureFlagManager.switchToNextGenEngine() - โ†’ Sets: enableKuzuDB=true, enableParallelProcessing=true -``` - -### 3. Processing with Fallback -```typescript -// Engine Manager routes to correct engine -if (engine === 'nextgen') { - try { - return await nextGenEngine.process(input); - } catch (error) { - // ๐Ÿ”„ AUTO FALLBACK WITH LOGGING - featureFlagManager.logEngineFallback(error.message); - console.warn("๐Ÿ”„ Engine Fallback: Next-gen โ†’ Legacy"); - return await legacyEngine.process(input); - } -} -``` - -### 4. User Feedback -```typescript -// UI shows fallback notification - -``` - -## ๐Ÿ“Š Benefits Achieved - -### โœ… **Clean Separation** -- Legacy and Next-Gen systems completely isolated -- No code mixing between engines -- Easy to modify each system independently - -### โœ… **Safe Migration** -- Legacy always works as fallback -- Next-Gen is opt-in with validation -- Auto-fallback prevents data loss - -### โœ… **Better UX** -- Clear engine selection interface -- Real-time status and performance metrics -- Transparent fallback notifications - -### โœ… **Maintainable Code** -- HomePage reduced from 1161 โ†’ ~200 lines -- Logic extracted to focused hooks -- Components have single responsibilities - -### โœ… **Performance Monitoring** -- Engine performance comparison -- Processing time tracking -- Success rate monitoring - -## ๐Ÿ”„ Fallback Logging Implementation - -When Next-Gen engine fails and falls back to Legacy: - -```typescript -// Engine Manager detects failure -catch (error) { - // Log the fallback event - featureFlagManager.logEngineFallback(error.message); - - // Console output: - // ๐Ÿ”„ Engine Fallback: Next-gen engine failed, falling back to legacy engine - // ๐Ÿ”„ Fallback reason: KuzuDB connection failed - // ๐Ÿ”„ Auto-fallback enabled: true - - // UI shows notification - callbacks?.onEngineFailure?.('nextgen', 'legacy', error.message); -} -``` - -## ๐Ÿงช Validation System - -Created comprehensive validation (`src/core/validation/dual-track-validation.ts`): - -1. โœ… Feature flag management -2. โœ… Engine manager initialization -3. โœ… Engine validation -4. โœ… GitNexus facade functionality -5. โœ… Engine switching -6. โœ… Fallback logging -7. โœ… Utility functions - -## ๐ŸŽฏ Usage Examples - -### Switch to Next-Gen Engine -```typescript -await facade.switchEngine('nextgen', 'User wants parallel processing'); -``` - -### Process with Automatic Fallback -```typescript -const result = await facade.processGitHubRepository( - 'https://github.com/user/repo', - { - engine: 'nextgen', - onEngineSwitch: (from, to) => { - console.log(`Fallback: ${from} โ†’ ${to}`); - } - } -); -``` - -### Monitor Engine Performance -```typescript -const comparison = facade.getPerformanceComparison(); -// Shows speedup factor, processing times, etc. -``` - -## ๐Ÿ Conclusion - -The dual-track system is now fully implemented with: - -- **Zero breaking changes** to existing functionality -- **Complete engine separation** for maintainability -- **Robust fallback system** with logging for reliability -- **Clean UI architecture** for better developer experience -- **Performance monitoring** for data-driven decisions - -The system is ready for production use and provides a solid foundation for future engine improvements and migrations. \ No newline at end of file diff --git a/FILTERING_FIX.md b/FILTERING_FIX.md deleted file mode 100644 index 12dbeba57..000000000 --- a/FILTERING_FIX.md +++ /dev/null @@ -1,132 +0,0 @@ -# ๐Ÿ”ง Directory Filtering Fix - .venv and Ignored Directories Hidden - -## ๐Ÿšจ **Issue Identified and Fixed** - -**Problem**: `.venv` and other ignored directories were still appearing in the Knowledge Graph despite filtering implementation. - -**Root Cause**: The two-stage filtering was only filtering **file parsing**, but ignored directory **nodes** were still being created and displayed in the KG. - -## โœ… **Solution Implemented** - -### **Enhanced StructureProcessor** - -#### **Directory Hiding Logic** -```typescript -// Added to StructureProcessor -private shouldHideDirectory(dirPath: string): boolean { - const pathSegments = dirPath.split('/'); - - // Check if any segment matches ignore patterns - const hasIgnoredSegment = pathSegments.some(segment => - StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - return hasIgnoredSegment || this.matchesAdditionalPatterns(dirPath); -} -``` - -#### **Filtered Node Creation** -```typescript -// Filter directories before creating nodes -const visibleDirectories = directories.filter(dir => !this.shouldHideDirectory(dir)); -const hiddenDirectoriesCount = directories.length - visibleDirectories.length; - -console.log(`StructureProcessor: Hiding ${hiddenDirectoriesCount} ignored directories from KG`); - -// Create nodes only for visible directories -const directoryNodes = this.createDirectoryNodes(visibleDirectories); -``` - -#### **Smart Relationship Handling** -```typescript -// Handle files in hidden directories by connecting to nearest visible parent -private findVisibleParent(path: string, projectId: string): string { - if (path === '') return projectId; - - const parentPath = this.getParentPath(path); - const parentId = this.nodeIdMap.get(parentPath); - - if (parentId) { - return parentId; // Found visible parent - } - - // Recursively look for visible parent - return this.findVisibleParent(parentPath, projectId); -} -``` - -## ๐ŸŽฏ **What's Now Hidden from KG** - -### **Directories Completely Hidden** -- โœ… `.venv`, `venv`, `env`, `virtualenv` (Python virtual environments) -- โœ… `node_modules`, `bower_components` (Package dependencies) -- โœ… `.git`, `.svn`, `.hg` (Version control) -- โœ… `build`, `dist`, `out`, `target` (Build outputs) -- โœ… `.vs`, `.vscode`, `.idea` (IDE directories) -- โœ… `__pycache__`, `.pytest_cache` (Python cache) -- โœ… `coverage`, `.coverage` (Test coverage) -- โœ… `.cache`, `.next`, `.nuxt` (Framework cache) -- โœ… `tmp`, `temp`, `logs` (Temporary directories) - -### **Special Handling** -- โœ… `.github` directory **remains visible** (important for workflows) -- โœ… Files in hidden directories connect to nearest visible parent -- โœ… Complete structure discovery still happens (for performance benefits) - -## ๐Ÿ“Š **Before vs After** - -### **Before (The Problem)** -``` -Knowledge Graph showing: -โ”œโ”€โ”€ src/ โœ… Visible -โ”œโ”€โ”€ tests/ โœ… Visible -โ”œโ”€โ”€ .venv/ โŒ Unwanted visibility -โ”œโ”€โ”€ node_modules/ โŒ Unwanted visibility -โ”œโ”€โ”€ __pycache__/ โŒ Unwanted visibility -โ””โ”€โ”€ package.json โœ… Visible -``` - -### **After (Fixed)** -``` -Knowledge Graph showing: -โ”œโ”€โ”€ src/ โœ… Visible -โ”œโ”€โ”€ tests/ โœ… Visible -โ”œโ”€โ”€ .github/ โœ… Visible (important) -โ””โ”€โ”€ package.json โœ… Visible - -Hidden from view: -- .venv/ (and all contents) -- node_modules/ (and all contents) -- __pycache__/ (and all contents) -``` - -## ๐ŸŽฏ **Technical Implementation** - -### **Two-Level Filtering** -1. **StructureProcessor**: Hides directory **nodes** from KG -2. **ParsingProcessor**: Skips **file parsing** in ignored directories - -### **Performance Benefits Maintained** -- โœ… **Complete Discovery**: Still discovers all paths for performance optimization -- โœ… **Smart Filtering**: Skips expensive parsing operations -- โœ… **Clean Visualization**: Users see only relevant directories -- โœ… **Accurate Relationships**: Files connect to appropriate visible parents - -### **Logging Enhanced** -``` -StructureProcessor: Found 1,247 directories and 892 files -StructureProcessor: Hiding 156 ignored directories from KG -StructureProcessor: Created 983 nodes total (156 directories hidden) -``` - -## ๐Ÿš€ **Result** - -**Perfect Fix!** Now: - -1. **โœ… .venv is Hidden**: No longer appears in Knowledge Graph -2. **โœ… Clean Visualization**: Only relevant directories shown -3. **โœ… Performance Maintained**: Still skip expensive parsing operations -4. **โœ… Accurate Structure**: Files properly connected to visible parents -5. **โœ… Comprehensive Coverage**: All common ignored directories hidden - -The directory filtering is now **working correctly** and `.venv` (along with other ignored directories) will no longer clutter the Knowledge Graph! ๐ŸŽ‰ \ No newline at end of file diff --git a/GITHUB_ARCHIVE_IMPLEMENTATION.md b/GITHUB_ARCHIVE_IMPLEMENTATION.md deleted file mode 100644 index cb186bfbb..000000000 --- a/GITHUB_ARCHIVE_IMPLEMENTATION.md +++ /dev/null @@ -1,334 +0,0 @@ -# ๐Ÿš€ GitHub Archive Implementation Guide - -## ๐Ÿ“‹ Overview - -This implementation adds **5-10x faster GitHub repository processing** to GitNexus by using GitHub's archive download feature instead of individual API calls. The system automatically chooses the best method based on repository size and user preferences. - -## ๐ŸŽฏ Key Features - -- **โšก 5-10x Faster Processing**: Archive downloads are much faster than API calls -- **๐Ÿค– Smart Method Selection**: Automatically chooses archive vs API based on repository size -- **๐Ÿ”„ Automatic Fallback**: Falls back to API if archive method fails -- **๐Ÿ“Š Real-time Progress**: Shows detailed progress with method and stage information -- **โš™๏ธ Configurable Settings**: User can control archive preferences and size limits -- **๐ŸŒฟ Branch Support**: Process specific branches instead of just default - -## ๐Ÿ“ File Structure - -``` -src/services/ -โ”œโ”€โ”€ github-archive.ts # Core archive download service -โ”œโ”€โ”€ hybrid-github.ts # Smart method selection service -โ””โ”€โ”€ github.ts # Existing API service - -src/ui/pages/ -โ””โ”€โ”€ HomePage.tsx # Updated with hybrid service integration - -src/lib/ -โ””โ”€โ”€ github-archive-test.ts # Test functions for verification -``` - -## ๐Ÿ”ง Core Components - -### 1. GitHubArchiveService (`src/services/github-archive.ts`) - -**Purpose**: Downloads and processes GitHub repository archives using ZIP downloads. - -**Key Methods**: -- `getRepositoryArchive()` - Main method for downloading and processing -- `checkRepositoryAccess()` - Verify repository exists and is accessible -- `estimateRepositorySize()` - Get repository size before downloading -- `getBranches()` - Get available branches for a repository - -**Features**: -- Progress tracking with detailed stages -- Batch processing to avoid memory issues -- Smart file filtering (skips binaries, large files, common directories) -- Error handling and recovery - -### 2. HybridGitHubService (`src/services/hybrid-github.ts`) - -**Purpose**: Intelligently chooses between archive and API methods. - -**Key Methods**: -- `getRepositoryStructure()` - Main method with smart method selection -- `compareMethods()` - Compare performance between methods -- `getRepositoryViaArchive()` - Archive method implementation -- `getRepositoryViaAPI()` - API method implementation - -**Features**: -- Automatic method selection based on repository size -- Performance comparison and recommendations -- Seamless fallback between methods -- Unified interface for both approaches - -### 3. Updated HomePage (`src/ui/pages/HomePage.tsx`) - -**New Features**: -- Enhanced progress display with method and stage information -- Branch selection input -- Archive method settings in settings modal -- Real-time progress bars and status updates - -## ๐Ÿš€ Usage - -### Basic Usage - -```typescript -import { HybridGitHubService } from './services/hybrid-github.js'; - -const hybridService = HybridGitHubService.getInstance(); - -// Process repository with automatic method selection -const result = await hybridService.getRepositoryStructure( - 'owner', - 'repo', - 'main', - { - preferArchive: true, - maxArchiveSizeMB: 100, - fallbackToAPI: true - }, - (progress) => { - console.log(`${progress.method}: ${progress.stage} - ${progress.progress}%`); - } -); -``` - -### Method Comparison - -```typescript -// Compare methods for a repository -const comparison = await hybridService.compareMethods('facebook', 'react'); -console.log('Recommended method:', comparison.recommended); -console.log('Archive time:', comparison.archive.estimatedTime); -console.log('API time:', comparison.api.estimatedTime); -``` - -### Direct Archive Usage - -```typescript -import { GitHubArchiveService } from './services/github-archive.js'; - -const archiveService = GitHubArchiveService.getInstance(); - -const result = await archiveService.getRepositoryArchive( - 'owner', - 'repo', - 'main', - (progress) => { - console.log(`${progress.stage}: ${progress.progress}%`); - } -); -``` - -## โš™๏ธ Configuration - -### User Settings - -Users can configure archive behavior in the settings modal: - -- **Prefer Fast Archive Method**: Enable/disable archive preference -- **Maximum Archive Size**: Set size limit for archive method (default: 100MB) -- **GitHub Token**: Optional token for higher rate limits - -### Default Behavior - -- **Small repos (< 100MB)**: Use archive method by default -- **Large repos (> 100MB)**: Use API method by default -- **Archive fails**: Automatically fallback to API method -- **No token**: Works with public repositories - -## ๐Ÿ“Š Performance Comparison - -### Archive Method -- **Speed**: 5-10x faster than API -- **Network**: Single ZIP download vs hundreds of API calls -- **Rate Limits**: No API rate limit impact -- **Memory**: Higher memory usage during extraction -- **Best For**: Small to medium repositories - -### API Method -- **Speed**: Slower but more reliable -- **Network**: Multiple API calls -- **Rate Limits**: Subject to GitHub API limits -- **Memory**: Lower memory usage -- **Best For**: Large repositories or when archive fails - -## ๐Ÿงช Testing - -### Browser Console Testing - -```javascript -// Test archive service -testGitHubArchive(); - -// Test hybrid service -testHybridGitHub(); - -// Compare methods for a specific repository -compareMethods('facebook', 'react'); -``` - -### Test Functions Available - -- `testGitHubArchive()` - Test archive service functionality -- `testHybridGitHub()` - Test hybrid service functionality -- `compareMethods(owner, repo)` - Compare methods for a repository - -## ๐Ÿ” Error Handling - -### Common Issues - -1. **Repository Not Found** - - Error: Repository doesn't exist or is private - - Solution: Check URL and repository access - -2. **Archive Too Large** - - Error: Repository exceeds size limit - - Solution: Automatically falls back to API method - -3. **Network Issues** - - Error: Download fails - - Solution: Automatic fallback to API method - -4. **Rate Limiting** - - Error: API rate limit exceeded - - Solution: Use GitHub token or wait for reset - -### Fallback Strategy - -1. Try archive method first (if enabled and size allows) -2. If archive fails, automatically try API method -3. If both fail, show error message to user - -## ๐ŸŽจ UI Enhancements - -### Progress Display - -The UI now shows: -- **Method**: ARCHIVE or API -- **Stage**: downloading, extracting, processing, complete -- **Progress Bar**: Visual progress indicator -- **File Count**: Files processed vs total files - -### Settings Integration - -New settings in the settings modal: -- Archive method preferences -- Size limits -- Performance options - -### Branch Selection - -Users can now specify: -- Custom branch names -- Default branch fallback -- Branch validation - -## ๐Ÿ”ฎ Future Enhancements - -### Planned Features - -1. **Caching**: Cache downloaded archives for faster re-processing -2. **Parallel Downloads**: Download multiple repositories simultaneously -3. **Incremental Updates**: Only download changed files -4. **Advanced Filtering**: More granular file filtering options -5. **Performance Analytics**: Track and optimize performance - -### Potential Improvements - -1. **WebSocket Progress**: Real-time progress updates -2. **Background Processing**: Process repositories in background -3. **Queue Management**: Handle multiple repository requests -4. **Smart Caching**: Intelligent cache invalidation -5. **Performance Metrics**: Detailed performance reporting - -## ๐Ÿ“ˆ Performance Metrics - -### Expected Improvements - -- **Small repos (< 10MB)**: 10x faster -- **Medium repos (10-50MB)**: 5-8x faster -- **Large repos (50-100MB)**: 3-5x faster -- **Very large repos (> 100MB)**: Uses API method - -### Memory Usage - -- **Archive Method**: Higher memory usage during extraction -- **API Method**: Lower memory usage, distributed over time -- **Optimization**: Batch processing prevents memory spikes - -## ๐Ÿ› ๏ธ Troubleshooting - -### Common Problems - -1. **Archive Download Fails** - - Check network connection - - Verify repository is public - - Try API method as fallback - -2. **Memory Issues** - - Reduce batch size in archive service - - Use API method for very large repositories - - Clear browser cache - -3. **Progress Not Updating** - - Check console for errors - - Verify progress callback is working - - Refresh page and try again - -### Debug Mode - -Enable debug logging: -```javascript -localStorage.setItem('debug_github_archive', 'true'); -``` - -## ๐Ÿ“š API Reference - -### GitHubArchiveService - -```typescript -class GitHubArchiveService { - static getInstance(): GitHubArchiveService; - - async getRepositoryArchive( - owner: string, - repo: string, - branch?: string, - onProgress?: (progress: ArchiveProgress) => void - ): Promise; - - async checkRepositoryAccess(owner: string, repo: string): Promise; - async estimateRepositorySize(owner: string, repo: string): Promise; - async getBranches(owner: string, repo: string): Promise; -} -``` - -### HybridGitHubService - -```typescript -class HybridGitHubService { - static getInstance(): HybridGitHubService; - - async getRepositoryStructure( - owner: string, - repo: string, - branch?: string, - options?: HybridOptions, - onProgress?: (progress: HybridProgress) => void - ): Promise; - - async compareMethods(owner: string, repo: string): Promise; - async checkRepositoryAccess(owner: string, repo: string): Promise; - async estimateRepositorySize(owner: string, repo: string): Promise; - async getBranches(owner: string, repo: string): Promise; -} -``` - -## ๐ŸŽ‰ Conclusion - -The GitHub Archive implementation provides significant performance improvements for GitNexus users, especially for small to medium-sized repositories. The hybrid approach ensures reliability while maximizing speed, and the enhanced UI provides better user experience with detailed progress tracking. - -The implementation is production-ready and includes comprehensive error handling, testing, and documentation for future maintenance and enhancement. diff --git a/GITNEXUS_README.md b/GITNEXUS_README.md deleted file mode 100644 index fca2d3247..000000000 --- a/GITNEXUS_README.md +++ /dev/null @@ -1,557 +0,0 @@ -# ๐Ÿ” CodeNexus - Edge Knowledge Graph Creator with Graph RAG - -**Transform any codebase into an interactive knowledge graph in your browser. No servers, no setup - just instant Graph RAG-powered code intelligence.** - -CodeNexus is a client-side knowledge graph creator that runs entirely in your browser. Drop in a GitHub repo or ZIP file, and get an interactive knowledge graph with AI-powered chat interface. Perfect for code exploration, documentation, and understanding complex codebases through Graph RAG (Retrieval-Augmented Generation). - -## โœจ Features - -### ๐Ÿ“Š **Code Analysis & Visualization** -- **GitHub Integration**: Analyze any public GitHub repository directly from URL -- **ZIP File Support**: Upload and analyze local code archives -- **Interactive Knowledge Graph**: Visualize code structure with Cytoscape.js -- **Multi-language Support**: Currently optimized for Python with extensible architecture -- **Smart Filtering**: Directory and file pattern filters to focus analysis scope -- **Performance Optimization**: Configurable file limits with confirmation dialogs for large repositories - -### ๐Ÿค– **AI-Powered Chat Interface** -- **Multiple LLM Providers**: OpenAI, Anthropic (Claude), Google Gemini -- **ReAct Agent Pattern**: Uses proper LangChain ReAct implementation for reasoning -- **Tool-Augmented Responses**: Graph queries, code retrieval, file search -- **Context-Aware**: Maintains conversation history with configurable memory - -### ๐Ÿ”ง **Advanced Processing Pipeline** -- **3-Pass Ingestion Strategy**: - 1. **Structure Analysis**: Project hierarchy and file organization - 2. **Code Parsing**: AST-based extraction using Tree-sitter - 3. **Call Resolution**: Function/method call relationship mapping -- **Web Worker Processing**: Non-blocking UI with progress tracking -- **Intelligent Caching**: AST and processing result optimization -- **Error Resilience**: Comprehensive error boundaries and recovery mechanisms - -### ๐ŸŽจ **Modern UI/UX** -- **Responsive Design**: Adaptive layout for different screen sizes -- **Real-time Progress**: Live updates during repository processing -- **Interactive Graph**: Node selection, zooming, panning -- **Split-Panel Layout**: Graph visualization + AI chat interface -- **Settings Management**: Persistent configuration for API keys and preferences -- **Export Functionality**: Download knowledge graphs as JSON with metadata -- **Performance Controls**: File limits, filtering, and optimization settings - -### ๐Ÿ›ก๏ธ **Reliability & Performance** -- **Error Boundaries**: Graceful error handling with user-friendly recovery options -- **Performance Monitoring**: Real-time processing statistics and export size calculation -- **Memory Management**: Efficient handling of large repositories with configurable limits -- **Progress Tracking**: Detailed progress indicators with phase-specific messaging -- **Confirmation Dialogs**: Smart warnings for potentially expensive operations - -## ๐Ÿ—๏ธ Architecture - -### **Frontend Stack** -- **React 18** with TypeScript -- **Vite** for fast development and building -- **Cytoscape.js** for graph visualization -- **Custom CSS** with modern design patterns -- **Error Boundaries** for robust error handling - -### **Processing Engine** -- **Deno Runtime** for TypeScript execution -- **Tree-sitter WASM** for syntax parsing -- **Web Workers** for background processing -- **Comlink** for worker communication - -### **AI Integration** -- **LangChain.js** with proper ReAct agent implementation -- **Multiple LLM Support**: OpenAI, Anthropic, Gemini -- **Tool-based Architecture**: Graph queries, code retrieval, file search -- **Cypher Query Generation**: Natural language to graph queries - -### **Services Layer** -``` -src/ -โ”œโ”€โ”€ services/ # External API integrations -โ”‚ โ”œโ”€โ”€ github.ts # GitHub REST API client -โ”‚ โ””โ”€โ”€ zip.ts # ZIP file processing -โ”œโ”€โ”€ core/ # Core processing logic -โ”‚ โ”œโ”€โ”€ graph/ # Knowledge graph types -โ”‚ โ”œโ”€โ”€ ingestion/ # 3-pass processing pipeline -โ”‚ โ””โ”€โ”€ tree-sitter/ # Syntax parsing infrastructure -โ”œโ”€โ”€ ai/ # AI and RAG components -โ”‚ โ”œโ”€โ”€ llm-service.ts # Multi-provider LLM client -โ”‚ โ”œโ”€โ”€ cypher-generator.ts # NL to Cypher translation -โ”‚ โ”œโ”€โ”€ orchestrator.ts # Custom ReAct implementation -โ”‚ โ””โ”€โ”€ langchain-orchestrator.ts # Standard LangChain ReAct -โ”œโ”€โ”€ workers/ # Web Worker implementations -โ”œโ”€โ”€ ui/ # React components and pages -โ”‚ โ”œโ”€โ”€ components/ # Reusable UI components -โ”‚ โ”‚ โ”œโ”€โ”€ ErrorBoundary.tsx # Error handling component -โ”‚ โ”‚ โ”œโ”€โ”€ graph/ # Graph visualization components -โ”‚ โ”‚ โ””โ”€โ”€ chat/ # Chat interface components -โ”‚ โ””โ”€โ”€ pages/ # Application pages -โ”œโ”€โ”€ lib/ # Shared utilities -โ”‚ โ””โ”€โ”€ export.ts # Graph export functionality -โ””โ”€โ”€ App.tsx # Main application entry point -``` - -## ๐Ÿš€ Getting Started - -### Prerequisites -- **Node.js 18+** and **npm/yarn** -- **Deno 1.40+** for development -- **API Keys** for AI features (OpenAI, Anthropic, or Gemini) - -### Installation - -1. **Clone the repository** - ```bash - git clone - cd gitnexus - ``` - -2. **Install dependencies** - ```bash - npm install - ``` - -3. **Start development server** - ```bash - npm run dev - ``` - -4. **Open in browser** - ``` - http://localhost:5173 - ``` - -### Configuration - -1. **GitHub Token (Optional)** - - Increases rate limit from 60 to 5,000 requests/hour - - Generate at: https://github.com/settings/tokens - - Requires no special permissions for public repos - -2. **AI API Keys** - - **OpenAI**: Get from https://platform.openai.com/api-keys - - **Anthropic**: Get from https://console.anthropic.com/ - - **Gemini**: Get from https://makersuite.google.com/app/apikey - -3. **Performance Settings** - - **File Limit**: Configure maximum files to process (default: 500) - - **Directory Filters**: Focus on specific directories (e.g., "src", "lib") - - **File Patterns**: Filter by file types (e.g., "*.py", "*.js", "*.ts") - -## ๐Ÿ’ก Usage - -### Analyzing a Repository - -1. **GitHub Repository** - ``` - 1. Enter GitHub URL: https://github.com/owner/repo - 2. Optional: Set directory/file filters to focus analysis - 3. Click "Analyze" - 4. For large repos: Confirm processing or adjust filters - 5. Wait for processing (structure โ†’ parsing โ†’ call resolution) - 6. Explore the interactive graph - ``` - -2. **ZIP File Upload** - ``` - 1. Click "Choose File" and select a .zip file - 2. Optional: Configure filters before processing - 3. Click "Analyze" - 4. Processing will extract and analyze text files - 5. Explore results in the graph visualization - ``` - -### Performance Optimization - -1. **Directory Filtering** - ``` - - Enter directory names: "src", "lib", "components" - - Focuses analysis on specific parts of the codebase - - Reduces processing time and memory usage - ``` - -2. **File Pattern Filtering** - ``` - - Use patterns: "*.py", "*.js", "*.ts" - - Supports wildcards: "test*.py", "*util*" - - Comma-separated: "*.py,*.js,*.ts" - ``` - -3. **File Limits** - ``` - - Default limit: 500 files - - Configurable in settings (50-2000 files) - - Large repositories show confirmation dialog - - Automatic truncation to limit if confirmed - ``` - -### Using the AI Chat - -1. **Configure API Key** - ``` - 1. Click the โš™๏ธ settings button - 2. Choose your preferred LLM provider - 3. Enter your API key - 4. Select model (e.g., gpt-4o-mini, claude-3-haiku) - ``` - -2. **Ask Questions** - ``` - - "What functions are in the main.py file?" - - "Show me all classes that inherit from BaseClass" - - "How does the authentication system work?" - - "Find all functions that call the database" - ``` - -### Exporting Data - -1. **Export Knowledge Graph** - ``` - 1. Click the ๐Ÿ“ฅ Export button after processing - 2. Downloads JSON file with graph data and metadata - 3. Includes processing statistics and timestamps - 4. File size shown in UI before export - ``` - -2. **Export Format** - ```json - { - "metadata": { - "exportedAt": "2024-01-01T12:00:00.000Z", - "version": "1.0.0", - "nodeCount": 150, - "relationshipCount": 200, - "fileCount": 25, - "processingDuration": 5000 - }, - "graph": { - "nodes": [...], - "relationships": [...] - }, - "fileContents": {...} - } - ``` - -### Graph Interaction - -- **Node Selection**: Click any node to highlight and view details -- **Zoom & Pan**: Mouse wheel to zoom, drag to pan -- **Node Types**: Different colors/shapes for files, functions, classes, etc. -- **Relationships**: Arrows show CONTAINS, CALLS, INHERITS relationships - -## ๐Ÿ”ง Development - -### Project Structure -``` -GitNexus/ -โ”œโ”€โ”€ src/ -โ”‚ โ”œโ”€โ”€ services/ # External integrations -โ”‚ โ”œโ”€โ”€ core/ # Processing pipeline -โ”‚ โ”œโ”€โ”€ ai/ # AI and RAG systems -โ”‚ โ”œโ”€โ”€ workers/ # Web Workers -โ”‚ โ”œโ”€โ”€ ui/ # React components -โ”‚ โ”‚ โ”œโ”€โ”€ components/ # Reusable components -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ ErrorBoundary.tsx -โ”‚ โ”‚ โ”‚ โ”œโ”€โ”€ graph/ # Graph components -โ”‚ โ”‚ โ”‚ โ””โ”€โ”€ chat/ # Chat components -โ”‚ โ”‚ โ””โ”€โ”€ pages/ # Application pages -โ”‚ โ”œโ”€โ”€ lib/ # Utilities -โ”‚ โ”‚ โ””โ”€โ”€ export.ts # Export functionality -โ”‚ โ””โ”€โ”€ App.tsx # Main application -โ”œโ”€โ”€ public/ -โ”‚ โ””โ”€โ”€ wasm/ # Tree-sitter WASM files -โ”œโ”€โ”€ package.json -โ”œโ”€โ”€ vite.config.ts -โ””โ”€โ”€ tsconfig.json -``` - -### Key Components - -#### **Error Handling** -```typescript -// ErrorBoundary component with recovery options - { - console.error('Application error:', error); - }} -> - - -``` - -#### **Performance Optimization** -```typescript -// File filtering and limits -const filterFiles = (files: any[]) => { - return files - .filter(file => matchesDirectoryFilter(file)) - .filter(file => matchesPatternFilter(file)) - .slice(0, maxFiles); -}; -``` - -#### **Export Functionality** -```typescript -// Export with metadata -exportAndDownloadGraph(graph, { - projectName: 'my-project', - includeMetadata: true, - prettyPrint: true -}, fileContents, { duration: 5000 }); -``` - -#### **Processing Pipeline** -```typescript -// 3-pass ingestion strategy with progress tracking -const pipeline = new GraphPipeline(); -const result = await pipeline.run({ - projectRoot: '/', - projectName: 'MyProject', - filePaths: ['src/main.py', 'src/utils.py'], - fileContents: new Map([ - ['src/main.py', 'def main(): pass'], - ['src/utils.py', 'def helper(): pass'] - ]) -}); -``` - -#### **AI Integration** -```typescript -// LangChain ReAct agent with error handling -const orchestrator = new LangChainRAGOrchestrator(llmService, cypherGenerator); -await orchestrator.setContext({ graph, fileContents }, llmConfig); -const response = await orchestrator.answerQuestion("How does auth work?"); -``` - -#### **Graph Visualization** -```typescript -// Interactive graph component with error boundaries - - setSelectedNode(nodeId)} - /> - -``` - -### Adding New Features - -1. **New Language Support** - ```typescript - // Add parser in core/tree-sitter/ - export const loadJavaScriptParser = async () => { - // Load JS Tree-sitter grammar - }; - ``` - -2. **Custom AI Tools** - ```typescript - // Add tools in ai/langchain-orchestrator.ts - const customTool = tool( - async (input: { query: string }) => { - // Tool implementation - }, - { - name: "custom_tool", - description: "Custom functionality", - schema: z.object({ query: z.string() }) - } - ); - ``` - -3. **Export Formats** - ```typescript - // Add new export formats in lib/export.ts - export function exportToCSV(graph: KnowledgeGraph): string { - // CSV export implementation - } - ``` - -## ๐Ÿงช Testing & Quality Assurance - -### Error Handling -- **Error Boundaries**: Catch and display JavaScript errors gracefully -- **User Recovery**: Allow users to reset component state after errors -- **Detailed Logging**: Console logging for debugging and error reporting -- **Fallback UI**: User-friendly error messages with recovery options - -### Performance Testing -1. **Large Repository Handling** - - Test with repositories containing 1000+ files - - Verify confirmation dialogs for file limits - - Monitor memory usage during processing - - Test filtering effectiveness - -2. **UI Responsiveness** - - Ensure non-blocking processing with Web Workers - - Verify progress indicators update correctly - - Test error recovery mechanisms - - Validate export functionality with large graphs - -3. **Error Scenarios** - - Network failures during GitHub API calls - - Corrupted ZIP files - - Invalid API keys - - Memory exhaustion scenarios - -### Manual Testing Checklist -- [ ] GitHub repository analysis with various sizes -- [ ] ZIP file upload and extraction -- [ ] Directory and file pattern filtering -- [ ] Large repository confirmation dialog -- [ ] Export functionality with different options -- [ ] Error boundary activation and recovery -- [ ] API key validation for all providers -- [ ] Settings persistence across sessions -- [ ] Graph visualization interactions -- [ ] Chat interface with different LLM providers - -## ๐Ÿš€ Deployment - -### Production Build -```bash -npm run build -npm run preview -``` - -### Environment Variables -```env -# Optional: Pre-configure API keys -VITE_OPENAI_API_KEY=sk-... -VITE_ANTHROPIC_API_KEY=sk-ant-... -VITE_GEMINI_API_KEY=... - -# Performance settings -VITE_DEFAULT_MAX_FILES=500 -VITE_ENABLE_DEBUG_LOGGING=false -``` - -### Docker Deployment -```dockerfile -FROM node:18-alpine -WORKDIR /app -COPY package*.json ./ -RUN npm ci --only=production -COPY . . -RUN npm run build -EXPOSE 3000 -CMD ["npm", "run", "preview", "--", "--host", "0.0.0.0"] -``` - -### Performance Monitoring -```javascript -// Add performance monitoring -const observer = new PerformanceObserver((list) => { - for (const entry of list.getEntries()) { - if (entry.entryType === 'measure') { - console.log(`${entry.name}: ${entry.duration}ms`); - } - } -}); -observer.observe({ entryTypes: ['measure'] }); -``` - -## ๐Ÿ”’ Security & Privacy - -- **Client-Side Processing**: All analysis happens in your browser -- **API Keys**: Stored locally, never transmitted to our servers -- **GitHub Access**: Uses public API, respects repository permissions -- **Data Privacy**: No code or analysis results are stored remotely -- **Error Logging**: Sensitive data excluded from error reports -- **Export Security**: User-controlled data export with no server interaction - -## ๐Ÿค Contributing - -### Development Setup -1. Fork the repository -2. Create feature branch: `git checkout -b feature/amazing-feature` -3. Make changes and test thoroughly -4. Run the testing checklist above -5. Commit: `git commit -m 'Add amazing feature'` -6. Push: `git push origin feature/amazing-feature` -7. Open a Pull Request - -### Code Style -- **TypeScript**: Strict mode enabled -- **ESLint**: Follow configured rules -- **Prettier**: Auto-formatting -- **Comments**: Minimal, only when necessary -- **Error Handling**: Comprehensive error boundaries and recovery -- **Performance**: Consider memory usage and processing time - -### Testing Guidelines -- Test error scenarios and edge cases -- Verify performance with large datasets -- Ensure graceful degradation -- Test all export functionality -- Validate error boundary behavior - -## ๐Ÿ“š Technical Details - -### Knowledge Graph Schema -```typescript -interface KnowledgeGraph { - nodes: GraphNode[]; // Code entities - relationships: GraphRelationship[]; // Connections -} - -// Node types: Project, Folder, File, Module, Class, Function, Method, Variable -// Relationship types: CONTAINS, CALLS, INHERITS, OVERRIDES, IMPORTS -``` - -### Export Format -```typescript -interface ExportedGraph { - metadata: { - exportedAt: string; - version: string; - nodeCount: number; - relationshipCount: number; - fileCount?: number; - processingDuration?: number; - }; - graph: KnowledgeGraph; - fileContents?: Record; -} -``` - -### Error Boundary Implementation -- **Component-Level**: Individual components wrapped for isolation -- **Application-Level**: Top-level boundary for catastrophic failures -- **Recovery Options**: Reset state, reload page, or continue with fallback -- **Error Reporting**: Detailed technical information for developers - -### Performance Optimizations -- **Web Workers**: Non-blocking processing -- **AST Caching**: Reuse parsed syntax trees -- **Progressive Loading**: Stream results as available -- **Memory Management**: Efficient data structures -- **File Filtering**: Reduce processing scope -- **Confirmation Dialogs**: Prevent accidental expensive operations - -### ReAct Agent Implementation -- **Standard LangChain**: Uses `createReactAgent` from `@langchain/langgraph/prebuilt` -- **Custom Implementation**: Manual ReAct loop for educational purposes -- **Tools**: Graph queries, code retrieval, file search -- **Memory**: Conversation persistence with thread management -- **Error Recovery**: Graceful handling of API failures - -## ๐Ÿ“„ License - -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. - -## ๐Ÿ™ Acknowledgments - -- **Tree-sitter**: Syntax parsing infrastructure -- **LangChain.js**: AI agent framework -- **Cytoscape.js**: Graph visualization -- **React**: UI framework with error boundaries -- **Vite**: Build tool and dev server - ---- - -**CodeNexus** - Edge Knowledge Graph Creator with instant Graph RAG. Zero setup, maximum insight. ๐Ÿš€ - -*Browser-native code intelligence that runs anywhere, anytime - no servers required.* \ No newline at end of file diff --git a/ISOLATED_NODES_FIX.md b/ISOLATED_NODES_FIX.md deleted file mode 100644 index d0153e0df..000000000 --- a/ISOLATED_NODES_FIX.md +++ /dev/null @@ -1,301 +0,0 @@ -# ๐Ÿ”ง Fixing Isolated Nodes in Graph Visualization - -## Problem Description -You're seeing nodes "flying away" with no connections in the graph visualization. This indicates **isolated nodes** - nodes that have no relationships to other nodes in the graph. - -## Root Causes - -### 1. **File Parsing Failures** (Most Common) -- Files fail to parse during **Pass 2** of ingestion -- File nodes get created but no functions/classes are extracted -- Results in isolated file nodes - -### 2. **Unsupported File Types** -- Files with extensions not recognized by the parser -- Configuration files, documentation, etc. without code content - -### 3. **Syntax Errors** -- Malformed code that the AST parser can't understand -- Missing imports or exports -- Language-specific syntax issues - -### 4. **Import Resolution Failures** -- **Pass 3** fails to resolve import relationships -- Files exist but aren't connected via imports - -### 5. **Call Resolution Failures** -- **Pass 4** fails to find function calls between files -- Functions exist but no call relationships are created - -## ๐Ÿ› ๏ธ How to Diagnose - -### Step 1: Use the New Diagnostic Tool -1. Load your repository in GitNexus -2. Click the **๐Ÿฉบ Diagnose** button in the chat interface -3. Check the statistics and follow the suggested steps - -### Step 2: Check Browser Console -1. Open Developer Tools (F12) -2. Look for console warnings during ingestion: - - `โš ๏ธ Found X isolated nodes` - - `โš ๏ธ Found X files without definitions` - - `Source files without definitions: [...]` - -### Step 3: Review Console Logs -Look for these specific log messages: -``` -๐Ÿ“ Pass 1: Analyzing project structure... -๐Ÿ” Pass 2: Parsing code and extracting definitions... -๐Ÿ”— Pass 3: Resolving imports and building dependency map... -๐Ÿ“ž Pass 4: Resolving function calls with 3-stage strategy... -``` - -## ๐Ÿ” Diagnostic Information - -The enhanced pipeline now shows: -- **Node counts by type** (Project, Folder, File, Function, Class, etc.) -- **Relationship counts by type** (CONTAINS, CALLS, IMPORTS, etc.) -- **Isolated nodes** with examples -- **Files without definitions** -- **Graph integrity issues** - -## โœ… Recent Fixes Applied - -### **1. Reduced Console Noise (Fixed)** -- **Issue**: Thousands of "Failed to resolve call" messages for Python built-ins like `int`, `str`, `len`, etc. -- **Fix**: Added `shouldIgnoreCall()` method to filter out Python built-in functions and standard library calls -- **Result**: Console output is now much cleaner and shows only relevant failures - -### **2. Improved Python Import Resolution (Fixed)** -- **Issue**: Python imports weren't being resolved correctly, causing "No import relationships found" -- **Fix**: Enhanced `resolveModulePath()` with better pattern matching for complex project structures -- **Features Added**: - - Multiple resolution patterns for Python modules - - Partial path matching for complex project structures - - Better handling of package imports - - Enhanced debugging with import resolution statistics - -### **3. Enhanced Diagnostic Reporting (Added)** -- **New**: Comprehensive graph integrity validation -- **New**: Import resolution success rate reporting -- **New**: Detailed breakdown of isolated nodes by type -- **New**: ๐Ÿฉบ Diagnose button in chat interface - -## ๐Ÿš€ Solutions - -### For File Parsing Issues: -1. **Check file extensions**: Ensure files are `.js`, `.ts`, `.jsx`, `.tsx`, `.py`, etc. -2. **Verify syntax**: Make sure code files have valid syntax -3. **Check file size**: Very large files might timeout during parsing -4. **Review file content**: Empty files or files with only comments won't generate nodes - -### For Import Issues (Now Improved): -1. **Check import syntax**: Ensure proper `import`/`export` or `require()` statements -2. **Verify file paths**: Relative imports should resolve correctly -3. **Check module resolution**: External libraries might not be resolved -4. **Monitor import resolution rate**: Should be >50% for healthy projects - -### For Call Issues: -1. **Function calls**: Ensure functions are actually called between files -2. **Method calls**: Class methods should be invoked -3. **Export/import**: Functions need to be properly exported and imported - -## ๐Ÿ”ง Quick Fixes - -### 1. **Filter Out Non-Code Files** -Use GitNexus filtering options to exclude: -- Documentation files (`.md`, `.txt`) -- Configuration files (`.json`, `.yaml`, `.xml`) -- Asset files (`.png`, `.jpg`, `.css`) - -### 2. **Focus on Core Directories** -- Include only `src/`, `lib/`, `app/` directories -- Exclude `node_modules/`, `.git/`, `dist/`, `build/` - -### 3. **Check File Limits** -- Large repositories might hit processing limits -- Consider processing smaller subsets first - -## ๐Ÿ“Š Expected Results After Fixes - -A healthy graph should show: -- **Project** โ†’ **Folders** โ†’ **Files** (CONTAINS relationships) -- **Files** โ†’ **Functions/Classes** (CONTAINS relationships) -- **Files** โ†’ **Files** (IMPORTS relationships) - **Now working better** -- **Functions** โ†’ **Functions** (CALLS relationships) - **Cleaner console output** - -### **Typical Success Rates**: -- **Import Resolution**: 40-70% (up from 0%) -- **Call Resolution**: 20-30% (excluding built-ins) -- **File Parsing**: 80-95% for source files - -## ๐Ÿ†˜ Still Having Issues? - -1. **Try the diagnostic tool**: Click ๐Ÿฉบ Diagnose button -2. **Check console output**: Look for specific error patterns -3. **Share diagnostic info**: Copy the improved console logs -4. **Test incrementally**: Try with smaller subsets of files - -## ๐Ÿ“ˆ What You Should See Now - -After the fixes, your console output should show: -``` -ImportProcessor: Found 45 imports, resolved 28 (62.2%) -CallProcessor: Success rate: 45.9% (excluding built-ins) -๐Ÿ“Š Graph Statistics: -Relationships by type: {CONTAINS: 395, DECORATES: 48, IMPORTS: 13, CALLS: 47} -โœ… Graph integrity validation passed -``` - -### **Latest Improvements (v2)**: -- **Expanded built-ins filtering**: Now ignores 100+ Python string methods, math functions, and third-party library calls -- **Better diagnostics**: Identifies source files with zero function calls (potential parsing issues) -- **Pattern-based filtering**: Automatically ignores dunder methods (`__init__`, `__str__`) and private methods - -### **Expected Results After All Fixes**: -- **Console noise reduction**: 95%+ reduction in irrelevant error messages -- **Import relationships**: 10-20+ IMPORTS relationships created -- **Call resolution success**: 40-60% (realistic for complex codebases) -- **Failed calls**: Only legitimate issues (domain-specific functions, missing imports) - -Instead of thousands of failed call resolutions, you'll see much cleaner output focused on actual issues that need attention! - -## ๐ŸŽฏ **FINAL STATUS - Issue Resolved!** - -### **๐Ÿ” MAJOR DISCOVERY - Python Call Extraction Bug Found!** - -**Latest diagnostic output revealed a critical issue:** -``` -๐Ÿ“Š Debug: config.py has 38 call nodes, 0 definitions -๐Ÿ“Š Debug: assessment_db.py has 120 call nodes, 4 definitions -๐Ÿ“Š Debug: sonar_analyzer.py has 30 call nodes, 2 definitions -``` - -**This shows that:** -- โœ… **AST parsing works perfectly** - files have hundreds of call nodes -- โŒ **Call extraction is broken** - 0 function calls extracted from files with 120+ call nodes -- ๐Ÿ”ง **Root cause identified** - Python call extraction logic needs fixes - -### **๐Ÿ› ๏ธ Latest Fix Applied** -- **Enhanced Python call extraction** with better node type handling -- **Comprehensive debugging** to identify what's being filtered vs extracted -- **Improved function name extraction** for complex Python call patterns -- **Added support for** subscript calls, nested calls, and more node types - -### **โœ… Latest Results (Your Console Output)** -``` -โœ… Parsing Success: 38 successful, 0 failed (100%) -โœ… Import Resolution: 102 imports found, 61 resolved (59.8%) -โœ… Call Resolution: 129 calls processed, 56.6% success rate -โœ… Graph Health: 367 nodes, 506 relationships -โœ… Console Cleanliness: Only 56 legitimate failures (93% noise reduction) -``` - -### **๐Ÿ” Enhanced Diagnostics Added** -- **Zero-call file detection**: Identifies source files with parsing issues -- **AST node counting**: Shows `call` nodes vs definitions for debugging -- **Suspicious file flagging**: Highlights files with definitions but no calls -- **Comprehensive built-ins filtering**: 100+ Python functions ignored - -### **๐Ÿ“Š Your Graph is Now Healthy!** - -**Before the fixes:** -- โŒ 814+ failed call messages (noise) -- โŒ No import relationships -- โŒ Isolated nodes everywhere -- โŒ Unreadable console output - -**After the fixes:** -- โœ… **56.6% call resolution success** (excellent!) -- โœ… **59.8% import resolution success** (great!) -- โœ… **13 IMPORTS relationships** created -- โœ… **47 CALLS relationships** created -- โœ… **Clean, readable diagnostics** - -### **๐Ÿ”ฌ Remaining Issues Are Expected** - -#### **1. Zero-Call Files (Normal)** -Files showing "No function calls found" are often: -- **Model/config files**: Only contain class definitions -- **Pure data files**: Constants, configurations -- **Interface files**: Abstract base classes -- **Files with only imports**: Router configurations - -#### **2. Failed Calls (Legitimate)** -The remaining 56 failed calls are **appropriate failures**: -- **External libraries**: LangGraph, FastAPI, Azure OpenAI -- **Domain-specific**: Business logic libraries (`ruleset`, `assert_fact`) -- **Custom models**: Application-specific classes - -### **๐ŸŽ‰ Problem Solved!** - -Your **"nodes flying away with no connections"** issue is **fully resolved**: - -1. โœ… **Files connect properly** via CONTAINS relationships -2. โœ… **Import relationships work** (13 created) -3. โœ… **Function calls connect** (47 relationships) -4. โœ… **Console is clean** and diagnostic -5. โœ… **Success rates are realistic** for complex codebases - -### **๐Ÿš€ What You Should See Now** - -When you reload your repository, expect: -- **Significantly fewer isolated nodes** -- **Connected file clusters** via imports -- **Function-to-function connections** within files -- **Clean console output** focusing on real issues -- **Better graph connectivity** overall - -The isolated nodes that remain will be: -- **Configuration files** (expected) -- **Documentation files** (expected) -- **Empty or comment-only files** (expected) -- **External library references** (expected) - -## **๐Ÿ† Mission Accomplished!** - -Your graph now has proper connectivity with realistic success rates. The diagnostic tools will help you identify any remaining issues that need attention. The isolated nodes problem is **solved**! ๐ŸŽฏโœจ - -## ๐ŸŽ›๏ธ **NEW FEATURE: Hide External Libraries Toggle** - -### **โœจ What's New** -Added a **"Hide external libraries"** toggle in the graph visualization that lets you: -- โœ… **Clean view**: Hide isolated external library nodes for cleaner visualization -- โœ… **Full view**: Show all nodes including external dependencies for complete context -- โœ… **Smart filtering**: Automatically identifies external library patterns -- โœ… **Live counter**: Shows how many external nodes are hidden/visible - -### **๐ŸŽฏ How It Works** -The toggle uses intelligent filtering to identify external library nodes: -- **Isolated nodes**: Nodes with no relationships (not connected to your code) -- **External patterns**: Recognizes common library functions like: - - `when_all`, `ruleset` (durable rules) - - `APIRouter`, `FastAPI` (FastAPI framework) - - `StateGraph`, `AsyncAzureOpenAI` (AI libraries) - - CamelCase patterns (often external classes) - -### **๐Ÿš€ When to Use Each Mode** - -#### **Hide External Libraries (Clean View)** -**Best for:** -- ๐Ÿ“Š **Architecture review** - Focus on your internal code structure -- ๐Ÿ” **Code navigation** - See relationships between your functions/classes -- ๐Ÿ“ˆ **Presentations** - Clean, professional visualization -- ๐ŸŽฏ **Debugging** - Trace internal call paths without distractions - -#### **Show External Libraries (Full View)** -**Best for:** -- ๐Ÿ”— **Dependency analysis** - See what external libraries you use -- ๐Ÿ—๏ธ **System design** - Understand integration points -- ๐Ÿ“‹ **Documentation** - Complete picture of your tech stack -- ๐Ÿ”ง **Troubleshooting** - Identify external dependency issues - -### **๐Ÿ’ก Pro Tips** -- **Default state**: External libraries are **visible by default** for complete context -- **Toggle anytime**: Switch between views without reloading the graph -- **Persistent**: Your preference is remembered during the session -- **Smart counting**: See exactly how many external nodes are being hidden - -This gives you the **best of both worlds** - clean focused views when you need them, and complete architectural context when you want it! - -Instead of thousands of failed call resolutions, you'll see much cleaner output focused on actual issues that need attention! \ No newline at end of file diff --git a/LRU_CACHE_IMPLEMENTATION.md b/LRU_CACHE_IMPLEMENTATION.md deleted file mode 100644 index 76d3a8e9a..000000000 --- a/LRU_CACHE_IMPLEMENTATION.md +++ /dev/null @@ -1,332 +0,0 @@ -# LRU Cache Implementation for Parsing Processor - -## ๐ŸŽฏ Overview - -The LRU (Least Recently Used) cache implementation provides **significant performance improvements** for the parsing processor by caching parsed files, Tree-sitter query results, and language parsers. This reduces redundant parsing operations and speeds up processing of large codebases. - -## ๐Ÿš€ Performance Benefits - -### **Expected Improvements:** -- **File Parsing**: 2-5x faster for repeated files -- **Query Execution**: 3-8x faster for repeated Tree-sitter queries -- **Parser Loading**: 10-20x faster for language parser reuse -- **Memory Efficiency**: Automatic eviction of least recently used items - -### **Key Features:** -- **Multi-level caching** - Files, queries, and parsers cached separately -- **Content-based invalidation** - Files cached with content hash -- **Automatic eviction** - LRU algorithm prevents memory bloat -- **Performance monitoring** - Hit rates and statistics tracking - ---- - -## ๐Ÿ“ Files Created/Modified - -### **New Files:** -- `src/lib/lru-cache-service.ts` - Main LRU cache service -- `src/lib/lru-cache-test.ts` - Test suite for cache functionality - -### **Modified Files:** -- `src/core/ingestion/parsing-processor.ts` - Integrated LRU caching - ---- - -## ๐Ÿ”ง Core Components - -### **1. LRUCacheService Class** -```typescript -import { LRUCacheService } from './src/lib/lru-cache-service.js'; - -const cache = LRUCacheService.getInstance({ - max: 500, // Maximum items - ttl: 30 * 60 * 1000, // 30 minutes TTL - maxSize: 100 * 1024 * 1024 // 100MB max size -}); -``` - -**Three Cache Types:** -1. **File Cache** - Parsed ASTs and definitions -2. **Query Cache** - Tree-sitter query results -3. **Parser Cache** - Language parser instances - -### **2. Cache Configuration** -```typescript -interface CacheOptions { - max?: number; // Maximum number of items - ttl?: number; // Time to live in milliseconds - maxSize?: number; // Maximum size in bytes - allowStale?: boolean; // Allow stale items - updateAgeOnGet?: boolean; // Update age on access -} -``` - ---- - -## ๐ŸŽฏ Integration Points - -### **ParsingProcessor Integration:** -```typescript -export class ParsingProcessor { - private lruCache: LRUCacheService; - - constructor() { - this.lruCache = LRUCacheService.getInstance(); - } - - // Cache-aware file parsing - private async parseFile(graph: KnowledgeGraph, filePath: string, content: string): Promise { - const contentHash = this.generateContentHash(content); - const cacheKey = this.lruCache.generateFileCacheKey(filePath, contentHash); - - // Check cache first - const cachedResult = this.lruCache.getParsedFile(cacheKey); - if (cachedResult) { - console.log(`Cache hit for file: ${filePath}`); - // Use cached result - return; - } - - // Parse and cache result - // ... parsing logic ... - this.lruCache.setParsedFile(cacheKey, parsedData); - } -} -``` - -### **Cache Key Generation:** -```typescript -// File cache keys include content hash for invalidation -const fileKey = cache.generateFileCacheKey('src/main.ts', contentHash); - -// Query cache keys include language and query string -const queryKey = cache.generateQueryCacheKey('typescript', queryString); -``` - ---- - -## ๐Ÿ“Š Cache Statistics & Monitoring - -### **Performance Metrics:** -```typescript -// Get cache statistics -const stats = cache.getStats(); -console.log('Cache Stats:', { - fileCache: { size: 150, max: 200, hitRatio: 0.85 }, - queryCache: { size: 800, max: 1000, hitRatio: 0.92 }, - parserCache: { size: 3, max: 10, hitRatio: 0.95 } -}); - -// Get hit rates -const hitRate = cache.getCacheHitRate(); -console.log('Hit Rates:', { - fileCache: '85.2%', - queryCache: '92.1%', - parserCache: '95.8%' -}); -``` - -### **Cache Performance Logging:** -```typescript -// Automatic logging in ParsingProcessor -console.log('ParsingProcessor: Cache Statistics:', { - fileCache: { size: 150, hitRate: '85.2%' }, - queryCache: { size: 800, hitRate: '92.1%' }, - parserCache: { size: 3, hitRate: '95.8%' } -}); -``` - ---- - -## ๐Ÿ”„ Cache Lifecycle - -### **1. Cache Initialization:** -```typescript -// Singleton pattern ensures single cache instance -const cache = LRUCacheService.getInstance(options); -``` - -### **2. Cache Operations:** -```typescript -// Set cache items -cache.setParsedFile(key, data); -cache.setQueryResult(key, data); -cache.setParser(language, parser); - -// Get cache items -const fileData = cache.getParsedFile(key); -const queryData = cache.getQueryResult(key); -const parser = cache.getParser(language); -``` - -### **3. Cache Eviction:** -- **LRU Algorithm**: Least recently used items evicted first -- **Size Limits**: Automatic eviction when cache is full -- **TTL Expiration**: Items expire based on time-to-live -- **Memory Pressure**: Size-based eviction for large items - -### **4. Cache Cleanup:** -```typescript -// Clear specific caches -cache.clearFileCache(); -cache.clearQueryCache(); -cache.clearParserCache(); - -// Clear all caches -cache.clearAll(); -``` - ---- - -## ๐Ÿงช Testing - -### **Test Suite:** -```typescript -// Browser console testing -window.testLRUCacheBasic() // Basic functionality -window.testLRUCacheEviction() // LRU eviction behavior -window.testCacheKeyGeneration() // Key generation -window.runLRUCacheTests() // Run all tests -``` - -### **Test Coverage:** -- **Basic Operations**: Set, get, has, delete -- **LRU Eviction**: Automatic removal of least used items -- **Key Generation**: File and query cache keys -- **Statistics**: Hit rates and cache sizes -- **Performance**: Memory usage and eviction behavior - ---- - -## ๐ŸŽฏ Usage Examples - -### **Basic Usage:** -```typescript -import { LRUCacheService } from './src/lib/lru-cache-service.js'; - -// Get cache instance -const cache = LRUCacheService.getInstance(); - -// Cache parsed file -cache.setParsedFile('src/main.ts', { - ast: parsedAST, - definitions: extractedDefinitions, - language: 'typescript', - lastModified: Date.now(), - fileSize: content.length -}); - -// Retrieve from cache -const cached = cache.getParsedFile('src/main.ts'); -if (cached) { - console.log('Cache hit!'); - // Use cached data -} -``` - -### **Advanced Configuration:** -```typescript -// Custom cache configuration -const cache = LRUCacheService.getInstance({ - max: 1000, // 1000 items max - ttl: 1000 * 60 * 60, // 1 hour TTL - maxSize: 200 * 1024 * 1024, // 200MB max size - allowStale: true, // Allow stale items - updateAgeOnGet: true // Update age on access -}); -``` - ---- - -## ๐Ÿšจ Error Handling & Fallbacks - -### **Cache Miss Handling:** -- Graceful fallback to parsing when cache miss occurs -- No impact on functionality when cache is unavailable -- Automatic cache recovery after errors - -### **Memory Management:** -- Automatic eviction prevents memory bloat -- Size-based limits protect against large files -- TTL expiration ensures fresh data - ---- - -## ๐Ÿ“ˆ Performance Impact - -### **Expected Improvements by Cache Type:** - -#### **File Cache:** -- **First Run**: No improvement (cache population) -- **Subsequent Runs**: 2-5x faster for unchanged files -- **Large Codebases**: Significant improvement for repeated processing - -#### **Query Cache:** -- **Repeated Queries**: 3-8x faster execution -- **Similar Files**: High hit rate for similar code patterns -- **Batch Processing**: Excellent for multiple files with similar structure - -#### **Parser Cache:** -- **Parser Loading**: 10-20x faster after first load -- **Language Switching**: Instant parser availability -- **Memory Efficiency**: Reuse expensive parser instances - ---- - -## ๐Ÿ”ง Configuration Options - -### **Default Settings:** -```typescript -const defaultOptions = { - max: 500, // 500 items max - ttl: 1000 * 60 * 30, // 30 minutes TTL - maxSize: 100 * 1024 * 1024, // 100MB max size - allowStale: false, // No stale items - updateAgeOnGet: true // Update age on access -}; -``` - -### **Cache-Specific Settings:** -- **File Cache**: 200 items, 1 hour TTL -- **Query Cache**: 1000 items, 15 minutes TTL -- **Parser Cache**: 10 items, 24 hours TTL - ---- - -## ๐ŸŽฏ Success Metrics - -### **Performance Improvements:** -- โœ… 2-5x faster file parsing for cached files -- โœ… 3-8x faster query execution for repeated queries -- โœ… 10-20x faster parser loading after initial load -- โœ… Reduced memory pressure through automatic eviction - -### **Code Quality:** -- โœ… Comprehensive error handling -- โœ… Extensive test coverage -- โœ… Performance monitoring and statistics -- โœ… Graceful fallback mechanisms - -### **User Experience:** -- โœ… Faster processing of large codebases -- โœ… Reduced waiting time for repeated operations -- โœ… Automatic cache management (no user intervention) -- โœ… Detailed performance insights - ---- - -## ๐Ÿš€ Future Enhancements - -### **Planned Improvements:** -1. **Persistent Caching**: Save cache to IndexedDB for session persistence -2. **Compression**: Compress cached data to reduce memory usage -3. **Predictive Caching**: Pre-cache likely-to-be-used files -4. **Distributed Caching**: Share cache across browser tabs -5. **Adaptive TTL**: Adjust TTL based on file change frequency - -### **Performance Optimizations:** -1. **Lazy Loading**: Load cache items on demand -2. **Background Prefetching**: Pre-cache files in background -3. **Cache Warming**: Pre-populate cache with common patterns -4. **Memory Optimization**: Better size calculation algorithms - -This LRU cache implementation provides significant performance improvements for the parsing processor while maintaining memory efficiency and providing comprehensive monitoring capabilities. diff --git a/PRUNING_IMPLEMENTATION.md b/PRUNING_IMPLEMENTATION.md deleted file mode 100644 index a56a5be68..000000000 --- a/PRUNING_IMPLEMENTATION.md +++ /dev/null @@ -1,165 +0,0 @@ -# ๐ŸŽฏ Two-Stage Filtering Implementation - Complete - -## ๐Ÿš€ **Successfully Implemented!** - -We have successfully implemented the sophisticated two-stage filtering architecture that decouples structural discovery from content analysis. - -## ๐Ÿ—๏ธ **Architecture Overview** - -### **Stage 1: Complete Structural Discovery** -- **โœ… GitHub Service**: Discovers ALL files and directories (including `node_modules`, `.git`, etc.) -- **โœ… ZIP Service**: Extracts ALL paths from archives (complete structure) -- **โœ… StructureProcessor**: Creates nodes for EVERY path discovered -- **โœ… Result**: Knowledge graph contains complete, accurate repository structure - -### **Stage 2: Intelligent Pruning Before Parsing** -- **โœ… ParsingProcessor**: Applies sophisticated filtering before content analysis -- **โœ… Ignore Patterns**: Comprehensive list of directories to skip during parsing -- **โœ… User Filters**: Directory and extension filters still work as before -- **โœ… Result**: Only relevant files get their content parsed and analyzed - -## ๐ŸŽฏ **Implementation Details** - -### **Enhanced ParsingProcessor** - -#### **Comprehensive Ignore Patterns** -```typescript -private static readonly IGNORE_PATTERNS = new Set([ - // Version Control - '.git', '.svn', '.hg', - - // Package Managers & Dependencies - 'node_modules', 'bower_components', 'vendor', 'deps', - - // Python Virtual Environments & Cache - 'venv', 'env', '.venv', 'virtualenv', '__pycache__', - - // Build & Distribution - 'build', 'dist', 'out', 'target', 'bin', 'obj', - - // IDE & Editor Directories - '.vs', '.vscode', '.idea', '.eclipse', - - // Temporary & Logs - 'tmp', 'temp', 'logs', 'log', - - // Coverage & Testing - 'coverage', '.coverage', 'htmlcov', - - // Cache Directories - '.cache', '.next', '.nuxt' -]); -``` - -#### **Two-Stage Filtering Logic** -```typescript -private applyFiltering(allPaths: string[], fileContents: Map, options?: FilterOptions): string[] { - // STAGE 1: Prune ignored directories - let filesToProcess = this.pruneIgnoredPaths(allPaths.filter(path => fileContents.has(path))); - - // STAGE 2: Apply user filters - if (options?.directoryFilter) { /* existing user filter logic */ } - if (options?.fileExtensions) { /* existing user filter logic */ } - - return filesToProcess; -} -``` - -#### **Intelligent Pruning** -```typescript -private pruneIgnoredPaths(filePaths: string[]): string[] { - return filePaths.filter(path => { - const pathSegments = path.split('/'); - - // Check if any segment matches ignore patterns - const hasIgnoredSegment = pathSegments.some(segment => - ParsingProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - return !hasIgnoredSegment && !this.matchesIgnorePatterns(path); - }); -} -``` - -### **Complete Structure Discovery** - -#### **GitHub Service Enhancement** -- **Removed**: `shouldSkipDirectory()` checks in `collectPathsAndContent()` -- **Result**: Discovers ALL directories, including `node_modules`, `.git`, etc. - -#### **ZIP Service Enhancement** -- **Removed**: `shouldSkipDirectory()` checks in `extractCompleteStructure()` -- **Result**: Extracts ALL paths from ZIP archives - -## ๐Ÿ“Š **Before vs After** - -| Aspect | โŒ **Before** | โœ… **After** | -|--------|---------------|--------------| -| **Structure Discovery** | Filtered early, missed directories | Complete discovery of all paths | -| **node_modules Visibility** | Missing from KG | Visible as folder node | -| **Content Parsing** | Parsed everything discovered | Intelligently skips ignored directories | -| **Performance** | Slow (parsed dependencies) | Fast (skips massive directories) | -| **KG Accuracy** | Incomplete structure | Perfect mirror of repository | -| **User Experience** | Cluttered with dependencies | Clean, focused on source code | - -## ๐ŸŽฏ **Benefits Achieved** - -### **1. Complete Accurate Structure** -``` -โœ… Repository Structure in KG: -โ”œโ”€โ”€ src/ (visible, parsed) -โ”œโ”€โ”€ tests/ (visible, parsed) -โ”œโ”€โ”€ node_modules/ (visible, NOT parsed) ๐ŸŽฏ -โ”œโ”€โ”€ .git/ (visible, NOT parsed) ๐ŸŽฏ -โ”œโ”€โ”€ dist/ (visible, NOT parsed) ๐ŸŽฏ -โ””โ”€โ”€ package.json (visible, parsed) -``` - -### **2. Performance Improvements** -- **โšก Skip Massive Directories**: No parsing of `node_modules` (thousands of files) -- **โšก Faster Processing**: Focus on actual source code -- **โšก Smaller Graphs**: Fewer definition nodes to render -- **โšก Better Memory Usage**: Avoid loading massive dependency files - -### **3. Professional User Experience** -- **๐Ÿ“Š Accurate Representation**: Users see complete project structure -- **๐ŸŽฏ Clean Analysis**: Focus on relevant code, not dependencies -- **๐Ÿ” Better Navigation**: Easy to distinguish project code from dependencies -- **๐Ÿ“ˆ Trust**: KG accurately mirrors their actual repository - -## ๐Ÿ” **Technical Highlights** - -### **Sophisticated Pattern Matching** -- **Directory Segments**: Checks each path segment against ignore patterns -- **Pattern-Based**: Handles `.egg-info`, `site-packages`, etc. -- **Hidden Directories**: Smart handling of `.github` (keep) vs `.vscode` (ignore) - -### **Logging & Visibility** -``` -ParsingProcessor: Starting with 1,247 files with content -ParsingProcessor: After pruning ignored directories: 1,247 -> 89 files -ParsingProcessor: Directory filter applied: 89 -> 45 files -``` - -### **Browser Compatibility** -- **โœ… No Node.js Dependencies**: Pure browser implementation -- **โœ… Memory Efficient**: Batched processing with size limits -- **โœ… Performance Optimized**: Skip expensive operations on ignored files - -## ๐Ÿš€ **Deployment Status** - -- **โœ… Build Success**: All TypeScript compilation passes -- **โœ… Architecture Complete**: Two-stage filtering fully implemented -- **โœ… Backward Compatible**: Existing functionality preserved -- **โœ… Production Ready**: Ready for real-world repository analysis - -## ๐ŸŽ‰ **Result** - -**Perfect Implementation!** We now have: - -1. **Complete Structure Discovery**: Every directory appears in the KG -2. **Intelligent Content Filtering**: Skip parsing massive dependency directories -3. **Optimal Performance**: Fast processing focused on relevant code -4. **Professional UX**: Clean, accurate knowledge graphs - -The two-stage filtering architecture is **successfully implemented** and ready for production! ๐Ÿš€ \ No newline at end of file diff --git a/README.md b/README.md index d1dd1fe7d..15924209c 100644 --- a/README.md +++ b/README.md @@ -1,1863 +1,380 @@ -# GitNexus: Edge-Based Code Knowledge Graph Generator for Deno - Step-by-Step Implementation Guide +# GitNexus - Edge Knowledge Graph Creator with Graph RAG -This guide will walk you through building a fully edge-based code knowledge graph generator from scratch using Deno. I'll explain each concept before showing the implementation, so you understand **why** we're doing something, not just **how** to do it. +**Transform any codebase into an interactive knowledge graph in your browser. No servers, no setup - just instant Graph RAG-powered code intelligence.** -## Phase 1: Project Setup & Core Infrastructure +GitNexus is a client-side knowledge graph creator that runs entirely in your browser. Drop in a GitHub repo or ZIP file, and get an interactive knowledge graph with AI-powered chat interface. Perfect for code exploration, documentation, and understanding complex codebases through Graph RAG (Retrieval-Augmented Generation). -### Step 1: Project Structure and Tooling Setup +## โœจ Features -**Why this matters:** Before writing any code, we need to set up our development environment properly. A well-structured project makes it easier to add features later and keeps everything organized. +### ๐Ÿ“Š **Code Analysis & Visualization** +- **GitHub Integration**: Analyze any public GitHub repository directly from URL +- **ZIP File Support**: Upload and analyze local code archives +- **Interactive Knowledge Graph**: Visualize code structure with Cytoscape.js +- **Multi-language Support**: TypeScript, JavaScript, Python, and more with extensible architecture +- **Smart Filtering**: Directory and file pattern filters to focus analysis scope +- **Performance Optimization**: Configurable file limits with confirmation dialogs for large repositories -**Key concepts:** +### ๐Ÿค– **AI-Powered Chat Interface** +- **Multiple LLM Providers**: OpenAI, Anthropic (Claude), Google Gemini, Azure OpenAI +- **ReAct Agent Pattern**: Uses proper LangChain ReAct implementation for reasoning +- **Tool-Augmented Responses**: Graph queries, code retrieval, file search +- **Context-Aware**: Maintains conversation history with configurable memory -- We're using Vite (a modern build tool) with React and TypeScript -- We need special configuration for WebAssembly (WASM) files -- A clear directory structure helps us scale to multiple languages later +### ๐Ÿ”ง **Advanced Processing Pipeline** +- **Four-Pass Ingestion System**: + 1. **Structure Analysis**: Project hierarchy and file organization + 2. **Code Parsing**: AST-based extraction using Tree-sitter + 3. **Import Resolution**: Module and import relationship mapping + 4. **Call Resolution**: Function/method call relationship mapping +- **Parallel Processing**: Multi-threaded processing using Web Worker Pool +- **Intelligent Caching**: AST and processing result optimization +- **Error Resilience**: Comprehensive error boundaries and recovery mechanisms -**Implementation Steps:** +### ๐ŸŽจ **Modern UI/UX** +- **Responsive Design**: Adaptive layout for different screen sizes +- **Real-time Progress**: Live updates during repository processing +- **Interactive Graph**: Node selection, zooming, panning +- **Split-Panel Layout**: Graph visualization + AI chat interface +- **Settings Management**: Persistent configuration for API keys and preferences +- **Export Functionality**: Download knowledge graphs as JSON/CSV with metadata +- **Performance Controls**: File limits, filtering, and optimization settings -1. **Create the base project:** +## ๐Ÿ—๏ธ Architecture +### **Frontend Stack** +- **React 18** with TypeScript +- **Vite** for fast development and building +- **Cytoscape.js** for graph visualization +- **Custom CSS** with modern design patterns +- **Error Boundaries** for robust error handling + +### **Processing Engine** +- **Tree-sitter WASM** for syntax parsing +- **Web Worker Pool** for parallel processing +- **Comlink** for worker communication +- **LRU Cache** for performance optimization + +### **AI Integration** +- **LangChain.js** with proper ReAct agent implementation +- **Multiple LLM Support**: OpenAI, Anthropic, Gemini, Azure OpenAI +- **Tool-based Architecture**: Graph queries, code retrieval, file search +- **Cypher Query Generation**: Natural language to graph queries + +### **Graph Database** +- **KuzuDB WASM**: Embedded graph database running in the browser +- **Cypher Queries**: Powerful graph querying capabilities +- **Persistent Storage**: Data stored in browser's IndexedDB +- **Performance**: Significantly faster queries than in-memory objects + +### **Four-Pass Ingestion Pipeline** +The GitNexus processing pipeline follows a consistent four-phase execution model: + +```mermaid +flowchart TD + A[Start Pipeline] --> B[Pass 1: Structure Analysis] + B --> C[Pass 2: Code Parsing & Definition Extraction] + C --> D[Pass 3: Import Resolution] + D --> E[Pass 4: Call Resolution] + E --> F[Return Knowledge Graph] + + subgraph "Phase 1: Structure Analysis" + B1[Identify Project Root] + B2[Discover All Paths] + B3[Categorize as Files/Directories] + B4[Create Project, Folder, File Nodes] + B5[Establish CONTAINS Relationships] + end + + subgraph "Phase 2: Code Parsing" + C1[Filter Processable Files] + C2[Initialize Tree-Sitter Parser] + C3[Parse Each File to AST] + C4[Extract Definitions: Functions, Classes, etc.] + C5[Store ASTs and Function Registry] + end + + subgraph "Phase 3: Import Resolution" + D1[Extract Import Statements from ASTs] + D2[Determine Language-Specific Import Patterns] + D3[Resolve Target File Paths] + D4[Build Import Map] + D5[Create IMPORTS Relationships] + end + + subgraph "Phase 4: Call Resolution" + E1[Extract Function Calls from ASTs] + E2[Stage 1: Exact Match via Import Map] + E3[Stage 2: Fuzzy Matching for Unresolved Calls] + E4[Create CALLS Relationships] + end +``` + +### **Dual-Engine Architecture** +GitNexus implements a dual-engine architecture to support both current stable and next-generation processing: + +```mermaid +graph TD + UI[User Interface] --> EM[Engine Manager] + EM --> LEG[Legacy Engine] + EM --> NG[Next-Gen Engine] + + subgraph "Legacy Engine (Current)" + LEG --> GP[GraphPipeline - Sequential] + GP --> PP[ParsingProcessor - Single Thread] + GP --> IM[In-Memory Storage] + end + + subgraph "Next-Gen Engine (In Progress)" + NG --> PLP[ParallelPipeline - Concurrent] + PLP --> PPP[ParallelParsingProcessor - Multi-Thread] + PLP --> KD[KuzuDB Storage] + KD --> KW[KuzuDB WASM] + end +``` + +### **Services Layer** +``` +src/ +โ”œโ”€โ”€ services/ # External API integrations +โ”‚ โ”œโ”€โ”€ github.ts # GitHub REST API client +โ”‚ โ””โ”€โ”€ zip.ts # ZIP file processing +โ”œโ”€โ”€ core/ # Core processing logic +โ”‚ โ”œโ”€โ”€ graph/ # Knowledge graph types and engines +โ”‚ โ”œโ”€โ”€ ingestion/ # Multi-pass processing pipeline +โ”‚ โ””โ”€โ”€ tree-sitter/ # Syntax parsing infrastructure +โ”œโ”€โ”€ ai/ # AI and RAG components +โ”‚ โ”œโ”€โ”€ llm-service.ts # Multi-provider LLM client +โ”‚ โ”œโ”€โ”€ cypher-generator.ts # NL to Cypher translation +โ”‚ โ””โ”€โ”€ kuzu-rag-orchestrator.ts # KuzuDB-enhanced RAG +โ”œโ”€โ”€ workers/ # Web Worker implementations +โ”œโ”€โ”€ ui/ # React components and pages +โ”‚ โ”œโ”€โ”€ components/ # Reusable UI components +โ”‚ โ”‚ โ”œโ”€โ”€ ErrorBoundary.tsx +โ”‚ โ”‚ โ”œโ”€โ”€ graph/ # Graph visualization components +โ”‚ โ”‚ โ””โ”€โ”€ chat/ # Chat interface components +โ”‚ โ””โ”€โ”€ pages/ # Application pages +โ”œโ”€โ”€ lib/ # Shared utilities +โ”‚ โ”œโ”€โ”€ web-worker-pool.ts # Worker pool implementation +โ”‚ โ”œโ”€โ”€ export.ts # Graph export functionality +โ”‚ โ””โ”€โ”€ lru-cache-service.ts # Caching service +โ””โ”€โ”€ App.tsx # Main application entry point +``` + +## ๐Ÿš€ Getting Started + +### Prerequisites +- **Node.js 18+** and **npm/yarn** +- **API Keys** for AI features (OpenAI, Anthropic, or Gemini) + +### Installation + +1. **Clone the repository** + ```bash + git clone + cd gitnexus + ``` + +2. **Install dependencies** + ```bash + npm install + ``` + +3. **Start development server** + ```bash + npm run dev + ``` + +4. **Open in browser** + ``` + http://localhost:5173 + ``` + +### Configuration + +1. **GitHub Token (Optional)** + - Increases rate limit from 60 to 5,000 requests/hour + - Generate at: https://github.com/settings/tokens + - Requires no special permissions for public repos + +2. **AI API Keys** + - **OpenAI**: Get from https://platform.openai.com/api-keys + - **Anthropic**: Get from https://console.anthropic.com/ + - **Gemini**: Get from https://makersuite.google.com/app/apikey + - **Azure OpenAI**: Configure endpoint and deployment settings + +3. **Performance Settings** + - **File Limit**: Configure maximum files to process (default: 500) + - **Directory Filters**: Focus on specific directories (e.g., "src", "lib") + - **File Patterns**: Filter by file types (e.g., "*.ts", "*.js", "*.py") + +## ๐Ÿ’ก Usage + +### Analyzing a Repository + +1. **GitHub Repository** + ``` + 1. Enter GitHub URL: https://github.com/owner/repo + 2. Optional: Set directory/file filters to focus analysis + 3. Click "Analyze" + 4. For large repos: Confirm processing or adjust filters + 5. Wait for processing (structure โ†’ parsing โ†’ import โ†’ call resolution) + 6. Explore the interactive graph + ``` + +2. **ZIP File Upload** + ``` + 1. Click "Choose File" and select a .zip file + 2. Optional: Configure filters before processing + 3. Click "Analyze" + 4. Processing will extract and analyze text files + 5. Explore results in the graph visualization + ``` + +### Engine Selection +GitNexus supports both legacy (stable) and next-gen (parallel/KuzuDB) processing engines: +- Use the engine selector in the UI to switch between engines +- Next-gen engine provides parallel processing and KuzuDB storage +- Legacy engine provides stable, in-memory processing +- System automatically falls back to legacy engine if next-gen fails + +### Using the AI Chat + +1. **Configure API Key** + ``` + 1. Click the โš™๏ธ settings button + 2. Choose your preferred LLM provider + 3. Enter your API key + 4. Select model (e.g., gpt-4o-mini, claude-3-haiku) + ``` + +2. **Ask Questions** + ``` + - "What functions are in the main.py file?" + - "Show me all classes that inherit from BaseClass" + - "How does the authentication system work?" + - "Find all functions that call the database" + ``` + +### Exporting Data + +1. **Export Knowledge Graph** + ``` + 1. Click the ๐Ÿ“ฅ Export button after processing + 2. Choose format (JSON or CSV) + 3. Downloads file with graph data and metadata + 4. File size shown in UI before export + ``` + +## ๐Ÿ”„ Work in Progress + +GitNexus is currently operating with a dual-engine architecture that supports both stable and next-generation processing: + +### Current Architecture (Stable - Default) +- **Single-threaded Processing**: Code analysis runs on the main browser thread using sequential processing +- **In-Memory Storage**: Knowledge graph stored as JSON objects in memory +- **Four-Pass Ingestion Pipeline**: Structure analysis โ†’ Code parsing โ†’ Import resolution โ†’ Call resolution +- **Limited Scalability**: Performance degrades with large codebases (500+ files) + +### Next-Gen Architecture (Feature Flag Enabled) +- **Parallel Processing**: Multi-threaded analysis using Web Worker Pool for massive performance gains +- **KuzuDB Integration**: Embedded graph database for persistent, high-performance graph queries +- **Cypher Queries**: AI agents can directly query the knowledge graph using Cypher, enabling more sophisticated analysis +- **Enhanced Scalability**: Handles larger repositories with better memory management + +### Transition Status +The project currently defaults to the stable legacy engine but has the next-generation engine available through feature flags. The next-gen engine includes: + +1. **Worker Pool Infrastructure**: Fully implemented Web Worker Pool for parallel processing +2. **KuzuDB Integration**: Complete implementation of KuzuDB WASM with Cypher query support +3. **Parallel Pipeline**: ParallelGraphPipeline with ParallelParsingProcessor ready for use +4. **Feature Flags**: All next-gen features enabled by default in feature flags + +Users can switch between engines using the engine selection interface, with automatic fallback to the legacy engine if issues occur. + +### Benefits of Next-Gen Architecture +- **4-8x faster processing** for large codebases through parallel execution +- **Persistent storage** that survives browser refreshes using IndexedDB +- **More powerful AI analysis** through direct database queries with Cypher +- **Better memory management** for large repositories through database storage + +## ๐Ÿงช Testing & Quality Assurance + +### Error Handling +- **Error Boundaries**: Catch and display JavaScript errors gracefully +- **User Recovery**: Allow users to reset component state after errors +- **Detailed Logging**: Console logging for debugging and error reporting +- **Fallback UI**: User-friendly error messages with recovery options + +### Performance Testing +1. **Large Repository Handling** + - Test with repositories containing 1000+ files + - Verify confirmation dialogs for file limits + - Monitor memory usage during processing + - Test filtering effectiveness + +2. **UI Responsiveness** + - Ensure non-blocking processing with Web Workers + - Verify progress indicators update correctly + - Test error recovery mechanisms + - Validate export functionality with large graphs + +## ๐Ÿš€ Deployment + +### Production Build ```bash -# Create project root -mkdir GitNexus -cd GitNexus - -# Initialize Vite project with React and TypeScript -npm create vite@latest . -- --template react-ts - -# Initialize Deno project -deno init +npm run build +npm run preview ``` -2. **Create the application directory structure:** +### Environment Variables +```env +# Optional: Pre-configure API keys +VITE_OPENAI_API_KEY=sk-... +VITE_ANTHROPIC_API_KEY=sk-ant-... +VITE_GEMINI_API_KEY=... -```bash -# Create directories for our core components -mkdir -p src/{core,core/tree-sitter,core/graph,core/ingestion,services,ai,ai/agents,ai/prompts,ui,ui/components,ui/components/graph,ui/components/chat,ui/hooks,workers,lib,config,store} +# Performance settings +VITE_DEFAULT_MAX_FILES=500 +VITE_ENABLE_DEBUG_LOGGING=false ``` -**Why this structure?** - -- `core/`: Contains the engine that builds the knowledge graph -- `services/`: Handles external interactions (GitHub API, ZIP processing) -- `ai/`: Contains the RAG and chat functionality -- `ui/`: All user interface components -- `workers/`: Web Workers for heavy processing (keeps UI responsive) -- `lib/`: Utility functions used throughout the app - -### Step 2: Configure Build Tools for WASM - -**Why this matters:** WebAssembly (WASM) is how we'll run the Tree-sitter parsers in the browser. We need special configuration to handle these binary files correctly. - -**Key concepts:** - -- WASM files are binary files that run at near-native speed in browsers -- Vite needs special configuration to handle them properly -- We want to avoid inlining large WASM files in our JavaScript bundles - -**Implementation:** - -1. **Update `vite.config.ts`:** - -```typescript -import { defineConfig } from 'vite' -import react from '@vitejs/plugin-react' -export default defineConfig({ - plugins: [react()], - worker: { - format: 'es' - }, - assetsInclude: ['**/*.wasm'], - build: { - target: 'esnext', - assetsInlineLimit: 0 // Don't inline WASM files - } -}) -``` - -**What this does:** - -- `assetsInclude: ['**/*.wasm']` tells Vite to treat WASM files as assets -- `assetsInlineLimit: 0` ensures WASM files aren't inlined into JavaScript (they're too large) -- `worker: { format: 'es' }` configures Web Workers to use ES modules - -2. **Configure TypeScript** with a `tsconfig.json` that has strict settings for better code quality. - -**Why strict settings?** They help catch errors early and make the code more maintainable as the project grows. - -### Step 3: Set Up WASM Parser Infrastructure - -**Why this matters:** Tree-sitter is the engine that parses code into ASTs (Abstract Syntax Trees). We need to get these parsers working in the browser via WASM. - -**Key concepts:** - -- Tree-sitter parsers for different languages are written in C -- We compile them to WASM so they can run in browsers -- We need to load these parsers on demand - -**Implementation:** - -1. **Create a public directory for WASM files:** - -```bash -mkdir -p public/wasm/python -``` - -2. **Download the Tree-sitter Python parser:** - - Get `tree-sitter-python.wasm` from [tree-sitter-python releases](https://github.com/tree-sitter/tree-sitter-python/releases) - - Place it in `public/wasm/python/` - -**Why host WASM files separately?** Browsers can't access the user's file system directly for security reasons. We need to serve the WASM files from a URL. - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase 1 -- Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase - Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase - Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -I'm building a Deno 2.4.1-based edge code knowledge graph generator called GitNexus. I've completed Phase 1 (project setup and core infrastructure) and now need to implement Phase 2 (Code Acquisition Module) with Deno 2.4.1 compatibility. - -Please generate the following two services with these specific requirements: - -## 1. GitHub Service (Deno 2.4.1 Implementation) - -Create a GitHubService class in `src/services/github.ts` that: - -- Uses Deno 2.4.1's native fetch API (no Node.js dependencies) -- Handles GitHub API authentication via personal access tokens -- Implements rate limit handling (GitHub allows 5,000 requests/hour with token) -- Includes methods to: - * getRepoContents(owner: string, repo: string, path = '') - fetches directory structure - * getFileContent(owner: string, repo: string, filePath: string) - fetches individual file content -- Properly handles GitHub API rate limits and errors -- Uses Deno 2.4.1-specific error handling patterns -- Includes TypeScript interfaces for return types -- Has comprehensive comments explaining key implementation choices - -Important Deno 2.4.1 considerations: - -- Use ES modules (no CommonJS) -- No Node.js-specific modules (use Deno's built-in APIs where possible) -- Handle fetch responses with proper Deno error patterns -- Include proper types for all functions -- Follow Deno 2.4.1's security model (permissions) - -## 2. ZIP Processing Service (Deno 2.4.1 Implementation) - -Create a ZipService class in `src/services/zip.ts` that: - -- Uses Deno-compatible ZIP processing (use `https://deno.land/x/zip@v1.2.3/mod.ts` instead of JSZip) -- Processes uploaded ZIP files containing code repositories -- Extracts file paths and contents into a Map -- Handles binary data properly in Deno 2.4.1 environment -- Includes error handling for corrupted ZIP files -- Has TypeScript interfaces for all types -- Includes comprehensive comments - -Important Deno 2.4.1 considerations: - -- Use Deno's file system APIs where appropriate -- Handle file reading with Deno.readFile() -- Process ZIP entries without blocking the event loop -- Use Deno's native text decoding for file contents -- Implement streaming where possible for large ZIP files -- Note that this is for a browser-based application, so the ZIP service should work with File objects from HTML inputs - -## Additional Requirements - -- All code must be Deno 2.4.1 compatible -- Use strict TypeScript with deno-lint directives where needed -- Include proper error messages that help with debugging -- Add unit test stubs for both services (using Deno's built-in test runner) -- Follow the same directory structure as Phase 1 (services directory already exists) -- Maintain the same coding style and patterns established in Phase - Include necessary imports from Deno's standard library -- Document any Deno-specific permissions required - -3. **Create a loader for Tree-sitter parsers:** - -```typescript -import WebTreeSitter from 'web-tree-sitter'; -let parserInstance: WebTreeSitter | null = null; -const parserCache = new Map(); - -export async function initTreeSitter() { - if (parserInstance) return parserInstance; - parserInstance = await WebTreeSitter.init(); - return parserInstance; -} - -export async function loadPythonParser(): Promise { - if (parserCache.has('python')) { - return parserCache.get('python')!; - } - const Parser = await initTreeSitter(); - const pythonLang = await Parser.Language.load( - '/wasm/python/tree-sitter-python.wasm' - ); - parserCache.set('python', pythonLang); - return pythonLang; -} -``` - -**How this works:** - -1. `initTreeSitter()` initializes the WebAssembly module once -2. `loadPythonParser()` loads the Python parser from the WASM file -3. We cache parsers to avoid reloading them multiple times - -**Why cache parsers?** Loading WASM files is relatively slow, so we want to do it once and reuse the parsers. - -## Phase 2: Code Acquisition Module - -### Step 4: Implement GitHub API Integration - -**Why this matters:** Users will want to analyze public GitHub repositories, so we need a way to fetch code from GitHub. - -**Key concepts:** - -- GitHub has a REST API for accessing repository contents -- We need to handle rate limits (GitHub limits how many requests you can make) -- We'll let users provide their own API tokens for higher limits - -**Implementation:** - -```typescript -export class GitHubService { - private token: string | null = null; - - setToken(token: string) { - this.token = token; - } - - async getRepoContents(owner: string, repo: string, path = '') { - const headers: HeadersInit = { - 'Accept': 'application/vnd.github.v3+json' - }; - if (this.token) { - headers['Authorization'] = `token ${this.token}`; - } - - const response = await fetch( - `https://api.github.com/repos/${owner}/${repo}/contents/${path}`, - { headers } - ); - - if (!response.ok) { - throw new Error(`GitHub API error: ${response.status}`); - } - - return response.json(); - } -} -``` - -**How this works:** - -- `getRepoContents()` fetches the directory structure of a repository -- It uses the GitHub API with proper headers -- It handles authentication via a token - -**Important note:** GitHub API has rate limits. For unauthenticated requests, it's about 60 requests/hour. With a token, it's 5,000/hour. - -### Step 5: Implement ZIP Processing - -**Why this matters:** Not all code is on GitHub. Users might want to analyze local code or private repositories by uploading a ZIP file. - -**Key concepts:** - -- JSZip is a library for handling ZIP files in JavaScript -- We need to extract files and their contents from the ZIP -- We'll use a Map to store file paths and contents - -**Implementation:** - -```typescript -import JSZip from 'jszip'; - -export class ZipService { - async processZip(file: File): Promise> { - const zip = await JSZip.loadAsync(file); - const files = new Map(); - - for (const [filePath, zipEntry] of Object.entries(zip.files)) { - if (!zipEntry.dir) { - const content = await zipEntry.async('text'); - files.set(filePath, content); - } - } - - return files; - } -} -``` - -**How this works:** - -1. `JSZip.loadAsync(file)` loads the ZIP file -2. We iterate through all entries in the ZIP -3. For each file (not directory), we extract its content as text -4. We store the file path and content in a Map - -**Why use a Map?** It provides O(1) lookups by file path, which is important when we need to find files during graph construction. - -## Phase 3: Graph Construction Pipeline - -### Step 6: Define Graph Data Structures - -**Why this matters:** Before we can build a graph, we need to define what nodes and relationships look like. - -**Key concepts:** - -- A knowledge graph consists of nodes and relationships -- Nodes represent code elements (functions, classes, etc.) -- Relationships represent connections between elements (calls, contains, etc.) - -**Implementation:** - -```typescript -export type NodeLabel = - | 'Project' - | 'Package' - | 'Module' - | 'Folder' - | 'File' - | 'Class' - | 'Function' - | 'Method' - | 'Variable'; - -export interface GraphNode { - id: string; - label: NodeLabel; - properties: Record; -} - -export type RelationshipType = - | 'CONTAINS' - | 'CALLS' - | 'INHERITS' - | 'OVERRIDES' - | 'IMPORTS'; - -export interface GraphRelationship { - id: string; - type: RelationshipType; - source: string; - target: string; - properties?: Record; -} - -export interface KnowledgeGraph { - nodes: GraphNode[]; - relationships: GraphRelationship[]; -} -``` - -**Why these specific types?** - -- `NodeLabel` defines all possible types of code elements we'll track -- `RelationshipType` defines how code elements connect to each other -- `KnowledgeGraph` is the complete structure we'll build - -**Important relationships:** - -- `CONTAINS`: A folder contains files, a file contains functions -- `CALLS`: A function calls another function -- `IMPORTS`: One module imports from another - -### Step 7: Implement the 3-Pass Ingestion Pipeline - -**Why this matters:** Building a complete knowledge graph requires multiple passes to handle cross-file references properly. - -**Key concepts:** - -- **Pass 1**: Identify the overall structure (folders, modules) -- **Pass 2**: Parse individual files and cache ASTs -- **Pass 3**: Process function calls across files (the hardest part) - -This three-pass approach solves the "island problem" - where functions in different files appear disconnected. - -#### Pass 1: Structure Identification - -```typescript -export class StructureProcessor { - private graph: KnowledgeGraph; - private projectRoot: string; - private projectName: string; - - constructor(graph: KnowledgeGraph, projectRoot: string, projectName: string) { - this.graph = graph; - this.projectRoot = projectRoot; - this.projectName = projectName; - } - - identifyStructure(filePaths: string[]): void { - // Add Project node - this.graph.nodes.push({ - id: `project:${this.projectName}`, - label: 'Project', - properties: { name: this.projectName } - }); - - // Track directory structure - const directories = new Set(); - for (const filePath of filePaths) { - const dirPath = filePath.substring(0, filePath.lastIndexOf('/')); - if (dirPath && !directories.has(dirPath)) { - directories.add(dirPath); - // Create Folder node - this.graph.nodes.push({ - id: `folder:${dirPath}`, - label: 'Folder', - properties: { path: dirPath } - }); - - // Create CONTAINS relationship with parent - if (dirPath.includes('/')) { - const parentPath = dirPath.substring(0, dirPath.lastIndexOf('/')); - this.graph.relationships.push({ - id: `rel:folder:${dirPath}:parent`, - type: 'CONTAINS', - source: `folder:${parentPath}`, - target: `folder:${dirPath}` - }); - } else { - // Root folder connects to project - this.graph.relationships.push({ - id: `rel:folder:${dirPath}:project`, - type: 'CONTAINS', - source: `project:${this.projectName}`, - target: `folder:${dirPath}` - }); - } - } - } - } -} -``` - -**How this works:** - -1. Creates a root Project node -2. Walks through all file paths to identify directories -3. Creates Folder nodes and CONTAINS relationships - -**Why identify structure first?** We need to know the overall organization before parsing individual files. - -#### Pass 2: File Parsing - -```typescript -export class ParsingProcessor { - private graph: KnowledgeGraph; - private astCache = new Map(); - - constructor(graph: KnowledgeGraph) { - this.graph = graph; - } - - async parseFiles(filePaths: string[], fileContents: Map): Promise> { - for (const [filePath, content] of fileContents) { - if (filePath.endsWith('.py')) { - await this.parsePythonFile(filePath, content); - } - } - return this.astCache; - } - - private async parsePythonFile(filePath: string, content: string): Promise { - const parser = await loadPythonParser(); - const tree = parser.parse(content); - // Cache the AST - this.astCache.set(filePath, tree); - // Extract definitions from the AST - this.extractDefinitions(filePath, tree, content); - } - - private extractDefinitions(filePath: string, tree: any, content: string): void { - // Extract modules - this.graph.nodes.push({ - id: `module:${filePath}`, - label: 'Module', - properties: { - path: filePath, - name: filePath.split('/').pop()!.replace('.py', ''), - extension: '.py' - } - }); - - // Extract functions from the AST - const rootNode = tree.rootNode; - const functionDefs = rootNode.descendantsOfType('function_definition'); - for (const funcNode of functionDefs) { - const nameNode = funcNode.childForFieldName('name'); - const name = nameNode ? nameNode.text : 'unknown'; - - // Calculate position - const startLine = funcNode.startPosition.row + 1; - - // Create function node - this.graph.nodes.push({ - id: `function:${filePath}:${name}`, - label: 'Function', - properties: { - name, - qualified_name: `${this.getModuleName(filePath)}.${name}`, - path: filePath, - start_line: startLine - } - }); - - // Create CONTAINS relationship with module - this.graph.relationships.push({ - id: `rel:function:${filePath}:${name}:module`, - type: 'CONTAINS', - source: `module:${filePath}`, - target: `function:${filePath}:${name}` - }); - } - } -} -``` - -**How this works:** - -1. Parses each file with the appropriate Tree-sitter parser -2. Caches the AST for later use -3. Extracts definitions (functions, classes) from the AST -4. Creates nodes and relationships in the graph - -**Why cache ASTs?** We need them in Pass 3 to resolve cross-file function calls. - -#### Pass 3: Call Resolution - -```typescript -export class CallProcessor { - private graph: KnowledgeGraph; - private astCache: Map; - private projectRoot: string; - private projectName: string; - - constructor( - graph: KnowledgeGraph, - astCache: Map, - projectRoot: string, - projectName: string - ) { - this.graph = graph; - this.astCache = astCache; - this.projectRoot = projectRoot; - this.projectName = projectName; - } - - processCalls(): void { - for (const [filePath, tree] of this.astCache) { - if (filePath.endsWith('.py')) { - this.processPythonCalls(filePath, tree); - } - } - } - - private processPythonCalls(filePath: string, tree: any): void { - const rootNode = tree.rootNode; - // Find all call expressions - const callExpressions = rootNode.descendantsOfType('call'); - for (const callNode of callExpressions) { - const functionNameNode = callNode.childForFieldName('function'); - if (!functionNameNode) continue; - - // Handle different types of function references - let targetFunctionName = ''; - if (functionNameNode.type === 'identifier') { - targetFunctionName = functionNameNode.text; - } else if (functionNameNode.type === 'attribute') { - // Handle method calls like obj.method() - const attrNode = functionNameNode; - const objectNode = attrNode.childForFieldName('object'); - const attrNameNode = attrNode.childForFieldName('attribute'); - if (objectNode && attrNameNode) { - const objectName = objectNode.text; - const methodName = attrNameNode.text; - targetFunctionName = `${objectName}.${methodName}`; - } - } - - if (!targetFunctionName) continue; - - // Try to resolve the target function - const targetNode = this.resolveTargetFunction(targetFunctionName, filePath); - if (targetNode) { - // Create CALLS relationship - const callerId = this.getCallerId(callNode, filePath); - this.graph.relationships.push({ - id: `rel:call:${callerId}:${targetNode.id}`, - type: 'CALLS', - source: callerId, - target: targetNode.id - }); - } - } - } - - private resolveTargetFunction(targetName: string, currentFilePath: string): { id: string; type: string } | null { - // 1. Check if it's a built-in function - if (this.isBuiltInFunction(targetName)) { - return { - id: `builtin:${targetName}`, - type: 'builtin' - }; - } - - // 2. Check if it's an imported function - const importInfo = this.findImportForFunction(targetName, currentFilePath); - if (importInfo) { - const targetId = `function:${importInfo.sourceFile}:${importInfo.targetName}`; - return { - id: targetId, - type: 'imported' - }; - } - - // 3. Check if it's defined in the current file - for (const node of this.graph.nodes) { - if (node.label === 'Function' && - node.properties.name === targetName && - node.properties.path === currentFilePath) { - return { - id: node.id, - type: 'local' - }; - } - } - - return null; - } -} -``` - -**How this works:** - -1. Finds all function calls in the AST -2. Determines what function is being called -3. Resolves the target function across files using imports -4. Creates CALLS relationships in the graph - -**Why is this the hardest part?** Resolving cross-file references requires understanding: - -- How imports work in the language -- How to map a simple name to a fully qualified name -- Handling edge cases like aliases (`import helper as h`) - -### Step 8: Implement Web Workers for Performance - -**Why this matters:** Parsing code and building graphs can be CPU-intensive. Web Workers keep the UI responsive. - -**Key concepts:** - -- Web Workers run JavaScript in background threads -- They can't access the DOM directly -- We use Comlink to simplify communication - -**Implementation:** - -```typescript -// src/workers/ingestion.worker.ts -import { expose } from 'comlink'; -import { GraphPipeline } from '../core/ingestion/pipeline'; - -class IngestionWorker { - async processRepository( - projectRoot: string, - projectName: string, - filePaths: string[], - fileContents: Record - ) { - const pipeline = new GraphPipeline(projectRoot, projectName); - return pipeline.run(filePaths, new Map(Object.entries(fileContents))); - } -} - -expose(new IngestionWorker()); -``` - -**How this works:** - -1. The worker runs the heavy processing in a background thread -2. We expose methods via Comlink to call them from the main thread -3. The main thread can call these methods without blocking the UI - -**Why use Web Workers?** Without them, large repositories would freeze the browser tab while processing. - -## Phase 4: Graph Visualization - -### Step 9: Implement Graph Visualization Components - -**Why this matters:** A knowledge graph is useless if users can't see and interact with it. - -**Key concepts:** - -- Cytoscape.js is a powerful graph visualization library -- We need to convert our graph data to Cytoscape's format -- Users need controls to filter and navigate the graph - -**Implementation:** - -```tsx -import React, { useEffect, useRef } from 'react'; -import cytoscape from 'cytoscape'; -import dagre from 'cytoscape-dagre'; -import { KnowledgeGraph } from '@/core/graph/types'; - -cytoscape.use(dagre); - -interface GraphVisualizationProps { - graph: KnowledgeGraph; - onNodeClick?: (nodeId: string) => void; - filter?: (node: any) => boolean; -} - -export const GraphVisualization: React.FC = ({ - graph, - onNodeClick, - filter -}) => { - const containerRef = useRef(null); - const cyRef = useRef(null); - - useEffect(() => { - if (!containerRef.current) return; - - // Clean up previous instance - if (cyRef.current) { - cyRef.current.destroy(); - } - - // Convert our graph to Cytoscape format - const cyElements = convertToCytoscapeElements(graph, filter); - - const cy = cytoscape({ - container: containerRef.current, - elements: cyElements, - style: [ - { - selector: 'node', - style: { - 'label': 'data(label)', - 'width': 'mapData(size, 0, 100, 20, 80)', - 'height': 'mapData(size, 0, 100, 20, 80)', - 'background-color': 'data(color)', - 'text-valign': 'center', - 'text-halign': 'center', - 'font-size': '8px' - } - }, - { - selector: 'edge', - style: { - 'width': 2, - 'line-color': '#ccc', - 'target-arrow-color': '#ccc', - 'target-arrow-shape': 'triangle' - } - } - ], - layout: { - name: 'dagre', - rankDir: 'TB', - padding: 20 - } - }); - - // Add interactions - cy.on('tap', 'node', (event) => { - const node = event.target; - const nodeId = node.data('id'); - if (onNodeClick) { - onNodeClick(nodeId); - } - }); - - cyRef.current = cy; - - return () => { - if (cyRef.current) { - cyRef.current.destroy(); - cyRef.current = null; - } - }; - }, [graph, filter]); - - return ( -
- ); -}; - -function convertToCytoscapeElements( - graph: KnowledgeGraph, - filter?: (node: any) => boolean -) { - const elements: any[] = []; - - // Add nodes - for (const node of graph.nodes) { - if (filter && !filter(node)) continue; - elements.push({ - data: { - id: node.id, - label: getNodeLabel(node), - type: node.label, - color: getNodeColor(node.label), - size: getNodeSize(node) - } - }); - } - - // Add edges - for (const rel of graph.relationships) { - elements.push({ - data: { - id: rel.id, - source: rel.source, - target: rel.target, - label: rel.type - } - }); - } - - return elements; -} -``` - -**How this works:** - -1. Converts our graph data to Cytoscape's format -2. Sets up visual styles based on node type -3. Applies a hierarchical layout (dagre) -4. Adds interaction handlers for node clicks - -**Why use Cytoscape.js?** It's specifically designed for graph visualization with: - -- Multiple layout algorithms -- Good performance for medium-sized graphs -- Extensive customization options - -### Step 10: Create Source Code Viewer - -**Why this matters:** Seeing the graph isn't enough - users need to see the actual code behind the nodes. - -**Implementation:** - -```tsx -import React, { useState, useEffect } from 'react'; -import { KnowledgeGraph } from '@/core/graph/types'; - -interface SourceViewerProps { - graph: KnowledgeGraph; - selectedNodeId: string | null; -} - -export const SourceViewer: React.FC = ({ graph, selectedNodeId }) => { - const [sourceCode, setSourceCode] = useState(''); - const [fileName, setFileName] = useState(''); - const [lineNumber, setLineNumber] = useState(null); - - useEffect(() => { - if (!selectedNodeId) { - setSourceCode(''); - setFileName(''); - setLineNumber(null); - return; - } - - // Find the node in the graph - const node = graph.nodes.find(n => n.id === selectedNodeId); - if (!node) return; - - // For functions, get the source code - if (node.label === 'Function' || node.label === 'Method') { - const filePath = node.properties.path; - const startLine = node.properties.start_line; - - // In a real implementation, you'd have the source code available - setFileName(filePath); - setLineNumber(startLine); - setSourceCode(`# Source code for ${node.properties.qualified_name} -# Line ${startLine} and following...`); - } - }, [graph, selectedNodeId]); - - if (!selectedNodeId || !sourceCode) { - return ( -
-

Select a node to view source code

-
- ); - } - - return ( -
-
- {fileName} - {lineNumber && ( - Line {lineNumber} - )} -
-
-
{sourceCode}
-
-
- ); -}; -``` - -**How this works:** - -1. When a node is selected, it finds the corresponding code element -2. It displays the source code with line numbers -3. It highlights the relevant part of the code - -**Why is this important?** It bridges the gap between the abstract graph and the concrete code, helping users understand what they're seeing. - -## Phase 5: RAG Chat Interface - -### Step 11: Implement LLM Service - -**Why this matters:** The chat interface needs to connect to LLMs (Large Language Models) to translate natural language to graph queries. - -**Key concepts:** - -- We'll support multiple LLM providers (OpenAI, Anthropic, Gemini) -- Users provide their own API keys (privacy-focused) -- We need a consistent interface for different providers - -**Implementation:** - -```typescript -import { ChatOpenAI } from 'langchain/chat_models/openai'; -import { ChatAnthropic } from 'langchain/chat_models/anthropic'; -import { ChatGoogleGenerativeAI } from '@langchain/google-genai'; - -export type LLMProvider = 'openai' | 'anthropic' | 'gemini'; - -export interface LLMConfig { - provider: LLMProvider; - apiKey: string; - model?: string; -} - -export class LLMService { - private config: LLMConfig; - - constructor(config: LLMConfig) { - this.config = config; - } - - getChatModel() { - switch (this.config.provider) { - case 'openai': - return new ChatOpenAI({ - apiKey: this.config.apiKey, - modelName: this.config.model || 'gpt-4-turbo', - temperature: 0 - }); - case 'anthropic': - return new ChatAnthropic({ - apiKey: this.config.apiKey, - modelName: this.config.model || 'claude-3-sonnet-20240229', - temperature: 0 - }); - case 'gemini': - return new ChatGoogleGenerativeAI({ - apiKey: this.config.apiKey, - modelName: this.config.model || 'gemini-1.5-pro-latest', - temperature: 0 - }); - default: - throw new Error(`Unsupported LLM provider: ${this.config.provider}`); - } - } -} -``` - -**How this works:** - -1. The service takes an LLM configuration (provider, API key, model) -2. It returns a consistent chat model interface regardless of provider -3. It handles provider-specific initialization - -**Why support multiple providers?** Different users have different preferences and API key availability. - -### Step 12: Implement Cypher Generator - -**Why this matters:** The core of the RAG system - translating natural language questions to graph queries. - -**Key concepts:** - -- We use a system prompt to instruct the LLM -- The prompt includes our graph schema -- We clean the response to get a valid Cypher query - -**Implementation:** - -```typescript -import { BaseChatModel } from 'langchain/chat_models/base'; -import { CYPHER_SYSTEM_PROMPT } from '../prompts/cypher'; - -export class CypherGenerator { - private llm: BaseChatModel; - - constructor(llm: BaseChatModel) { - this.llm = llm; - } - - async generate(naturalLanguageQuery: string): Promise { - const response = await this.llm.call([ - { role: 'system', content: CYPHER_SYSTEM_PROMPT }, - { role: 'user', content: naturalLanguageQuery } - ]); - - return this.cleanResponse(response.content); - } - - private cleanResponse(response: string): string { - // Remove markdown code blocks - let cleaned = response.replace(/```cypher/g, '').replace(/```/g, ''); - // Ensure it ends with a semicolon - if (!cleaned.trim().endsWith(';')) { - cleaned = cleaned.trim() + ';'; - } - return cleaned; - } -} -``` - -**How this works:** - -1. It sends the natural language query with a system prompt to the LLM -2. The system prompt teaches the LLM about our graph structure -3. It cleans the response to extract a valid Cypher query - -**Why is the system prompt important?** It provides the LLM with the context it needs to generate correct queries. Without it, the LLM wouldn't know about our graph schema. - -### Step 13: Implement RAG Orchestrator - -**Why this matters:** This is the "brain" of the system that coordinates the query process. - -**Key concepts:** - -- It follows a ReAct (Reason + Act) pattern -- It plans steps, uses tools, observes results, and responds -- It prevents hallucination by sticking to tool results - -**Implementation:** - -```typescript -import { BaseChatModel } from 'langchain/chat_models/base'; -import { RAG_ORCHESTRATOR_SYSTEM_PROMPT } from '../prompts/rag-orchestrator'; - -export class RAGOrchestrator { - private llm: BaseChatModel; - - constructor(llm: BaseChatModel) { - this.llm = llm; - } - - async query( - userQuery: string, - queryGraph: (cypher: string) => Promise, - retrieveCode: (nodeId: string) => Promise - ) { - // Start with the system prompt - let conversation = [ - { role: 'system', content: RAG_ORCHESTRATOR_SYSTEM_PROMPT } - ]; - - // Add the user's question - conversation.push({ role: 'user', content: userQuery }); - - // Simple ReAct loop - for (let i = 0; i < 5; i++) { // Max 5 steps - const response = await this.llm.call(conversation); - const responseContent = response.content; - - // Check if the response contains a tool call - if (responseContent.includes('Action: query_graph')) { - const match = responseContent.match(/Action Input: (.*)/); - if (match) { - const cypherQuery = match[1].trim(); - - // Execute the query - const queryResults = await queryGraph(cypherQuery); - - // Add the observation to the conversation - conversation.push({ - role: 'assistant', - content: responseContent - }); - - conversation.push({ - role: 'system', - content: `Observation: ${JSON.stringify(queryResults)}` - }); - - // If we have results, we might be done - if (queryResults.length > 0) { - break; - } - } - } - else if (responseContent.includes('Action: retrieve_code')) { - // Similar handling for code retrieval - } - else { - // This appears to be the final answer - return responseContent; - } - } - - // If we got here without a final answer, generate one - conversation.push({ - role: 'user', - content: 'Please provide your final answer based on the information gathered.' - }); - - const finalResponse = await this.llm.call(conversation); - return finalResponse.content; - } -} -``` - -**How this works:** - -1. It starts with a system prompt that defines the rules -2. It sends the user's query to the LLM -3. The LLM responds with either: - - A tool call (query_graph or retrieve_code) - - A final answer -4. If it's a tool call, it executes the tool and adds the result to the conversation -5. It repeats until it gets a final answer or hits the step limit - -**Why the step limit?** To prevent infinite loops if the LLM gets stuck. - -## Phase 6: Main Application Integration - -### Step 14: Create Main Application Component - -**Why this matters:** This brings all the pieces together into a cohesive UI. - -**Implementation:** - -```tsx -import React, { useState, useRef } from 'react'; -import { GraphVisualization } from '@/ui/components/graph/Visualization'; -import { GraphControls } from '@/ui/components/graph/Controls'; -import { SourceViewer } from '@/ui/components/graph/SourceViewer'; -import { ChatInterface } from '@/ui/components/chat/ChatInterface'; -import { KnowledgeGraph } from '@/core/graph/types'; -import { GitHubService } from '@/services/github'; -import { ZipService } from '@/services/zip'; -import { ingestionWorkerApi } from '@/lib/workerUtils'; - -export const HomePage = () => { - const [graph, setGraph] = useState(null); - const [selectedNodeId, setSelectedNodeId] = useState(null); - const [isLoading, setIsLoading] = useState(false); - const [error, setError] = useState(null); - const [repoUrl, setRepoUrl] = useState(''); - const fileInputRef = useRef(null); - - const handleRepoSubmit = async (e: React.FormEvent) => { - e.preventDefault(); - if (!repoUrl.trim() || isLoading) return; - - setIsLoading(true); - setError(null); - - try { - // Parse the GitHub URL - const urlMatch = repoUrl.match(/github\.com\/([^/]+)\/([^/]+)/); - if (!urlMatch) { - throw new Error('Invalid GitHub repository URL'); - } - - const owner = urlMatch[1]; - const repo = urlMatch[2].replace(/\.git$/, ''); - - // Fetch repository contents - const githubService = new GitHubService(); - const contents = await githubService.getRepoContents(owner, repo); - - // Filter for Python files - const pythonFiles = contents - .filter((item: any) => item.type === 'file' && item.name.endsWith('.py')) - .map((item: any) => item.path); - - // Fetch file contents - const fileContents: Record = {}; - for (const filePath of pythonFiles) { - fileContents[filePath] = await githubService.getFileContent(owner, repo, filePath); - } - - // Process the repository - const projectName = `${owner}/${repo}`; - const processedGraph = await ingestionWorkerApi.processRepository( - repoUrl, - projectName, - pythonFiles, - fileContents - ); - - setGraph(processedGraph); - } catch (err) { - setError(err instanceof Error ? err.message : 'Failed to process repository'); - console.error('Processing error:', err); - } finally { - setIsLoading(false); - } - }; - - const handleQuery = async (query: string): Promise => { - if (!graph) { - throw new Error('No graph available'); - } - - // In a real implementation, this would use the RAG orchestrator - return `I found information related to "${query}" in the codebase.`; - }; - - return ( -
- {/* Header with repository input */} -
-
-

GitNexus

- -
-
-
- setRepoUrl(e.target.value)} - placeholder="https://github.com/owner/repo.git" - className="flex-1 px-3 py-2 border border-gray-300 rounded-md focus:outline-none focus:ring-1 focus:ring-blue-500 focus:border-blue-500" - /> - -
-
- -
- or - - -
-
-
-
- -
- {/* Graph Visualization Pane */} -
-
- {graph ? ( - <> - {}} - onLayoutChange={() => {}} - /> -
- -
- - ) : ( -
- {isLoading ? ( -
-
-

Processing repository...

-
- ) : ( -

Enter a repository URL or upload a ZIP to get started

- )} -
- )} -
-
- - {/* Right Panel */} -
- {/* Source Viewer */} -
-
-

Source Code

-
- {graph ? ( - - ) : ( -
-

Select a node to view source

-
- )} -
- - {/* Chat Interface */} -
-
-

Ask About Code

-
- {graph ? ( - - ) : ( -
-

Process a repository to ask questions about the code

-
- )} -
-
-
-
- ); -}; -``` - -**How this works:** - -1. The header has inputs for GitHub URLs and ZIP uploads -2. The main area has two panes: - - Left: Graph visualization - - Right: Source viewer and chat interface -3. When a repository is processed, the graph is displayed -4. Users can click nodes to see source code and ask questions - -**Why this layout?** It provides a cohesive experience where users can: - -- See the big picture (graph) -- Drill down to specific code (source viewer) -- Ask questions about what they're seeing (chat) - -## Final Steps: Testing and Optimization - -### Step 15: Add Error Boundaries - -**Why this matters:** Inevitably, something will go wrong. We want to handle errors gracefully. - -**Implementation:** - -```tsx -import React, { Component, ErrorInfo, ReactNode } from 'react'; - -interface Props { - children: ReactNode; -} - -interface State { - hasError: boolean; - error: Error | null; -} - -export class ErrorBoundary extends Component { - public state: State = { - hasError: false, - error: null - }; - - public static getDerivedStateFromError(error: Error): State { - return { hasError: true, error }; - } - - public componentDidCatch(error: Error, errorInfo: ErrorInfo) { - console.error("Uncaught error:", error, errorInfo); - } - - public render() { - if (this.state.hasError) { - return ( -
-

Something went wrong

-

{this.state.error?.message}

- -
- ); - } - - return this.props.children; - } -} -``` - -**How this works:** - -- It catches JavaScript errors in child components -- It displays a friendly error message instead of a blank screen -- It allows users to try again without losing their work - -**Why use error boundaries?** They prevent a single error from breaking the entire application. - -### Step 16: Implement Performance Optimizations - -**Why this matters:** Large repositories can be slow to process. We need to keep the UI responsive. - -**Key optimizations:** - -1. **Web Workers**: Already implemented for graph processing -2. **Progress Indicators**: Show users what's happening -3. **File Filtering**: Only process relevant files -4. **Lazy Loading**: Load components as needed - -**Implementation:** - -```tsx -// Add to your GitHub processing function -const MAX_FILES = 500; // Limit for free tier -if (pythonFiles.length > MAX_FILES) { - // Offer to filter by directory or file pattern - const shouldFilter = window.confirm( - `Repository has ${pythonFiles.length} Python files (max ${MAX_FILES}). ` + - `Would you like to filter by directory or file pattern?` - ); - if (shouldFilter) { - const filterPattern = prompt( - "Enter a directory path or file pattern to filter (e.g., 'src/', '*.py')", - "src/" - ); - if (filterPattern) { - const filteredFiles = pythonFiles.filter(file => - file.includes(filterPattern) || file.endsWith(filterPattern) - ); - pythonFiles = filteredFiles; - } - } -} -``` - -**Why limit file processing?** Processing too many files can: - -- Freeze the browser tab -- Exceed GitHub API rate limits -- Use excessive memory - -### Step 17: Add Export Functionality - -**Why this matters:** Users might want to save or share their generated graphs. - -**Implementation:** - -```tsx -export function exportGraphToJson(graph: KnowledgeGraph): string { - return JSON.stringify(graph, null, 2); -} - -export function downloadGraph(graph: KnowledgeGraph, filename: string = 'gitnexus-graph.json') { - const json = exportGraphToJson(graph); - const blob = new Blob([json], { type: 'application/json' }); - const url = URL.createObjectURL(blob); - const a = document.createElement('a'); - a.href = url; - a.download = filename; - document.body.appendChild(a); - a.click(); - document.body.removeChild(a); - URL.revokeObjectURL(url); -} -``` - -**How this works:** - -1. Converts the graph to JSON -2. Creates a downloadable file -3. Triggers a download - -**Why include export?** It allows users to: - -- Save their work for later -- Share graphs with teammates -- Use the data in other tools - -## Conclusion - -This implementation guide has walked you through building a complete edge-based code knowledge graph generator using Deno. By following these steps, you'll create a privacy-focused tool that runs entirely in the user's browser. - -**Key advantages of this approach:** - -- **Zero server costs**: All processing happens in the user's browser -- **Strong privacy**: Code never leaves the user's machine -- **Modular architecture**: Easy to add more languages later -- **Clear separation of concerns**: Makes the codebase maintainable -- **Deno compatibility**: Modern runtime with built-in TypeScript support - -Remember to start small (Python support only) and iterate, adding more features and language support as you validate the core functionality. The most important part is getting the graph construction pipeline working correctly - everything else builds on that foundation. +## ๐Ÿ”’ Security & Privacy + +- **Client-Side Processing**: All analysis happens in your browser +- **API Keys**: Stored locally, never transmitted to our servers +- **GitHub Access**: Uses public API, respects repository permissions +- **Data Privacy**: No code or analysis results are stored remotely +- **Error Logging**: Sensitive data excluded from error reports +- **Export Security**: User-controlled data export with no server interaction + +## ๐Ÿค Contributing + +### Development Setup +1. Fork the repository +2. Create feature branch: `git checkout -b feature/amazing-feature` +3. Make changes and test thoroughly +4. Run the testing checklist +5. Commit: `git commit -m 'Add amazing feature'` +6. Push: `git push origin feature/amazing-feature` +7. Open a Pull Request + +### Code Style +- **TypeScript**: Strict mode enabled +- **ESLint**: Follow configured rules +- **Prettier**: Auto-formatting +- **Comments**: Minimal, only when necessary +- **Error Handling**: Comprehensive error boundaries and recovery +- **Performance**: Consider memory usage and processing time + +## ๐Ÿ“„ License + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. + +## ๐Ÿ™ Acknowledgments + +- **Tree-sitter**: Syntax parsing infrastructure +- **LangChain.js**: AI agent framework +- **Cytoscape.js**: Graph visualization +- **React**: UI framework with error boundaries +- **Vite**: Build tool and dev server +- **KuzuDB**: Embedded graph database \ No newline at end of file diff --git a/REFACTORING_ANALYSIS.md b/REFACTORING_ANALYSIS.md deleted file mode 100644 index b95c49a63..000000000 --- a/REFACTORING_ANALYSIS.md +++ /dev/null @@ -1,524 +0,0 @@ -# GitNexus Codebase Separation Analysis & Refactoring Plan - -## Executive Summary - -The GitNexus codebase already has a **dual-track architecture** with Legacy (sequential + in-memory) and Next-Gen (parallel + KuzuDB) processing engines. However, there are several areas where separation of concerns can be improved without changing functionality or appearance. - -**Current Status**: โœ… Engine abstractions exist, โŒ Service layer has duplication, โŒ UI has engine-specific logic - -**Goal**: Create clear separation while maintaining exact current functionality and appearance. - -## Current Architecture Analysis - -### 1. Engine Layer (โœ… WELL SEPARATED) - -**Location**: `src/core/engines/` - -**Current State**: EXCELLENT separation already exists -- โœ… `engine-interface.ts` - Clean abstraction layer -- โœ… `legacy/legacy-engine.ts` - Wraps `IngestionService` + `GraphPipeline` -- โœ… `nextgen/nextgen-engine.ts` - Wraps `KuzuIngestionService` + `KuzuGraphPipeline` - -**Strengths**: -- Common `ProcessingEngine` interface -- Proper fallback mechanisms -- Performance monitoring for both tracks -- Engine validation and health checks - -**Minor Issues**: -- Services instantiated in constructors (tight coupling) -- No service injection pattern - -### 2. Service Layer (โŒ MAJOR DUPLICATION FOUND) - -**Location**: `src/services/` - -**Current State**: SIGNIFICANT code duplication between services - -#### Issues Identified: - -**A) Ingestion Services Duplication (90% identical code)** -- `ingestion.service.ts` (195 lines) vs `kuzu-ingestion.service.ts` (198 lines) -- Both use identical: GitHub/ZIP services, URL parsing, path normalization -- Only difference: Pipeline type (`GraphPipeline` vs `KuzuGraphPipeline`) - -**B) Shared Services with Mixed Concerns** -- `github.ts` and `zip.ts` are shared but have no engine-specific optimizations -- Both ingestion services use identical logic for repository discovery - -**Code Analysis**: -```typescript -// ingestion.service.ts -class IngestionService { - async processGitHubRepo(url, options) { - // 1. Parse GitHub URL (identical) - // 2. Get repository structure (identical) - // 3. Normalize paths (identical) - // 4. Use GraphPipeline (different) - } - - private normalizeZipPaths() { /* Identical 40 lines */ } -} - -// kuzu-ingestion.service.ts -class KuzuIngestionService { - async processGitHubRepo(url, options) { - // 1. Parse GitHub URL (identical) - // 2. Get repository structure (identical) - // 3. Normalize paths (identical) - // 4. Use KuzuGraphPipeline (different) - } - - private normalizeZipPaths() { /* Identical 40 lines */ } -} -``` - -### 3. Pipeline Layer (โœ… GOOD SEPARATION) - -**Location**: `src/core/ingestion/` - -**Current State**: PROPER separation with shared processors - -**Pipelines**: -- โœ… `pipeline.ts` (`GraphPipeline`) - Sequential processing -- โœ… `kuzu-pipeline.ts` (`KuzuGraphPipeline`) - Parallel processing -- โœ… `parallel-pipeline.ts` (`ParallelGraphPipeline`) - Alternative parallel implementation - -**Processors**: Shared appropriately -- โœ… `StructureProcessor`, `ImportProcessor`, `CallProcessor` - Shared (good) -- โœ… `ParsingProcessor` vs `ParallelParsingProcessor` - Separate (good) - -**Assessment**: This layer already has excellent separation. - -### 4. Graph Interface Layer (โš ๏ธ PARTIAL UNIFICATION) - -**Location**: `src/core/graph/` - -**Current State**: Two graph types with some unification - -```typescript -// SimpleKnowledgeGraph (Legacy) -interface KnowledgeGraph { - nodes: GraphNode[]; - relationships: GraphRelationship[]; -} - -// KuzuKnowledgeGraph (Next-Gen) -interface KuzuKnowledgeGraphInterface extends KnowledgeGraph { - getNodeCount(): number; - getRelationshipCount(): number; - query(cypher: string): Promise; -} -``` - -**Issues**: -- UI uses conditional logic: `graph.getNodeCount ? graph.getNodeCount() : graph.nodes.length` -- Different capabilities not properly abstracted - -### 5. UI Components (โš ๏ธ SOME ENGINE-SPECIFIC LOGIC) - -**Location**: `src/ui/` - -**Current State**: Generally well structured but has some engine coupling - -**Well Separated**: -- โœ… `hooks/useEngine.ts` - Engine management -- โœ… `components/engine/` - Engine selection components -- โœ… `hooks/useGitNexus.ts` - Main orchestration - -**Issues Found**: -- Processing status shows engine-specific details -- Export functionality not engine-aware -- Some conditional logic based on engine capabilities - -### 6. Configuration System (โš ๏ธ NEEDS ENGINE SECTIONS) - -**Location**: `src/config/` - -**Current State**: Basic configuration without engine-specific sections - -**Existing**: -- โœ… `config.ts` - Core configuration -- โœ… `feature-flags.ts` - Feature toggles -- โš ๏ธ Missing engine-specific configurations - -## Implementation Progress - -### โœ… Phase 1 Completed: Service Layer Refactoring - -**Status**: COMPLETED - All service layer refactoring implemented successfully - -#### โœ… Step 1.1: Base Ingestion Service Created -- โœ… `src/services/common/base-ingestion.service.ts` - Abstract base class with 90% shared logic -- โœ… Extracted: GitHub URL parsing, repository discovery, ZIP normalization, progress reporting -- โœ… Template method pattern implemented for pipeline processing - -#### โœ… Step 1.2: Service Factory Implementation -- โœ… `src/services/service.factory.ts` - Centralized service creation -- โœ… Dynamic imports to avoid circular dependencies -- โœ… Engine validation and fallback support -- โœ… Clean abstraction for service instantiation - -#### โœ… Step 1.3: Engine-Specific Service Implementations -- โœ… `src/services/legacy/legacy-ingestion.service.ts` - Legacy engine implementation -- โœ… `src/services/nextgen/nextgen-ingestion.service.ts` - Next-Gen engine implementation -- โœ… Both extend BaseIngestionService with engine-specific pipeline logic - -#### โœ… Step 1.4: Backward Compatibility Maintained -- โœ… `src/services/ingestion.service.ts` - Updated to use service factory internally -- โœ… `src/services/kuzu-ingestion.service.ts` - Updated to use service factory internally -- โœ… Existing API completely preserved for zero breaking changes - -#### โœ… Step 1.5: Engine Wrappers Updated -- โœ… `src/core/engines/legacy/legacy-engine.ts` - Now uses ServiceFactory -- โœ… `src/core/engines/nextgen/nextgen-engine.ts` - Now uses ServiceFactory -- โœ… Proper dependency injection pattern implemented - -**Results Achieved**: -- โœ… 90% code duplication eliminated between ingestion services -- โœ… Clear separation between Legacy and Next-Gen implementations -- โœ… Zero breaking changes - existing code works unchanged -- โœ… Proper abstraction layers with dependency injection -- โœ… No compilation errors - -### โœ… Phase 2 Completed: Configuration Enhancement - -**Status**: COMPLETED - Engine-specific configuration system implemented - -#### โœ… Step 2.1: Engine Configuration Schema Added -- โœ… `LegacyEngineConfigSchema` - Memory limits, processing settings, worker configuration -- โœ… `NextGenEngineConfigSchema` - KuzuDB settings, parallel processing, worker pool configuration -- โœ… `EngineConfigSchema` - Runtime settings, fallback configuration, performance monitoring -- โœ… Proper Zod validation with sensible defaults - -#### โœ… Step 2.2: Environment Variable Support -- โœ… `ENGINE_DEFAULT` - Set default engine (legacy/nextgen) -- โœ… `ENGINE_ALLOW_FALLBACK` - Enable/disable engine fallback -- โœ… `ENGINE_LEGACY_*` - Legacy engine specific settings -- โœ… `ENGINE_NEXTGEN_*` - Next-Gen engine specific settings -- โœ… Hardware-aware defaults (worker count based on CPU cores) - -#### โœ… Step 2.3: Configuration Integration -- โœ… Updated `ConfigService` to load engine configuration -- โœ… Enhanced validation with engine-specific checks -- โœ… Proper configuration fallbacks and error handling - -#### โœ… Step 2.4: Engine Wrapper Integration -- โœ… Legacy engine validates configuration and logs settings -- โœ… Next-Gen engine validates configuration and logs settings -- โœ… Engines respect enabled/disabled state from configuration - -#### โœ… Step 2.5: Configuration Helper Created -- โœ… `src/config/engine-config.helper.ts` - UI-friendly configuration access -- โœ… Engine availability checking, fallback determination -- โœ… Display information for UI components -- โœ… Processing options and validation utilities - -**Results Achieved**: -- โœ… Complete engine-specific configuration system -- โœ… Environment variable support for all settings -- โœ… Runtime engine switching capabilities -- โœ… Hardware-aware configuration defaults -- โœ… Comprehensive validation and error handling - -### โœ… Phase 3 Completed: UI Component Enhancement - -**Status**: COMPLETED - UI components enhanced with configuration awareness - -#### โœ… Step 3.1: Engine Selector Enhancement -- โœ… Enhanced `EngineSelector.tsx` to use `EngineConfigHelper` -- โœ… Shows engine status (enabled/disabled) from configuration -- โœ… Displays engine features and descriptions from config -- โœ… Shows fallback status and configuration warnings -- โœ… Better visual indicators for engine states - -#### โœ… Step 3.2: Processing Status Enhancement -- โœ… Enhanced `ProcessingStatus.tsx` with configuration awareness -- โœ… Shows engine-specific processing options during execution -- โœ… Displays configuration-based engine descriptions -- โœ… Performance indicators based on engine type -- โœ… More detailed engine information in success state - -#### โœ… Step 3.3: Configuration Documentation -- โœ… Created `.env.example` with comprehensive engine configuration -- โœ… Documented all engine-specific environment variables -- โœ… Provided example configurations for different use cases -- โœ… Clear separation between Legacy and Next-Gen settings - -**Results Achieved**: -- โœ… UI components now configuration-aware -- โœ… Engine status properly reflected in interface -- โœ… User-friendly display of engine capabilities -- โœ… Clear documentation for customization - -### โœ… REFACTORING COMPLETED SUCCESSFULLY! - -**Final Status**: ALL PHASES COMPLETED - Clear separation of concerns achieved - -#### ๐ŸŽ† Summary of Achievements: - -**1. Service Layer Refactoring** โœ… -- 90% code duplication eliminated -- Clear Legacy/Next-Gen separation -- Service factory pattern implemented -- Backward compatibility maintained - -**2. Configuration System** โœ… -- Engine-specific configuration schemas -- Environment variable support -- Runtime configuration validation -- UI-friendly configuration helpers - -**3. UI Component Enhancement** โœ… -- Configuration-aware components -- Engine status visibility -- User-friendly engine information -- Comprehensive documentation - -#### ๐Ÿ“Š Benefits Realized: - -**Maintainability**: -- Clear separation between Legacy and Next-Gen code -- No code duplication in service layer -- Proper abstraction layers with dependency injection - -**Configurability**: -- Easy engine switching via configuration -- Hardware-aware defaults -- Environment-specific settings -- Runtime configuration validation - -**User Experience**: -- Transparent engine operation -- Clear engine status indicators -- Comprehensive configuration options -- Excellent fallback mechanisms - -**Developer Experience**: -- Well-documented configuration -- Clear architectural boundaries -- Easy to extend and maintain -- Comprehensive error handling - -### ๐Ÿ”„ Current Status: Ready for Production - -**โœ… All Requirements Met**: -- โœ… Exact same functionality and appearance -- โœ… Clear separation of concerns -- โœ… Maintainable code structure -- โœ… Easy engine switching via configuration -- โœ… Zero breaking changes - -**Next Steps for User**: -1. Copy `.env.example` to `.env` and customize as needed -2. Test both engines work correctly -3. Configure default engine preference -4. Optionally enable performance monitoring - ---- - -## Detailed Refactoring Plan - -### Phase 1: Service Layer Refactoring (HIGH PRIORITY) - -#### Step 1.1: Create Base Ingestion Service - -**Goal**: Extract 90% shared logic from both ingestion services - -**New File**: `src/services/common/base-ingestion.service.ts` - -```typescript -// Abstract base class with shared logic -abstract class BaseIngestionService { - protected githubService: GitHubService; - protected zipService: ZipService; - - // Shared methods: - // - GitHub URL parsing - // - Repository structure discovery - // - ZIP path normalization - // - Progress reporting - - // Abstract method for pipeline creation - protected abstract createPipeline(): Pipeline; -} -``` - -**Files to Update**: -- `src/services/ingestion.service.ts` โ†’ Extend base class -- `src/services/kuzu-ingestion.service.ts` โ†’ Extend base class - -#### Step 1.2: Service Factory Pattern - -**New File**: `src/services/service.factory.ts` - -```typescript -class ServiceFactory { - static createIngestionService(engine: ProcessingEngineType, token?: string): BaseIngestionService { - switch (engine) { - case 'legacy': return new LegacyIngestionService(token); - case 'nextgen': return new NextGenIngestionService(token); - } - } -} -``` - -#### Step 1.3: Update Engine Wrappers - -**Files to Update**: -- `src/core/engines/legacy/legacy-engine.ts` -- `src/core/engines/nextgen/nextgen-engine.ts` - -**Change**: Use ServiceFactory instead of direct instantiation - -### Phase 2: Configuration Enhancement - -#### Step 2.1: Engine-Specific Configuration - -**File to Update**: `src/config/config.ts` - -**Add Engine Configuration Schema**: -```typescript -interface EngineConfig { - legacy: { - enabled: boolean; - memoryLimits: { maxMemoryMB: number; gcIntervalMs: number }; - processing: { batchSize: number; timeoutMs: number }; - }; - nextgen: { - enabled: boolean; - kuzu: { databasePath: string; bufferPoolSize: number }; - parallel: { maxWorkers: number; batchSize: number }; - }; - runtime: { - defaultEngine: 'legacy' | 'nextgen'; - allowFallback: boolean; - performanceMonitoring: boolean; - }; -} -``` - -### Phase 3: UI Component Updates (LOW PRIORITY) - -#### Step 3.1: Engine-Aware Components - -**Files to Update**: -- `src/ui/components/graph/GraphExplorer.tsx` - Remove conditional logic -- `src/ui/components/ExportFormatModal.tsx` - Make engine-aware -- `src/ui/pages/HomePage/HomePage.tsx` - Clean up engine-specific styling - -#### Step 3.2: Enhanced Status Components - -**File to Update**: `src/ui/components/engine/ProcessingStatus.tsx` - -**Enhancement**: Show unified status regardless of engine - -### Phase 4: Graph Interface Unification (MEDIUM PRIORITY) - -#### Step 4.1: Unified Graph Interface - -**File to Update**: `src/core/graph/types.ts` - -**Goal**: Create unified interface that works with both graph types - -```typescript -interface UnifiedKnowledgeGraph { - // Common interface - nodes: GraphNode[]; - relationships: GraphRelationship[]; - - // Unified methods - getEngineType(): 'legacy' | 'nextgen'; - getCapabilities(): string[]; - - // Optional advanced methods - query?(cypher: string): Promise; -} -``` - -## Implementation Strategy - -### Week 1: Service Layer Refactoring -1. Create `BaseIngestionService` abstract class -2. Extract shared logic (URL parsing, normalization, progress reporting) -3. Update existing services to extend base class -4. Create `ServiceFactory` for centralized service creation -5. Update engine wrappers to use factory - -### Week 2: Configuration Enhancement -1. Add engine-specific configuration schema -2. Implement environment variable support -3. Add runtime engine selection configuration -4. Update configuration service - -### Week 3: UI Component Cleanup -1. Remove engine-specific conditional logic from components -2. Enhance status and export components -3. Clean up engine-specific styling - -### Week 4: Testing & Validation -1. Ensure exact same functionality and appearance -2. Test engine switching -3. Verify performance characteristics remain unchanged -4. Test fallback mechanisms - -## File Impact Analysis - -### Files to Create: -- `src/services/common/base-ingestion.service.ts` -- `src/services/service.factory.ts` - -### Files to Modify (Major Changes): -- `src/services/ingestion.service.ts` -- `src/services/kuzu-ingestion.service.ts` -- `src/core/engines/legacy/legacy-engine.ts` -- `src/core/engines/nextgen/nextgen-engine.ts` -- `src/config/config.ts` - -### Files to Modify (Minor Changes): -- `src/ui/components/graph/GraphExplorer.tsx` -- `src/ui/components/ExportFormatModal.tsx` -- `src/ui/pages/HomePage/HomePage.tsx` -- `src/core/graph/types.ts` - -## Risk Assessment - -### Low Risk โœ… -- Service layer refactoring (extracting shared logic) -- Configuration enhancements -- Factory pattern implementation - -### Medium Risk โš ๏ธ -- Graph interface unification -- UI component updates - -### Zero Risk โœ… -- Pipeline layer (already well separated) -- Engine wrapper layer (already excellent) - -## Success Criteria - -### Functional Requirements โœ… -- [ ] Exact same UI appearance and behavior -- [ ] Both engines work identically to current implementation -- [ ] Engine switching works flawlessly -- [ ] No performance degradation -- [ ] All existing features work unchanged - -### Code Quality Requirements โœ… -- [ ] 90% reduction in service layer duplication -- [ ] Clear separation between Legacy and Next-Gen implementations -- [ ] Maintainable code structure -- [ ] Proper abstraction layers -- [ ] Comprehensive configuration system - ---- - -## Next Steps - -1. **Validate Current Functionality**: Test both engines work correctly -2. **Start with Service Layer**: Begin Phase 1 refactoring -3. **Incremental Testing**: Test after each major change -4. **Maintain Backward Compatibility**: Ensure no breaking changes - -This refactoring will create clear separation of concerns while maintaining the exact same functionality and appearance. \ No newline at end of file diff --git a/REFACTORING_COMPLETE.md b/REFACTORING_COMPLETE.md deleted file mode 100644 index 972c3508e..000000000 --- a/REFACTORING_COMPLETE.md +++ /dev/null @@ -1,204 +0,0 @@ -# GitNexus Refactoring Completion Summary - -## ๐ŸŽ‰ Refactoring Successfully Completed! - -Your GitNexus codebase has been successfully refactored for **clear separation of concerns** while maintaining **100% backward compatibility**. The application works exactly the same as before, but now has a much cleaner and more maintainable architecture. - -## โœ… What Was Accomplished - -### 1. Service Layer Refactoring (90% Code Duplication Eliminated) - -**Before**: Two nearly identical ingestion services with 90% duplicated code -**After**: Clean inheritance hierarchy with shared base class - -``` -New Architecture: -src/services/ -โ”œโ”€โ”€ common/base-ingestion.service.ts # Shared logic (90% of code) -โ”œโ”€โ”€ legacy/legacy-ingestion.service.ts # Legacy-specific (10% of code) -โ”œโ”€โ”€ nextgen/nextgen-ingestion.service.ts # Next-Gen-specific (10% of code) -โ”œโ”€โ”€ service.factory.ts # Centralized service creation -โ”œโ”€โ”€ ingestion.service.ts # Backward compatibility wrapper -โ””โ”€โ”€ kuzu-ingestion.service.ts # Backward compatibility wrapper -``` - -**Benefits**: -- ๐Ÿ”ง Eliminated 160+ lines of duplicated code -- ๐ŸŽฏ Clear separation between Legacy and Next-Gen implementations -- ๐Ÿ”„ Zero breaking changes - existing code works unchanged -- ๐Ÿ—๏ธ Proper dependency injection with service factory pattern - -### 2. Engine-Specific Configuration System - -**Before**: Basic configuration without engine-specific settings -**After**: Comprehensive configuration system with engine separation - -``` -New Configuration: -src/config/ -โ”œโ”€โ”€ config.ts # Enhanced with engine schemas -โ”œโ”€โ”€ engine-config.helper.ts # UI-friendly configuration access -โ””โ”€โ”€ .env.example # Complete configuration examples -``` - -**Features**: -- โš™๏ธ Separate Legacy and Next-Gen engine configuration -- ๐ŸŒ Environment variable support for all settings -- ๐Ÿ”ง Hardware-aware defaults (CPU cores detection) -- โœ… Runtime configuration validation -- ๐ŸŽ›๏ธ Easy engine switching via configuration - -### 3. Enhanced UI Components - -**Before**: Basic engine selection with minimal configuration awareness -**After**: Configuration-aware components with detailed engine information - -**Enhancements**: -- ๐ŸŽจ Engine status indicators (enabled/disabled) -- ๐Ÿ“Š Configuration-based engine descriptions -- ๐Ÿ”„ Fallback status display -- โšก Processing options shown during execution -- ๐Ÿ† Performance indicators based on engine type - -## ๐Ÿ”ง How to Use the New Configuration - -### 1. Basic Setup - -Copy the example configuration: -```bash -cp .env.example .env -``` - -### 2. Engine Selection - -Choose your default engine: -```bash -# For stable, sequential processing -ENGINE_DEFAULT=legacy - -# For high-performance, parallel processing -ENGINE_DEFAULT=nextgen -``` - -### 3. Engine-Specific Tuning - -**Legacy Engine (Sequential + In-Memory)**: -```bash -ENGINE_LEGACY_MEMORY_LIMIT_MB=512 -ENGINE_LEGACY_BATCH_SIZE=10 -ENGINE_LEGACY_USE_WORKERS=true -``` - -**Next-Gen Engine (Parallel + KuzuDB)**: -```bash -ENGINE_NEXTGEN_MAX_WORKERS=4 -ENGINE_NEXTGEN_BATCH_SIZE=20 -ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=256 -``` - -### 4. Safety Features - -Enable fallback for maximum reliability: -```bash -ENGINE_ALLOW_FALLBACK=true -ENGINE_PERFORMANCE_MONITORING=true -``` - -## ๐Ÿ“ New File Structure - -``` -src/ -โ”œโ”€โ”€ services/ -โ”‚ โ”œโ”€โ”€ common/ -โ”‚ โ”‚ โ””โ”€โ”€ base-ingestion.service.ts # โœจ NEW: Shared logic -โ”‚ โ”œโ”€โ”€ legacy/ -โ”‚ โ”‚ โ””โ”€โ”€ legacy-ingestion.service.ts # โœจ NEW: Legacy implementation -โ”‚ โ”œโ”€โ”€ nextgen/ -โ”‚ โ”‚ โ””โ”€โ”€ nextgen-ingestion.service.ts # โœจ NEW: Next-Gen implementation -โ”‚ โ””โ”€โ”€ service.factory.ts # โœจ NEW: Service factory -โ”œโ”€โ”€ config/ -โ”‚ โ”œโ”€โ”€ config.ts # ๐Ÿ”ง ENHANCED: Engine configuration -โ”‚ โ””โ”€โ”€ engine-config.helper.ts # โœจ NEW: Configuration helper -โ”œโ”€โ”€ core/engines/ -โ”‚ โ”œโ”€โ”€ legacy/legacy-engine.ts # ๐Ÿ”ง ENHANCED: Uses service factory -โ”‚ โ””โ”€โ”€ nextgen/nextgen-engine.ts # ๐Ÿ”ง ENHANCED: Uses service factory -โ””โ”€โ”€ ui/components/engine/ - โ”œโ”€โ”€ EngineSelector.tsx # ๐Ÿ”ง ENHANCED: Configuration-aware - โ””โ”€โ”€ ProcessingStatus.tsx # ๐Ÿ”ง ENHANCED: Shows engine details -``` - -## ๐Ÿš€ Benefits Achieved - -### For Developers -- **Maintainability**: Clear separation makes code easier to understand and modify -- **Testability**: Each engine can be tested independently -- **Extensibility**: Easy to add new engines or modify existing ones -- **Debugging**: Clear boundaries help isolate issues - -### For Users -- **Reliability**: Fallback mechanisms ensure processing always works -- **Performance**: Choose the right engine for your use case -- **Transparency**: Clear visibility into which engine is being used -- **Customization**: Fine-tune processing for your specific needs - -### For Operations -- **Configuration**: Comprehensive environment variable support -- **Monitoring**: Built-in performance monitoring -- **Flexibility**: Runtime engine switching without code changes -- **Documentation**: Clear examples and configuration guidance - -## ๐Ÿ›ก๏ธ Backward Compatibility Guarantee - -**Zero Breaking Changes**: All existing code continues to work exactly as before: - -- โœ… `IngestionService` still works the same way -- โœ… `KuzuIngestionService` still works the same way -- โœ… Engine wrappers maintain the same interface -- โœ… UI components look and behave identically -- โœ… All existing functionality preserved - -## ๐Ÿ”„ Migration Path - -**Immediate** (Already Done): -- โœ… Service factory architecture implemented -- โœ… Configuration system enhanced -- โœ… UI components improved -- โœ… Documentation created - -**Optional Next Steps**: -1. **Customize Configuration**: Edit `.env` file for your preferences -2. **Test Both Engines**: Verify both Legacy and Next-Gen work for your use cases -3. **Set Default Engine**: Choose your preferred engine -4. **Enable Monitoring**: Turn on performance monitoring if desired - -## ๐Ÿ“Š Current Status - -- ๐ŸŸข **Application Status**: Running perfectly -- ๐ŸŸข **Legacy Engine**: Fully functional with new architecture -- ๐ŸŸข **Next-Gen Engine**: Fully functional with new architecture -- ๐ŸŸข **Configuration**: Complete and tested -- ๐ŸŸข **UI Components**: Enhanced and working -- ๐ŸŸข **Documentation**: Comprehensive and complete - -## ๐ŸŽฏ What's Next - -The refactoring is **complete and ready for production use**. You can now: - -1. **Continue Development**: Focus on your parallel + KuzuDB work with clear separation -2. **Easy Testing**: Switch between engines via configuration to test new features -3. **Gradual Migration**: Keep Legacy as fallback while perfecting Next-Gen -4. **Team Collaboration**: Clear boundaries make team development easier - -## ๐Ÿ” Quick Test - -To verify everything works: - -1. Application should be running at `http://localhost:5173` -2. UI should look identical to before -3. Both engines should be available in the engine selector -4. Processing should work exactly as before -5. Configuration changes should take effect after restart - ---- - -**๐ŸŽ‰ Congratulations!** Your codebase now has crystal-clear separation of concerns while maintaining full backward compatibility. The foundation is set for easy maintenance and continued development of your parallel + KuzuDB features! \ No newline at end of file diff --git a/STRUCTURE_FIX_SUMMARY.md b/STRUCTURE_FIX_SUMMARY.md deleted file mode 100644 index becbd5ed2..000000000 --- a/STRUCTURE_FIX_SUMMARY.md +++ /dev/null @@ -1,176 +0,0 @@ -# ๐Ÿ”ง GitNexus Structure Discovery Fix - Complete Architecture Overhaul - -## ๐Ÿšจ **Critical Flaw Identified and Fixed** - -### **The Problem** -The original GitNexus architecture had a **fatal flaw** in repository structure discovery: - -- **Flawed Logic**: `StructureProcessor` inferred directory existence from filtered file paths -- **Critical Bug**: Empty directories or directories containing only filtered-out files were **completely missing** from the knowledge graph -- **Result**: Incomplete and inaccurate codebase representation - -### **Root Cause Analysis** -``` -โŒ OLD BROKEN FLOW: -GitHub/ZIP Service โ†’ Filter Files โ†’ Pass Filtered Paths โ†’ Infer Structure - โ†‘ FILTERING HERE BREAKS STRUCTURE DISCOVERY -``` - -**The Fundamental Issue**: Filtering happened **before** structure discovery, causing the `StructureProcessor` to never see paths for directories that contained only filtered-out files. - -## ๐Ÿ—๏ธ **The New Robust Architecture** - -### **Core Principle** -> **Discover Complete Structure First, Filter During Parsing** - -``` -โœ… NEW ROBUST FLOW: -GitHub/ZIP Service โ†’ Discover ALL Paths โ†’ Build Complete Structure โ†’ Filter During Parsing - โ†‘ NO FILTERING YET โ†‘ COMPLETE STRUCTURE โ†‘ FILTERING HERE -``` - -### **Architectural Changes** - -## **1. Data Acquisition Services (Complete Structure Discovery)** - -### **GitHub Service (`src/services/github.ts`)** -- โœ… **New Method**: `getCompleteRepositoryStructure()` -- โœ… **Returns**: `CompleteRepositoryStructure` with `allPaths` + `fileContents` -- โœ… **Behavior**: Discovers **every file and directory** in the repository -- โœ… **No Filtering**: Collects all content regardless of user filters - -### **ZIP Service (`src/services/zip.ts`)** -- โœ… **New Method**: `extractCompleteStructure()` -- โœ… **Returns**: `CompleteZipStructure` with `allPaths` + `fileContents` -- โœ… **Enhanced Logic**: Explicitly tracks directories and intermediate paths -- โœ… **Path Normalization**: Handles common top-level folder removal - -## **2. Ingestion Service (Pipeline Orchestration)** - -### **Updated Methods (`src/services/ingestion.service.ts`)** -- โœ… **`processGitHubRepo()`**: Uses complete structure discovery -- โœ… **`processZipFile()`**: Uses complete structure discovery -- โœ… **No Filtering**: Passes **all discovered paths** to pipeline -- โœ… **Clean Architecture**: Filtering responsibility moved to `ParsingProcessor` - -## **3. Structure Processor (Direct Path Processing)** - -### **Complete Rewrite (`src/core/ingestion/structure-processor.ts`)** -```typescript -// OLD: Infer structure from filtered file paths -const folderPaths = this.extractFolderPaths(filePaths); // โŒ BROKEN - -// NEW: Process complete discovered structure directly -const { directories, files } = this.categorizePaths(filePaths); // โœ… ROBUST -``` - -#### **Key Improvements**: -- โœ… **Direct Processing**: No inference, direct path categorization -- โœ… **Complete Structure**: Processes **all** discovered paths -- โœ… **Smart Categorization**: Distinguishes files from directories algorithmically -- โœ… **Intermediate Directories**: Automatically adds missing intermediate paths -- โœ… **Perfect Mirror**: KG structure exactly matches repository file system - -## **4. Parsing Processor (Filtering During Parsing)** - -### **New Filtering Logic (`src/core/ingestion/parsing-processor.ts`)** -```typescript -// NEW: Filtering happens here, during parsing -private applyFiltering( - allPaths: string[], - fileContents: Map, - options?: { directoryFilter?: string; fileExtensions?: string } -): string[] -``` - -#### **Filtering Strategy**: -- โœ… **Input**: Receives **all** paths from structure discovery -- โœ… **Apply Filters**: Directory and extension filters applied here -- โœ… **Parse Only Filtered**: Only processes files that pass filters -- โœ… **Structure Intact**: All directories remain in graph, regardless of filtering - -## **5. Pipeline Integration** - -### **Updated Pipeline (`src/core/ingestion/pipeline.ts`)** -- โœ… **4-Pass Architecture**: Maintains existing pass structure -- โœ… **Options Passing**: Filtering options passed to `ParsingProcessor` -- โœ… **Complete Structure**: `StructureProcessor` gets all paths -- โœ… **Filtered Parsing**: `ParsingProcessor` applies user filters - -## ๐Ÿ“Š **Before vs After Comparison** - -| Aspect | โŒ **Before (Broken)** | โœ… **After (Robust)** | -|--------|------------------------|----------------------| -| **Structure Discovery** | Inferred from filtered files | Direct discovery of all paths | -| **Empty Directories** | Missing from KG | Present in KG | -| **Filtered Directories** | Missing if all files filtered | Present in KG | -| **Filtering Location** | Before structure discovery | During parsing phase | -| **KG Completeness** | Incomplete, inaccurate | Complete, accurate mirror | -| **Architecture** | Monolithic, coupled | Decoupled, robust | - -## ๐ŸŽฏ **Results and Benefits** - -### **Immediate Fixes** -1. **โœ… Empty Directories**: Now appear in knowledge graph -2. **โœ… Filtered Directories**: Directories with only filtered files now appear -3. **โœ… Complete Structure**: KG is a perfect mirror of repository structure -4. **โœ… Accurate Representation**: No missing parts of codebase - -### **Architectural Improvements** -1. **๐Ÿ”ง Separation of Concerns**: Structure discovery โ‰  Content filtering -2. **๐Ÿ”ง Robust Design**: No inference, direct discovery -3. **๐Ÿ”ง Maintainable**: Clear responsibility boundaries -4. **๐Ÿ”ง Extensible**: Easy to add new file types or filtering logic - -### **User Experience** -1. **๐Ÿ“ˆ Accurate Graphs**: Users see complete repository structure -2. **๐Ÿ“ˆ Better Navigation**: All directories visible for exploration -3. **๐Ÿ“ˆ Consistent Results**: Same structure regardless of filter settings -4. **๐Ÿ“ˆ Trust**: KG accurately represents their codebase - -## ๐Ÿ” **Technical Implementation Details** - -### **Path Categorization Algorithm** -```typescript -// Smart algorithm to distinguish files from directories -const isDirectory = allPaths.some(otherPath => - otherPath !== path && otherPath.startsWith(path + '/') -); -``` - -### **Intermediate Directory Discovery** -```typescript -// Automatically discover missing intermediate directories -for (let i = 1; i < parts.length; i++) { - const intermediatePath = parts.slice(0, i).join('/'); - if (intermediatePath && !pathSet.has(intermediatePath)) { - allIntermediateDirs.add(intermediatePath); - } -} -``` - -### **Filtering During Parsing** -```typescript -// Apply user filters only during parsing phase -if (options.directoryFilter?.trim()) { - filesToProcess = filesToProcess.filter(path => - dirPatterns.some(pattern => path.toLowerCase().includes(pattern)) - ); -} -``` - -## ๐Ÿš€ **Deployment Status** - -- โœ… **Build Status**: All components compile successfully -- โœ… **Integration**: Complete end-to-end pipeline updated -- โœ… **Testing Ready**: Architecture ready for validation -- โœ… **Backward Compatible**: Existing functionality preserved -- โœ… **Performance**: Optimized with batching and memory management - -## ๐ŸŽ‰ **Conclusion** - -This architectural overhaul transforms GitNexus from a **flawed, incomplete** structure discovery system into a **robust, accurate** repository analysis tool. - -**The critical flaw is now fixed**: GitNexus will discover and represent the **complete** repository structure, ensuring users get an accurate and comprehensive knowledge graph of their codebase. - -**Key Success Metric**: The knowledge graph structure is now a **perfect mirror** of the actual repository file system, regardless of user filtering preferences. \ No newline at end of file diff --git a/WORKER_POOL_IMPLEMENTATION.md b/WORKER_POOL_IMPLEMENTATION.md deleted file mode 100644 index 67b4237cf..000000000 --- a/WORKER_POOL_IMPLEMENTATION.md +++ /dev/null @@ -1,448 +0,0 @@ -# Worker Pool Implementation Guide - -## ๐ŸŽฏ Overview - -The Worker Pool implementation provides **massive performance improvements** for large codebases by parallelizing CPU-intensive operations like Tree-sitter parsing, AST analysis, and code processing. - -## ๐Ÿš€ Performance Benefits - -### **Expected Speedup:** -- **Small codebases (< 100 files)**: 1.5-2x speedup -- **Medium codebases (100-1000 files)**: 2-4x speedup -- **Large codebases (1000+ files)**: 4-8x speedup - -### **Key Improvements:** -- **Parallel file parsing** - Multiple files processed simultaneously -- **Concurrent Tree-sitter operations** - AST generation in parallel -- **Better CPU utilization** - Leverages all available cores -- **Improved UI responsiveness** - Main thread freed up - ---- - -## ๐Ÿ“ File Structure - -``` -src/ -โ”œโ”€โ”€ lib/ -โ”‚ โ”œโ”€โ”€ web-worker-pool.ts # Main worker pool implementation -โ”‚ โ””โ”€โ”€ worker-pool-test.ts # Test suite -โ”œโ”€โ”€ core/ingestion/ -โ”‚ โ”œโ”€โ”€ parallel-parsing-processor.ts # Parallel parsing with workers -โ”‚ โ””โ”€โ”€ parallel-pipeline.ts # Parallel pipeline integration -โ””โ”€โ”€ config/ - โ””โ”€โ”€ feature-flags.ts # Worker pool feature flags - -public/workers/ -โ”œโ”€โ”€ tree-sitter-worker.js # Tree-sitter parsing worker -โ”œโ”€โ”€ generic-worker.js # Generic processing worker -โ””โ”€โ”€ file-processing-worker.js # File analysis worker -``` - ---- - -## ๐Ÿ”ง Core Components - -### **1. WebWorkerPool Class** -```typescript -import { WebWorkerPool } from './src/lib/web-worker-pool.js'; - -const workerPool = new WebWorkerPool({ - maxWorkers: navigator.hardwareConcurrency, - workerScript: '/workers/tree-sitter-worker.js', - timeout: 30000, - name: 'MyWorkerPool' -}); -``` - -**Key Features:** -- **Automatic worker management** - Creates, recycles, and terminates workers -- **Task queuing** - Handles task distribution and load balancing -- **Error recovery** - Graceful handling of worker failures -- **Progress tracking** - Real-time progress callbacks -- **Statistics** - Detailed performance metrics - -### **2. ParallelParsingProcessor** -```typescript -import { ParallelParsingProcessor } from './src/core/ingestion/parallel-parsing-processor.ts'; - -const processor = new ParallelParsingProcessor(); -await processor.process(graph, { - filePaths: ['file1.ts', 'file2.ts'], - fileContents: fileContentsMap, - options: { useParallelProcessing: true } -}); -``` - -**Key Features:** -- **Parallel file parsing** - Processes multiple files simultaneously -- **Worker pool integration** - Uses WebWorkerPool for CPU-intensive tasks -- **Memory optimization** - Efficient result aggregation -- **Error handling** - Continues processing even if some files fail - -### **3. ParallelGraphPipeline** -```typescript -import { ParallelGraphPipeline } from './src/core/ingestion/parallel-pipeline.ts'; - -const pipeline = new ParallelGraphPipeline(); -pipeline.setProgressCallback((progress) => { - console.log(`${progress.phase}: ${progress.progress}%`); -}); - -const graph = await pipeline.run({ - projectRoot: '/path/to/project', - projectName: 'MyProject', - filePaths: allFilePaths, - fileContents: fileContentsMap, - options: { useParallelProcessing: true } -}); -``` - ---- - -## ๐ŸŽฎ Usage Examples - -### **Basic Worker Pool Usage** -```typescript -import { WebWorkerPool } from './src/lib/web-worker-pool.js'; - -// Create worker pool -const workerPool = new WebWorkerPool({ - maxWorkers: 4, - workerScript: '/workers/generic-worker.js' -}); - -// Execute single task -const result = await workerPool.execute({ - taskType: 'textAnalysis', - text: 'Hello world!', - analysisType: 'wordCount' -}); - -// Execute multiple tasks in parallel -const results = await workerPool.executeAll([ - { taskType: 'textAnalysis', text: 'Task 1', analysisType: 'wordCount' }, - { taskType: 'textAnalysis', text: 'Task 2', analysisType: 'wordCount' } -]); - -// Execute with progress tracking -const results = await workerPool.executeWithProgress( - tasks, - (completed, total) => { - console.log(`Progress: ${(completed/total)*100}%`); - } -); - -// Cleanup -await workerPool.shutdown(); -``` - -### **File Processing with Workers** -```typescript -import { WebWorkerPoolUtils } from './src/lib/web-worker-pool.js'; - -// Create specialized file processing pool -const filePool = WebWorkerPoolUtils.createCPUPool({ - workerScript: '/workers/file-processing-worker.js' -}); - -// Analyze file structure -const analysis = await filePool.execute({ - processorType: 'analyzeStructure', - filePath: '/src/main.ts', - content: fileContent -}); - -// Extract dependencies -const dependencies = await filePool.execute({ - processorType: 'extractDependencies', - filePath: '/src/main.ts', - content: fileContent -}); -``` - -### **Parallel Pipeline Integration** -```typescript -import { ParallelGraphPipeline } from './src/core/ingestion/parallel-pipeline.ts'; - -// Check if parallel processing is supported -if (ParallelGraphPipeline.isParallelProcessingSupported()) { - const pipeline = new ParallelGraphPipeline(); - - // Set up progress tracking - pipeline.setProgressCallback((progress) => { - console.log(`${progress.phase}: ${progress.message} (${progress.progress}%)`); - }); - - // Run parallel processing - const graph = await pipeline.run({ - projectRoot: '/path/to/project', - projectName: 'MyProject', - filePaths: allFilePaths, - fileContents: fileContentsMap, - options: { - useParallelProcessing: true, - maxWorkers: ParallelGraphPipeline.getOptimalWorkerCount() - } - }); - - // Get worker pool statistics - const stats = pipeline.getWorkerPoolStats(); - console.log('Worker pool stats:', stats); -} -``` - ---- - -## โš™๏ธ Configuration - -### **Feature Flags** -```typescript -import { featureFlags } from './src/config/feature-flags.js'; - -// Enable worker pool features -featureFlags.enableWorkerPool(); - -// Or configure individually -featureFlags.setFlags({ - enableWorkerPool: true, - enableParallelParsing: true, - enableParallelProcessing: true -}); - -// Check if enabled -if (featureFlags.getFlag('enableWorkerPool')) { - // Use worker pool -} -``` - -### **Worker Pool Options** -```typescript -const workerPoolOptions = { - maxWorkers: navigator.hardwareConcurrency, // Number of workers - workerScript: '/workers/tree-sitter-worker.js', // Worker script path - timeout: 30000, // Task timeout in milliseconds - name: 'MyWorkerPool' // Pool name for logging -}; -``` - -### **Optimal Worker Counts** -```typescript -import { WebWorkerPoolUtils } from './src/lib/web-worker-pool.js'; - -// Get optimal worker count for different task types -const cpuWorkers = WebWorkerPoolUtils.getOptimalWorkerCount('cpu'); -const ioWorkers = WebWorkerPoolUtils.getOptimalWorkerCount('io'); -const mixedWorkers = WebWorkerPoolUtils.getOptimalWorkerCount('mixed'); - -// Get hardware concurrency -const concurrency = WebWorkerPoolUtils.getHardwareConcurrency(); -``` - ---- - -## ๐Ÿงช Testing - -### **Run Test Suite** -```typescript -import { runWorkerPoolTests } from './src/lib/worker-pool-test.js'; - -// Run all tests -await runWorkerPoolTests(); -``` - -### **Individual Tests** -```typescript -import { - testWorkerPoolBasic, - testFileProcessingPool, - testWorkerPoolPerformance, - testWorkerPoolErrorHandling -} from './src/lib/worker-pool-test.js'; - -// Test basic functionality -await testWorkerPoolBasic(); - -// Test file processing -await testFileProcessingPool(); - -// Test performance -await testWorkerPoolPerformance(); - -// Test error handling -await testWorkerPoolErrorHandling(); -``` - -### **Browser Console Testing** -```javascript -// Available globally in browser -await window.runWorkerPoolTests(); -await window.testWorkerPoolBasic(); -await window.testWorkerPoolPerformance(); -``` - ---- - -## ๐Ÿ“Š Performance Monitoring - -### **Worker Pool Statistics** -```typescript -const stats = workerPool.getStats(); -console.log({ - totalWorkers: stats.totalWorkers, - availableWorkers: stats.availableWorkers, - activeTasks: stats.activeTasks, - queuedTasks: stats.queuedTasks, - maxWorkers: stats.maxWorkers, - memoryUsage: stats.memoryUsage -}); -``` - -### **Performance Metrics** -```typescript -// Track processing time -const startTime = performance.now(); -const results = await workerPool.executeAll(tasks); -const endTime = performance.now(); - -console.log({ - totalTime: endTime - startTime, - averageTimePerTask: (endTime - startTime) / tasks.length, - processingRate: tasks.length / ((endTime - startTime) / 1000) -}); -``` - ---- - -## ๐Ÿšจ Error Handling - -### **Worker Errors** -```typescript -workerPool.on('workerError', (data) => { - console.warn(`Worker ${data.workerId} error:`, data.error); -}); - -workerPool.on('workerCreated', (data) => { - console.log(`Worker ${data.workerId} created`); -}); - -workerPool.on('shutdown', () => { - console.log('Worker pool shutdown'); -}); -``` - -### **Task Timeouts** -```typescript -try { - const result = await workerPool.execute(task); -} catch (error) { - if (error.message.includes('timed out')) { - console.warn('Task timed out, retrying...'); - // Implement retry logic - } -} -``` - -### **Fallback to Sequential Processing** -```typescript -if (ParallelGraphPipeline.isParallelProcessingSupported()) { - // Use parallel processing - const pipeline = new ParallelGraphPipeline(); -} else { - // Fallback to sequential processing - const pipeline = new GraphPipeline(); -} -``` - ---- - -## ๐Ÿ”„ Migration Guide - -### **From Sequential to Parallel Processing** - -**Before (Sequential):** -```typescript -import { GraphPipeline } from './src/core/ingestion/pipeline.ts'; - -const pipeline = new GraphPipeline(); -const graph = await pipeline.run(input); -``` - -**After (Parallel):** -```typescript -import { ParallelGraphPipeline } from './src/core/ingestion/parallel-pipeline.ts'; - -const pipeline = new ParallelGraphPipeline(); -pipeline.setProgressCallback((progress) => { - console.log(`${progress.phase}: ${progress.progress}%`); -}); - -const graph = await pipeline.run({ - ...input, - options: { useParallelProcessing: true } -}); -``` - -### **From BatchProcessor to WorkerPool** - -**Before (Sequential batches):** -```typescript -const batchProcessor = new BatchProcessor(10, async (files) => { - for (const file of files) { - await processFile(file); // Sequential within batch - } -}); -``` - -**After (Parallel workers):** -```typescript -const workerPool = new WebWorkerPool({ - maxWorkers: 4, - workerScript: '/workers/tree-sitter-worker.js' -}); - -const results = await workerPool.executeAll( - files.map(file => ({ filePath: file, content: fileContents.get(file) })) -); -``` - ---- - -## ๐ŸŽฏ Best Practices - -### **1. Worker Pool Configuration** -- **CPU-intensive tasks**: Use `navigator.hardwareConcurrency` workers -- **I/O-intensive tasks**: Use 2-4x more workers than CPU cores -- **Mixed tasks**: Use 2-8 workers depending on workload - -### **2. Task Design** -- **Keep tasks independent** - Avoid shared state between workers -- **Serialize data efficiently** - Minimize data transfer overhead -- **Handle errors gracefully** - Implement proper error recovery - -### **3. Memory Management** -- **Monitor memory usage** - Use `performance.memory` API -- **Clean up resources** - Always call `workerPool.shutdown()` -- **Batch large datasets** - Process in chunks to avoid memory issues - -### **4. Performance Optimization** -- **Profile worker performance** - Monitor task execution times -- **Adjust worker count** - Find optimal balance for your workload -- **Use appropriate timeouts** - Set realistic timeout values - ---- - -## ๐Ÿš€ Ready to Use! - -The Worker Pool implementation is **fully functional** and ready for production use. It provides: - -โœ… **Massive performance improvements** for large codebases -โœ… **Automatic worker management** and error recovery -โœ… **Progress tracking** and performance monitoring -โœ… **Easy integration** with existing pipeline -โœ… **Comprehensive testing** and documentation - -**Next Steps:** -1. Test with your codebase to measure performance gains -2. Adjust worker counts based on your system capabilities -3. Monitor memory usage and optimize as needed -4. Enjoy faster, more responsive code analysis! ๐ŸŽ‰ diff --git a/log.txt b/log.txt deleted file mode 100644 index ac062b0d4..000000000 --- a/log.txt +++ /dev/null @@ -1,492 +0,0 @@ -Starting ZIP processing... trumio-cortex-core.zip -zip.ts:64 Starting complete ZIP extraction of: trumio-cortex-core.zip (27513697 bytes) -zip.ts:118 Skipping large file: trumio-cortex-core/.git/objects/pack/pack-4d5e84ce4d4c68c3dcdaf68f49c25a0693180f62.pack (27326542 bytes) -(anonymous) @ zip.ts:118 -forEach @ jszip.js?v=7c98553f:349 -extractCompleteStructure @ zip.ts:102 -await in extractCompleteStructure -processZipFile @ ingestion.service.ts:94 -handleFileUpload @ HomePage.tsx:118 -callCallback2 @ chunk-YZVM2MHU.js?v=5b2c6c96:3674 -invokeGuardedCallbackDev @ chunk-YZVM2MHU.js?v=5b2c6c96:3699 -invokeGuardedCallback @ chunk-YZVM2MHU.js?v=5b2c6c96:3733 -invokeGuardedCallbackAndCatchFirstError @ chunk-YZVM2MHU.js?v=5b2c6c96:3736 -executeDispatch @ chunk-YZVM2MHU.js?v=5b2c6c96:7014 -processDispatchQueueItemsInOrder @ chunk-YZVM2MHU.js?v=5b2c6c96:7034 -processDispatchQueue @ chunk-YZVM2MHU.js?v=5b2c6c96:7043 -dispatchEventsForPlugins @ chunk-YZVM2MHU.js?v=5b2c6c96:7051 -(anonymous) @ chunk-YZVM2MHU.js?v=5b2c6c96:7174 -batchedUpdates$1 @ chunk-YZVM2MHU.js?v=5b2c6c96:18913 -batchedUpdates @ chunk-YZVM2MHU.js?v=5b2c6c96:3579 -dispatchEventForPluginEventSystem @ chunk-YZVM2MHU.js?v=5b2c6c96:7173 -dispatchEventWithEnableCapturePhaseSelectiveHydrationWithoutDiscreteEventReplay @ chunk-YZVM2MHU.js?v=5b2c6c96:5478 -dispatchEvent @ chunk-YZVM2MHU.js?v=5b2c6c96:5472 -dispatchDiscreteEvent @ chunk-YZVM2MHU.js?v=5b2c6c96:5449Understand this warning -zip.ts:146 ZIP: Discovered 271 total paths, 109 files with content -zip.ts:147 ZIP: Total extracted size: 401503 bytes -workerUtils.ts:43 Ingestion worker initialized successfully -ingestion.worker.ts?worker_file&type=module:15 IngestionWorker: Starting processing with 271 files -pipeline.ts:19 ๐Ÿš€ Starting 4-pass ingestion for project: trumio-cortex-core -pipeline.ts:20 ๐Ÿ“ Pass 1: Analyzing project structure... -structure-processor.ts:71 StructureProcessor: Processing 271 complete paths -structure-processor.ts:75 StructureProcessor: Found 74 directories and 197 files -structure-processor.ts:79 StructureProcessor: Hiding 33 ignored directories from KG -structure-processor.ts:86 StructureProcessor: Hiding 82 files in ignored directories from KG -structure-processor.ts:227 StructureProcessor: Created 156 CONTAINS relationships -structure-processor.ts:92 StructureProcessor: Created 157 nodes total (115 items hidden) -pipeline.ts:26 ๐Ÿ” Pass 2: Parsing code and extracting definitions... -parsing-processor.ts:40 ParsingProcessor: Processing 271 total paths -parsing-processor.ts:42 Memory status: 0MB used, 0 files cached -parsing-processor.ts:44 ParsingProcessor: After filtering: 32 files to parse -parsing-processor.ts:49 ParsingProcessor: Found 22 source files and 3 config files, processing in batches of 10 -parser-loader.ts:65 Loading TypeScript parser from: /wasm/typescript/tree-sitter-typescript.wasm -parser-loader.ts:68 TypeScript parser loaded successfully -parsing-processor.ts:121 typescript parser loaded successfully. -parser-loader.ts:49 Loading JavaScript parser from: /wasm/javascript/tree-sitter-javascript.wasm -parser-loader.ts:52 JavaScript parser loaded successfully -parsing-processor.ts:121 javascript parser loaded successfully. -parser-loader.ts:33 Loading Python parser from: /wasm/python/tree-sitter-python.wasm -parser-loader.ts:36 Python parser loaded successfully -parsing-processor.ts:121 python parser loaded successfully. -parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/docker-compose.yaml -parseFile @ parsing-processor.ts:203 -(anonymous) @ parsing-processor.ts:112 -await in (anonymous) -processAll @ shared-utils.ts:274 -await in processAll -process @ parsing-processor.ts:122 -await in process -run @ pipeline.ts:49 -await in run -processRepository @ ingestion.worker.ts:68 -callback @ comlink.js?v=4af0e45c:91Understand this warning -parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/manifests/uat/deployment_cortex.yml -parseFile @ parsing-processor.ts:203 -(anonymous) @ parsing-processor.ts:112 -await in (anonymous) -processAll @ shared-utils.ts:274 -await in processAll -process @ parsing-processor.ts:122 -await in process -run @ pipeline.ts:49 -await in run -processRepository @ ingestion.worker.ts:68 -callback @ comlink.js?v=4af0e45c:91Understand this warning -parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/manifests/uat/service_cortex.yml -parseFile @ parsing-processor.ts:203 -(anonymous) @ parsing-processor.ts:112 -await in (anonymous) -processAll @ shared-utils.ts:274 -await in processAll -process @ parsing-processor.ts:122 -await in process -run @ pipeline.ts:49 -await in run -processRepository @ ingestion.worker.ts:68 -callback @ comlink.js?v=4af0e45c:91Understand this warning -parsing-processor.ts:72 ParsingProcessor: Successfully processed 25 files -pipeline.ts:36 ๐Ÿ”— Pass 3: Resolving imports and building dependency map... -import-processor.ts:42 ImportProcessor: Starting import resolution... -import-processor.ts:52 ImportProcessor: Completed import resolution -import-processor.ts:53 ImportProcessor: Found 160 imports, resolved 90 (56.3%) -import-processor.ts:54 ImportProcessor: Built import map for 22 files -pipeline.ts:38 ๐Ÿ“ž Pass 4: Resolving function calls with 3-stage strategy... -call-processor.ts:33 CallProcessor: Starting call resolution with 3-stage strategy... -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in check_prompt.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: AsyncIOMotorClient in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: limit in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: sort in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: find in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: close in check_prompt.py -call-processor.ts:553 ๐Ÿ” Filtered out: run in check_prompt.py -call-processor.ts:65 CallProcessor: Found 1 function calls in trumio-cortex-core/check_prompt.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: AsyncIOMotorClient in debug_channel_messages.py -2call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: find_one in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: now in debug_channel_messages.py -call-processor.ts:65 CallProcessor: Found 5 function calls in trumio-cortex-core/debug_channel_messages.py -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:24 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:31 -call-processor.ts:86 โŒ Failed to resolve call: timedelta in trumio-cortex-core/debug_channel_messages.py:49 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:51 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in mcp_auth_service.py -call-processor.ts:65 CallProcessor: Found 14 function calls in trumio-cortex-core/src/python/common_services/mcp_auth_service.py -call-processor.ts:86 โŒ Failed to resolve call: HTTPBaseModel in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:32 -call-processor.ts:86 โŒ Failed to resolve call: hexdigest in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:99 -call-processor.ts:86 โŒ Failed to resolve call: new in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:99 -call-processor.ts:86 โŒ Failed to resolve call: compare_digest in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:130 -call-processor.ts:86 โŒ Failed to resolve call: body in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:227 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in openai_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in openai_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ValueError in openai_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: OpenAI in openai_service.py -call-processor.ts:65 CallProcessor: Found 12 function calls in trumio-cortex-core/src/python/common_services/openai_service.py -call-processor.ts:86 โŒ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:30 -call-processor.ts:86 โŒ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:31 -call-processor.ts:86 โŒ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:32 -call-processor.ts:86 โŒ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:33 -call-processor.ts:86 โŒ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:89 -call-processor.ts:86 โŒ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:90 -call-processor.ts:86 โŒ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:91 -call-processor.ts:86 โŒ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:92 -call-processor.ts:86 โŒ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:122 -call-processor.ts:86 โŒ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:123 -call-processor.ts:86 โŒ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:124 -call-processor.ts:86 โŒ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:125 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: isinstance in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: TypeError in prompt_registry_service.py -2call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ValueError in prompt_registry_service.py -2call-processor.ts:553 ๐Ÿ” Filtered out: now in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: insert_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: str in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: limit in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: sort in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: error in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ValueError in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: now in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: update_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ObjectId in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: delete_one in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: ObjectId in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: find in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in prompt_registry_service.py -call-processor.ts:553 ๐Ÿ” Filtered out: limit in prompt_registry_service.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/common_services/prompt_registry_service.py -call-processor.ts:58 ๐Ÿ“Š Debug: prompt_registry_service.py has 37 call nodes, 7 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: append in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: HTTPException in chat.py -call-processor.ts:553 ๐Ÿ” Filtered out: str in chat.py -call-processor.ts:65 CallProcessor: Found 7 function calls in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:15 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:28 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:29 -11call-processor.ts:553 ๐Ÿ” Filtered out: Field in chat.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/demo_app/app/schemas/chat.py -call-processor.ts:58 ๐Ÿ“Š Debug: chat.py has 11 call nodes, 4 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: basicConfig in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: StreamHandler in start-service.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in start-service.py -call-processor.ts:65 CallProcessor: Found 15 function calls in trumio-cortex-core/src/python/demo_app/start-service.py -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:26 -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:27 -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:28 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:37 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:40 -call-processor.ts:86 โŒ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:43 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:59 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:62 -call-processor.ts:86 โŒ Failed to resolve call: get_motor_client in trumio-cortex-core/src/python/demo_app/start-service.py:77 -call-processor.ts:86 โŒ Failed to resolve call: compile_graphs in trumio-cortex-core/src/python/demo_app/start-service.py:87 -call-processor.ts:86 โŒ Failed to resolve call: close_motor_client in trumio-cortex-core/src/python/demo_app/start-service.py:99 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:132 -call-processor.ts:86 โŒ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:135 -call-processor.ts:86 โŒ Failed to resolve call: RedirectResponse in trumio-cortex-core/src/python/demo_app/start-service.py:141 -call-processor.ts:86 โŒ Failed to resolve call: get_swagger_ui_html in trumio-cortex-core/src/python/demo_app/start-service.py:145 -call-processor.ts:553 ๐Ÿ” Filtered out: append in interactions_endpoint.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in interactions_endpoint.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in interactions_endpoint.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in interactions_endpoint.py -call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:14 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:16 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:18 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:19 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:20 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:21 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:22 -call-processor.ts:86 โŒ Failed to resolve call: fetch_and_process_project_chats_for_period in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:32 -call-processor.ts:553 ๐Ÿ” Filtered out: append in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in mcp_auth.py -call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py:14 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py:69 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in config.py -9call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: lower in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -3call-processor.ts:553 ๐Ÿ” Filtered out: bool in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in config.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/config.py -call-processor.ts:58 ๐Ÿ“Š Debug: config.py has 19 call nodes, 0 definitions -10call-processor.ts:553 ๐Ÿ” Filtered out: Field in processed_text_unit.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/schemas/processed_text_unit.py -call-processor.ts:58 ๐Ÿ“Š Debug: processed_text_unit.py has 10 call nodes, 1 definitions -2call-processor.ts:553 ๐Ÿ” Filtered out: Field in watermark.py -call-processor.ts:553 ๐Ÿ” Filtered out: now in watermark.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/schemas/watermark.py -call-processor.ts:58 ๐Ÿ“Š Debug: watermark.py has 3 call nodes, 2 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: append in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in path_utils.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/utils/path_utils.py -call-processor.ts:58 ๐Ÿ“Š Debug: path_utils.py has 9 call nodes, 1 definitions -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: AsyncIOMotorClient in debug_channel_messages.py -3call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: aggregate in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: aggregate in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: to_list in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in debug_channel_messages.py -call-processor.ts:65 CallProcessor: Found 6 function calls in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:24 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:54 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:61 -call-processor.ts:86 โŒ Failed to resolve call: timedelta in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:77 -call-processor.ts:86 โŒ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:79 -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: basicConfig in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: StreamHandler in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:65 CallProcessor: Found 5 function calls in trumio-cortex-core/src/python/ms_teams_connector/start-service.py -call-processor.ts:86 โŒ Failed to resolve call: connect_db_client in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:44 -call-processor.ts:86 โŒ Failed to resolve call: disconnect_db_client in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:50 -call-processor.ts:86 โŒ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:79 -call-processor.ts:553 ๐Ÿ” Filtered out: append in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in mcp_auth.py -call-processor.ts:553 ๐Ÿ” Filtered out: get in mcp_auth.py -call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py:14 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py:69 -call-processor.ts:553 ๐Ÿ” Filtered out: append in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: critical in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: error in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: HTTPException in quality_analyzer.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in quality_analyzer.py -call-processor.ts:65 CallProcessor: Found 23 function calls in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py -call-processor.ts:86 โŒ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:36 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:38 -call-processor.ts:86 โŒ Failed to resolve call: compile_graphs in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:46 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:74 -call-processor.ts:86 โŒ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:80 -call-processor.ts:86 โŒ Failed to resolve call: calculate_date_range in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:100 -call-processor.ts:86 โŒ Failed to resolve call: invoke_weekly_user_fetch_graph in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:104 -call-processor.ts:86 โŒ Failed to resolve call: run_full_weekly_analysis in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:126 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:148 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:157 -call-processor.ts:86 โŒ Failed to resolve call: Query in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:158 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:159 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:160 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:179 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:187 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:197 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:205 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:206 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:214 -call-processor.ts:86 โŒ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:220 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:228 -call-processor.ts:86 โŒ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:229 -call-processor.ts:86 โŒ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:237 -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -10call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: strip in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -2call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -5call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: int in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: lower in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in config.py -6call-processor.ts:553 ๐Ÿ” Filtered out: bool in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in config.py -call-processor.ts:553 ๐Ÿ” Filtered out: warning in config.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/pr_quality_service/app/config.py -call-processor.ts:58 ๐Ÿ“Š Debug: config.py has 38 call nodes, 0 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: append in path_utils.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in path_utils.py -call-processor.ts:52 โš ๏ธ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/pr_quality_service/app/utils/path_utils.py -call-processor.ts:58 ๐Ÿ“Š Debug: path_utils.py has 9 call nodes, 1 definitions -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: upper in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getenv in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: basicConfig in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: StreamHandler in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: getLogger in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:553 ๐Ÿ” Filtered out: info in start-service.py -call-processor.ts:65 CallProcessor: Found 6 function calls in trumio-cortex-core/src/python/pr_quality_service/start-service.py -call-processor.ts:86 โŒ Failed to resolve call: connect_db_client in trumio-cortex-core/src/python/pr_quality_service/start-service.py:45 -call-processor.ts:86 โŒ Failed to resolve call: disconnect_db_client in trumio-cortex-core/src/python/pr_quality_service/start-service.py:54 -call-processor.ts:86 โŒ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/pr_quality_service/start-service.py:97 -call-processor.ts:86 โŒ Failed to resolve call: get_database in trumio-cortex-core/src/python/pr_quality_service/start-service.py:104 -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: dirname in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: abspath in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: join in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: exists in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: load_dotenv in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: append in prompt_test.py -6call-processor.ts:553 ๐Ÿ” Filtered out: print in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: print in prompt_test.py -call-processor.ts:553 ๐Ÿ” Filtered out: run in prompt_test.py -call-processor.ts:65 CallProcessor: Found 3 function calls in trumio-cortex-core/src/python/pr_quality_service/test/prompt_test.py -call-processor.ts:86 โŒ Failed to resolve call: get_weekly_prompts in trumio-cortex-core/src/python/pr_quality_service/test/prompt_test.py:31 -call-processor.ts:723 ๐Ÿ“Š CallProcessor Resolution Statistics: -call-processor.ts:724 Total calls processed: 121 -call-processor.ts:725 โœ… Exact matches (Stage 1): 11 (9.1%) -call-processor.ts:726 โœ… Same-file matches (Stage 2): 19 (15.7%) -call-processor.ts:727 ๐ŸŽฏ Heuristic matches (Stage 3): 4 (3.3%) -call-processor.ts:728 โŒ Failed resolutions: 87 (71.9%) -call-processor.ts:729 Success rate: 28.1% -pipeline.ts:41 Ingestion complete. Graph contains 417 nodes and 446 relationships. -pipeline.ts:50 ๐Ÿ“Š Graph Statistics: -pipeline.ts:51 Nodes by type: {Project: 1, Folder: 41, File: 140, Import: 160, Function: 52,ย โ€ฆ} -pipeline.ts:52 Relationships by type: {CONTAINS: 156, DEFINES: 235, IMPORTS: 23, CALLS: 32} -pipeline.ts:60 โš ๏ธ Found 3 isolated nodes: -run @ pipeline.ts:60 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:65 Isolated nodes by type: {File: 3} -run @ pipeline.ts:65 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:66 Sample isolated nodes: (3)ย [{โ€ฆ}, {โ€ฆ}, {โ€ฆ}] -run @ pipeline.ts:66 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:82 โš ๏ธ Found 140 files without definitions: -run @ pipeline.ts:82 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:83 Files without content: (5)ย ['trumio-cortex-core/', 'trumio-cortex-core/.github/', 'trumio-cortex-core/.github/workflows/', 'trumio-cortex-core/.github/workflows/cortex-ci-cd.yml', 'trumio-cortex-core/.github/workflows/readme.md'] -run @ pipeline.ts:83 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:94 ๐Ÿ” Validating graph integrity... -pipeline.ts:130 Source files without definitions: (3)ย ['trumio-cortex-core/check_prompt.py', 'trumio-cortex-core/debug_channel_messages.py', 'trumio-cortex-core/src/python/common_services/mcp_auth_service.py'] -validateGraphIntegrity @ pipeline.ts:130 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:158 โš ๏ธ Graph integrity issues found: -validateGraphIntegrity @ pipeline.ts:158 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:159 1. 25 files not connected to project structure -(anonymous) @ pipeline.ts:159 -validateGraphIntegrity @ pipeline.ts:159 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:159 2. 59 source files contain no parsed definitions -(anonymous) @ pipeline.ts:159 -validateGraphIntegrity @ pipeline.ts:159 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -pipeline.ts:159 3. 66 definitions not connected to files -(anonymous) @ pipeline.ts:159 -validateGraphIntegrity @ pipeline.ts:159 -run @ pipeline.ts:87 -await in run -processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34 -callback @ comlink.js?v=4af0e45c:91Understand this warning -:5173/src/workers/ingestion.worker.ts?worker_file&type=module:40 IngestionWorker: Processing completed successfully -:5173/src/workers/ingestion.worker.ts?worker_file&type=module:41 Graph contains 417 nodes and 446 relationships -HomePage.tsx:123 ZIP processing completed: {nodeCount: 417, relationshipCount: 446, fileCount: 109} -:5173/src/workers/ingestion.worker.ts?worker_file&type=module:150 Ingestion worker terminated -workerUtils.ts:127 Ingestion worker terminate \ No newline at end of file diff --git a/src/core/orchestration/engine-manager.ts b/src/core/orchestration/engine-manager.ts index d5d76b5d4..218ca6395 100644 --- a/src/core/orchestration/engine-manager.ts +++ b/src/core/orchestration/engine-manager.ts @@ -62,6 +62,7 @@ export class EngineManager { async process(input: ProcessingInput, callbacks?: ProcessingCallbacks): Promise { const selectedEngine = featureFlagManager.getProcessingEngine(); console.log(`๐ŸŽฏ Engine Manager: Selected engine - ${selectedEngine}`); + console.log(`ENGINE-CHECK: Currently using ${selectedEngine} engine - this engine is set from .env`); try { // First, try the selected engine diff --git a/src/lib/import-extraction-test.ts b/src/lib/import-extraction-test.ts index ade195e68..404835c6c 100644 --- a/src/lib/import-extraction-test.ts +++ b/src/lib/import-extraction-test.ts @@ -444,3 +444,6 @@ export function runImportExtractionTests(): void { } + + + diff --git a/src/lib/real-ast-test.ts b/src/lib/real-ast-test.ts index 57c1cf616..910e5cada 100644 --- a/src/lib/real-ast-test.ts +++ b/src/lib/real-ast-test.ts @@ -331,3 +331,6 @@ function detectLanguage(filePath: string): 'python' | 'javascript' | 'typescript } + + + diff --git a/src/lib/test-runner.ts b/src/lib/test-runner.ts index 751498e2b..57d0e7211 100644 --- a/src/lib/test-runner.ts +++ b/src/lib/test-runner.ts @@ -24,3 +24,6 @@ if (typeof window !== 'undefined') { } + + + diff --git a/vercel.json b/vercel.json index c2477ee9a..957bfecbc 100644 --- a/vercel.json +++ b/vercel.json @@ -38,3 +38,6 @@ } } + + +