mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-05 08:06:02 +00:00
Fixed readme and deleted unwanted markdown files
This commit is contained in:
abhigyantrumio
parent
d4a6be6b78
commit
fad3afa750
24 changed files with 473 additions and 6747 deletions
95
.env
Normal file
95
.env
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
|
||||
# ========================================
|
||||
# ENGINE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Default engine to use ('legacy' or 'nextgen')
|
||||
ENGINE_DEFAULT=nextgen
|
||||
|
||||
# Enable/disable engines
|
||||
ENGINE_LEGACY_ENABLED=true
|
||||
ENGINE_NEXTGEN_ENABLED=true
|
||||
|
||||
# Allow fallback between engines if one fails
|
||||
ENGINE_ALLOW_FALLBACK=true
|
||||
|
||||
# Enable performance monitoring
|
||||
ENGINE_PERFORMANCE_MONITORING=true
|
||||
|
||||
# Auto engine selection based on repository size (experimental)
|
||||
ENGINE_AUTO_SELECTION=false
|
||||
|
||||
# Time threshold before falling back to another engine (ms)
|
||||
ENGINE_FALLBACK_THRESHOLD_MS=10000
|
||||
|
||||
# ========================================
|
||||
# LEGACY ENGINE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Memory limits for Legacy engine
|
||||
ENGINE_LEGACY_MEMORY_LIMIT_MB=512
|
||||
ENGINE_LEGACY_GC_INTERVAL_MS=30000
|
||||
|
||||
# Processing settings for Legacy engine
|
||||
ENGINE_LEGACY_BATCH_SIZE=10
|
||||
ENGINE_LEGACY_TIMEOUT_MS=30000
|
||||
ENGINE_LEGACY_USE_WORKERS=true
|
||||
|
||||
# ========================================
|
||||
# NEXT-GEN ENGINE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# KuzuDB settings
|
||||
ENGINE_NEXTGEN_KUZU_DB_PATH=gitnexus.kuzu
|
||||
ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=256
|
||||
ENGINE_NEXTGEN_KUZU_ENABLE_WAL=true
|
||||
ENGINE_NEXTGEN_KUZU_ENABLE_COMPRESSION=true
|
||||
|
||||
# Parallel processing settings
|
||||
ENGINE_NEXTGEN_MAX_WORKERS=4
|
||||
ENGINE_NEXTGEN_BATCH_SIZE=20
|
||||
ENGINE_NEXTGEN_WORKER_TIMEOUT_MS=60000
|
||||
ENGINE_NEXTGEN_ENABLE_PARALLEL_PARSING=true
|
||||
|
||||
# ========================================
|
||||
# GENERAL APPLICATION CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# GitHub token for API access (increases rate limits)
|
||||
GITHUB_TOKEN=your_github_token_here
|
||||
|
||||
# Logging level
|
||||
LOG_LEVEL=info
|
||||
|
||||
# Enable performance metrics
|
||||
LOG_ENABLE_METRICS=true
|
||||
LOG_ENABLE_PERFORMANCE=true
|
||||
|
||||
# Memory configuration
|
||||
MEMORY_MAX_MB=512
|
||||
MEMORY_CLEANUP_THRESHOLD_MB=400
|
||||
|
||||
# ========================================
|
||||
# EXAMPLES AND RECOMMENDED CONFIGURATIONS
|
||||
# ========================================
|
||||
|
||||
# For large repositories (prefer Next-Gen engine):
|
||||
# ENGINE_DEFAULT=nextgen
|
||||
# ENGINE_NEXTGEN_MAX_WORKERS=8
|
||||
# ENGINE_NEXTGEN_BATCH_SIZE=50
|
||||
# ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=512
|
||||
|
||||
# For stable processing (prefer Legacy engine):
|
||||
# ENGINE_DEFAULT=legacy
|
||||
# ENGINE_LEGACY_MEMORY_LIMIT_MB=1024
|
||||
# ENGINE_LEGACY_BATCH_SIZE=5
|
||||
|
||||
# For development/testing (enable both with fallback):
|
||||
# ENGINE_DEFAULT=nextgen
|
||||
# ENGINE_ALLOW_FALLBACK=true
|
||||
# ENGINE_PERFORMANCE_MONITORING=true
|
||||
|
||||
# For production (disable experimental features):
|
||||
# ENGINE_AUTO_SELECTION=false
|
||||
# ENGINE_NEXTGEN_ENABLE_PARALLEL_PARSING=false
|
||||
# LOG_LEVEL=warn
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
# 🚀 Knowledge Graph Generation Architecture Upgrade
|
||||
|
||||
## Overview
|
||||
|
||||
The knowledge graph generation pipeline has been completely restructured to address two critical weaknesses:
|
||||
|
||||
1. **Inaccurate Call Resolution** - Previous monolithic approach lacked project-wide visibility
|
||||
2. **Inefficient Definition Lookups** - Simple map-based storage limited advanced resolution heuristics
|
||||
|
||||
## 🏗️ New Architecture: 4-Pass Decoupled Pipeline
|
||||
|
||||
### **Pass 1: Structure Analysis** 📁
|
||||
- **Processor**: `StructureProcessor`
|
||||
- **Purpose**: Build project hierarchy (folders, files)
|
||||
- **Output**: Basic graph structure with CONTAINS relationships
|
||||
|
||||
### **Pass 2: Code Parsing & Definition Extraction** 🔍
|
||||
- **Processor**: `ParsingProcessor` (Enhanced)
|
||||
- **Purpose**: Parse source code and extract definitions
|
||||
- **Key Enhancement**: Populates `FunctionRegistryTrie` for efficient lookups
|
||||
- **Output**: Function/class/method nodes + optimized search structure
|
||||
|
||||
### **Pass 3: Import Resolution** 🔗
|
||||
- **Processor**: `ImportProcessor` (NEW)
|
||||
- **Purpose**: Build comprehensive project-wide import map
|
||||
- **Key Features**:
|
||||
- Resolves all aliases and relative paths
|
||||
- Handles Python, JavaScript, TypeScript imports
|
||||
- Creates accurate IMPORTS relationships
|
||||
- **Output**: Complete dependency graph + import map
|
||||
|
||||
### **Pass 4: Call Resolution** 📞
|
||||
- **Processor**: `CallProcessor` (Completely Rewritten)
|
||||
- **Purpose**: Resolve function calls using 3-stage strategy
|
||||
- **Input**: Import map + Function registry trie
|
||||
- **Output**: Accurate CALLS relationships
|
||||
|
||||
## 🧠 Key Innovations
|
||||
|
||||
### 1. FunctionRegistryTrie (`src/core/graph/trie.ts`)
|
||||
|
||||
**Purpose**: Optimized data structure for function definition lookups
|
||||
|
||||
**Key Features**:
|
||||
- **Suffix-based search**: `findEndingWith(name)` for heuristic matching
|
||||
- **Qualified names**: Stores full paths like `myProject.services.api.fetchUser`
|
||||
- **Import distance calculation**: Smart scoring for best match selection
|
||||
|
||||
**Example Usage**:
|
||||
```typescript
|
||||
// Find all functions ending with "fetchUser" across the project
|
||||
const candidates = trie.findEndingWith("fetchUser");
|
||||
// Returns: [
|
||||
// { qualifiedName: "services.api.fetchUser", filePath: "services/api.py" },
|
||||
// { qualifiedName: "utils.cache.fetchUser", filePath: "utils/cache.js" }
|
||||
// ]
|
||||
```
|
||||
|
||||
### 2. ImportProcessor (`src/core/ingestion/import-processor.ts`)
|
||||
|
||||
**Purpose**: Dedicated import resolution with project-wide visibility
|
||||
|
||||
**Key Features**:
|
||||
- **Language Support**: Python (`import`, `from...import`) and JS/TS (`import`, `require`)
|
||||
- **Path Resolution**: Handles relative imports (`.`, `..`) and absolute imports
|
||||
- **Alias Tracking**: Maps local names to actual exported functions
|
||||
- **Validation**: Checks against actual project files
|
||||
|
||||
**Example Output**:
|
||||
```typescript
|
||||
importMap = {
|
||||
"src/api.js": {
|
||||
"fetchUser": {
|
||||
targetFile: "src/services/user.js",
|
||||
exportedName: "fetchUser",
|
||||
importType: "named"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Advanced Call Resolution Strategy
|
||||
|
||||
**3-Stage Resolution Process**:
|
||||
|
||||
#### Stage 1: Exact Match (High Confidence)
|
||||
- Uses import map for direct resolution
|
||||
- Example: `import { fetchUser } from './services'` → Direct link to `services/fetchUser`
|
||||
|
||||
#### Stage 2: Same-Module Match (High Confidence)
|
||||
- Checks for function definitions within the same file
|
||||
- Example: Local function calls within a module
|
||||
|
||||
#### Stage 3: Heuristic Fallback (Intelligent Guessing)
|
||||
- Uses `FunctionRegistryTrie.findEndingWith()` to find candidates
|
||||
- Applies **import distance** algorithm for best match
|
||||
- **Distance Formula**: `max(caller_parts, candidate_parts) - common_prefix_length`
|
||||
- **Sibling Bonus**: -1 for functions in same parent directory
|
||||
|
||||
**Example Heuristic Resolution**:
|
||||
```
|
||||
Call: fetchUser() in "src/components/UserList.js"
|
||||
Candidates found:
|
||||
- src/services/user.js:fetchUser (distance: 2)
|
||||
- src/utils/api.js:fetchUser (distance: 2)
|
||||
- src/components/utils.js:fetchUser (distance: 1) ← SELECTED (sibling bonus)
|
||||
```
|
||||
|
||||
## 📊 Performance & Accuracy Improvements
|
||||
|
||||
### Resolution Statistics
|
||||
The new CallProcessor provides detailed statistics:
|
||||
- **Exact matches** (Stage 1): Highest confidence
|
||||
- **Same-file matches** (Stage 2): High confidence
|
||||
- **Heuristic matches** (Stage 3): Medium confidence with distance scoring
|
||||
- **Failed resolutions**: Tracked for debugging
|
||||
|
||||
### Expected Improvements
|
||||
- **🎯 Higher Accuracy**: Project-wide visibility eliminates cross-file resolution failures
|
||||
- **⚡ Better Performance**: Trie-based lookups vs linear searches
|
||||
- **🔍 Smarter Heuristics**: Distance-based scoring for ambiguous cases
|
||||
- **📈 Detailed Metrics**: Comprehensive resolution statistics
|
||||
|
||||
## 🔧 Technical Implementation Details
|
||||
|
||||
### File Structure
|
||||
```
|
||||
src/core/
|
||||
├── graph/
|
||||
│ └── trie.ts # FunctionRegistryTrie implementation
|
||||
├── ingestion/
|
||||
│ ├── pipeline.ts # Updated 4-pass orchestration
|
||||
│ ├── structure-processor.ts
|
||||
│ ├── parsing-processor.ts # Enhanced with trie population
|
||||
│ ├── import-processor.ts # NEW: Dedicated import resolution
|
||||
│ └── call-processor.ts # Completely rewritten
|
||||
```
|
||||
|
||||
### Integration Points
|
||||
1. **ParsingProcessor** populates the trie during definition extraction
|
||||
2. **ImportProcessor** builds the complete import map
|
||||
3. **CallProcessor** uses both trie and import map for resolution
|
||||
4. **Pipeline** orchestrates the sequence with proper data flow
|
||||
|
||||
### Browser Compatibility
|
||||
- Custom path utilities replace Node.js `path` module
|
||||
- All processors work in browser environment
|
||||
- Maintains existing WASM tree-sitter integration
|
||||
|
||||
## 🚀 Usage
|
||||
|
||||
The new architecture is fully integrated into the existing pipeline. No changes required for:
|
||||
- UI components
|
||||
- Worker integration
|
||||
- Export functionality
|
||||
- Statistics display
|
||||
|
||||
The system automatically uses the new 4-pass architecture for all repository processing.
|
||||
|
||||
## 🎯 Results
|
||||
|
||||
This architecture upgrade transforms the knowledge graph generation from a basic parser into an intelligent code analysis system capable of:
|
||||
|
||||
- **Accurate cross-file call resolution**
|
||||
- **Smart import dependency tracking**
|
||||
- **Heuristic-based intelligent guessing**
|
||||
- **Comprehensive project-wide visibility**
|
||||
- **Detailed resolution analytics**
|
||||
|
||||
The result is a significantly more accurate and comprehensive knowledge graph that truly represents the structure and relationships within a codebase.
|
||||
387
Agent.md
387
Agent.md
|
|
@ -1,387 +0,0 @@
|
|||
# GitNexus - Complete Agent Documentation
|
||||
|
||||
## 🎯 Project Overview
|
||||
|
||||
**GitNexus** is a client-side, edge-based code knowledge graph generator that transforms any codebase into an interactive knowledge graph with AI-powered Graph RAG capabilities. It runs entirely in the browser with zero server dependencies.
|
||||
|
||||
### Core Mission
|
||||
|
||||
- **Zero-Setup Code Intelligence**: Analyze codebases without servers or configuration
|
||||
- **Graph RAG-Powered**: Use knowledge graphs for AI-powered code understanding
|
||||
- **Multi-Language Support**: Currently Python-focused with extensible architecture
|
||||
- **Browser-Native**: All processing happens client-side using WebAssembly and Web Workers
|
||||
|
||||
### Key Capabilities
|
||||
|
||||
- **GitHub Integration**: Direct repository analysis via GitHub API
|
||||
- **ZIP Processing**: Local archive analysis with intelligent filtering
|
||||
- **Interactive Visualization**: Cytoscape.js-powered knowledge graphs
|
||||
- **AI Chat Interface**: Multi-LLM support (OpenAI, Anthropic, Gemini)
|
||||
- **Advanced Parsing**: Tree-sitter WASM for accurate AST analysis
|
||||
|
||||
## 🏗️ Architecture Deep Dive
|
||||
|
||||
### System Architecture Layers
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ User Interface Layer │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │
|
||||
│ │ HomePage │ │ Chat UI │ │ Graph Explorer │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────────────┘ │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Service Layer │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │
|
||||
│ │ GitHub API │ │ ZIP Service │ │ Ingestion Service │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────────────┘ │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Processing Pipeline │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │
|
||||
│ │ Structure │ │ Parsing │ │ Call Resolution │ │
|
||||
│ │ Processor │ │ Processor │ │ 3-Stage Strategy │ │
|
||||
│ └─────────────┘ └─────────────┘ └─────────────────────┘ │
|
||||
├─────────────────────────────────────────────────────────────┤
|
||||
│ Core Engine │
|
||||
│ ┌─────────────┐ ┌─────────────┐ ┌─────────────────────┐ │
|
||||
│ │ Graph │ │ Function │ │ Import │ │
|
||||
│ │ Types │ │ Registry │ │ Resolution │ │
|
||||
│ └─────────────┘ │ (Trie) │ └─────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Technology Stack
|
||||
|
||||
**Frontend Framework**: React 18 + TypeScript + Vite
|
||||
**Graph Visualization**: Cytoscape.js + d3.js
|
||||
**Code Parsing**: Tree-sitter WebAssembly
|
||||
**AI Integration**: LangChain.js with ReAct pattern
|
||||
**State Management**: React Context + custom hooks
|
||||
**Build System**: Vite with WASM support
|
||||
|
||||
## 📊 4-Pass Processing Pipeline
|
||||
|
||||
### Pass 1: Structure Analysis (`StructureProcessor`)
|
||||
|
||||
**Purpose**: Discover complete repository structure without filtering
|
||||
|
||||
**Key Innovations**:
|
||||
|
||||
- **Complete Discovery**: Finds ALL directories and files (including ignored ones)
|
||||
- **No Early Filtering**: Preserves complete structure for accurate representation
|
||||
- **Smart Categorization**: Distinguishes files from directories algorithmically
|
||||
- **Intermediate Paths**: Automatically discovers missing directory levels
|
||||
|
||||
**Implementation Details**:
|
||||
|
||||
```typescript
|
||||
// Direct path processing instead of inference
|
||||
const { directories, files } = this.categorizePaths(allPaths);
|
||||
```
|
||||
|
||||
### Pass 2: Code Parsing & Definition Extraction (`ParsingProcessor`)
|
||||
|
||||
**Purpose**: Parse source code and extract definitions while applying intelligent filtering
|
||||
|
||||
**Key Components**:
|
||||
|
||||
- **Tree-sitter Integration**: WASM-based parsing for multiple languages
|
||||
- **Function Registry Trie**: Optimized data structure for definition lookups
|
||||
- **Two-Stage Filtering**:
|
||||
- Stage 1: Prune ignored directories (node_modules, .git, etc.)
|
||||
- Stage 2: Apply user filters (directory patterns, file extensions)
|
||||
|
||||
**Ignore Patterns**:
|
||||
|
||||
```typescript
|
||||
IGNORE_PATTERNS = [
|
||||
'.git', 'node_modules', '__pycache__', '.venv', 'build', 'dist',
|
||||
'.vscode', '.idea', 'tmp', 'logs', 'coverage'
|
||||
]
|
||||
```
|
||||
|
||||
### Pass 3: Import Resolution (`ImportProcessor`)
|
||||
|
||||
**Purpose**: Build comprehensive project-wide import map
|
||||
|
||||
**Features**:
|
||||
|
||||
- **Multi-language Support**: Python, JavaScript, TypeScript imports
|
||||
- **Path Resolution**: Handles relative and absolute imports
|
||||
- **Alias Tracking**: Maps local names to actual exported functions
|
||||
- **Validation**: Checks against actual project files
|
||||
|
||||
**Resolution Patterns**:
|
||||
|
||||
- Python: `import`, `from...import`
|
||||
- JS/TS: `import`, `require()`, `export`
|
||||
- Path normalization for complex project structures
|
||||
|
||||
### Pass 4: Call Resolution (`CallProcessor`)
|
||||
|
||||
**Purpose**: Resolve function calls using 3-stage strategy
|
||||
|
||||
**3-Stage Resolution Strategy**:
|
||||
|
||||
1. **Exact Match** (High Confidence): Uses import map for direct resolution
|
||||
2. **Same-Module Match** (High Confidence): Local function calls within files
|
||||
3. **Heuristic Fallback** (Intelligent): Uses FunctionRegistryTrie with distance scoring
|
||||
|
||||
**Heuristic Algorithm**:
|
||||
|
||||
```typescript
|
||||
// Distance-based scoring for ambiguous calls
|
||||
const distance = max(caller_parts, candidate_parts) - common_prefix_length
|
||||
const score = distance - sibling_bonus
|
||||
```
|
||||
|
||||
## 🤖 AI Integration Architecture
|
||||
|
||||
### ReAct Agent Implementation
|
||||
|
||||
**Pattern**: Reasoning + Acting for complex code queries
|
||||
|
||||
**Agent Components**:
|
||||
|
||||
- **LLM Service**: Multi-provider support (OpenAI, Anthropic, Gemini)
|
||||
- **Cypher Generator**: Natural language to graph query translation
|
||||
- **Tool System**: Graph queries, code retrieval, file search
|
||||
- **Memory Management**: Configurable conversation history
|
||||
|
||||
### Available Tools
|
||||
|
||||
1. **query_graph**: Execute Cypher queries on knowledge graph
|
||||
2. **get_code**: Retrieve specific code snippets
|
||||
3. **search_files**: Find files by name or content patterns
|
||||
4. **get_file_content**: Get complete file contents
|
||||
|
||||
### Debug Mode Features
|
||||
|
||||
- **Reasoning Steps**: Complete ReAct process visualization
|
||||
- **Cypher Queries**: Generated queries with explanations
|
||||
- **Configuration**: LLM settings and performance metrics
|
||||
- **Context Info**: Graph statistics and source attribution
|
||||
|
||||
## 🔧 Service Layer Details
|
||||
|
||||
### GitHub Service (`src/services/github.ts`)
|
||||
|
||||
**Purpose**: GitHub API integration with rate limiting and error handling
|
||||
|
||||
**Key Features**:
|
||||
|
||||
- **Rate Limit Handling**: 5,000 requests/hour with token, 60 without
|
||||
- **Error Recovery**: Comprehensive error handling with user-friendly messages
|
||||
- **Authentication**: Personal access token support
|
||||
- **Content Retrieval**: Efficient file and directory fetching
|
||||
|
||||
**API Methods**:
|
||||
|
||||
```typescript
|
||||
getRepositoryContents(owner, repo, path) // Directory structure
|
||||
getFileContent(owner, repo, path) // Individual file content
|
||||
downloadFileRaw(owner, repo, path) // Raw file download
|
||||
```
|
||||
|
||||
### ZIP Service (`src/services/zip.ts`)
|
||||
|
||||
**Purpose**: Local archive processing with complete structure discovery
|
||||
|
||||
**Features**:
|
||||
|
||||
- **Complete Structure**: Extracts all paths regardless of filtering
|
||||
- **Memory Efficient**: Streaming processing for large archives
|
||||
- **Path Normalization**: Handles common top-level folder removal
|
||||
- **Content Mapping**: Efficient Map<string, string> for file contents
|
||||
|
||||
### Ingestion Service (`src/services/ingestion.service.ts`)
|
||||
|
||||
**Purpose**: Orchestrate the complete ingestion pipeline
|
||||
|
||||
**Orchestration Methods**:
|
||||
|
||||
```typescript
|
||||
processGitHubRepo(url, options) // GitHub repository processing
|
||||
processZipFile(file, options) // ZIP archive processing
|
||||
```
|
||||
|
||||
## 📈 Data Models & Types
|
||||
|
||||
### Core Graph Types
|
||||
|
||||
```typescript
|
||||
interface KnowledgeGraph {
|
||||
nodes: GraphNode[]
|
||||
relationships: Relationship[]
|
||||
}
|
||||
|
||||
interface GraphNode {
|
||||
id: string
|
||||
label: 'Project' | 'Folder' | 'File' | 'Function' | 'Class' | 'Method' | 'Variable'
|
||||
properties: Record<string, any>
|
||||
}
|
||||
|
||||
interface Relationship {
|
||||
id: string
|
||||
type: 'CONTAINS' | 'CALLS' | 'IMPORTS' | 'DECORATES'
|
||||
source: string
|
||||
target: string
|
||||
properties: Record<string, any>
|
||||
}
|
||||
```
|
||||
|
||||
### Function Registry Trie
|
||||
|
||||
**Purpose**: Optimized function definition lookups
|
||||
|
||||
**Key Features**:
|
||||
|
||||
- **Suffix-based search**: `findEndingWith(name)` for heuristic matching
|
||||
- **Qualified names**: Full paths like `myProject.services.api.fetchUser`
|
||||
- **Import distance**: Smart scoring for best match selection
|
||||
|
||||
## 🎨 User Interface Architecture
|
||||
|
||||
### Component Structure
|
||||
|
||||
```
|
||||
App.tsx
|
||||
├── HomePage.tsx (Main application page)
|
||||
├── GraphExplorer.tsx (Interactive graph visualization)
|
||||
├── ChatInterface.tsx (AI chat with debug mode)
|
||||
├── SourceViewer.tsx (Code display with syntax highlighting)
|
||||
└── ErrorBoundary.tsx (Comprehensive error handling)
|
||||
```
|
||||
|
||||
### Interactive Features
|
||||
|
||||
- **Graph Navigation**: Node selection, zooming, panning
|
||||
- **Real-time Progress**: Live updates during processing
|
||||
- **Split-Panel Layout**: Graph + chat interface
|
||||
- **Settings Management**: Persistent configuration
|
||||
- **Export Functionality**: JSON export with metadata
|
||||
|
||||
## 🔍 Performance Optimization
|
||||
|
||||
### Processing Optimizations
|
||||
|
||||
- **Web Workers**: Background processing to keep UI responsive
|
||||
- **Intelligent Filtering**: Skip massive directories (node_modules, .git)
|
||||
- **Batch Processing**: Chunked processing for large repositories
|
||||
- **Memory Management**: Configurable file limits (default: 500 files)
|
||||
|
||||
### Graph Optimization
|
||||
|
||||
- **Node Limiting**: Smart truncation for large graphs
|
||||
- **Relationship Pruning**: Focus on high-confidence connections
|
||||
- **Caching**: AST and processing result caching
|
||||
- **Lazy Loading**: On-demand content loading
|
||||
|
||||
## 🛡️ Error Handling & Reliability
|
||||
|
||||
### Error Boundaries
|
||||
|
||||
- **Component-level**: Graceful degradation for UI components
|
||||
- **Worker-level**: Web Worker error recovery
|
||||
- **Service-level**: API and processing error handling
|
||||
|
||||
### User Experience
|
||||
|
||||
- **Progress Indicators**: Detailed phase-specific messaging
|
||||
- **Confirmation Dialogs**: Smart warnings for expensive operations
|
||||
- **Recovery Options**: Clear guidance for error resolution
|
||||
- **Debug Information**: Comprehensive logging for troubleshooting
|
||||
|
||||
## 📊 Development Setup
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- **Node.js 18+** and **npm/yarn**
|
||||
- **GitHub Token** (optional, increases rate limits)
|
||||
- **AI API Keys**: OpenAI, Anthropic, or Gemini
|
||||
|
||||
### Installation
|
||||
|
||||
```bash
|
||||
npm install
|
||||
npm run dev # Development server on http://localhost:5173
|
||||
npm run build # Production build
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
- **GitHub Token**: Settings → GitHub Token
|
||||
- **AI Keys**: Settings → AI Provider Configuration
|
||||
- **Performance**: Settings → File limits and filtering
|
||||
|
||||
## 🎯 Usage Patterns
|
||||
|
||||
### GitHub Repository Analysis
|
||||
|
||||
1. **URL Input**: Enter GitHub repository URL
|
||||
2. **Filtering**: Optional directory and file extension filters
|
||||
3. **Processing**: 4-pass pipeline with progress tracking
|
||||
4. **Exploration**: Interactive graph with AI chat
|
||||
|
||||
### ZIP Archive Analysis
|
||||
|
||||
1. **File Upload**: Select local ZIP archive
|
||||
2. **Configuration**: Set processing limits and filters
|
||||
3. **Analysis**: Complete repository structure discovery
|
||||
4. **Results**: Knowledge graph with code intelligence
|
||||
|
||||
### Best Practices
|
||||
|
||||
- **Start Small**: Begin with focused directories
|
||||
- **Use Filters**: Exclude dependencies and build artifacts
|
||||
- **Monitor Progress**: Watch console for processing insights
|
||||
- **Leverage AI**: Use chat interface for code exploration
|
||||
|
||||
## 🔮 Future Enhancements
|
||||
|
||||
### Language Support
|
||||
|
||||
- **JavaScript/TypeScript**: Enhanced parsing and analysis
|
||||
- **Java**: Class and method relationship mapping
|
||||
- **C++**: Template and inheritance analysis
|
||||
- **Go**: Package and interface resolution
|
||||
|
||||
### Advanced Features
|
||||
|
||||
- **Code Metrics**: Complexity and quality analysis
|
||||
- **Security Scanning**: Vulnerability detection
|
||||
- **Documentation Generation**: Auto-generated docs
|
||||
- **Team Collaboration**: Shared graph exploration
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
- **Incremental Processing**: Update existing graphs
|
||||
- **Distributed Processing**: Multiple worker threads
|
||||
- **Caching Layer**: Persistent processing cache
|
||||
- **Streaming Analysis**: Real-time code changes
|
||||
|
||||
---
|
||||
|
||||
## 📝 Quick Reference
|
||||
|
||||
### Key Files
|
||||
|
||||
- **Main Entry**: `src/App.tsx`
|
||||
- **Pipeline**: `src/core/ingestion/pipeline.ts`
|
||||
- **Services**: `src/services/`
|
||||
- **AI Logic**: `src/ai/`
|
||||
- **UI Components**: `src/ui/components/`
|
||||
|
||||
### Debug Commands
|
||||
|
||||
- **Enable Debug**: Click "🔍 Debug" in chat interface
|
||||
- **Check Console**: F12 → Console for processing logs
|
||||
- **Diagnose Issues**: Use "🩺 Diagnose" button
|
||||
|
||||
### Common Issues
|
||||
|
||||
- **Rate Limits**: Add GitHub token for higher limits
|
||||
- **Large Repos**: Adjust file limits in settings
|
||||
- **Parsing Errors**: Check file syntax and extensions
|
||||
- **Memory Issues**: Reduce processing scope with filters
|
||||
|
||||
This documentation provides complete context for any agent working on GitNexus, from architecture understanding to implementation details and troubleshooting guidance.
|
||||
|
|
@ -1,54 +0,0 @@
|
|||
# Deno to Node.js Conversion Summary
|
||||
|
||||
## Overview
|
||||
Successfully converted the GitNexus repository from Deno to Node.js while maintaining all functionality.
|
||||
|
||||
## Changes Made
|
||||
|
||||
### 1. Configuration Files
|
||||
- **Removed**: `deno.json`, `deno.lock`
|
||||
- **Updated**: `package.json` with all dependencies from `deno.json`
|
||||
- Added all npm dependencies: jszip, axios, cytoscape, web-tree-sitter, langchain packages, etc.
|
||||
- Updated version to 1.0.0
|
||||
- Kept existing build scripts (Vite-based)
|
||||
|
||||
### 2. Import Statements
|
||||
- **Removed**: All `npm:` prefixes from import statements
|
||||
- **Removed**: All `@ts-expect-error` comments related to npm: imports
|
||||
- **Files affected**: 13+ TypeScript files across the codebase
|
||||
|
||||
### 3. Dependencies Successfully Converted
|
||||
- `react` & `react-dom` (already present)
|
||||
- `jszip` for ZIP file processing
|
||||
- `axios` for HTTP requests
|
||||
- `cytoscape` & `cytoscape-dagre` for graph visualization
|
||||
- `web-tree-sitter` for code parsing
|
||||
- `comlink` for web workers
|
||||
- `@langchain/*` packages for AI functionality
|
||||
- `zod` for schema validation
|
||||
|
||||
### 4. Build System
|
||||
- **Unchanged**: Vite configuration remains the same
|
||||
- **Unchanged**: TypeScript configuration
|
||||
- **Working**: Development server starts successfully on port 5173
|
||||
- **Note**: Some TypeScript errors remain but don't prevent the dev server from running
|
||||
|
||||
## Current Status
|
||||
✅ **Development server running** - The application starts and runs on Node.js
|
||||
✅ **All dependencies installed** - npm install completed successfully
|
||||
✅ **Import statements fixed** - All Deno-style imports converted to Node.js style
|
||||
⚠️ **TypeScript errors** - Some type errors remain but don't block functionality
|
||||
|
||||
## Next Steps (Optional)
|
||||
The conversion is complete and functional, but to achieve a clean build:
|
||||
1. Fix TypeScript errors in langchain imports
|
||||
2. Update type definitions for cytoscape
|
||||
3. Fix unused variable warnings
|
||||
4. Address JSZip type compatibility issues
|
||||
|
||||
## Files Modified
|
||||
- `package.json` - Added all dependencies
|
||||
- 13+ TypeScript files - Removed npm: prefixes and Deno comments
|
||||
- Removed `deno.json` and `deno.lock`
|
||||
|
||||
The repository is now fully converted to Node.js and ready for development!
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
# 🔍 Debug Mode Features
|
||||
|
||||
The GitNexus chat interface now includes a comprehensive debug mode that shows the internal workings of the Graph RAG agent.
|
||||
|
||||
## 📝 Markdown Formatting
|
||||
|
||||
**NEW**: The chat interface now supports full markdown formatting for better readability!
|
||||
|
||||
### Supported Markdown Features:
|
||||
- **Headers** (# ## ###) for organizing information
|
||||
- **Bold** and *italic* text for emphasis
|
||||
- `Inline code` for function names and file paths
|
||||
- Code blocks with syntax highlighting for multiple languages
|
||||
- Bullet points and numbered lists
|
||||
- Tables for structured data
|
||||
- Blockquotes for important notes
|
||||
- Links (automatically open in new tabs)
|
||||
|
||||
### Enhanced Debug Display:
|
||||
- **Reasoning observations** are now rendered with markdown
|
||||
- **Query explanations** support formatted text
|
||||
- **Tool outputs** preserve formatting and structure
|
||||
- **Code snippets** get proper syntax highlighting
|
||||
|
||||
## How to Use Debug Mode
|
||||
|
||||
1. **Toggle Debug Mode**: Click the `🔍 Debug` button in the chat interface header
|
||||
2. **Ask Questions**: When debug mode is enabled, all assistant responses will include detailed debug information
|
||||
3. **Explore Tabs**: The debug panel includes multiple tabs showing different aspects of the processing
|
||||
|
||||
## Debug Panel Tabs
|
||||
|
||||
### 🧠 Reasoning Steps
|
||||
Shows the complete ReAct (Reasoning + Acting) process:
|
||||
- **Step-by-step thinking**: See how the agent reasons about your question
|
||||
- **Actions taken**: View which tools the agent decides to use
|
||||
- **Tool inputs**: See the exact parameters passed to each tool
|
||||
- **Observations**: View the results returned by each tool
|
||||
- **Success/failure status**: Monitor tool execution success
|
||||
|
||||
### 🔍 Cypher Queries
|
||||
Displays generated graph queries:
|
||||
- **Generated Cypher**: See the exact database queries created
|
||||
- **Query explanations**: Understand why each query was generated
|
||||
- **Confidence scores**: View how confident the system is in each query
|
||||
- **Syntax highlighting**: Cypher queries displayed with proper formatting
|
||||
|
||||
### ⚙️ Configuration
|
||||
Shows system configuration:
|
||||
- **LLM Settings**: Provider, model, temperature, max tokens
|
||||
- **RAG Options**: Reasoning steps, strict mode, temperature
|
||||
- **Performance Metrics**: Execution time, confidence scores
|
||||
|
||||
### 📊 Context Info
|
||||
Displays knowledge graph statistics:
|
||||
- **Graph Nodes**: Total number of code entities in the graph
|
||||
- **Files Indexed**: Number of source files processed
|
||||
- **Sources Used**: Files referenced in the current response
|
||||
- **Referenced Sources**: List of specific files used for the answer
|
||||
|
||||
## What You Can Learn
|
||||
|
||||
### Understanding Agent Behavior
|
||||
- See how the agent breaks down complex questions
|
||||
- Understand the reasoning process step-by-step
|
||||
- Monitor which tools are used and why
|
||||
|
||||
### Query Optimization
|
||||
- View generated Cypher queries to understand graph traversal
|
||||
- See query confidence scores to assess reliability
|
||||
- Learn about query patterns for different question types
|
||||
|
||||
### Performance Analysis
|
||||
- Monitor execution times for different operations
|
||||
- Understand the relationship between question complexity and processing time
|
||||
- Identify bottlenecks in the reasoning process
|
||||
|
||||
### Context Awareness
|
||||
- See how much of your codebase is being used
|
||||
- Understand which files are most relevant to your questions
|
||||
- Monitor the scope of graph traversal
|
||||
|
||||
## Debug Mode Benefits
|
||||
|
||||
1. **Transparency**: Complete visibility into AI decision-making
|
||||
2. **Learning**: Understand how Graph RAG works internally
|
||||
3. **Debugging**: Identify issues with queries or reasoning
|
||||
4. **Optimization**: Fine-tune your questions for better results
|
||||
5. **Trust**: Build confidence through explainable AI
|
||||
|
||||
## Example Debug Output
|
||||
|
||||
When you ask "How many functions are in this project?", debug mode shows:
|
||||
|
||||
**Reasoning Steps:**
|
||||
1. **Thought**: "I need to count all functions in the project using a graph query"
|
||||
2. **Action**: query_graph
|
||||
3. **Input**: "Count all functions in the project"
|
||||
4. **Observation**: Generated Cypher query and results
|
||||
|
||||
**Generated Query:**
|
||||
```cypher
|
||||
MATCH (f:Function) RETURN COUNT(f)
|
||||
```
|
||||
|
||||
**Configuration:**
|
||||
- Model: gpt-4o-mini
|
||||
- Temperature: 0.1
|
||||
- Execution Time: 1,234ms
|
||||
|
||||
This level of detail helps you understand exactly how your question was processed and answered.
|
||||
|
|
@ -1,182 +0,0 @@
|
|||
# GitNexus Deployment Guide
|
||||
|
||||
## 🚀 Deploying to Vercel
|
||||
|
||||
### Quick Start (Recommended)
|
||||
|
||||
1. **Push your code to GitHub**
|
||||
```bash
|
||||
git add .
|
||||
git commit -m "Prepare for deployment"
|
||||
git push origin main
|
||||
```
|
||||
|
||||
2. **Deploy via Vercel Dashboard**
|
||||
- Go to [vercel.com](https://vercel.com) and sign in
|
||||
- Click "New Project"
|
||||
- Import your GitHub repository
|
||||
- Vercel will auto-detect it's a Vite project
|
||||
- Click "Deploy"
|
||||
|
||||
### Manual Deployment
|
||||
|
||||
1. **Install Vercel CLI**
|
||||
```bash
|
||||
npm install -g vercel
|
||||
```
|
||||
|
||||
2. **Login to Vercel**
|
||||
```bash
|
||||
vercel login
|
||||
```
|
||||
|
||||
3. **Deploy from your project directory**
|
||||
```bash
|
||||
vercel
|
||||
```
|
||||
|
||||
4. **Follow the prompts:**
|
||||
- Set up and deploy? `Y`
|
||||
- Which scope? `[Your account]`
|
||||
- Link to existing project? `N`
|
||||
- Project name: `gitnexus` (or your preferred name)
|
||||
- Directory: `./` (current directory)
|
||||
- Override settings? `N`
|
||||
|
||||
### Environment Variables
|
||||
|
||||
Set these in your Vercel project dashboard under Settings → Environment Variables:
|
||||
|
||||
```env
|
||||
# Optional: Pre-configure API keys for users
|
||||
VITE_OPENAI_API_KEY=sk-...
|
||||
VITE_ANTHROPIC_API_KEY=sk-ant-...
|
||||
VITE_GEMINI_API_KEY=...
|
||||
|
||||
# Performance settings
|
||||
VITE_DEFAULT_MAX_FILES=500
|
||||
VITE_ENABLE_DEBUG_LOGGING=false
|
||||
```
|
||||
|
||||
### Build Configuration
|
||||
|
||||
The `vercel.json` file is already configured with:
|
||||
|
||||
- **Build Command**: `npm run build`
|
||||
- **Output Directory**: `dist`
|
||||
- **Framework**: `vite`
|
||||
- **WASM Support**: Proper headers for WebAssembly files
|
||||
- **CORS Headers**: Required for SharedArrayBuffer support
|
||||
|
||||
### Custom Domain (Optional)
|
||||
|
||||
1. Go to your Vercel project dashboard
|
||||
2. Navigate to Settings → Domains
|
||||
3. Add your custom domain
|
||||
4. Update DNS records as instructed
|
||||
|
||||
### Deployment Verification
|
||||
|
||||
After deployment, verify:
|
||||
|
||||
1. **Homepage loads** - Should show the GitNexus interface
|
||||
2. **WASM files load** - Check browser console for WASM loading errors
|
||||
3. **GitHub integration works** - Test repository analysis
|
||||
4. **AI features work** - Test chat interface (if API keys configured)
|
||||
|
||||
### Troubleshooting
|
||||
|
||||
#### Build Failures
|
||||
|
||||
**Error**: "Module not found" or TypeScript errors
|
||||
```bash
|
||||
# Fix locally first
|
||||
npm run build
|
||||
# If successful locally, the issue is resolved
|
||||
```
|
||||
|
||||
**Error**: WASM files not found
|
||||
- Ensure `public/wasm/` directory is included in your repository
|
||||
- Check that WASM files are not gitignored
|
||||
|
||||
#### Runtime Issues
|
||||
|
||||
**Error**: "SharedArrayBuffer not available"
|
||||
- This is expected in development
|
||||
- Vercel automatically sets the required CORS headers
|
||||
|
||||
**Error**: "Cross-origin isolation required"
|
||||
- The `vercel.json` headers should handle this
|
||||
- If issues persist, check browser console for specific errors
|
||||
|
||||
#### Performance Issues
|
||||
|
||||
**Slow loading**:
|
||||
- Check bundle size in Vercel dashboard
|
||||
- Consider code splitting for large dependencies
|
||||
- Optimize WASM file loading
|
||||
|
||||
### Monitoring & Analytics
|
||||
|
||||
1. **Vercel Analytics** (Optional)
|
||||
```bash
|
||||
npm install @vercel/analytics
|
||||
```
|
||||
|
||||
2. **Add to your app**:
|
||||
```typescript
|
||||
import { Analytics } from '@vercel/analytics/react';
|
||||
|
||||
function App() {
|
||||
return (
|
||||
<>
|
||||
{/* Your app content */}
|
||||
<Analytics />
|
||||
</>
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
### Continuous Deployment
|
||||
|
||||
Vercel automatically deploys on:
|
||||
- Push to `main` branch → Production
|
||||
- Push to other branches → Preview deployments
|
||||
- Pull requests → Preview deployments
|
||||
|
||||
### Rollback
|
||||
|
||||
If you need to rollback:
|
||||
1. Go to Vercel dashboard
|
||||
2. Navigate to Deployments
|
||||
3. Find the working deployment
|
||||
4. Click "Promote to Production"
|
||||
|
||||
### Security Considerations
|
||||
|
||||
- API keys in environment variables are secure
|
||||
- Client-side code is public (as expected for this app)
|
||||
- No server-side secrets are exposed
|
||||
- WASM files are served as static assets
|
||||
|
||||
### Cost Optimization
|
||||
|
||||
Vercel Hobby Plan (Free) includes:
|
||||
- 100GB bandwidth/month
|
||||
- 100GB storage
|
||||
- 100GB function execution time
|
||||
- Custom domains
|
||||
- Automatic HTTPS
|
||||
|
||||
For higher usage, consider Vercel Pro ($20/month).
|
||||
|
||||
## 🎯 Next Steps
|
||||
|
||||
1. **Deploy your first version**
|
||||
2. **Test all features thoroughly**
|
||||
3. **Set up monitoring** (optional)
|
||||
4. **Configure custom domain** (optional)
|
||||
5. **Set up environment variables** for AI features
|
||||
|
||||
Your GitNexus application is now ready for production deployment! 🚀
|
||||
|
||||
|
|
@ -1,330 +0,0 @@
|
|||
# GitNexus Dual-Track System Implementation Summary
|
||||
|
||||
## 🎯 Overview
|
||||
|
||||
Successfully implemented a dual-track processing system for GitNexus that provides:
|
||||
|
||||
1. **Complete separation** between Legacy (Sequential + In-Memory) and Next-Gen (Parallel + KuzuDB) engines
|
||||
2. **Clean UI refactoring** from monolithic 700+ line HomePage to focused components
|
||||
3. **Robust fallback system** with logging when Next-Gen fails
|
||||
4. **Feature toggle capability** for easy engine switching
|
||||
5. **Maintainable architecture** with clear separation of concerns
|
||||
|
||||
## 📁 New Directory Structure
|
||||
|
||||
```
|
||||
src/
|
||||
├── config/
|
||||
│ ├── feature-flags.ts # ✅ Enhanced with engine switching
|
||||
│ └── ...
|
||||
├── core/
|
||||
│ ├── engines/
|
||||
│ │ ├── engine-interface.ts # ✅ Common engine interface
|
||||
│ │ ├── legacy/
|
||||
│ │ │ └── legacy-engine.ts # ✅ Wrapper for current system
|
||||
│ │ └── nextgen/
|
||||
│ │ └── nextgen-engine.ts # ✅ Wrapper for parallel+kuzu
|
||||
│ ├── orchestration/
|
||||
│ │ └── engine-manager.ts # ✅ Engine switching & fallback
|
||||
│ └── validation/
|
||||
│ └── dual-track-validation.ts # ✅ System validation tests
|
||||
├── services/
|
||||
│ ├── facade/
|
||||
│ │ └── gitnexus-facade.ts # ✅ Simplified API for UI
|
||||
│ └── ...
|
||||
├── ui/
|
||||
│ ├── components/
|
||||
│ │ ├── engine/
|
||||
│ │ │ ├── EngineSelector.tsx # ✅ Engine switching UI
|
||||
│ │ │ └── ProcessingStatus.tsx # ✅ Engine-aware status
|
||||
│ │ └── repository/
|
||||
│ │ └── RepositoryInput.tsx # ✅ GitHub/ZIP input
|
||||
│ ├── hooks/
|
||||
│ │ ├── useEngine.ts # ✅ Engine management
|
||||
│ │ ├── useProcessing.ts # ✅ Processing operations
|
||||
│ │ ├── useSettings.ts # ✅ Settings management
|
||||
│ │ └── useGitNexus.ts # ✅ Main application hook
|
||||
│ └── pages/
|
||||
│ └── HomePage/
|
||||
│ ├── HomePage.tsx # ✅ New lightweight container
|
||||
│ └── index.ts
|
||||
```
|
||||
|
||||
## 🔧 Key Components Implemented
|
||||
|
||||
### 1. Enhanced Feature Flags (`src/config/feature-flags.ts`)
|
||||
|
||||
```typescript
|
||||
interface FeatureFlags {
|
||||
// NEW: Engine Selection
|
||||
processingEngine: ProcessingEngineType;
|
||||
autoFallbackOnError: boolean;
|
||||
enablePerformanceComparison: boolean;
|
||||
// ... existing flags
|
||||
}
|
||||
|
||||
// NEW: Engine switching methods
|
||||
featureFlagManager.switchToLegacyEngine();
|
||||
featureFlagManager.switchToNextGenEngine();
|
||||
featureFlagManager.logEngineFallback(error);
|
||||
```
|
||||
|
||||
### 2. Engine Interface (`src/core/engines/engine-interface.ts`)
|
||||
|
||||
```typescript
|
||||
interface ProcessingEngine {
|
||||
readonly name: string;
|
||||
readonly type: ProcessingEngineType;
|
||||
readonly capabilities: string[];
|
||||
|
||||
process(input: ProcessingInput): Promise<ProcessingResult>;
|
||||
validate(): Promise<boolean>;
|
||||
cleanup(): Promise<void>;
|
||||
getStatus(): EngineStatus;
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Legacy Engine Wrapper (`src/core/engines/legacy/legacy-engine.ts`)
|
||||
|
||||
- Wraps existing `IngestionService`
|
||||
- Uses `GraphPipeline` + `SimpleKnowledgeGraph`
|
||||
- Capabilities: `['sequential-processing', 'in-memory-storage', 'basic-queries']`
|
||||
|
||||
### 4. Next-Gen Engine Wrapper (`src/core/engines/nextgen/nextgen-engine.ts`)
|
||||
|
||||
- Wraps `KuzuIngestionService`
|
||||
- Uses `KuzuGraphPipeline` + `ParallelProcessing` + `KuzuKnowledgeGraph`
|
||||
- Capabilities: `['parallel-processing', 'kuzu-db-storage', 'advanced-queries']`
|
||||
|
||||
### 5. Engine Manager (`src/core/orchestration/engine-manager.ts`)
|
||||
|
||||
```typescript
|
||||
class EngineManager {
|
||||
async process(input: ProcessingInput): Promise<ProcessingResult> {
|
||||
try {
|
||||
// Try selected engine
|
||||
return await this.processWithEngine(selectedEngine, input);
|
||||
} catch (error) {
|
||||
// Auto-fallback if enabled
|
||||
if (this.config.autoFallback && selectedEngine === 'nextgen') {
|
||||
featureFlagManager.logEngineFallback(error.message);
|
||||
return await this.processWithEngine('legacy', input);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 6. GitNexus Facade (`src/services/facade/gitnexus-facade.ts`)
|
||||
|
||||
Simplified API for UI:
|
||||
|
||||
```typescript
|
||||
class GitNexusFacade {
|
||||
async processGitHubRepository(url: string): Promise<GitNexusResult>;
|
||||
async processZipFile(file: File): Promise<GitNexusResult>;
|
||||
async switchEngine(engine: ProcessingEngineType): Promise<void>;
|
||||
getCurrentEngine(): EngineInfo;
|
||||
getAvailableEngines(): EngineInfo[];
|
||||
}
|
||||
```
|
||||
|
||||
### 7. UI Components
|
||||
|
||||
#### EngineSelector (`src/ui/components/engine/EngineSelector.tsx`)
|
||||
- Dropdown for engine selection
|
||||
- Real-time engine status display
|
||||
- Performance info toggle
|
||||
|
||||
#### ProcessingStatus (`src/ui/components/engine/ProcessingStatus.tsx`)
|
||||
- Engine-aware progress display
|
||||
- Fallback notifications with logging
|
||||
- Success/error states with metrics
|
||||
|
||||
#### RepositoryInput (`src/ui/components/repository/RepositoryInput.tsx`)
|
||||
- Tabbed interface (GitHub/ZIP)
|
||||
- Drag-and-drop ZIP support
|
||||
- Input validation
|
||||
|
||||
### 8. Custom Hooks
|
||||
|
||||
#### useEngine (`src/ui/hooks/useEngine.ts`)
|
||||
- Engine switching logic
|
||||
- Status monitoring
|
||||
- Performance comparison
|
||||
|
||||
#### useProcessing (`src/ui/hooks/useProcessing.ts`)
|
||||
- GitHub/ZIP processing
|
||||
- Progress tracking
|
||||
- Error handling
|
||||
|
||||
#### useGitNexus (`src/ui/hooks/useGitNexus.ts`)
|
||||
- Main application state
|
||||
- Combines all functionality
|
||||
- Clean API for components
|
||||
|
||||
### 9. New HomePage (`src/ui/pages/HomePage/HomePage.tsx`)
|
||||
|
||||
**Before**: 1161 lines, monolithic
|
||||
**After**: ~200 lines, focused components
|
||||
|
||||
```typescript
|
||||
const HomePage = () => {
|
||||
const { state, engine, processing, settings, ... } = useGitNexus();
|
||||
|
||||
return (
|
||||
<div className="app">
|
||||
<EngineSelector currentEngine={engine.currentEngine} />
|
||||
<ProcessingStatus {...processing.state} />
|
||||
<RepositoryInput onGitHubSubmit={handleGitHub} />
|
||||
<GraphExplorer graph={state.graph} />
|
||||
<ChatInterface />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
## 🚀 How Engine Switching Works
|
||||
|
||||
### 1. UI Selection
|
||||
```typescript
|
||||
<EngineSelector
|
||||
currentEngine="legacy"
|
||||
onEngineChange={(engine) => switchEngine(engine)}
|
||||
options={[
|
||||
{ value: 'legacy', label: '🔧 Stable (Sequential + In-Memory)' },
|
||||
{ value: 'nextgen', label: '🚀 Advanced (Parallel + KuzuDB)' }
|
||||
]}
|
||||
/>
|
||||
```
|
||||
|
||||
### 2. Feature Flag Update
|
||||
```typescript
|
||||
// User selects Next-Gen
|
||||
switchEngine('nextgen')
|
||||
→ featureFlagManager.switchToNextGenEngine()
|
||||
→ Sets: enableKuzuDB=true, enableParallelProcessing=true
|
||||
```
|
||||
|
||||
### 3. Processing with Fallback
|
||||
```typescript
|
||||
// Engine Manager routes to correct engine
|
||||
if (engine === 'nextgen') {
|
||||
try {
|
||||
return await nextGenEngine.process(input);
|
||||
} catch (error) {
|
||||
// 🔄 AUTO FALLBACK WITH LOGGING
|
||||
featureFlagManager.logEngineFallback(error.message);
|
||||
console.warn("🔄 Engine Fallback: Next-gen → Legacy");
|
||||
return await legacyEngine.process(input);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 4. User Feedback
|
||||
```typescript
|
||||
// UI shows fallback notification
|
||||
<ProcessingStatus
|
||||
hadFallback={true}
|
||||
fallbackEngine="legacy"
|
||||
// Shows: "🔄 Used fallback engine: Legacy Engine"
|
||||
/>
|
||||
```
|
||||
|
||||
## 📊 Benefits Achieved
|
||||
|
||||
### ✅ **Clean Separation**
|
||||
- Legacy and Next-Gen systems completely isolated
|
||||
- No code mixing between engines
|
||||
- Easy to modify each system independently
|
||||
|
||||
### ✅ **Safe Migration**
|
||||
- Legacy always works as fallback
|
||||
- Next-Gen is opt-in with validation
|
||||
- Auto-fallback prevents data loss
|
||||
|
||||
### ✅ **Better UX**
|
||||
- Clear engine selection interface
|
||||
- Real-time status and performance metrics
|
||||
- Transparent fallback notifications
|
||||
|
||||
### ✅ **Maintainable Code**
|
||||
- HomePage reduced from 1161 → ~200 lines
|
||||
- Logic extracted to focused hooks
|
||||
- Components have single responsibilities
|
||||
|
||||
### ✅ **Performance Monitoring**
|
||||
- Engine performance comparison
|
||||
- Processing time tracking
|
||||
- Success rate monitoring
|
||||
|
||||
## 🔄 Fallback Logging Implementation
|
||||
|
||||
When Next-Gen engine fails and falls back to Legacy:
|
||||
|
||||
```typescript
|
||||
// Engine Manager detects failure
|
||||
catch (error) {
|
||||
// Log the fallback event
|
||||
featureFlagManager.logEngineFallback(error.message);
|
||||
|
||||
// Console output:
|
||||
// 🔄 Engine Fallback: Next-gen engine failed, falling back to legacy engine
|
||||
// 🔄 Fallback reason: KuzuDB connection failed
|
||||
// 🔄 Auto-fallback enabled: true
|
||||
|
||||
// UI shows notification
|
||||
callbacks?.onEngineFailure?.('nextgen', 'legacy', error.message);
|
||||
}
|
||||
```
|
||||
|
||||
## 🧪 Validation System
|
||||
|
||||
Created comprehensive validation (`src/core/validation/dual-track-validation.ts`):
|
||||
|
||||
1. ✅ Feature flag management
|
||||
2. ✅ Engine manager initialization
|
||||
3. ✅ Engine validation
|
||||
4. ✅ GitNexus facade functionality
|
||||
5. ✅ Engine switching
|
||||
6. ✅ Fallback logging
|
||||
7. ✅ Utility functions
|
||||
|
||||
## 🎯 Usage Examples
|
||||
|
||||
### Switch to Next-Gen Engine
|
||||
```typescript
|
||||
await facade.switchEngine('nextgen', 'User wants parallel processing');
|
||||
```
|
||||
|
||||
### Process with Automatic Fallback
|
||||
```typescript
|
||||
const result = await facade.processGitHubRepository(
|
||||
'https://github.com/user/repo',
|
||||
{
|
||||
engine: 'nextgen',
|
||||
onEngineSwitch: (from, to) => {
|
||||
console.log(`Fallback: ${from} → ${to}`);
|
||||
}
|
||||
}
|
||||
);
|
||||
```
|
||||
|
||||
### Monitor Engine Performance
|
||||
```typescript
|
||||
const comparison = facade.getPerformanceComparison();
|
||||
// Shows speedup factor, processing times, etc.
|
||||
```
|
||||
|
||||
## 🏁 Conclusion
|
||||
|
||||
The dual-track system is now fully implemented with:
|
||||
|
||||
- **Zero breaking changes** to existing functionality
|
||||
- **Complete engine separation** for maintainability
|
||||
- **Robust fallback system** with logging for reliability
|
||||
- **Clean UI architecture** for better developer experience
|
||||
- **Performance monitoring** for data-driven decisions
|
||||
|
||||
The system is ready for production use and provides a solid foundation for future engine improvements and migrations.
|
||||
132
FILTERING_FIX.md
132
FILTERING_FIX.md
|
|
@ -1,132 +0,0 @@
|
|||
# 🔧 Directory Filtering Fix - .venv and Ignored Directories Hidden
|
||||
|
||||
## 🚨 **Issue Identified and Fixed**
|
||||
|
||||
**Problem**: `.venv` and other ignored directories were still appearing in the Knowledge Graph despite filtering implementation.
|
||||
|
||||
**Root Cause**: The two-stage filtering was only filtering **file parsing**, but ignored directory **nodes** were still being created and displayed in the KG.
|
||||
|
||||
## ✅ **Solution Implemented**
|
||||
|
||||
### **Enhanced StructureProcessor**
|
||||
|
||||
#### **Directory Hiding Logic**
|
||||
```typescript
|
||||
// Added to StructureProcessor
|
||||
private shouldHideDirectory(dirPath: string): boolean {
|
||||
const pathSegments = dirPath.split('/');
|
||||
|
||||
// Check if any segment matches ignore patterns
|
||||
const hasIgnoredSegment = pathSegments.some(segment =>
|
||||
StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase())
|
||||
);
|
||||
|
||||
return hasIgnoredSegment || this.matchesAdditionalPatterns(dirPath);
|
||||
}
|
||||
```
|
||||
|
||||
#### **Filtered Node Creation**
|
||||
```typescript
|
||||
// Filter directories before creating nodes
|
||||
const visibleDirectories = directories.filter(dir => !this.shouldHideDirectory(dir));
|
||||
const hiddenDirectoriesCount = directories.length - visibleDirectories.length;
|
||||
|
||||
console.log(`StructureProcessor: Hiding ${hiddenDirectoriesCount} ignored directories from KG`);
|
||||
|
||||
// Create nodes only for visible directories
|
||||
const directoryNodes = this.createDirectoryNodes(visibleDirectories);
|
||||
```
|
||||
|
||||
#### **Smart Relationship Handling**
|
||||
```typescript
|
||||
// Handle files in hidden directories by connecting to nearest visible parent
|
||||
private findVisibleParent(path: string, projectId: string): string {
|
||||
if (path === '') return projectId;
|
||||
|
||||
const parentPath = this.getParentPath(path);
|
||||
const parentId = this.nodeIdMap.get(parentPath);
|
||||
|
||||
if (parentId) {
|
||||
return parentId; // Found visible parent
|
||||
}
|
||||
|
||||
// Recursively look for visible parent
|
||||
return this.findVisibleParent(parentPath, projectId);
|
||||
}
|
||||
```
|
||||
|
||||
## 🎯 **What's Now Hidden from KG**
|
||||
|
||||
### **Directories Completely Hidden**
|
||||
- ✅ `.venv`, `venv`, `env`, `virtualenv` (Python virtual environments)
|
||||
- ✅ `node_modules`, `bower_components` (Package dependencies)
|
||||
- ✅ `.git`, `.svn`, `.hg` (Version control)
|
||||
- ✅ `build`, `dist`, `out`, `target` (Build outputs)
|
||||
- ✅ `.vs`, `.vscode`, `.idea` (IDE directories)
|
||||
- ✅ `__pycache__`, `.pytest_cache` (Python cache)
|
||||
- ✅ `coverage`, `.coverage` (Test coverage)
|
||||
- ✅ `.cache`, `.next`, `.nuxt` (Framework cache)
|
||||
- ✅ `tmp`, `temp`, `logs` (Temporary directories)
|
||||
|
||||
### **Special Handling**
|
||||
- ✅ `.github` directory **remains visible** (important for workflows)
|
||||
- ✅ Files in hidden directories connect to nearest visible parent
|
||||
- ✅ Complete structure discovery still happens (for performance benefits)
|
||||
|
||||
## 📊 **Before vs After**
|
||||
|
||||
### **Before (The Problem)**
|
||||
```
|
||||
Knowledge Graph showing:
|
||||
├── src/ ✅ Visible
|
||||
├── tests/ ✅ Visible
|
||||
├── .venv/ ❌ Unwanted visibility
|
||||
├── node_modules/ ❌ Unwanted visibility
|
||||
├── __pycache__/ ❌ Unwanted visibility
|
||||
└── package.json ✅ Visible
|
||||
```
|
||||
|
||||
### **After (Fixed)**
|
||||
```
|
||||
Knowledge Graph showing:
|
||||
├── src/ ✅ Visible
|
||||
├── tests/ ✅ Visible
|
||||
├── .github/ ✅ Visible (important)
|
||||
└── package.json ✅ Visible
|
||||
|
||||
Hidden from view:
|
||||
- .venv/ (and all contents)
|
||||
- node_modules/ (and all contents)
|
||||
- __pycache__/ (and all contents)
|
||||
```
|
||||
|
||||
## 🎯 **Technical Implementation**
|
||||
|
||||
### **Two-Level Filtering**
|
||||
1. **StructureProcessor**: Hides directory **nodes** from KG
|
||||
2. **ParsingProcessor**: Skips **file parsing** in ignored directories
|
||||
|
||||
### **Performance Benefits Maintained**
|
||||
- ✅ **Complete Discovery**: Still discovers all paths for performance optimization
|
||||
- ✅ **Smart Filtering**: Skips expensive parsing operations
|
||||
- ✅ **Clean Visualization**: Users see only relevant directories
|
||||
- ✅ **Accurate Relationships**: Files connect to appropriate visible parents
|
||||
|
||||
### **Logging Enhanced**
|
||||
```
|
||||
StructureProcessor: Found 1,247 directories and 892 files
|
||||
StructureProcessor: Hiding 156 ignored directories from KG
|
||||
StructureProcessor: Created 983 nodes total (156 directories hidden)
|
||||
```
|
||||
|
||||
## 🚀 **Result**
|
||||
|
||||
**Perfect Fix!** Now:
|
||||
|
||||
1. **✅ .venv is Hidden**: No longer appears in Knowledge Graph
|
||||
2. **✅ Clean Visualization**: Only relevant directories shown
|
||||
3. **✅ Performance Maintained**: Still skip expensive parsing operations
|
||||
4. **✅ Accurate Structure**: Files properly connected to visible parents
|
||||
5. **✅ Comprehensive Coverage**: All common ignored directories hidden
|
||||
|
||||
The directory filtering is now **working correctly** and `.venv` (along with other ignored directories) will no longer clutter the Knowledge Graph! 🎉
|
||||
|
|
@ -1,334 +0,0 @@
|
|||
# 🚀 GitHub Archive Implementation Guide
|
||||
|
||||
## 📋 Overview
|
||||
|
||||
This implementation adds **5-10x faster GitHub repository processing** to GitNexus by using GitHub's archive download feature instead of individual API calls. The system automatically chooses the best method based on repository size and user preferences.
|
||||
|
||||
## 🎯 Key Features
|
||||
|
||||
- **⚡ 5-10x Faster Processing**: Archive downloads are much faster than API calls
|
||||
- **🤖 Smart Method Selection**: Automatically chooses archive vs API based on repository size
|
||||
- **🔄 Automatic Fallback**: Falls back to API if archive method fails
|
||||
- **📊 Real-time Progress**: Shows detailed progress with method and stage information
|
||||
- **⚙️ Configurable Settings**: User can control archive preferences and size limits
|
||||
- **🌿 Branch Support**: Process specific branches instead of just default
|
||||
|
||||
## 📁 File Structure
|
||||
|
||||
```
|
||||
src/services/
|
||||
├── github-archive.ts # Core archive download service
|
||||
├── hybrid-github.ts # Smart method selection service
|
||||
└── github.ts # Existing API service
|
||||
|
||||
src/ui/pages/
|
||||
└── HomePage.tsx # Updated with hybrid service integration
|
||||
|
||||
src/lib/
|
||||
└── github-archive-test.ts # Test functions for verification
|
||||
```
|
||||
|
||||
## 🔧 Core Components
|
||||
|
||||
### 1. GitHubArchiveService (`src/services/github-archive.ts`)
|
||||
|
||||
**Purpose**: Downloads and processes GitHub repository archives using ZIP downloads.
|
||||
|
||||
**Key Methods**:
|
||||
- `getRepositoryArchive()` - Main method for downloading and processing
|
||||
- `checkRepositoryAccess()` - Verify repository exists and is accessible
|
||||
- `estimateRepositorySize()` - Get repository size before downloading
|
||||
- `getBranches()` - Get available branches for a repository
|
||||
|
||||
**Features**:
|
||||
- Progress tracking with detailed stages
|
||||
- Batch processing to avoid memory issues
|
||||
- Smart file filtering (skips binaries, large files, common directories)
|
||||
- Error handling and recovery
|
||||
|
||||
### 2. HybridGitHubService (`src/services/hybrid-github.ts`)
|
||||
|
||||
**Purpose**: Intelligently chooses between archive and API methods.
|
||||
|
||||
**Key Methods**:
|
||||
- `getRepositoryStructure()` - Main method with smart method selection
|
||||
- `compareMethods()` - Compare performance between methods
|
||||
- `getRepositoryViaArchive()` - Archive method implementation
|
||||
- `getRepositoryViaAPI()` - API method implementation
|
||||
|
||||
**Features**:
|
||||
- Automatic method selection based on repository size
|
||||
- Performance comparison and recommendations
|
||||
- Seamless fallback between methods
|
||||
- Unified interface for both approaches
|
||||
|
||||
### 3. Updated HomePage (`src/ui/pages/HomePage.tsx`)
|
||||
|
||||
**New Features**:
|
||||
- Enhanced progress display with method and stage information
|
||||
- Branch selection input
|
||||
- Archive method settings in settings modal
|
||||
- Real-time progress bars and status updates
|
||||
|
||||
## 🚀 Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```typescript
|
||||
import { HybridGitHubService } from './services/hybrid-github.js';
|
||||
|
||||
const hybridService = HybridGitHubService.getInstance();
|
||||
|
||||
// Process repository with automatic method selection
|
||||
const result = await hybridService.getRepositoryStructure(
|
||||
'owner',
|
||||
'repo',
|
||||
'main',
|
||||
{
|
||||
preferArchive: true,
|
||||
maxArchiveSizeMB: 100,
|
||||
fallbackToAPI: true
|
||||
},
|
||||
(progress) => {
|
||||
console.log(`${progress.method}: ${progress.stage} - ${progress.progress}%`);
|
||||
}
|
||||
);
|
||||
```
|
||||
|
||||
### Method Comparison
|
||||
|
||||
```typescript
|
||||
// Compare methods for a repository
|
||||
const comparison = await hybridService.compareMethods('facebook', 'react');
|
||||
console.log('Recommended method:', comparison.recommended);
|
||||
console.log('Archive time:', comparison.archive.estimatedTime);
|
||||
console.log('API time:', comparison.api.estimatedTime);
|
||||
```
|
||||
|
||||
### Direct Archive Usage
|
||||
|
||||
```typescript
|
||||
import { GitHubArchiveService } from './services/github-archive.js';
|
||||
|
||||
const archiveService = GitHubArchiveService.getInstance();
|
||||
|
||||
const result = await archiveService.getRepositoryArchive(
|
||||
'owner',
|
||||
'repo',
|
||||
'main',
|
||||
(progress) => {
|
||||
console.log(`${progress.stage}: ${progress.progress}%`);
|
||||
}
|
||||
);
|
||||
```
|
||||
|
||||
## ⚙️ Configuration
|
||||
|
||||
### User Settings
|
||||
|
||||
Users can configure archive behavior in the settings modal:
|
||||
|
||||
- **Prefer Fast Archive Method**: Enable/disable archive preference
|
||||
- **Maximum Archive Size**: Set size limit for archive method (default: 100MB)
|
||||
- **GitHub Token**: Optional token for higher rate limits
|
||||
|
||||
### Default Behavior
|
||||
|
||||
- **Small repos (< 100MB)**: Use archive method by default
|
||||
- **Large repos (> 100MB)**: Use API method by default
|
||||
- **Archive fails**: Automatically fallback to API method
|
||||
- **No token**: Works with public repositories
|
||||
|
||||
## 📊 Performance Comparison
|
||||
|
||||
### Archive Method
|
||||
- **Speed**: 5-10x faster than API
|
||||
- **Network**: Single ZIP download vs hundreds of API calls
|
||||
- **Rate Limits**: No API rate limit impact
|
||||
- **Memory**: Higher memory usage during extraction
|
||||
- **Best For**: Small to medium repositories
|
||||
|
||||
### API Method
|
||||
- **Speed**: Slower but more reliable
|
||||
- **Network**: Multiple API calls
|
||||
- **Rate Limits**: Subject to GitHub API limits
|
||||
- **Memory**: Lower memory usage
|
||||
- **Best For**: Large repositories or when archive fails
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
### Browser Console Testing
|
||||
|
||||
```javascript
|
||||
// Test archive service
|
||||
testGitHubArchive();
|
||||
|
||||
// Test hybrid service
|
||||
testHybridGitHub();
|
||||
|
||||
// Compare methods for a specific repository
|
||||
compareMethods('facebook', 'react');
|
||||
```
|
||||
|
||||
### Test Functions Available
|
||||
|
||||
- `testGitHubArchive()` - Test archive service functionality
|
||||
- `testHybridGitHub()` - Test hybrid service functionality
|
||||
- `compareMethods(owner, repo)` - Compare methods for a repository
|
||||
|
||||
## 🔍 Error Handling
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Repository Not Found**
|
||||
- Error: Repository doesn't exist or is private
|
||||
- Solution: Check URL and repository access
|
||||
|
||||
2. **Archive Too Large**
|
||||
- Error: Repository exceeds size limit
|
||||
- Solution: Automatically falls back to API method
|
||||
|
||||
3. **Network Issues**
|
||||
- Error: Download fails
|
||||
- Solution: Automatic fallback to API method
|
||||
|
||||
4. **Rate Limiting**
|
||||
- Error: API rate limit exceeded
|
||||
- Solution: Use GitHub token or wait for reset
|
||||
|
||||
### Fallback Strategy
|
||||
|
||||
1. Try archive method first (if enabled and size allows)
|
||||
2. If archive fails, automatically try API method
|
||||
3. If both fail, show error message to user
|
||||
|
||||
## 🎨 UI Enhancements
|
||||
|
||||
### Progress Display
|
||||
|
||||
The UI now shows:
|
||||
- **Method**: ARCHIVE or API
|
||||
- **Stage**: downloading, extracting, processing, complete
|
||||
- **Progress Bar**: Visual progress indicator
|
||||
- **File Count**: Files processed vs total files
|
||||
|
||||
### Settings Integration
|
||||
|
||||
New settings in the settings modal:
|
||||
- Archive method preferences
|
||||
- Size limits
|
||||
- Performance options
|
||||
|
||||
### Branch Selection
|
||||
|
||||
Users can now specify:
|
||||
- Custom branch names
|
||||
- Default branch fallback
|
||||
- Branch validation
|
||||
|
||||
## 🔮 Future Enhancements
|
||||
|
||||
### Planned Features
|
||||
|
||||
1. **Caching**: Cache downloaded archives for faster re-processing
|
||||
2. **Parallel Downloads**: Download multiple repositories simultaneously
|
||||
3. **Incremental Updates**: Only download changed files
|
||||
4. **Advanced Filtering**: More granular file filtering options
|
||||
5. **Performance Analytics**: Track and optimize performance
|
||||
|
||||
### Potential Improvements
|
||||
|
||||
1. **WebSocket Progress**: Real-time progress updates
|
||||
2. **Background Processing**: Process repositories in background
|
||||
3. **Queue Management**: Handle multiple repository requests
|
||||
4. **Smart Caching**: Intelligent cache invalidation
|
||||
5. **Performance Metrics**: Detailed performance reporting
|
||||
|
||||
## 📈 Performance Metrics
|
||||
|
||||
### Expected Improvements
|
||||
|
||||
- **Small repos (< 10MB)**: 10x faster
|
||||
- **Medium repos (10-50MB)**: 5-8x faster
|
||||
- **Large repos (50-100MB)**: 3-5x faster
|
||||
- **Very large repos (> 100MB)**: Uses API method
|
||||
|
||||
### Memory Usage
|
||||
|
||||
- **Archive Method**: Higher memory usage during extraction
|
||||
- **API Method**: Lower memory usage, distributed over time
|
||||
- **Optimization**: Batch processing prevents memory spikes
|
||||
|
||||
## 🛠️ Troubleshooting
|
||||
|
||||
### Common Problems
|
||||
|
||||
1. **Archive Download Fails**
|
||||
- Check network connection
|
||||
- Verify repository is public
|
||||
- Try API method as fallback
|
||||
|
||||
2. **Memory Issues**
|
||||
- Reduce batch size in archive service
|
||||
- Use API method for very large repositories
|
||||
- Clear browser cache
|
||||
|
||||
3. **Progress Not Updating**
|
||||
- Check console for errors
|
||||
- Verify progress callback is working
|
||||
- Refresh page and try again
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Enable debug logging:
|
||||
```javascript
|
||||
localStorage.setItem('debug_github_archive', 'true');
|
||||
```
|
||||
|
||||
## 📚 API Reference
|
||||
|
||||
### GitHubArchiveService
|
||||
|
||||
```typescript
|
||||
class GitHubArchiveService {
|
||||
static getInstance(): GitHubArchiveService;
|
||||
|
||||
async getRepositoryArchive(
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch?: string,
|
||||
onProgress?: (progress: ArchiveProgress) => void
|
||||
): Promise<ArchiveRepositoryStructure>;
|
||||
|
||||
async checkRepositoryAccess(owner: string, repo: string): Promise<boolean>;
|
||||
async estimateRepositorySize(owner: string, repo: string): Promise<number>;
|
||||
async getBranches(owner: string, repo: string): Promise<string[]>;
|
||||
}
|
||||
```
|
||||
|
||||
### HybridGitHubService
|
||||
|
||||
```typescript
|
||||
class HybridGitHubService {
|
||||
static getInstance(): HybridGitHubService;
|
||||
|
||||
async getRepositoryStructure(
|
||||
owner: string,
|
||||
repo: string,
|
||||
branch?: string,
|
||||
options?: HybridOptions,
|
||||
onProgress?: (progress: HybridProgress) => void
|
||||
): Promise<CompleteRepositoryStructure>;
|
||||
|
||||
async compareMethods(owner: string, repo: string): Promise<MethodComparison>;
|
||||
async checkRepositoryAccess(owner: string, repo: string): Promise<boolean>;
|
||||
async estimateRepositorySize(owner: string, repo: string): Promise<number>;
|
||||
async getBranches(owner: string, repo: string): Promise<string[]>;
|
||||
}
|
||||
```
|
||||
|
||||
## 🎉 Conclusion
|
||||
|
||||
The GitHub Archive implementation provides significant performance improvements for GitNexus users, especially for small to medium-sized repositories. The hybrid approach ensures reliability while maximizing speed, and the enhanced UI provides better user experience with detailed progress tracking.
|
||||
|
||||
The implementation is production-ready and includes comprehensive error handling, testing, and documentation for future maintenance and enhancement.
|
||||
|
|
@ -1,557 +0,0 @@
|
|||
# 🔍 CodeNexus - Edge Knowledge Graph Creator with Graph RAG
|
||||
|
||||
**Transform any codebase into an interactive knowledge graph in your browser. No servers, no setup - just instant Graph RAG-powered code intelligence.**
|
||||
|
||||
CodeNexus is a client-side knowledge graph creator that runs entirely in your browser. Drop in a GitHub repo or ZIP file, and get an interactive knowledge graph with AI-powered chat interface. Perfect for code exploration, documentation, and understanding complex codebases through Graph RAG (Retrieval-Augmented Generation).
|
||||
|
||||
## ✨ Features
|
||||
|
||||
### 📊 **Code Analysis & Visualization**
|
||||
- **GitHub Integration**: Analyze any public GitHub repository directly from URL
|
||||
- **ZIP File Support**: Upload and analyze local code archives
|
||||
- **Interactive Knowledge Graph**: Visualize code structure with Cytoscape.js
|
||||
- **Multi-language Support**: Currently optimized for Python with extensible architecture
|
||||
- **Smart Filtering**: Directory and file pattern filters to focus analysis scope
|
||||
- **Performance Optimization**: Configurable file limits with confirmation dialogs for large repositories
|
||||
|
||||
### 🤖 **AI-Powered Chat Interface**
|
||||
- **Multiple LLM Providers**: OpenAI, Anthropic (Claude), Google Gemini
|
||||
- **ReAct Agent Pattern**: Uses proper LangChain ReAct implementation for reasoning
|
||||
- **Tool-Augmented Responses**: Graph queries, code retrieval, file search
|
||||
- **Context-Aware**: Maintains conversation history with configurable memory
|
||||
|
||||
### 🔧 **Advanced Processing Pipeline**
|
||||
- **3-Pass Ingestion Strategy**:
|
||||
1. **Structure Analysis**: Project hierarchy and file organization
|
||||
2. **Code Parsing**: AST-based extraction using Tree-sitter
|
||||
3. **Call Resolution**: Function/method call relationship mapping
|
||||
- **Web Worker Processing**: Non-blocking UI with progress tracking
|
||||
- **Intelligent Caching**: AST and processing result optimization
|
||||
- **Error Resilience**: Comprehensive error boundaries and recovery mechanisms
|
||||
|
||||
### 🎨 **Modern UI/UX**
|
||||
- **Responsive Design**: Adaptive layout for different screen sizes
|
||||
- **Real-time Progress**: Live updates during repository processing
|
||||
- **Interactive Graph**: Node selection, zooming, panning
|
||||
- **Split-Panel Layout**: Graph visualization + AI chat interface
|
||||
- **Settings Management**: Persistent configuration for API keys and preferences
|
||||
- **Export Functionality**: Download knowledge graphs as JSON with metadata
|
||||
- **Performance Controls**: File limits, filtering, and optimization settings
|
||||
|
||||
### 🛡️ **Reliability & Performance**
|
||||
- **Error Boundaries**: Graceful error handling with user-friendly recovery options
|
||||
- **Performance Monitoring**: Real-time processing statistics and export size calculation
|
||||
- **Memory Management**: Efficient handling of large repositories with configurable limits
|
||||
- **Progress Tracking**: Detailed progress indicators with phase-specific messaging
|
||||
- **Confirmation Dialogs**: Smart warnings for potentially expensive operations
|
||||
|
||||
## 🏗️ Architecture
|
||||
|
||||
### **Frontend Stack**
|
||||
- **React 18** with TypeScript
|
||||
- **Vite** for fast development and building
|
||||
- **Cytoscape.js** for graph visualization
|
||||
- **Custom CSS** with modern design patterns
|
||||
- **Error Boundaries** for robust error handling
|
||||
|
||||
### **Processing Engine**
|
||||
- **Deno Runtime** for TypeScript execution
|
||||
- **Tree-sitter WASM** for syntax parsing
|
||||
- **Web Workers** for background processing
|
||||
- **Comlink** for worker communication
|
||||
|
||||
### **AI Integration**
|
||||
- **LangChain.js** with proper ReAct agent implementation
|
||||
- **Multiple LLM Support**: OpenAI, Anthropic, Gemini
|
||||
- **Tool-based Architecture**: Graph queries, code retrieval, file search
|
||||
- **Cypher Query Generation**: Natural language to graph queries
|
||||
|
||||
### **Services Layer**
|
||||
```
|
||||
src/
|
||||
├── services/ # External API integrations
|
||||
│ ├── github.ts # GitHub REST API client
|
||||
│ └── zip.ts # ZIP file processing
|
||||
├── core/ # Core processing logic
|
||||
│ ├── graph/ # Knowledge graph types
|
||||
│ ├── ingestion/ # 3-pass processing pipeline
|
||||
│ └── tree-sitter/ # Syntax parsing infrastructure
|
||||
├── ai/ # AI and RAG components
|
||||
│ ├── llm-service.ts # Multi-provider LLM client
|
||||
│ ├── cypher-generator.ts # NL to Cypher translation
|
||||
│ ├── orchestrator.ts # Custom ReAct implementation
|
||||
│ └── langchain-orchestrator.ts # Standard LangChain ReAct
|
||||
├── workers/ # Web Worker implementations
|
||||
├── ui/ # React components and pages
|
||||
│ ├── components/ # Reusable UI components
|
||||
│ │ ├── ErrorBoundary.tsx # Error handling component
|
||||
│ │ ├── graph/ # Graph visualization components
|
||||
│ │ └── chat/ # Chat interface components
|
||||
│ └── pages/ # Application pages
|
||||
├── lib/ # Shared utilities
|
||||
│ └── export.ts # Graph export functionality
|
||||
└── App.tsx # Main application entry point
|
||||
```
|
||||
|
||||
## 🚀 Getting Started
|
||||
|
||||
### Prerequisites
|
||||
- **Node.js 18+** and **npm/yarn**
|
||||
- **Deno 1.40+** for development
|
||||
- **API Keys** for AI features (OpenAI, Anthropic, or Gemini)
|
||||
|
||||
### Installation
|
||||
|
||||
1. **Clone the repository**
|
||||
```bash
|
||||
git clone <repository-url>
|
||||
cd gitnexus
|
||||
```
|
||||
|
||||
2. **Install dependencies**
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
3. **Start development server**
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
4. **Open in browser**
|
||||
```
|
||||
http://localhost:5173
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
1. **GitHub Token (Optional)**
|
||||
- Increases rate limit from 60 to 5,000 requests/hour
|
||||
- Generate at: https://github.com/settings/tokens
|
||||
- Requires no special permissions for public repos
|
||||
|
||||
2. **AI API Keys**
|
||||
- **OpenAI**: Get from https://platform.openai.com/api-keys
|
||||
- **Anthropic**: Get from https://console.anthropic.com/
|
||||
- **Gemini**: Get from https://makersuite.google.com/app/apikey
|
||||
|
||||
3. **Performance Settings**
|
||||
- **File Limit**: Configure maximum files to process (default: 500)
|
||||
- **Directory Filters**: Focus on specific directories (e.g., "src", "lib")
|
||||
- **File Patterns**: Filter by file types (e.g., "*.py", "*.js", "*.ts")
|
||||
|
||||
## 💡 Usage
|
||||
|
||||
### Analyzing a Repository
|
||||
|
||||
1. **GitHub Repository**
|
||||
```
|
||||
1. Enter GitHub URL: https://github.com/owner/repo
|
||||
2. Optional: Set directory/file filters to focus analysis
|
||||
3. Click "Analyze"
|
||||
4. For large repos: Confirm processing or adjust filters
|
||||
5. Wait for processing (structure → parsing → call resolution)
|
||||
6. Explore the interactive graph
|
||||
```
|
||||
|
||||
2. **ZIP File Upload**
|
||||
```
|
||||
1. Click "Choose File" and select a .zip file
|
||||
2. Optional: Configure filters before processing
|
||||
3. Click "Analyze"
|
||||
4. Processing will extract and analyze text files
|
||||
5. Explore results in the graph visualization
|
||||
```
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
1. **Directory Filtering**
|
||||
```
|
||||
- Enter directory names: "src", "lib", "components"
|
||||
- Focuses analysis on specific parts of the codebase
|
||||
- Reduces processing time and memory usage
|
||||
```
|
||||
|
||||
2. **File Pattern Filtering**
|
||||
```
|
||||
- Use patterns: "*.py", "*.js", "*.ts"
|
||||
- Supports wildcards: "test*.py", "*util*"
|
||||
- Comma-separated: "*.py,*.js,*.ts"
|
||||
```
|
||||
|
||||
3. **File Limits**
|
||||
```
|
||||
- Default limit: 500 files
|
||||
- Configurable in settings (50-2000 files)
|
||||
- Large repositories show confirmation dialog
|
||||
- Automatic truncation to limit if confirmed
|
||||
```
|
||||
|
||||
### Using the AI Chat
|
||||
|
||||
1. **Configure API Key**
|
||||
```
|
||||
1. Click the ⚙️ settings button
|
||||
2. Choose your preferred LLM provider
|
||||
3. Enter your API key
|
||||
4. Select model (e.g., gpt-4o-mini, claude-3-haiku)
|
||||
```
|
||||
|
||||
2. **Ask Questions**
|
||||
```
|
||||
- "What functions are in the main.py file?"
|
||||
- "Show me all classes that inherit from BaseClass"
|
||||
- "How does the authentication system work?"
|
||||
- "Find all functions that call the database"
|
||||
```
|
||||
|
||||
### Exporting Data
|
||||
|
||||
1. **Export Knowledge Graph**
|
||||
```
|
||||
1. Click the 📥 Export button after processing
|
||||
2. Downloads JSON file with graph data and metadata
|
||||
3. Includes processing statistics and timestamps
|
||||
4. File size shown in UI before export
|
||||
```
|
||||
|
||||
2. **Export Format**
|
||||
```json
|
||||
{
|
||||
"metadata": {
|
||||
"exportedAt": "2024-01-01T12:00:00.000Z",
|
||||
"version": "1.0.0",
|
||||
"nodeCount": 150,
|
||||
"relationshipCount": 200,
|
||||
"fileCount": 25,
|
||||
"processingDuration": 5000
|
||||
},
|
||||
"graph": {
|
||||
"nodes": [...],
|
||||
"relationships": [...]
|
||||
},
|
||||
"fileContents": {...}
|
||||
}
|
||||
```
|
||||
|
||||
### Graph Interaction
|
||||
|
||||
- **Node Selection**: Click any node to highlight and view details
|
||||
- **Zoom & Pan**: Mouse wheel to zoom, drag to pan
|
||||
- **Node Types**: Different colors/shapes for files, functions, classes, etc.
|
||||
- **Relationships**: Arrows show CONTAINS, CALLS, INHERITS relationships
|
||||
|
||||
## 🔧 Development
|
||||
|
||||
### Project Structure
|
||||
```
|
||||
GitNexus/
|
||||
├── src/
|
||||
│ ├── services/ # External integrations
|
||||
│ ├── core/ # Processing pipeline
|
||||
│ ├── ai/ # AI and RAG systems
|
||||
│ ├── workers/ # Web Workers
|
||||
│ ├── ui/ # React components
|
||||
│ │ ├── components/ # Reusable components
|
||||
│ │ │ ├── ErrorBoundary.tsx
|
||||
│ │ │ ├── graph/ # Graph components
|
||||
│ │ │ └── chat/ # Chat components
|
||||
│ │ └── pages/ # Application pages
|
||||
│ ├── lib/ # Utilities
|
||||
│ │ └── export.ts # Export functionality
|
||||
│ └── App.tsx # Main application
|
||||
├── public/
|
||||
│ └── wasm/ # Tree-sitter WASM files
|
||||
├── package.json
|
||||
├── vite.config.ts
|
||||
└── tsconfig.json
|
||||
```
|
||||
|
||||
### Key Components
|
||||
|
||||
#### **Error Handling**
|
||||
```typescript
|
||||
// ErrorBoundary component with recovery options
|
||||
<ErrorBoundary
|
||||
onError={(error, errorInfo) => {
|
||||
console.error('Application error:', error);
|
||||
}}
|
||||
>
|
||||
<App />
|
||||
</ErrorBoundary>
|
||||
```
|
||||
|
||||
#### **Performance Optimization**
|
||||
```typescript
|
||||
// File filtering and limits
|
||||
const filterFiles = (files: any[]) => {
|
||||
return files
|
||||
.filter(file => matchesDirectoryFilter(file))
|
||||
.filter(file => matchesPatternFilter(file))
|
||||
.slice(0, maxFiles);
|
||||
};
|
||||
```
|
||||
|
||||
#### **Export Functionality**
|
||||
```typescript
|
||||
// Export with metadata
|
||||
exportAndDownloadGraph(graph, {
|
||||
projectName: 'my-project',
|
||||
includeMetadata: true,
|
||||
prettyPrint: true
|
||||
}, fileContents, { duration: 5000 });
|
||||
```
|
||||
|
||||
#### **Processing Pipeline**
|
||||
```typescript
|
||||
// 3-pass ingestion strategy with progress tracking
|
||||
const pipeline = new GraphPipeline();
|
||||
const result = await pipeline.run({
|
||||
projectRoot: '/',
|
||||
projectName: 'MyProject',
|
||||
filePaths: ['src/main.py', 'src/utils.py'],
|
||||
fileContents: new Map([
|
||||
['src/main.py', 'def main(): pass'],
|
||||
['src/utils.py', 'def helper(): pass']
|
||||
])
|
||||
});
|
||||
```
|
||||
|
||||
#### **AI Integration**
|
||||
```typescript
|
||||
// LangChain ReAct agent with error handling
|
||||
const orchestrator = new LangChainRAGOrchestrator(llmService, cypherGenerator);
|
||||
await orchestrator.setContext({ graph, fileContents }, llmConfig);
|
||||
const response = await orchestrator.answerQuestion("How does auth work?");
|
||||
```
|
||||
|
||||
#### **Graph Visualization**
|
||||
```typescript
|
||||
// Interactive graph component with error boundaries
|
||||
<ErrorBoundary>
|
||||
<GraphExplorer
|
||||
graph={knowledgeGraph}
|
||||
onNodeSelect={(nodeId) => setSelectedNode(nodeId)}
|
||||
/>
|
||||
</ErrorBoundary>
|
||||
```
|
||||
|
||||
### Adding New Features
|
||||
|
||||
1. **New Language Support**
|
||||
```typescript
|
||||
// Add parser in core/tree-sitter/
|
||||
export const loadJavaScriptParser = async () => {
|
||||
// Load JS Tree-sitter grammar
|
||||
};
|
||||
```
|
||||
|
||||
2. **Custom AI Tools**
|
||||
```typescript
|
||||
// Add tools in ai/langchain-orchestrator.ts
|
||||
const customTool = tool(
|
||||
async (input: { query: string }) => {
|
||||
// Tool implementation
|
||||
},
|
||||
{
|
||||
name: "custom_tool",
|
||||
description: "Custom functionality",
|
||||
schema: z.object({ query: z.string() })
|
||||
}
|
||||
);
|
||||
```
|
||||
|
||||
3. **Export Formats**
|
||||
```typescript
|
||||
// Add new export formats in lib/export.ts
|
||||
export function exportToCSV(graph: KnowledgeGraph): string {
|
||||
// CSV export implementation
|
||||
}
|
||||
```
|
||||
|
||||
## 🧪 Testing & Quality Assurance
|
||||
|
||||
### Error Handling
|
||||
- **Error Boundaries**: Catch and display JavaScript errors gracefully
|
||||
- **User Recovery**: Allow users to reset component state after errors
|
||||
- **Detailed Logging**: Console logging for debugging and error reporting
|
||||
- **Fallback UI**: User-friendly error messages with recovery options
|
||||
|
||||
### Performance Testing
|
||||
1. **Large Repository Handling**
|
||||
- Test with repositories containing 1000+ files
|
||||
- Verify confirmation dialogs for file limits
|
||||
- Monitor memory usage during processing
|
||||
- Test filtering effectiveness
|
||||
|
||||
2. **UI Responsiveness**
|
||||
- Ensure non-blocking processing with Web Workers
|
||||
- Verify progress indicators update correctly
|
||||
- Test error recovery mechanisms
|
||||
- Validate export functionality with large graphs
|
||||
|
||||
3. **Error Scenarios**
|
||||
- Network failures during GitHub API calls
|
||||
- Corrupted ZIP files
|
||||
- Invalid API keys
|
||||
- Memory exhaustion scenarios
|
||||
|
||||
### Manual Testing Checklist
|
||||
- [ ] GitHub repository analysis with various sizes
|
||||
- [ ] ZIP file upload and extraction
|
||||
- [ ] Directory and file pattern filtering
|
||||
- [ ] Large repository confirmation dialog
|
||||
- [ ] Export functionality with different options
|
||||
- [ ] Error boundary activation and recovery
|
||||
- [ ] API key validation for all providers
|
||||
- [ ] Settings persistence across sessions
|
||||
- [ ] Graph visualization interactions
|
||||
- [ ] Chat interface with different LLM providers
|
||||
|
||||
## 🚀 Deployment
|
||||
|
||||
### Production Build
|
||||
```bash
|
||||
npm run build
|
||||
npm run preview
|
||||
```
|
||||
|
||||
### Environment Variables
|
||||
```env
|
||||
# Optional: Pre-configure API keys
|
||||
VITE_OPENAI_API_KEY=sk-...
|
||||
VITE_ANTHROPIC_API_KEY=sk-ant-...
|
||||
VITE_GEMINI_API_KEY=...
|
||||
|
||||
# Performance settings
|
||||
VITE_DEFAULT_MAX_FILES=500
|
||||
VITE_ENABLE_DEBUG_LOGGING=false
|
||||
```
|
||||
|
||||
### Docker Deployment
|
||||
```dockerfile
|
||||
FROM node:18-alpine
|
||||
WORKDIR /app
|
||||
COPY package*.json ./
|
||||
RUN npm ci --only=production
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
EXPOSE 3000
|
||||
CMD ["npm", "run", "preview", "--", "--host", "0.0.0.0"]
|
||||
```
|
||||
|
||||
### Performance Monitoring
|
||||
```javascript
|
||||
// Add performance monitoring
|
||||
const observer = new PerformanceObserver((list) => {
|
||||
for (const entry of list.getEntries()) {
|
||||
if (entry.entryType === 'measure') {
|
||||
console.log(`${entry.name}: ${entry.duration}ms`);
|
||||
}
|
||||
}
|
||||
});
|
||||
observer.observe({ entryTypes: ['measure'] });
|
||||
```
|
||||
|
||||
## 🔒 Security & Privacy
|
||||
|
||||
- **Client-Side Processing**: All analysis happens in your browser
|
||||
- **API Keys**: Stored locally, never transmitted to our servers
|
||||
- **GitHub Access**: Uses public API, respects repository permissions
|
||||
- **Data Privacy**: No code or analysis results are stored remotely
|
||||
- **Error Logging**: Sensitive data excluded from error reports
|
||||
- **Export Security**: User-controlled data export with no server interaction
|
||||
|
||||
## 🤝 Contributing
|
||||
|
||||
### Development Setup
|
||||
1. Fork the repository
|
||||
2. Create feature branch: `git checkout -b feature/amazing-feature`
|
||||
3. Make changes and test thoroughly
|
||||
4. Run the testing checklist above
|
||||
5. Commit: `git commit -m 'Add amazing feature'`
|
||||
6. Push: `git push origin feature/amazing-feature`
|
||||
7. Open a Pull Request
|
||||
|
||||
### Code Style
|
||||
- **TypeScript**: Strict mode enabled
|
||||
- **ESLint**: Follow configured rules
|
||||
- **Prettier**: Auto-formatting
|
||||
- **Comments**: Minimal, only when necessary
|
||||
- **Error Handling**: Comprehensive error boundaries and recovery
|
||||
- **Performance**: Consider memory usage and processing time
|
||||
|
||||
### Testing Guidelines
|
||||
- Test error scenarios and edge cases
|
||||
- Verify performance with large datasets
|
||||
- Ensure graceful degradation
|
||||
- Test all export functionality
|
||||
- Validate error boundary behavior
|
||||
|
||||
## 📚 Technical Details
|
||||
|
||||
### Knowledge Graph Schema
|
||||
```typescript
|
||||
interface KnowledgeGraph {
|
||||
nodes: GraphNode[]; // Code entities
|
||||
relationships: GraphRelationship[]; // Connections
|
||||
}
|
||||
|
||||
// Node types: Project, Folder, File, Module, Class, Function, Method, Variable
|
||||
// Relationship types: CONTAINS, CALLS, INHERITS, OVERRIDES, IMPORTS
|
||||
```
|
||||
|
||||
### Export Format
|
||||
```typescript
|
||||
interface ExportedGraph {
|
||||
metadata: {
|
||||
exportedAt: string;
|
||||
version: string;
|
||||
nodeCount: number;
|
||||
relationshipCount: number;
|
||||
fileCount?: number;
|
||||
processingDuration?: number;
|
||||
};
|
||||
graph: KnowledgeGraph;
|
||||
fileContents?: Record<string, string>;
|
||||
}
|
||||
```
|
||||
|
||||
### Error Boundary Implementation
|
||||
- **Component-Level**: Individual components wrapped for isolation
|
||||
- **Application-Level**: Top-level boundary for catastrophic failures
|
||||
- **Recovery Options**: Reset state, reload page, or continue with fallback
|
||||
- **Error Reporting**: Detailed technical information for developers
|
||||
|
||||
### Performance Optimizations
|
||||
- **Web Workers**: Non-blocking processing
|
||||
- **AST Caching**: Reuse parsed syntax trees
|
||||
- **Progressive Loading**: Stream results as available
|
||||
- **Memory Management**: Efficient data structures
|
||||
- **File Filtering**: Reduce processing scope
|
||||
- **Confirmation Dialogs**: Prevent accidental expensive operations
|
||||
|
||||
### ReAct Agent Implementation
|
||||
- **Standard LangChain**: Uses `createReactAgent` from `@langchain/langgraph/prebuilt`
|
||||
- **Custom Implementation**: Manual ReAct loop for educational purposes
|
||||
- **Tools**: Graph queries, code retrieval, file search
|
||||
- **Memory**: Conversation persistence with thread management
|
||||
- **Error Recovery**: Graceful handling of API failures
|
||||
|
||||
## 📄 License
|
||||
|
||||
This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
|
||||
|
||||
## 🙏 Acknowledgments
|
||||
|
||||
- **Tree-sitter**: Syntax parsing infrastructure
|
||||
- **LangChain.js**: AI agent framework
|
||||
- **Cytoscape.js**: Graph visualization
|
||||
- **React**: UI framework with error boundaries
|
||||
- **Vite**: Build tool and dev server
|
||||
|
||||
---
|
||||
|
||||
**CodeNexus** - Edge Knowledge Graph Creator with instant Graph RAG. Zero setup, maximum insight. 🚀
|
||||
|
||||
*Browser-native code intelligence that runs anywhere, anytime - no servers required.*
|
||||
|
|
@ -1,301 +0,0 @@
|
|||
# 🔧 Fixing Isolated Nodes in Graph Visualization
|
||||
|
||||
## Problem Description
|
||||
You're seeing nodes "flying away" with no connections in the graph visualization. This indicates **isolated nodes** - nodes that have no relationships to other nodes in the graph.
|
||||
|
||||
## Root Causes
|
||||
|
||||
### 1. **File Parsing Failures** (Most Common)
|
||||
- Files fail to parse during **Pass 2** of ingestion
|
||||
- File nodes get created but no functions/classes are extracted
|
||||
- Results in isolated file nodes
|
||||
|
||||
### 2. **Unsupported File Types**
|
||||
- Files with extensions not recognized by the parser
|
||||
- Configuration files, documentation, etc. without code content
|
||||
|
||||
### 3. **Syntax Errors**
|
||||
- Malformed code that the AST parser can't understand
|
||||
- Missing imports or exports
|
||||
- Language-specific syntax issues
|
||||
|
||||
### 4. **Import Resolution Failures**
|
||||
- **Pass 3** fails to resolve import relationships
|
||||
- Files exist but aren't connected via imports
|
||||
|
||||
### 5. **Call Resolution Failures**
|
||||
- **Pass 4** fails to find function calls between files
|
||||
- Functions exist but no call relationships are created
|
||||
|
||||
## 🛠️ How to Diagnose
|
||||
|
||||
### Step 1: Use the New Diagnostic Tool
|
||||
1. Load your repository in GitNexus
|
||||
2. Click the **🩺 Diagnose** button in the chat interface
|
||||
3. Check the statistics and follow the suggested steps
|
||||
|
||||
### Step 2: Check Browser Console
|
||||
1. Open Developer Tools (F12)
|
||||
2. Look for console warnings during ingestion:
|
||||
- `⚠️ Found X isolated nodes`
|
||||
- `⚠️ Found X files without definitions`
|
||||
- `Source files without definitions: [...]`
|
||||
|
||||
### Step 3: Review Console Logs
|
||||
Look for these specific log messages:
|
||||
```
|
||||
📁 Pass 1: Analyzing project structure...
|
||||
🔍 Pass 2: Parsing code and extracting definitions...
|
||||
🔗 Pass 3: Resolving imports and building dependency map...
|
||||
📞 Pass 4: Resolving function calls with 3-stage strategy...
|
||||
```
|
||||
|
||||
## 🔍 Diagnostic Information
|
||||
|
||||
The enhanced pipeline now shows:
|
||||
- **Node counts by type** (Project, Folder, File, Function, Class, etc.)
|
||||
- **Relationship counts by type** (CONTAINS, CALLS, IMPORTS, etc.)
|
||||
- **Isolated nodes** with examples
|
||||
- **Files without definitions**
|
||||
- **Graph integrity issues**
|
||||
|
||||
## ✅ Recent Fixes Applied
|
||||
|
||||
### **1. Reduced Console Noise (Fixed)**
|
||||
- **Issue**: Thousands of "Failed to resolve call" messages for Python built-ins like `int`, `str`, `len`, etc.
|
||||
- **Fix**: Added `shouldIgnoreCall()` method to filter out Python built-in functions and standard library calls
|
||||
- **Result**: Console output is now much cleaner and shows only relevant failures
|
||||
|
||||
### **2. Improved Python Import Resolution (Fixed)**
|
||||
- **Issue**: Python imports weren't being resolved correctly, causing "No import relationships found"
|
||||
- **Fix**: Enhanced `resolveModulePath()` with better pattern matching for complex project structures
|
||||
- **Features Added**:
|
||||
- Multiple resolution patterns for Python modules
|
||||
- Partial path matching for complex project structures
|
||||
- Better handling of package imports
|
||||
- Enhanced debugging with import resolution statistics
|
||||
|
||||
### **3. Enhanced Diagnostic Reporting (Added)**
|
||||
- **New**: Comprehensive graph integrity validation
|
||||
- **New**: Import resolution success rate reporting
|
||||
- **New**: Detailed breakdown of isolated nodes by type
|
||||
- **New**: 🩺 Diagnose button in chat interface
|
||||
|
||||
## 🚀 Solutions
|
||||
|
||||
### For File Parsing Issues:
|
||||
1. **Check file extensions**: Ensure files are `.js`, `.ts`, `.jsx`, `.tsx`, `.py`, etc.
|
||||
2. **Verify syntax**: Make sure code files have valid syntax
|
||||
3. **Check file size**: Very large files might timeout during parsing
|
||||
4. **Review file content**: Empty files or files with only comments won't generate nodes
|
||||
|
||||
### For Import Issues (Now Improved):
|
||||
1. **Check import syntax**: Ensure proper `import`/`export` or `require()` statements
|
||||
2. **Verify file paths**: Relative imports should resolve correctly
|
||||
3. **Check module resolution**: External libraries might not be resolved
|
||||
4. **Monitor import resolution rate**: Should be >50% for healthy projects
|
||||
|
||||
### For Call Issues:
|
||||
1. **Function calls**: Ensure functions are actually called between files
|
||||
2. **Method calls**: Class methods should be invoked
|
||||
3. **Export/import**: Functions need to be properly exported and imported
|
||||
|
||||
## 🔧 Quick Fixes
|
||||
|
||||
### 1. **Filter Out Non-Code Files**
|
||||
Use GitNexus filtering options to exclude:
|
||||
- Documentation files (`.md`, `.txt`)
|
||||
- Configuration files (`.json`, `.yaml`, `.xml`)
|
||||
- Asset files (`.png`, `.jpg`, `.css`)
|
||||
|
||||
### 2. **Focus on Core Directories**
|
||||
- Include only `src/`, `lib/`, `app/` directories
|
||||
- Exclude `node_modules/`, `.git/`, `dist/`, `build/`
|
||||
|
||||
### 3. **Check File Limits**
|
||||
- Large repositories might hit processing limits
|
||||
- Consider processing smaller subsets first
|
||||
|
||||
## 📊 Expected Results After Fixes
|
||||
|
||||
A healthy graph should show:
|
||||
- **Project** → **Folders** → **Files** (CONTAINS relationships)
|
||||
- **Files** → **Functions/Classes** (CONTAINS relationships)
|
||||
- **Files** → **Files** (IMPORTS relationships) - **Now working better**
|
||||
- **Functions** → **Functions** (CALLS relationships) - **Cleaner console output**
|
||||
|
||||
### **Typical Success Rates**:
|
||||
- **Import Resolution**: 40-70% (up from 0%)
|
||||
- **Call Resolution**: 20-30% (excluding built-ins)
|
||||
- **File Parsing**: 80-95% for source files
|
||||
|
||||
## 🆘 Still Having Issues?
|
||||
|
||||
1. **Try the diagnostic tool**: Click 🩺 Diagnose button
|
||||
2. **Check console output**: Look for specific error patterns
|
||||
3. **Share diagnostic info**: Copy the improved console logs
|
||||
4. **Test incrementally**: Try with smaller subsets of files
|
||||
|
||||
## 📈 What You Should See Now
|
||||
|
||||
After the fixes, your console output should show:
|
||||
```
|
||||
ImportProcessor: Found 45 imports, resolved 28 (62.2%)
|
||||
CallProcessor: Success rate: 45.9% (excluding built-ins)
|
||||
📊 Graph Statistics:
|
||||
Relationships by type: {CONTAINS: 395, DECORATES: 48, IMPORTS: 13, CALLS: 47}
|
||||
✅ Graph integrity validation passed
|
||||
```
|
||||
|
||||
### **Latest Improvements (v2)**:
|
||||
- **Expanded built-ins filtering**: Now ignores 100+ Python string methods, math functions, and third-party library calls
|
||||
- **Better diagnostics**: Identifies source files with zero function calls (potential parsing issues)
|
||||
- **Pattern-based filtering**: Automatically ignores dunder methods (`__init__`, `__str__`) and private methods
|
||||
|
||||
### **Expected Results After All Fixes**:
|
||||
- **Console noise reduction**: 95%+ reduction in irrelevant error messages
|
||||
- **Import relationships**: 10-20+ IMPORTS relationships created
|
||||
- **Call resolution success**: 40-60% (realistic for complex codebases)
|
||||
- **Failed calls**: Only legitimate issues (domain-specific functions, missing imports)
|
||||
|
||||
Instead of thousands of failed call resolutions, you'll see much cleaner output focused on actual issues that need attention!
|
||||
|
||||
## 🎯 **FINAL STATUS - Issue Resolved!**
|
||||
|
||||
### **🔍 MAJOR DISCOVERY - Python Call Extraction Bug Found!**
|
||||
|
||||
**Latest diagnostic output revealed a critical issue:**
|
||||
```
|
||||
📊 Debug: config.py has 38 call nodes, 0 definitions
|
||||
📊 Debug: assessment_db.py has 120 call nodes, 4 definitions
|
||||
📊 Debug: sonar_analyzer.py has 30 call nodes, 2 definitions
|
||||
```
|
||||
|
||||
**This shows that:**
|
||||
- ✅ **AST parsing works perfectly** - files have hundreds of call nodes
|
||||
- ❌ **Call extraction is broken** - 0 function calls extracted from files with 120+ call nodes
|
||||
- 🔧 **Root cause identified** - Python call extraction logic needs fixes
|
||||
|
||||
### **🛠️ Latest Fix Applied**
|
||||
- **Enhanced Python call extraction** with better node type handling
|
||||
- **Comprehensive debugging** to identify what's being filtered vs extracted
|
||||
- **Improved function name extraction** for complex Python call patterns
|
||||
- **Added support for** subscript calls, nested calls, and more node types
|
||||
|
||||
### **✅ Latest Results (Your Console Output)**
|
||||
```
|
||||
✅ Parsing Success: 38 successful, 0 failed (100%)
|
||||
✅ Import Resolution: 102 imports found, 61 resolved (59.8%)
|
||||
✅ Call Resolution: 129 calls processed, 56.6% success rate
|
||||
✅ Graph Health: 367 nodes, 506 relationships
|
||||
✅ Console Cleanliness: Only 56 legitimate failures (93% noise reduction)
|
||||
```
|
||||
|
||||
### **🔍 Enhanced Diagnostics Added**
|
||||
- **Zero-call file detection**: Identifies source files with parsing issues
|
||||
- **AST node counting**: Shows `call` nodes vs definitions for debugging
|
||||
- **Suspicious file flagging**: Highlights files with definitions but no calls
|
||||
- **Comprehensive built-ins filtering**: 100+ Python functions ignored
|
||||
|
||||
### **📊 Your Graph is Now Healthy!**
|
||||
|
||||
**Before the fixes:**
|
||||
- ❌ 814+ failed call messages (noise)
|
||||
- ❌ No import relationships
|
||||
- ❌ Isolated nodes everywhere
|
||||
- ❌ Unreadable console output
|
||||
|
||||
**After the fixes:**
|
||||
- ✅ **56.6% call resolution success** (excellent!)
|
||||
- ✅ **59.8% import resolution success** (great!)
|
||||
- ✅ **13 IMPORTS relationships** created
|
||||
- ✅ **47 CALLS relationships** created
|
||||
- ✅ **Clean, readable diagnostics**
|
||||
|
||||
### **🔬 Remaining Issues Are Expected**
|
||||
|
||||
#### **1. Zero-Call Files (Normal)**
|
||||
Files showing "No function calls found" are often:
|
||||
- **Model/config files**: Only contain class definitions
|
||||
- **Pure data files**: Constants, configurations
|
||||
- **Interface files**: Abstract base classes
|
||||
- **Files with only imports**: Router configurations
|
||||
|
||||
#### **2. Failed Calls (Legitimate)**
|
||||
The remaining 56 failed calls are **appropriate failures**:
|
||||
- **External libraries**: LangGraph, FastAPI, Azure OpenAI
|
||||
- **Domain-specific**: Business logic libraries (`ruleset`, `assert_fact`)
|
||||
- **Custom models**: Application-specific classes
|
||||
|
||||
### **🎉 Problem Solved!**
|
||||
|
||||
Your **"nodes flying away with no connections"** issue is **fully resolved**:
|
||||
|
||||
1. ✅ **Files connect properly** via CONTAINS relationships
|
||||
2. ✅ **Import relationships work** (13 created)
|
||||
3. ✅ **Function calls connect** (47 relationships)
|
||||
4. ✅ **Console is clean** and diagnostic
|
||||
5. ✅ **Success rates are realistic** for complex codebases
|
||||
|
||||
### **🚀 What You Should See Now**
|
||||
|
||||
When you reload your repository, expect:
|
||||
- **Significantly fewer isolated nodes**
|
||||
- **Connected file clusters** via imports
|
||||
- **Function-to-function connections** within files
|
||||
- **Clean console output** focusing on real issues
|
||||
- **Better graph connectivity** overall
|
||||
|
||||
The isolated nodes that remain will be:
|
||||
- **Configuration files** (expected)
|
||||
- **Documentation files** (expected)
|
||||
- **Empty or comment-only files** (expected)
|
||||
- **External library references** (expected)
|
||||
|
||||
## **🏆 Mission Accomplished!**
|
||||
|
||||
Your graph now has proper connectivity with realistic success rates. The diagnostic tools will help you identify any remaining issues that need attention. The isolated nodes problem is **solved**! 🎯✨
|
||||
|
||||
## 🎛️ **NEW FEATURE: Hide External Libraries Toggle**
|
||||
|
||||
### **✨ What's New**
|
||||
Added a **"Hide external libraries"** toggle in the graph visualization that lets you:
|
||||
- ✅ **Clean view**: Hide isolated external library nodes for cleaner visualization
|
||||
- ✅ **Full view**: Show all nodes including external dependencies for complete context
|
||||
- ✅ **Smart filtering**: Automatically identifies external library patterns
|
||||
- ✅ **Live counter**: Shows how many external nodes are hidden/visible
|
||||
|
||||
### **🎯 How It Works**
|
||||
The toggle uses intelligent filtering to identify external library nodes:
|
||||
- **Isolated nodes**: Nodes with no relationships (not connected to your code)
|
||||
- **External patterns**: Recognizes common library functions like:
|
||||
- `when_all`, `ruleset` (durable rules)
|
||||
- `APIRouter`, `FastAPI` (FastAPI framework)
|
||||
- `StateGraph`, `AsyncAzureOpenAI` (AI libraries)
|
||||
- CamelCase patterns (often external classes)
|
||||
|
||||
### **🚀 When to Use Each Mode**
|
||||
|
||||
#### **Hide External Libraries (Clean View)**
|
||||
**Best for:**
|
||||
- 📊 **Architecture review** - Focus on your internal code structure
|
||||
- 🔍 **Code navigation** - See relationships between your functions/classes
|
||||
- 📈 **Presentations** - Clean, professional visualization
|
||||
- 🎯 **Debugging** - Trace internal call paths without distractions
|
||||
|
||||
#### **Show External Libraries (Full View)**
|
||||
**Best for:**
|
||||
- 🔗 **Dependency analysis** - See what external libraries you use
|
||||
- 🏗️ **System design** - Understand integration points
|
||||
- 📋 **Documentation** - Complete picture of your tech stack
|
||||
- 🔧 **Troubleshooting** - Identify external dependency issues
|
||||
|
||||
### **💡 Pro Tips**
|
||||
- **Default state**: External libraries are **visible by default** for complete context
|
||||
- **Toggle anytime**: Switch between views without reloading the graph
|
||||
- **Persistent**: Your preference is remembered during the session
|
||||
- **Smart counting**: See exactly how many external nodes are being hidden
|
||||
|
||||
This gives you the **best of both worlds** - clean focused views when you need them, and complete architectural context when you want it!
|
||||
|
||||
Instead of thousands of failed call resolutions, you'll see much cleaner output focused on actual issues that need attention!
|
||||
|
|
@ -1,332 +0,0 @@
|
|||
# LRU Cache Implementation for Parsing Processor
|
||||
|
||||
## 🎯 Overview
|
||||
|
||||
The LRU (Least Recently Used) cache implementation provides **significant performance improvements** for the parsing processor by caching parsed files, Tree-sitter query results, and language parsers. This reduces redundant parsing operations and speeds up processing of large codebases.
|
||||
|
||||
## 🚀 Performance Benefits
|
||||
|
||||
### **Expected Improvements:**
|
||||
- **File Parsing**: 2-5x faster for repeated files
|
||||
- **Query Execution**: 3-8x faster for repeated Tree-sitter queries
|
||||
- **Parser Loading**: 10-20x faster for language parser reuse
|
||||
- **Memory Efficiency**: Automatic eviction of least recently used items
|
||||
|
||||
### **Key Features:**
|
||||
- **Multi-level caching** - Files, queries, and parsers cached separately
|
||||
- **Content-based invalidation** - Files cached with content hash
|
||||
- **Automatic eviction** - LRU algorithm prevents memory bloat
|
||||
- **Performance monitoring** - Hit rates and statistics tracking
|
||||
|
||||
---
|
||||
|
||||
## 📁 Files Created/Modified
|
||||
|
||||
### **New Files:**
|
||||
- `src/lib/lru-cache-service.ts` - Main LRU cache service
|
||||
- `src/lib/lru-cache-test.ts` - Test suite for cache functionality
|
||||
|
||||
### **Modified Files:**
|
||||
- `src/core/ingestion/parsing-processor.ts` - Integrated LRU caching
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Core Components
|
||||
|
||||
### **1. LRUCacheService Class**
|
||||
```typescript
|
||||
import { LRUCacheService } from './src/lib/lru-cache-service.js';
|
||||
|
||||
const cache = LRUCacheService.getInstance({
|
||||
max: 500, // Maximum items
|
||||
ttl: 30 * 60 * 1000, // 30 minutes TTL
|
||||
maxSize: 100 * 1024 * 1024 // 100MB max size
|
||||
});
|
||||
```
|
||||
|
||||
**Three Cache Types:**
|
||||
1. **File Cache** - Parsed ASTs and definitions
|
||||
2. **Query Cache** - Tree-sitter query results
|
||||
3. **Parser Cache** - Language parser instances
|
||||
|
||||
### **2. Cache Configuration**
|
||||
```typescript
|
||||
interface CacheOptions {
|
||||
max?: number; // Maximum number of items
|
||||
ttl?: number; // Time to live in milliseconds
|
||||
maxSize?: number; // Maximum size in bytes
|
||||
allowStale?: boolean; // Allow stale items
|
||||
updateAgeOnGet?: boolean; // Update age on access
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Integration Points
|
||||
|
||||
### **ParsingProcessor Integration:**
|
||||
```typescript
|
||||
export class ParsingProcessor {
|
||||
private lruCache: LRUCacheService;
|
||||
|
||||
constructor() {
|
||||
this.lruCache = LRUCacheService.getInstance();
|
||||
}
|
||||
|
||||
// Cache-aware file parsing
|
||||
private async parseFile(graph: KnowledgeGraph, filePath: string, content: string): Promise<void> {
|
||||
const contentHash = this.generateContentHash(content);
|
||||
const cacheKey = this.lruCache.generateFileCacheKey(filePath, contentHash);
|
||||
|
||||
// Check cache first
|
||||
const cachedResult = this.lruCache.getParsedFile(cacheKey);
|
||||
if (cachedResult) {
|
||||
console.log(`Cache hit for file: ${filePath}`);
|
||||
// Use cached result
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse and cache result
|
||||
// ... parsing logic ...
|
||||
this.lruCache.setParsedFile(cacheKey, parsedData);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### **Cache Key Generation:**
|
||||
```typescript
|
||||
// File cache keys include content hash for invalidation
|
||||
const fileKey = cache.generateFileCacheKey('src/main.ts', contentHash);
|
||||
|
||||
// Query cache keys include language and query string
|
||||
const queryKey = cache.generateQueryCacheKey('typescript', queryString);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Cache Statistics & Monitoring
|
||||
|
||||
### **Performance Metrics:**
|
||||
```typescript
|
||||
// Get cache statistics
|
||||
const stats = cache.getStats();
|
||||
console.log('Cache Stats:', {
|
||||
fileCache: { size: 150, max: 200, hitRatio: 0.85 },
|
||||
queryCache: { size: 800, max: 1000, hitRatio: 0.92 },
|
||||
parserCache: { size: 3, max: 10, hitRatio: 0.95 }
|
||||
});
|
||||
|
||||
// Get hit rates
|
||||
const hitRate = cache.getCacheHitRate();
|
||||
console.log('Hit Rates:', {
|
||||
fileCache: '85.2%',
|
||||
queryCache: '92.1%',
|
||||
parserCache: '95.8%'
|
||||
});
|
||||
```
|
||||
|
||||
### **Cache Performance Logging:**
|
||||
```typescript
|
||||
// Automatic logging in ParsingProcessor
|
||||
console.log('ParsingProcessor: Cache Statistics:', {
|
||||
fileCache: { size: 150, hitRate: '85.2%' },
|
||||
queryCache: { size: 800, hitRate: '92.1%' },
|
||||
parserCache: { size: 3, hitRate: '95.8%' }
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Cache Lifecycle
|
||||
|
||||
### **1. Cache Initialization:**
|
||||
```typescript
|
||||
// Singleton pattern ensures single cache instance
|
||||
const cache = LRUCacheService.getInstance(options);
|
||||
```
|
||||
|
||||
### **2. Cache Operations:**
|
||||
```typescript
|
||||
// Set cache items
|
||||
cache.setParsedFile(key, data);
|
||||
cache.setQueryResult(key, data);
|
||||
cache.setParser(language, parser);
|
||||
|
||||
// Get cache items
|
||||
const fileData = cache.getParsedFile(key);
|
||||
const queryData = cache.getQueryResult(key);
|
||||
const parser = cache.getParser(language);
|
||||
```
|
||||
|
||||
### **3. Cache Eviction:**
|
||||
- **LRU Algorithm**: Least recently used items evicted first
|
||||
- **Size Limits**: Automatic eviction when cache is full
|
||||
- **TTL Expiration**: Items expire based on time-to-live
|
||||
- **Memory Pressure**: Size-based eviction for large items
|
||||
|
||||
### **4. Cache Cleanup:**
|
||||
```typescript
|
||||
// Clear specific caches
|
||||
cache.clearFileCache();
|
||||
cache.clearQueryCache();
|
||||
cache.clearParserCache();
|
||||
|
||||
// Clear all caches
|
||||
cache.clearAll();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
### **Test Suite:**
|
||||
```typescript
|
||||
// Browser console testing
|
||||
window.testLRUCacheBasic() // Basic functionality
|
||||
window.testLRUCacheEviction() // LRU eviction behavior
|
||||
window.testCacheKeyGeneration() // Key generation
|
||||
window.runLRUCacheTests() // Run all tests
|
||||
```
|
||||
|
||||
### **Test Coverage:**
|
||||
- **Basic Operations**: Set, get, has, delete
|
||||
- **LRU Eviction**: Automatic removal of least used items
|
||||
- **Key Generation**: File and query cache keys
|
||||
- **Statistics**: Hit rates and cache sizes
|
||||
- **Performance**: Memory usage and eviction behavior
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Usage Examples
|
||||
|
||||
### **Basic Usage:**
|
||||
```typescript
|
||||
import { LRUCacheService } from './src/lib/lru-cache-service.js';
|
||||
|
||||
// Get cache instance
|
||||
const cache = LRUCacheService.getInstance();
|
||||
|
||||
// Cache parsed file
|
||||
cache.setParsedFile('src/main.ts', {
|
||||
ast: parsedAST,
|
||||
definitions: extractedDefinitions,
|
||||
language: 'typescript',
|
||||
lastModified: Date.now(),
|
||||
fileSize: content.length
|
||||
});
|
||||
|
||||
// Retrieve from cache
|
||||
const cached = cache.getParsedFile('src/main.ts');
|
||||
if (cached) {
|
||||
console.log('Cache hit!');
|
||||
// Use cached data
|
||||
}
|
||||
```
|
||||
|
||||
### **Advanced Configuration:**
|
||||
```typescript
|
||||
// Custom cache configuration
|
||||
const cache = LRUCacheService.getInstance({
|
||||
max: 1000, // 1000 items max
|
||||
ttl: 1000 * 60 * 60, // 1 hour TTL
|
||||
maxSize: 200 * 1024 * 1024, // 200MB max size
|
||||
allowStale: true, // Allow stale items
|
||||
updateAgeOnGet: true // Update age on access
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚨 Error Handling & Fallbacks
|
||||
|
||||
### **Cache Miss Handling:**
|
||||
- Graceful fallback to parsing when cache miss occurs
|
||||
- No impact on functionality when cache is unavailable
|
||||
- Automatic cache recovery after errors
|
||||
|
||||
### **Memory Management:**
|
||||
- Automatic eviction prevents memory bloat
|
||||
- Size-based limits protect against large files
|
||||
- TTL expiration ensures fresh data
|
||||
|
||||
---
|
||||
|
||||
## 📈 Performance Impact
|
||||
|
||||
### **Expected Improvements by Cache Type:**
|
||||
|
||||
#### **File Cache:**
|
||||
- **First Run**: No improvement (cache population)
|
||||
- **Subsequent Runs**: 2-5x faster for unchanged files
|
||||
- **Large Codebases**: Significant improvement for repeated processing
|
||||
|
||||
#### **Query Cache:**
|
||||
- **Repeated Queries**: 3-8x faster execution
|
||||
- **Similar Files**: High hit rate for similar code patterns
|
||||
- **Batch Processing**: Excellent for multiple files with similar structure
|
||||
|
||||
#### **Parser Cache:**
|
||||
- **Parser Loading**: 10-20x faster after first load
|
||||
- **Language Switching**: Instant parser availability
|
||||
- **Memory Efficiency**: Reuse expensive parser instances
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Configuration Options
|
||||
|
||||
### **Default Settings:**
|
||||
```typescript
|
||||
const defaultOptions = {
|
||||
max: 500, // 500 items max
|
||||
ttl: 1000 * 60 * 30, // 30 minutes TTL
|
||||
maxSize: 100 * 1024 * 1024, // 100MB max size
|
||||
allowStale: false, // No stale items
|
||||
updateAgeOnGet: true // Update age on access
|
||||
};
|
||||
```
|
||||
|
||||
### **Cache-Specific Settings:**
|
||||
- **File Cache**: 200 items, 1 hour TTL
|
||||
- **Query Cache**: 1000 items, 15 minutes TTL
|
||||
- **Parser Cache**: 10 items, 24 hours TTL
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Success Metrics
|
||||
|
||||
### **Performance Improvements:**
|
||||
- ✅ 2-5x faster file parsing for cached files
|
||||
- ✅ 3-8x faster query execution for repeated queries
|
||||
- ✅ 10-20x faster parser loading after initial load
|
||||
- ✅ Reduced memory pressure through automatic eviction
|
||||
|
||||
### **Code Quality:**
|
||||
- ✅ Comprehensive error handling
|
||||
- ✅ Extensive test coverage
|
||||
- ✅ Performance monitoring and statistics
|
||||
- ✅ Graceful fallback mechanisms
|
||||
|
||||
### **User Experience:**
|
||||
- ✅ Faster processing of large codebases
|
||||
- ✅ Reduced waiting time for repeated operations
|
||||
- ✅ Automatic cache management (no user intervention)
|
||||
- ✅ Detailed performance insights
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Future Enhancements
|
||||
|
||||
### **Planned Improvements:**
|
||||
1. **Persistent Caching**: Save cache to IndexedDB for session persistence
|
||||
2. **Compression**: Compress cached data to reduce memory usage
|
||||
3. **Predictive Caching**: Pre-cache likely-to-be-used files
|
||||
4. **Distributed Caching**: Share cache across browser tabs
|
||||
5. **Adaptive TTL**: Adjust TTL based on file change frequency
|
||||
|
||||
### **Performance Optimizations:**
|
||||
1. **Lazy Loading**: Load cache items on demand
|
||||
2. **Background Prefetching**: Pre-cache files in background
|
||||
3. **Cache Warming**: Pre-populate cache with common patterns
|
||||
4. **Memory Optimization**: Better size calculation algorithms
|
||||
|
||||
This LRU cache implementation provides significant performance improvements for the parsing processor while maintaining memory efficiency and providing comprehensive monitoring capabilities.
|
||||
|
|
@ -1,165 +0,0 @@
|
|||
# 🎯 Two-Stage Filtering Implementation - Complete
|
||||
|
||||
## 🚀 **Successfully Implemented!**
|
||||
|
||||
We have successfully implemented the sophisticated two-stage filtering architecture that decouples structural discovery from content analysis.
|
||||
|
||||
## 🏗️ **Architecture Overview**
|
||||
|
||||
### **Stage 1: Complete Structural Discovery**
|
||||
- **✅ GitHub Service**: Discovers ALL files and directories (including `node_modules`, `.git`, etc.)
|
||||
- **✅ ZIP Service**: Extracts ALL paths from archives (complete structure)
|
||||
- **✅ StructureProcessor**: Creates nodes for EVERY path discovered
|
||||
- **✅ Result**: Knowledge graph contains complete, accurate repository structure
|
||||
|
||||
### **Stage 2: Intelligent Pruning Before Parsing**
|
||||
- **✅ ParsingProcessor**: Applies sophisticated filtering before content analysis
|
||||
- **✅ Ignore Patterns**: Comprehensive list of directories to skip during parsing
|
||||
- **✅ User Filters**: Directory and extension filters still work as before
|
||||
- **✅ Result**: Only relevant files get their content parsed and analyzed
|
||||
|
||||
## 🎯 **Implementation Details**
|
||||
|
||||
### **Enhanced ParsingProcessor**
|
||||
|
||||
#### **Comprehensive Ignore Patterns**
|
||||
```typescript
|
||||
private static readonly IGNORE_PATTERNS = new Set([
|
||||
// Version Control
|
||||
'.git', '.svn', '.hg',
|
||||
|
||||
// Package Managers & Dependencies
|
||||
'node_modules', 'bower_components', 'vendor', 'deps',
|
||||
|
||||
// Python Virtual Environments & Cache
|
||||
'venv', 'env', '.venv', 'virtualenv', '__pycache__',
|
||||
|
||||
// Build & Distribution
|
||||
'build', 'dist', 'out', 'target', 'bin', 'obj',
|
||||
|
||||
// IDE & Editor Directories
|
||||
'.vs', '.vscode', '.idea', '.eclipse',
|
||||
|
||||
// Temporary & Logs
|
||||
'tmp', 'temp', 'logs', 'log',
|
||||
|
||||
// Coverage & Testing
|
||||
'coverage', '.coverage', 'htmlcov',
|
||||
|
||||
// Cache Directories
|
||||
'.cache', '.next', '.nuxt'
|
||||
]);
|
||||
```
|
||||
|
||||
#### **Two-Stage Filtering Logic**
|
||||
```typescript
|
||||
private applyFiltering(allPaths: string[], fileContents: Map<string, string>, options?: FilterOptions): string[] {
|
||||
// STAGE 1: Prune ignored directories
|
||||
let filesToProcess = this.pruneIgnoredPaths(allPaths.filter(path => fileContents.has(path)));
|
||||
|
||||
// STAGE 2: Apply user filters
|
||||
if (options?.directoryFilter) { /* existing user filter logic */ }
|
||||
if (options?.fileExtensions) { /* existing user filter logic */ }
|
||||
|
||||
return filesToProcess;
|
||||
}
|
||||
```
|
||||
|
||||
#### **Intelligent Pruning**
|
||||
```typescript
|
||||
private pruneIgnoredPaths(filePaths: string[]): string[] {
|
||||
return filePaths.filter(path => {
|
||||
const pathSegments = path.split('/');
|
||||
|
||||
// Check if any segment matches ignore patterns
|
||||
const hasIgnoredSegment = pathSegments.some(segment =>
|
||||
ParsingProcessor.IGNORE_PATTERNS.has(segment.toLowerCase())
|
||||
);
|
||||
|
||||
return !hasIgnoredSegment && !this.matchesIgnorePatterns(path);
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
### **Complete Structure Discovery**
|
||||
|
||||
#### **GitHub Service Enhancement**
|
||||
- **Removed**: `shouldSkipDirectory()` checks in `collectPathsAndContent()`
|
||||
- **Result**: Discovers ALL directories, including `node_modules`, `.git`, etc.
|
||||
|
||||
#### **ZIP Service Enhancement**
|
||||
- **Removed**: `shouldSkipDirectory()` checks in `extractCompleteStructure()`
|
||||
- **Result**: Extracts ALL paths from ZIP archives
|
||||
|
||||
## 📊 **Before vs After**
|
||||
|
||||
| Aspect | ❌ **Before** | ✅ **After** |
|
||||
|--------|---------------|--------------|
|
||||
| **Structure Discovery** | Filtered early, missed directories | Complete discovery of all paths |
|
||||
| **node_modules Visibility** | Missing from KG | Visible as folder node |
|
||||
| **Content Parsing** | Parsed everything discovered | Intelligently skips ignored directories |
|
||||
| **Performance** | Slow (parsed dependencies) | Fast (skips massive directories) |
|
||||
| **KG Accuracy** | Incomplete structure | Perfect mirror of repository |
|
||||
| **User Experience** | Cluttered with dependencies | Clean, focused on source code |
|
||||
|
||||
## 🎯 **Benefits Achieved**
|
||||
|
||||
### **1. Complete Accurate Structure**
|
||||
```
|
||||
✅ Repository Structure in KG:
|
||||
├── src/ (visible, parsed)
|
||||
├── tests/ (visible, parsed)
|
||||
├── node_modules/ (visible, NOT parsed) 🎯
|
||||
├── .git/ (visible, NOT parsed) 🎯
|
||||
├── dist/ (visible, NOT parsed) 🎯
|
||||
└── package.json (visible, parsed)
|
||||
```
|
||||
|
||||
### **2. Performance Improvements**
|
||||
- **⚡ Skip Massive Directories**: No parsing of `node_modules` (thousands of files)
|
||||
- **⚡ Faster Processing**: Focus on actual source code
|
||||
- **⚡ Smaller Graphs**: Fewer definition nodes to render
|
||||
- **⚡ Better Memory Usage**: Avoid loading massive dependency files
|
||||
|
||||
### **3. Professional User Experience**
|
||||
- **📊 Accurate Representation**: Users see complete project structure
|
||||
- **🎯 Clean Analysis**: Focus on relevant code, not dependencies
|
||||
- **🔍 Better Navigation**: Easy to distinguish project code from dependencies
|
||||
- **📈 Trust**: KG accurately mirrors their actual repository
|
||||
|
||||
## 🔍 **Technical Highlights**
|
||||
|
||||
### **Sophisticated Pattern Matching**
|
||||
- **Directory Segments**: Checks each path segment against ignore patterns
|
||||
- **Pattern-Based**: Handles `.egg-info`, `site-packages`, etc.
|
||||
- **Hidden Directories**: Smart handling of `.github` (keep) vs `.vscode` (ignore)
|
||||
|
||||
### **Logging & Visibility**
|
||||
```
|
||||
ParsingProcessor: Starting with 1,247 files with content
|
||||
ParsingProcessor: After pruning ignored directories: 1,247 -> 89 files
|
||||
ParsingProcessor: Directory filter applied: 89 -> 45 files
|
||||
```
|
||||
|
||||
### **Browser Compatibility**
|
||||
- **✅ No Node.js Dependencies**: Pure browser implementation
|
||||
- **✅ Memory Efficient**: Batched processing with size limits
|
||||
- **✅ Performance Optimized**: Skip expensive operations on ignored files
|
||||
|
||||
## 🚀 **Deployment Status**
|
||||
|
||||
- **✅ Build Success**: All TypeScript compilation passes
|
||||
- **✅ Architecture Complete**: Two-stage filtering fully implemented
|
||||
- **✅ Backward Compatible**: Existing functionality preserved
|
||||
- **✅ Production Ready**: Ready for real-world repository analysis
|
||||
|
||||
## 🎉 **Result**
|
||||
|
||||
**Perfect Implementation!** We now have:
|
||||
|
||||
1. **Complete Structure Discovery**: Every directory appears in the KG
|
||||
2. **Intelligent Content Filtering**: Skip parsing massive dependency directories
|
||||
3. **Optimal Performance**: Fast processing focused on relevant code
|
||||
4. **Professional UX**: Clean, accurate knowledge graphs
|
||||
|
||||
The two-stage filtering architecture is **successfully implemented** and ready for production! 🚀
|
||||
|
|
@ -1,524 +0,0 @@
|
|||
# GitNexus Codebase Separation Analysis & Refactoring Plan
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The GitNexus codebase already has a **dual-track architecture** with Legacy (sequential + in-memory) and Next-Gen (parallel + KuzuDB) processing engines. However, there are several areas where separation of concerns can be improved without changing functionality or appearance.
|
||||
|
||||
**Current Status**: ✅ Engine abstractions exist, ❌ Service layer has duplication, ❌ UI has engine-specific logic
|
||||
|
||||
**Goal**: Create clear separation while maintaining exact current functionality and appearance.
|
||||
|
||||
## Current Architecture Analysis
|
||||
|
||||
### 1. Engine Layer (✅ WELL SEPARATED)
|
||||
|
||||
**Location**: `src/core/engines/`
|
||||
|
||||
**Current State**: EXCELLENT separation already exists
|
||||
- ✅ `engine-interface.ts` - Clean abstraction layer
|
||||
- ✅ `legacy/legacy-engine.ts` - Wraps `IngestionService` + `GraphPipeline`
|
||||
- ✅ `nextgen/nextgen-engine.ts` - Wraps `KuzuIngestionService` + `KuzuGraphPipeline`
|
||||
|
||||
**Strengths**:
|
||||
- Common `ProcessingEngine` interface
|
||||
- Proper fallback mechanisms
|
||||
- Performance monitoring for both tracks
|
||||
- Engine validation and health checks
|
||||
|
||||
**Minor Issues**:
|
||||
- Services instantiated in constructors (tight coupling)
|
||||
- No service injection pattern
|
||||
|
||||
### 2. Service Layer (❌ MAJOR DUPLICATION FOUND)
|
||||
|
||||
**Location**: `src/services/`
|
||||
|
||||
**Current State**: SIGNIFICANT code duplication between services
|
||||
|
||||
#### Issues Identified:
|
||||
|
||||
**A) Ingestion Services Duplication (90% identical code)**
|
||||
- `ingestion.service.ts` (195 lines) vs `kuzu-ingestion.service.ts` (198 lines)
|
||||
- Both use identical: GitHub/ZIP services, URL parsing, path normalization
|
||||
- Only difference: Pipeline type (`GraphPipeline` vs `KuzuGraphPipeline`)
|
||||
|
||||
**B) Shared Services with Mixed Concerns**
|
||||
- `github.ts` and `zip.ts` are shared but have no engine-specific optimizations
|
||||
- Both ingestion services use identical logic for repository discovery
|
||||
|
||||
**Code Analysis**:
|
||||
```typescript
|
||||
// ingestion.service.ts
|
||||
class IngestionService {
|
||||
async processGitHubRepo(url, options) {
|
||||
// 1. Parse GitHub URL (identical)
|
||||
// 2. Get repository structure (identical)
|
||||
// 3. Normalize paths (identical)
|
||||
// 4. Use GraphPipeline (different)
|
||||
}
|
||||
|
||||
private normalizeZipPaths() { /* Identical 40 lines */ }
|
||||
}
|
||||
|
||||
// kuzu-ingestion.service.ts
|
||||
class KuzuIngestionService {
|
||||
async processGitHubRepo(url, options) {
|
||||
// 1. Parse GitHub URL (identical)
|
||||
// 2. Get repository structure (identical)
|
||||
// 3. Normalize paths (identical)
|
||||
// 4. Use KuzuGraphPipeline (different)
|
||||
}
|
||||
|
||||
private normalizeZipPaths() { /* Identical 40 lines */ }
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Pipeline Layer (✅ GOOD SEPARATION)
|
||||
|
||||
**Location**: `src/core/ingestion/`
|
||||
|
||||
**Current State**: PROPER separation with shared processors
|
||||
|
||||
**Pipelines**:
|
||||
- ✅ `pipeline.ts` (`GraphPipeline`) - Sequential processing
|
||||
- ✅ `kuzu-pipeline.ts` (`KuzuGraphPipeline`) - Parallel processing
|
||||
- ✅ `parallel-pipeline.ts` (`ParallelGraphPipeline`) - Alternative parallel implementation
|
||||
|
||||
**Processors**: Shared appropriately
|
||||
- ✅ `StructureProcessor`, `ImportProcessor`, `CallProcessor` - Shared (good)
|
||||
- ✅ `ParsingProcessor` vs `ParallelParsingProcessor` - Separate (good)
|
||||
|
||||
**Assessment**: This layer already has excellent separation.
|
||||
|
||||
### 4. Graph Interface Layer (⚠️ PARTIAL UNIFICATION)
|
||||
|
||||
**Location**: `src/core/graph/`
|
||||
|
||||
**Current State**: Two graph types with some unification
|
||||
|
||||
```typescript
|
||||
// SimpleKnowledgeGraph (Legacy)
|
||||
interface KnowledgeGraph {
|
||||
nodes: GraphNode[];
|
||||
relationships: GraphRelationship[];
|
||||
}
|
||||
|
||||
// KuzuKnowledgeGraph (Next-Gen)
|
||||
interface KuzuKnowledgeGraphInterface extends KnowledgeGraph {
|
||||
getNodeCount(): number;
|
||||
getRelationshipCount(): number;
|
||||
query(cypher: string): Promise<QueryResult>;
|
||||
}
|
||||
```
|
||||
|
||||
**Issues**:
|
||||
- UI uses conditional logic: `graph.getNodeCount ? graph.getNodeCount() : graph.nodes.length`
|
||||
- Different capabilities not properly abstracted
|
||||
|
||||
### 5. UI Components (⚠️ SOME ENGINE-SPECIFIC LOGIC)
|
||||
|
||||
**Location**: `src/ui/`
|
||||
|
||||
**Current State**: Generally well structured but has some engine coupling
|
||||
|
||||
**Well Separated**:
|
||||
- ✅ `hooks/useEngine.ts` - Engine management
|
||||
- ✅ `components/engine/` - Engine selection components
|
||||
- ✅ `hooks/useGitNexus.ts` - Main orchestration
|
||||
|
||||
**Issues Found**:
|
||||
- Processing status shows engine-specific details
|
||||
- Export functionality not engine-aware
|
||||
- Some conditional logic based on engine capabilities
|
||||
|
||||
### 6. Configuration System (⚠️ NEEDS ENGINE SECTIONS)
|
||||
|
||||
**Location**: `src/config/`
|
||||
|
||||
**Current State**: Basic configuration without engine-specific sections
|
||||
|
||||
**Existing**:
|
||||
- ✅ `config.ts` - Core configuration
|
||||
- ✅ `feature-flags.ts` - Feature toggles
|
||||
- ⚠️ Missing engine-specific configurations
|
||||
|
||||
## Implementation Progress
|
||||
|
||||
### ✅ Phase 1 Completed: Service Layer Refactoring
|
||||
|
||||
**Status**: COMPLETED - All service layer refactoring implemented successfully
|
||||
|
||||
#### ✅ Step 1.1: Base Ingestion Service Created
|
||||
- ✅ `src/services/common/base-ingestion.service.ts` - Abstract base class with 90% shared logic
|
||||
- ✅ Extracted: GitHub URL parsing, repository discovery, ZIP normalization, progress reporting
|
||||
- ✅ Template method pattern implemented for pipeline processing
|
||||
|
||||
#### ✅ Step 1.2: Service Factory Implementation
|
||||
- ✅ `src/services/service.factory.ts` - Centralized service creation
|
||||
- ✅ Dynamic imports to avoid circular dependencies
|
||||
- ✅ Engine validation and fallback support
|
||||
- ✅ Clean abstraction for service instantiation
|
||||
|
||||
#### ✅ Step 1.3: Engine-Specific Service Implementations
|
||||
- ✅ `src/services/legacy/legacy-ingestion.service.ts` - Legacy engine implementation
|
||||
- ✅ `src/services/nextgen/nextgen-ingestion.service.ts` - Next-Gen engine implementation
|
||||
- ✅ Both extend BaseIngestionService with engine-specific pipeline logic
|
||||
|
||||
#### ✅ Step 1.4: Backward Compatibility Maintained
|
||||
- ✅ `src/services/ingestion.service.ts` - Updated to use service factory internally
|
||||
- ✅ `src/services/kuzu-ingestion.service.ts` - Updated to use service factory internally
|
||||
- ✅ Existing API completely preserved for zero breaking changes
|
||||
|
||||
#### ✅ Step 1.5: Engine Wrappers Updated
|
||||
- ✅ `src/core/engines/legacy/legacy-engine.ts` - Now uses ServiceFactory
|
||||
- ✅ `src/core/engines/nextgen/nextgen-engine.ts` - Now uses ServiceFactory
|
||||
- ✅ Proper dependency injection pattern implemented
|
||||
|
||||
**Results Achieved**:
|
||||
- ✅ 90% code duplication eliminated between ingestion services
|
||||
- ✅ Clear separation between Legacy and Next-Gen implementations
|
||||
- ✅ Zero breaking changes - existing code works unchanged
|
||||
- ✅ Proper abstraction layers with dependency injection
|
||||
- ✅ No compilation errors
|
||||
|
||||
### ✅ Phase 2 Completed: Configuration Enhancement
|
||||
|
||||
**Status**: COMPLETED - Engine-specific configuration system implemented
|
||||
|
||||
#### ✅ Step 2.1: Engine Configuration Schema Added
|
||||
- ✅ `LegacyEngineConfigSchema` - Memory limits, processing settings, worker configuration
|
||||
- ✅ `NextGenEngineConfigSchema` - KuzuDB settings, parallel processing, worker pool configuration
|
||||
- ✅ `EngineConfigSchema` - Runtime settings, fallback configuration, performance monitoring
|
||||
- ✅ Proper Zod validation with sensible defaults
|
||||
|
||||
#### ✅ Step 2.2: Environment Variable Support
|
||||
- ✅ `ENGINE_DEFAULT` - Set default engine (legacy/nextgen)
|
||||
- ✅ `ENGINE_ALLOW_FALLBACK` - Enable/disable engine fallback
|
||||
- ✅ `ENGINE_LEGACY_*` - Legacy engine specific settings
|
||||
- ✅ `ENGINE_NEXTGEN_*` - Next-Gen engine specific settings
|
||||
- ✅ Hardware-aware defaults (worker count based on CPU cores)
|
||||
|
||||
#### ✅ Step 2.3: Configuration Integration
|
||||
- ✅ Updated `ConfigService` to load engine configuration
|
||||
- ✅ Enhanced validation with engine-specific checks
|
||||
- ✅ Proper configuration fallbacks and error handling
|
||||
|
||||
#### ✅ Step 2.4: Engine Wrapper Integration
|
||||
- ✅ Legacy engine validates configuration and logs settings
|
||||
- ✅ Next-Gen engine validates configuration and logs settings
|
||||
- ✅ Engines respect enabled/disabled state from configuration
|
||||
|
||||
#### ✅ Step 2.5: Configuration Helper Created
|
||||
- ✅ `src/config/engine-config.helper.ts` - UI-friendly configuration access
|
||||
- ✅ Engine availability checking, fallback determination
|
||||
- ✅ Display information for UI components
|
||||
- ✅ Processing options and validation utilities
|
||||
|
||||
**Results Achieved**:
|
||||
- ✅ Complete engine-specific configuration system
|
||||
- ✅ Environment variable support for all settings
|
||||
- ✅ Runtime engine switching capabilities
|
||||
- ✅ Hardware-aware configuration defaults
|
||||
- ✅ Comprehensive validation and error handling
|
||||
|
||||
### ✅ Phase 3 Completed: UI Component Enhancement
|
||||
|
||||
**Status**: COMPLETED - UI components enhanced with configuration awareness
|
||||
|
||||
#### ✅ Step 3.1: Engine Selector Enhancement
|
||||
- ✅ Enhanced `EngineSelector.tsx` to use `EngineConfigHelper`
|
||||
- ✅ Shows engine status (enabled/disabled) from configuration
|
||||
- ✅ Displays engine features and descriptions from config
|
||||
- ✅ Shows fallback status and configuration warnings
|
||||
- ✅ Better visual indicators for engine states
|
||||
|
||||
#### ✅ Step 3.2: Processing Status Enhancement
|
||||
- ✅ Enhanced `ProcessingStatus.tsx` with configuration awareness
|
||||
- ✅ Shows engine-specific processing options during execution
|
||||
- ✅ Displays configuration-based engine descriptions
|
||||
- ✅ Performance indicators based on engine type
|
||||
- ✅ More detailed engine information in success state
|
||||
|
||||
#### ✅ Step 3.3: Configuration Documentation
|
||||
- ✅ Created `.env.example` with comprehensive engine configuration
|
||||
- ✅ Documented all engine-specific environment variables
|
||||
- ✅ Provided example configurations for different use cases
|
||||
- ✅ Clear separation between Legacy and Next-Gen settings
|
||||
|
||||
**Results Achieved**:
|
||||
- ✅ UI components now configuration-aware
|
||||
- ✅ Engine status properly reflected in interface
|
||||
- ✅ User-friendly display of engine capabilities
|
||||
- ✅ Clear documentation for customization
|
||||
|
||||
### ✅ REFACTORING COMPLETED SUCCESSFULLY!
|
||||
|
||||
**Final Status**: ALL PHASES COMPLETED - Clear separation of concerns achieved
|
||||
|
||||
#### 🎆 Summary of Achievements:
|
||||
|
||||
**1. Service Layer Refactoring** ✅
|
||||
- 90% code duplication eliminated
|
||||
- Clear Legacy/Next-Gen separation
|
||||
- Service factory pattern implemented
|
||||
- Backward compatibility maintained
|
||||
|
||||
**2. Configuration System** ✅
|
||||
- Engine-specific configuration schemas
|
||||
- Environment variable support
|
||||
- Runtime configuration validation
|
||||
- UI-friendly configuration helpers
|
||||
|
||||
**3. UI Component Enhancement** ✅
|
||||
- Configuration-aware components
|
||||
- Engine status visibility
|
||||
- User-friendly engine information
|
||||
- Comprehensive documentation
|
||||
|
||||
#### 📊 Benefits Realized:
|
||||
|
||||
**Maintainability**:
|
||||
- Clear separation between Legacy and Next-Gen code
|
||||
- No code duplication in service layer
|
||||
- Proper abstraction layers with dependency injection
|
||||
|
||||
**Configurability**:
|
||||
- Easy engine switching via configuration
|
||||
- Hardware-aware defaults
|
||||
- Environment-specific settings
|
||||
- Runtime configuration validation
|
||||
|
||||
**User Experience**:
|
||||
- Transparent engine operation
|
||||
- Clear engine status indicators
|
||||
- Comprehensive configuration options
|
||||
- Excellent fallback mechanisms
|
||||
|
||||
**Developer Experience**:
|
||||
- Well-documented configuration
|
||||
- Clear architectural boundaries
|
||||
- Easy to extend and maintain
|
||||
- Comprehensive error handling
|
||||
|
||||
### 🔄 Current Status: Ready for Production
|
||||
|
||||
**✅ All Requirements Met**:
|
||||
- ✅ Exact same functionality and appearance
|
||||
- ✅ Clear separation of concerns
|
||||
- ✅ Maintainable code structure
|
||||
- ✅ Easy engine switching via configuration
|
||||
- ✅ Zero breaking changes
|
||||
|
||||
**Next Steps for User**:
|
||||
1. Copy `.env.example` to `.env` and customize as needed
|
||||
2. Test both engines work correctly
|
||||
3. Configure default engine preference
|
||||
4. Optionally enable performance monitoring
|
||||
|
||||
---
|
||||
|
||||
## Detailed Refactoring Plan
|
||||
|
||||
### Phase 1: Service Layer Refactoring (HIGH PRIORITY)
|
||||
|
||||
#### Step 1.1: Create Base Ingestion Service
|
||||
|
||||
**Goal**: Extract 90% shared logic from both ingestion services
|
||||
|
||||
**New File**: `src/services/common/base-ingestion.service.ts`
|
||||
|
||||
```typescript
|
||||
// Abstract base class with shared logic
|
||||
abstract class BaseIngestionService {
|
||||
protected githubService: GitHubService;
|
||||
protected zipService: ZipService;
|
||||
|
||||
// Shared methods:
|
||||
// - GitHub URL parsing
|
||||
// - Repository structure discovery
|
||||
// - ZIP path normalization
|
||||
// - Progress reporting
|
||||
|
||||
// Abstract method for pipeline creation
|
||||
protected abstract createPipeline(): Pipeline;
|
||||
}
|
||||
```
|
||||
|
||||
**Files to Update**:
|
||||
- `src/services/ingestion.service.ts` → Extend base class
|
||||
- `src/services/kuzu-ingestion.service.ts` → Extend base class
|
||||
|
||||
#### Step 1.2: Service Factory Pattern
|
||||
|
||||
**New File**: `src/services/service.factory.ts`
|
||||
|
||||
```typescript
|
||||
class ServiceFactory {
|
||||
static createIngestionService(engine: ProcessingEngineType, token?: string): BaseIngestionService {
|
||||
switch (engine) {
|
||||
case 'legacy': return new LegacyIngestionService(token);
|
||||
case 'nextgen': return new NextGenIngestionService(token);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Step 1.3: Update Engine Wrappers
|
||||
|
||||
**Files to Update**:
|
||||
- `src/core/engines/legacy/legacy-engine.ts`
|
||||
- `src/core/engines/nextgen/nextgen-engine.ts`
|
||||
|
||||
**Change**: Use ServiceFactory instead of direct instantiation
|
||||
|
||||
### Phase 2: Configuration Enhancement
|
||||
|
||||
#### Step 2.1: Engine-Specific Configuration
|
||||
|
||||
**File to Update**: `src/config/config.ts`
|
||||
|
||||
**Add Engine Configuration Schema**:
|
||||
```typescript
|
||||
interface EngineConfig {
|
||||
legacy: {
|
||||
enabled: boolean;
|
||||
memoryLimits: { maxMemoryMB: number; gcIntervalMs: number };
|
||||
processing: { batchSize: number; timeoutMs: number };
|
||||
};
|
||||
nextgen: {
|
||||
enabled: boolean;
|
||||
kuzu: { databasePath: string; bufferPoolSize: number };
|
||||
parallel: { maxWorkers: number; batchSize: number };
|
||||
};
|
||||
runtime: {
|
||||
defaultEngine: 'legacy' | 'nextgen';
|
||||
allowFallback: boolean;
|
||||
performanceMonitoring: boolean;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### Phase 3: UI Component Updates (LOW PRIORITY)
|
||||
|
||||
#### Step 3.1: Engine-Aware Components
|
||||
|
||||
**Files to Update**:
|
||||
- `src/ui/components/graph/GraphExplorer.tsx` - Remove conditional logic
|
||||
- `src/ui/components/ExportFormatModal.tsx` - Make engine-aware
|
||||
- `src/ui/pages/HomePage/HomePage.tsx` - Clean up engine-specific styling
|
||||
|
||||
#### Step 3.2: Enhanced Status Components
|
||||
|
||||
**File to Update**: `src/ui/components/engine/ProcessingStatus.tsx`
|
||||
|
||||
**Enhancement**: Show unified status regardless of engine
|
||||
|
||||
### Phase 4: Graph Interface Unification (MEDIUM PRIORITY)
|
||||
|
||||
#### Step 4.1: Unified Graph Interface
|
||||
|
||||
**File to Update**: `src/core/graph/types.ts`
|
||||
|
||||
**Goal**: Create unified interface that works with both graph types
|
||||
|
||||
```typescript
|
||||
interface UnifiedKnowledgeGraph {
|
||||
// Common interface
|
||||
nodes: GraphNode[];
|
||||
relationships: GraphRelationship[];
|
||||
|
||||
// Unified methods
|
||||
getEngineType(): 'legacy' | 'nextgen';
|
||||
getCapabilities(): string[];
|
||||
|
||||
// Optional advanced methods
|
||||
query?(cypher: string): Promise<QueryResult>;
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Strategy
|
||||
|
||||
### Week 1: Service Layer Refactoring
|
||||
1. Create `BaseIngestionService` abstract class
|
||||
2. Extract shared logic (URL parsing, normalization, progress reporting)
|
||||
3. Update existing services to extend base class
|
||||
4. Create `ServiceFactory` for centralized service creation
|
||||
5. Update engine wrappers to use factory
|
||||
|
||||
### Week 2: Configuration Enhancement
|
||||
1. Add engine-specific configuration schema
|
||||
2. Implement environment variable support
|
||||
3. Add runtime engine selection configuration
|
||||
4. Update configuration service
|
||||
|
||||
### Week 3: UI Component Cleanup
|
||||
1. Remove engine-specific conditional logic from components
|
||||
2. Enhance status and export components
|
||||
3. Clean up engine-specific styling
|
||||
|
||||
### Week 4: Testing & Validation
|
||||
1. Ensure exact same functionality and appearance
|
||||
2. Test engine switching
|
||||
3. Verify performance characteristics remain unchanged
|
||||
4. Test fallback mechanisms
|
||||
|
||||
## File Impact Analysis
|
||||
|
||||
### Files to Create:
|
||||
- `src/services/common/base-ingestion.service.ts`
|
||||
- `src/services/service.factory.ts`
|
||||
|
||||
### Files to Modify (Major Changes):
|
||||
- `src/services/ingestion.service.ts`
|
||||
- `src/services/kuzu-ingestion.service.ts`
|
||||
- `src/core/engines/legacy/legacy-engine.ts`
|
||||
- `src/core/engines/nextgen/nextgen-engine.ts`
|
||||
- `src/config/config.ts`
|
||||
|
||||
### Files to Modify (Minor Changes):
|
||||
- `src/ui/components/graph/GraphExplorer.tsx`
|
||||
- `src/ui/components/ExportFormatModal.tsx`
|
||||
- `src/ui/pages/HomePage/HomePage.tsx`
|
||||
- `src/core/graph/types.ts`
|
||||
|
||||
## Risk Assessment
|
||||
|
||||
### Low Risk ✅
|
||||
- Service layer refactoring (extracting shared logic)
|
||||
- Configuration enhancements
|
||||
- Factory pattern implementation
|
||||
|
||||
### Medium Risk ⚠️
|
||||
- Graph interface unification
|
||||
- UI component updates
|
||||
|
||||
### Zero Risk ✅
|
||||
- Pipeline layer (already well separated)
|
||||
- Engine wrapper layer (already excellent)
|
||||
|
||||
## Success Criteria
|
||||
|
||||
### Functional Requirements ✅
|
||||
- [ ] Exact same UI appearance and behavior
|
||||
- [ ] Both engines work identically to current implementation
|
||||
- [ ] Engine switching works flawlessly
|
||||
- [ ] No performance degradation
|
||||
- [ ] All existing features work unchanged
|
||||
|
||||
### Code Quality Requirements ✅
|
||||
- [ ] 90% reduction in service layer duplication
|
||||
- [ ] Clear separation between Legacy and Next-Gen implementations
|
||||
- [ ] Maintainable code structure
|
||||
- [ ] Proper abstraction layers
|
||||
- [ ] Comprehensive configuration system
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. **Validate Current Functionality**: Test both engines work correctly
|
||||
2. **Start with Service Layer**: Begin Phase 1 refactoring
|
||||
3. **Incremental Testing**: Test after each major change
|
||||
4. **Maintain Backward Compatibility**: Ensure no breaking changes
|
||||
|
||||
This refactoring will create clear separation of concerns while maintaining the exact same functionality and appearance.
|
||||
|
|
@ -1,204 +0,0 @@
|
|||
# GitNexus Refactoring Completion Summary
|
||||
|
||||
## 🎉 Refactoring Successfully Completed!
|
||||
|
||||
Your GitNexus codebase has been successfully refactored for **clear separation of concerns** while maintaining **100% backward compatibility**. The application works exactly the same as before, but now has a much cleaner and more maintainable architecture.
|
||||
|
||||
## ✅ What Was Accomplished
|
||||
|
||||
### 1. Service Layer Refactoring (90% Code Duplication Eliminated)
|
||||
|
||||
**Before**: Two nearly identical ingestion services with 90% duplicated code
|
||||
**After**: Clean inheritance hierarchy with shared base class
|
||||
|
||||
```
|
||||
New Architecture:
|
||||
src/services/
|
||||
├── common/base-ingestion.service.ts # Shared logic (90% of code)
|
||||
├── legacy/legacy-ingestion.service.ts # Legacy-specific (10% of code)
|
||||
├── nextgen/nextgen-ingestion.service.ts # Next-Gen-specific (10% of code)
|
||||
├── service.factory.ts # Centralized service creation
|
||||
├── ingestion.service.ts # Backward compatibility wrapper
|
||||
└── kuzu-ingestion.service.ts # Backward compatibility wrapper
|
||||
```
|
||||
|
||||
**Benefits**:
|
||||
- 🔧 Eliminated 160+ lines of duplicated code
|
||||
- 🎯 Clear separation between Legacy and Next-Gen implementations
|
||||
- 🔄 Zero breaking changes - existing code works unchanged
|
||||
- 🏗️ Proper dependency injection with service factory pattern
|
||||
|
||||
### 2. Engine-Specific Configuration System
|
||||
|
||||
**Before**: Basic configuration without engine-specific settings
|
||||
**After**: Comprehensive configuration system with engine separation
|
||||
|
||||
```
|
||||
New Configuration:
|
||||
src/config/
|
||||
├── config.ts # Enhanced with engine schemas
|
||||
├── engine-config.helper.ts # UI-friendly configuration access
|
||||
└── .env.example # Complete configuration examples
|
||||
```
|
||||
|
||||
**Features**:
|
||||
- ⚙️ Separate Legacy and Next-Gen engine configuration
|
||||
- 🌍 Environment variable support for all settings
|
||||
- 🔧 Hardware-aware defaults (CPU cores detection)
|
||||
- ✅ Runtime configuration validation
|
||||
- 🎛️ Easy engine switching via configuration
|
||||
|
||||
### 3. Enhanced UI Components
|
||||
|
||||
**Before**: Basic engine selection with minimal configuration awareness
|
||||
**After**: Configuration-aware components with detailed engine information
|
||||
|
||||
**Enhancements**:
|
||||
- 🎨 Engine status indicators (enabled/disabled)
|
||||
- 📊 Configuration-based engine descriptions
|
||||
- 🔄 Fallback status display
|
||||
- ⚡ Processing options shown during execution
|
||||
- 🏆 Performance indicators based on engine type
|
||||
|
||||
## 🔧 How to Use the New Configuration
|
||||
|
||||
### 1. Basic Setup
|
||||
|
||||
Copy the example configuration:
|
||||
```bash
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
### 2. Engine Selection
|
||||
|
||||
Choose your default engine:
|
||||
```bash
|
||||
# For stable, sequential processing
|
||||
ENGINE_DEFAULT=legacy
|
||||
|
||||
# For high-performance, parallel processing
|
||||
ENGINE_DEFAULT=nextgen
|
||||
```
|
||||
|
||||
### 3. Engine-Specific Tuning
|
||||
|
||||
**Legacy Engine (Sequential + In-Memory)**:
|
||||
```bash
|
||||
ENGINE_LEGACY_MEMORY_LIMIT_MB=512
|
||||
ENGINE_LEGACY_BATCH_SIZE=10
|
||||
ENGINE_LEGACY_USE_WORKERS=true
|
||||
```
|
||||
|
||||
**Next-Gen Engine (Parallel + KuzuDB)**:
|
||||
```bash
|
||||
ENGINE_NEXTGEN_MAX_WORKERS=4
|
||||
ENGINE_NEXTGEN_BATCH_SIZE=20
|
||||
ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=256
|
||||
```
|
||||
|
||||
### 4. Safety Features
|
||||
|
||||
Enable fallback for maximum reliability:
|
||||
```bash
|
||||
ENGINE_ALLOW_FALLBACK=true
|
||||
ENGINE_PERFORMANCE_MONITORING=true
|
||||
```
|
||||
|
||||
## 📁 New File Structure
|
||||
|
||||
```
|
||||
src/
|
||||
├── services/
|
||||
│ ├── common/
|
||||
│ │ └── base-ingestion.service.ts # ✨ NEW: Shared logic
|
||||
│ ├── legacy/
|
||||
│ │ └── legacy-ingestion.service.ts # ✨ NEW: Legacy implementation
|
||||
│ ├── nextgen/
|
||||
│ │ └── nextgen-ingestion.service.ts # ✨ NEW: Next-Gen implementation
|
||||
│ └── service.factory.ts # ✨ NEW: Service factory
|
||||
├── config/
|
||||
│ ├── config.ts # 🔧 ENHANCED: Engine configuration
|
||||
│ └── engine-config.helper.ts # ✨ NEW: Configuration helper
|
||||
├── core/engines/
|
||||
│ ├── legacy/legacy-engine.ts # 🔧 ENHANCED: Uses service factory
|
||||
│ └── nextgen/nextgen-engine.ts # 🔧 ENHANCED: Uses service factory
|
||||
└── ui/components/engine/
|
||||
├── EngineSelector.tsx # 🔧 ENHANCED: Configuration-aware
|
||||
└── ProcessingStatus.tsx # 🔧 ENHANCED: Shows engine details
|
||||
```
|
||||
|
||||
## 🚀 Benefits Achieved
|
||||
|
||||
### For Developers
|
||||
- **Maintainability**: Clear separation makes code easier to understand and modify
|
||||
- **Testability**: Each engine can be tested independently
|
||||
- **Extensibility**: Easy to add new engines or modify existing ones
|
||||
- **Debugging**: Clear boundaries help isolate issues
|
||||
|
||||
### For Users
|
||||
- **Reliability**: Fallback mechanisms ensure processing always works
|
||||
- **Performance**: Choose the right engine for your use case
|
||||
- **Transparency**: Clear visibility into which engine is being used
|
||||
- **Customization**: Fine-tune processing for your specific needs
|
||||
|
||||
### For Operations
|
||||
- **Configuration**: Comprehensive environment variable support
|
||||
- **Monitoring**: Built-in performance monitoring
|
||||
- **Flexibility**: Runtime engine switching without code changes
|
||||
- **Documentation**: Clear examples and configuration guidance
|
||||
|
||||
## 🛡️ Backward Compatibility Guarantee
|
||||
|
||||
**Zero Breaking Changes**: All existing code continues to work exactly as before:
|
||||
|
||||
- ✅ `IngestionService` still works the same way
|
||||
- ✅ `KuzuIngestionService` still works the same way
|
||||
- ✅ Engine wrappers maintain the same interface
|
||||
- ✅ UI components look and behave identically
|
||||
- ✅ All existing functionality preserved
|
||||
|
||||
## 🔄 Migration Path
|
||||
|
||||
**Immediate** (Already Done):
|
||||
- ✅ Service factory architecture implemented
|
||||
- ✅ Configuration system enhanced
|
||||
- ✅ UI components improved
|
||||
- ✅ Documentation created
|
||||
|
||||
**Optional Next Steps**:
|
||||
1. **Customize Configuration**: Edit `.env` file for your preferences
|
||||
2. **Test Both Engines**: Verify both Legacy and Next-Gen work for your use cases
|
||||
3. **Set Default Engine**: Choose your preferred engine
|
||||
4. **Enable Monitoring**: Turn on performance monitoring if desired
|
||||
|
||||
## 📊 Current Status
|
||||
|
||||
- 🟢 **Application Status**: Running perfectly
|
||||
- 🟢 **Legacy Engine**: Fully functional with new architecture
|
||||
- 🟢 **Next-Gen Engine**: Fully functional with new architecture
|
||||
- 🟢 **Configuration**: Complete and tested
|
||||
- 🟢 **UI Components**: Enhanced and working
|
||||
- 🟢 **Documentation**: Comprehensive and complete
|
||||
|
||||
## 🎯 What's Next
|
||||
|
||||
The refactoring is **complete and ready for production use**. You can now:
|
||||
|
||||
1. **Continue Development**: Focus on your parallel + KuzuDB work with clear separation
|
||||
2. **Easy Testing**: Switch between engines via configuration to test new features
|
||||
3. **Gradual Migration**: Keep Legacy as fallback while perfecting Next-Gen
|
||||
4. **Team Collaboration**: Clear boundaries make team development easier
|
||||
|
||||
## 🔍 Quick Test
|
||||
|
||||
To verify everything works:
|
||||
|
||||
1. Application should be running at `http://localhost:5173`
|
||||
2. UI should look identical to before
|
||||
3. Both engines should be available in the engine selector
|
||||
4. Processing should work exactly as before
|
||||
5. Configuration changes should take effect after restart
|
||||
|
||||
---
|
||||
|
||||
**🎉 Congratulations!** Your codebase now has crystal-clear separation of concerns while maintaining full backward compatibility. The foundation is set for easy maintenance and continued development of your parallel + KuzuDB features!
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
# 🔧 GitNexus Structure Discovery Fix - Complete Architecture Overhaul
|
||||
|
||||
## 🚨 **Critical Flaw Identified and Fixed**
|
||||
|
||||
### **The Problem**
|
||||
The original GitNexus architecture had a **fatal flaw** in repository structure discovery:
|
||||
|
||||
- **Flawed Logic**: `StructureProcessor` inferred directory existence from filtered file paths
|
||||
- **Critical Bug**: Empty directories or directories containing only filtered-out files were **completely missing** from the knowledge graph
|
||||
- **Result**: Incomplete and inaccurate codebase representation
|
||||
|
||||
### **Root Cause Analysis**
|
||||
```
|
||||
❌ OLD BROKEN FLOW:
|
||||
GitHub/ZIP Service → Filter Files → Pass Filtered Paths → Infer Structure
|
||||
↑ FILTERING HERE BREAKS STRUCTURE DISCOVERY
|
||||
```
|
||||
|
||||
**The Fundamental Issue**: Filtering happened **before** structure discovery, causing the `StructureProcessor` to never see paths for directories that contained only filtered-out files.
|
||||
|
||||
## 🏗️ **The New Robust Architecture**
|
||||
|
||||
### **Core Principle**
|
||||
> **Discover Complete Structure First, Filter During Parsing**
|
||||
|
||||
```
|
||||
✅ NEW ROBUST FLOW:
|
||||
GitHub/ZIP Service → Discover ALL Paths → Build Complete Structure → Filter During Parsing
|
||||
↑ NO FILTERING YET ↑ COMPLETE STRUCTURE ↑ FILTERING HERE
|
||||
```
|
||||
|
||||
### **Architectural Changes**
|
||||
|
||||
## **1. Data Acquisition Services (Complete Structure Discovery)**
|
||||
|
||||
### **GitHub Service (`src/services/github.ts`)**
|
||||
- ✅ **New Method**: `getCompleteRepositoryStructure()`
|
||||
- ✅ **Returns**: `CompleteRepositoryStructure` with `allPaths` + `fileContents`
|
||||
- ✅ **Behavior**: Discovers **every file and directory** in the repository
|
||||
- ✅ **No Filtering**: Collects all content regardless of user filters
|
||||
|
||||
### **ZIP Service (`src/services/zip.ts`)**
|
||||
- ✅ **New Method**: `extractCompleteStructure()`
|
||||
- ✅ **Returns**: `CompleteZipStructure` with `allPaths` + `fileContents`
|
||||
- ✅ **Enhanced Logic**: Explicitly tracks directories and intermediate paths
|
||||
- ✅ **Path Normalization**: Handles common top-level folder removal
|
||||
|
||||
## **2. Ingestion Service (Pipeline Orchestration)**
|
||||
|
||||
### **Updated Methods (`src/services/ingestion.service.ts`)**
|
||||
- ✅ **`processGitHubRepo()`**: Uses complete structure discovery
|
||||
- ✅ **`processZipFile()`**: Uses complete structure discovery
|
||||
- ✅ **No Filtering**: Passes **all discovered paths** to pipeline
|
||||
- ✅ **Clean Architecture**: Filtering responsibility moved to `ParsingProcessor`
|
||||
|
||||
## **3. Structure Processor (Direct Path Processing)**
|
||||
|
||||
### **Complete Rewrite (`src/core/ingestion/structure-processor.ts`)**
|
||||
```typescript
|
||||
// OLD: Infer structure from filtered file paths
|
||||
const folderPaths = this.extractFolderPaths(filePaths); // ❌ BROKEN
|
||||
|
||||
// NEW: Process complete discovered structure directly
|
||||
const { directories, files } = this.categorizePaths(filePaths); // ✅ ROBUST
|
||||
```
|
||||
|
||||
#### **Key Improvements**:
|
||||
- ✅ **Direct Processing**: No inference, direct path categorization
|
||||
- ✅ **Complete Structure**: Processes **all** discovered paths
|
||||
- ✅ **Smart Categorization**: Distinguishes files from directories algorithmically
|
||||
- ✅ **Intermediate Directories**: Automatically adds missing intermediate paths
|
||||
- ✅ **Perfect Mirror**: KG structure exactly matches repository file system
|
||||
|
||||
## **4. Parsing Processor (Filtering During Parsing)**
|
||||
|
||||
### **New Filtering Logic (`src/core/ingestion/parsing-processor.ts`)**
|
||||
```typescript
|
||||
// NEW: Filtering happens here, during parsing
|
||||
private applyFiltering(
|
||||
allPaths: string[],
|
||||
fileContents: Map<string, string>,
|
||||
options?: { directoryFilter?: string; fileExtensions?: string }
|
||||
): string[]
|
||||
```
|
||||
|
||||
#### **Filtering Strategy**:
|
||||
- ✅ **Input**: Receives **all** paths from structure discovery
|
||||
- ✅ **Apply Filters**: Directory and extension filters applied here
|
||||
- ✅ **Parse Only Filtered**: Only processes files that pass filters
|
||||
- ✅ **Structure Intact**: All directories remain in graph, regardless of filtering
|
||||
|
||||
## **5. Pipeline Integration**
|
||||
|
||||
### **Updated Pipeline (`src/core/ingestion/pipeline.ts`)**
|
||||
- ✅ **4-Pass Architecture**: Maintains existing pass structure
|
||||
- ✅ **Options Passing**: Filtering options passed to `ParsingProcessor`
|
||||
- ✅ **Complete Structure**: `StructureProcessor` gets all paths
|
||||
- ✅ **Filtered Parsing**: `ParsingProcessor` applies user filters
|
||||
|
||||
## 📊 **Before vs After Comparison**
|
||||
|
||||
| Aspect | ❌ **Before (Broken)** | ✅ **After (Robust)** |
|
||||
|--------|------------------------|----------------------|
|
||||
| **Structure Discovery** | Inferred from filtered files | Direct discovery of all paths |
|
||||
| **Empty Directories** | Missing from KG | Present in KG |
|
||||
| **Filtered Directories** | Missing if all files filtered | Present in KG |
|
||||
| **Filtering Location** | Before structure discovery | During parsing phase |
|
||||
| **KG Completeness** | Incomplete, inaccurate | Complete, accurate mirror |
|
||||
| **Architecture** | Monolithic, coupled | Decoupled, robust |
|
||||
|
||||
## 🎯 **Results and Benefits**
|
||||
|
||||
### **Immediate Fixes**
|
||||
1. **✅ Empty Directories**: Now appear in knowledge graph
|
||||
2. **✅ Filtered Directories**: Directories with only filtered files now appear
|
||||
3. **✅ Complete Structure**: KG is a perfect mirror of repository structure
|
||||
4. **✅ Accurate Representation**: No missing parts of codebase
|
||||
|
||||
### **Architectural Improvements**
|
||||
1. **🔧 Separation of Concerns**: Structure discovery ≠ Content filtering
|
||||
2. **🔧 Robust Design**: No inference, direct discovery
|
||||
3. **🔧 Maintainable**: Clear responsibility boundaries
|
||||
4. **🔧 Extensible**: Easy to add new file types or filtering logic
|
||||
|
||||
### **User Experience**
|
||||
1. **📈 Accurate Graphs**: Users see complete repository structure
|
||||
2. **📈 Better Navigation**: All directories visible for exploration
|
||||
3. **📈 Consistent Results**: Same structure regardless of filter settings
|
||||
4. **📈 Trust**: KG accurately represents their codebase
|
||||
|
||||
## 🔍 **Technical Implementation Details**
|
||||
|
||||
### **Path Categorization Algorithm**
|
||||
```typescript
|
||||
// Smart algorithm to distinguish files from directories
|
||||
const isDirectory = allPaths.some(otherPath =>
|
||||
otherPath !== path && otherPath.startsWith(path + '/')
|
||||
);
|
||||
```
|
||||
|
||||
### **Intermediate Directory Discovery**
|
||||
```typescript
|
||||
// Automatically discover missing intermediate directories
|
||||
for (let i = 1; i < parts.length; i++) {
|
||||
const intermediatePath = parts.slice(0, i).join('/');
|
||||
if (intermediatePath && !pathSet.has(intermediatePath)) {
|
||||
allIntermediateDirs.add(intermediatePath);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### **Filtering During Parsing**
|
||||
```typescript
|
||||
// Apply user filters only during parsing phase
|
||||
if (options.directoryFilter?.trim()) {
|
||||
filesToProcess = filesToProcess.filter(path =>
|
||||
dirPatterns.some(pattern => path.toLowerCase().includes(pattern))
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
## 🚀 **Deployment Status**
|
||||
|
||||
- ✅ **Build Status**: All components compile successfully
|
||||
- ✅ **Integration**: Complete end-to-end pipeline updated
|
||||
- ✅ **Testing Ready**: Architecture ready for validation
|
||||
- ✅ **Backward Compatible**: Existing functionality preserved
|
||||
- ✅ **Performance**: Optimized with batching and memory management
|
||||
|
||||
## 🎉 **Conclusion**
|
||||
|
||||
This architectural overhaul transforms GitNexus from a **flawed, incomplete** structure discovery system into a **robust, accurate** repository analysis tool.
|
||||
|
||||
**The critical flaw is now fixed**: GitNexus will discover and represent the **complete** repository structure, ensuring users get an accurate and comprehensive knowledge graph of their codebase.
|
||||
|
||||
**Key Success Metric**: The knowledge graph structure is now a **perfect mirror** of the actual repository file system, regardless of user filtering preferences.
|
||||
|
|
@ -1,448 +0,0 @@
|
|||
# Worker Pool Implementation Guide
|
||||
|
||||
## 🎯 Overview
|
||||
|
||||
The Worker Pool implementation provides **massive performance improvements** for large codebases by parallelizing CPU-intensive operations like Tree-sitter parsing, AST analysis, and code processing.
|
||||
|
||||
## 🚀 Performance Benefits
|
||||
|
||||
### **Expected Speedup:**
|
||||
- **Small codebases (< 100 files)**: 1.5-2x speedup
|
||||
- **Medium codebases (100-1000 files)**: 2-4x speedup
|
||||
- **Large codebases (1000+ files)**: 4-8x speedup
|
||||
|
||||
### **Key Improvements:**
|
||||
- **Parallel file parsing** - Multiple files processed simultaneously
|
||||
- **Concurrent Tree-sitter operations** - AST generation in parallel
|
||||
- **Better CPU utilization** - Leverages all available cores
|
||||
- **Improved UI responsiveness** - Main thread freed up
|
||||
|
||||
---
|
||||
|
||||
## 📁 File Structure
|
||||
|
||||
```
|
||||
src/
|
||||
├── lib/
|
||||
│ ├── web-worker-pool.ts # Main worker pool implementation
|
||||
│ └── worker-pool-test.ts # Test suite
|
||||
├── core/ingestion/
|
||||
│ ├── parallel-parsing-processor.ts # Parallel parsing with workers
|
||||
│ └── parallel-pipeline.ts # Parallel pipeline integration
|
||||
└── config/
|
||||
└── feature-flags.ts # Worker pool feature flags
|
||||
|
||||
public/workers/
|
||||
├── tree-sitter-worker.js # Tree-sitter parsing worker
|
||||
├── generic-worker.js # Generic processing worker
|
||||
└── file-processing-worker.js # File analysis worker
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔧 Core Components
|
||||
|
||||
### **1. WebWorkerPool Class**
|
||||
```typescript
|
||||
import { WebWorkerPool } from './src/lib/web-worker-pool.js';
|
||||
|
||||
const workerPool = new WebWorkerPool({
|
||||
maxWorkers: navigator.hardwareConcurrency,
|
||||
workerScript: '/workers/tree-sitter-worker.js',
|
||||
timeout: 30000,
|
||||
name: 'MyWorkerPool'
|
||||
});
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- **Automatic worker management** - Creates, recycles, and terminates workers
|
||||
- **Task queuing** - Handles task distribution and load balancing
|
||||
- **Error recovery** - Graceful handling of worker failures
|
||||
- **Progress tracking** - Real-time progress callbacks
|
||||
- **Statistics** - Detailed performance metrics
|
||||
|
||||
### **2. ParallelParsingProcessor**
|
||||
```typescript
|
||||
import { ParallelParsingProcessor } from './src/core/ingestion/parallel-parsing-processor.ts';
|
||||
|
||||
const processor = new ParallelParsingProcessor();
|
||||
await processor.process(graph, {
|
||||
filePaths: ['file1.ts', 'file2.ts'],
|
||||
fileContents: fileContentsMap,
|
||||
options: { useParallelProcessing: true }
|
||||
});
|
||||
```
|
||||
|
||||
**Key Features:**
|
||||
- **Parallel file parsing** - Processes multiple files simultaneously
|
||||
- **Worker pool integration** - Uses WebWorkerPool for CPU-intensive tasks
|
||||
- **Memory optimization** - Efficient result aggregation
|
||||
- **Error handling** - Continues processing even if some files fail
|
||||
|
||||
### **3. ParallelGraphPipeline**
|
||||
```typescript
|
||||
import { ParallelGraphPipeline } from './src/core/ingestion/parallel-pipeline.ts';
|
||||
|
||||
const pipeline = new ParallelGraphPipeline();
|
||||
pipeline.setProgressCallback((progress) => {
|
||||
console.log(`${progress.phase}: ${progress.progress}%`);
|
||||
});
|
||||
|
||||
const graph = await pipeline.run({
|
||||
projectRoot: '/path/to/project',
|
||||
projectName: 'MyProject',
|
||||
filePaths: allFilePaths,
|
||||
fileContents: fileContentsMap,
|
||||
options: { useParallelProcessing: true }
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎮 Usage Examples
|
||||
|
||||
### **Basic Worker Pool Usage**
|
||||
```typescript
|
||||
import { WebWorkerPool } from './src/lib/web-worker-pool.js';
|
||||
|
||||
// Create worker pool
|
||||
const workerPool = new WebWorkerPool({
|
||||
maxWorkers: 4,
|
||||
workerScript: '/workers/generic-worker.js'
|
||||
});
|
||||
|
||||
// Execute single task
|
||||
const result = await workerPool.execute({
|
||||
taskType: 'textAnalysis',
|
||||
text: 'Hello world!',
|
||||
analysisType: 'wordCount'
|
||||
});
|
||||
|
||||
// Execute multiple tasks in parallel
|
||||
const results = await workerPool.executeAll([
|
||||
{ taskType: 'textAnalysis', text: 'Task 1', analysisType: 'wordCount' },
|
||||
{ taskType: 'textAnalysis', text: 'Task 2', analysisType: 'wordCount' }
|
||||
]);
|
||||
|
||||
// Execute with progress tracking
|
||||
const results = await workerPool.executeWithProgress(
|
||||
tasks,
|
||||
(completed, total) => {
|
||||
console.log(`Progress: ${(completed/total)*100}%`);
|
||||
}
|
||||
);
|
||||
|
||||
// Cleanup
|
||||
await workerPool.shutdown();
|
||||
```
|
||||
|
||||
### **File Processing with Workers**
|
||||
```typescript
|
||||
import { WebWorkerPoolUtils } from './src/lib/web-worker-pool.js';
|
||||
|
||||
// Create specialized file processing pool
|
||||
const filePool = WebWorkerPoolUtils.createCPUPool({
|
||||
workerScript: '/workers/file-processing-worker.js'
|
||||
});
|
||||
|
||||
// Analyze file structure
|
||||
const analysis = await filePool.execute({
|
||||
processorType: 'analyzeStructure',
|
||||
filePath: '/src/main.ts',
|
||||
content: fileContent
|
||||
});
|
||||
|
||||
// Extract dependencies
|
||||
const dependencies = await filePool.execute({
|
||||
processorType: 'extractDependencies',
|
||||
filePath: '/src/main.ts',
|
||||
content: fileContent
|
||||
});
|
||||
```
|
||||
|
||||
### **Parallel Pipeline Integration**
|
||||
```typescript
|
||||
import { ParallelGraphPipeline } from './src/core/ingestion/parallel-pipeline.ts';
|
||||
|
||||
// Check if parallel processing is supported
|
||||
if (ParallelGraphPipeline.isParallelProcessingSupported()) {
|
||||
const pipeline = new ParallelGraphPipeline();
|
||||
|
||||
// Set up progress tracking
|
||||
pipeline.setProgressCallback((progress) => {
|
||||
console.log(`${progress.phase}: ${progress.message} (${progress.progress}%)`);
|
||||
});
|
||||
|
||||
// Run parallel processing
|
||||
const graph = await pipeline.run({
|
||||
projectRoot: '/path/to/project',
|
||||
projectName: 'MyProject',
|
||||
filePaths: allFilePaths,
|
||||
fileContents: fileContentsMap,
|
||||
options: {
|
||||
useParallelProcessing: true,
|
||||
maxWorkers: ParallelGraphPipeline.getOptimalWorkerCount()
|
||||
}
|
||||
});
|
||||
|
||||
// Get worker pool statistics
|
||||
const stats = pipeline.getWorkerPoolStats();
|
||||
console.log('Worker pool stats:', stats);
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## ⚙️ Configuration
|
||||
|
||||
### **Feature Flags**
|
||||
```typescript
|
||||
import { featureFlags } from './src/config/feature-flags.js';
|
||||
|
||||
// Enable worker pool features
|
||||
featureFlags.enableWorkerPool();
|
||||
|
||||
// Or configure individually
|
||||
featureFlags.setFlags({
|
||||
enableWorkerPool: true,
|
||||
enableParallelParsing: true,
|
||||
enableParallelProcessing: true
|
||||
});
|
||||
|
||||
// Check if enabled
|
||||
if (featureFlags.getFlag('enableWorkerPool')) {
|
||||
// Use worker pool
|
||||
}
|
||||
```
|
||||
|
||||
### **Worker Pool Options**
|
||||
```typescript
|
||||
const workerPoolOptions = {
|
||||
maxWorkers: navigator.hardwareConcurrency, // Number of workers
|
||||
workerScript: '/workers/tree-sitter-worker.js', // Worker script path
|
||||
timeout: 30000, // Task timeout in milliseconds
|
||||
name: 'MyWorkerPool' // Pool name for logging
|
||||
};
|
||||
```
|
||||
|
||||
### **Optimal Worker Counts**
|
||||
```typescript
|
||||
import { WebWorkerPoolUtils } from './src/lib/web-worker-pool.js';
|
||||
|
||||
// Get optimal worker count for different task types
|
||||
const cpuWorkers = WebWorkerPoolUtils.getOptimalWorkerCount('cpu');
|
||||
const ioWorkers = WebWorkerPoolUtils.getOptimalWorkerCount('io');
|
||||
const mixedWorkers = WebWorkerPoolUtils.getOptimalWorkerCount('mixed');
|
||||
|
||||
// Get hardware concurrency
|
||||
const concurrency = WebWorkerPoolUtils.getHardwareConcurrency();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🧪 Testing
|
||||
|
||||
### **Run Test Suite**
|
||||
```typescript
|
||||
import { runWorkerPoolTests } from './src/lib/worker-pool-test.js';
|
||||
|
||||
// Run all tests
|
||||
await runWorkerPoolTests();
|
||||
```
|
||||
|
||||
### **Individual Tests**
|
||||
```typescript
|
||||
import {
|
||||
testWorkerPoolBasic,
|
||||
testFileProcessingPool,
|
||||
testWorkerPoolPerformance,
|
||||
testWorkerPoolErrorHandling
|
||||
} from './src/lib/worker-pool-test.js';
|
||||
|
||||
// Test basic functionality
|
||||
await testWorkerPoolBasic();
|
||||
|
||||
// Test file processing
|
||||
await testFileProcessingPool();
|
||||
|
||||
// Test performance
|
||||
await testWorkerPoolPerformance();
|
||||
|
||||
// Test error handling
|
||||
await testWorkerPoolErrorHandling();
|
||||
```
|
||||
|
||||
### **Browser Console Testing**
|
||||
```javascript
|
||||
// Available globally in browser
|
||||
await window.runWorkerPoolTests();
|
||||
await window.testWorkerPoolBasic();
|
||||
await window.testWorkerPoolPerformance();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📊 Performance Monitoring
|
||||
|
||||
### **Worker Pool Statistics**
|
||||
```typescript
|
||||
const stats = workerPool.getStats();
|
||||
console.log({
|
||||
totalWorkers: stats.totalWorkers,
|
||||
availableWorkers: stats.availableWorkers,
|
||||
activeTasks: stats.activeTasks,
|
||||
queuedTasks: stats.queuedTasks,
|
||||
maxWorkers: stats.maxWorkers,
|
||||
memoryUsage: stats.memoryUsage
|
||||
});
|
||||
```
|
||||
|
||||
### **Performance Metrics**
|
||||
```typescript
|
||||
// Track processing time
|
||||
const startTime = performance.now();
|
||||
const results = await workerPool.executeAll(tasks);
|
||||
const endTime = performance.now();
|
||||
|
||||
console.log({
|
||||
totalTime: endTime - startTime,
|
||||
averageTimePerTask: (endTime - startTime) / tasks.length,
|
||||
processingRate: tasks.length / ((endTime - startTime) / 1000)
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🚨 Error Handling
|
||||
|
||||
### **Worker Errors**
|
||||
```typescript
|
||||
workerPool.on('workerError', (data) => {
|
||||
console.warn(`Worker ${data.workerId} error:`, data.error);
|
||||
});
|
||||
|
||||
workerPool.on('workerCreated', (data) => {
|
||||
console.log(`Worker ${data.workerId} created`);
|
||||
});
|
||||
|
||||
workerPool.on('shutdown', () => {
|
||||
console.log('Worker pool shutdown');
|
||||
});
|
||||
```
|
||||
|
||||
### **Task Timeouts**
|
||||
```typescript
|
||||
try {
|
||||
const result = await workerPool.execute(task);
|
||||
} catch (error) {
|
||||
if (error.message.includes('timed out')) {
|
||||
console.warn('Task timed out, retrying...');
|
||||
// Implement retry logic
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### **Fallback to Sequential Processing**
|
||||
```typescript
|
||||
if (ParallelGraphPipeline.isParallelProcessingSupported()) {
|
||||
// Use parallel processing
|
||||
const pipeline = new ParallelGraphPipeline();
|
||||
} else {
|
||||
// Fallback to sequential processing
|
||||
const pipeline = new GraphPipeline();
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🔄 Migration Guide
|
||||
|
||||
### **From Sequential to Parallel Processing**
|
||||
|
||||
**Before (Sequential):**
|
||||
```typescript
|
||||
import { GraphPipeline } from './src/core/ingestion/pipeline.ts';
|
||||
|
||||
const pipeline = new GraphPipeline();
|
||||
const graph = await pipeline.run(input);
|
||||
```
|
||||
|
||||
**After (Parallel):**
|
||||
```typescript
|
||||
import { ParallelGraphPipeline } from './src/core/ingestion/parallel-pipeline.ts';
|
||||
|
||||
const pipeline = new ParallelGraphPipeline();
|
||||
pipeline.setProgressCallback((progress) => {
|
||||
console.log(`${progress.phase}: ${progress.progress}%`);
|
||||
});
|
||||
|
||||
const graph = await pipeline.run({
|
||||
...input,
|
||||
options: { useParallelProcessing: true }
|
||||
});
|
||||
```
|
||||
|
||||
### **From BatchProcessor to WorkerPool**
|
||||
|
||||
**Before (Sequential batches):**
|
||||
```typescript
|
||||
const batchProcessor = new BatchProcessor(10, async (files) => {
|
||||
for (const file of files) {
|
||||
await processFile(file); // Sequential within batch
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
**After (Parallel workers):**
|
||||
```typescript
|
||||
const workerPool = new WebWorkerPool({
|
||||
maxWorkers: 4,
|
||||
workerScript: '/workers/tree-sitter-worker.js'
|
||||
});
|
||||
|
||||
const results = await workerPool.executeAll(
|
||||
files.map(file => ({ filePath: file, content: fileContents.get(file) }))
|
||||
);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 🎯 Best Practices
|
||||
|
||||
### **1. Worker Pool Configuration**
|
||||
- **CPU-intensive tasks**: Use `navigator.hardwareConcurrency` workers
|
||||
- **I/O-intensive tasks**: Use 2-4x more workers than CPU cores
|
||||
- **Mixed tasks**: Use 2-8 workers depending on workload
|
||||
|
||||
### **2. Task Design**
|
||||
- **Keep tasks independent** - Avoid shared state between workers
|
||||
- **Serialize data efficiently** - Minimize data transfer overhead
|
||||
- **Handle errors gracefully** - Implement proper error recovery
|
||||
|
||||
### **3. Memory Management**
|
||||
- **Monitor memory usage** - Use `performance.memory` API
|
||||
- **Clean up resources** - Always call `workerPool.shutdown()`
|
||||
- **Batch large datasets** - Process in chunks to avoid memory issues
|
||||
|
||||
### **4. Performance Optimization**
|
||||
- **Profile worker performance** - Monitor task execution times
|
||||
- **Adjust worker count** - Find optimal balance for your workload
|
||||
- **Use appropriate timeouts** - Set realistic timeout values
|
||||
|
||||
---
|
||||
|
||||
## 🚀 Ready to Use!
|
||||
|
||||
The Worker Pool implementation is **fully functional** and ready for production use. It provides:
|
||||
|
||||
✅ **Massive performance improvements** for large codebases
|
||||
✅ **Automatic worker management** and error recovery
|
||||
✅ **Progress tracking** and performance monitoring
|
||||
✅ **Easy integration** with existing pipeline
|
||||
✅ **Comprehensive testing** and documentation
|
||||
|
||||
**Next Steps:**
|
||||
1. Test with your codebase to measure performance gains
|
||||
2. Adjust worker counts based on your system capabilities
|
||||
3. Monitor memory usage and optimize as needed
|
||||
4. Enjoy faster, more responsive code analysis! 🎉
|
||||
492
log.txt
492
log.txt
|
|
@ -1,492 +0,0 @@
|
|||
Starting ZIP processing... trumio-cortex-core.zip
|
||||
zip.ts:64 Starting complete ZIP extraction of: trumio-cortex-core.zip (27513697 bytes)
|
||||
zip.ts:118 Skipping large file: trumio-cortex-core/.git/objects/pack/pack-4d5e84ce4d4c68c3dcdaf68f49c25a0693180f62.pack (27326542 bytes)
|
||||
(anonymous) @ zip.ts:118
|
||||
forEach @ jszip.js?v=7c98553f:349
|
||||
extractCompleteStructure @ zip.ts:102
|
||||
await in extractCompleteStructure
|
||||
processZipFile @ ingestion.service.ts:94
|
||||
handleFileUpload @ HomePage.tsx:118
|
||||
callCallback2 @ chunk-YZVM2MHU.js?v=5b2c6c96:3674
|
||||
invokeGuardedCallbackDev @ chunk-YZVM2MHU.js?v=5b2c6c96:3699
|
||||
invokeGuardedCallback @ chunk-YZVM2MHU.js?v=5b2c6c96:3733
|
||||
invokeGuardedCallbackAndCatchFirstError @ chunk-YZVM2MHU.js?v=5b2c6c96:3736
|
||||
executeDispatch @ chunk-YZVM2MHU.js?v=5b2c6c96:7014
|
||||
processDispatchQueueItemsInOrder @ chunk-YZVM2MHU.js?v=5b2c6c96:7034
|
||||
processDispatchQueue @ chunk-YZVM2MHU.js?v=5b2c6c96:7043
|
||||
dispatchEventsForPlugins @ chunk-YZVM2MHU.js?v=5b2c6c96:7051
|
||||
(anonymous) @ chunk-YZVM2MHU.js?v=5b2c6c96:7174
|
||||
batchedUpdates$1 @ chunk-YZVM2MHU.js?v=5b2c6c96:18913
|
||||
batchedUpdates @ chunk-YZVM2MHU.js?v=5b2c6c96:3579
|
||||
dispatchEventForPluginEventSystem @ chunk-YZVM2MHU.js?v=5b2c6c96:7173
|
||||
dispatchEventWithEnableCapturePhaseSelectiveHydrationWithoutDiscreteEventReplay @ chunk-YZVM2MHU.js?v=5b2c6c96:5478
|
||||
dispatchEvent @ chunk-YZVM2MHU.js?v=5b2c6c96:5472
|
||||
dispatchDiscreteEvent @ chunk-YZVM2MHU.js?v=5b2c6c96:5449Understand this warning
|
||||
zip.ts:146 ZIP: Discovered 271 total paths, 109 files with content
|
||||
zip.ts:147 ZIP: Total extracted size: 401503 bytes
|
||||
workerUtils.ts:43 Ingestion worker initialized successfully
|
||||
ingestion.worker.ts?worker_file&type=module:15 IngestionWorker: Starting processing with 271 files
|
||||
pipeline.ts:19 🚀 Starting 4-pass ingestion for project: trumio-cortex-core
|
||||
pipeline.ts:20 📁 Pass 1: Analyzing project structure...
|
||||
structure-processor.ts:71 StructureProcessor: Processing 271 complete paths
|
||||
structure-processor.ts:75 StructureProcessor: Found 74 directories and 197 files
|
||||
structure-processor.ts:79 StructureProcessor: Hiding 33 ignored directories from KG
|
||||
structure-processor.ts:86 StructureProcessor: Hiding 82 files in ignored directories from KG
|
||||
structure-processor.ts:227 StructureProcessor: Created 156 CONTAINS relationships
|
||||
structure-processor.ts:92 StructureProcessor: Created 157 nodes total (115 items hidden)
|
||||
pipeline.ts:26 🔍 Pass 2: Parsing code and extracting definitions...
|
||||
parsing-processor.ts:40 ParsingProcessor: Processing 271 total paths
|
||||
parsing-processor.ts:42 Memory status: 0MB used, 0 files cached
|
||||
parsing-processor.ts:44 ParsingProcessor: After filtering: 32 files to parse
|
||||
parsing-processor.ts:49 ParsingProcessor: Found 22 source files and 3 config files, processing in batches of 10
|
||||
parser-loader.ts:65 Loading TypeScript parser from: /wasm/typescript/tree-sitter-typescript.wasm
|
||||
parser-loader.ts:68 TypeScript parser loaded successfully
|
||||
parsing-processor.ts:121 typescript parser loaded successfully.
|
||||
parser-loader.ts:49 Loading JavaScript parser from: /wasm/javascript/tree-sitter-javascript.wasm
|
||||
parser-loader.ts:52 JavaScript parser loaded successfully
|
||||
parsing-processor.ts:121 javascript parser loaded successfully.
|
||||
parser-loader.ts:33 Loading Python parser from: /wasm/python/tree-sitter-python.wasm
|
||||
parser-loader.ts:36 Python parser loaded successfully
|
||||
parsing-processor.ts:121 python parser loaded successfully.
|
||||
parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/docker-compose.yaml
|
||||
parseFile @ parsing-processor.ts:203
|
||||
(anonymous) @ parsing-processor.ts:112
|
||||
await in (anonymous)
|
||||
processAll @ shared-utils.ts:274
|
||||
await in processAll
|
||||
process @ parsing-processor.ts:122
|
||||
await in process
|
||||
run @ pipeline.ts:49
|
||||
await in run
|
||||
processRepository @ ingestion.worker.ts:68
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/manifests/uat/deployment_cortex.yml
|
||||
parseFile @ parsing-processor.ts:203
|
||||
(anonymous) @ parsing-processor.ts:112
|
||||
await in (anonymous)
|
||||
processAll @ shared-utils.ts:274
|
||||
await in processAll
|
||||
process @ parsing-processor.ts:122
|
||||
await in process
|
||||
run @ pipeline.ts:49
|
||||
await in run
|
||||
processRepository @ ingestion.worker.ts:68
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
parsing-processor.ts:131 No parser available for language: generic. Skipping file: trumio-cortex-core/manifests/uat/service_cortex.yml
|
||||
parseFile @ parsing-processor.ts:203
|
||||
(anonymous) @ parsing-processor.ts:112
|
||||
await in (anonymous)
|
||||
processAll @ shared-utils.ts:274
|
||||
await in processAll
|
||||
process @ parsing-processor.ts:122
|
||||
await in process
|
||||
run @ pipeline.ts:49
|
||||
await in run
|
||||
processRepository @ ingestion.worker.ts:68
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
parsing-processor.ts:72 ParsingProcessor: Successfully processed 25 files
|
||||
pipeline.ts:36 🔗 Pass 3: Resolving imports and building dependency map...
|
||||
import-processor.ts:42 ImportProcessor: Starting import resolution...
|
||||
import-processor.ts:52 ImportProcessor: Completed import resolution
|
||||
import-processor.ts:53 ImportProcessor: Found 160 imports, resolved 90 (56.3%)
|
||||
import-processor.ts:54 ImportProcessor: Built import map for 22 files
|
||||
pipeline.ts:38 📞 Pass 4: Resolving function calls with 3-stage strategy...
|
||||
call-processor.ts:33 CallProcessor: Starting call resolution with 3-stage strategy...
|
||||
call-processor.ts:553 🔍 Filtered out: load_dotenv in check_prompt.py
|
||||
2call-processor.ts:553 🔍 Filtered out: getenv in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: AsyncIOMotorClient in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: limit in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: sort in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: find in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: to_list in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: close in check_prompt.py
|
||||
call-processor.ts:553 🔍 Filtered out: run in check_prompt.py
|
||||
call-processor.ts:65 CallProcessor: Found 1 function calls in trumio-cortex-core/check_prompt.py
|
||||
2call-processor.ts:553 🔍 Filtered out: getenv in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: AsyncIOMotorClient in debug_channel_messages.py
|
||||
2call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: find_one in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: get in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: get in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: get in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: now in debug_channel_messages.py
|
||||
call-processor.ts:65 CallProcessor: Found 5 function calls in trumio-cortex-core/debug_channel_messages.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:24
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:31
|
||||
call-processor.ts:86 ❌ Failed to resolve call: timedelta in trumio-cortex-core/debug_channel_messages.py:49
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/debug_channel_messages.py:51
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in mcp_auth_service.py
|
||||
call-processor.ts:65 CallProcessor: Found 14 function calls in trumio-cortex-core/src/python/common_services/mcp_auth_service.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: HTTPBaseModel in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:32
|
||||
call-processor.ts:86 ❌ Failed to resolve call: hexdigest in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:99
|
||||
call-processor.ts:86 ❌ Failed to resolve call: new in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:99
|
||||
call-processor.ts:86 ❌ Failed to resolve call: compare_digest in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:130
|
||||
call-processor.ts:86 ❌ Failed to resolve call: body in trumio-cortex-core/src/python/common_services/mcp_auth_service.py:227
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in openai_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in openai_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: ValueError in openai_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: OpenAI in openai_service.py
|
||||
call-processor.ts:65 CallProcessor: Found 12 function calls in trumio-cortex-core/src/python/common_services/openai_service.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:30
|
||||
call-processor.ts:86 ❌ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:31
|
||||
call-processor.ts:86 ❌ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:32
|
||||
call-processor.ts:86 ❌ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:33
|
||||
call-processor.ts:86 ❌ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:89
|
||||
call-processor.ts:86 ❌ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:90
|
||||
call-processor.ts:86 ❌ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:91
|
||||
call-processor.ts:86 ❌ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:92
|
||||
call-processor.ts:86 ❌ Failed to resolve call: retry in trumio-cortex-core/src/python/common_services/openai_service.py:122
|
||||
call-processor.ts:86 ❌ Failed to resolve call: retry_if_exception_type in trumio-cortex-core/src/python/common_services/openai_service.py:123
|
||||
call-processor.ts:86 ❌ Failed to resolve call: stop_after_attempt in trumio-cortex-core/src/python/common_services/openai_service.py:124
|
||||
call-processor.ts:86 ❌ Failed to resolve call: wait_exponential in trumio-cortex-core/src/python/common_services/openai_service.py:125
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: isinstance in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: TypeError in prompt_registry_service.py
|
||||
2call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: find_one in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: warning in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: ValueError in prompt_registry_service.py
|
||||
2call-processor.ts:553 🔍 Filtered out: now in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: insert_one in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: str in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: find_one in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: limit in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: sort in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: find in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: to_list in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: error in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: ValueError in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: now in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: update_one in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: ObjectId in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: warning in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: delete_one in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: ObjectId in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: warning in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: find in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: to_list in prompt_registry_service.py
|
||||
call-processor.ts:553 🔍 Filtered out: limit in prompt_registry_service.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/common_services/prompt_registry_service.py
|
||||
call-processor.ts:58 📊 Debug: prompt_registry_service.py has 37 call nodes, 7 definitions
|
||||
call-processor.ts:553 🔍 Filtered out: append in chat.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in chat.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in chat.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in chat.py
|
||||
call-processor.ts:553 🔍 Filtered out: HTTPException in chat.py
|
||||
call-processor.ts:553 🔍 Filtered out: str in chat.py
|
||||
call-processor.ts:65 CallProcessor: Found 7 function calls in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:15
|
||||
call-processor.ts:86 ❌ Failed to resolve call: post in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:28
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/demo_app/app/api/v1/endpoints/chat.py:29
|
||||
11call-processor.ts:553 🔍 Filtered out: Field in chat.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/demo_app/app/schemas/chat.py
|
||||
call-processor.ts:58 📊 Debug: chat.py has 11 call nodes, 4 definitions
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: exists in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: load_dotenv in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: upper in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: basicConfig in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: StreamHandler in start-service.py
|
||||
2call-processor.ts:553 🔍 Filtered out: getLogger in start-service.py
|
||||
call-processor.ts:65 CallProcessor: Found 15 function calls in trumio-cortex-core/src/python/demo_app/start-service.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:26
|
||||
call-processor.ts:86 ❌ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:27
|
||||
call-processor.ts:86 ❌ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:28
|
||||
call-processor.ts:86 ❌ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:37
|
||||
call-processor.ts:86 ❌ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:40
|
||||
call-processor.ts:86 ❌ Failed to resolve call: setLevel in trumio-cortex-core/src/python/demo_app/start-service.py:43
|
||||
call-processor.ts:86 ❌ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:59
|
||||
call-processor.ts:86 ❌ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:62
|
||||
call-processor.ts:86 ❌ Failed to resolve call: get_motor_client in trumio-cortex-core/src/python/demo_app/start-service.py:77
|
||||
call-processor.ts:86 ❌ Failed to resolve call: compile_graphs in trumio-cortex-core/src/python/demo_app/start-service.py:87
|
||||
call-processor.ts:86 ❌ Failed to resolve call: close_motor_client in trumio-cortex-core/src/python/demo_app/start-service.py:99
|
||||
call-processor.ts:86 ❌ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:132
|
||||
call-processor.ts:86 ❌ Failed to resolve call: exit in trumio-cortex-core/src/python/demo_app/start-service.py:135
|
||||
call-processor.ts:86 ❌ Failed to resolve call: RedirectResponse in trumio-cortex-core/src/python/demo_app/start-service.py:141
|
||||
call-processor.ts:86 ❌ Failed to resolve call: get_swagger_ui_html in trumio-cortex-core/src/python/demo_app/start-service.py:145
|
||||
call-processor.ts:553 🔍 Filtered out: append in interactions_endpoint.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in interactions_endpoint.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in interactions_endpoint.py
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in interactions_endpoint.py
|
||||
call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:14
|
||||
call-processor.ts:86 ❌ Failed to resolve call: post in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:16
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:18
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:19
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Query in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:20
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:21
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:22
|
||||
call-processor.ts:86 ❌ Failed to resolve call: fetch_and_process_project_chats_for_period in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/interactions_endpoint.py:32
|
||||
call-processor.ts:553 🔍 Filtered out: append in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: get in mcp_auth.py
|
||||
call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py:14
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/ms_teams_connector/app/api/v1/endpoints/mcp_auth.py:69
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in config.py
|
||||
9call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: int in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: lower in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
3call-processor.ts:553 🔍 Filtered out: bool in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: warning in config.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/config.py
|
||||
call-processor.ts:58 📊 Debug: config.py has 19 call nodes, 0 definitions
|
||||
10call-processor.ts:553 🔍 Filtered out: Field in processed_text_unit.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/schemas/processed_text_unit.py
|
||||
call-processor.ts:58 📊 Debug: processed_text_unit.py has 10 call nodes, 1 definitions
|
||||
2call-processor.ts:553 🔍 Filtered out: Field in watermark.py
|
||||
call-processor.ts:553 🔍 Filtered out: now in watermark.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/schemas/watermark.py
|
||||
call-processor.ts:58 📊 Debug: watermark.py has 3 call nodes, 2 definitions
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: append in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in path_utils.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/ms_teams_connector/app/utils/path_utils.py
|
||||
call-processor.ts:58 📊 Debug: path_utils.py has 9 call nodes, 1 definitions
|
||||
2call-processor.ts:553 🔍 Filtered out: getenv in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: AsyncIOMotorClient in debug_channel_messages.py
|
||||
3call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: aggregate in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: to_list in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: aggregate in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: to_list in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in debug_channel_messages.py
|
||||
call-processor.ts:65 CallProcessor: Found 6 function calls in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:24
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:54
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:61
|
||||
call-processor.ts:86 ❌ Failed to resolve call: timedelta in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:77
|
||||
call-processor.ts:86 ❌ Failed to resolve call: count_documents in trumio-cortex-core/src/python/ms_teams_connector/debug_channel_messages.py:79
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: exists in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: load_dotenv in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: upper in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: basicConfig in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: StreamHandler in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in start-service.py
|
||||
call-processor.ts:65 CallProcessor: Found 5 function calls in trumio-cortex-core/src/python/ms_teams_connector/start-service.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: connect_db_client in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:44
|
||||
call-processor.ts:86 ❌ Failed to resolve call: disconnect_db_client in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:50
|
||||
call-processor.ts:86 ❌ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/ms_teams_connector/start-service.py:79
|
||||
call-processor.ts:553 🔍 Filtered out: append in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in mcp_auth.py
|
||||
call-processor.ts:553 🔍 Filtered out: get in mcp_auth.py
|
||||
call-processor.ts:65 CallProcessor: Found 8 function calls in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py:14
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/mcp_auth.py:69
|
||||
call-processor.ts:553 🔍 Filtered out: append in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: critical in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: error in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: HTTPException in quality_analyzer.py
|
||||
call-processor.ts:553 🔍 Filtered out: warning in quality_analyzer.py
|
||||
call-processor.ts:65 CallProcessor: Found 23 function calls in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: APIRouter in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:36
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:38
|
||||
call-processor.ts:86 ❌ Failed to resolve call: compile_graphs in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:46
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:74
|
||||
call-processor.ts:86 ❌ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:80
|
||||
call-processor.ts:86 ❌ Failed to resolve call: calculate_date_range in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:100
|
||||
call-processor.ts:86 ❌ Failed to resolve call: invoke_weekly_user_fetch_graph in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:104
|
||||
call-processor.ts:86 ❌ Failed to resolve call: run_full_weekly_analysis in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:126
|
||||
call-processor.ts:86 ❌ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:148
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Query in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:157
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Query in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:158
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:159
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:160
|
||||
call-processor.ts:86 ❌ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:179
|
||||
call-processor.ts:86 ❌ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:187
|
||||
call-processor.ts:86 ❌ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:197
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:205
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:206
|
||||
call-processor.ts:86 ❌ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:214
|
||||
call-processor.ts:86 ❌ Failed to resolve call: post in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:220
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:228
|
||||
call-processor.ts:86 ❌ Failed to resolve call: Depends in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:229
|
||||
call-processor.ts:86 ❌ Failed to resolve call: add_task in trumio-cortex-core/src/python/pr_quality_service/app/api/v1/endpoints/quality_analyzer.py:237
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: int in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: upper in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
10call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: strip in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
2call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: int in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
5call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: int in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: lower in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in config.py
|
||||
6call-processor.ts:553 🔍 Filtered out: bool in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in config.py
|
||||
call-processor.ts:553 🔍 Filtered out: warning in config.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/pr_quality_service/app/config.py
|
||||
call-processor.ts:58 📊 Debug: config.py has 38 call nodes, 0 definitions
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: append in path_utils.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in path_utils.py
|
||||
call-processor.ts:52 ⚠️ CallProcessor: No function calls found in source file: trumio-cortex-core/src/python/pr_quality_service/app/utils/path_utils.py
|
||||
call-processor.ts:58 📊 Debug: path_utils.py has 9 call nodes, 1 definitions
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: exists in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: load_dotenv in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: upper in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: getenv in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: basicConfig in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: StreamHandler in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: getLogger in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in start-service.py
|
||||
call-processor.ts:553 🔍 Filtered out: info in start-service.py
|
||||
call-processor.ts:65 CallProcessor: Found 6 function calls in trumio-cortex-core/src/python/pr_quality_service/start-service.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: connect_db_client in trumio-cortex-core/src/python/pr_quality_service/start-service.py:45
|
||||
call-processor.ts:86 ❌ Failed to resolve call: disconnect_db_client in trumio-cortex-core/src/python/pr_quality_service/start-service.py:54
|
||||
call-processor.ts:86 ❌ Failed to resolve call: is_motor_client_open in trumio-cortex-core/src/python/pr_quality_service/start-service.py:97
|
||||
call-processor.ts:86 ❌ Failed to resolve call: get_database in trumio-cortex-core/src/python/pr_quality_service/start-service.py:104
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: dirname in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: abspath in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: join in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: exists in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: load_dotenv in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: append in prompt_test.py
|
||||
6call-processor.ts:553 🔍 Filtered out: print in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: print in prompt_test.py
|
||||
call-processor.ts:553 🔍 Filtered out: run in prompt_test.py
|
||||
call-processor.ts:65 CallProcessor: Found 3 function calls in trumio-cortex-core/src/python/pr_quality_service/test/prompt_test.py
|
||||
call-processor.ts:86 ❌ Failed to resolve call: get_weekly_prompts in trumio-cortex-core/src/python/pr_quality_service/test/prompt_test.py:31
|
||||
call-processor.ts:723 📊 CallProcessor Resolution Statistics:
|
||||
call-processor.ts:724 Total calls processed: 121
|
||||
call-processor.ts:725 ✅ Exact matches (Stage 1): 11 (9.1%)
|
||||
call-processor.ts:726 ✅ Same-file matches (Stage 2): 19 (15.7%)
|
||||
call-processor.ts:727 🎯 Heuristic matches (Stage 3): 4 (3.3%)
|
||||
call-processor.ts:728 ❌ Failed resolutions: 87 (71.9%)
|
||||
call-processor.ts:729 Success rate: 28.1%
|
||||
pipeline.ts:41 Ingestion complete. Graph contains 417 nodes and 446 relationships.
|
||||
pipeline.ts:50 📊 Graph Statistics:
|
||||
pipeline.ts:51 Nodes by type: {Project: 1, Folder: 41, File: 140, Import: 160, Function: 52, …}
|
||||
pipeline.ts:52 Relationships by type: {CONTAINS: 156, DEFINES: 235, IMPORTS: 23, CALLS: 32}
|
||||
pipeline.ts:60 ⚠️ Found 3 isolated nodes:
|
||||
run @ pipeline.ts:60
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:65 Isolated nodes by type: {File: 3}
|
||||
run @ pipeline.ts:65
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:66 Sample isolated nodes: (3) [{…}, {…}, {…}]
|
||||
run @ pipeline.ts:66
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:82 ⚠️ Found 140 files without definitions:
|
||||
run @ pipeline.ts:82
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:83 Files without content: (5) ['trumio-cortex-core/', 'trumio-cortex-core/.github/', 'trumio-cortex-core/.github/workflows/', 'trumio-cortex-core/.github/workflows/cortex-ci-cd.yml', 'trumio-cortex-core/.github/workflows/readme.md']
|
||||
run @ pipeline.ts:83
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:94 🔍 Validating graph integrity...
|
||||
pipeline.ts:130 Source files without definitions: (3) ['trumio-cortex-core/check_prompt.py', 'trumio-cortex-core/debug_channel_messages.py', 'trumio-cortex-core/src/python/common_services/mcp_auth_service.py']
|
||||
validateGraphIntegrity @ pipeline.ts:130
|
||||
run @ pipeline.ts:87
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:158 ⚠️ Graph integrity issues found:
|
||||
validateGraphIntegrity @ pipeline.ts:158
|
||||
run @ pipeline.ts:87
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:159 1. 25 files not connected to project structure
|
||||
(anonymous) @ pipeline.ts:159
|
||||
validateGraphIntegrity @ pipeline.ts:159
|
||||
run @ pipeline.ts:87
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:159 2. 59 source files contain no parsed definitions
|
||||
(anonymous) @ pipeline.ts:159
|
||||
validateGraphIntegrity @ pipeline.ts:159
|
||||
run @ pipeline.ts:87
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
pipeline.ts:159 3. 66 definitions not connected to files
|
||||
(anonymous) @ pipeline.ts:159
|
||||
validateGraphIntegrity @ pipeline.ts:159
|
||||
run @ pipeline.ts:87
|
||||
await in run
|
||||
processRepository @ :5173/src/workers/ingestion.worker.ts?worker_file&type=module:34
|
||||
callback @ comlink.js?v=4af0e45c:91Understand this warning
|
||||
:5173/src/workers/ingestion.worker.ts?worker_file&type=module:40 IngestionWorker: Processing completed successfully
|
||||
:5173/src/workers/ingestion.worker.ts?worker_file&type=module:41 Graph contains 417 nodes and 446 relationships
|
||||
HomePage.tsx:123 ZIP processing completed: {nodeCount: 417, relationshipCount: 446, fileCount: 109}
|
||||
:5173/src/workers/ingestion.worker.ts?worker_file&type=module:150 Ingestion worker terminated
|
||||
workerUtils.ts:127 Ingestion worker terminate
|
||||
|
|
@ -62,6 +62,7 @@ export class EngineManager {
|
|||
async process(input: ProcessingInput, callbacks?: ProcessingCallbacks): Promise<ProcessingResult> {
|
||||
const selectedEngine = featureFlagManager.getProcessingEngine();
|
||||
console.log(`🎯 Engine Manager: Selected engine - ${selectedEngine}`);
|
||||
console.log(`ENGINE-CHECK: Currently using ${selectedEngine} engine - this engine is set from .env`);
|
||||
|
||||
try {
|
||||
// First, try the selected engine
|
||||
|
|
|
|||
|
|
@ -444,3 +444,6 @@ export function runImportExtractionTests(): void {
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -331,3 +331,6 @@ function detectLanguage(filePath: string): 'python' | 'javascript' | 'typescript
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -24,3 +24,6 @@ if (typeof window !== 'undefined') {
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,3 +38,6 @@
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue