mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-07 08:26:11 +00:00
removed .env, refactoiring
This commit is contained in:
abhigyantrumio
parent
7f8ee8c01e
commit
3d6b88938c
12 changed files with 377 additions and 366 deletions
94
.env
94
.env
|
|
@ -1,94 +0,0 @@
|
|||
|
||||
# ========================================
|
||||
# PROCESSING CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Enable performance monitoring
|
||||
PROCESSING_PERFORMANCE_MONITORING=true
|
||||
|
||||
# ========================================
|
||||
# PARALLEL PROCESSING CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Worker pool settings for parallel processing mode
|
||||
PARALLEL_MAX_WORKERS=4
|
||||
PARALLEL_BATCH_SIZE=20
|
||||
PARALLEL_WORKER_TIMEOUT_MS=60000
|
||||
|
||||
# ========================================
|
||||
# PARSING MODE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Choose parsing approach: 'parallel' for multi-threaded workers, 'single' for single-threaded
|
||||
# - parallel: Uses Web Workers for faster processing (recommended for large projects)
|
||||
# - single: Uses main thread processing (more stable, good for debugging)
|
||||
# Note: VITE_ prefix is required for Vite to expose this to the client
|
||||
VITE_PARSING_MODE=parallel
|
||||
|
||||
# ========================================
|
||||
# GENERAL APPLICATION CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# GitHub token for API access (increases rate limits)
|
||||
GITHUB_TOKEN=your_github_token_here
|
||||
|
||||
# Logging level
|
||||
LOG_LEVEL=info
|
||||
|
||||
# Enable performance metrics
|
||||
LOG_ENABLE_METRICS=true
|
||||
LOG_ENABLE_PERFORMANCE=true
|
||||
|
||||
# Memory configuration
|
||||
MEMORY_MAX_MB=512
|
||||
MEMORY_CLEANUP_THRESHOLD_MB=400
|
||||
|
||||
# ========================================
|
||||
# EXAMPLES AND RECOMMENDED CONFIGURATIONS
|
||||
# ========================================
|
||||
|
||||
# For large repositories (use parallel processing):
|
||||
# VITE_PARSING_MODE=parallel
|
||||
# PARALLEL_MAX_WORKERS=8
|
||||
# PARALLEL_BATCH_SIZE=50
|
||||
|
||||
# For stable processing (use single-threaded):
|
||||
# VITE_PARSING_MODE=single
|
||||
# MEMORY_MAX_MB=1024
|
||||
|
||||
# For development/testing (use parallel with debugging):
|
||||
# VITE_PARSING_MODE=parallel
|
||||
# LOG_LEVEL=info
|
||||
# LOG_ENABLE_PERFORMANCE=true
|
||||
|
||||
# For production (stable single-threaded):
|
||||
# VITE_PARSING_MODE=single
|
||||
# LOG_LEVEL=warn
|
||||
|
||||
# ========================================
|
||||
# QUICK CONFIGURATION EXAMPLES
|
||||
# ========================================
|
||||
|
||||
# To use single-threaded processing (for debugging or stability):
|
||||
# VITE_PARSING_MODE=single
|
||||
|
||||
# To use parallel processing (for performance):
|
||||
# VITE_PARSING_MODE=parallel
|
||||
|
||||
# To switch between modes, just change VITE_PARSING_MODE and restart the server
|
||||
|
||||
# ========================================
|
||||
# KUZU DB CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Enable KuzuDB dual-write mode (stores data in both JSON and KuzuDB)
|
||||
# Values: true/false, 1/0, yes/no
|
||||
VITE_KUZU_ENABLED=true
|
||||
# KuzuDB is now enabled! This will:
|
||||
# - Write data to both JSON (primary) and KuzuDB (secondary)
|
||||
# - Provide enhanced logging and statistics
|
||||
# - Gracefully fall back to JSON-only if KuzuDB fails
|
||||
# - Maintain full backward compatibility
|
||||
|
||||
# To disable KuzuDB and use JSON-only:
|
||||
# VITE_KUZU_ENABLED=false
|
||||
97
.env.example
97
.env.example
|
|
@ -1,97 +0,0 @@
|
|||
# GitNexus Engine Configuration
|
||||
# Copy this file to .env to customize engine settings
|
||||
|
||||
# ========================================
|
||||
# ENGINE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Default engine to use ('legacy' or 'nextgen')
|
||||
ENGINE_DEFAULT=legacy
|
||||
|
||||
# Enable/disable engines
|
||||
ENGINE_LEGACY_ENABLED=true
|
||||
ENGINE_NEXTGEN_ENABLED=true
|
||||
|
||||
# Allow fallback between engines if one fails
|
||||
ENGINE_ALLOW_FALLBACK=true
|
||||
|
||||
# Enable performance monitoring
|
||||
ENGINE_PERFORMANCE_MONITORING=true
|
||||
|
||||
# Auto engine selection based on repository size (experimental)
|
||||
ENGINE_AUTO_SELECTION=false
|
||||
|
||||
# Time threshold before falling back to another engine (ms)
|
||||
ENGINE_FALLBACK_THRESHOLD_MS=10000
|
||||
|
||||
# ========================================
|
||||
# LEGACY ENGINE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# Memory limits for Legacy engine
|
||||
ENGINE_LEGACY_MEMORY_LIMIT_MB=512
|
||||
ENGINE_LEGACY_GC_INTERVAL_MS=30000
|
||||
|
||||
# Processing settings for Legacy engine
|
||||
ENGINE_LEGACY_BATCH_SIZE=10
|
||||
ENGINE_LEGACY_TIMEOUT_MS=30000
|
||||
ENGINE_LEGACY_USE_WORKERS=true
|
||||
|
||||
# ========================================
|
||||
# NEXT-GEN ENGINE CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# KuzuDB settings
|
||||
ENGINE_NEXTGEN_KUZU_DB_PATH=gitnexus.kuzu
|
||||
ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=256
|
||||
ENGINE_NEXTGEN_KUZU_ENABLE_WAL=true
|
||||
ENGINE_NEXTGEN_KUZU_ENABLE_COMPRESSION=true
|
||||
|
||||
# Parallel processing settings
|
||||
ENGINE_NEXTGEN_MAX_WORKERS=4
|
||||
ENGINE_NEXTGEN_BATCH_SIZE=20
|
||||
ENGINE_NEXTGEN_WORKER_TIMEOUT_MS=60000
|
||||
ENGINE_NEXTGEN_ENABLE_PARALLEL_PARSING=true
|
||||
|
||||
# ========================================
|
||||
# GENERAL APPLICATION CONFIGURATION
|
||||
# ========================================
|
||||
|
||||
# GitHub token for API access (increases rate limits)
|
||||
GITHUB_TOKEN=your_github_token_here
|
||||
|
||||
# Logging level
|
||||
LOG_LEVEL=info
|
||||
|
||||
# Enable performance metrics
|
||||
LOG_ENABLE_METRICS=true
|
||||
LOG_ENABLE_PERFORMANCE=true
|
||||
|
||||
# Memory configuration
|
||||
MEMORY_MAX_MB=512
|
||||
MEMORY_CLEANUP_THRESHOLD_MB=400
|
||||
|
||||
# ========================================
|
||||
# EXAMPLES AND RECOMMENDED CONFIGURATIONS
|
||||
# ========================================
|
||||
|
||||
# For large repositories (prefer Next-Gen engine):
|
||||
# ENGINE_DEFAULT=nextgen
|
||||
# ENGINE_NEXTGEN_MAX_WORKERS=8
|
||||
# ENGINE_NEXTGEN_BATCH_SIZE=50
|
||||
# ENGINE_NEXTGEN_KUZU_BUFFER_POOL_SIZE=512
|
||||
|
||||
# For stable processing (prefer Legacy engine):
|
||||
# ENGINE_DEFAULT=legacy
|
||||
# ENGINE_LEGACY_MEMORY_LIMIT_MB=1024
|
||||
# ENGINE_LEGACY_BATCH_SIZE=5
|
||||
|
||||
# For development/testing (enable both with fallback):
|
||||
# ENGINE_DEFAULT=nextgen
|
||||
# ENGINE_ALLOW_FALLBACK=true
|
||||
# ENGINE_PERFORMANCE_MONITORING=true
|
||||
|
||||
# For production (disable experimental features):
|
||||
# ENGINE_AUTO_SELECTION=false
|
||||
# ENGINE_NEXTGEN_ENABLE_PARALLEL_PARSING=false
|
||||
# LOG_LEVEL=warn
|
||||
|
|
@ -13,8 +13,19 @@ export interface GitNexusConfig {
|
|||
// ========================================
|
||||
processing: {
|
||||
mode: 'parallel' | 'single';
|
||||
workers: {
|
||||
mode: 'auto' | 'manual';
|
||||
auto: {
|
||||
enabled: boolean;
|
||||
maxWorkers: number;
|
||||
memoryPerWorkerMB: number;
|
||||
cpuMultiplier: number;
|
||||
};
|
||||
manual: {
|
||||
count: number;
|
||||
};
|
||||
};
|
||||
parallel: {
|
||||
maxWorkers: number;
|
||||
batchSize: number;
|
||||
workerTimeoutMs: number;
|
||||
};
|
||||
|
|
@ -135,8 +146,8 @@ export interface GitNexusConfig {
|
|||
/**
|
||||
* Default GitNexus Configuration
|
||||
*
|
||||
* These are the default values. Environment variables can override them.
|
||||
* Uses import.meta.env for browser compatibility (Vite environment).
|
||||
* Centralized configuration for the client-side application.
|
||||
* No environment variables needed - all settings are defined here.
|
||||
*/
|
||||
const config: GitNexusConfig = {
|
||||
|
||||
|
|
@ -144,39 +155,50 @@ const config: GitNexusConfig = {
|
|||
// PROCESSING CONFIGURATION
|
||||
// ========================================
|
||||
processing: {
|
||||
mode: (import.meta.env.VITE_PARSING_MODE as 'parallel' | 'single') ?? 'parallel',
|
||||
mode: 'parallel', // Use parallel processing by default
|
||||
workers: {
|
||||
mode: 'auto', // Use automatic hardware-based worker scaling
|
||||
auto: {
|
||||
enabled: true,
|
||||
maxWorkers: 20, // Maximum workers allowed (increased from user preference)
|
||||
memoryPerWorkerMB: 60, // Memory estimation per worker
|
||||
cpuMultiplier: 0.75 // Use 75% of CPU cores for safety
|
||||
},
|
||||
manual: {
|
||||
count: 4 // Fallback for manual mode
|
||||
}
|
||||
},
|
||||
parallel: {
|
||||
maxWorkers: parseInt(import.meta.env.VITE_PARALLEL_MAX_WORKERS ?? '4'),
|
||||
batchSize: parseInt(import.meta.env.VITE_PARALLEL_BATCH_SIZE ?? '20'),
|
||||
workerTimeoutMs: parseInt(import.meta.env.VITE_PARALLEL_WORKER_TIMEOUT_MS ?? '60000')
|
||||
batchSize: 20, // Files processed per batch
|
||||
workerTimeoutMs: 60000 // 60 seconds timeout per worker
|
||||
},
|
||||
memory: {
|
||||
maxMB: parseInt(import.meta.env.VITE_MEMORY_MAX_MB ?? '512'),
|
||||
cleanupThresholdMB: parseInt(import.meta.env.VITE_MEMORY_CLEANUP_THRESHOLD_MB ?? '400'),
|
||||
gcIntervalMs: parseInt(import.meta.env.VITE_MEMORY_GC_INTERVAL_MS ?? '30000'),
|
||||
maxFileSizeMB: parseInt(import.meta.env.VITE_MEMORY_MAX_FILE_SIZE_MB ?? '10'),
|
||||
maxFilesInMemory: parseInt(import.meta.env.VITE_MEMORY_MAX_FILES ?? '1000')
|
||||
maxMB: 512,
|
||||
cleanupThresholdMB: 400,
|
||||
gcIntervalMs: 30000,
|
||||
maxFileSizeMB: 10,
|
||||
maxFilesInMemory: 1000
|
||||
},
|
||||
fileExtensions: import.meta.env.VITE_PROCESSING_FILE_EXTENSIONS?.split(',') ?? [
|
||||
fileExtensions: [
|
||||
'.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp',
|
||||
'.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.dart',
|
||||
'.json', '.yaml', '.yml', '.xml', '.toml', '.ini', '.cfg', '.properties'
|
||||
],
|
||||
performanceMonitoring: import.meta.env.VITE_PROCESSING_PERFORMANCE_MONITORING !== 'false'
|
||||
performanceMonitoring: true
|
||||
},
|
||||
|
||||
// ========================================
|
||||
// KUZU DB CONFIGURATION
|
||||
// ========================================
|
||||
kuzu: {
|
||||
enabled: import.meta.env.VITE_KUZU_ENABLED === 'true',
|
||||
persistence: import.meta.env.VITE_KUZU_PERSISTENCE !== 'false',
|
||||
dualWrite: import.meta.env.VITE_KUZU_DUAL_WRITE !== 'false',
|
||||
fallbackToJson: import.meta.env.VITE_KUZU_FALLBACK_JSON !== 'false',
|
||||
enabled: true, // Enable KuzuDB dual-write mode
|
||||
persistence: true,
|
||||
dualWrite: true,
|
||||
fallbackToJson: true,
|
||||
performance: {
|
||||
enableCache: import.meta.env.VITE_KUZU_ENABLE_CACHE !== 'false',
|
||||
cacheSize: parseInt(import.meta.env.VITE_KUZU_CACHE_SIZE ?? '1000'),
|
||||
queryTimeout: parseInt(import.meta.env.VITE_KUZU_QUERY_TIMEOUT ?? '30000')
|
||||
enableCache: true,
|
||||
cacheSize: 1000,
|
||||
queryTimeout: 30000
|
||||
}
|
||||
},
|
||||
|
||||
|
|
@ -185,41 +207,24 @@ const config: GitNexusConfig = {
|
|||
// ========================================
|
||||
ai: {
|
||||
cypher: {
|
||||
defaultLimit: parseInt(import.meta.env.VITE_AI_CYPHER_DEFAULT_LIMIT ?? '20'),
|
||||
maxLimit: parseInt(import.meta.env.VITE_AI_CYPHER_MAX_LIMIT ?? '100'),
|
||||
timeoutMs: parseInt(import.meta.env.VITE_AI_CYPHER_TIMEOUT_MS ?? '30000'),
|
||||
enableValidation: import.meta.env.VITE_AI_CYPHER_ENABLE_VALIDATION !== 'false',
|
||||
enableLimiting: import.meta.env.VITE_AI_CYPHER_ENABLE_LIMITING !== 'false',
|
||||
enableTruncation: import.meta.env.VITE_AI_CYPHER_ENABLE_TRUNCATION !== 'false'
|
||||
defaultLimit: 20,
|
||||
maxLimit: 100,
|
||||
timeoutMs: 30000,
|
||||
enableValidation: true,
|
||||
enableLimiting: true,
|
||||
enableTruncation: true
|
||||
},
|
||||
llm: {
|
||||
defaultProvider: (import.meta.env.VITE_LLM_DEFAULT_PROVIDER as 'openai' | 'azure' | 'anthropic' | 'gemini') ?? 'openai',
|
||||
defaultProvider: 'openai',
|
||||
providers: {
|
||||
openai: import.meta.env.VITE_OPENAI_API_KEY ? {
|
||||
apiKey: import.meta.env.VITE_OPENAI_API_KEY,
|
||||
model: import.meta.env.VITE_OPENAI_MODEL ?? 'gpt-4o-mini',
|
||||
maxTokens: parseInt(import.meta.env.VITE_OPENAI_MAX_TOKENS ?? '4000'),
|
||||
temperature: parseFloat(import.meta.env.VITE_OPENAI_TEMPERATURE ?? '0.1')
|
||||
} : undefined,
|
||||
azure: import.meta.env.VITE_AZURE_API_KEY ? {
|
||||
apiKey: import.meta.env.VITE_AZURE_API_KEY,
|
||||
endpoint: import.meta.env.VITE_AZURE_ENDPOINT,
|
||||
deployment: import.meta.env.VITE_AZURE_DEPLOYMENT,
|
||||
maxTokens: parseInt(import.meta.env.VITE_AZURE_MAX_TOKENS ?? '4000'),
|
||||
temperature: parseFloat(import.meta.env.VITE_AZURE_TEMPERATURE ?? '0.1')
|
||||
} : undefined,
|
||||
anthropic: import.meta.env.VITE_ANTHROPIC_API_KEY ? {
|
||||
apiKey: import.meta.env.VITE_ANTHROPIC_API_KEY,
|
||||
model: import.meta.env.VITE_ANTHROPIC_MODEL ?? 'claude-3-sonnet-20240229',
|
||||
maxTokens: parseInt(import.meta.env.VITE_ANTHROPIC_MAX_TOKENS ?? '4000'),
|
||||
temperature: parseFloat(import.meta.env.VITE_ANTHROPIC_TEMPERATURE ?? '0.1')
|
||||
} : undefined,
|
||||
gemini: import.meta.env.VITE_GEMINI_API_KEY ? {
|
||||
apiKey: import.meta.env.VITE_GEMINI_API_KEY,
|
||||
model: import.meta.env.VITE_GEMINI_MODEL ?? 'gemini-pro',
|
||||
maxTokens: parseInt(import.meta.env.VITE_GEMINI_MAX_TOKENS ?? '4000'),
|
||||
temperature: parseFloat(import.meta.env.VITE_GEMINI_TEMPERATURE ?? '0.1')
|
||||
} : undefined
|
||||
// API keys will be provided via UI settings
|
||||
// No hardcoded keys in configuration
|
||||
openai: {
|
||||
apiKey: '', // Set via UI
|
||||
model: 'gpt-4o-mini',
|
||||
maxTokens: 4000,
|
||||
temperature: 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -228,8 +233,8 @@ const config: GitNexusConfig = {
|
|||
// IGNORE PATTERNS (CENTRALIZED!)
|
||||
// ========================================
|
||||
ignore: {
|
||||
enabled: import.meta.env.VITE_IGNORE_ENABLED !== 'false',
|
||||
patterns: import.meta.env.VITE_IGNORE_PATTERNS?.split(',') ?? [
|
||||
enabled: true,
|
||||
patterns: [
|
||||
// Version Control
|
||||
'.git', '.svn', '.hg',
|
||||
// Package Managers & Dependencies
|
||||
|
|
@ -254,8 +259,8 @@ const config: GitNexusConfig = {
|
|||
// Cache Directories
|
||||
'.cache', '.parcel-cache', '.next', '.nuxt'
|
||||
],
|
||||
suffixes: import.meta.env.VITE_IGNORE_SUFFIXES?.split(',') ?? ['.tmp', '~', '.bak', '.swp', '.swo'],
|
||||
fileExtensions: import.meta.env.VITE_IGNORE_FILE_EXTENSIONS?.split(',') ?? [
|
||||
suffixes: ['.tmp', '~', '.bak', '.swp', '.swo'],
|
||||
fileExtensions: [
|
||||
// Compiled/Binary
|
||||
'.pyc', '.pyo', '.pyd', '.so', '.dll', '.exe', '.jar', '.war', '.ear',
|
||||
// Archives
|
||||
|
|
@ -269,40 +274,40 @@ const config: GitNexusConfig = {
|
|||
// Minified/Generated
|
||||
'.min.js', '.min.css', '.map'
|
||||
],
|
||||
customPatterns: import.meta.env.VITE_IGNORE_CUSTOM_PATTERNS?.split(',') ?? []
|
||||
customPatterns: []
|
||||
},
|
||||
|
||||
// ========================================
|
||||
// LOGGING & DEBUGGING
|
||||
// ========================================
|
||||
logging: {
|
||||
level: (import.meta.env.VITE_LOG_LEVEL as 'debug' | 'info' | 'warn' | 'error') ?? 'info',
|
||||
enableMetrics: import.meta.env.VITE_LOG_ENABLE_METRICS !== 'false',
|
||||
enablePerformance: import.meta.env.VITE_LOG_ENABLE_PERFORMANCE !== 'false',
|
||||
maxEntries: parseInt(import.meta.env.VITE_LOG_MAX_ENTRIES ?? '1000'),
|
||||
monitoringIntervalMs: parseInt(import.meta.env.VITE_LOG_MONITORING_INTERVAL_MS ?? '30000')
|
||||
level: 'info',
|
||||
enableMetrics: true,
|
||||
enablePerformance: true,
|
||||
maxEntries: 1000,
|
||||
monitoringIntervalMs: 30000
|
||||
},
|
||||
|
||||
// ========================================
|
||||
// GITHUB INTEGRATION
|
||||
// ========================================
|
||||
github: {
|
||||
token: import.meta.env.VITE_GITHUB_TOKEN,
|
||||
apiUrl: import.meta.env.VITE_GITHUB_API_URL ?? 'https://api.github.com',
|
||||
token: '', // Will be set via UI settings - no hardcoded tokens
|
||||
apiUrl: 'https://api.github.com',
|
||||
rateLimit: {
|
||||
maxRequests: parseInt(import.meta.env.VITE_GITHUB_RATE_LIMIT_MAX ?? '60'),
|
||||
windowMs: parseInt(import.meta.env.VITE_GITHUB_RATE_LIMIT_WINDOW_MS ?? '60000')
|
||||
maxRequests: 60, // GitHub default for unauthenticated requests
|
||||
windowMs: 60000 // 1 minute window
|
||||
},
|
||||
retry: {
|
||||
maxRetries: parseInt(import.meta.env.VITE_GITHUB_RETRY_MAX ?? '3'),
|
||||
backoffMs: parseInt(import.meta.env.VITE_GITHUB_RETRY_BACKOFF_MS ?? '1000')
|
||||
maxRetries: 3,
|
||||
backoffMs: 1000
|
||||
}
|
||||
},
|
||||
|
||||
// ========================================
|
||||
// ENVIRONMENT & DEPLOYMENT
|
||||
// ========================================
|
||||
environment: (import.meta.env.MODE as 'development' | 'staging' | 'production') ?? 'development'
|
||||
environment: 'development' // Can be changed for different deployments
|
||||
};
|
||||
|
||||
export default config;
|
||||
|
|
|
|||
|
|
@ -11,8 +11,19 @@ import { z } from 'zod';
|
|||
|
||||
const ProcessingConfigSchema = z.object({
|
||||
mode: z.enum(['parallel', 'single']),
|
||||
workers: z.object({
|
||||
mode: z.enum(['auto', 'manual']),
|
||||
auto: z.object({
|
||||
enabled: z.boolean(),
|
||||
maxWorkers: z.number().min(1).max(32),
|
||||
memoryPerWorkerMB: z.number().min(20).max(200),
|
||||
cpuMultiplier: z.number().min(0.1).max(2.0)
|
||||
}),
|
||||
manual: z.object({
|
||||
count: z.number().min(1).max(32)
|
||||
})
|
||||
}),
|
||||
parallel: z.object({
|
||||
maxWorkers: z.number().min(1).max(16),
|
||||
batchSize: z.number().min(1).max(100),
|
||||
workerTimeoutMs: z.number().min(10000).max(300000)
|
||||
}),
|
||||
|
|
@ -182,8 +193,19 @@ export class ConfigLoader {
|
|||
return {
|
||||
processing: {
|
||||
mode: 'parallel',
|
||||
workers: {
|
||||
mode: 'auto',
|
||||
auto: {
|
||||
enabled: true,
|
||||
maxWorkers: 16,
|
||||
memoryPerWorkerMB: 60,
|
||||
cpuMultiplier: 0.75
|
||||
},
|
||||
manual: {
|
||||
count: 4
|
||||
}
|
||||
},
|
||||
parallel: {
|
||||
maxWorkers: 4,
|
||||
batchSize: 20,
|
||||
workerTimeoutMs: 60000
|
||||
},
|
||||
|
|
|
|||
|
|
@ -42,10 +42,10 @@ export const DEFAULT_FEATURE_FLAGS: FeatureFlags = {
|
|||
enableParallelParsing: true,
|
||||
enableParallelProcessing: true,
|
||||
|
||||
// KuzuDB Features (disabled by default for safety)
|
||||
enableKuzuDB: false,
|
||||
enableKuzuDBPersistence: false,
|
||||
enableKuzuDBPerformanceMonitoring: false,
|
||||
// KuzuDB Features (enabled by default to match gitnexus.config.ts)
|
||||
enableKuzuDB: true,
|
||||
enableKuzuDBPersistence: true,
|
||||
enableKuzuDBPerformanceMonitoring: true,
|
||||
|
||||
// Debug Features
|
||||
enableDebugMode: false,
|
||||
|
|
|
|||
|
|
@ -34,6 +34,8 @@ export class GraphQueryEngine {
|
|||
return this.executeAggregationQuery(parsedQuery, limit, offset);
|
||||
case 'MATCH_RELATIONSHIP':
|
||||
return this.executeRelationshipQuery(parsedQuery, limit, offset);
|
||||
case 'COUNT_ALL':
|
||||
return this.executeCountAllQuery(parsedQuery);
|
||||
default:
|
||||
throw new Error(`Unsupported query type: ${parsedQuery.type}`);
|
||||
}
|
||||
|
|
@ -79,6 +81,20 @@ export class GraphQueryEngine {
|
|||
};
|
||||
}
|
||||
|
||||
// Count all nodes pattern: MATCH (n) RETURN COUNT(n)
|
||||
const countAllPattern = /MATCH\s+\((\w+)\)\s+RETURN\s+COUNT\(\1\)(?:\s+as\s+(\w+))?(?:\s+LIMIT\s+\d+)?/i;
|
||||
const countAllMatch = cypher.match(countAllPattern);
|
||||
|
||||
if (countAllMatch) {
|
||||
const [, variable, alias] = countAllMatch;
|
||||
return {
|
||||
type: 'COUNT_ALL',
|
||||
variable,
|
||||
alias: alias || 'count',
|
||||
returnClause: `COUNT(${variable})`
|
||||
};
|
||||
}
|
||||
|
||||
// Aggregation pattern: MATCH (n:Label) RETURN COUNT(n)
|
||||
const aggregationPattern = /MATCH\s+\((\w+):(\w+)(?:\s*\{([^}]*)\})?\)\s+RETURN\s+(COUNT|COLLECT|AVG|SUM)\(([^)]+)\)/i;
|
||||
const aggMatch = cypher.match(aggregationPattern);
|
||||
|
|
@ -462,6 +478,19 @@ export class GraphQueryEngine {
|
|||
});
|
||||
}
|
||||
|
||||
private executeCountAllQuery(query: any): QueryResult {
|
||||
const totalCount = this.graph.nodes.length;
|
||||
|
||||
const result: Record<string, any> = {};
|
||||
result[query.alias] = totalCount;
|
||||
|
||||
return {
|
||||
nodes: [],
|
||||
relationships: [],
|
||||
data: [result]
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get all relationships for a node
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -67,11 +67,8 @@ export interface ParallelParsingResult {
|
|||
|
||||
constructor() {
|
||||
this.memoryManager = MemoryManager.getInstance();
|
||||
this.workerPool = WebWorkerPoolUtils.createCPUPool({
|
||||
workerScript: '/workers/tree-sitter-worker.js',
|
||||
name: 'ParallelParsingPool',
|
||||
timeout: 60000 // 60 seconds for parsing
|
||||
});
|
||||
// Worker pool will be initialized asynchronously in initializeWorkerPool()
|
||||
this.workerPool = null as any; // Temporary until initialization
|
||||
}
|
||||
|
||||
public getASTMap(): Map<string, ParsedAST> {
|
||||
|
|
@ -96,6 +93,12 @@ export interface ParallelParsingResult {
|
|||
throw new Error('Web Workers are not supported in this environment');
|
||||
}
|
||||
|
||||
// Create worker pool using configuration
|
||||
this.workerPool = await WebWorkerPoolUtils.createWorkerPool({
|
||||
workerScript: '/workers/tree-sitter-worker.js',
|
||||
name: 'ParallelParsingPool'
|
||||
});
|
||||
|
||||
// Set up worker pool event listeners
|
||||
this.workerPool.on('workerCreated', (data: unknown) => {
|
||||
const { workerId, totalWorkers } = data as { workerId: number, totalWorkers: number };
|
||||
|
|
@ -629,7 +632,8 @@ export interface ParallelParsingResult {
|
|||
}
|
||||
|
||||
// Monitor worker pool memory usage
|
||||
WebWorkerPoolUtils.monitorMemoryUsage();
|
||||
// Memory monitoring is now handled by the MemoryManager
|
||||
console.log('💾 Memory monitoring active via MemoryManager');
|
||||
|
||||
} catch (error) {
|
||||
console.warn('Error monitoring memory usage:', error);
|
||||
|
|
|
|||
|
|
@ -240,8 +240,15 @@ export class ParallelGraphPipeline {
|
|||
/**
|
||||
* Get optimal worker count for current system
|
||||
*/
|
||||
public static getOptimalWorkerCount(): number {
|
||||
return WebWorkerPoolUtils.getOptimalWorkerCount('cpu');
|
||||
public static async getOptimalWorkerCount(): Promise<number> {
|
||||
// Worker count is now determined by configuration
|
||||
const { ConfigLoader } = await import('../../config/config-loader.ts');
|
||||
const { calculateWorkerCount } = await import('../../lib/worker-calculator.ts');
|
||||
|
||||
const config = await ConfigLoader.getInstance().loadConfig();
|
||||
const workerCalc = await calculateWorkerCount(config);
|
||||
|
||||
return workerCalc.workerCount;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -480,48 +480,44 @@ export class FileProcessingPool extends WebWorkerPool {
|
|||
}
|
||||
|
||||
/**
|
||||
* Worker pool utilities
|
||||
* Worker pool utilities - simplified for GitNexus
|
||||
*/
|
||||
export const WebWorkerPoolUtils = {
|
||||
/**
|
||||
* Create a specialized worker pool for CPU-intensive tasks
|
||||
* Create a worker pool using GitNexus configuration
|
||||
*/
|
||||
createCPUPool(options: Partial<WorkerPoolOptions> = {}): WebWorkerPool {
|
||||
return new WebWorkerPool({
|
||||
maxWorkers: navigator.hardwareConcurrency || 4,
|
||||
timeout: 60000, // 1 minute
|
||||
name: 'CPUPool',
|
||||
...options
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Create a worker pool for I/O operations
|
||||
*/
|
||||
createIOPool(options: Partial<WorkerPoolOptions> = {}): WebWorkerPool {
|
||||
return new WebWorkerPool({
|
||||
maxWorkers: Math.min(20, (navigator.hardwareConcurrency || 4) * 4), // More workers for I/O
|
||||
timeout: 30000, // 30 seconds
|
||||
name: 'IOPool',
|
||||
...options
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Get optimal worker count for different task types
|
||||
*/
|
||||
getOptimalWorkerCount(taskType: 'cpu' | 'io' | 'mixed' = 'mixed'): number {
|
||||
const cpuCount = navigator.hardwareConcurrency || 4;
|
||||
async createWorkerPool(options: Partial<WorkerPoolOptions> = {}): Promise<WebWorkerPool> {
|
||||
// Import config loader dynamically to avoid circular dependencies
|
||||
const { ConfigLoader } = await import('../config/config-loader.ts');
|
||||
const { calculateWorkerCount } = await import('./worker-calculator.ts');
|
||||
|
||||
switch (taskType) {
|
||||
case 'cpu':
|
||||
return cpuCount;
|
||||
case 'io':
|
||||
return Math.min(20, cpuCount * 4);
|
||||
case 'mixed':
|
||||
default:
|
||||
return Math.max(2, Math.min(8, cpuCount));
|
||||
}
|
||||
const config = await ConfigLoader.getInstance().loadConfig();
|
||||
const workerCalc = await calculateWorkerCount(config);
|
||||
|
||||
console.log(''); // Empty line for better readability
|
||||
console.log('🔧 GitNexus Worker Pool Initialization');
|
||||
console.log('='.repeat(50));
|
||||
|
||||
const workerPool = new WebWorkerPool({
|
||||
maxWorkers: workerCalc.workerCount,
|
||||
timeout: config.processing.parallel.workerTimeoutMs,
|
||||
name: 'GitNexusWorkerPool',
|
||||
...options
|
||||
});
|
||||
|
||||
console.log('='.repeat(50));
|
||||
console.log('✅ Worker pool created successfully');
|
||||
console.log(''); // Empty line for better readability
|
||||
|
||||
return workerPool;
|
||||
},
|
||||
|
||||
/**
|
||||
* Legacy method - redirects to new simplified approach
|
||||
* @deprecated Use createWorkerPool() instead
|
||||
*/
|
||||
createCPUPool(options: Partial<WorkerPoolOptions> = {}): Promise<WebWorkerPool> {
|
||||
return this.createWorkerPool(options);
|
||||
},
|
||||
|
||||
/**
|
||||
|
|
@ -532,60 +528,22 @@ export const WebWorkerPoolUtils = {
|
|||
},
|
||||
|
||||
/**
|
||||
* Get hardware concurrency
|
||||
* Get system information for debugging
|
||||
*/
|
||||
getHardwareConcurrency(): number {
|
||||
return navigator.hardwareConcurrency || 4;
|
||||
async getSystemInfo() {
|
||||
const { getSystemInfoForDebug } = await import('./worker-calculator.ts');
|
||||
return getSystemInfoForDebug();
|
||||
},
|
||||
|
||||
/**
|
||||
* Cleanup all singleton worker pool instances
|
||||
*/
|
||||
async cleanupAllPools(): Promise<void> {
|
||||
await FileProcessingPool.shutdownInstance();
|
||||
},
|
||||
|
||||
/**
|
||||
* Monitor memory usage and trigger cleanup if needed
|
||||
*/
|
||||
monitorMemoryUsage(): void {
|
||||
if (typeof performance !== 'undefined' && (performance as PerformanceWithMemory).memory) {
|
||||
const memInfo = (performance as PerformanceWithMemory).memory!
|
||||
const usedMemoryMB = memInfo.usedJSHeapSize / (1024 * 1024);
|
||||
const totalMemoryMB = memInfo.totalJSHeapSize / (1024 * 1024);
|
||||
|
||||
console.log(`Memory usage: ${usedMemoryMB.toFixed(2)}MB / ${totalMemoryMB.toFixed(2)}MB`);
|
||||
|
||||
// If memory usage is over 80%, trigger cleanup
|
||||
if (usedMemoryMB / totalMemoryMB > 0.8) {
|
||||
console.warn('High memory usage detected, triggering cleanup...');
|
||||
this.cleanupAllPools();
|
||||
|
||||
// Suggest garbage collection if available
|
||||
if (typeof window !== 'undefined' && (window as WindowWithGC).gc) {
|
||||
(window as WindowWithGC).gc!();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
/**
|
||||
* Setup global cleanup handlers for when the app/page unloads
|
||||
*/
|
||||
setupGlobalCleanup(): void {
|
||||
if (typeof window !== 'undefined') {
|
||||
// Cleanup on page unload
|
||||
window.addEventListener('beforeunload', async () => {
|
||||
await this.cleanupAllPools();
|
||||
});
|
||||
|
||||
// Cleanup on page visibility change (when user switches tabs)
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.hidden) {
|
||||
// Page is now hidden, good time to cleanup
|
||||
this.cleanupAllPools();
|
||||
}
|
||||
});
|
||||
// Cleanup FileProcessingPool if it exists
|
||||
if (FileProcessingPool.hasInstance()) {
|
||||
await FileProcessingPool.shutdownInstance();
|
||||
}
|
||||
|
||||
console.log('🧹 Worker pool cleanup completed');
|
||||
}
|
||||
};
|
||||
|
|
|
|||
47
src/lib/worker-banner.ts
Normal file
47
src/lib/worker-banner.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Worker Banner - Display worker configuration at startup
|
||||
*/
|
||||
|
||||
import type { GitNexusConfig } from '../../gitnexus.config.ts';
|
||||
|
||||
/**
|
||||
* Display a startup banner with worker configuration
|
||||
*/
|
||||
export function displayWorkerBanner(config: GitNexusConfig, workerCount: number): void {
|
||||
const workerConfig = config.processing.workers;
|
||||
|
||||
console.log('');
|
||||
console.log('🚀 GitNexus Worker Configuration');
|
||||
console.log('='.repeat(60));
|
||||
console.log(`Mode: ${workerConfig.mode.toUpperCase()}`);
|
||||
|
||||
if (workerConfig.mode === 'auto') {
|
||||
console.log('Auto Configuration:');
|
||||
console.log(` • Max Workers: ${workerConfig.auto.maxWorkers}`);
|
||||
console.log(` • CPU Multiplier: ${workerConfig.auto.cpuMultiplier}`);
|
||||
console.log(` • Memory per Worker: ${workerConfig.auto.memoryPerWorkerMB}MB`);
|
||||
} else {
|
||||
console.log('Manual Configuration:');
|
||||
console.log(` • Fixed Worker Count: ${workerConfig.manual.count}`);
|
||||
}
|
||||
|
||||
console.log('');
|
||||
console.log(`✨ Active Workers: ${workerCount}`);
|
||||
console.log(`⏱️ Worker Timeout: ${config.processing.parallel.workerTimeoutMs / 1000}s`);
|
||||
console.log(`📦 Batch Size: ${config.processing.parallel.batchSize}`);
|
||||
console.log('='.repeat(60));
|
||||
console.log('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Display environment variable overrides available
|
||||
*/
|
||||
export function displayWorkerEnvHelp(): void {
|
||||
console.log('💡 Environment Variable Overrides:');
|
||||
console.log(' VITE_WORKER_MODE=auto|manual');
|
||||
console.log(' VITE_WORKER_MAX_WORKERS=16');
|
||||
console.log(' VITE_WORKER_MANUAL_COUNT=4');
|
||||
console.log(' VITE_WORKER_CPU_MULTIPLIER=0.75');
|
||||
console.log(' VITE_WORKER_MEMORY_PER_WORKER_MB=60');
|
||||
console.log('');
|
||||
}
|
||||
135
src/lib/worker-calculator.ts
Normal file
135
src/lib/worker-calculator.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
/**
|
||||
* Simple Worker Calculator
|
||||
*
|
||||
* Calculates optimal worker count based on hardware and configuration
|
||||
* without complex categorization or unused pool types.
|
||||
*/
|
||||
|
||||
import type { GitNexusConfig } from '../../gitnexus.config.ts';
|
||||
|
||||
interface PerformanceMemory {
|
||||
usedJSHeapSize: number;
|
||||
totalJSHeapSize: number;
|
||||
jsHeapSizeLimit: number;
|
||||
}
|
||||
|
||||
interface PerformanceWithMemory extends Performance {
|
||||
memory?: PerformanceMemory;
|
||||
}
|
||||
|
||||
interface SystemInfo {
|
||||
cpuCores: number;
|
||||
availableMemoryMB?: number;
|
||||
}
|
||||
|
||||
interface WorkerCalculationResult {
|
||||
workerCount: number;
|
||||
reasoning: string;
|
||||
limitations: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Get system hardware information
|
||||
*/
|
||||
function getSystemInfo(): SystemInfo {
|
||||
const cpuCores = navigator.hardwareConcurrency || 4;
|
||||
|
||||
let availableMemoryMB: number | undefined;
|
||||
if (typeof performance !== 'undefined' && (performance as PerformanceWithMemory).memory) {
|
||||
// Use 75% of available heap as safe limit
|
||||
availableMemoryMB = Math.floor(((performance as PerformanceWithMemory).memory!.jsHeapSizeLimit * 0.75) / (1024 * 1024));
|
||||
}
|
||||
|
||||
return { cpuCores, availableMemoryMB };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate optimal worker count based on configuration and hardware
|
||||
*/
|
||||
export async function calculateWorkerCount(config: GitNexusConfig): Promise<WorkerCalculationResult> {
|
||||
const systemInfo = getSystemInfo();
|
||||
const workerConfig = config.processing.workers;
|
||||
const limitations: string[] = [];
|
||||
|
||||
let workerCount: number;
|
||||
let reasoning: string;
|
||||
|
||||
// Log system information
|
||||
console.log('🖥️ System Information:');
|
||||
console.log(` CPU cores: ${systemInfo.cpuCores}`);
|
||||
if (systemInfo.availableMemoryMB) {
|
||||
console.log(` Available memory: ${systemInfo.availableMemoryMB}MB`);
|
||||
} else {
|
||||
console.log(' Available memory: Unknown (performance.memory not available)');
|
||||
}
|
||||
|
||||
if (workerConfig.mode === 'manual') {
|
||||
workerCount = workerConfig.manual.count;
|
||||
reasoning = `Manual mode: ${workerCount} workers configured`;
|
||||
|
||||
console.log('🎛️ Worker Mode: MANUAL');
|
||||
console.log(` Configured workers: ${workerCount}`);
|
||||
} else {
|
||||
// Auto mode calculation
|
||||
const { cpuCores, availableMemoryMB } = systemInfo;
|
||||
const { maxWorkers, memoryPerWorkerMB, cpuMultiplier } = workerConfig.auto;
|
||||
|
||||
console.log('🤖 Worker Mode: AUTO');
|
||||
console.log(` CPU multiplier: ${cpuMultiplier}`);
|
||||
console.log(` Memory per worker: ${memoryPerWorkerMB}MB`);
|
||||
console.log(` Max workers limit: ${maxWorkers}`);
|
||||
|
||||
// CPU-based calculation
|
||||
const cpuBasedWorkers = Math.floor(cpuCores * cpuMultiplier);
|
||||
limitations.push(`CPU: ${cpuBasedWorkers} (${cpuCores} cores × ${cpuMultiplier})`);
|
||||
|
||||
// Memory-based calculation (if available)
|
||||
let memoryBasedWorkers = maxWorkers;
|
||||
if (availableMemoryMB) {
|
||||
memoryBasedWorkers = Math.floor(availableMemoryMB / memoryPerWorkerMB);
|
||||
limitations.push(`Memory: ${memoryBasedWorkers} (${availableMemoryMB}MB ÷ ${memoryPerWorkerMB}MB per worker)`);
|
||||
} else {
|
||||
limitations.push(`Memory: Unknown (using max limit: ${maxWorkers})`);
|
||||
}
|
||||
|
||||
// Take the minimum of all constraints
|
||||
workerCount = Math.min(cpuBasedWorkers, memoryBasedWorkers, maxWorkers);
|
||||
limitations.push(`Config max: ${maxWorkers}`);
|
||||
|
||||
// Safety minimum
|
||||
workerCount = Math.max(1, workerCount);
|
||||
|
||||
reasoning = `Auto mode: min(${limitations.join(', ')}) = ${workerCount}`;
|
||||
|
||||
console.log(' Calculation breakdown:');
|
||||
limitations.forEach(limitation => console.log(` ${limitation}`));
|
||||
}
|
||||
|
||||
console.log(`🚀 Final Result: ${workerCount} workers enabled`);
|
||||
console.log(` Reasoning: ${reasoning}`);
|
||||
|
||||
// Display configuration banner
|
||||
const { displayWorkerBanner } = await import('./worker-banner.ts');
|
||||
displayWorkerBanner(config, workerCount);
|
||||
|
||||
return {
|
||||
workerCount,
|
||||
reasoning,
|
||||
limitations
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current system information for debugging
|
||||
*/
|
||||
export function getSystemInfoForDebug() {
|
||||
const systemInfo = getSystemInfo();
|
||||
return {
|
||||
...systemInfo,
|
||||
memoryInfo: (performance as PerformanceWithMemory).memory ? {
|
||||
usedMB: Math.round((performance as PerformanceWithMemory).memory!.usedJSHeapSize / (1024 * 1024)),
|
||||
totalMB: Math.round((performance as PerformanceWithMemory).memory!.totalJSHeapSize / (1024 * 1024)),
|
||||
limitMB: Math.round((performance as PerformanceWithMemory).memory!.jsHeapSizeLimit / (1024 * 1024))
|
||||
} : null
|
||||
};
|
||||
}
|
||||
|
|
@ -14,13 +14,8 @@ import { WebWorkerPoolUtils } from './web-worker-pool.js';
|
|||
export function initializeWorkerPoolCleanup(): void {
|
||||
console.log('Initializing worker pool cleanup handlers...');
|
||||
|
||||
// Setup global cleanup handlers
|
||||
WebWorkerPoolUtils.setupGlobalCleanup();
|
||||
|
||||
// Start periodic memory monitoring (every 2 minutes)
|
||||
setInterval(() => {
|
||||
WebWorkerPoolUtils.monitorMemoryUsage();
|
||||
}, 120000);
|
||||
// Memory monitoring is now handled by MemoryManager and individual processors
|
||||
console.log('💾 Memory monitoring delegated to MemoryManager');
|
||||
|
||||
console.log('Worker pool cleanup handlers initialized');
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue