diff --git a/.env b/.env index b65c471ec..eeba95568 100644 --- a/.env +++ b/.env @@ -1,39 +1,10 @@ # ======================================== -# ENGINE CONFIGURATION +# PROCESSING CONFIGURATION # ======================================== -# Default engine to use ('legacy' or 'nextgen') -ENGINE_DEFAULT=nextgen - -# Enable/disable engines -ENGINE_LEGACY_ENABLED=true -ENGINE_NEXTGEN_ENABLED=true - -# Allow fallback between engines if one fails -ENGINE_ALLOW_FALLBACK=true - # Enable performance monitoring -ENGINE_PERFORMANCE_MONITORING=true - -# Auto engine selection based on repository size (experimental) -ENGINE_AUTO_SELECTION=false - -# Time threshold before falling back to another engine (ms) -ENGINE_FALLBACK_THRESHOLD_MS=10000 - -# ======================================== -# LEGACY ENGINE CONFIGURATION -# ======================================== - -# Memory limits for Legacy engine -ENGINE_LEGACY_MEMORY_LIMIT_MB=512 -ENGINE_LEGACY_GC_INTERVAL_MS=30000 - -# Processing settings for Legacy engine -ENGINE_LEGACY_BATCH_SIZE=10 -ENGINE_LEGACY_TIMEOUT_MS=30000 -ENGINE_LEGACY_USE_WORKERS=true +PROCESSING_PERFORMANCE_MONITORING=true # ======================================== # PARALLEL PROCESSING CONFIGURATION diff --git a/gitnexus.config.ts b/gitnexus.config.ts new file mode 100644 index 000000000..15e805ff7 --- /dev/null +++ b/gitnexus.config.ts @@ -0,0 +1,307 @@ +/** + * GitNexus Configuration + * + * Centralized configuration file for all GitNexus settings. + * This replaces scattered .env variables and hardcoded values. + * + * Environment variables can still override these values for deployment. + */ + +export interface GitNexusConfig { + // ======================================== + // PROCESSING CONFIGURATION + // ======================================== + processing: { + mode: 'parallel' | 'single'; + parallel: { + maxWorkers: number; + batchSize: number; + workerTimeoutMs: number; + }; + memory: { + maxMB: number; + cleanupThresholdMB: number; + gcIntervalMs: number; + maxFileSizeMB: number; + maxFilesInMemory: number; + }; + fileExtensions: string[]; + performanceMonitoring: boolean; + }; + + // ======================================== + // KUZU DB CONFIGURATION + // ======================================== + kuzu: { + enabled: boolean; + persistence: boolean; + dualWrite: boolean; + fallbackToJson: boolean; + performance: { + enableCache: boolean; + cacheSize: number; + queryTimeout: number; + }; + }; + + // ======================================== + // AI & QUERY CONFIGURATION + // ======================================== + ai: { + cypher: { + defaultLimit: number; + maxLimit: number; + timeoutMs: number; + enableValidation: boolean; + enableLimiting: boolean; // Enable/disable automatic LIMIT addition + enableTruncation: boolean; // Enable/disable response truncation + }; + llm: { + defaultProvider: 'openai' | 'azure' | 'anthropic' | 'gemini'; + providers: { + openai?: { + apiKey?: string; + model: string; + maxTokens: number; + temperature: number; + }; + azure?: { + apiKey?: string; + endpoint?: string; + deployment?: string; + maxTokens: number; + temperature: number; + }; + anthropic?: { + apiKey?: string; + model: string; + maxTokens: number; + temperature: number; + }; + gemini?: { + apiKey?: string; + model: string; + maxTokens: number; + temperature: number; + }; + }; + }; + }; + + // ======================================== + // IGNORE PATTERNS (CENTRALIZED!) + // ======================================== + ignore: { + enabled: boolean; + patterns: string[]; + suffixes: string[]; + fileExtensions: string[]; + customPatterns: string[]; + }; + + // ======================================== + // LOGGING & DEBUGGING + // ======================================== + logging: { + level: 'debug' | 'info' | 'warn' | 'error'; + enableMetrics: boolean; + enablePerformance: boolean; + maxEntries: number; + monitoringIntervalMs: number; + }; + + // ======================================== + // GITHUB INTEGRATION + // ======================================== + github: { + token?: string; + apiUrl: string; + rateLimit: { + maxRequests: number; + windowMs: number; + }; + retry: { + maxRetries: number; + backoffMs: number; + }; + }; + + // ======================================== + // ENVIRONMENT & DEPLOYMENT + // ======================================== + environment: 'development' | 'staging' | 'production'; +} + +/** + * Default GitNexus Configuration + * + * These are the default values. Environment variables can override them. + */ +const config: GitNexusConfig = { + + // ======================================== + // PROCESSING CONFIGURATION + // ======================================== + processing: { + mode: (process.env.VITE_PARSING_MODE as 'parallel' | 'single') ?? 'parallel', + parallel: { + maxWorkers: parseInt(process.env.PARALLEL_MAX_WORKERS ?? '4'), + batchSize: parseInt(process.env.PARALLEL_BATCH_SIZE ?? '20'), + workerTimeoutMs: parseInt(process.env.PARALLEL_WORKER_TIMEOUT_MS ?? '60000') + }, + memory: { + maxMB: parseInt(process.env.MEMORY_MAX_MB ?? '512'), + cleanupThresholdMB: parseInt(process.env.MEMORY_CLEANUP_THRESHOLD_MB ?? '400'), + gcIntervalMs: parseInt(process.env.MEMORY_GC_INTERVAL_MS ?? '30000'), + maxFileSizeMB: parseInt(process.env.MEMORY_MAX_FILE_SIZE_MB ?? '10'), + maxFilesInMemory: parseInt(process.env.MEMORY_MAX_FILES ?? '1000') + }, + fileExtensions: process.env.PROCESSING_FILE_EXTENSIONS?.split(',') ?? [ + '.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', + '.cs', '.php', '.rb', '.go', '.rs', '.swift', '.kt', '.scala', '.dart', + '.json', '.yaml', '.yml', '.xml', '.toml', '.ini', '.cfg', '.properties' + ], + performanceMonitoring: process.env.PROCESSING_PERFORMANCE_MONITORING !== 'false' + }, + + // ======================================== + // KUZU DB CONFIGURATION + // ======================================== + kuzu: { + enabled: process.env.VITE_KUZU_ENABLED === 'true', + persistence: process.env.KUZU_PERSISTENCE !== 'false', + dualWrite: process.env.KUZU_DUAL_WRITE !== 'false', + fallbackToJson: process.env.KUZU_FALLBACK_JSON !== 'false', + performance: { + enableCache: process.env.KUZU_ENABLE_CACHE !== 'false', + cacheSize: parseInt(process.env.KUZU_CACHE_SIZE ?? '1000'), + queryTimeout: parseInt(process.env.KUZU_QUERY_TIMEOUT ?? '30000') + } + }, + + // ======================================== + // AI & QUERY CONFIGURATION + // ======================================== + ai: { + cypher: { + defaultLimit: parseInt(process.env.AI_CYPHER_DEFAULT_LIMIT ?? '20'), + maxLimit: parseInt(process.env.AI_CYPHER_MAX_LIMIT ?? '100'), + timeoutMs: parseInt(process.env.AI_CYPHER_TIMEOUT_MS ?? '30000'), + enableValidation: process.env.AI_CYPHER_ENABLE_VALIDATION !== 'false', + enableLimiting: process.env.AI_CYPHER_ENABLE_LIMITING !== 'false', + enableTruncation: process.env.AI_CYPHER_ENABLE_TRUNCATION !== 'false' + }, + llm: { + defaultProvider: (process.env.LLM_DEFAULT_PROVIDER as 'openai' | 'azure' | 'anthropic' | 'gemini') ?? 'openai', + providers: { + openai: process.env.OPENAI_API_KEY ? { + apiKey: process.env.OPENAI_API_KEY, + model: process.env.OPENAI_MODEL ?? 'gpt-4o-mini', + maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS ?? '4000'), + temperature: parseFloat(process.env.OPENAI_TEMPERATURE ?? '0.1') + } : undefined, + azure: process.env.AZURE_API_KEY ? { + apiKey: process.env.AZURE_API_KEY, + endpoint: process.env.AZURE_ENDPOINT, + deployment: process.env.AZURE_DEPLOYMENT, + maxTokens: parseInt(process.env.AZURE_MAX_TOKENS ?? '4000'), + temperature: parseFloat(process.env.AZURE_TEMPERATURE ?? '0.1') + } : undefined, + anthropic: process.env.ANTHROPIC_API_KEY ? { + apiKey: process.env.ANTHROPIC_API_KEY, + model: process.env.ANTHROPIC_MODEL ?? 'claude-3-sonnet-20240229', + maxTokens: parseInt(process.env.ANTHROPIC_MAX_TOKENS ?? '4000'), + temperature: parseFloat(process.env.ANTHROPIC_TEMPERATURE ?? '0.1') + } : undefined, + gemini: process.env.GEMINI_API_KEY ? { + apiKey: process.env.GEMINI_API_KEY, + model: process.env.GEMINI_MODEL ?? 'gemini-pro', + maxTokens: parseInt(process.env.GEMINI_MAX_TOKENS ?? '4000'), + temperature: parseFloat(process.env.GEMINI_TEMPERATURE ?? '0.1') + } : undefined + } + } + }, + + // ======================================== + // IGNORE PATTERNS (CENTRALIZED!) + // ======================================== + ignore: { + enabled: process.env.IGNORE_ENABLED !== 'false', + patterns: process.env.IGNORE_PATTERNS?.split(',') ?? [ + // Version Control + '.git', '.svn', '.hg', + // Package Managers & Dependencies + 'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps', + // Python Virtual Environments & Cache + 'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__', + '.pytest_cache', '.mypy_cache', '.tox', + // Build & Distribution Directories + 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', + // Static Assets and Public Directories + 'public', 'assets', 'static', + // IDE & Editor Directories + '.vs', '.vscode', '.idea', '.eclipse', '.settings', + // Temporary & Log Directories + 'tmp', '.tmp', 'temp', 'logs', 'log', + // Coverage & Testing + 'coverage', '.coverage', 'htmlcov', '.nyc_output', + // OS & System + '.DS_Store', 'Thumbs.db', + // Documentation Build Output + '_site', '.docusaurus', + // Cache Directories + '.cache', '.parcel-cache', '.next', '.nuxt' + ], + suffixes: process.env.IGNORE_SUFFIXES?.split(',') ?? ['.tmp', '~', '.bak', '.swp', '.swo'], + fileExtensions: process.env.IGNORE_FILE_EXTENSIONS?.split(',') ?? [ + // Compiled/Binary + '.pyc', '.pyo', '.pyd', '.so', '.dll', '.exe', '.jar', '.war', '.ear', + // Archives + '.zip', '.tar', '.rar', '.7z', '.gz', + // Media + '.jpg', '.jpeg', '.png', '.gif', '.bmp', '.svg', '.ico', '.mp4', '.avi', '.mp3', '.wav', + // Documents + '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + // Fonts + '.woff', '.woff2', '.ttf', '.eot', '.otf', + // Minified/Generated + '.min.js', '.min.css', '.map' + ], + customPatterns: process.env.IGNORE_CUSTOM_PATTERNS?.split(',') ?? [] + }, + + // ======================================== + // LOGGING & DEBUGGING + // ======================================== + logging: { + level: (process.env.LOG_LEVEL as 'debug' | 'info' | 'warn' | 'error') ?? 'info', + enableMetrics: process.env.LOG_ENABLE_METRICS !== 'false', + enablePerformance: process.env.LOG_ENABLE_PERFORMANCE !== 'false', + maxEntries: parseInt(process.env.LOG_MAX_ENTRIES ?? '1000'), + monitoringIntervalMs: parseInt(process.env.LOG_MONITORING_INTERVAL_MS ?? '30000') + }, + + // ======================================== + // GITHUB INTEGRATION + // ======================================== + github: { + token: process.env.GITHUB_TOKEN, + apiUrl: process.env.GITHUB_API_URL ?? 'https://api.github.com', + rateLimit: { + maxRequests: parseInt(process.env.GITHUB_RATE_LIMIT_MAX ?? '60'), + windowMs: parseInt(process.env.GITHUB_RATE_LIMIT_WINDOW_MS ?? '60000') + }, + retry: { + maxRetries: parseInt(process.env.GITHUB_RETRY_MAX ?? '3'), + backoffMs: parseInt(process.env.GITHUB_RETRY_BACKOFF_MS ?? '1000') + } + }, + + // ======================================== + // ENVIRONMENT & DEPLOYMENT + // ======================================== + environment: (process.env.NODE_ENV as 'development' | 'staging' | 'production') ?? 'development' +}; + +export default config; diff --git a/scripts/migrate-config.ts b/scripts/migrate-config.ts new file mode 100644 index 000000000..2e9455914 --- /dev/null +++ b/scripts/migrate-config.ts @@ -0,0 +1,135 @@ +#!/usr/bin/env tsx +/** + * Configuration Migration Script + * + * Helps migrate from .env variables to the new gitnexus.config.ts system. + * This script analyzes your current .env file and suggests config updates. + */ + +import { readFileSync, existsSync } from 'fs'; +import { join } from 'path'; + +interface EnvMapping { + envKey: string; + configPath: string; + transform?: (value: string) => any; +} + +const ENV_MAPPINGS: EnvMapping[] = [ + // Engine Configuration + { envKey: 'ENGINE_DEFAULT', configPath: 'engine.default' }, + { envKey: 'ENGINE_LEGACY_ENABLED', configPath: 'engine.legacy.enabled', transform: (v) => v === 'true' }, + { envKey: 'ENGINE_LEGACY_MEMORY_LIMIT_MB', configPath: 'engine.legacy.memoryLimitMB', transform: parseInt }, + { envKey: 'ENGINE_LEGACY_BATCH_SIZE', configPath: 'engine.legacy.batchSize', transform: parseInt }, + + // Processing Configuration + { envKey: 'VITE_PARSING_MODE', configPath: 'processing.mode' }, + { envKey: 'PARALLEL_MAX_WORKERS', configPath: 'processing.parallel.maxWorkers', transform: parseInt }, + { envKey: 'PARALLEL_BATCH_SIZE', configPath: 'processing.parallel.batchSize', transform: parseInt }, + { envKey: 'MEMORY_MAX_MB', configPath: 'processing.memory.maxMB', transform: parseInt }, + + // KuzuDB Configuration + { envKey: 'VITE_KUZU_ENABLED', configPath: 'kuzu.enabled', transform: (v) => v === 'true' }, + + // Logging Configuration + { envKey: 'LOG_LEVEL', configPath: 'logging.level' }, + { envKey: 'LOG_ENABLE_METRICS', configPath: 'logging.enableMetrics', transform: (v) => v === 'true' }, + + // GitHub Configuration + { envKey: 'GITHUB_TOKEN', configPath: 'github.token' }, + { envKey: 'GITHUB_API_URL', configPath: 'github.apiUrl' } +]; + +function parseEnvFile(filePath: string): Record { + if (!existsSync(filePath)) { + console.log(`āŒ .env file not found at ${filePath}`); + return {}; + } + + const content = readFileSync(filePath, 'utf-8'); + const env: Record = {}; + + content.split('\n').forEach(line => { + line = line.trim(); + if (line && !line.startsWith('#')) { + const [key, ...valueParts] = line.split('='); + if (key && valueParts.length > 0) { + env[key.trim()] = valueParts.join('=').trim(); + } + } + }); + + return env; +} + +function generateConfigSuggestions(env: Record): string { + const suggestions: string[] = []; + const foundMappings: EnvMapping[] = []; + + // Find matching environment variables + ENV_MAPPINGS.forEach(mapping => { + if (env[mapping.envKey]) { + foundMappings.push(mapping); + } + }); + + if (foundMappings.length === 0) { + return 'No matching environment variables found for migration.'; + } + + suggestions.push('// Suggested updates for gitnexus.config.ts based on your .env file:\n'); + + foundMappings.forEach(mapping => { + const envValue = env[mapping.envKey]; + const transformedValue = mapping.transform ? mapping.transform(envValue) : `'${envValue}'`; + + suggestions.push(`// ${mapping.envKey}=${envValue}`); + suggestions.push(`${mapping.configPath}: ${transformedValue},\n`); + }); + + return suggestions.join('\n'); +} + +function analyzeCurrentConfig(): void { + console.log('šŸ” GitNexus Configuration Migration Analysis\n'); + + // Check for .env file + const envPath = join(process.cwd(), '.env'); + const env = parseEnvFile(envPath); + + if (Object.keys(env).length === 0) { + console.log('No .env file found or it\'s empty.'); + return; + } + + console.log(`šŸ“‹ Found ${Object.keys(env).length} environment variables in .env\n`); + + // Generate suggestions + const suggestions = generateConfigSuggestions(env); + console.log('šŸ’” Configuration Suggestions:'); + console.log('=' .repeat(50)); + console.log(suggestions); + console.log('=' .repeat(50)); + + // Check if gitnexus.config.ts exists + const configPath = join(process.cwd(), 'gitnexus.config.ts'); + if (existsSync(configPath)) { + console.log('\nāœ… gitnexus.config.ts already exists'); + console.log('You can update it with the suggestions above.'); + } else { + console.log('\nāš ļø gitnexus.config.ts not found'); + console.log('Please create it first, then apply the suggestions above.'); + } + + // Show next steps + console.log('\nšŸ“ Next Steps:'); + console.log('1. Update gitnexus.config.ts with the suggested values'); + console.log('2. Test your application to ensure everything works'); + console.log('3. Consider removing the corresponding .env variables'); + console.log('4. Update your deployment scripts to use the new config system'); +} + +// Run the analysis +if (require.main === module) { + analyzeCurrentConfig(); +} diff --git a/src/ai/react-agent.ts b/src/ai/react-agent.ts index a41339552..259b2d79f 100644 --- a/src/ai/react-agent.ts +++ b/src/ai/react-agent.ts @@ -4,6 +4,7 @@ import type { LLMService, LLMConfig } from './llm-service.ts'; import type { CypherGenerator } from './cypher-generator.ts'; import type { KnowledgeGraph } from '../core/graph/types.ts'; import type { LocalStorageChatHistory } from '../lib/chat-history.ts'; +import { configLoader } from '../config/config-loader.ts'; export interface ReActContext { graph: KnowledgeGraph; @@ -380,21 +381,26 @@ CRITICAL: The action field must be exactly one of these four values: query_graph maxRetries: 3 }); - // Add LIMIT if not present to prevent JSON truncation issues + // Get config once for efficiency + const config = await configLoader.loadConfig(); + + // Add LIMIT if not present and limiting is enabled let finalCypher = cypherQuery.cypher; - if (!finalCypher.toLowerCase().includes('limit')) { - finalCypher += ' LIMIT 20'; + if (config.ai.cypher.enableLimiting && !finalCypher.toLowerCase().includes('limit')) { + const defaultLimit = config.ai.cypher.defaultLimit; + finalCypher += ` LIMIT ${defaultLimit}`; } // Execute the query const results = await this.executeGraphQuery(finalCypher); - // Truncate large responses to prevent JSON parsing issues - if (results.rows && results.rows.length > 20) { - results.rows = results.rows.slice(0, 20); - results.rowCount = 20; + // Truncate large responses if truncation is enabled + if (config.ai.cypher.enableTruncation && results.rows && results.rows.length > config.ai.cypher.maxLimit) { + const maxLimit = config.ai.cypher.maxLimit; + results.rows = results.rows.slice(0, maxLimit); + results.rowCount = maxLimit; results.truncated = true; - results.summary += ' (showing first 20 results)'; + results.summary += ` (showing first ${maxLimit} results)`; } output = JSON.stringify(results, null, 2); diff --git a/src/config/config-loader.ts b/src/config/config-loader.ts new file mode 100644 index 000000000..b42c7b579 --- /dev/null +++ b/src/config/config-loader.ts @@ -0,0 +1,279 @@ +/** + * Configuration Loader + * + * Loads and validates configuration from the root gitnexus.config.ts file + * with Zod schema validation and type safety. + */ + +import { z } from 'zod'; + +// Zod schemas for validation + +const ProcessingConfigSchema = z.object({ + mode: z.enum(['parallel', 'single']), + parallel: z.object({ + maxWorkers: z.number().min(1).max(16), + batchSize: z.number().min(1).max(100), + workerTimeoutMs: z.number().min(10000).max(300000) + }), + memory: z.object({ + maxMB: z.number().min(100).max(4096), + cleanupThresholdMB: z.number().min(50).max(2048), + gcIntervalMs: z.number().min(5000).max(60000), + maxFileSizeMB: z.number().min(1).max(100), + maxFilesInMemory: z.number().min(100).max(10000) + }), + fileExtensions: z.array(z.string()), + performanceMonitoring: z.boolean() +}); + +const KuzuConfigSchema = z.object({ + enabled: z.boolean(), + persistence: z.boolean(), + dualWrite: z.boolean(), + fallbackToJson: z.boolean(), + performance: z.object({ + enableCache: z.boolean(), + cacheSize: z.number().min(100).max(10000), + queryTimeout: z.number().min(1000).max(60000) + }) +}); + +const AIConfigSchema = z.object({ + cypher: z.object({ + defaultLimit: z.number().min(1).max(1000), + maxLimit: z.number().min(10).max(1000), + timeoutMs: z.number().min(1000).max(60000), + enableValidation: z.boolean(), + enableLimiting: z.boolean(), + enableTruncation: z.boolean() + }), + llm: z.object({ + defaultProvider: z.enum(['openai', 'azure', 'anthropic', 'gemini']), + providers: z.object({ + openai: z.object({ + apiKey: z.string().optional(), + model: z.string(), + maxTokens: z.number().min(100).max(10000), + temperature: z.number().min(0).max(2) + }).optional(), + azure: z.object({ + apiKey: z.string().optional(), + endpoint: z.string().optional(), + deployment: z.string().optional(), + maxTokens: z.number().min(100).max(10000), + temperature: z.number().min(0).max(2) + }).optional(), + anthropic: z.object({ + apiKey: z.string().optional(), + model: z.string(), + maxTokens: z.number().min(100).max(10000), + temperature: z.number().min(0).max(2) + }).optional(), + gemini: z.object({ + apiKey: z.string().optional(), + model: z.string(), + maxTokens: z.number().min(100).max(10000), + temperature: z.number().min(0).max(2) + }).optional() + }) + }) +}); + +const IgnoreConfigSchema = z.object({ + enabled: z.boolean(), + patterns: z.array(z.string()), + suffixes: z.array(z.string()), + fileExtensions: z.array(z.string()), + customPatterns: z.array(z.string()) +}); + +const LoggingConfigSchema = z.object({ + level: z.enum(['debug', 'info', 'warn', 'error']), + enableMetrics: z.boolean(), + enablePerformance: z.boolean(), + maxEntries: z.number().min(100).max(10000), + monitoringIntervalMs: z.number().min(5000).max(60000) +}); + +const GitHubConfigSchema = z.object({ + token: z.string().optional(), + apiUrl: z.string().url(), + rateLimit: z.object({ + maxRequests: z.number().min(1).max(5000), + windowMs: z.number().min(1000).max(3600000) + }), + retry: z.object({ + maxRetries: z.number().min(0).max(10), + backoffMs: z.number().min(100).max(10000) + }) +}); + +const GitNexusConfigSchema = z.object({ + processing: ProcessingConfigSchema, + kuzu: KuzuConfigSchema, + ai: AIConfigSchema, + ignore: IgnoreConfigSchema, + logging: LoggingConfigSchema, + github: GitHubConfigSchema, + environment: z.enum(['development', 'staging', 'production']) +}); + +export type ValidatedGitNexusConfig = z.infer; + +/** + * Configuration Loader Service + */ +export class ConfigLoader { + private static instance: ConfigLoader; + private config: ValidatedGitNexusConfig | null = null; + private validationErrors: string[] = []; + + private constructor() {} + + public static getInstance(): ConfigLoader { + if (!ConfigLoader.instance) { + ConfigLoader.instance = new ConfigLoader(); + } + return ConfigLoader.instance; + } + + /** + * Load and validate configuration from gitnexus.config.ts + */ + public async loadConfig(): Promise { + if (this.config) { + return this.config; + } + + try { + // Import the config file + const configModule = await import('../../gitnexus.config.ts'); + const rawConfig = configModule.default; + + // Validate with Zod + const result = GitNexusConfigSchema.safeParse(rawConfig); + + if (!result.success) { + this.validationErrors = result.error.errors.map(e => + `${e.path.join('.')}: ${e.message}` + ); + console.error('āŒ Configuration validation errors:', this.validationErrors); + + // Return a minimal valid config as fallback + this.config = this.getMinimalConfig(); + } else { + this.config = result.data; + console.log('āœ… Configuration loaded and validated successfully'); + } + + return this.config; + } catch (error) { + console.error('āŒ Failed to load gitnexus.config.ts:', error); + this.config = this.getMinimalConfig(); + return this.config; + } + } + + /** + * Get minimal fallback configuration + */ + private getMinimalConfig(): ValidatedGitNexusConfig { + return { + processing: { + mode: 'parallel', + parallel: { + maxWorkers: 4, + batchSize: 20, + workerTimeoutMs: 60000 + }, + memory: { + maxMB: 512, + cleanupThresholdMB: 400, + gcIntervalMs: 30000, + maxFileSizeMB: 10, + maxFilesInMemory: 1000 + }, + fileExtensions: ['.js', '.ts', '.jsx', '.tsx', '.py'], + performanceMonitoring: true + }, + kuzu: { + enabled: true, + persistence: true, + dualWrite: true, + fallbackToJson: true, + performance: { + enableCache: true, + cacheSize: 1000, + queryTimeout: 30000 + } + }, + ai: { + cypher: { + defaultLimit: 20, + maxLimit: 100, + timeoutMs: 30000, + enableValidation: true, + enableLimiting: true, + enableTruncation: true + }, + llm: { + defaultProvider: 'openai', + providers: {} + } + }, + ignore: { + enabled: true, + patterns: ['node_modules', '.git', 'build', 'dist'], + suffixes: ['.tmp', '~'], + fileExtensions: ['.pyc', '.zip', '.jpg'], + customPatterns: [] + }, + logging: { + level: 'info', + enableMetrics: true, + enablePerformance: true, + maxEntries: 1000, + monitoringIntervalMs: 30000 + }, + github: { + apiUrl: 'https://api.github.com', + rateLimit: { + maxRequests: 60, + windowMs: 60000 + }, + retry: { + maxRetries: 3, + backoffMs: 1000 + } + }, + environment: 'development' + }; + } + + /** + * Get current configuration + */ + public getConfig(): ValidatedGitNexusConfig | null { + return this.config; + } + + /** + * Get validation errors + */ + public getValidationErrors(): string[] { + return [...this.validationErrors]; + } + + /** + * Reload configuration (useful for development) + */ + public async reloadConfig(): Promise { + this.config = null; + this.validationErrors = []; + return this.loadConfig(); + } +} + +// Export singleton instance +export const configLoader = ConfigLoader.getInstance(); diff --git a/src/config/config.ts b/src/config/config.ts index 2ba564f14..359042f47 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -1,9 +1,16 @@ /** - * Centralized Configuration System - * Provides type-safe configuration with environment validation and defaults + * Legacy Configuration System (DEPRECATED) + * + * This file is being replaced by the new centralized config system: + * - Root config: gitnexus.config.ts + * - Config loader: src/config/config-loader.ts + * - Ignore service: src/config/ignore-service.ts + * + * TODO: Migrate remaining usage to the new system */ import { z } from 'zod'; +import { configLoader, type ValidatedGitNexusConfig } from './config-loader.ts'; // Configuration schemas with validation const MemoryConfigSchema = z.object({ @@ -99,16 +106,32 @@ export type ProcessingConfig = z.infer; export type LoggingConfig = z.infer; /** - * Configuration service with environment validation + * Configuration service with environment validation (LEGACY) + * + * This is a compatibility layer that bridges to the new centralized config system. + * New code should use configLoader directly. */ export class ConfigService { private static instance: ConfigService; private config: AppConfig; private validationErrors: string[] = []; + private newConfig: ValidatedGitNexusConfig | null = null; private constructor() { this.config = this.loadConfiguration(); this.validateEnvironment(); + this.loadNewConfig(); + } + + /** + * Load the new centralized configuration + */ + private async loadNewConfig(): Promise { + try { + this.newConfig = await configLoader.loadConfig(); + } catch (error) { + console.warn('Failed to load new config system, using legacy config:', error); + } } public static getInstance(): ConfigService { diff --git a/src/config/ignore-service.ts b/src/config/ignore-service.ts new file mode 100644 index 000000000..c5ef7e6b7 --- /dev/null +++ b/src/config/ignore-service.ts @@ -0,0 +1,284 @@ +/** + * Centralized Ignore Service + * + * Provides robust file and directory filtering based on centralized configuration. + * Implements the same logic as the Python example with directory-component matching. + */ + +import { configLoader, type ValidatedGitNexusConfig } from './config-loader.ts'; + +export class IgnoreService { + private static instance: IgnoreService; + private config: ValidatedGitNexusConfig | null = null; + private patternsSet: Set = new Set(); + private suffixesSet: Set = new Set(); + private extensionsSet: Set = new Set(); + private customRegexes: RegExp[] = []; + + private constructor() {} + + public static getInstance(): IgnoreService { + if (!IgnoreService.instance) { + IgnoreService.instance = new IgnoreService(); + } + return IgnoreService.instance; + } + + /** + * Initialize the ignore service with configuration + */ + public async initialize(): Promise { + this.config = await configLoader.loadConfig(); + this.updatePatterns(); + } + + /** + * Update internal pattern sets from configuration + */ + private updatePatterns(): void { + if (!this.config) return; + + const { ignore } = this.config; + + // Convert patterns to lowercase Set for O(1) lookup + this.patternsSet = new Set(ignore.patterns.map(p => p.toLowerCase())); + this.suffixesSet = new Set(ignore.suffixes.map(s => s.toLowerCase())); + this.extensionsSet = new Set(ignore.fileExtensions.map(e => e.toLowerCase())); + + // Compile custom regex patterns + this.customRegexes = ignore.customPatterns + .map(pattern => { + try { + return new RegExp(pattern, 'i'); + } catch (error) { + console.warn(`Invalid regex pattern: ${pattern}`, error); + return null; + } + }) + .filter((regex): regex is RegExp => regex !== null); + + console.log(`šŸ”§ IgnoreService initialized with ${this.patternsSet.size} patterns, ${this.suffixesSet.size} suffixes, ${this.extensionsSet.size} extensions`); + } + + /** + * Check if a file path should be ignored + * + * Uses directory-component matching like the Python example: + * - Splits path into components + * - Checks each component against ignore patterns + * - Prevents false positives (e.g., "my_node_modules_notes.txt") + */ + public shouldIgnorePath(filePath: string): boolean { + if (!this.config?.ignore.enabled) { + return false; + } + + // Normalize path separators and remove leading/trailing slashes + const normalizedPath = filePath.replace(/\\/g, '/').replace(/^\/+|\/+$/g, ''); + + // Split path into components for directory-part matching + const pathComponents = normalizedPath.split('/').filter(Boolean); + + // Check each component against ignore patterns (case-insensitive) + for (const component of pathComponents) { + const lowerComponent = component.toLowerCase(); + + if (this.patternsSet.has(lowerComponent)) { + return true; + } + } + + // Check file suffixes + const lowerPath = normalizedPath.toLowerCase(); + for (const suffix of this.suffixesSet) { + if (lowerPath.endsWith(suffix)) { + return true; + } + } + + // Check file extensions + const fileName = pathComponents[pathComponents.length - 1] || ''; + const lowerFileName = fileName.toLowerCase(); + + for (const ext of this.extensionsSet) { + if (lowerFileName.endsWith(ext)) { + return true; + } + } + + // Check custom regex patterns + for (const regex of this.customRegexes) { + if (regex.test(normalizedPath)) { + return true; + } + } + + return false; + } + + /** + * Filter an array of paths, removing ignored ones + * + * Optimized for batch processing with early returns + */ + public filterPaths(paths: string[]): string[] { + if (!this.config?.ignore.enabled) { + return paths; + } + + const startTime = performance.now(); + const filtered = paths.filter(path => !this.shouldIgnorePath(path)); + const endTime = performance.now(); + + const filteredCount = paths.length - filtered.length; + if (filteredCount > 0) { + console.log(`šŸš« IgnoreService filtered ${filteredCount}/${paths.length} paths in ${(endTime - startTime).toFixed(2)}ms`); + } + + return filtered; + } + + /** + * Check if a directory should be ignored + * + * Specialized method for directory filtering during traversal + */ + public shouldIgnoreDirectory(dirPath: string): boolean { + if (!this.config?.ignore.enabled) { + return false; + } + + // For directories, we only check the directory name itself + const dirName = dirPath.split('/').pop()?.toLowerCase() || ''; + return this.patternsSet.has(dirName); + } + + /** + * Get current ignore statistics + */ + public getStats(): { + enabled: boolean; + patterns: number; + suffixes: number; + extensions: number; + customPatterns: number; + } { + return { + enabled: this.config?.ignore.enabled ?? false, + patterns: this.patternsSet.size, + suffixes: this.suffixesSet.size, + extensions: this.extensionsSet.size, + customPatterns: this.customRegexes.length + }; + } + + /** + * Update ignore patterns at runtime + */ + public updateIgnorePatterns(updates: Partial): void { + if (!this.config) return; + + // Update configuration + this.config.ignore = { ...this.config.ignore, ...updates }; + + // Rebuild pattern sets + this.updatePatterns(); + + console.log('šŸ”„ IgnoreService patterns updated'); + } + + /** + * Add custom patterns at runtime + */ + public addCustomPatterns(patterns: string[], type: 'patterns' | 'suffixes' | 'extensions' | 'regex' = 'patterns'): void { + if (!this.config) return; + + switch (type) { + case 'patterns': + this.config.ignore.patterns.push(...patterns); + break; + case 'suffixes': + this.config.ignore.suffixes.push(...patterns); + break; + case 'extensions': + this.config.ignore.fileExtensions.push(...patterns); + break; + case 'regex': + this.config.ignore.customPatterns.push(...patterns); + break; + } + + this.updatePatterns(); + console.log(`āž• Added ${patterns.length} ${type} to IgnoreService`); + } + + /** + * Test a path against ignore patterns (for debugging) + */ + public testPath(filePath: string): { + ignored: boolean; + reason?: string; + matchedPattern?: string; + } { + if (!this.config?.ignore.enabled) { + return { ignored: false }; + } + + const normalizedPath = filePath.replace(/\\/g, '/').replace(/^\/+|\/+$/g, ''); + const pathComponents = normalizedPath.split('/').filter(Boolean); + + // Check directory components + for (const component of pathComponents) { + const lowerComponent = component.toLowerCase(); + if (this.patternsSet.has(lowerComponent)) { + return { + ignored: true, + reason: 'directory pattern', + matchedPattern: component + }; + } + } + + // Check suffixes + const lowerPath = normalizedPath.toLowerCase(); + for (const suffix of this.suffixesSet) { + if (lowerPath.endsWith(suffix)) { + return { + ignored: true, + reason: 'file suffix', + matchedPattern: suffix + }; + } + } + + // Check extensions + const fileName = pathComponents[pathComponents.length - 1] || ''; + const lowerFileName = fileName.toLowerCase(); + for (const ext of this.extensionsSet) { + if (lowerFileName.endsWith(ext)) { + return { + ignored: true, + reason: 'file extension', + matchedPattern: ext + }; + } + } + + // Check regex patterns + for (let i = 0; i < this.customRegexes.length; i++) { + const regex = this.customRegexes[i]; + if (regex.test(normalizedPath)) { + return { + ignored: true, + reason: 'custom regex', + matchedPattern: this.config.ignore.customPatterns[i] + }; + } + } + + return { ignored: false }; + } +} + +// Export singleton instance +export const ignoreService = IgnoreService.getInstance(); diff --git a/src/config/language-config.ts b/src/config/language-config.ts index f5473697a..ee7b928a3 100644 --- a/src/config/language-config.ts +++ b/src/config/language-config.ts @@ -161,42 +161,8 @@ const TYPESCRIPT_BUILTINS = new Set([ 'InstanceType', 'ThisParameterType', 'OmitThisParameter', 'ThisType' ]); -// Centralized ignore patterns -export const IGNORE_PATTERNS = new Set([ - // Version Control - '.git', '.svn', '.hg', - - // Package Managers & Dependencies - 'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps', - - // Python Virtual Environments & Cache - 'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__', - '.pytest_cache', '.mypy_cache', '.tox', - - // Build & Distribution Directories - 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', - - // Static Assets and Public Directories - 'public', 'assets', 'static', - - // IDE & Editor Directories - '.vs', '.vscode', '.idea', '.eclipse', '.settings', - - // Temporary & Log Directories - 'tmp', '.tmp', 'temp', 'logs', 'log', - - // Coverage & Testing - 'coverage', '.coverage', 'htmlcov', '.nyc_output', - - // OS & System - '.DS_Store', 'Thumbs.db', - - // Documentation Build Output - '_site', '.docusaurus', - - // Cache Directories - '.cache', '.parcel-cache', '.next', '.nuxt' -]); +// Note: Ignore patterns have been moved to the centralized IgnoreService +// See src/config/ignore-service.ts and gitnexus.config.ts // Language configurations export const LANGUAGE_CONFIGS: Record = { diff --git a/src/core/ingestion/parallel-parsing-processor.ts b/src/core/ingestion/parallel-parsing-processor.ts index 1c0d9a925..fd0dbf95a 100644 --- a/src/core/ingestion/parallel-parsing-processor.ts +++ b/src/core/ingestion/parallel-parsing-processor.ts @@ -6,7 +6,7 @@ import { DuplicateDetector, pathUtils } from '../../lib/shared-utils.js'; -import { IGNORE_PATTERNS } from '../../config/language-config.js'; +import { ignoreService } from '../../config/ignore-service.js'; import { WebWorkerPool, WebWorkerPoolUtils } from '../../lib/web-worker-pool.js'; import { FunctionRegistryTrie, FunctionDefinition } from '../graph/trie.js'; import { generateDeterministicId } from '../../lib/utils.ts'; @@ -478,54 +478,9 @@ export interface ParallelParsingResult { filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext))); } - // Apply ignore patterns (be more selective to avoid over-filtering) - EXACT MATCH TO SINGLE-THREADED + // Apply centralized ignore patterns const beforeIgnoreFilter = filtered.length; - filtered = filtered.filter(path => { - // More precise ignore pattern matching - for (const pattern of IGNORE_PATTERNS) { - if (typeof pattern === 'string') { - // Only ignore if the pattern is a complete directory component - if (path.includes(`/${pattern}/`) || - path.startsWith(`${pattern}/`) || - path.endsWith(`/${pattern}`) || - path === pattern) { - return false; - } - } - } - - // Additional filtering for files that shouldn't be in KG - const fileName = path.split('/').pop()?.toLowerCase() || ''; - - // Skip documentation and readme files - if (fileName.includes('readme') || - fileName.includes('license') || - fileName.includes('changelog') || - fileName.includes('contributing') || - fileName.includes('authors') || - fileName.includes('maintainers')) { - return false; - } - - // Skip git and version control files - if (fileName.startsWith('.git') || - fileName.includes('.gitignore') || - fileName.includes('.gitattributes')) { - return false; - } - - // Skip common non-source files - if (fileName.includes('dockerfile') || - fileName.includes('docker-compose') || - fileName.endsWith('.md') || - fileName.endsWith('.txt') || - fileName.endsWith('.log') || - fileName.endsWith('.lock')) { - return false; - } - - return true; - }); + filtered = ignoreService.filterPaths(filtered); // Apply content filter (only exclude truly empty files) const beforeContentFilter = filtered.length; diff --git a/src/core/ingestion/parsing-processor.ts b/src/core/ingestion/parsing-processor.ts index ba728e5f0..4eee8666b 100644 --- a/src/core/ingestion/parsing-processor.ts +++ b/src/core/ingestion/parsing-processor.ts @@ -7,7 +7,7 @@ import { DuplicateDetector, BatchProcessor } from '../../lib/shared-utils.js'; -import { IGNORE_PATTERNS } from '../../config/language-config.js'; +import { ignoreService } from '../../config/ignore-service.js'; import Parser from 'web-tree-sitter'; import { TYPESCRIPT_QUERIES, JAVASCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries'; import { initTreeSitter, loadTypeScriptParser, loadPythonParser, loadJavaScriptParser } from '../tree-sitter/parser-loader.js'; @@ -148,54 +148,9 @@ export class ParsingProcessor implements GraphProcessor { filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext))); } - // Apply ignore patterns (be more selective to avoid over-filtering) + // Apply centralized ignore patterns const beforeIgnoreFilter = filtered.length; - filtered = filtered.filter(path => { - // More precise ignore pattern matching - for (const pattern of IGNORE_PATTERNS) { - if (typeof pattern === 'string') { - // Only ignore if the pattern is a complete directory component - if (path.includes(`/${pattern}/`) || - path.startsWith(`${pattern}/`) || - path.endsWith(`/${pattern}`) || - path === pattern) { - return false; - } - } - } - - // Additional filtering for files that shouldn't be in KG - const fileName = path.split('/').pop()?.toLowerCase() || ''; - - // Skip documentation and readme files - if (fileName.includes('readme') || - fileName.includes('license') || - fileName.includes('changelog') || - fileName.includes('contributing') || - fileName.includes('authors') || - fileName.includes('maintainers')) { - return false; - } - - // Skip git and version control files - if (fileName.startsWith('.git') || - fileName.includes('.gitignore') || - fileName.includes('.gitattributes')) { - return false; - } - - // Skip common non-source files - if (fileName.includes('dockerfile') || - fileName.includes('docker-compose') || - fileName.endsWith('.md') || - fileName.endsWith('.txt') || - fileName.endsWith('.log') || - fileName.endsWith('.lock')) { - return false; - } - - return true; - }); + filtered = ignoreService.filterPaths(filtered); // Apply content filter (only exclude truly empty files) const beforeContentFilter = filtered.length; diff --git a/src/core/ingestion/structure-processor.ts b/src/core/ingestion/structure-processor.ts index caba1c23e..5ae6951ba 100644 --- a/src/core/ingestion/structure-processor.ts +++ b/src/core/ingestion/structure-processor.ts @@ -1,6 +1,7 @@ import type { KnowledgeGraph } from '../graph/graph.ts'; import type { GraphNode, GraphRelationship } from '../graph/types.ts'; import { generateDeterministicId } from '../../lib/utils.ts'; +import { ignoreService } from '../../config/ignore-service.js'; export interface StructureInput { projectRoot: string; @@ -16,42 +17,6 @@ export class StructureProcessor { relationshipsProcessed: 0 }; - // Import ignore patterns from ParsingProcessor - private static readonly IGNORE_PATTERNS = new Set([ - // Version Control - '.git', '.svn', '.hg', - - // Package Managers & Dependencies - 'node_modules', 'bower_components', 'jspm_packages', 'vendor', 'deps', - - // Python Virtual Environments & Cache - 'venv', 'env', '.venv', '.env', 'envs', 'virtualenv', '__pycache__', - '.pytest_cache', '.mypy_cache', '.tox', - - // Build & Distribution Directories - 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', - - // Static Assets and Public Directories - 'public', 'assets', 'static', - - // IDE & Editor Directories - '.vs', '.vscode', '.idea', '.eclipse', '.settings', - - // Temporary & Log Directories - 'tmp', '.tmp', 'temp', 'logs', 'log', - - // Coverage & Testing - 'coverage', '.coverage', 'htmlcov', '.nyc_output', - - // OS & System - '.DS_Store', 'Thumbs.db', - - // Documentation Build Output - '_site', '.docusaurus', - - // Cache Directories - '.cache', '.parcel-cache', '.next', '.nuxt' - ]); /** * Process complete repository structure directly from discovered paths @@ -328,80 +293,18 @@ export class StructureProcessor { /** * Check if a directory should be hidden from the KG visualization - * This matches the ignore patterns used in ParsingProcessor + * Uses centralized ignore service */ private shouldHideDirectory(dirPath: string): boolean { - const pathSegments = dirPath.split('/'); - - // Check if any segment of the path matches an ignore pattern - const hasIgnoredSegment = pathSegments.some(segment => - StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - if (hasIgnoredSegment) { - return true; - } - - // Additional pattern matching - const lowerPath = dirPath.toLowerCase(); - - // Hide Python egg-info directories - if (lowerPath.includes('.egg-info')) { - return true; - } - - // Hide site-packages directories - if (lowerPath.includes('site-packages')) { - return true; - } - - // Hide most hidden directories (except important ones like .github) - for (const segment of pathSegments) { - if (segment.startsWith('.') && segment !== '.github') { - return true; - } - } - - return false; + return ignoreService.shouldIgnoreDirectory(dirPath); } /** * Check if a file should be hidden from the KG visualization - * This matches the ignore patterns used in ParsingProcessor + * Uses centralized ignore service */ private shouldHideFile(filePath: string): boolean { - const pathSegments = filePath.split('/'); - - // Check if any segment of the path matches an ignore pattern - const hasIgnoredSegment = pathSegments.some(segment => - StructureProcessor.IGNORE_PATTERNS.has(segment.toLowerCase()) - ); - - if (hasIgnoredSegment) { - return true; - } - - // Additional pattern matching - const lowerPath = filePath.toLowerCase(); - - // Hide Python egg-info directories - if (lowerPath.includes('.egg-info')) { - return true; - } - - // Hide site-packages directories - if (lowerPath.includes('site-packages')) { - return true; - } - - // Hide most hidden directories (except important ones like .github) - for (const segment of pathSegments) { - if (segment.startsWith('.') && segment !== '.github') { - return true; - } - } - - return false; + return ignoreService.shouldIgnorePath(filePath); } /** diff --git a/src/lib/preload.ts b/src/lib/preload.ts index 0844fb192..2a5a91e24 100644 --- a/src/lib/preload.ts +++ b/src/lib/preload.ts @@ -10,5 +10,11 @@ import '../core/ingestion/parsing-processor'; import '../core/ingestion/call-processor'; import '../core/ingestion/structure-processor'; import '../core/tree-sitter/parser-loader'; +import { ignoreService } from '../config/ignore-service'; + +// Initialize ignore service +ignoreService.initialize().catch(error => { + console.warn('Failed to initialize ignore service:', error); +}); console.log('Dependencies preloaded to prevent runtime optimization'); \ No newline at end of file diff --git a/src/services/github.ts b/src/services/github.ts index fbcbb1602..778f098be 100644 --- a/src/services/github.ts +++ b/src/services/github.ts @@ -1,4 +1,5 @@ import axios, { type AxiosInstance, type AxiosResponse } from 'axios'; +import { ignoreService } from '../config/ignore-service.js'; interface GitHubFile { name: string; @@ -253,202 +254,16 @@ export class GitHubService { private shouldSkipDirectory(path: string): boolean { if (!path) return true; // Skip if path is undefined/null - - const skipDirs = [ - // Git version control - '.git', - // JavaScript dependencies (common in full-stack projects) - 'node_modules', - // Python bytecode cache - '__pycache__', - // Python virtual environments - 'venv', - 'env', - '.venv', - 'envs', - 'virtualenv', - // Build, distribution, and temporary directories - 'build', - 'dist', - 'logs', - 'tmp', - '.tmp', - // Static assets and public files - 'public', - 'assets', - 'static', - // Additional common directories to skip - 'coverage', - '.coverage', - 'htmlcov', - 'vendor', - 'deps', - '_build', - '.gradle', - 'bin', - 'obj', - '.vs', - '.vscode', - '.idea', - 'temp' - ]; - - // Check each directory component in the path - const pathParts = path.split('/'); - for (const part of pathParts) { - const dirName = part.toLowerCase(); - - // Check for exact matches - if (skipDirs.includes(dirName) || dirName.startsWith('.')) { - return true; - } - - // Check for .egg-info directories - if (dirName.endsWith('.egg-info')) { - return true; - } - } - - // Check for virtual environment patterns anywhere in the path - const fullPathLower = path.toLowerCase(); - const venvPatterns = [ - '/.venv/', - '/venv/', - '/env/', - '/.env/', - '/envs/', - '/virtualenv/', - '/site-packages/', - '/lib/python', - '/lib64/python', - '/scripts/', - '/bin/python' - ]; - - if (venvPatterns.some(pattern => fullPathLower.includes(pattern))) { - return true; - } - - return false; + return ignoreService.shouldIgnoreDirectory(path); } private shouldIncludeFile(path: string): boolean { if (!path) return false; // Skip if path is undefined/null - - const fileName = path.split('/').pop() || ''; - - // Skip hidden files except specific config files - if (fileName.startsWith('.') && !fileName.endsWith('.env.example')) { - return false; - } - - // Skip Python-specific file patterns - const skipPatterns = [ - // Python compiled bytecode - /\.pyc$/, - /\.pyo$/, - // Python extension modules (binary) - /\.pyd$/, - /\.so$/, - // Python packages - /\.egg$/, - /\.whl$/, - // Lock files - /\.lock$/, - /poetry\.lock$/, - /Pipfile\.lock$/, - // Editor swap files - /\..*\.swp$/, - /\..*\.swo$/, - // OS metadata files - /^Thumbs\.db$/, - /^\.DS_Store$/, - // General binary and archive files - /\.zip$/, - /\.tar$/, - /\.rar$/, - /\.7z$/, - /\.gz$/, - // Media files - /\.(jpg|jpeg|png|gif|bmp|svg|ico|tiff|webp)$/i, - /\.(mp4|avi|mov|wmv|flv|webm|mkv|m4v)$/i, - /\.(mp3|wav|flac|aac|ogg|m4a|wma)$/i, - // Document files - /\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i, - // Other binary files - /\.(exe|dll|dylib|wasm|jar|war|ear|deb|rpm|dmg|msi|pkg|apk|ipa)$/i, - // Font files - /\.(woff|woff2|ttf|eot|otf)$/i, - // Minified files and source maps - /\.min\.(js|css)$/, - /\.map$/, - // Log and temporary files - /\.log$/, - /\.tmp$/, - /\.cache$/, - /\.pid$/, - /\.seed$/ - ]; - - if (skipPatterns.some(pattern => pattern.test(fileName))) { - return false; - } - - // Include common source and important config files - const extension = '.' + (fileName.split('.').pop() || '').toLowerCase(); - - const includeSourceExts = new Set(['.py', '.js', '.jsx', '.ts', '.tsx']); - if (includeSourceExts.has(extension)) { - return true; - } - - const importantConfigFiles = new Set([ - // Python - 'pyproject.toml', 'setup.py', 'requirements.txt', 'setup.cfg', 'tox.ini', 'pytest.ini', 'pipfile', 'poetry.toml', - '__init__.py', - // JS/TS ecosystem - 'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', - 'tsconfig.json', 'tsconfig.base.json', - 'vite.config.ts', 'vite.config.js', - '.eslintrc', '.eslintrc.json', '.eslintrc.js', '.prettierrc', '.prettierrc.json', - // Docs/licenses - 'readme.md', 'license', 'changelog.md', 'manifest.in' - ]); - if (importantConfigFiles.has(fileName.toLowerCase())) { - return true; - } - - return false; + return !ignoreService.shouldIgnorePath(path); } private shouldSkipFileForContent(path: string): boolean { - const pathLower = path.toLowerCase(); - - // Skip .git files and directories (these can be massive and not useful for code analysis) - if (pathLower.includes('/.git/') || pathLower.startsWith('.git/')) { - return true; - } - - // Skip binary files that shouldn't have content - const binaryExtensions = [ - // Images - '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp', - // Fonts - '.woff', '.woff2', '.ttf', '.eot', '.otf', - // Documents - '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', - // Archives - '.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma', - // Executables & Libraries - '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm', - // Package formats - '.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa', - // Media - '.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v', - '.aac', '.ogg', '.flac', '.m4a', '.wma' - ]; - - return binaryExtensions.some(ext => pathLower.endsWith(ext)); + return ignoreService.shouldIgnorePath(path); } public async getAllPathsRecursively(owner: string, repo: string, path: string = ''): Promise { diff --git a/src/services/zip.ts b/src/services/zip.ts index b535c0ea8..6dd4db742 100644 --- a/src/services/zip.ts +++ b/src/services/zip.ts @@ -1,4 +1,5 @@ import JSZip from 'jszip'; +import { ignoreService } from '../config/ignore-service.js'; interface JSZipObjectWithData extends JSZip.JSZipObject { _data?: { @@ -378,216 +379,21 @@ export class ZipService { } private shouldSkipPath(filePath: string): boolean { - // Skip directories that shouldn't be processed - if (this.shouldSkipDirectory(filePath)) { - return true; - } - - // Skip files that shouldn't be processed - if (!this.shouldIncludeFile(filePath)) { - return true; - } - - return false; + return ignoreService.shouldIgnorePath(filePath); } private shouldSkipDirectory(path: string): boolean { if (!path) return true; // Skip if path is undefined/null - - const skipDirs = [ - // Git version control - '.git', - // JavaScript dependencies (common in full-stack projects) - 'node_modules', - // Python bytecode cache - '__pycache__', - // Python virtual environments - 'venv', - 'env', - '.venv', - 'envs', - 'virtualenv', - // Build, distribution, and temporary directories - 'build', - 'dist', - 'logs', - 'tmp', - '.tmp', - // Static assets and public files - 'public', - 'assets', - 'static', - // Additional common directories to skip - 'coverage', - '.coverage', - 'htmlcov', - 'vendor', - 'deps', - '_build', - '.gradle', - 'bin', - 'obj', - '.vs', - '.vscode', - '.idea', - 'temp' - ]; - - // Check each directory component in the path - const pathParts = path.split('/'); - for (const part of pathParts) { - const dirName = part.toLowerCase(); - - // Check for exact matches - if (skipDirs.includes(dirName) || dirName.startsWith('.')) { - return true; - } - - // Check for .egg-info directories - if (dirName.endsWith('.egg-info')) { - return true; - } - } - - // Check for virtual environment patterns anywhere in the path - const fullPathLower = path.toLowerCase(); - const venvPatterns = [ - '/.venv/', - '/venv/', - '/env/', - '/.env/', - '/envs/', - '/virtualenv/', - '/site-packages/', - '/lib/python', - '/lib64/python', - '/scripts/', - '/bin/python' - ]; - - if (venvPatterns.some(pattern => fullPathLower.includes(pattern))) { - return true; - } - - return false; + return ignoreService.shouldIgnoreDirectory(path); } private shouldIncludeFile(path: string): boolean { if (!path) return false; // Skip if path is undefined/null - - const fileName = path.split('/').pop() || ''; - - // Skip hidden files except specific config files - if (fileName.startsWith('.') && !fileName.endsWith('.env.example')) { - return false; - } - - // Skip common compiled/binary patterns - const skipPatterns = [ - // Python compiled bytecode - /\.pyc$/, - /\.pyo$/, - // Python extension modules (binary) - /\.pyd$/, - /\.so$/, - // Python packages - /\.egg$/, - /\.whl$/, - // Lock files - /\.lock$/, - /poetry\.lock$/, - /Pipfile\.lock$/, - // Editor swap files - /\..*\.swp$/, - /\..*\.swo$/, - // OS metadata files - /^Thumbs\.db$/, - /^\.DS_Store$/, - // General binary and archive files - /\.zip$/, - /\.tar$/, - /\.rar$/, - /\.7z$/, - /\.gz$/, - // Media files - /\.(jpg|jpeg|png|gif|bmp|svg|ico|tiff|webp)$/i, - /\.(mp4|avi|mov|wmv|flv|webm|mkv|m4v)$/i, - /\.(mp3|wav|flac|aac|ogg|m4a|wma)$/i, - // Document files - /\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i, - // Other binary files - /\.(exe|dll|dylib|wasm|jar|war|ear|deb|rpm|dmg|msi|pkg|apk|ipa)$/i, - // Font files - /\.(woff|woff2|ttf|eot|otf)$/i, - // Minified files and source maps - /\.min\.(js|css)$/, - /\.map$/, - // Log and temporary files - /\.log$/, - /\.tmp$/, - /\.cache$/, - /\.pid$/, - /\.seed$/ - ]; - - if (skipPatterns.some(pattern => pattern.test(fileName))) { - return false; - } - - // Include common source and important config files - const extension = '.' + (fileName.split('.').pop() || '').toLowerCase(); - const includeSourceExts = new Set(['.py', '.js', '.jsx', '.ts', '.tsx']); - if (includeSourceExts.has(extension)) { - return true; - } - - const importantConfigFiles = new Set([ - // Python - 'pyproject.toml', 'setup.py', 'requirements.txt', 'setup.cfg', 'tox.ini', 'pytest.ini', 'pipfile', 'poetry.toml', - '__init__.py', - // JS/TS ecosystem - 'package.json', 'package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', - 'tsconfig.json', 'tsconfig.base.json', - 'vite.config.ts', 'vite.config.js', - '.eslintrc', '.eslintrc.json', '.eslintrc.js', '.prettierrc', '.prettierrc.json', - // Docs/licenses - 'readme.md', 'license', 'changelog.md', 'manifest.in' - ]); - if (importantConfigFiles.has(fileName.toLowerCase())) { - return true; - } - - return false; + return !ignoreService.shouldIgnorePath(path); } private shouldSkipFileForContent(path: string): boolean { - const pathLower = path.toLowerCase(); - - // Skip .git files and directories (these can be massive and not useful for code analysis) - if (pathLower.includes('/.git/') || pathLower.startsWith('.git/')) { - return true; - } - - // Skip binary files that shouldn't have content - const binaryExtensions = [ - // Images - '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp', - // Fonts - '.woff', '.woff2', '.ttf', '.eot', '.otf', - // Documents - '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', - // Archives - '.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma', - // Executables & Libraries - '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm', - // Package formats - '.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa', - // Media - '.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v', - '.aac', '.ogg', '.flac', '.m4a', '.wma' - ]; - - return binaryExtensions.some(ext => pathLower.endsWith(ext)); + return ignoreService.shouldIgnorePath(path); } public getDefaultTextExtensions(): string[] { diff --git a/test-ignore-service.js b/test-ignore-service.js new file mode 100644 index 000000000..0519ecba6 --- /dev/null +++ b/test-ignore-service.js @@ -0,0 +1 @@ + \ No newline at end of file