diff --git a/src/config/language-config.ts b/src/config/language-config.ts index 42da4537d..f5473697a 100644 --- a/src/config/language-config.ts +++ b/src/config/language-config.ts @@ -176,6 +176,9 @@ export const IGNORE_PATTERNS = new Set([ // Build & Distribution Directories 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', + // Static Assets and Public Directories + 'public', 'assets', 'static', + // IDE & Editor Directories '.vs', '.vscode', '.idea', '.eclipse', '.settings', diff --git a/src/core/ingestion/parsing-processor.ts b/src/core/ingestion/parsing-processor.ts index 06017135a..b963dec87 100644 --- a/src/core/ingestion/parsing-processor.ts +++ b/src/core/ingestion/parsing-processor.ts @@ -155,41 +155,202 @@ export class ParsingProcessor implements GraphProcessor { options?: { directoryFilter?: string; fileExtensions?: string }): string[] { let filtered = filePaths; + let filteringStats = { + initial: filtered.length, + afterDirectoryFilter: 0, + afterExtensionFilter: 0, + afterIgnorePatterns: 0, + afterContentFilter: 0, + final: 0 + }; + // Apply directory filter if specified if (options?.directoryFilter) { filtered = filtered.filter(path => path.includes(options.directoryFilter ?? '')); + console.log(`Filtering by directory '${options.directoryFilter}': ${filteringStats.initial} -> ${filtered.length} files`); } + filteringStats.afterDirectoryFilter = filtered.length; + // Apply extension filter if specified if (options?.fileExtensions) { const extensions = options.fileExtensions.split(',').map(ext => ext.trim()).filter(ext => ext.length); filtered = filtered.filter(path => extensions.some(ext => path.endsWith(ext))); + console.log(`Filtering by extensions [${extensions.join(', ')}]: ${filteringStats.afterDirectoryFilter} -> ${filtered.length} files`); } + filteringStats.afterExtensionFilter = filtered.length; - filtered = filtered.filter(path => - ![...IGNORE_PATTERNS].some(pattern => { - if (typeof pattern === 'string') { - return path.includes(pattern); - } - return false; - }) - ); - + // Apply ignore patterns (be more selective to avoid over-filtering) + const beforeIgnoreFilter = filtered.length; filtered = filtered.filter(path => { - const content = fileContents.get(path); - return content && content.trim().length > 0; + // More precise ignore pattern matching + for (const pattern of IGNORE_PATTERNS) { + if (typeof pattern === 'string') { + // Only ignore if the pattern is a complete directory component + if (path.includes(`/${pattern}/`) || + path.startsWith(`${pattern}/`) || + path.endsWith(`/${pattern}`) || + path === pattern) { + return false; + } + } + } + + // Additional filtering for files that shouldn't be in KG + const fileName = path.split('/').pop()?.toLowerCase() || ''; + + // Skip documentation and readme files + if (fileName.includes('readme') || + fileName.includes('license') || + fileName.includes('changelog') || + fileName.includes('contributing') || + fileName.includes('authors') || + fileName.includes('maintainers')) { + return false; + } + + // Skip git and version control files + if (fileName.startsWith('.git') || + fileName.includes('.gitignore') || + fileName.includes('.gitattributes')) { + return false; + } + + // Skip common non-source files + if (fileName.includes('dockerfile') || + fileName.includes('docker-compose') || + fileName.endsWith('.md') || + fileName.endsWith('.txt') || + fileName.endsWith('.log') || + fileName.endsWith('.lock')) { + return false; + } + + return true; }); + if (beforeIgnoreFilter !== filtered.length) { + const excludedCount = beforeIgnoreFilter - filtered.length; + console.log(`Filtering by ignore patterns and file types: ${beforeIgnoreFilter} -> ${filtered.length} files (${excludedCount} excluded)`); + + // Log what types of files were excluded for transparency + if (excludedCount <= 10) { + const excluded = filePaths.slice(0, beforeIgnoreFilter).filter(path => !filtered.includes(path)); + console.log('๐Ÿ“‹ Excluded files sample:', excluded.slice(0, 5).map(p => p.split('/').pop())); + } + } + filteringStats.afterIgnorePatterns = filtered.length; + + // Apply content filter (only exclude truly empty files) + const beforeContentFilter = filtered.length; + const emptyFiles: string[] = []; + filtered = filtered.filter(path => { + const content = fileContents.get(path); + if (!content || content.trim().length === 0) { + emptyFiles.push(path); + return false; + } + return true; + }); + if (emptyFiles.length > 0) { + console.log(`Filtering empty files: ${beforeContentFilter} -> ${filtered.length} files (${emptyFiles.length} empty)`); + if (emptyFiles.length <= 5) { + console.log('Empty files:', emptyFiles); + } + } + filteringStats.afterContentFilter = filtered.length; + filteringStats.final = filtered.length; + + // Log comprehensive filtering summary + console.log('๐Ÿ“Š File Filtering Summary:', filteringStats); + + // Show sample of filtered files by type + const filesByExtension = filtered.reduce((acc, path) => { + const ext = pathUtils.extname(path).toLowerCase() || 'no-extension'; + acc[ext] = (acc[ext] || 0) + 1; + return acc; + }, {} as Record); + console.log('๐Ÿ“‹ Files by extension:', filesByExtension); + + // Debug: Check for specific TSX/TS files + const tsxFiles = filtered.filter(path => path.endsWith('.tsx')); + const tsFiles = filtered.filter(path => path.endsWith('.ts')); + if (tsxFiles.length > 0) { + console.log(`๐Ÿ” TSX FILES: Found ${tsxFiles.length} TSX files:`, tsxFiles.slice(0, 5).map(p => p.split('/').pop())); + } + if (tsFiles.length > 0) { + console.log(`๐Ÿ” TS FILES: Found ${tsFiles.length} TS files:`, tsFiles.slice(0, 5).map(p => p.split('/').pop())); + } + + // Specifically check for ChatInterface.tsx + const chatInterface = filtered.find(path => path.includes('ChatInterface.tsx')); + if (chatInterface) { + console.log(`๐ŸŽฏ FOUND ChatInterface.tsx in filtered files: ${chatInterface}`); + } else { + const originalChatInterface = filePaths.find(path => path.includes('ChatInterface.tsx')); + if (originalChatInterface) { + console.warn(`โš ๏ธ ChatInterface.tsx was FILTERED OUT: ${originalChatInterface}`); + } + } + + // Warn about potentially inappropriate files + const suspiciousFiles = filtered.filter(path => { + const fileName = path.split('/').pop()?.toLowerCase() || ''; + return fileName.includes('test') || + fileName.includes('spec') || + fileName.includes('mock') || + fileName.includes('fixture') || + path.includes('/test/') || + path.includes('/tests/') || + path.includes('/__tests__/') || + path.includes('/spec/'); + }); + + if (suspiciousFiles.length > 0) { + console.warn(`โš ๏ธ Found ${suspiciousFiles.length} test/spec files that will be processed:`); + console.warn('Test files sample:', suspiciousFiles.slice(0, 3).map(p => p.split('/').pop())); + console.warn('Consider if these should be excluded from the knowledge graph'); + } return filtered; } private isSourceFile(filePath: string): boolean { - const sourceExtensions = ['.js', '.ts', '.jsx', '.tsx', '.py', '.java', '.cpp', '.c', '.h', '.hpp', '.cs', '.php', '.rb', '.go', '.rs']; + // Only include actual programming language source files + const sourceExtensions = [ + // JavaScript/TypeScript (core web technologies) + '.js', '.ts', '.jsx', '.tsx', + // Python + '.py', + // Java + '.java', + // C/C++ + '.cpp', '.c', '.cc', '.cxx', '.h', '.hpp', '.hxx', + // C# + '.cs', + // Only include other languages if they're commonly used + '.php', '.rb', '.go', '.rs' + // Removed: .mjs, .cjs (might be build artifacts) + // Removed: .html, .htm, .xml (markup, not source code) + // Removed: .vue, .svelte (framework-specific) + // Removed: .kt, .scala, .swift (less common) + ]; return sourceExtensions.some(ext => filePath.toLowerCase().endsWith(ext)); } private isConfigFile(filePath: string): boolean { - const configFiles = ['package.json', 'tsconfig.json', 'webpack.config.js', 'vite.config.ts', '.eslintrc.js', '.prettierrc']; - const configExtensions = ['.json', '.yaml', '.yml', '.toml']; + // Only include config files that might contain meaningful definitions + const configFiles = [ + 'package.json', 'tsconfig.json', 'jsconfig.json', + 'webpack.config.js', 'vite.config.ts', 'vite.config.js', + '.eslintrc.js', '.eslintrc.json', + 'babel.config.js', 'rollup.config.js' + // Removed: .prettierrc (formatting, no definitions) + // Removed: pyproject.toml, setup.py (might be worth including if Python project) + // Removed: requirements.txt (just dependencies) + // Removed: Dockerfile, docker-compose.yml (deployment, not source) + // Removed: .gitignore, .gitattributes (git config, no definitions) + // Removed: README.md, LICENSE (documentation, no definitions) + ]; + const configExtensions = ['.json']; // Only JSON configs, removed .yaml, .yml, .toml, .ini, .cfg return configFiles.some(name => filePath.endsWith(name)) || configExtensions.some(ext => filePath.toLowerCase().endsWith(ext)); @@ -226,10 +387,11 @@ export class ParsingProcessor implements GraphProcessor { private async parseFile(graph: KnowledgeGraph, filePath: string, content: string): Promise { const language = this.detectLanguage(filePath); + const fileName = pathUtils.getFileName(filePath); // Skip compiled/minified files for JavaScript if (language === 'javascript' && this.isCompiledOrMinified(content, filePath)) { - console.log(`Skipping compiled/minified file: ${filePath}`); + console.log(`Skipping compiled/minified file: ${fileName}`); await this.parseGenericFile(graph, filePath, content); return; } @@ -240,7 +402,7 @@ export class ParsingProcessor implements GraphProcessor { // Check cache first - TEMPORARILY DISABLED FOR DEBUGGING const cachedResult = null; // this.lruCache.getParsedFile(cacheKey); if (cachedResult) { - console.log(`Cache hit for file: ${filePath}`); + console.log(`Cache hit for file: ${fileName}`); this.astMap.set(filePath, { tree: cachedResult.ast }); await this.addDefinitionsToGraph(graph, filePath, cachedResult.definitions); return; @@ -249,125 +411,278 @@ export class ParsingProcessor implements GraphProcessor { const langParser = this.languageParsers.get(language); if (!langParser || !this.parser) { - // Skip file with reduced logging to avoid console spam + console.warn(`โš ๏ธ SKIPPING: ${fileName} - No parser available (language: ${language}, hasLangParser: ${!!langParser}, hasMainParser: ${!!this.parser})`); await this.parseGenericFile(graph, filePath, content); return; } - this.parser.setLanguage(langParser); - const tree = this.parser.parse(content); - this.astMap.set(filePath, { tree }); - const definitions: ParsedDefinition[] = []; + try { + this.parser.setLanguage(langParser); + const tree = this.parser.parse(content); + this.astMap.set(filePath, { tree }); + const definitions: ParsedDefinition[] = []; - const queries = this.getQueriesForLanguage(language); - if (!queries) { - console.warn(`No queries available for language: ${language}.`); - return; - } - - // Process queries with caching - for (const [queryName, queryString] of Object.entries(queries)) { - const queryCacheKey = this.lruCache.generateQueryCacheKey(language, queryString); - let queryResults: Parser.QueryMatch[] = []; - - // Check query cache - TEMPORARILY DISABLED FOR DEBUGGING - const cachedQuery = null; // this.lruCache.getQueryResult(queryCacheKey); - if (cachedQuery) { - queryResults = cachedQuery.results; - } else { - const query = langParser.query(queryString as string); - queryResults = query.matches(tree.rootNode); - - // Cache query results - this.lruCache.setQueryResult(queryCacheKey, { - query: queryString, - results: queryResults, - timestamp: Date.now() - }); + const queries = this.getQueriesForLanguage(language); + if (!queries) { + console.warn(`No queries available for language: ${language} (${fileName}).`); + await this.parseGenericFile(graph, filePath, content); + return; } - for (const match of queryResults) { - for (const capture of match.captures) { - const node = capture.node; - const definition = this.extractDefinition(node, queryName, filePath); - if (definition) { - definitions.push(definition); + let totalMatches = 0; + // Process queries with caching + for (const [queryName, queryString] of Object.entries(queries)) { + const queryCacheKey = this.lruCache.generateQueryCacheKey(language, queryString); + let queryResults: Parser.QueryMatch[] = []; + + try { + // Check query cache - TEMPORARILY DISABLED FOR DEBUGGING + const cachedQuery = null; // this.lruCache.getQueryResult(queryCacheKey); + if (cachedQuery) { + queryResults = cachedQuery.results; + } else { + const query = langParser.query(queryString as string); + queryResults = query.matches(tree.rootNode); + totalMatches += queryResults.length; - // Debug: Log what specific definitions we're finding - if (language === 'python' && filePath.endsWith('.py')) { - console.log(`๐Ÿ” DEBUG: Found ${queryName} -> ${definition.type}: "${definition.name}" in ${filePath.split('/').pop()}`); + // Cache query results + this.lruCache.setQueryResult(queryCacheKey, { + query: queryString, + results: queryResults, + timestamp: Date.now() + }); + } + + for (const match of queryResults) { + for (const capture of match.captures) { + const node = capture.node; + const definition = this.extractDefinition(node, queryName, filePath); + if (definition) { + definitions.push(definition); + + // Debug: Log what specific definitions we're finding for certain files + if (fileName.endsWith('.py') || fileName.endsWith('.ts') || fileName.endsWith('.js')) { + console.log(`๐Ÿ” DEBUG: Found ${queryName} -> ${definition.type}: "${definition.name}" in ${fileName}:${definition.startLine}`); + } + } } } + } catch (queryError) { + console.warn(`Query error in ${fileName} for ${queryName}:`, queryError); } } - } - // Cache the parsed file results - this.lruCache.setParsedFile(cacheKey, { - ast: tree, - definitions, - language, - lastModified: Date.now(), - fileSize: content.length - }); + // Cache the parsed file results + this.lruCache.setParsedFile(cacheKey, { + ast: tree, + definitions, + language, + lastModified: Date.now(), + fileSize: content.length + }); - // Debug: Log definition extraction results for Python files - if (language === 'python' && filePath.endsWith('.py')) { - console.log(`๐Ÿ” DEBUG: ${filePath.split('/').pop()} -> ${definitions.length} definitions extracted (content: ${content.length} chars, hash: ${contentHash.substring(0, 8)})`); + // Enhanced debug logging + console.log(`๐Ÿ” PARSING: ${fileName} (${language}) -> ${definitions.length} definitions from ${totalMatches} matches (content: ${content.length} chars)`); // Log definition types for debugging - const definitionTypes = definitions.reduce((acc, def) => { - acc[def.type] = (acc[def.type] || 0) + 1; - return acc; - }, {}); - if (definitions.length > 0) { - console.log(`๐Ÿ” DEBUG: Definition types: ${JSON.stringify(definitionTypes)}`); - } - - if (definitions.length === 0 && content.length > 100) { - console.log(`๐Ÿšจ DEBUG: Large Python file with no definitions: ${filePath} (${content.length} chars)`); + const definitionTypes = definitions.reduce((acc, def) => { + acc[def.type] = (acc[def.type] || 0) + 1; + return acc; + }, {} as Record); + console.log(`๐Ÿ” TYPES: ${fileName} -> ${JSON.stringify(definitionTypes)}`); + } else if (content.length > 100 && language !== 'generic') { + console.warn(`๐Ÿšจ NO DEFINITIONS: ${fileName} (${language}, ${content.length} chars, ${totalMatches} total matches)`); // Log first few lines to understand the content - const firstLines = content.split('\n').slice(0, 3).join('\\n'); - console.log(`๐Ÿ” DEBUG: File starts with: ${firstLines}`); + const firstLines = content.split('\n').slice(0, 3).join('\\n').substring(0, 150); + console.warn(`๐Ÿ” CONTENT PREVIEW: ${firstLines}...`); } - } - await this.addDefinitionsToGraph(graph, filePath, definitions); + await this.addDefinitionsToGraph(graph, filePath, definitions); + } catch (parseError) { + console.error(`Error parsing ${fileName}:`, parseError); + await this.parseGenericFile(graph, filePath, content); + } } private extractDefinition(node: Parser.SyntaxNode, queryName: string, filePath: string): ParsedDefinition | null { - const nameNode = node.childForFieldName('name'); - const name = nameNode ? nameNode.text : null; + let nameNode = node.childForFieldName('name'); + let name = nameNode ? nameNode.text : null; + + // Handle different naming patterns for different query types + if (!name) { + // Try alternative naming strategies based on query type + switch (queryName) { + case 'variables': + case 'constDeclarations': + case 'global_variables': + // For variable assignments, look for identifier in left side + const leftChild = node.namedChildren.find(child => child.type === 'identifier'); + if (leftChild) name = leftChild.text; + break; + + case 'hookCalls': + case 'hookDestructuring': + // For React hooks, try to get the variable name + const hookVar = node.namedChildren.find(child => child.type === 'variable_declarator'); + if (hookVar) { + const hookName = hookVar.childForFieldName('name'); + if (hookName) { + // Handle array destructuring for useState pattern + if (hookName.type === 'array_pattern') { + const elements = hookName.namedChildren.filter(child => child.type === 'identifier'); + if (elements.length > 0) { + name = elements.map(el => el.text).join(', '); + } + } else { + name = hookName.text; + } + } + } + break; + + case 'reactComponents': + case 'reactConstComponents': + case 'defaultExportArrows': + // For React components, get the component name + const componentVar = node.namedChildren.find(child => child.type === 'variable_declarator'); + if (componentVar) { + const componentName = componentVar.childForFieldName('name'); + if (componentName) name = componentName.text; + } + break; + + case 'moduleExports': + // For module.exports = something, get the property name + const memberExpr = node.namedChildren.find(child => child.type === 'member_expression'); + if (memberExpr) { + const property = memberExpr.childForFieldName('property'); + if (property) name = property.text; + } + break; + + case 'decorators': + // For decorators, get the decorator name + const decoratorChild = node.namedChildren.find(child => child.type === 'identifier'); + if (decoratorChild) name = decoratorChild.text; + break; + + default: + // Try to find any identifier child + const identifierChild = node.namedChildren.find(child => child.type === 'identifier'); + if (identifierChild) name = identifierChild.text; + } + } // Skip anonymous definitions - they're usually from compiled/minified code if (!name || name === 'anonymous' || name.trim().length === 0) { return null; } + + // Skip very short names that are likely noise (but keep single-letter variables like 'i', 'x') + if (name.length === 1 && queryName !== 'variables' && queryName !== 'constDeclarations') { + return null; + } + + // Skip common noise patterns + const noisePatterns = ['_', '__', '___', 'temp', 'tmp']; + if (noisePatterns.includes(name.toLowerCase())) { + return null; + } - return { + const definition: ParsedDefinition = { name, type: this.getDefinitionType(queryName), startLine: node.startPosition.row + 1, endLine: node.endPosition.row + 1, - filePath, - } as ParsedDefinition; + }; + + // Extract additional metadata based on definition type + if (definition.type === 'function' || definition.type === 'method') { + // Try to extract parameters + const parametersNode = node.childForFieldName('parameters'); + if (parametersNode) { + const params: string[] = []; + for (const param of parametersNode.namedChildren) { + if (param.type === 'identifier' || param.type === 'formal_parameter') { + params.push(param.text); + } + } + if (params.length > 0) { + definition.parameters = params; + } + } + + // Check for async functions - removed async queries as they were causing Tree-sitter errors + // if (queryName === 'async_functions' || queryName === 'async_methods') { + // definition.isAsync = true; + // } + + // Mark React components + if (queryName === 'reactComponents' || queryName === 'reactConstComponents') { + definition.isAsync = false; // React components are not async by default + definition.exportType = 'default'; // Most React components are default exports + } + } + + if (definition.type === 'class') { + // Try to extract inheritance information + const superclassNode = node.childForFieldName('superclass'); + if (superclassNode) { + definition.extends = [superclassNode.text]; + } + } + + // Handle variable types with additional context + if (definition.type === 'variable') { + if (queryName === 'hookCalls' || queryName === 'hookDestructuring') { + definition.exportType = 'named'; // React hooks are typically named exports + + // Try to extract hook type from call expression + const callExpr = node.descendantsOfType('call_expression')[0]; + if (callExpr) { + const funcNode = callExpr.childForFieldName('function'); + if (funcNode && funcNode.type === 'identifier') { + definition.returnType = funcNode.text; // Store hook function name + } + } + } + } + + return definition; } private getDefinitionType(queryName: string): ParsedDefinition['type'] { switch (queryName) { - case 'classes': return 'class'; - case 'methods': return 'method'; + case 'classes': + case 'exportClasses': return 'class'; + case 'methods': + case 'properties': + case 'staticmethods': + case 'classmethods': return 'method'; case 'functions': case 'arrowFunctions': + case 'reactComponents': + case 'reactConstComponents': + case 'defaultExportArrows': case 'variableAssignments': - case 'objectMethods': return 'function'; + case 'objectMethods': + case 'exportFunctions': + case 'defaultExportFunctions': + case 'functionExpressions': return 'function'; + case 'variables': + case 'constDeclarations': + case 'hookCalls': + case 'hookDestructuring': + case 'global_variables': return 'variable'; case 'imports': case 'from_imports': return 'import'; case 'exports': - case 'defaultExports': return 'function'; // Exports usually export functions + case 'defaultExports': + case 'moduleExports': return 'function'; // Exports usually export functions case 'interfaces': return 'interface'; case 'types': return 'type'; + case 'enums': return 'type'; case 'decorators': return 'decorator'; default: console.warn(`Unknown query type: ${queryName}, defaulting to 'function'`); @@ -413,12 +728,22 @@ export class ParsingProcessor implements GraphProcessor { switch (extension) { case '.ts': - case '.tsx': return 'typescript'; + case '.tsx': + console.log(`๐Ÿ” LANGUAGE: ${filePath.split('/').pop()} -> typescript (${extension})`); + return 'typescript'; case '.js': - case '.jsx': return 'javascript'; - case '.py': return 'python'; - case '.java': return 'java'; - default: return 'generic'; + case '.jsx': + console.log(`๐Ÿ” LANGUAGE: ${filePath.split('/').pop()} -> javascript (${extension})`); + return 'javascript'; + case '.py': + console.log(`๐Ÿ” LANGUAGE: ${filePath.split('/').pop()} -> python (${extension})`); + return 'python'; + case '.java': + console.log(`๐Ÿ” LANGUAGE: ${filePath.split('/').pop()} -> java (${extension})`); + return 'java'; + default: + console.log(`๐Ÿ” LANGUAGE: ${filePath.split('/').pop()} -> generic (${extension})`); + return 'generic'; } } @@ -633,4 +958,106 @@ export class ParsingProcessor implements GraphProcessor { this.memoryManager.clearCache(); this.lruCache.clearAll(); } + + /** + * Diagnostic method to analyze why files might not have definitions + */ + public getDiagnosticInfo(): { + processedFiles: number; + skippedFiles: number; + totalDefinitions: number; + definitionsByType: Record; + definitionsByFile: Record; + processingErrors: string[]; + } { + const definitionsByType: Record = {}; + const definitionsByFile: Record = {}; + let totalDefinitions = 0; + + // Analyze function registry + const allDefs = this.functionTrie.getAllDefinitions(); + allDefs.forEach(def => { + totalDefinitions++; + definitionsByType[def.type] = (definitionsByType[def.type] || 0) + 1; + definitionsByFile[def.filePath] = (definitionsByFile[def.filePath] || 0) + 1; + }); + + return { + processedFiles: this.processedFiles.size, + skippedFiles: 0, // Would need to track this during processing + totalDefinitions, + definitionsByType, + definitionsByFile, + processingErrors: [] // Would need to track errors during processing + }; + } + + /** + * Method to analyze a specific file and explain why it might not have definitions + */ + public async analyzeFile(filePath: string, content: string): Promise<{ + language: string; + isSourceFile: boolean; + isConfigFile: boolean; + isCompiled: boolean; + contentLength: number; + queryResults: Record; + extractionIssues: string[]; + }> { + const language = this.detectLanguage(filePath); + const isSourceFile = this.isSourceFile(filePath); + const isConfigFile = this.isConfigFile(filePath); + const isCompiled = language === 'javascript' && this.isCompiledOrMinified(content, filePath); + const extractionIssues: string[] = []; + const queryResults: Record = {}; + + if (!isSourceFile && !isConfigFile) { + extractionIssues.push('File is not recognized as a source or config file'); + } + + if (isCompiled) { + extractionIssues.push('File appears to be compiled/minified and is skipped'); + } + + if (content.trim().length === 0) { + extractionIssues.push('File is empty'); + } + + const langParser = this.languageParsers.get(language); + if (!langParser || !this.parser) { + extractionIssues.push(`No parser available for language: ${language}`); + } else { + try { + this.parser.setLanguage(langParser); + const tree = this.parser.parse(content); + + const queries = this.getQueriesForLanguage(language); + if (queries) { + for (const [queryName, queryString] of Object.entries(queries)) { + try { + const query = langParser.query(queryString as string); + const matches = query.matches(tree.rootNode); + queryResults[queryName] = matches.length; + } catch (queryError) { + extractionIssues.push(`Query '${queryName}' failed: ${queryError}`); + } + } + } else { + extractionIssues.push(`No queries available for language: ${language}`); + } + } catch (parseError) { + extractionIssues.push(`Parse error: ${parseError}`); + } + } + + return { + language, + isSourceFile, + isConfigFile, + isCompiled, + contentLength: content.length, + queryResults, + extractionIssues + }; + } } \ No newline at end of file diff --git a/src/core/ingestion/pipeline.ts b/src/core/ingestion/pipeline.ts index 2f929abea..707edeadf 100644 --- a/src/core/ingestion/pipeline.ts +++ b/src/core/ingestion/pipeline.ts @@ -135,6 +135,20 @@ export class GraphPipeline { return graph; } + /** + * Get detailed diagnostic information about parsing results + */ + public getParsingDiagnostics() { + return this.parsingProcessor.getDiagnosticInfo(); + } + + /** + * Analyze a specific file to understand why it might not have definitions + */ + public async analyzeSpecificFile(filePath: string, content: string) { + return await this.parsingProcessor.analyzeFile(filePath, content); + } + /** * Validate graph integrity and identify potential issues */ diff --git a/src/core/ingestion/structure-processor.ts b/src/core/ingestion/structure-processor.ts index a385a46f9..6986014e0 100644 --- a/src/core/ingestion/structure-processor.ts +++ b/src/core/ingestion/structure-processor.ts @@ -26,6 +26,9 @@ export class StructureProcessor { // Build & Distribution Directories 'build', 'dist', 'out', 'target', 'bin', 'obj', '.gradle', '_build', + // Static Assets and Public Directories + 'public', 'assets', 'static', + // IDE & Editor Directories '.vs', '.vscode', '.idea', '.eclipse', '.settings', diff --git a/src/core/ingestion/tree-sitter-queries.ts b/src/core/ingestion/tree-sitter-queries.ts index 5b172f107..e97b0023e 100644 --- a/src/core/ingestion/tree-sitter-queries.ts +++ b/src/core/ingestion/tree-sitter-queries.ts @@ -18,12 +18,96 @@ export const TYPESCRIPT_QUERIES = { name: (identifier) @name value: (arrow_function))) @arrow_function `, + // React functional components with type annotations (simplified) + reactComponents: ` + (lexical_declaration + (variable_declarator + name: (identifier) @name + value: (arrow_function + (type_annotation)))) @react_component + `, + // React functional components as const declarations (simplified) + reactConstComponents: ` + (lexical_declaration + (variable_declarator + name: (identifier) @name + value: (as_expression + (arrow_function)))) @react_const_component + `, + // Default export arrow functions (common React pattern) + defaultExportArrows: ` + (export_statement + (lexical_declaration + (variable_declarator + name: (identifier) @name + value: (arrow_function)))) @default_export_arrow + `, + // React hooks (useState, useEffect, etc.) + hookCalls: ` + (lexical_declaration + (variable_declarator + name: (identifier) @hook_name + value: (call_expression + function: (identifier) @hook_function + (#match? @hook_function "^use[A-Z].*")))) @hook_call + `, + // Hook calls with array destructuring (useState pattern) + hookDestructuring: ` + (lexical_declaration + (variable_declarator + name: (array_pattern) @hook_pattern + value: (call_expression + function: (identifier) @hook_function + (#match? @hook_function "^use[A-Z].*")))) @hook_destructuring + `, + variables: ` + (lexical_declaration + (variable_declarator + name: (identifier) @variable)) @var_declaration + `, + constDeclarations: ` + (lexical_declaration + (variable_declarator + name: (identifier) @const + value: _)) @const_declaration + `, + // Function expressions assigned to variables + functionExpressions: ` + (lexical_declaration + (variable_declarator + name: (identifier) @name + value: (function_expression))) @function_expression + `, + exports: ` + (export_statement) @export + `, + exportFunctions: ` + (export_statement + (function_declaration) @export_function) + `, + exportClasses: ` + (export_statement + (class_declaration) @export_class) + `, + // Default exports + defaultExports: ` + (export_statement + (identifier) @default_export) + `, + // Default export functions + defaultExportFunctions: ` + (export_statement + (function_declaration) @default_export_function) + `, interfaces: ` (interface_declaration) @interface `, types: ` (type_alias_declaration) @type `, + enums: ` + (enum_declaration) @enum + `, }; // JavaScript queries - similar to TypeScript but without TS-specific syntax @@ -46,6 +130,17 @@ export const JAVASCRIPT_QUERIES = { name: (identifier) @name value: (arrow_function))) @arrow_function `, + variables: ` + (variable_declaration + (variable_declarator + name: (identifier) @variable)) @var_declaration + `, + constDeclarations: ` + (lexical_declaration + (variable_declarator + name: (identifier) @const + value: _)) @const_declaration + `, exports: ` (export_statement) @export `, @@ -53,6 +148,14 @@ export const JAVASCRIPT_QUERIES = { (export_statement (identifier) @default_export) `, + exportFunctions: ` + (export_statement + (function_declaration) @export_function) + `, + exportClasses: ` + (export_statement + (class_declaration) @export_class) + `, variableAssignments: ` (variable_declaration (variable_declarator @@ -65,7 +168,18 @@ export const JAVASCRIPT_QUERIES = { property: (property_identifier) @name) right: (function_expression)) @obj_method `, - // Note: No interfaces or types for pure JavaScript + moduleExports: ` + (assignment_expression + left: (member_expression + object: (identifier) @module + property: (property_identifier) @export_name) + right: _) @module_export + `, + functionExpressions: ` + (assignment_expression + left: (identifier) @name + right: (function_expression)) @func_expr + `, }; export const PYTHON_QUERIES = { @@ -86,10 +200,48 @@ export const PYTHON_QUERIES = { body: (block (function_definition) @method)) `, + variables: ` + (assignment + left: (identifier) @variable + right: _) @var_assignment + `, + global_variables: ` + (module + (assignment + left: (identifier) @global_var + right: _)) @global_assignment + `, decorators: ` (decorated_definition (decorator) @decorator) `, + properties: ` + (class_definition + body: (block + (decorated_definition + (decorator + (identifier) @property_decorator + (#eq? @property_decorator "property")) + (function_definition) @property))) + `, + staticmethods: ` + (class_definition + body: (block + (decorated_definition + (decorator + (identifier) @static_decorator + (#eq? @static_decorator "staticmethod")) + (function_definition) @static_method))) + `, + classmethods: ` + (class_definition + body: (block + (decorated_definition + (decorator + (identifier) @class_decorator + (#eq? @class_decorator "classmethod")) + (function_definition) @class_method))) + `, }; export const JAVA_QUERIES = { diff --git a/src/lib/kg-file-analyzer.ts b/src/lib/kg-file-analyzer.ts new file mode 100644 index 000000000..1a0ac2a94 --- /dev/null +++ b/src/lib/kg-file-analyzer.ts @@ -0,0 +1,178 @@ +/** + * Knowledge Graph File Analysis Utility + * Helps identify if unnecessary files are being included in the KG + */ + +import type { KnowledgeGraph } from '../core/graph/types'; + +export interface FileAnalysisResult { + totalFiles: number; + filesByType: Record; + suspiciousFiles: Array<{ + path: string; + reason: string; + hasDefinitions: boolean; + }>; + recommendations: string[]; +} + +export function analyzeKnowledgeGraphFiles( + graph: KnowledgeGraph, + fileContents?: Map +): FileAnalysisResult { + const fileNodes = graph.nodes.filter(node => node.label === 'File'); + const filesByType: Record = {}; + const suspiciousFiles: Array<{ path: string; reason: string; hasDefinitions: boolean }> = []; + const recommendations: string[] = []; + + // Analyze each file node + fileNodes.forEach(fileNode => { + const filePath = (fileNode.properties.filePath || fileNode.properties.path) as string; + const fileName = filePath.split('/').pop()?.toLowerCase() || ''; + const extension = fileName.split('.').pop()?.toLowerCase() || 'no-extension'; + + // Count by extension + filesByType[`.${extension}`] = (filesByType[`.${extension}`] || 0) + 1; + + // Check if file has any definitions + const hasDefinitions = graph.relationships.some(rel => + rel.source === fileNode.id && + rel.type === 'DEFINES' && + graph.nodes.some(targetNode => + targetNode.id === rel.target && + ['Function', 'Class', 'Method', 'Variable', 'Interface', 'Type'].includes(targetNode.label) + ) + ); + + // Check for suspicious files + const suspiciousPatterns = [ + { pattern: fileName.includes('readme'), reason: 'Documentation file (README)' }, + { pattern: fileName.includes('license'), reason: 'License file' }, + { pattern: fileName.includes('changelog'), reason: 'Changelog file' }, + { pattern: fileName.includes('dockerfile'), reason: 'Docker configuration' }, + { pattern: fileName.includes('docker-compose'), reason: 'Docker Compose file' }, + { pattern: fileName.includes('.gitignore'), reason: 'Git ignore file' }, + { pattern: fileName.includes('.gitattributes'), reason: 'Git attributes file' }, + { pattern: fileName.endsWith('.md'), reason: 'Markdown documentation' }, + { pattern: fileName.endsWith('.txt'), reason: 'Text file' }, + { pattern: fileName.endsWith('.log'), reason: 'Log file' }, + { pattern: fileName.endsWith('.lock'), reason: 'Lock file' }, + { pattern: fileName.includes('package-lock'), reason: 'Package lock file' }, + { pattern: fileName.includes('yarn.lock'), reason: 'Yarn lock file' }, + { pattern: filePath.includes('/test/') || filePath.includes('/tests/'), reason: 'Test directory file' }, + { pattern: filePath.includes('/__tests__/'), reason: 'Jest test file' }, + { pattern: fileName.includes('.test.') || fileName.includes('.spec.'), reason: 'Test/spec file' }, + { pattern: fileName.includes('mock'), reason: 'Mock file' }, + { pattern: fileName.includes('fixture'), reason: 'Test fixture' }, + { pattern: !hasDefinitions && fileName.endsWith('.js'), reason: 'JavaScript file with no definitions' }, + { pattern: !hasDefinitions && fileName.endsWith('.ts'), reason: 'TypeScript file with no definitions' }, + { pattern: !hasDefinitions && fileName.endsWith('.py'), reason: 'Python file with no definitions' }, + ]; + + for (const { pattern, reason } of suspiciousPatterns) { + if (pattern) { + suspiciousFiles.push({ + path: filePath, + reason, + hasDefinitions + }); + break; // Only add one reason per file + } + } + }); + + // Generate recommendations + const suspiciousCount = suspiciousFiles.length; + const totalFiles = fileNodes.length; + + if (suspiciousCount > totalFiles * 0.2) { + recommendations.push(`โš ๏ธ ${suspiciousCount} out of ${totalFiles} files (${Math.round(suspiciousCount/totalFiles*100)}%) seem suspicious. Consider tightening file filtering.`); + } + + const documentationFiles = suspiciousFiles.filter(f => + f.reason.includes('Documentation') || + f.reason.includes('Markdown') || + f.reason.includes('README') || + f.reason.includes('License') + ); + + if (documentationFiles.length > 0) { + recommendations.push(`๐Ÿ“š ${documentationFiles.length} documentation files found. These typically don't contain code definitions and could be excluded.`); + } + + const testFiles = suspiciousFiles.filter(f => + f.reason.includes('Test') || + f.reason.includes('Mock') || + f.reason.includes('fixture') + ); + + if (testFiles.length > 0) { + recommendations.push(`๐Ÿงช ${testFiles.length} test-related files found. Consider if test files should be included in your knowledge graph.`); + } + + const noDefinitionFiles = suspiciousFiles.filter(f => + !f.hasDefinitions && + (f.reason.includes('no definitions')) + ); + + if (noDefinitionFiles.length > 0) { + recommendations.push(`๐Ÿ“„ ${noDefinitionFiles.length} source files have no extracted definitions. These might be empty, have parsing issues, or contain only comments.`); + } + + return { + totalFiles, + filesByType, + suspiciousFiles, + recommendations + }; +} + +/** + * Print a formatted analysis report to console + */ +export function printFileAnalysisReport(analysis: FileAnalysisResult): void { + console.log('\n๐Ÿ” KNOWLEDGE GRAPH FILE ANALYSIS REPORT'); + console.log('====================================='); + + console.log(`\n๐Ÿ“Š Overview:`); + console.log(`Total files in KG: ${analysis.totalFiles}`); + console.log(`Suspicious files: ${analysis.suspiciousFiles.length} (${Math.round(analysis.suspiciousFiles.length/analysis.totalFiles*100)}%)`); + + console.log(`\n๐Ÿ“‹ Files by type:`); + Object.entries(analysis.filesByType) + .sort(([,a], [,b]) => b - a) + .forEach(([type, count]) => { + console.log(` ${type}: ${count} files`); + }); + + if (analysis.suspiciousFiles.length > 0) { + console.log(`\nโš ๏ธ Suspicious files (first 10):`); + analysis.suspiciousFiles.slice(0, 10).forEach(file => { + const fileName = file.path.split('/').pop(); + const definitionStatus = file.hasDefinitions ? 'โœ…' : 'โŒ'; + console.log(` ${definitionStatus} ${fileName} - ${file.reason}`); + }); + + if (analysis.suspiciousFiles.length > 10) { + console.log(` ... and ${analysis.suspiciousFiles.length - 10} more`); + } + } + + if (analysis.recommendations.length > 0) { + console.log(`\n๐Ÿ’ก Recommendations:`); + analysis.recommendations.forEach((rec, index) => { + console.log(` ${index + 1}. ${rec}`); + }); + } + + console.log('\n=====================================\n'); +} + +// Make it available globally for browser console use +if (typeof window !== 'undefined') { + (window as any).analyzeKGFiles = (graph: any, fileContents?: Map) => { + const analysis = analyzeKnowledgeGraphFiles(graph, fileContents); + printFileAnalysisReport(analysis); + return analysis; + }; +} \ No newline at end of file diff --git a/src/services/github-archive.ts b/src/services/github-archive.ts index d0a657605..6ef268e70 100644 --- a/src/services/github-archive.ts +++ b/src/services/github-archive.ts @@ -236,10 +236,21 @@ export class GitHubArchiveService { */ private isBinaryFile(path: string): boolean { const binaryExtensions = [ - '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', - '.ttf', '.eot', '.pdf', '.zip', '.tar', '.gz', '.rar', '.7z', - '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', - '.mp4', '.mp3', '.wav', '.avi', '.mov', '.webm', '.webp' + // Images + '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp', + // Fonts + '.woff', '.woff2', '.ttf', '.eot', '.otf', + // Documents + '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + // Archives + '.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma', + // Executables & Libraries + '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm', + // Package formats + '.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa', + // Media + '.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v', + '.aac', '.ogg', '.flac', '.m4a', '.wma' ]; return binaryExtensions.some(ext => path.toLowerCase().endsWith(ext)); diff --git a/src/services/github.ts b/src/services/github.ts index ced2e858e..fbcbb1602 100644 --- a/src/services/github.ts +++ b/src/services/github.ts @@ -273,6 +273,10 @@ export class GitHubService { 'logs', 'tmp', '.tmp', + // Static assets and public files + 'public', + 'assets', + 'static', // Additional common directories to skip 'coverage', '.coverage', @@ -366,13 +370,15 @@ export class GitHubService { /\.7z$/, /\.gz$/, // Media files - /\.(jpg|jpeg|png|gif|bmp|svg|ico)$/i, - /\.(mp4|avi|mov|wmv|flv|webm)$/i, - /\.(mp3|wav|flac|aac|ogg)$/i, + /\.(jpg|jpeg|png|gif|bmp|svg|ico|tiff|webp)$/i, + /\.(mp4|avi|mov|wmv|flv|webm|mkv|m4v)$/i, + /\.(mp3|wav|flac|aac|ogg|m4a|wma)$/i, // Document files /\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i, // Other binary files - /\.(exe|dll|dylib)$/i, + /\.(exe|dll|dylib|wasm|jar|war|ear|deb|rpm|dmg|msi|pkg|apk|ipa)$/i, + // Font files + /\.(woff|woff2|ttf|eot|otf)$/i, // Minified files and source maps /\.min\.(js|css)$/, /\.map$/, @@ -425,10 +431,21 @@ export class GitHubService { // Skip binary files that shouldn't have content const binaryExtensions = [ - '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', - '.ttf', '.eot', '.pdf', '.zip', '.tar', '.gz', '.rar', '.7z', - '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', - '.mp4', '.mp3', '.wav', '.avi', '.mov', '.webm', '.webp' + // Images + '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp', + // Fonts + '.woff', '.woff2', '.ttf', '.eot', '.otf', + // Documents + '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + // Archives + '.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma', + // Executables & Libraries + '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm', + // Package formats + '.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa', + // Media + '.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v', + '.aac', '.ogg', '.flac', '.m4a', '.wma' ]; return binaryExtensions.some(ext => pathLower.endsWith(ext)); diff --git a/src/services/zip.ts b/src/services/zip.ts index 749368159..b535c0ea8 100644 --- a/src/services/zip.ts +++ b/src/services/zip.ts @@ -413,6 +413,10 @@ export class ZipService { 'logs', 'tmp', '.tmp', + // Static assets and public files + 'public', + 'assets', + 'static', // Additional common directories to skip 'coverage', '.coverage', @@ -506,13 +510,15 @@ export class ZipService { /\.7z$/, /\.gz$/, // Media files - /\.(jpg|jpeg|png|gif|bmp|svg|ico)$/i, - /\.(mp4|avi|mov|wmv|flv|webm)$/i, - /\.(mp3|wav|flac|aac|ogg)$/i, + /\.(jpg|jpeg|png|gif|bmp|svg|ico|tiff|webp)$/i, + /\.(mp4|avi|mov|wmv|flv|webm|mkv|m4v)$/i, + /\.(mp3|wav|flac|aac|ogg|m4a|wma)$/i, // Document files /\.(pdf|doc|docx|xls|xlsx|ppt|pptx)$/i, // Other binary files - /\.(exe|dll|dylib)$/i, + /\.(exe|dll|dylib|wasm|jar|war|ear|deb|rpm|dmg|msi|pkg|apk|ipa)$/i, + // Font files + /\.(woff|woff2|ttf|eot|otf)$/i, // Minified files and source maps /\.min\.(js|css)$/, /\.map$/, @@ -564,10 +570,21 @@ export class ZipService { // Skip binary files that shouldn't have content const binaryExtensions = [ - '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.woff', '.woff2', - '.ttf', '.eot', '.pdf', '.zip', '.tar', '.gz', '.rar', '.7z', - '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', - '.mp4', '.mp3', '.wav', '.avi', '.mov', '.webm', '.webp' + // Images + '.png', '.jpg', '.jpeg', '.gif', '.svg', '.ico', '.bmp', '.tiff', '.webp', + // Fonts + '.woff', '.woff2', '.ttf', '.eot', '.otf', + // Documents + '.pdf', '.doc', '.docx', '.xls', '.xlsx', '.ppt', '.pptx', + // Archives + '.zip', '.tar', '.gz', '.rar', '.7z', '.bz2', '.xz', '.lzma', + // Executables & Libraries + '.exe', '.dll', '.so', '.dylib', '.class', '.pyc', '.o', '.a', '.lib', '.wasm', + // Package formats + '.jar', '.war', '.ear', '.deb', '.rpm', '.dmg', '.msi', '.pkg', '.apk', '.ipa', + // Media + '.mp4', '.mp3', '.wav', '.avi', '.mov', '.wmv', '.flv', '.webm', '.mkv', '.m4v', + '.aac', '.ogg', '.flac', '.m4a', '.wma' ]; return binaryExtensions.some(ext => pathLower.endsWith(ext)); diff --git a/src/ui/components/DiagnosticPanel.tsx b/src/ui/components/DiagnosticPanel.tsx new file mode 100644 index 000000000..5c4295a43 --- /dev/null +++ b/src/ui/components/DiagnosticPanel.tsx @@ -0,0 +1,209 @@ +import React, { useState } from 'react'; + +interface DiagnosticInfo { + processedFiles: number; + skippedFiles: number; + totalDefinitions: number; + definitionsByType: Record; + definitionsByFile: Record; + processingErrors: string[]; +} + +interface FileAnalysis { + language: string; + isSourceFile: boolean; + isConfigFile: boolean; + isCompiled: boolean; + contentLength: number; + queryResults: Record; + extractionIssues: string[]; +} + +interface DiagnosticPanelProps { + onGetDiagnostics?: () => DiagnosticInfo | null; + onAnalyzeFile?: (filePath: string, content: string) => Promise; + fileContents?: Map; +} + +export default function DiagnosticPanel({ + onGetDiagnostics, + onAnalyzeFile, + fileContents +}: DiagnosticPanelProps) { + const [diagnostics, setDiagnostics] = useState(null); + const [fileAnalysis, setFileAnalysis] = useState(null); + const [selectedFile, setSelectedFile] = useState(''); + const [isAnalyzing, setIsAnalyzing] = useState(false); + + const handleGetDiagnostics = () => { + if (onGetDiagnostics) { + const result = onGetDiagnostics(); + setDiagnostics(result); + } + }; + + const handleAnalyzeFile = async () => { + if (!onAnalyzeFile || !selectedFile || !fileContents) return; + + const content = fileContents.get(selectedFile); + if (!content) { + alert('File content not found'); + return; + } + + setIsAnalyzing(true); + try { + const result = await onAnalyzeFile(selectedFile, content); + setFileAnalysis(result); + } catch (error) { + console.error('Error analyzing file:', error); + alert('Error analyzing file: ' + error); + } finally { + setIsAnalyzing(false); + } + }; + + const fileList = fileContents ? Array.from(fileContents.keys()).sort() : []; + + const containerStyle: React.CSSProperties = { + padding: '20px', + backgroundColor: '#f8f9fa', + border: '1px solid #e0e0e0', + borderRadius: '8px', + margin: '10px', + fontFamily: 'Arial, sans-serif' + }; + + const sectionStyle: React.CSSProperties = { + marginBottom: '20px', + padding: '15px', + backgroundColor: 'white', + borderRadius: '6px', + border: '1px solid #ddd' + }; + + const buttonStyle: React.CSSProperties = { + padding: '8px 16px', + backgroundColor: '#007bff', + color: 'white', + border: 'none', + borderRadius: '4px', + cursor: 'pointer', + marginRight: '10px' + }; + + const selectStyle: React.CSSProperties = { + padding: '8px', + marginRight: '10px', + borderRadius: '4px', + border: '1px solid #ccc', + minWidth: '300px' + }; + + return ( +
+

๐Ÿ” Parsing Diagnostics

+ + {/* Overall Diagnostics Section */} +
+

Overall Parsing Statistics

+ + + {diagnostics && ( +
+

๐Ÿ“Š Summary

+
    +
  • Processed Files: {diagnostics.processedFiles}
    • +
  • Total Definitions: {diagnostics.totalDefinitions}
    • +
+ +

๐Ÿ“‹ Definitions by Type

+
    + {Object.entries(diagnostics.definitionsByType).map(([type, count]) => ( +
  • {type}: {count}
    • + ))} +
+ +

๐Ÿ“ Top Files by Definition Count

+
    + {Object.entries(diagnostics.definitionsByFile) + .sort(([,a], [,b]) => b - a) + .slice(0, 10) + .map(([file, count]) => ( +
  • {file.split('/').pop()}: {count} definitions
    • + ))} +
+
+ )} +
+ + {/* Individual File Analysis Section */} +
+

Individual File Analysis

+
+ + +
+ + {fileAnalysis && ( +
+

๐Ÿ“„ Analysis Results for: {selectedFile.split('/').pop()}

+ +
+ Language: {fileAnalysis.language}
+ Source File: {fileAnalysis.isSourceFile ? 'โœ… Yes' : 'โŒ No'}
+ Config File: {fileAnalysis.isConfigFile ? 'โœ… Yes' : 'โŒ No'}
+ Compiled/Minified: {fileAnalysis.isCompiled ? 'โš ๏ธ Yes' : 'โœ… No'}
+ Content Length: {fileAnalysis.contentLength} characters +
+ +
๐Ÿ” Query Results
+
    + {Object.entries(fileAnalysis.queryResults).map(([query, count]) => ( +
  • {query}: {count} matches
    • + ))} +
+ + {fileAnalysis.extractionIssues.length > 0 && ( + <> +
โš ๏ธ Extraction Issues
+
    + {fileAnalysis.extractionIssues.map((issue, index) => ( +
  • {issue}
    • + ))} +
+ + )} + + {fileAnalysis.extractionIssues.length === 0 && + Object.values(fileAnalysis.queryResults).every(count => count === 0) && ( +
+ โ„น๏ธ No extraction issues found, but no definitions were extracted. + This might be a file with no parseable definitions (like documentation, empty files, or pure data files). +
+ )} +
+ )} +
+
+ ); +} \ No newline at end of file diff --git a/src/ui/components/graph/FloatingSourceViewer.tsx b/src/ui/components/graph/FloatingSourceViewer.tsx index 750cfed90..f97421e87 100644 --- a/src/ui/components/graph/FloatingSourceViewer.tsx +++ b/src/ui/components/graph/FloatingSourceViewer.tsx @@ -259,8 +259,8 @@ const FloatingSourceViewer: React.FC = ({ } // If not found through relationships, search through file contents - // BUT ONLY for Function, Method, Class, and Variable nodes - NOT for Folder or File nodes - if (!filePath && fileContents && ['Function', 'Method', 'Class', 'Variable'].includes(nodeType)) { + // For Function, Method, Class, Variable nodes OR if this is a File node that doesn't have proper path + if (!filePath && fileContents && (['Function', 'Method', 'Class', 'Variable'].includes(nodeType) || nodeType === 'File')) { console.log('FloatingSourceViewer - Searching file contents for node:', nodeName, 'of type:', nodeType); // Use more specific search patterns instead of just checking if content includes nodeName @@ -337,28 +337,165 @@ const FloatingSourceViewer: React.FC = ({ }; } - // Return mock content for nodes without file association - const mockContent = `// ${nodeType}: ${nodeName} -// This is a ${nodeType.toLowerCase()} definition in the knowledge graph + // Special handling for File nodes - display actual file content + if (nodeType === 'File') { + const fileNodePath = node.properties.filePath as string || node.properties.path as string || nodeName; + + // Try to find the file content using various path resolution strategies + let content = fileContents.get(fileNodePath); + + if (!content) { + // Try different path variations + const pathVariations = [ + fileNodePath, + fileNodePath.replace(/^[./]*/, ''), // Remove leading ./ or / + fileNodePath.startsWith('/') ? fileNodePath.substring(1) : `/${fileNodePath}`, // Toggle leading slash + `src/${fileNodePath}`, // Try under src/ + fileNodePath.replace(/\\/g, '/'), // Convert backslashes to forward slashes + fileNodePath.replace(/\//g, '\\') // Convert forward slashes to backslashes + ]; + + for (const variation of pathVariations) { + content = fileContents.get(variation); + if (content) { + break; + } + } + } + + if (content) { + // Found file content - display it + const language = fileNodePath.split('.').pop() || 'text'; + return { + fileName: fileNodePath.split('/').pop() || fileNodePath, + filePath: fileNodePath, + content: content, + nodeType, + nodeName, + language + }; + } + + // If no content found, show file info with explanation + return { + fileName: fileNodePath.split('/').pop() || fileNodePath, + filePath: 'file-not-found', + content: `# File: ${nodeName} +โš ๏ธ **File Content Not Available** + +This file was detected in the project structure but its content could not be loaded. + +## Possible Reasons: + +1. **File Filtering**: The file may have been filtered out during upload +2. **File Size**: Large files might be excluded from processing +3. **Path Mismatch**: The file path in the graph doesn't match the uploaded content +4. **Upload Issues**: The file might not have been included in the ZIP upload + +## File Information: +- **Expected Path**: \`${fileNodePath}\` +- **Node Type**: ${nodeType} +- **Available Files**: ${fileContents.size} files in memory + +## Debug Information: +Tried these path variations: +${['- `' + fileNodePath + '`', '- `' + fileNodePath.replace(/^[./]*/, '') + '`', '- `' + (fileNodePath.startsWith('/') ? fileNodePath.substring(1) : `/${fileNodePath}`) + '`'].join('\n')}`, + nodeType, + nodeName, + language: 'markdown' + }; + } + + // Return detailed information for nodes without file association + const isStructuralNode = ['Project', 'Folder'].includes(nodeType); // File nodes should show content, not placeholders + const isDefinitionNode = ['Function', 'Class', 'Method', 'Variable', 'Interface', 'Type'].includes(nodeType); + + let mockContent = ''; + + if (isStructuralNode) { + mockContent = `# ${nodeType}: ${nodeName} + +This is a structural node in the knowledge graph representing ${nodeType.toLowerCase()} organization. + +## Information: +- **Type**: ${nodeType} +- **Name**: ${nodeName} +- **Status**: Structural element (no source code content) + +## Why no content? +Structural nodes like projects, folders, and some files represent organizational elements rather than code definitions. They help organize the knowledge graph but don't contain executable code. + +## To view code content: +1. Look for definition nodes (Function, Class, Method) within this ${nodeType.toLowerCase()} +2. Check if this ${nodeType.toLowerCase()} contains any parsed source files +3. Verify that the ingestion pipeline successfully processed files in this location`; + } else if (isDefinitionNode) { + mockContent = `# ${nodeType}: ${nodeName} + +โš ๏ธ **Content Not Available** + +This ${nodeType.toLowerCase()} definition was found in the knowledge graph but the source content is not accessible. + +## Possible Reasons: + +### 1. External Library +This may be a function/class from an external library or framework that wasn't included in the project analysis. + +### 2. Parsing Issues +The source file might have: +- Syntax errors that prevented parsing +- Unsupported language features +- Complex code patterns not captured by Tree-sitter queries + +### 3. File Access Issues +The original source file might be: +- Missing from the uploaded content +- Filtered out during ingestion +- Located in an ignored directory + +### 4. Incomplete Ingestion +The ingestion pipeline might have: +- Skipped this file due to size limits +- Failed to process this specific definition +- Encountered errors during Tree-sitter parsing + +## Debugging Tips: +1. Check the browser console for parsing errors +2. Verify the file was included in the upload +3. Look for related files that might contain this definition +4. Try re-running the ingestion process + +## Mock Implementation: +\`\`\`${nodeType === 'Function' ? 'javascript' : nodeType === 'Class' ? 'typescript' : 'text'} ${nodeType === 'Function' ? `function ${nodeName}() { - // Function implementation not available in current context - // This may be an external library function or incomplete parsing + // Implementation not available + // This may be an external library function + // or incomplete parsing result }` : nodeType === 'Class' ? `class ${nodeName} { - // Class definition not available in current context - // This may be an external library class or incomplete parsing -}` : nodeType === 'Method' ? `${nodeName}() { - // Method implementation not available in current context + // Class definition not available + // This may be an external library class + // or incomplete parsing result }` : `// ${nodeType} definition for ${nodeName} -// Content not available in current context`}`; +// Content not available in current context`} +\`\`\``; + } else { + mockContent = `# ${nodeType}: ${nodeName} + +This node represents a ${nodeType.toLowerCase()} in the knowledge graph. + +**Status**: Content not available + +**Note**: This may be a specialized node type or an element that doesn't have direct source code representation.`; + } return { - fileName: `${nodeName}`, + fileName: `${nodeName} (${nodeType})`, filePath: 'virtual-node', content: mockContent, nodeType, nodeName, - language: 'javascript' + language: 'markdown' }; }