diff --git a/.gitignore b/.gitignore index d0a559c32..88d57c8ad 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ dist dist-ssr *.local +# Auto-generated files +public/workers/compiled-queries.js + # Editor directories and files .vscode/* !.vscode/extensions.json diff --git a/package.json b/package.json index cc97478e3..b5713363e 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,9 @@ "version": "1.0.0", "type": "module", "scripts": { - "dev": "vite", - "build": "tsc -b && vite build", + "dev": "npm run compile-queries && vite", + "build": "npm run compile-queries && tsc -b && vite build", + "compile-queries": "node scripts/compile-queries.js", "lint": "eslint .", "preview": "vite preview", "test": "jest", diff --git a/public/workers/tree-sitter-worker.js b/public/workers/tree-sitter-worker.js index 7869582e1..61e029f27 100644 --- a/public/workers/tree-sitter-worker.js +++ b/public/workers/tree-sitter-worker.js @@ -5,6 +5,8 @@ // Import tree-sitter and language parsers import Parser from 'web-tree-sitter'; +// Import compiled queries (generated from TypeScript) +import { getQueriesForLanguage } from './compiled-queries.js'; // Initialize tree-sitter let parser = null; @@ -98,109 +100,61 @@ function extractDefinitions(tree, filePath) { return definitions; } -// Get queries for specific language -function getQueriesForLanguage(language) { - const queries = { - typescript: { - function_declaration: ` - (function_declaration - name: (identifier) @function.name - parameters: (formal_parameters) @function.parameters - body: (statement_block) @function.body - ) - `, - class_declaration: ` - (class_declaration - name: (identifier) @class.name - body: (class_body) @class.body - ) - `, - method_definition: ` - (method_definition - name: (property_identifier) @method.name - parameters: (formal_parameters) @method.parameters - body: (statement_block) @method.body - ) - `, - import_statement: ` - (import_statement - source: (string) @import.source - ) - ` - }, - javascript: { - function_declaration: ` - (function_declaration - name: (identifier) @function.name - parameters: (formal_parameters) @function.parameters - body: (statement_block) @function.body - ) - `, - arrow_function: ` - (arrow_function - parameters: (formal_parameters) @function.parameters - body: (statement_block) @function.body - ) - `, - class_declaration: ` - (class_declaration - name: (identifier) @class.name - body: (class_body) @class.body - ) - ` - }, - python: { - function_definition: ` - (function_definition - name: (identifier) @function.name - parameters: (parameters) @function.parameters - body: (block) @function.body - ) - `, - class_definition: ` - (class_definition - name: (identifier) @class.name - body: (block) @class.body - ) - `, - import_statement: ` - (import_statement - name: (dotted_name) @import.name - ) - ` - } - }; +// Get queries for specific language - now uses imported compiled queries +// This ensures consistency with the main thread parsing logic - return queries[language] || null; +// Helper function to map query names to definition types (matches main thread) +function getDefinitionType(queryName) { + switch (queryName) { + case 'classes': return 'class'; + case 'methods': return 'method'; + case 'functions': + case 'arrowFunctions': return 'function'; + case 'imports': + case 'from_imports': return 'import'; + case 'interfaces': return 'interface'; + case 'types': return 'type'; + case 'decorators': return 'decorator'; + default: return 'variable'; + } } // Process a query match into a definition +// Updated to match main thread's extractDefinition logic exactly function processMatch(match, filePath, queryType) { try { - const captures = match.captures; - const definition = { - type: queryType, - filePath, - startLine: match.node.startPosition.row, - endLine: match.node.endPosition.row, - startColumn: match.node.startPosition.column, - endColumn: match.node.endPosition.column - }; - - // Extract specific information based on query type - for (const capture of captures) { - const { name, node } = capture; + // Main thread processes each capture individually + // For each match, process all captures + const definitions = []; + + for (const capture of match.captures) { + const node = capture.node; - if (name.includes('name')) { - definition.name = node.text; - } else if (name.includes('parameters')) { - definition.parameters = node.text; - } else if (name.includes('source')) { - definition.importSource = node.text.replace(/['"]/g, ''); - } - } + // Extract name from the node (same logic as main thread) + const nameNode = node.childForFieldName('name'); + const name = nameNode ? nameNode.text : 'anonymous'; - return definition; + const definition = { + name: name, + type: getDefinitionType(queryType), // Use proper type mapping + filePath: filePath, + startLine: node.startPosition.row + 1, + endLine: node.endPosition.row + 1, + startColumn: node.startPosition.column, + endColumn: node.endPosition.column + }; + + // Extract additional fields if available + const parametersNode = node.childForFieldName('parameters'); + if (parametersNode) { + definition.parameters = parametersNode.text; + } + + definitions.push(definition); + } + + // Return the first definition (main thread processes one capture at a time) + return definitions.length > 0 ? definitions[0] : null; } catch (error) { console.warn('Worker: Error processing match:', error); return null; diff --git a/scripts/compile-queries.js b/scripts/compile-queries.js new file mode 100644 index 000000000..3340ab6a1 --- /dev/null +++ b/scripts/compile-queries.js @@ -0,0 +1,94 @@ +#!/usr/bin/env node + +/** + * Build-time script to compile TypeScript queries into JavaScript for Web Workers + * This allows workers to import the same query definitions as the main thread + */ + +import fs from 'fs'; +import path from 'path'; +import { fileURLToPath } from 'url'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// Path to the TypeScript queries file +const queriesPath = path.join(__dirname, '../src/core/ingestion/tree-sitter-queries.ts'); +// Output path for compiled JavaScript queries +const outputPath = path.join(__dirname, '../public/workers/compiled-queries.js'); + +function compileQueries() { + try { + console.log('๐Ÿ”จ Compiling Tree-sitter queries for Web Workers...'); + + // Read the TypeScript queries file + const queriesContent = fs.readFileSync(queriesPath, 'utf8'); + + // Extract the query objects using simple regex (since they're just object literals) + const typescriptMatch = queriesContent.match(/export const TYPESCRIPT_QUERIES = ({[\s\S]*?});/); + const javascriptMatch = queriesContent.match(/export const JAVASCRIPT_QUERIES = ({[\s\S]*?});/); + const pythonMatch = queriesContent.match(/export const PYTHON_QUERIES = ({[\s\S]*?});/); + const javaMatch = queriesContent.match(/export const JAVA_QUERIES = ({[\s\S]*?});/); + + if (!typescriptMatch || !javascriptMatch || !pythonMatch || !javaMatch) { + throw new Error('Could not extract queries from TypeScript file'); + } + + // Create JavaScript module content + const jsContent = `/** + * AUTO-GENERATED FILE - DO NOT EDIT MANUALLY + * Generated from src/core/ingestion/tree-sitter-queries.ts + * Run 'npm run compile-queries' to regenerate + */ + +export const TYPESCRIPT_QUERIES = ${typescriptMatch[1]}; + +export const JAVASCRIPT_QUERIES = ${javascriptMatch[1]}; + +export const PYTHON_QUERIES = ${pythonMatch[1]}; + +export const JAVA_QUERIES = ${javaMatch[1]}; + +// Helper function to get queries for a specific language +export function getQueriesForLanguage(language) { + switch (language) { + case 'typescript': + return TYPESCRIPT_QUERIES; + case 'javascript': + return JAVASCRIPT_QUERIES; + case 'python': + return PYTHON_QUERIES; + case 'java': + return JAVA_QUERIES; + default: + return null; + } +} + +// Export individual query sets for backward compatibility +export const queries = { + typescript: TYPESCRIPT_QUERIES, + javascript: JAVASCRIPT_QUERIES, + python: PYTHON_QUERIES, + java: JAVA_QUERIES +}; +`; + + // Ensure output directory exists + const outputDir = path.dirname(outputPath); + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + // Write the compiled JavaScript file + fs.writeFileSync(outputPath, jsContent, 'utf8'); + + console.log(`โœ… Queries compiled successfully to: ${outputPath}`); + + } catch (error) { + console.error('โŒ Failed to compile queries:', error.message); + process.exit(1); + } +} + +compileQueries(); \ No newline at end of file diff --git a/scripts/test-wasm.js b/scripts/test-wasm.js new file mode 100644 index 000000000..3964f0f68 --- /dev/null +++ b/scripts/test-wasm.js @@ -0,0 +1,43 @@ +#!/usr/bin/env node + +/** + * TEMPORARY WASM TEST RUNNER + * Run this to verify Tree-sitter WASM functionality + * DELETE THIS FILE after confirming everything works + */ + +import { spawn } from 'child_process'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const projectRoot = join(__dirname, '..'); + +console.log('๐Ÿ” Starting WASM Verification Tests...'); +console.log('๐Ÿ“ Project root:', projectRoot); + +// Run the WASM verification tests +const testProcess = spawn('npm', ['run', 'test', 'src/tests/wasm-verification.test.ts'], { + cwd: projectRoot, + stdio: 'inherit', + shell: true +}); + +testProcess.on('close', (code) => { + if (code === 0) { + console.log('\nโœ… All WASM verification tests passed!'); + console.log('๐Ÿงน You can now safely delete:'); + console.log(' - src/tests/wasm-verification.test.ts'); + console.log(' - scripts/test-wasm.js'); + } else { + console.log('\nโŒ WASM verification tests failed!'); + console.log('๐Ÿ”ง Please check the issues above before proceeding.'); + } + process.exit(code); +}); + +testProcess.on('error', (error) => { + console.error('โŒ Failed to run tests:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/scripts/verify-wasm-direct.js b/scripts/verify-wasm-direct.js new file mode 100644 index 000000000..b17c65683 --- /dev/null +++ b/scripts/verify-wasm-direct.js @@ -0,0 +1,223 @@ +#!/usr/bin/env node + +/** + * DIRECT WASM VERIFICATION SCRIPT + * Tests Tree-sitter WASM functionality without Jest + * DELETE THIS FILE after confirming everything works + */ + +import { createRequire } from 'module'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; +import fs from 'fs'; + +const require = createRequire(import.meta.url); +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const projectRoot = dirname(__dirname); + +console.log('๐Ÿ” Direct WASM Verification Starting...'); +console.log('๐Ÿ“ Project root:', projectRoot); + +// Test 1: Check if WASM files exist +async function checkWasmFiles() { + console.log('\n๐Ÿ“‹ Test 1: Checking WASM file existence...'); + + const wasmFiles = [ + 'public/wasm/python/tree-sitter-python.wasm', + 'public/wasm/javascript/tree-sitter-javascript.wasm', + 'public/wasm/typescript/tree-sitter-typescript.wasm', + 'public/wasm/tree-sitter.wasm' + ]; + + let allExist = true; + + for (const wasmPath of wasmFiles) { + const fullPath = join(projectRoot, wasmPath); + if (fs.existsSync(fullPath)) { + const stats = fs.statSync(fullPath); + console.log(`โœ… ${wasmPath} exists (${(stats.size / 1024).toFixed(1)}KB)`); + } else { + console.log(`โŒ ${wasmPath} missing`); + allExist = false; + } + } + + return allExist; +} + +// Test 2: Try to load Tree-sitter in Node.js environment +async function testTreeSitterLoad() { + console.log('\n๐Ÿ“‹ Test 2: Testing Tree-sitter module loading...'); + + try { + // Try dynamic import + const Parser = await import('web-tree-sitter'); + console.log('โœ… web-tree-sitter module imported successfully'); + console.log(` Default export type: ${typeof Parser.default}`); + + if (Parser.default && typeof Parser.default === 'function') { + console.log('โœ… Parser constructor available'); + return true; + } else { + console.log('โŒ Parser constructor not found'); + return false; + } + } catch (error) { + console.log('โŒ Failed to import web-tree-sitter:', error.message); + return false; + } +} + +// Test 3: Check compiled queries +async function testCompiledQueries() { + console.log('\n๐Ÿ“‹ Test 3: Testing compiled queries...'); + + try { + const compiledQueriesPath = join(projectRoot, 'public/workers/compiled-queries.js'); + + if (!fs.existsSync(compiledQueriesPath)) { + console.log('โŒ compiled-queries.js not found'); + return false; + } + + console.log('โœ… compiled-queries.js exists'); + + // Read and basic parse check + const content = fs.readFileSync(compiledQueriesPath, 'utf8'); + + if (content.includes('PYTHON_QUERIES') && content.includes('TYPESCRIPT_QUERIES')) { + console.log('โœ… Compiled queries contain expected exports'); + + // Count Python queries + const pythonQueryMatch = content.match(/PYTHON_QUERIES = ({[\\s\\S]*?});/); + if (pythonQueryMatch) { + const queryCount = (pythonQueryMatch[1].match(/:\\s*`/g) || []).length; + console.log(`โœ… Python queries: ${queryCount} query types found`); + } + + return true; + } else { + console.log('โŒ Compiled queries missing expected exports'); + return false; + } + } catch (error) { + console.log('โŒ Error checking compiled queries:', error.message); + return false; + } +} + +// Test 4: Check parser loader module +async function testParserLoader() { + console.log('\n๐Ÿ“‹ Test 4: Testing parser loader module...'); + + try { + const parserLoaderPath = join(projectRoot, 'src/core/tree-sitter/parser-loader.ts'); + + if (!fs.existsSync(parserLoaderPath)) { + console.log('โŒ parser-loader.ts not found'); + return false; + } + + console.log('โœ… parser-loader.ts exists'); + + // Try importing the module + const loaderModule = await import(`file://${parserLoaderPath}`); + + const expectedExports = ['initTreeSitter', 'loadPythonParser', 'loadJavaScriptParser', 'loadTypeScriptParser']; + let allExportsPresent = true; + + for (const exportName of expectedExports) { + if (typeof loaderModule[exportName] === 'function') { + console.log(`โœ… ${exportName} function exported`); + } else { + console.log(`โŒ ${exportName} function missing`); + allExportsPresent = false; + } + } + + return allExportsPresent; + } catch (error) { + console.log('โŒ Error testing parser loader:', error.message); + return false; + } +} + +// Test 5: Verify Tree-sitter queries syntax +async function testQuerySyntax() { + console.log('\n๐Ÿ“‹ Test 5: Testing Tree-sitter query syntax...'); + + try { + const queriesPath = join(projectRoot, 'src/core/ingestion/tree-sitter-queries.ts'); + + if (!fs.existsSync(queriesPath)) { + console.log('โŒ tree-sitter-queries.ts not found'); + return false; + } + + const content = fs.readFileSync(queriesPath, 'utf8'); + + // Basic syntax checks + if (!content.includes('PYTHON_QUERIES') || !content.includes('TYPESCRIPT_QUERIES')) { + console.log('โŒ Missing expected query exports'); + return false; + } + + // Check for the async function query issue we fixed + if (content.includes('async_functions')) { + console.log('โŒ async_functions query still present (should be removed)'); + return false; + } + + console.log('โœ… Query file structure looks correct'); + console.log('โœ… No problematic async_functions query found'); + + return true; + } catch (error) { + console.log('โŒ Error checking query syntax:', error.message); + return false; + } +} + +// Run all tests +async function runAllTests() { + console.log('๐Ÿš€ Starting comprehensive WASM verification...\n'); + + const results = []; + + results.push(await checkWasmFiles()); + results.push(await testTreeSitterLoad()); + results.push(await testCompiledQueries()); + results.push(await testParserLoader()); + results.push(await testQuerySyntax()); + + const passedTests = results.filter(Boolean).length; + const totalTests = results.length; + + console.log(`\n๐Ÿ“Š Results: ${passedTests}/${totalTests} tests passed`); + + if (passedTests === totalTests) { + console.log('\nโœ… All WASM verification tests PASSED!'); + console.log('๐ŸŽ‰ Tree-sitter WASM setup appears to be working correctly'); + console.log('\n๐Ÿงน You can now safely delete these test files:'); + console.log(' - scripts/verify-wasm-direct.js'); + console.log(' - scripts/test-wasm.js'); + console.log(' - src/tests/wasm-verification.test.ts'); + console.log(' - src/__tests__/setup.ts'); + console.log('\n๐Ÿš€ Ready to proceed with ZIP upload testing!'); + return true; + } else { + console.log('\nโŒ Some WASM verification tests FAILED!'); + console.log('๐Ÿ”ง Please fix the issues above before proceeding.'); + return false; + } +} + +runAllTests() + .then(success => { + process.exit(success ? 0 : 1); + }) + .catch(error => { + console.error('๐Ÿ’ฅ Unexpected error:', error); + process.exit(1); + }); \ No newline at end of file diff --git a/src/__tests__/setup.ts b/src/__tests__/setup.ts index 76e0edebe..143251f68 100644 --- a/src/__tests__/setup.ts +++ b/src/__tests__/setup.ts @@ -82,4 +82,52 @@ export const cleanup = () => { // Clean up any test artifacts jest.clearAllTimers(); jest.clearAllMocks(); -}; \ No newline at end of file +}; + +/** + * Jest test setup for WASM verification + * TEMPORARY FILE - DELETE after WASM verification + */ + +// Mock fetch for WASM file loading +global.fetch = jest.fn(); + +// Mock URL.createObjectURL and revokeObjectURL for worker tests +global.URL.createObjectURL = jest.fn(() => 'mock-blob-url'); +global.URL.revokeObjectURL = jest.fn(); + +// Mock Worker for worker tests +global.Worker = jest.fn().mockImplementation(() => ({ + postMessage: jest.fn(), + terminate: jest.fn(), + addEventListener: jest.fn(), + removeEventListener: jest.fn(), + onmessage: null, + onerror: null +})); + +// Setup console for test output +const originalConsoleLog = console.log; +console.log = (...args) => { + // Allow our test messages through + if (args[0] && typeof args[0] === 'string' && ( + args[0].includes('โœ…') || + args[0].includes('โŒ') || + args[0].includes('๐Ÿ”') || + args[0].includes('๐Ÿงช') + )) { + originalConsoleLog(...args); + } +}; + +// Mock WebAssembly for basic checks +if (typeof WebAssembly === 'undefined') { + global.WebAssembly = { + compile: jest.fn(), + instantiate: jest.fn(), + Module: jest.fn(), + Instance: jest.fn() + }; +} + +console.log('๐Ÿ”ง Jest setup completed for WASM verification tests'); diff --git a/src/core/ingestion/call-processor.ts b/src/core/ingestion/call-processor.ts index 4d2d3bffa..d6b6ffbfd 100644 --- a/src/core/ingestion/call-processor.ts +++ b/src/core/ingestion/call-processor.ts @@ -46,7 +46,13 @@ export class CallProcessor { sameFileMatches: 0, heuristicMatches: 0, failed: 0, - callTypes: {} as Record + callTypes: {} as Record, + // Failure categorization + failuresByCategory: { + externalLibraries: 0, // Calls to external/stdlib functions (expected) + pythonBuiltins: 0, // Python built-in functions (expected) + actualFailures: 0 // Real resolution failures (unexpected) + } }; constructor(functionTrie: FunctionRegistryTrie) { @@ -92,26 +98,10 @@ export class CallProcessor { if (calls.length === 0) { // Only log for source files that should have function calls if (this.isSourceFile(filePath)) { - console.log(`โš ๏ธ CallProcessor: No function calls found in source file: ${filePath}`); - - // Debug: Check if this file has any 'call' nodes at all - if (filePath.endsWith('.py')) { - const callNodeCount = this.countNodeType(ast.tree!.rootNode, 'call'); - const definitionCount = graph.nodes.filter(n => - (n.label === 'Function' || n.label === 'Class' || n.label === 'Method') && - n.properties.filePath === filePath - ).length; - - console.log(` ๐Ÿ“Š Debug: ${filePath.split('/').pop()} has ${callNodeCount} call nodes, ${definitionCount} definitions`); - - // If we have definitions but no calls, that's suspicious - if (definitionCount > 0 && callNodeCount === 0) { - console.log(` ๐Ÿšจ Suspicious: File has definitions but no call nodes - possible AST parsing issue`); - } - } + // Reduced logging - track but don't spam console for each file } } else { - console.log(`CallProcessor: Found ${calls.length} function calls in ${filePath}`); + // Reduced logging - only log summary instead of per-file details } for (const call of calls) { @@ -137,7 +127,8 @@ export class CallProcessor { } } else { this.stats.failed++; - console.log(`โŒ Failed to resolve call: ${call.functionName} in ${call.callerFile}:${call.startLine}`); + // Categorize the failure for better statistics + this.categorizeFailureWithReason(call, this.diagnoseFailure(call)); } } } @@ -192,16 +183,44 @@ export class CallProcessor { if (importInfo) { // We have an import for this function name - const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def => - def.filePath === importInfo.targetFile && - (def.functionName === importInfo.exportedName || - (importInfo.exportedName === 'default' && def.functionName === call.functionName)) - ); + const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def => { + // Match file path + if (def.filePath !== importInfo.targetFile) { + return false; + } + + // Handle different import types + if (importInfo.importType === 'default') { + // For default imports, the function name could be anything + // Look for functions that could be the default export + return def.functionName === call.functionName || + def.functionName === importInfo.exportedName || + // Common default export patterns + (def.type === 'function' && def.startLine === 1) || + (def.type === 'class' && def.functionName === 'default'); + } else if (importInfo.importType === 'named') { + // For named imports, match the exported name + return def.functionName === importInfo.exportedName; + } else if (importInfo.importType === 'namespace') { + // For namespace imports like * as utils, + // the call would be utils.someFunction, so we need to handle this differently + return def.functionName === call.functionName; + } + + return false; + }); if (targetDefinitions.length > 0) { + // Prefer functions over other types for function calls + const preferred = targetDefinitions.find(def => + call.callType === 'function_call' ? def.type === 'function' : + call.callType === 'method_call' ? def.type === 'method' : + call.callType === 'constructor_call' ? def.type === 'class' : true + ) || targetDefinitions[0]; + return { success: true, - targetNodeId: targetDefinitions[0].nodeId, + targetNodeId: preferred.nodeId, stage: 'exact', confidence: 'high' }; @@ -407,6 +426,101 @@ export class CallProcessor { return true; } + // JavaScript/TypeScript built-ins and common library functions + const jsBuiltins = new Set([ + // Core JavaScript functions + 'parseInt', 'parseFloat', 'isNaN', 'isFinite', 'encodeURI', 'decodeURI', + 'encodeURIComponent', 'decodeURIComponent', 'escape', 'unescape', + // Array methods + 'push', 'pop', 'shift', 'unshift', 'slice', 'splice', 'concat', 'join', + 'reverse', 'sort', 'indexOf', 'lastIndexOf', 'includes', 'find', 'findIndex', + 'filter', 'map', 'reduce', 'reduceRight', 'forEach', 'some', 'every', + 'flat', 'flatMap', 'fill', 'copyWithin', 'from', 'of', 'isArray', + // Object methods + 'keys', 'values', 'entries', 'assign', 'create', 'defineProperty', + 'defineProperties', 'freeze', 'seal', 'preventExtensions', 'hasOwnProperty', + 'isPrototypeOf', 'propertyIsEnumerable', 'toString', 'valueOf', 'toLocaleString', + // String methods + 'charAt', 'charCodeAt', 'codePointAt', 'concat', 'endsWith', 'includes', + 'indexOf', 'lastIndexOf', 'localeCompare', 'match', 'normalize', 'padEnd', + 'padStart', 'repeat', 'replace', 'search', 'slice', 'split', 'startsWith', + 'substring', 'substr', 'toLowerCase', 'toUpperCase', 'trim', 'trimEnd', + 'trimStart', 'trimLeft', 'trimRight', + // Number methods + 'toFixed', 'toExponential', 'toPrecision', 'isInteger', 'isSafeInteger', + 'isFinite', 'isNaN', 'parseFloat', 'parseInt', + // Date methods + 'getTime', 'getDate', 'getDay', 'getFullYear', 'getHours', 'getMinutes', + 'getSeconds', 'getMilliseconds', 'getMonth', 'setDate', 'setFullYear', + 'setHours', 'setMinutes', 'setSeconds', 'setMilliseconds', 'setMonth', + 'toDateString', 'toTimeString', 'toISOString', 'toJSON', 'now', 'parse', + // Promise methods + 'then', 'catch', 'finally', 'resolve', 'reject', 'all', 'race', 'allSettled', + // Console methods + 'log', 'error', 'warn', 'info', 'debug', 'trace', 'assert', 'clear', + 'count', 'dir', 'dirxml', 'group', 'groupCollapsed', 'groupEnd', 'table', + 'time', 'timeEnd', 'timeLog', 'profile', 'profileEnd', + // DOM methods (common ones) + 'getElementById', 'getElementsByClassName', 'getElementsByTagName', + 'querySelector', 'querySelectorAll', 'createElement', 'createTextNode', + 'appendChild', 'removeChild', 'insertBefore', 'replaceChild', 'cloneNode', + 'getAttribute', 'setAttribute', 'removeAttribute', 'hasAttribute', + 'addEventListener', 'removeEventListener', 'dispatchEvent', + 'preventDefault', 'stopPropagation', 'stopImmediatePropagation', + 'focus', 'blur', 'click', 'submit', 'reset', 'scrollIntoView', + // Common library methods (React, etc.) + 'useState', 'useEffect', 'useContext', 'useReducer', 'useCallback', + 'useMemo', 'useRef', 'useImperativeHandle', 'useLayoutEffect', 'useDebugValue', + 'memo', 'forwardRef', 'lazy', 'Suspense', 'Fragment', 'createElement', + 'cloneElement', 'isValidElement', 'render', 'hydrate', 'unmountComponentAtNode', + // HTTP/Fetch + 'fetch', 'get', 'post', 'put', 'delete', 'patch', 'head', 'options', + // JSON + 'stringify', 'parse', + // Math + 'abs', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'exp', 'floor', + 'log', 'max', 'min', 'pow', 'random', 'round', 'sin', 'sqrt', 'tan', + // Common testing functions + 'describe', 'it', 'test', 'expect', 'beforeEach', 'afterEach', 'beforeAll', 'afterAll', + 'mock', 'spy', 'stub', 'restore', 'reset', 'resetAllMocks', 'clearAllMocks', + // Node.js specific + 'require', 'module', 'exports', '__dirname', '__filename', 'process', 'global', + 'Buffer', 'setImmediate', 'clearImmediate', 'setInterval', 'clearInterval', + 'setTimeout', 'clearTimeout', + // Worker API + 'postMessage', 'onmessage', 'onerror', 'close', 'importScripts', + // JavaScript constructors and built-ins + 'Array', 'Object', 'String', 'Number', 'Boolean', 'Function', 'Date', + 'RegExp', 'Error', 'TypeError', 'ReferenceError', 'SyntaxError', + 'RangeError', 'EvalError', 'URIError', 'AggregateError', + 'Set', 'Map', 'WeakSet', 'WeakMap', 'Symbol', 'BigInt', + 'Promise', 'Proxy', 'Reflect', 'ArrayBuffer', 'SharedArrayBuffer', + 'DataView', 'Int8Array', 'Uint8Array', 'Int16Array', 'Uint16Array', + 'Int32Array', 'Uint32Array', 'Float32Array', 'Float64Array', + 'BigInt64Array', 'BigUint64Array', + // Configuration and build tools + 'config', 'define', 'plugin', 'preset', 'loader', 'rule', + 'extend', 'override', 'merge', 'concat', 'apply', + // ESLint specific + 'rules', 'extends', 'parser', 'parserOptions', 'env', 'globals', + // Bundler/build tools + 'bundle', 'chunk', 'entry', 'output', 'optimization', 'resolve', + 'devtool', 'target', 'externals', 'stats', 'performance', + // Process and execution + 'exec', 'spawn', 'fork', 'execSync', 'spawnSync', + // File system operations + 'readFile', 'writeFile', 'readdir', 'stat', 'mkdir', 'rmdir', + 'unlink', 'rename', 'copyFile', 'access', 'watch', 'createReadStream', + 'createWriteStream' + ]); + + // Check if it's a JS/TS file and the function is a built-in + if ((filePath.endsWith('.js') || filePath.endsWith('.ts') || + filePath.endsWith('.jsx') || filePath.endsWith('.tsx')) && + jsBuiltins.has(functionName)) { + return true; + } + // Ignore very short function names (likely built-ins or operators) if (functionName.length <= 2) { return true; @@ -453,26 +567,19 @@ export class CallProcessor { const functionName = this.extractPythonCallName(functionNode); // Debug: Log what we're finding vs filtering - if (functionName) { - const shouldIgnore = this.shouldIgnoreCall(functionName, filePath); - if (shouldIgnore) { - // Only log a few examples to avoid spam - if (calls.length < 3) { - console.log(`๐Ÿ” Filtered out: ${functionName} in ${filePath.split('/').pop()}`); - } - } else { - calls.push({ - callerFile: filePath, - functionName, - startLine: node.startPosition.row + 1, - endLine: node.endPosition.row + 1, - callType: 'function_call' - }); - } + if (functionName && !this.shouldIgnoreCall(functionName, filePath)) { + // Don't filter here - let all calls through to resolution + calls.push({ + callerFile: filePath, + functionName, + startLine: node.startPosition.row + 1, + endLine: node.endPosition.row + 1, + callType: 'function_call' + }); } else { - // Debug: Log when we can't extract function name + // Reduced logging - don't log every individual extraction failure if (calls.length < 3) { - console.log(`๐Ÿ” Could not extract function name from: ${functionNode.type} in ${filePath.split('/').pop()}`); + // Only log first few failures per file to understand patterns } } } @@ -509,13 +616,16 @@ export class CallProcessor { const constructorNode = node.childForFieldName('constructor'); if (constructorNode) { const constructorName = constructorNode.text; - calls.push({ - callerFile: filePath, - functionName: constructorName, - startLine: node.startPosition.row + 1, - endLine: node.endPosition.row + 1, - callType: 'constructor_call' - }); + // Don't filter constructor calls as strictly - they're important for the graph + if (!this.shouldIgnoreCall(constructorName, filePath)) { + calls.push({ + callerFile: filePath, + functionName: constructorName, + startLine: node.startPosition.row + 1, + endLine: node.endPosition.row + 1, + callType: 'constructor_call' + }); + } } } @@ -552,9 +662,22 @@ export class CallProcessor { } } - // Debug: Log unhandled node types (but limit spam) - if (Math.random() < 0.1) { // Only log 10% of cases to avoid spam - console.log(`๐Ÿ” Unhandled Python call node type: ${node.type} (text: "${node.text}")`); + // Handle additional Python call patterns + if (node.text && node.text.length > 0 && node.text.length < 100) { + // For simple cases, try using the text directly if it looks like a function name + const text = node.text.trim(); + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(text)) { + return text; + } + + // For attribute access, try to extract the last part + const parts = text.split('.'); + if (parts.length > 1) { + const lastPart = parts[parts.length - 1]; + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(lastPart)) { + return lastPart; + } + } } return null; @@ -570,7 +693,40 @@ export class CallProcessor { // For method calls like obj.method(), we want just 'method' const propertyNode = node.childForFieldName('property'); return propertyNode ? propertyNode.text : null; + } else if (node.type === 'call_expression') { + // Handle nested calls like getData().process() + const functionNode = node.childForFieldName('function'); + if (functionNode) { + return this.extractJSCallName(functionNode); + } + } else if (node.type === 'subscript_expression') { + // Handle array/object access like obj['method']() + const propertyNode = node.childForFieldName('index'); + if (propertyNode && propertyNode.type === 'string') { + // Extract string literal content + const propText = propertyNode.text.replace(/['"`]/g, ''); + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(propText)) { + return propText; + } + } } + + // Fallback: try to extract from text for simple patterns + if (node.text && node.text.length > 0 && node.text.length < 50) { + const text = node.text.trim(); + + // Handle simple function calls + if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(text)) { + return text; + } + + // Handle member expressions like obj.method + const memberMatch = text.match(/([a-zA-Z_][a-zA-Z0-9_]*)\s*$/); + if (memberMatch) { + return memberMatch[1]; + } + } + return null; } @@ -677,7 +833,12 @@ export class CallProcessor { sameFileMatches: 0, heuristicMatches: 0, failed: 0, - callTypes: {} + callTypes: {}, + failuresByCategory: { + externalLibraries: 0, + pythonBuiltins: 0, + actualFailures: 0 + } }; } @@ -691,6 +852,18 @@ export class CallProcessor { console.log(` โœ… Same-file matches (Stage 2): ${this.stats.sameFileMatches} (${((this.stats.sameFileMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`); console.log(` ๐ŸŽฏ Heuristic matches (Stage 3): ${this.stats.heuristicMatches} (${((this.stats.heuristicMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`); console.log(` โŒ Failed resolutions: ${this.stats.failed} (${((this.stats.failed / this.stats.totalCalls) * 100).toFixed(1)}%)`); + + // Enhanced failure breakdown + if (this.stats.failed > 0) { + const { externalLibraries, pythonBuiltins, actualFailures } = this.stats.failuresByCategory; + console.log(` ๐Ÿ“ฆ External libraries (expected): ${externalLibraries} (${((externalLibraries / this.stats.failed) * 100).toFixed(1)}% of failures)`); + console.log(` ๐Ÿ Python built-ins (expected): ${pythonBuiltins} (${((pythonBuiltins / this.stats.failed) * 100).toFixed(1)}% of failures)`); + console.log(` ๐Ÿšจ Actual failures (unexpected): ${actualFailures} (${((actualFailures / this.stats.failed) * 100).toFixed(1)}% of failures)`); + + const expectedFailures = externalLibraries + pythonBuiltins; + console.log(` Real success rate (excluding expected failures): ${(((this.stats.totalCalls - actualFailures) / this.stats.totalCalls) * 100).toFixed(1)}%`); + } + console.log(` Success rate: ${(((this.stats.totalCalls - this.stats.failed) / this.stats.totalCalls) * 100).toFixed(1)}%`); } @@ -710,6 +883,57 @@ export class CallProcessor { this.resetStats(); } + /** + * Categorize a failed call for statistics with detailed reason + */ + private categorizeFailureWithReason(call: CallInfo, reason: string): void { + // Check if this is an expected failure (external library or built-in) + if (this.shouldIgnoreCall(call.functionName, call.callerFile)) { + // It's a call we expect to fail (external library or built-in) + if (call.callerFile.endsWith('.py')) { + this.stats.failuresByCategory.pythonBuiltins++; + } else { + this.stats.failuresByCategory.externalLibraries++; + } + } else { + // It's a call to user code that we failed to resolve (unexpected) + this.stats.failuresByCategory.actualFailures++; + } + } + + /** + * Diagnose why a specific call failed + */ + private diagnoseFailure(call: CallInfo): string { + // Check if it's in import map but target not found + const importInfo = this.importMap[call.callerFile]?.[call.functionName]; + if (importInfo) { + const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def => + def.filePath === importInfo.targetFile && + (def.functionName === importInfo.exportedName || + (importInfo.exportedName === 'default' && def.functionName === call.functionName)) + ); + + if (targetDefinitions.length === 0) { + return `Imported from ${importInfo.targetFile} but definition not found`; + } + } + + // Check if function exists in same file + const sameFileDefinitions = this.functionTrie.findInSameFile(call.callerFile, call.functionName); + if (sameFileDefinitions.length === 0) { + // Check if any similar functions exist + const candidates = this.functionTrie.findEndingWith(call.functionName); + if (candidates.length === 0) { + return `No function named '${call.functionName}' found anywhere`; + } else { + return `Function '${call.functionName}' not in same file, ${candidates.length} candidates in other files`; + } + } + + return 'Unknown failure reason'; + } + /** * Check if a file is a source file that should contain function calls */ diff --git a/src/core/ingestion/import-processor.ts b/src/core/ingestion/import-processor.ts index 2d261fe2b..aada29f9f 100644 --- a/src/core/ingestion/import-processor.ts +++ b/src/core/ingestion/import-processor.ts @@ -102,10 +102,16 @@ export class ImportProcessor { ast: ParsedAST, graph: KnowledgeGraph ): Promise<{ found: number; resolved: number }> { - if (!ast.tree) return { found: 0, resolved: 0 }; + if (!ast.tree) { + return { found: 0, resolved: 0 }; + } + + const imports = this.extractImports(ast.tree.rootNode, filePath); + + if (imports.length === 0) return { found: 0, resolved: 0 }; // Initialize import map for this file @@ -227,17 +233,81 @@ export class ImportProcessor { filePath: string, imports: ImportInfo[] ): void { + + if (node.type === 'import_statement') { - const sourceNode = node.childForFieldName('source'); - if (!sourceNode) return; + + + // Try different approaches to find the source + let sourceNode = node.childForFieldName('source'); + if (!sourceNode) { + // Try finding string literal directly + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i); + if (child && (child.type === 'string' || child.type === 'string_literal')) { + sourceNode = child; + break; + } + } + } + + if (!sourceNode) { + return; + } const sourcePath = sourceNode.text.replace(/['"]/g, ''); const targetFile = this.resolveModulePath(sourcePath, filePath, 'javascript'); + + // Handle different import patterns - const importClauseNode = node.childForFieldName('import_clause'); + let importClauseNode: Parser.SyntaxNode | null = node.childForFieldName('import_clause'); + + // CRITICAL FIX: If field-based approach fails, search by node type + if (!importClauseNode) { + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i); + if (child?.type === 'import_clause') { + importClauseNode = child; + break; + } + } + } + if (importClauseNode) { this.processJSImportClause(importClauseNode, filePath, targetFile, imports); + } else { + + // Handle simple imports like: import 'module' + if (node.text.trim().startsWith('import') && !node.text.includes('{') && !node.text.includes('from')) { + imports.push({ + importingFile: filePath, + localName: '_side_effect_', + targetFile, + exportedName: '*', + importType: 'namespace' + }); + + } else { + // Try to extract import manually from text as fallback + const importText = node.text.trim(); + const match = importText.match(/import\s+(.+?)\s+from\s+['"]([^'"]+)['"]/);; + if (match) { + const importPart = match[1].trim(); + + // Handle simple default import + if (!importPart.includes('{') && !importPart.includes('*')) { + imports.push({ + importingFile: filePath, + localName: importPart, + targetFile, + exportedName: 'default', + importType: 'default' + }); + + } + } + } } } else if (node.type === 'variable_declaration') { // Handle CommonJS: const x = require('module') @@ -262,13 +332,44 @@ export class ImportProcessor { targetFile: string, imports: ImportInfo[] ): void { + // Track what we've processed to ensure we don't miss anything + let processedSomething = false; + for (let i = 0; i < importClauseNode.childCount; i++) { const child = importClauseNode.child(i); if (!child) continue; + if (child.type === 'identifier') { + // Default import - this is the most common case we're missing + imports.push({ + importingFile: filePath, + localName: child.text, + targetFile, + exportedName: 'default', + importType: 'default' + }); - if (child.type === 'import_specifier') { - // Named import: { name } or { name as alias } + processedSomething = true; + } else if (child.type === 'named_imports') { + // Process named imports: { a, b, c } + this.processNamedImportsNode(child, filePath, targetFile, imports); + processedSomething = true; + } else if (child.type === 'namespace_import') { + // Namespace import: * as name const nameNode = child.childForFieldName('name'); + if (nameNode) { + imports.push({ + importingFile: filePath, + localName: nameNode.text, + targetFile, + exportedName: '*', + importType: 'namespace' + }); + + processedSomething = true; + } + } else if (child.type === 'import_specifier') { + // Direct import specifier (should be handled by named_imports, but just in case) + const nameNode = child.childForFieldName('name') || child.child(0); const aliasNode = child.childForFieldName('alias'); if (nameNode) { @@ -282,28 +383,98 @@ export class ImportProcessor { exportedName, importType: 'named' }); + + processedSomething = true; } - } else if (child.type === 'namespace_import') { - // Namespace import: * as name - const nameNode = child.childForFieldName('name'); - if (nameNode) { + } + } + + // Fallback: If structured processing didn't work, try text parsing + if (!processedSomething) { + const clauseText = importClauseNode.text.trim(); + + if (clauseText.startsWith('{') && clauseText.endsWith('}')) { + // Named imports like { foo, bar } + const namedImports = clauseText.slice(1, -1) + .split(',') + .map(s => s.trim()) + .filter(s => s.length > 0); + + namedImports.forEach(importName => { imports.push({ importingFile: filePath, - localName: nameNode.text, + localName: importName, + targetFile, + exportedName: importName, + importType: 'named' + }); + }); + + } else if (clauseText.includes(' as ')) { + // Namespace import like * as foo + const namespaceMatch = clauseText.match(/\*\s+as\s+(\w+)/); + if (namespaceMatch) { + imports.push({ + importingFile: filePath, + localName: namespaceMatch[1], targetFile, exportedName: '*', importType: 'namespace' }); + } - } else if (child.type === 'identifier') { - // Default import + } else { + // Simple default import + const defaultName = clauseText.split(',')[0].trim(); // Handle mixed imports + if (defaultName && !defaultName.includes('{') && !defaultName.includes('*')) { + imports.push({ + importingFile: filePath, + localName: defaultName, + targetFile, + exportedName: 'default', + importType: 'default' + }); + + } + } + } + } + + private processNamedImportsNode( + namedImportsNode: Parser.SyntaxNode, + filePath: string, + targetFile: string, + imports: ImportInfo[] + ): void { + for (let j = 0; j < namedImportsNode.childCount; j++) { + const namedChild = namedImportsNode.child(j); + if (namedChild && namedChild.type === 'import_specifier') { + + const nameNode = namedChild.childForFieldName('name') || namedChild.child(0); + const aliasNode = namedChild.childForFieldName('alias'); + + if (nameNode) { + const exportedName = nameNode.text; + const localName = aliasNode ? aliasNode.text : exportedName; + + imports.push({ + importingFile: filePath, + localName, + targetFile, + exportedName, + importType: 'named' + }); + + } + } else if (namedChild && namedChild.type === 'identifier') { imports.push({ importingFile: filePath, - localName: child.text, + localName: namedChild.text, targetFile, - exportedName: 'default', - importType: 'default' + exportedName: namedChild.text, + importType: 'named' }); + } } } diff --git a/src/core/ingestion/parsing-processor.ts b/src/core/ingestion/parsing-processor.ts index bf473ad37..06017135a 100644 --- a/src/core/ingestion/parsing-processor.ts +++ b/src/core/ingestion/parsing-processor.ts @@ -9,7 +9,7 @@ import { } from '../../lib/shared-utils.js'; import { IGNORE_PATTERNS } from '../../config/language-config.js'; import Parser from 'web-tree-sitter'; -import { TYPESCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries'; +import { TYPESCRIPT_QUERIES, JAVASCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries'; import { initTreeSitter, loadTypeScriptParser, loadPythonParser, loadJavaScriptParser } from '../tree-sitter/parser-loader.js'; import { FunctionRegistryTrie, FunctionDefinition } from '../graph/trie.js'; import { LRUCacheService } from '../../lib/lru-cache-service.js'; @@ -226,11 +226,19 @@ export class ParsingProcessor implements GraphProcessor { private async parseFile(graph: KnowledgeGraph, filePath: string, content: string): Promise { const language = this.detectLanguage(filePath); + + // Skip compiled/minified files for JavaScript + if (language === 'javascript' && this.isCompiledOrMinified(content, filePath)) { + console.log(`Skipping compiled/minified file: ${filePath}`); + await this.parseGenericFile(graph, filePath, content); + return; + } + const contentHash = this.generateContentHash(content); const cacheKey = this.lruCache.generateFileCacheKey(filePath, contentHash); - // Check cache first - const cachedResult = this.lruCache.getParsedFile(cacheKey); + // Check cache first - TEMPORARILY DISABLED FOR DEBUGGING + const cachedResult = null; // this.lruCache.getParsedFile(cacheKey); if (cachedResult) { console.log(`Cache hit for file: ${filePath}`); this.astMap.set(filePath, { tree: cachedResult.ast }); @@ -241,7 +249,7 @@ export class ParsingProcessor implements GraphProcessor { const langParser = this.languageParsers.get(language); if (!langParser || !this.parser) { - console.warn(`No parser available for language: ${language}. Skipping file: ${filePath}`); + // Skip file with reduced logging to avoid console spam await this.parseGenericFile(graph, filePath, content); return; } @@ -262,8 +270,8 @@ export class ParsingProcessor implements GraphProcessor { const queryCacheKey = this.lruCache.generateQueryCacheKey(language, queryString); let queryResults: Parser.QueryMatch[] = []; - // Check query cache - const cachedQuery = this.lruCache.getQueryResult(queryCacheKey); + // Check query cache - TEMPORARILY DISABLED FOR DEBUGGING + const cachedQuery = null; // this.lruCache.getQueryResult(queryCacheKey); if (cachedQuery) { queryResults = cachedQuery.results; } else { @@ -284,6 +292,11 @@ export class ParsingProcessor implements GraphProcessor { const definition = this.extractDefinition(node, queryName, filePath); if (definition) { definitions.push(definition); + + // Debug: Log what specific definitions we're finding + if (language === 'python' && filePath.endsWith('.py')) { + console.log(`๐Ÿ” DEBUG: Found ${queryName} -> ${definition.type}: "${definition.name}" in ${filePath.split('/').pop()}`); + } } } } @@ -298,12 +311,39 @@ export class ParsingProcessor implements GraphProcessor { fileSize: content.length }); + // Debug: Log definition extraction results for Python files + if (language === 'python' && filePath.endsWith('.py')) { + console.log(`๐Ÿ” DEBUG: ${filePath.split('/').pop()} -> ${definitions.length} definitions extracted (content: ${content.length} chars, hash: ${contentHash.substring(0, 8)})`); + + // Log definition types for debugging + const definitionTypes = definitions.reduce((acc, def) => { + acc[def.type] = (acc[def.type] || 0) + 1; + return acc; + }, {}); + + if (definitions.length > 0) { + console.log(`๐Ÿ” DEBUG: Definition types: ${JSON.stringify(definitionTypes)}`); + } + + if (definitions.length === 0 && content.length > 100) { + console.log(`๐Ÿšจ DEBUG: Large Python file with no definitions: ${filePath} (${content.length} chars)`); + // Log first few lines to understand the content + const firstLines = content.split('\n').slice(0, 3).join('\\n'); + console.log(`๐Ÿ” DEBUG: File starts with: ${firstLines}`); + } + } + await this.addDefinitionsToGraph(graph, filePath, definitions); } private extractDefinition(node: Parser.SyntaxNode, queryName: string, filePath: string): ParsedDefinition | null { const nameNode = node.childForFieldName('name'); - const name = nameNode ? nameNode.text : 'anonymous'; + const name = nameNode ? nameNode.text : null; + + // Skip anonymous definitions - they're usually from compiled/minified code + if (!name || name === 'anonymous' || name.trim().length === 0) { + return null; + } return { name, @@ -319,16 +359,55 @@ export class ParsingProcessor implements GraphProcessor { case 'classes': return 'class'; case 'methods': return 'method'; case 'functions': - case 'arrowFunctions': return 'function'; + case 'arrowFunctions': + case 'variableAssignments': + case 'objectMethods': return 'function'; case 'imports': case 'from_imports': return 'import'; + case 'exports': + case 'defaultExports': return 'function'; // Exports usually export functions case 'interfaces': return 'interface'; case 'types': return 'type'; case 'decorators': return 'decorator'; - default: return 'variable'; + default: + console.warn(`Unknown query type: ${queryName}, defaulting to 'function'`); + return 'function'; // Better default than 'variable' } } + private isCompiledOrMinified(content: string, filePath: string): boolean { + // Check file name patterns for known compiled files + const fileName = filePath.split('/').pop()?.toLowerCase() || ''; + if (fileName.includes('.min.') || + fileName.includes('.bundle.') || + fileName.includes('tree-sitter.js') || + fileName.includes('kuzu_wasm_worker.js')) { + return true; + } + + // Check content characteristics for minified code + const lines = content.split('\n'); + if (lines.length > 0) { + const firstLine = lines[0]; + + // Very long first line (typical of minified code) + if (firstLine.length > 500) { + return true; + } + + // Contains typical minified patterns + if (firstLine.includes('var Module=void 0!==Module?Module:{}') || + firstLine.includes('__webpack_require__') || + firstLine.includes('!function(') || + content.includes('/*! ') || // Webpack/build tool comments + content.match(/^\s*!function\s*\(/)) { // IIFE patterns + return true; + } + } + + return false; + } + private detectLanguage(filePath: string): string { const extension = pathUtils.extname(filePath).toLowerCase(); @@ -346,8 +425,9 @@ export class ParsingProcessor implements GraphProcessor { private getQueriesForLanguage(language: string): Record | null { switch (language) { case 'typescript': - case 'javascript': return TYPESCRIPT_QUERIES; + case 'javascript': + return JAVASCRIPT_QUERIES; // Use separate JavaScript queries case 'python': return PYTHON_QUERIES; case 'java': @@ -358,18 +438,31 @@ export class ParsingProcessor implements GraphProcessor { } private async parseGenericFile(graph: KnowledgeGraph, filePath: string, _content: string): Promise { - const fileNode: GraphNode = { - id: generateId('file'), - label: 'File' as NodeLabel, - properties: { - name: pathUtils.getFileName(filePath), - path: filePath, - size: _content.length, - language: this.detectLanguage(filePath) - } - }; + // Find existing file node created by StructureProcessor + let fileNode = graph.nodes.find(node => + node.label === 'File' && + (node.properties.filePath === filePath || node.properties.path === filePath) + ); - graph.addNode(fileNode); + // If no existing file node found, create one (fallback) + if (!fileNode) { + fileNode = { + id: generateId(`file_${filePath}`), + label: 'File' as NodeLabel, + properties: { + name: pathUtils.getFileName(filePath), + path: filePath, + filePath: filePath, + size: _content.length, + language: this.detectLanguage(filePath) + } + }; + graph.addNode(fileNode); + } else { + // Update existing file node with additional properties + fileNode.properties.size = _content.length; + fileNode.properties.language = this.detectLanguage(filePath); + } } private async addDefinitionsToGraph( @@ -377,20 +470,30 @@ export class ParsingProcessor implements GraphProcessor { filePath: string, definitions: ParsedDefinition[] ): Promise { - const fileNode: GraphNode = { - id: generateId('file'), - label: 'File' as NodeLabel, - properties: { - name: pathUtils.getFileName(filePath), - path: filePath, - language: this.detectLanguage(filePath) - } - }; + // Find existing file node created by StructureProcessor + let fileNode = graph.nodes.find(node => + node.label === 'File' && + (node.properties.filePath === filePath || node.properties.path === filePath) + ); - graph.addNode(fileNode); + // If no existing file node found, create one (fallback) + if (!fileNode) { + fileNode = { + id: generateId(`file_${filePath}`), + label: 'File' as NodeLabel, + properties: { + name: pathUtils.getFileName(filePath), + path: filePath, + filePath: filePath, + language: this.detectLanguage(filePath) + } + }; + graph.addNode(fileNode); + } for (const def of definitions) { - const nodeId = generateId(def.type); + // Generate unique ID based on file path and definition name + const nodeId = generateId(`${def.type}_${filePath}_${def.name}_${def.startLine}`); if (this.duplicateDetector.checkAndMark(nodeId)) continue; diff --git a/src/core/ingestion/pipeline.ts b/src/core/ingestion/pipeline.ts index 2547bbdfa..2f929abea 100644 --- a/src/core/ingestion/pipeline.ts +++ b/src/core/ingestion/pipeline.ts @@ -113,10 +113,10 @@ export class GraphPipeline { const filesWithoutDefinitions = fileNodes.filter(fileNode => { const hasDefinitions = graph.relationships.some(rel => rel.source === fileNode.id && - rel.type === 'CONTAINS' && + rel.type === 'DEFINES' && graph.nodes.some(targetNode => targetNode.id === rel.target && - ['Function', 'Class', 'Method'].includes(targetNode.label) + ['Function', 'Class', 'Method', 'Variable'].includes(targetNode.label) ) ); return !hasDefinitions; @@ -181,7 +181,7 @@ export class GraphPipeline { const sourceFilesWithoutDefinitions = sourceFiles.filter(fileNode => { const hasDefinitions = graph.relationships.some(rel => rel.source === fileNode.id && - rel.type === 'CONTAINS' && + rel.type === 'DEFINES' && graph.nodes.some(n => n.id === rel.target && ['Function', 'Class', 'Method', 'Variable'].includes(n.label) @@ -205,7 +205,7 @@ export class GraphPipeline { const definitionsWithoutFiles = definitionNodes.filter(defNode => { const hasFileParent = graph.relationships.some(rel => rel.target === defNode.id && - rel.type === 'CONTAINS' && + rel.type === 'DEFINES' && graph.nodes.some(n => n.id === rel.source && n.label === 'File') ); return !hasFileParent; diff --git a/src/core/ingestion/structure-processor.ts b/src/core/ingestion/structure-processor.ts index 918cb93e9..a385a46f9 100644 --- a/src/core/ingestion/structure-processor.ts +++ b/src/core/ingestion/structure-processor.ts @@ -67,10 +67,6 @@ export class StructureProcessor { const visibleDirectories = directories.filter(dir => !this.shouldHideDirectory(dir)); const hiddenDirectoriesCount = directories.length - visibleDirectories.length; - if (hiddenDirectoriesCount > 0) { - console.log(`StructureProcessor: Hiding ${hiddenDirectoriesCount} ignored directories from KG`); - } - // Create directory nodes only for visible directories const directoryNodes = this.createDirectoryNodes(visibleDirectories); directoryNodes.forEach(node => graph.addNode(node)); @@ -79,10 +75,6 @@ export class StructureProcessor { const visibleFiles = files.filter(file => !this.shouldHideFile(file)); const hiddenFilesCount = files.length - visibleFiles.length; - if (hiddenFilesCount > 0) { - console.log(`StructureProcessor: Hiding ${hiddenFilesCount} files in ignored directories from KG`); - } - // Create file nodes only for visible files const fileNodes = this.createFileNodes(visibleFiles); fileNodes.forEach(node => graph.addNode(node)); @@ -91,7 +83,7 @@ export class StructureProcessor { this.createContainsRelationships(graph, projectNode.id, visibleDirectories, visibleFiles); const totalHidden = hiddenDirectoriesCount + hiddenFilesCount; - console.log(`StructureProcessor: Created ${graph.nodes.length} nodes total (${totalHidden} items hidden)`); + console.log(`StructureProcessor: Created ${graph.nodes.length} nodes total (${totalHidden} items filtered)`); } /** @@ -164,7 +156,7 @@ export class StructureProcessor { for (const dirPath of directoryPaths) { if (!dirPath) continue; - const id = generateId('folder'); + const id = generateId(`folder_${dirPath}`); this.nodeIdMap.set(dirPath, id); const pathParts = dirPath.split('/'); @@ -196,7 +188,7 @@ export class StructureProcessor { for (const filePath of filePaths) { if (!filePath) continue; - const id = generateId('file'); + const id = generateId(`file_${filePath}`); this.nodeIdMap.set(filePath, id); const fileName = filePath.split('/').pop() || filePath; diff --git a/src/core/ingestion/tree-sitter-queries.ts b/src/core/ingestion/tree-sitter-queries.ts index 786e95874..5b172f107 100644 --- a/src/core/ingestion/tree-sitter-queries.ts +++ b/src/core/ingestion/tree-sitter-queries.ts @@ -26,6 +26,48 @@ export const TYPESCRIPT_QUERIES = { `, }; +// JavaScript queries - similar to TypeScript but without TS-specific syntax +export const JAVASCRIPT_QUERIES = { + imports: ` + (import_statement) @import + `, + classes: ` + (class_declaration) @class + `, + methods: ` + (method_definition) @method + `, + functions: ` + (function_declaration) @function + `, + arrowFunctions: ` + (lexical_declaration + (variable_declarator + name: (identifier) @name + value: (arrow_function))) @arrow_function + `, + exports: ` + (export_statement) @export + `, + defaultExports: ` + (export_statement + (identifier) @default_export) + `, + variableAssignments: ` + (variable_declaration + (variable_declarator + name: (identifier) @name + value: (function_expression))) @var_function + `, + objectMethods: ` + (assignment_expression + left: (member_expression + property: (property_identifier) @name) + right: (function_expression)) @obj_method + `, + // Note: No interfaces or types for pure JavaScript +}; + export const PYTHON_QUERIES = { imports: ` (import_statement) @import @@ -39,8 +81,14 @@ export const PYTHON_QUERIES = { functions: ` (function_definition) @function `, + methods: ` + (class_definition + body: (block + (function_definition) @method)) + `, decorators: ` - (decorator) @decorator + (decorated_definition + (decorator) @decorator) `, }; diff --git a/src/services/github.ts b/src/services/github.ts index f2b6b6415..ced2e858e 100644 --- a/src/services/github.ts +++ b/src/services/github.ts @@ -477,7 +477,7 @@ export class GitHubService { await this.collectPathsAndContent(owner, repo, '', allPaths, fileContents); - console.log(`GitHub: Discovered ${allPaths.length} total paths, ${fileContents.size} files with content`); + console.log(`GitHub: Extracted ${allPaths.length} paths, ${fileContents.size} files`); return { allPaths, @@ -503,7 +503,7 @@ export class GitHubService { // Always try to get content for files, but skip unwanted files like .git files // Filtering will happen later in ParsingProcessor, but we can skip obvious files if (this.shouldSkipFileForContent(fullPath)) { - console.log(`Skipping file content for: ${fullPath}`); + // Reduced logging to avoid console spam } else { try { const content = await this.getFileContent(owner, repo, fullPath); diff --git a/src/services/ingestion.service.ts b/src/services/ingestion.service.ts index 80af479bc..1076d97eb 100644 --- a/src/services/ingestion.service.ts +++ b/src/services/ingestion.service.ts @@ -153,35 +153,44 @@ export class IngestionService { // Check if all paths start with the same top-level folder const potentialPrefix = pathParts[0] + '/'; - const allHaveSamePrefix = paths.every(path => path.startsWith(potentialPrefix)); + const pathsWithPrefix = paths.filter(path => path.startsWith(potentialPrefix)); - if (!allHaveSamePrefix) { - return structure; // No common prefix to remove - } + // If most paths (>80%) have the common prefix, normalize all paths + if (pathsWithPrefix.length > paths.length * 0.8) { + console.log(`Normalizing ZIP paths: removing common prefix "${potentialPrefix}" from ${pathsWithPrefix.length}/${paths.length} paths`); - console.log(`Normalizing ZIP paths: removing common prefix "${potentialPrefix}"`); + // Remove the common prefix from all paths + const normalizedPaths = paths.map(path => { + if (path.startsWith(potentialPrefix)) { + const withoutPrefix = path.substring(potentialPrefix.length); + return withoutPrefix || path; // Keep original if normalization would result in empty string + } + // For paths without prefix, keep as-is but filter out the bare container name + return path === pathParts[0] ? '' : path; + }).filter(path => path.length > 0); // Remove empty paths - // Remove the common prefix from all paths - const normalizedPaths = paths.map(path => { - const withoutPrefix = path.substring(potentialPrefix.length); - return withoutPrefix || path; // Keep original if normalization would result in empty string - }).filter(path => path.length > 0); // Remove empty paths - - // Normalize file contents map - const normalizedContents = new Map(); - for (const [originalPath, content] of structure.fileContents) { - const normalizedPath = originalPath.startsWith(potentialPrefix) - ? originalPath.substring(potentialPrefix.length) - : originalPath; - - if (normalizedPath) { - normalizedContents.set(normalizedPath, content); + // Normalize file contents map + const normalizedContents = new Map(); + for (const [originalPath, content] of structure.fileContents) { + let normalizedPath = originalPath; + if (originalPath.startsWith(potentialPrefix)) { + normalizedPath = originalPath.substring(potentialPrefix.length); + } else if (originalPath === pathParts[0]) { + // Skip the bare container directory + continue; + } + + if (normalizedPath && normalizedPath.length > 0) { + normalizedContents.set(normalizedPath, content); + } } + + return { + allPaths: normalizedPaths, + fileContents: normalizedContents + }; } - return { - allPaths: normalizedPaths, - fileContents: normalizedContents - }; + return structure; // No normalization if prefix isn't common enough } } \ No newline at end of file diff --git a/src/services/zip.ts b/src/services/zip.ts index e5cf42241..749368159 100644 --- a/src/services/zip.ts +++ b/src/services/zip.ts @@ -109,7 +109,7 @@ export class ZipService { // Skip .git files and other unwanted files during content extraction if (this.shouldSkipFileForContent(normalizedPath)) { - console.log(`Skipping file content for: ${normalizedPath}`); + // Reduced logging - only log summary at the end return; } @@ -149,8 +149,7 @@ export class ZipService { // Wait for all file extractions to complete await Promise.all(filePromises); - console.log(`ZIP: Discovered ${allPaths.length} total paths, ${fileContents.size} files with content`); - console.log(`ZIP: Total extracted size: ${totalExtractedSize} bytes`); + console.log(`ZIP: Extracted ${allPaths.length} paths, ${fileContents.size} files (${totalExtractedSize} bytes)`); return { allPaths: allPaths.sort(), // Sort for consistent ordering diff --git a/src/tests/wasm-verification.test.ts b/src/tests/wasm-verification.test.ts new file mode 100644 index 000000000..3b8aa673e --- /dev/null +++ b/src/tests/wasm-verification.test.ts @@ -0,0 +1,344 @@ +/** + * TEMPORARY WASM VERIFICATION TESTS + * These tests verify that Tree-sitter WASM files are loading and working correctly + * DELETE THIS FILE after confirming everything works + */ + +import { + initTreeSitter, + loadPythonParser, + loadJavaScriptParser, + loadTypeScriptParser +} from '../core/tree-sitter/parser-loader.js'; +import Parser from 'web-tree-sitter'; +import { PYTHON_QUERIES, TYPESCRIPT_QUERIES } from '../core/ingestion/tree-sitter-queries'; + +describe('๐Ÿ” WASM Verification Tests (TEMPORARY)', () => { + let parser: Parser; + + beforeAll(async () => { + console.log('๐Ÿงช Starting WASM verification tests...'); + }); + + afterAll(() => { + console.log('โœ… WASM verification tests completed'); + }); + + describe('Main Thread Tree-sitter Initialization', () => { + it('should initialize Tree-sitter successfully', async () => { + parser = await initTreeSitter(); + expect(parser).toBeDefined(); + expect(parser).toBeInstanceOf(Parser); + console.log('โœ… Tree-sitter main thread initialized successfully'); + }); + + it('should load Python WASM parser', async () => { + const pythonLang = await loadPythonParser(); + expect(pythonLang).toBeDefined(); + expect(typeof pythonLang.query).toBe('function'); + console.log('โœ… Python WASM parser loaded successfully'); + }); + + it('should load JavaScript WASM parser', async () => { + const jsLang = await loadJavaScriptParser(); + expect(jsLang).toBeDefined(); + expect(typeof jsLang.query).toBe('function'); + console.log('โœ… JavaScript WASM parser loaded successfully'); + }); + + it('should load TypeScript WASM parser', async () => { + const tsLang = await loadTypeScriptParser(); + expect(tsLang).toBeDefined(); + expect(typeof tsLang.query).toBe('function'); + console.log('โœ… TypeScript WASM parser loaded successfully'); + }); + }); + + describe('Python WASM Parsing Verification', () => { + let pythonLang: Parser.Language; + + beforeAll(async () => { + pythonLang = await loadPythonParser(); + parser.setLanguage(pythonLang); + }); + + it('should parse simple Python function', () => { + const pythonCode = ` +def test_function(param1, param2): + """Test function docstring""" + return param1 + param2 + +class TestClass: + def method_one(self): + pass + + async def async_method(self): + return "async result" + +variable_assignment = "test value" +`; + + const tree = parser.parse(pythonCode); + expect(tree).toBeDefined(); + expect(tree.rootNode).toBeDefined(); + expect(tree.rootNode.type).toBe('module'); + console.log('โœ… Python code parsed successfully'); + console.log(` Root node type: ${tree.rootNode.type}`); + console.log(` Child count: ${tree.rootNode.childCount}`); + }); + + it('should execute Python queries successfully', () => { + const pythonCode = ` +def hello_world(): + print("Hello, World!") + +class Calculator: + def add(self, a, b): + return a + b + + def subtract(self, a, b): + return a - b + +result = 42 +`; + + const tree = parser.parse(pythonCode); + + // Test each Python query + for (const [queryName, queryString] of Object.entries(PYTHON_QUERIES)) { + try { + const query = pythonLang.query(queryString as string); + const matches = query.matches(tree.rootNode); + + console.log(` โœ… ${queryName} query executed: ${matches.length} matches`); + + // Log some match details for verification + if (matches.length > 0) { + const firstMatch = matches[0]; + console.log(` First match captures: ${firstMatch.captures.length}`); + if (firstMatch.captures.length > 0) { + const firstCapture = firstMatch.captures[0]; + console.log(` First capture type: ${firstCapture.node.type}`); + console.log(` First capture text: "${firstCapture.node.text.substring(0, 50)}..."`); + } + } + } catch (error) { + console.error(` โŒ ${queryName} query failed:`, error); + throw error; + } + } + }); + + it('should extract definitions correctly', () => { + const pythonCode = ` +import os +from datetime import datetime + +def process_data(input_data): + return input_data.upper() + +class DataProcessor: + def __init__(self): + self.data = [] + + def add_item(self, item): + self.data.append(item) + + async def process_async(self): + return await some_async_operation() + +@decorator +def decorated_function(): + pass + +global_var = "test" +`; + + const tree = parser.parse(pythonCode); + let totalDefinitions = 0; + + for (const [queryName, queryString] of Object.entries(PYTHON_QUERIES)) { + const query = pythonLang.query(queryString as string); + const matches = query.matches(tree.rootNode); + + for (const match of matches) { + for (const capture of match.captures) { + totalDefinitions++; + const node = capture.node; + console.log(` Found ${queryName}: "${node.text.split('\n')[0]}" at line ${node.startPosition.row + 1}`); + } + } + } + + expect(totalDefinitions).toBeGreaterThan(0); + console.log(`โœ… Extracted ${totalDefinitions} total definitions`); + }); + }); + + describe('TypeScript WASM Parsing Verification', () => { + let typescriptLang: Parser.Language; + + beforeAll(async () => { + typescriptLang = await loadTypeScriptParser(); + parser.setLanguage(typescriptLang); + }); + + it('should parse TypeScript code', () => { + const tsCode = ` +import React from 'react'; + +interface User { + id: number; + name: string; +} + +class UserService { + getUser(id: number): User { + return { id, name: 'Test User' }; + } +} + +const createUser = (name: string): User => { + return { id: Date.now(), name }; +}; + +function processUser(user: User): void { + console.log(user.name); +} +`; + + const tree = parser.parse(tsCode); + expect(tree).toBeDefined(); + expect(tree.rootNode.type).toBe('program'); + console.log('โœ… TypeScript code parsed successfully'); + }); + + it('should execute TypeScript queries successfully', () => { + const tsCode = ` +class TestClass { + method(): string { + return "test"; + } +} + +function testFunction(): void {} + +const arrowFunc = () => "result"; + +interface TestInterface { + prop: string; +} +`; + + const tree = parser.parse(tsCode); + + for (const [queryName, queryString] of Object.entries(TYPESCRIPT_QUERIES)) { + try { + const query = typescriptLang.query(queryString as string); + const matches = query.matches(tree.rootNode); + console.log(` โœ… ${queryName} query executed: ${matches.length} matches`); + } catch (error) { + console.error(` โŒ ${queryName} query failed:`, error); + throw error; + } + } + }); + }); + + describe('Worker Thread WASM Verification', () => { + it('should verify worker can load and use WASM', async () => { + // Create a test worker to verify WASM loading + const workerCode = ` + import Parser from 'web-tree-sitter'; + + async function testWorkerWasm() { + try { + // Test worker WASM initialization + await Parser.init(); + const parser = new Parser(); + + // Test loading Python parser + const pythonLang = await Parser.Language.load('/wasm/python/tree-sitter-python.wasm'); + parser.setLanguage(pythonLang); + + // Test parsing + const tree = parser.parse('def test(): pass'); + + return { + success: true, + rootNodeType: tree.rootNode.type, + childCount: tree.rootNode.childCount + }; + } catch (error) { + return { + success: false, + error: error.message + }; + } + } + + testWorkerWasm().then(result => { + self.postMessage(result); + }); + `; + + const blob = new Blob([workerCode], { type: 'application/javascript' }); + const workerUrl = URL.createObjectURL(blob); + + const workerResult = await new Promise((resolve, reject) => { + const worker = new Worker(workerUrl, { type: 'module' }); + + worker.onmessage = (event) => { + worker.terminate(); + URL.revokeObjectURL(workerUrl); + resolve(event.data); + }; + + worker.onerror = (error) => { + worker.terminate(); + URL.revokeObjectURL(workerUrl); + reject(error); + }; + + // Timeout after 10 seconds + setTimeout(() => { + worker.terminate(); + URL.revokeObjectURL(workerUrl); + reject(new Error('Worker test timeout')); + }, 10000); + }); + + console.log('๐Ÿ” Worker WASM test result:', workerResult); + expect(workerResult).toHaveProperty('success'); + + if (!(workerResult as any).success) { + throw new Error(`Worker WASM failed: ${(workerResult as any).error}`); + } + + console.log('โœ… Worker WASM verification passed'); + }, 15000); // 15 second timeout for this test + }); + + describe('WASM File Accessibility Check', () => { + it('should verify WASM files are accessible', async () => { + const wasmFiles = [ + '/wasm/python/tree-sitter-python.wasm', + '/wasm/javascript/tree-sitter-javascript.wasm', + '/wasm/typescript/tree-sitter-typescript.wasm', + '/wasm/tree-sitter.wasm' + ]; + + for (const wasmPath of wasmFiles) { + try { + const response = await fetch(wasmPath); + expect(response.ok).toBe(true); + expect(response.headers.get('content-type')).toContain('wasm'); + console.log(`โœ… ${wasmPath} is accessible (${response.status})`); + } catch (error) { + console.error(`โŒ ${wasmPath} failed to load:`, error); + throw error; + } + } + }); + }); +}); \ No newline at end of file diff --git a/src/ui/components/graph/FloatingSourceViewer.tsx b/src/ui/components/graph/FloatingSourceViewer.tsx index 546167ec1..750cfed90 100644 --- a/src/ui/components/graph/FloatingSourceViewer.tsx +++ b/src/ui/components/graph/FloatingSourceViewer.tsx @@ -244,22 +244,43 @@ const FloatingSourceViewer: React.FC = ({ rel.type === 'CONTAINS' && rel.target === nodeId ); - if (containsRel) { - const sourceNode = graph.nodes.find(n => n.id === containsRel.source); + // If no CONTAINS relationship found, try DEFINES relationship (fallback) + const definesRel = !containsRel ? graph.relationships.find(rel => + rel.type === 'DEFINES' && rel.target === nodeId + ) : null; + + const fileRel = containsRel || definesRel; + + if (fileRel) { + const sourceNode = graph.nodes.find(n => n.id === fileRel.source); if (sourceNode && sourceNode.properties.filePath) { filePath = sourceNode.properties.filePath as string; } } // If not found through relationships, search through file contents - if (!filePath && fileContents) { + // BUT ONLY for Function, Method, Class, and Variable nodes - NOT for Folder or File nodes + if (!filePath && fileContents && ['Function', 'Method', 'Class', 'Variable'].includes(nodeType)) { + console.log('FloatingSourceViewer - Searching file contents for node:', nodeName, 'of type:', nodeType); + + // Use more specific search patterns instead of just checking if content includes nodeName + const searchPatterns = [ + `def ${nodeName}(`, // Python function + `function ${nodeName}(`, // JavaScript function + `const ${nodeName} =`, // JavaScript const + `class ${nodeName}`, // Class definition + ]; + for (const [path, content] of fileContents) { // Skip .git files, node_modules, and other unwanted directories if (shouldSkipFileForSearch(path)) { continue; } - if (content.includes(nodeName)) { + // Check if any specific pattern matches instead of just nodeName + const foundPattern = searchPatterns.find(pattern => content.includes(pattern)); + if (foundPattern) { + console.log(`FloatingSourceViewer - Found pattern "${foundPattern}" in ${path}`); filePath = path; break; } @@ -267,25 +288,73 @@ const FloatingSourceViewer: React.FC = ({ } if (!filePath) { + // Special handling for Folder nodes + if (nodeType === 'Folder') { + const folderPath = node.properties.path as string || nodeName; + const childFiles = graph?.nodes?.filter(n => + n.label === 'File' && + (n.properties.filePath as string || n.properties.path as string || '').startsWith(folderPath + '/') + ) || []; + + const childFolders = graph?.nodes?.filter(n => + n.label === 'Folder' && + (n.properties.path as string || '').startsWith(folderPath + '/') && + (n.properties.path as string || '').split('/').length === folderPath.split('/').length + 1 + ) || []; + + let directoryContent = `# Directory: ${folderPath}\n`; + directoryContent += `# This folder contains ${childFiles.length} files and ${childFolders.length} subfolders\n\n`; + + if (childFolders.length > 0) { + directoryContent += '## Subdirectories:\n'; + childFolders.forEach(folder => { + const name = folder.properties.name as string || 'Unknown'; + directoryContent += `- ๐Ÿ“ ${name}\n`; + }); + directoryContent += '\n'; + } + + if (childFiles.length > 0) { + directoryContent += '## Files:\n'; + childFiles.slice(0, 15).forEach(file => { + const name = file.properties.name as string || 'Unknown'; + const ext = file.properties.extension as string || ''; + const icon = ext === '.py' ? '๐Ÿ' : ext === '.js' ? '๐Ÿ“œ' : ext === '.ts' ? '๐Ÿ“˜' : '๐Ÿ“„'; + directoryContent += `- ${icon} ${name}\n`; + }); + if (childFiles.length > 15) { + directoryContent += `... and ${childFiles.length - 15} more files\n`; + } + } + + return { + fileName: folderPath.split('/').pop() || folderPath, + filePath: 'folder-listing', + content: directoryContent, + nodeType, + nodeName, + language: 'markdown' + }; + } + // Return mock content for nodes without file association const mockContent = `// ${nodeType}: ${nodeName} -// This ${nodeType.toLowerCase()} is part of the knowledge graph -// File path not available in the current context +// This is a ${nodeType.toLowerCase()} definition in the knowledge graph ${nodeType === 'Function' ? `function ${nodeName}() { - // Implementation details would be here - return true; + // Function implementation not available in current context + // This may be an external library function or incomplete parsing }` : nodeType === 'Class' ? `class ${nodeName} { - constructor() { - // Constructor implementation - } - - // Class methods would be here -}` : `// ${nodeType} definition for ${nodeName}`}`; + // Class definition not available in current context + // This may be an external library class or incomplete parsing +}` : nodeType === 'Method' ? `${nodeName}() { + // Method implementation not available in current context +}` : `// ${nodeType} definition for ${nodeName} +// Content not available in current context`}`; return { - fileName: `${nodeName}.${nodeType.toLowerCase()}`, - filePath: 'graph-node', + fileName: `${nodeName}`, + filePath: 'virtual-node', content: mockContent, nodeType, nodeName, @@ -293,20 +362,52 @@ ${nodeType === 'Function' ? `function ${nodeName}() { }; } - const content = fileContents.get(filePath); - if (!content) return null; + // Try to find the file content using various path resolution strategies + const nodeFilePath = node.properties.filePath as string || + node.properties.path as string || + node.properties.name as string; + + if (nodeFilePath) { + // Try exact match first + let content = fileContents.get(nodeFilePath); + + // If not found, try different path variations + if (!content) { + // Try relative paths starting from different roots + const pathVariations = [ + nodeFilePath, + nodeFilePath.replace(/^[./]*/, ''), // Remove leading ./ or / + nodeFilePath.startsWith('/') ? nodeFilePath.substring(1) : `/${nodeFilePath}`, // Toggle leading slash + `src/${nodeFilePath}`, // Try under src/ + nodeFilePath.replace(/\\/g, '/'), // Convert backslashes to forward slashes + nodeFilePath.replace(/\//g, '\\') // Convert forward slashes to backslashes + ]; + + for (const variation of pathVariations) { + content = fileContents.get(variation); + if (content) { + break; + } + } + } + + if (content) { + const extractedContent = extractRelevantContent(content, nodeName, nodeType); + const language = nodeFilePath.split('.').pop() || 'text'; - const extractedContent = extractRelevantContent(content, nodeName, nodeType); - const language = filePath.split('.').pop() || 'text'; - - return { - fileName: filePath.split('/').pop() || filePath, - filePath, - content: extractedContent || content.substring(0, 500) + '...', - nodeType, - nodeName, - language - }; + return { + fileName: nodeFilePath.split('/').pop() || nodeFilePath, + filePath: nodeFilePath, + content: extractedContent || content.substring(0, 500) + '...', + nodeType, + nodeName, + language + }; + } + } + + // If no content found, return null to trigger the mock content generation above + return null; }, [nodeId, graph, fileContents]); // Dragging functionality diff --git a/src/ui/components/graph/SourceViewer.tsx b/src/ui/components/graph/SourceViewer.tsx index 89af3fb91..04e52929c 100644 --- a/src/ui/components/graph/SourceViewer.tsx +++ b/src/ui/components/graph/SourceViewer.tsx @@ -423,17 +423,25 @@ def hasattr(obj: Any, name: str) -> bool: rel.type === 'CONTAINS' && rel.target === selectedNodeId ); - if (containsRelationship) { - const fileNode = graph.nodes.find(n => n.id === containsRelationship.source); + // If no CONTAINS relationship found, try DEFINES relationship (fallback) + const definesRelationship = !containsRelationship ? graph.relationships?.find(rel => + rel.type === 'DEFINES' && rel.target === selectedNodeId + ) : null; + + const fileRelationship = containsRelationship || definesRelationship; + + if (fileRelationship) { + const fileNode = graph.nodes.find(n => n.id === fileRelationship.source); if (fileNode && fileNode.label === 'File') { filePath = fileNode.properties.path as string || fileNode.properties.filePath as string; - console.log('SourceViewer - Found file through CONTAINS relationship:', filePath); + console.log('SourceViewer - Found file through relationship:', filePath, 'via', fileRelationship.type); } } // If still no file path, try reverse lookup by searching for the node name in file contents - if (!filePath && fileContents) { - console.log('SourceViewer - Searching file contents for node:', nodeName); + // BUT ONLY for Function, Method, Class, and Variable nodes - NOT for Folder or File nodes + if (!filePath && fileContents && ['Function', 'Method', 'Class', 'Variable'].includes(node.label)) { + console.log('SourceViewer - Searching file contents for node:', nodeName, 'of type:', node.label); // Try multiple search patterns for the function const searchPatterns = [ @@ -481,8 +489,9 @@ def hasattr(obj: Any, name: str) -> bool: } // If still not found, try a more lenient search (case-insensitive) - if (!filePath) { - console.log('SourceViewer - Trying case-insensitive search for:', nodeName); + // BUT ONLY for Function, Method, Class, and Variable nodes + if (!filePath && ['Function', 'Method', 'Class', 'Variable'].includes(node.label)) { + console.log('SourceViewer - Trying case-insensitive search for:', nodeName, 'of type:', node.label); for (const [path, content] of fileContents) { // Skip .git files, node_modules, and other unwanted directories if (shouldSkipFileForSearch(path)) { @@ -510,10 +519,20 @@ def hasattr(obj: Any, name: str) -> bool: console.log('SourceViewer - Final node details:', { nodeId: selectedNodeId, nodeName, + nodeLabel: node.label, + nodeType: node.label, filePath, fileName, - nodeLabel: node.label, nodeProperties: node.properties, + // CRITICAL DEBUG: Show what type this node actually is + actualNodeInfo: { + isFolder: node.label === 'Folder', + isFile: node.label === 'File', + isFunction: node.label === 'Function', + isClass: node.label === 'Class', + nodeLabel: node.label, + nodeName: nodeName + }, fileContentsSize: fileContents?.size || 0, // Add detailed relationship debugging allRelationships: graph?.relationships?.length || 0, @@ -536,7 +555,55 @@ def hasattr(obj: Any, name: str) -> bool: // Try to get actual file content let content = ''; - if (filePath && fileContents && fileContents.has(filePath)) { + // Special handling for Folder nodes + if (node.label === 'Folder') { + const folderPath = filePath || nodeName || 'Unknown Folder'; + console.log('SourceViewer - Processing folder node:', { + folderPath, + nodeName, + nodeProperties: node.properties + }); + + const childFiles = graph?.nodes?.filter(n => + n.label === 'File' && + (n.properties.filePath as string || n.properties.path as string || '').startsWith(folderPath + '/') + ) || []; + + const childFolders = graph?.nodes?.filter(n => + n.label === 'Folder' && + (n.properties.path as string || '').startsWith(folderPath + '/') && + (n.properties.path as string || '').split('/').length === folderPath.split('/').length + 1 + ) || []; + + content = `# Directory: ${folderPath}\n`; + content += `# Node Type: ${node.label}\n`; + content += `# This folder contains ${childFiles.length} files and ${childFolders.length} subfolders\n\n`; + + if (childFolders.length > 0) { + content += '## Subdirectories:\n'; + childFolders.forEach(folder => { + const name = folder.properties.name as string || 'Unknown'; + content += `- ๐Ÿ“ ${name}\n`; + }); + content += '\n'; + } + + if (childFiles.length > 0) { + content += '## Files:\n'; + childFiles.slice(0, 20).forEach(file => { + const name = file.properties.name as string || 'Unknown'; + const ext = file.properties.extension as string || ''; + const icon = ext === '.py' ? '๐Ÿ' : ext === '.js' ? '๐Ÿ“œ' : ext === '.ts' ? '๐Ÿ“˜' : '๐Ÿ“„'; + content += `- ${icon} ${name}\n`; + }); + if (childFiles.length > 20) { + content += `... and ${childFiles.length - 20} more files\n`; + } + } + + console.log('SourceViewer - Generated directory content for folder:', folderPath); + } + else if (filePath && fileContents && fileContents.has(filePath)) { content = fileContents.get(filePath)!; console.log('SourceViewer - Found file content for:', filePath); diff --git a/src/ui/components/graph/Visualization.tsx b/src/ui/components/graph/Visualization.tsx index 29956a46c..c8f8aa392 100644 --- a/src/ui/components/graph/Visualization.tsx +++ b/src/ui/components/graph/Visualization.tsx @@ -195,7 +195,7 @@ const GraphVisualization: React.FC = ({ return { id: node.id, - label: node.properties.name as string || node.id, + label: getNodeDisplayName(node), nodeType: node.label.toLowerCase(), properties: node.properties, color, @@ -258,6 +258,50 @@ const GraphVisualization: React.FC = ({ return { nodes, links }; }; + // Helper function to get proper display name for nodes + const getNodeDisplayName = (node: GraphNode): string => { + // Use the name property if available + if (node.properties.name && typeof node.properties.name === 'string') { + const name = node.properties.name; + + // For file nodes, show just the filename without path + if (node.label.toLowerCase() === 'file') { + const fileName = name.split('/').pop() || name; + return fileName; + } + + // For other nodes, use the name as-is + return name; + } + + // Fallback to filePath for file nodes + if (node.label.toLowerCase() === 'file' && node.properties.filePath) { + const filePath = node.properties.filePath as string; + const fileName = filePath.split('/').pop() || filePath; + return fileName; + } + + // For function/method/class nodes, try common property names + if (['function', 'method', 'class', 'interface'].includes(node.label.toLowerCase())) { + const functionName = node.properties.functionName || + node.properties.methodName || + node.properties.className || + node.properties.interfaceName; + if (functionName && typeof functionName === 'string') { + return functionName; + } + } + + // Last resort: use a cleaned version of the node ID + let displayName = node.id; + + // Remove common prefixes that might make it look like placeholder text + displayName = displayName.replace(/^(file|function|method|class)_?/i, ''); + displayName = displayName.replace(/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i, 'Unknown'); + + return displayName; + }; + // Helper function to adjust color brightness const adjustColorBrightness = (hex: string, percent: number): string => { const num = parseInt(hex.replace("#", ""), 16); diff --git a/src/workers/ingestion.worker.ts b/src/workers/ingestion.worker.ts index 78dea1718..f5b7b1e72 100644 --- a/src/workers/ingestion.worker.ts +++ b/src/workers/ingestion.worker.ts @@ -70,8 +70,8 @@ export class IngestionWorker { fileContents: fileContentsMap }); - // Clear file contents to free memory after processing - fileContentsMap.clear(); + // Note: Keeping file contents available for UI components + // fileContentsMap.clear(); // Commented out to preserve file contents for SourceViewer const duration = Date.now() - startTime; diff --git a/src/workers/kuzu-ingestion.worker.ts b/src/workers/kuzu-ingestion.worker.ts index 2f5f3f93e..57d83f3f0 100644 --- a/src/workers/kuzu-ingestion.worker.ts +++ b/src/workers/kuzu-ingestion.worker.ts @@ -71,8 +71,8 @@ export class KuzuIngestionWorker { fileContents: fileContentsMap }); - // Clear file contents to free memory after processing - fileContentsMap.clear(); + // Note: Keeping file contents available for UI components + // fileContentsMap.clear(); // Commented out to preserve file contents for SourceViewer const duration = Date.now() - startTime; diff --git a/test-export-functionality.html b/test-export-functionality.html new file mode 100644 index 000000000..9aafa6f11 --- /dev/null +++ b/test-export-functionality.html @@ -0,0 +1,753 @@ + + + + + + Export Functionality Test + + + +
+

๐Ÿงช Export Functionality Test

+

This test verifies that the export functionality works correctly after our import resolution improvements.

+ +
+

๐Ÿ“Š Test Knowledge Graph

+

We'll create a sample knowledge graph with the types of nodes our import resolution system now properly handles.

+ +
+
+ +
+

๐Ÿ” Validation Tests

+

Test graph validation before export:

+ + +
+
+ +
+

๐Ÿ“ฅ Export Tests

+

Test different export formats:

+ + + + +
+ +
+

๐Ÿ“ค Import Test

+

Test importing exported data:

+ +
+
+ +
+

๐Ÿ”ง Console Output

+
+
+
+ + + + \ No newline at end of file