mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-14 23:22:54 +00:00
log verbosity decreased, call resolutions improved for py and ts, multiple other changes
This commit is contained in:
abhigyantrumio
parent
5c9b1281f3
commit
e99a636bc3
23 changed files with 2510 additions and 289 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -12,6 +12,9 @@ dist
|
|||
dist-ssr
|
||||
*.local
|
||||
|
||||
# Auto-generated files
|
||||
public/workers/compiled-queries.js
|
||||
|
||||
# Editor directories and files
|
||||
.vscode/*
|
||||
!.vscode/extensions.json
|
||||
|
|
|
|||
|
|
@ -4,8 +4,9 @@
|
|||
"version": "1.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "tsc -b && vite build",
|
||||
"dev": "npm run compile-queries && vite",
|
||||
"build": "npm run compile-queries && tsc -b && vite build",
|
||||
"compile-queries": "node scripts/compile-queries.js",
|
||||
"lint": "eslint .",
|
||||
"preview": "vite preview",
|
||||
"test": "jest",
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@
|
|||
|
||||
// Import tree-sitter and language parsers
|
||||
import Parser from 'web-tree-sitter';
|
||||
// Import compiled queries (generated from TypeScript)
|
||||
import { getQueriesForLanguage } from './compiled-queries.js';
|
||||
|
||||
// Initialize tree-sitter
|
||||
let parser = null;
|
||||
|
|
@ -98,109 +100,61 @@ function extractDefinitions(tree, filePath) {
|
|||
return definitions;
|
||||
}
|
||||
|
||||
// Get queries for specific language
|
||||
function getQueriesForLanguage(language) {
|
||||
const queries = {
|
||||
typescript: {
|
||||
function_declaration: `
|
||||
(function_declaration
|
||||
name: (identifier) @function.name
|
||||
parameters: (formal_parameters) @function.parameters
|
||||
body: (statement_block) @function.body
|
||||
)
|
||||
`,
|
||||
class_declaration: `
|
||||
(class_declaration
|
||||
name: (identifier) @class.name
|
||||
body: (class_body) @class.body
|
||||
)
|
||||
`,
|
||||
method_definition: `
|
||||
(method_definition
|
||||
name: (property_identifier) @method.name
|
||||
parameters: (formal_parameters) @method.parameters
|
||||
body: (statement_block) @method.body
|
||||
)
|
||||
`,
|
||||
import_statement: `
|
||||
(import_statement
|
||||
source: (string) @import.source
|
||||
)
|
||||
`
|
||||
},
|
||||
javascript: {
|
||||
function_declaration: `
|
||||
(function_declaration
|
||||
name: (identifier) @function.name
|
||||
parameters: (formal_parameters) @function.parameters
|
||||
body: (statement_block) @function.body
|
||||
)
|
||||
`,
|
||||
arrow_function: `
|
||||
(arrow_function
|
||||
parameters: (formal_parameters) @function.parameters
|
||||
body: (statement_block) @function.body
|
||||
)
|
||||
`,
|
||||
class_declaration: `
|
||||
(class_declaration
|
||||
name: (identifier) @class.name
|
||||
body: (class_body) @class.body
|
||||
)
|
||||
`
|
||||
},
|
||||
python: {
|
||||
function_definition: `
|
||||
(function_definition
|
||||
name: (identifier) @function.name
|
||||
parameters: (parameters) @function.parameters
|
||||
body: (block) @function.body
|
||||
)
|
||||
`,
|
||||
class_definition: `
|
||||
(class_definition
|
||||
name: (identifier) @class.name
|
||||
body: (block) @class.body
|
||||
)
|
||||
`,
|
||||
import_statement: `
|
||||
(import_statement
|
||||
name: (dotted_name) @import.name
|
||||
)
|
||||
`
|
||||
}
|
||||
};
|
||||
// Get queries for specific language - now uses imported compiled queries
|
||||
// This ensures consistency with the main thread parsing logic
|
||||
|
||||
return queries[language] || null;
|
||||
// Helper function to map query names to definition types (matches main thread)
|
||||
function getDefinitionType(queryName) {
|
||||
switch (queryName) {
|
||||
case 'classes': return 'class';
|
||||
case 'methods': return 'method';
|
||||
case 'functions':
|
||||
case 'arrowFunctions': return 'function';
|
||||
case 'imports':
|
||||
case 'from_imports': return 'import';
|
||||
case 'interfaces': return 'interface';
|
||||
case 'types': return 'type';
|
||||
case 'decorators': return 'decorator';
|
||||
default: return 'variable';
|
||||
}
|
||||
}
|
||||
|
||||
// Process a query match into a definition
|
||||
// Updated to match main thread's extractDefinition logic exactly
|
||||
function processMatch(match, filePath, queryType) {
|
||||
try {
|
||||
const captures = match.captures;
|
||||
const definition = {
|
||||
type: queryType,
|
||||
filePath,
|
||||
startLine: match.node.startPosition.row,
|
||||
endLine: match.node.endPosition.row,
|
||||
startColumn: match.node.startPosition.column,
|
||||
endColumn: match.node.endPosition.column
|
||||
};
|
||||
|
||||
// Extract specific information based on query type
|
||||
for (const capture of captures) {
|
||||
const { name, node } = capture;
|
||||
// Main thread processes each capture individually
|
||||
// For each match, process all captures
|
||||
const definitions = [];
|
||||
|
||||
for (const capture of match.captures) {
|
||||
const node = capture.node;
|
||||
|
||||
if (name.includes('name')) {
|
||||
definition.name = node.text;
|
||||
} else if (name.includes('parameters')) {
|
||||
definition.parameters = node.text;
|
||||
} else if (name.includes('source')) {
|
||||
definition.importSource = node.text.replace(/['"]/g, '');
|
||||
}
|
||||
}
|
||||
// Extract name from the node (same logic as main thread)
|
||||
const nameNode = node.childForFieldName('name');
|
||||
const name = nameNode ? nameNode.text : 'anonymous';
|
||||
|
||||
return definition;
|
||||
const definition = {
|
||||
name: name,
|
||||
type: getDefinitionType(queryType), // Use proper type mapping
|
||||
filePath: filePath,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
startColumn: node.startPosition.column,
|
||||
endColumn: node.endPosition.column
|
||||
};
|
||||
|
||||
// Extract additional fields if available
|
||||
const parametersNode = node.childForFieldName('parameters');
|
||||
if (parametersNode) {
|
||||
definition.parameters = parametersNode.text;
|
||||
}
|
||||
|
||||
definitions.push(definition);
|
||||
}
|
||||
|
||||
// Return the first definition (main thread processes one capture at a time)
|
||||
return definitions.length > 0 ? definitions[0] : null;
|
||||
} catch (error) {
|
||||
console.warn('Worker: Error processing match:', error);
|
||||
return null;
|
||||
|
|
|
|||
94
scripts/compile-queries.js
Normal file
94
scripts/compile-queries.js
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Build-time script to compile TypeScript queries into JavaScript for Web Workers
|
||||
* This allows workers to import the same query definitions as the main thread
|
||||
*/
|
||||
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Path to the TypeScript queries file
|
||||
const queriesPath = path.join(__dirname, '../src/core/ingestion/tree-sitter-queries.ts');
|
||||
// Output path for compiled JavaScript queries
|
||||
const outputPath = path.join(__dirname, '../public/workers/compiled-queries.js');
|
||||
|
||||
function compileQueries() {
|
||||
try {
|
||||
console.log('🔨 Compiling Tree-sitter queries for Web Workers...');
|
||||
|
||||
// Read the TypeScript queries file
|
||||
const queriesContent = fs.readFileSync(queriesPath, 'utf8');
|
||||
|
||||
// Extract the query objects using simple regex (since they're just object literals)
|
||||
const typescriptMatch = queriesContent.match(/export const TYPESCRIPT_QUERIES = ({[\s\S]*?});/);
|
||||
const javascriptMatch = queriesContent.match(/export const JAVASCRIPT_QUERIES = ({[\s\S]*?});/);
|
||||
const pythonMatch = queriesContent.match(/export const PYTHON_QUERIES = ({[\s\S]*?});/);
|
||||
const javaMatch = queriesContent.match(/export const JAVA_QUERIES = ({[\s\S]*?});/);
|
||||
|
||||
if (!typescriptMatch || !javascriptMatch || !pythonMatch || !javaMatch) {
|
||||
throw new Error('Could not extract queries from TypeScript file');
|
||||
}
|
||||
|
||||
// Create JavaScript module content
|
||||
const jsContent = `/**
|
||||
* AUTO-GENERATED FILE - DO NOT EDIT MANUALLY
|
||||
* Generated from src/core/ingestion/tree-sitter-queries.ts
|
||||
* Run 'npm run compile-queries' to regenerate
|
||||
*/
|
||||
|
||||
export const TYPESCRIPT_QUERIES = ${typescriptMatch[1]};
|
||||
|
||||
export const JAVASCRIPT_QUERIES = ${javascriptMatch[1]};
|
||||
|
||||
export const PYTHON_QUERIES = ${pythonMatch[1]};
|
||||
|
||||
export const JAVA_QUERIES = ${javaMatch[1]};
|
||||
|
||||
// Helper function to get queries for a specific language
|
||||
export function getQueriesForLanguage(language) {
|
||||
switch (language) {
|
||||
case 'typescript':
|
||||
return TYPESCRIPT_QUERIES;
|
||||
case 'javascript':
|
||||
return JAVASCRIPT_QUERIES;
|
||||
case 'python':
|
||||
return PYTHON_QUERIES;
|
||||
case 'java':
|
||||
return JAVA_QUERIES;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Export individual query sets for backward compatibility
|
||||
export const queries = {
|
||||
typescript: TYPESCRIPT_QUERIES,
|
||||
javascript: JAVASCRIPT_QUERIES,
|
||||
python: PYTHON_QUERIES,
|
||||
java: JAVA_QUERIES
|
||||
};
|
||||
`;
|
||||
|
||||
// Ensure output directory exists
|
||||
const outputDir = path.dirname(outputPath);
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Write the compiled JavaScript file
|
||||
fs.writeFileSync(outputPath, jsContent, 'utf8');
|
||||
|
||||
console.log(`✅ Queries compiled successfully to: ${outputPath}`);
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Failed to compile queries:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
compileQueries();
|
||||
43
scripts/test-wasm.js
Normal file
43
scripts/test-wasm.js
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* TEMPORARY WASM TEST RUNNER
|
||||
* Run this to verify Tree-sitter WASM functionality
|
||||
* DELETE THIS FILE after confirming everything works
|
||||
*/
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const projectRoot = join(__dirname, '..');
|
||||
|
||||
console.log('🔍 Starting WASM Verification Tests...');
|
||||
console.log('📁 Project root:', projectRoot);
|
||||
|
||||
// Run the WASM verification tests
|
||||
const testProcess = spawn('npm', ['run', 'test', 'src/tests/wasm-verification.test.ts'], {
|
||||
cwd: projectRoot,
|
||||
stdio: 'inherit',
|
||||
shell: true
|
||||
});
|
||||
|
||||
testProcess.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
console.log('\n✅ All WASM verification tests passed!');
|
||||
console.log('🧹 You can now safely delete:');
|
||||
console.log(' - src/tests/wasm-verification.test.ts');
|
||||
console.log(' - scripts/test-wasm.js');
|
||||
} else {
|
||||
console.log('\n❌ WASM verification tests failed!');
|
||||
console.log('🔧 Please check the issues above before proceeding.');
|
||||
}
|
||||
process.exit(code);
|
||||
});
|
||||
|
||||
testProcess.on('error', (error) => {
|
||||
console.error('❌ Failed to run tests:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
223
scripts/verify-wasm-direct.js
Normal file
223
scripts/verify-wasm-direct.js
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* DIRECT WASM VERIFICATION SCRIPT
|
||||
* Tests Tree-sitter WASM functionality without Jest
|
||||
* DELETE THIS FILE after confirming everything works
|
||||
*/
|
||||
|
||||
import { createRequire } from 'module';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname, join } from 'path';
|
||||
import fs from 'fs';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const projectRoot = dirname(__dirname);
|
||||
|
||||
console.log('🔍 Direct WASM Verification Starting...');
|
||||
console.log('📁 Project root:', projectRoot);
|
||||
|
||||
// Test 1: Check if WASM files exist
|
||||
async function checkWasmFiles() {
|
||||
console.log('\n📋 Test 1: Checking WASM file existence...');
|
||||
|
||||
const wasmFiles = [
|
||||
'public/wasm/python/tree-sitter-python.wasm',
|
||||
'public/wasm/javascript/tree-sitter-javascript.wasm',
|
||||
'public/wasm/typescript/tree-sitter-typescript.wasm',
|
||||
'public/wasm/tree-sitter.wasm'
|
||||
];
|
||||
|
||||
let allExist = true;
|
||||
|
||||
for (const wasmPath of wasmFiles) {
|
||||
const fullPath = join(projectRoot, wasmPath);
|
||||
if (fs.existsSync(fullPath)) {
|
||||
const stats = fs.statSync(fullPath);
|
||||
console.log(`✅ ${wasmPath} exists (${(stats.size / 1024).toFixed(1)}KB)`);
|
||||
} else {
|
||||
console.log(`❌ ${wasmPath} missing`);
|
||||
allExist = false;
|
||||
}
|
||||
}
|
||||
|
||||
return allExist;
|
||||
}
|
||||
|
||||
// Test 2: Try to load Tree-sitter in Node.js environment
|
||||
async function testTreeSitterLoad() {
|
||||
console.log('\n📋 Test 2: Testing Tree-sitter module loading...');
|
||||
|
||||
try {
|
||||
// Try dynamic import
|
||||
const Parser = await import('web-tree-sitter');
|
||||
console.log('✅ web-tree-sitter module imported successfully');
|
||||
console.log(` Default export type: ${typeof Parser.default}`);
|
||||
|
||||
if (Parser.default && typeof Parser.default === 'function') {
|
||||
console.log('✅ Parser constructor available');
|
||||
return true;
|
||||
} else {
|
||||
console.log('❌ Parser constructor not found');
|
||||
return false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Failed to import web-tree-sitter:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Test 3: Check compiled queries
|
||||
async function testCompiledQueries() {
|
||||
console.log('\n📋 Test 3: Testing compiled queries...');
|
||||
|
||||
try {
|
||||
const compiledQueriesPath = join(projectRoot, 'public/workers/compiled-queries.js');
|
||||
|
||||
if (!fs.existsSync(compiledQueriesPath)) {
|
||||
console.log('❌ compiled-queries.js not found');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('✅ compiled-queries.js exists');
|
||||
|
||||
// Read and basic parse check
|
||||
const content = fs.readFileSync(compiledQueriesPath, 'utf8');
|
||||
|
||||
if (content.includes('PYTHON_QUERIES') && content.includes('TYPESCRIPT_QUERIES')) {
|
||||
console.log('✅ Compiled queries contain expected exports');
|
||||
|
||||
// Count Python queries
|
||||
const pythonQueryMatch = content.match(/PYTHON_QUERIES = ({[\\s\\S]*?});/);
|
||||
if (pythonQueryMatch) {
|
||||
const queryCount = (pythonQueryMatch[1].match(/:\\s*`/g) || []).length;
|
||||
console.log(`✅ Python queries: ${queryCount} query types found`);
|
||||
}
|
||||
|
||||
return true;
|
||||
} else {
|
||||
console.log('❌ Compiled queries missing expected exports');
|
||||
return false;
|
||||
}
|
||||
} catch (error) {
|
||||
console.log('❌ Error checking compiled queries:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Test 4: Check parser loader module
|
||||
async function testParserLoader() {
|
||||
console.log('\n📋 Test 4: Testing parser loader module...');
|
||||
|
||||
try {
|
||||
const parserLoaderPath = join(projectRoot, 'src/core/tree-sitter/parser-loader.ts');
|
||||
|
||||
if (!fs.existsSync(parserLoaderPath)) {
|
||||
console.log('❌ parser-loader.ts not found');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('✅ parser-loader.ts exists');
|
||||
|
||||
// Try importing the module
|
||||
const loaderModule = await import(`file://${parserLoaderPath}`);
|
||||
|
||||
const expectedExports = ['initTreeSitter', 'loadPythonParser', 'loadJavaScriptParser', 'loadTypeScriptParser'];
|
||||
let allExportsPresent = true;
|
||||
|
||||
for (const exportName of expectedExports) {
|
||||
if (typeof loaderModule[exportName] === 'function') {
|
||||
console.log(`✅ ${exportName} function exported`);
|
||||
} else {
|
||||
console.log(`❌ ${exportName} function missing`);
|
||||
allExportsPresent = false;
|
||||
}
|
||||
}
|
||||
|
||||
return allExportsPresent;
|
||||
} catch (error) {
|
||||
console.log('❌ Error testing parser loader:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Test 5: Verify Tree-sitter queries syntax
|
||||
async function testQuerySyntax() {
|
||||
console.log('\n📋 Test 5: Testing Tree-sitter query syntax...');
|
||||
|
||||
try {
|
||||
const queriesPath = join(projectRoot, 'src/core/ingestion/tree-sitter-queries.ts');
|
||||
|
||||
if (!fs.existsSync(queriesPath)) {
|
||||
console.log('❌ tree-sitter-queries.ts not found');
|
||||
return false;
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(queriesPath, 'utf8');
|
||||
|
||||
// Basic syntax checks
|
||||
if (!content.includes('PYTHON_QUERIES') || !content.includes('TYPESCRIPT_QUERIES')) {
|
||||
console.log('❌ Missing expected query exports');
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check for the async function query issue we fixed
|
||||
if (content.includes('async_functions')) {
|
||||
console.log('❌ async_functions query still present (should be removed)');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('✅ Query file structure looks correct');
|
||||
console.log('✅ No problematic async_functions query found');
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log('❌ Error checking query syntax:', error.message);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Run all tests
|
||||
async function runAllTests() {
|
||||
console.log('🚀 Starting comprehensive WASM verification...\n');
|
||||
|
||||
const results = [];
|
||||
|
||||
results.push(await checkWasmFiles());
|
||||
results.push(await testTreeSitterLoad());
|
||||
results.push(await testCompiledQueries());
|
||||
results.push(await testParserLoader());
|
||||
results.push(await testQuerySyntax());
|
||||
|
||||
const passedTests = results.filter(Boolean).length;
|
||||
const totalTests = results.length;
|
||||
|
||||
console.log(`\n📊 Results: ${passedTests}/${totalTests} tests passed`);
|
||||
|
||||
if (passedTests === totalTests) {
|
||||
console.log('\n✅ All WASM verification tests PASSED!');
|
||||
console.log('🎉 Tree-sitter WASM setup appears to be working correctly');
|
||||
console.log('\n🧹 You can now safely delete these test files:');
|
||||
console.log(' - scripts/verify-wasm-direct.js');
|
||||
console.log(' - scripts/test-wasm.js');
|
||||
console.log(' - src/tests/wasm-verification.test.ts');
|
||||
console.log(' - src/__tests__/setup.ts');
|
||||
console.log('\n🚀 Ready to proceed with ZIP upload testing!');
|
||||
return true;
|
||||
} else {
|
||||
console.log('\n❌ Some WASM verification tests FAILED!');
|
||||
console.log('🔧 Please fix the issues above before proceeding.');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
runAllTests()
|
||||
.then(success => {
|
||||
process.exit(success ? 0 : 1);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('💥 Unexpected error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
|
@ -82,4 +82,52 @@ export const cleanup = () => {
|
|||
// Clean up any test artifacts
|
||||
jest.clearAllTimers();
|
||||
jest.clearAllMocks();
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Jest test setup for WASM verification
|
||||
* TEMPORARY FILE - DELETE after WASM verification
|
||||
*/
|
||||
|
||||
// Mock fetch for WASM file loading
|
||||
global.fetch = jest.fn();
|
||||
|
||||
// Mock URL.createObjectURL and revokeObjectURL for worker tests
|
||||
global.URL.createObjectURL = jest.fn(() => 'mock-blob-url');
|
||||
global.URL.revokeObjectURL = jest.fn();
|
||||
|
||||
// Mock Worker for worker tests
|
||||
global.Worker = jest.fn().mockImplementation(() => ({
|
||||
postMessage: jest.fn(),
|
||||
terminate: jest.fn(),
|
||||
addEventListener: jest.fn(),
|
||||
removeEventListener: jest.fn(),
|
||||
onmessage: null,
|
||||
onerror: null
|
||||
}));
|
||||
|
||||
// Setup console for test output
|
||||
const originalConsoleLog = console.log;
|
||||
console.log = (...args) => {
|
||||
// Allow our test messages through
|
||||
if (args[0] && typeof args[0] === 'string' && (
|
||||
args[0].includes('✅') ||
|
||||
args[0].includes('❌') ||
|
||||
args[0].includes('🔍') ||
|
||||
args[0].includes('🧪')
|
||||
)) {
|
||||
originalConsoleLog(...args);
|
||||
}
|
||||
};
|
||||
|
||||
// Mock WebAssembly for basic checks
|
||||
if (typeof WebAssembly === 'undefined') {
|
||||
global.WebAssembly = {
|
||||
compile: jest.fn(),
|
||||
instantiate: jest.fn(),
|
||||
Module: jest.fn(),
|
||||
Instance: jest.fn()
|
||||
};
|
||||
}
|
||||
|
||||
console.log('🔧 Jest setup completed for WASM verification tests');
|
||||
|
|
|
|||
|
|
@ -46,7 +46,13 @@ export class CallProcessor {
|
|||
sameFileMatches: 0,
|
||||
heuristicMatches: 0,
|
||||
failed: 0,
|
||||
callTypes: {} as Record<string, number>
|
||||
callTypes: {} as Record<string, number>,
|
||||
// Failure categorization
|
||||
failuresByCategory: {
|
||||
externalLibraries: 0, // Calls to external/stdlib functions (expected)
|
||||
pythonBuiltins: 0, // Python built-in functions (expected)
|
||||
actualFailures: 0 // Real resolution failures (unexpected)
|
||||
}
|
||||
};
|
||||
|
||||
constructor(functionTrie: FunctionRegistryTrie) {
|
||||
|
|
@ -92,26 +98,10 @@ export class CallProcessor {
|
|||
if (calls.length === 0) {
|
||||
// Only log for source files that should have function calls
|
||||
if (this.isSourceFile(filePath)) {
|
||||
console.log(`⚠️ CallProcessor: No function calls found in source file: ${filePath}`);
|
||||
|
||||
// Debug: Check if this file has any 'call' nodes at all
|
||||
if (filePath.endsWith('.py')) {
|
||||
const callNodeCount = this.countNodeType(ast.tree!.rootNode, 'call');
|
||||
const definitionCount = graph.nodes.filter(n =>
|
||||
(n.label === 'Function' || n.label === 'Class' || n.label === 'Method') &&
|
||||
n.properties.filePath === filePath
|
||||
).length;
|
||||
|
||||
console.log(` 📊 Debug: ${filePath.split('/').pop()} has ${callNodeCount} call nodes, ${definitionCount} definitions`);
|
||||
|
||||
// If we have definitions but no calls, that's suspicious
|
||||
if (definitionCount > 0 && callNodeCount === 0) {
|
||||
console.log(` 🚨 Suspicious: File has definitions but no call nodes - possible AST parsing issue`);
|
||||
}
|
||||
}
|
||||
// Reduced logging - track but don't spam console for each file
|
||||
}
|
||||
} else {
|
||||
console.log(`CallProcessor: Found ${calls.length} function calls in ${filePath}`);
|
||||
// Reduced logging - only log summary instead of per-file details
|
||||
}
|
||||
|
||||
for (const call of calls) {
|
||||
|
|
@ -137,7 +127,8 @@ export class CallProcessor {
|
|||
}
|
||||
} else {
|
||||
this.stats.failed++;
|
||||
console.log(`❌ Failed to resolve call: ${call.functionName} in ${call.callerFile}:${call.startLine}`);
|
||||
// Categorize the failure for better statistics
|
||||
this.categorizeFailureWithReason(call, this.diagnoseFailure(call));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -192,16 +183,44 @@ export class CallProcessor {
|
|||
|
||||
if (importInfo) {
|
||||
// We have an import for this function name
|
||||
const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def =>
|
||||
def.filePath === importInfo.targetFile &&
|
||||
(def.functionName === importInfo.exportedName ||
|
||||
(importInfo.exportedName === 'default' && def.functionName === call.functionName))
|
||||
);
|
||||
const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def => {
|
||||
// Match file path
|
||||
if (def.filePath !== importInfo.targetFile) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Handle different import types
|
||||
if (importInfo.importType === 'default') {
|
||||
// For default imports, the function name could be anything
|
||||
// Look for functions that could be the default export
|
||||
return def.functionName === call.functionName ||
|
||||
def.functionName === importInfo.exportedName ||
|
||||
// Common default export patterns
|
||||
(def.type === 'function' && def.startLine === 1) ||
|
||||
(def.type === 'class' && def.functionName === 'default');
|
||||
} else if (importInfo.importType === 'named') {
|
||||
// For named imports, match the exported name
|
||||
return def.functionName === importInfo.exportedName;
|
||||
} else if (importInfo.importType === 'namespace') {
|
||||
// For namespace imports like * as utils,
|
||||
// the call would be utils.someFunction, so we need to handle this differently
|
||||
return def.functionName === call.functionName;
|
||||
}
|
||||
|
||||
return false;
|
||||
});
|
||||
|
||||
if (targetDefinitions.length > 0) {
|
||||
// Prefer functions over other types for function calls
|
||||
const preferred = targetDefinitions.find(def =>
|
||||
call.callType === 'function_call' ? def.type === 'function' :
|
||||
call.callType === 'method_call' ? def.type === 'method' :
|
||||
call.callType === 'constructor_call' ? def.type === 'class' : true
|
||||
) || targetDefinitions[0];
|
||||
|
||||
return {
|
||||
success: true,
|
||||
targetNodeId: targetDefinitions[0].nodeId,
|
||||
targetNodeId: preferred.nodeId,
|
||||
stage: 'exact',
|
||||
confidence: 'high'
|
||||
};
|
||||
|
|
@ -407,6 +426,101 @@ export class CallProcessor {
|
|||
return true;
|
||||
}
|
||||
|
||||
// JavaScript/TypeScript built-ins and common library functions
|
||||
const jsBuiltins = new Set([
|
||||
// Core JavaScript functions
|
||||
'parseInt', 'parseFloat', 'isNaN', 'isFinite', 'encodeURI', 'decodeURI',
|
||||
'encodeURIComponent', 'decodeURIComponent', 'escape', 'unescape',
|
||||
// Array methods
|
||||
'push', 'pop', 'shift', 'unshift', 'slice', 'splice', 'concat', 'join',
|
||||
'reverse', 'sort', 'indexOf', 'lastIndexOf', 'includes', 'find', 'findIndex',
|
||||
'filter', 'map', 'reduce', 'reduceRight', 'forEach', 'some', 'every',
|
||||
'flat', 'flatMap', 'fill', 'copyWithin', 'from', 'of', 'isArray',
|
||||
// Object methods
|
||||
'keys', 'values', 'entries', 'assign', 'create', 'defineProperty',
|
||||
'defineProperties', 'freeze', 'seal', 'preventExtensions', 'hasOwnProperty',
|
||||
'isPrototypeOf', 'propertyIsEnumerable', 'toString', 'valueOf', 'toLocaleString',
|
||||
// String methods
|
||||
'charAt', 'charCodeAt', 'codePointAt', 'concat', 'endsWith', 'includes',
|
||||
'indexOf', 'lastIndexOf', 'localeCompare', 'match', 'normalize', 'padEnd',
|
||||
'padStart', 'repeat', 'replace', 'search', 'slice', 'split', 'startsWith',
|
||||
'substring', 'substr', 'toLowerCase', 'toUpperCase', 'trim', 'trimEnd',
|
||||
'trimStart', 'trimLeft', 'trimRight',
|
||||
// Number methods
|
||||
'toFixed', 'toExponential', 'toPrecision', 'isInteger', 'isSafeInteger',
|
||||
'isFinite', 'isNaN', 'parseFloat', 'parseInt',
|
||||
// Date methods
|
||||
'getTime', 'getDate', 'getDay', 'getFullYear', 'getHours', 'getMinutes',
|
||||
'getSeconds', 'getMilliseconds', 'getMonth', 'setDate', 'setFullYear',
|
||||
'setHours', 'setMinutes', 'setSeconds', 'setMilliseconds', 'setMonth',
|
||||
'toDateString', 'toTimeString', 'toISOString', 'toJSON', 'now', 'parse',
|
||||
// Promise methods
|
||||
'then', 'catch', 'finally', 'resolve', 'reject', 'all', 'race', 'allSettled',
|
||||
// Console methods
|
||||
'log', 'error', 'warn', 'info', 'debug', 'trace', 'assert', 'clear',
|
||||
'count', 'dir', 'dirxml', 'group', 'groupCollapsed', 'groupEnd', 'table',
|
||||
'time', 'timeEnd', 'timeLog', 'profile', 'profileEnd',
|
||||
// DOM methods (common ones)
|
||||
'getElementById', 'getElementsByClassName', 'getElementsByTagName',
|
||||
'querySelector', 'querySelectorAll', 'createElement', 'createTextNode',
|
||||
'appendChild', 'removeChild', 'insertBefore', 'replaceChild', 'cloneNode',
|
||||
'getAttribute', 'setAttribute', 'removeAttribute', 'hasAttribute',
|
||||
'addEventListener', 'removeEventListener', 'dispatchEvent',
|
||||
'preventDefault', 'stopPropagation', 'stopImmediatePropagation',
|
||||
'focus', 'blur', 'click', 'submit', 'reset', 'scrollIntoView',
|
||||
// Common library methods (React, etc.)
|
||||
'useState', 'useEffect', 'useContext', 'useReducer', 'useCallback',
|
||||
'useMemo', 'useRef', 'useImperativeHandle', 'useLayoutEffect', 'useDebugValue',
|
||||
'memo', 'forwardRef', 'lazy', 'Suspense', 'Fragment', 'createElement',
|
||||
'cloneElement', 'isValidElement', 'render', 'hydrate', 'unmountComponentAtNode',
|
||||
// HTTP/Fetch
|
||||
'fetch', 'get', 'post', 'put', 'delete', 'patch', 'head', 'options',
|
||||
// JSON
|
||||
'stringify', 'parse',
|
||||
// Math
|
||||
'abs', 'acos', 'asin', 'atan', 'atan2', 'ceil', 'cos', 'exp', 'floor',
|
||||
'log', 'max', 'min', 'pow', 'random', 'round', 'sin', 'sqrt', 'tan',
|
||||
// Common testing functions
|
||||
'describe', 'it', 'test', 'expect', 'beforeEach', 'afterEach', 'beforeAll', 'afterAll',
|
||||
'mock', 'spy', 'stub', 'restore', 'reset', 'resetAllMocks', 'clearAllMocks',
|
||||
// Node.js specific
|
||||
'require', 'module', 'exports', '__dirname', '__filename', 'process', 'global',
|
||||
'Buffer', 'setImmediate', 'clearImmediate', 'setInterval', 'clearInterval',
|
||||
'setTimeout', 'clearTimeout',
|
||||
// Worker API
|
||||
'postMessage', 'onmessage', 'onerror', 'close', 'importScripts',
|
||||
// JavaScript constructors and built-ins
|
||||
'Array', 'Object', 'String', 'Number', 'Boolean', 'Function', 'Date',
|
||||
'RegExp', 'Error', 'TypeError', 'ReferenceError', 'SyntaxError',
|
||||
'RangeError', 'EvalError', 'URIError', 'AggregateError',
|
||||
'Set', 'Map', 'WeakSet', 'WeakMap', 'Symbol', 'BigInt',
|
||||
'Promise', 'Proxy', 'Reflect', 'ArrayBuffer', 'SharedArrayBuffer',
|
||||
'DataView', 'Int8Array', 'Uint8Array', 'Int16Array', 'Uint16Array',
|
||||
'Int32Array', 'Uint32Array', 'Float32Array', 'Float64Array',
|
||||
'BigInt64Array', 'BigUint64Array',
|
||||
// Configuration and build tools
|
||||
'config', 'define', 'plugin', 'preset', 'loader', 'rule',
|
||||
'extend', 'override', 'merge', 'concat', 'apply',
|
||||
// ESLint specific
|
||||
'rules', 'extends', 'parser', 'parserOptions', 'env', 'globals',
|
||||
// Bundler/build tools
|
||||
'bundle', 'chunk', 'entry', 'output', 'optimization', 'resolve',
|
||||
'devtool', 'target', 'externals', 'stats', 'performance',
|
||||
// Process and execution
|
||||
'exec', 'spawn', 'fork', 'execSync', 'spawnSync',
|
||||
// File system operations
|
||||
'readFile', 'writeFile', 'readdir', 'stat', 'mkdir', 'rmdir',
|
||||
'unlink', 'rename', 'copyFile', 'access', 'watch', 'createReadStream',
|
||||
'createWriteStream'
|
||||
]);
|
||||
|
||||
// Check if it's a JS/TS file and the function is a built-in
|
||||
if ((filePath.endsWith('.js') || filePath.endsWith('.ts') ||
|
||||
filePath.endsWith('.jsx') || filePath.endsWith('.tsx')) &&
|
||||
jsBuiltins.has(functionName)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Ignore very short function names (likely built-ins or operators)
|
||||
if (functionName.length <= 2) {
|
||||
return true;
|
||||
|
|
@ -453,26 +567,19 @@ export class CallProcessor {
|
|||
const functionName = this.extractPythonCallName(functionNode);
|
||||
|
||||
// Debug: Log what we're finding vs filtering
|
||||
if (functionName) {
|
||||
const shouldIgnore = this.shouldIgnoreCall(functionName, filePath);
|
||||
if (shouldIgnore) {
|
||||
// Only log a few examples to avoid spam
|
||||
if (calls.length < 3) {
|
||||
console.log(`🔍 Filtered out: ${functionName} in ${filePath.split('/').pop()}`);
|
||||
}
|
||||
} else {
|
||||
calls.push({
|
||||
callerFile: filePath,
|
||||
functionName,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
callType: 'function_call'
|
||||
});
|
||||
}
|
||||
if (functionName && !this.shouldIgnoreCall(functionName, filePath)) {
|
||||
// Don't filter here - let all calls through to resolution
|
||||
calls.push({
|
||||
callerFile: filePath,
|
||||
functionName,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
callType: 'function_call'
|
||||
});
|
||||
} else {
|
||||
// Debug: Log when we can't extract function name
|
||||
// Reduced logging - don't log every individual extraction failure
|
||||
if (calls.length < 3) {
|
||||
console.log(`🔍 Could not extract function name from: ${functionNode.type} in ${filePath.split('/').pop()}`);
|
||||
// Only log first few failures per file to understand patterns
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -509,13 +616,16 @@ export class CallProcessor {
|
|||
const constructorNode = node.childForFieldName('constructor');
|
||||
if (constructorNode) {
|
||||
const constructorName = constructorNode.text;
|
||||
calls.push({
|
||||
callerFile: filePath,
|
||||
functionName: constructorName,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
callType: 'constructor_call'
|
||||
});
|
||||
// Don't filter constructor calls as strictly - they're important for the graph
|
||||
if (!this.shouldIgnoreCall(constructorName, filePath)) {
|
||||
calls.push({
|
||||
callerFile: filePath,
|
||||
functionName: constructorName,
|
||||
startLine: node.startPosition.row + 1,
|
||||
endLine: node.endPosition.row + 1,
|
||||
callType: 'constructor_call'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -552,9 +662,22 @@ export class CallProcessor {
|
|||
}
|
||||
}
|
||||
|
||||
// Debug: Log unhandled node types (but limit spam)
|
||||
if (Math.random() < 0.1) { // Only log 10% of cases to avoid spam
|
||||
console.log(`🔍 Unhandled Python call node type: ${node.type} (text: "${node.text}")`);
|
||||
// Handle additional Python call patterns
|
||||
if (node.text && node.text.length > 0 && node.text.length < 100) {
|
||||
// For simple cases, try using the text directly if it looks like a function name
|
||||
const text = node.text.trim();
|
||||
if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
// For attribute access, try to extract the last part
|
||||
const parts = text.split('.');
|
||||
if (parts.length > 1) {
|
||||
const lastPart = parts[parts.length - 1];
|
||||
if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(lastPart)) {
|
||||
return lastPart;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
|
|
@ -570,7 +693,40 @@ export class CallProcessor {
|
|||
// For method calls like obj.method(), we want just 'method'
|
||||
const propertyNode = node.childForFieldName('property');
|
||||
return propertyNode ? propertyNode.text : null;
|
||||
} else if (node.type === 'call_expression') {
|
||||
// Handle nested calls like getData().process()
|
||||
const functionNode = node.childForFieldName('function');
|
||||
if (functionNode) {
|
||||
return this.extractJSCallName(functionNode);
|
||||
}
|
||||
} else if (node.type === 'subscript_expression') {
|
||||
// Handle array/object access like obj['method']()
|
||||
const propertyNode = node.childForFieldName('index');
|
||||
if (propertyNode && propertyNode.type === 'string') {
|
||||
// Extract string literal content
|
||||
const propText = propertyNode.text.replace(/['"`]/g, '');
|
||||
if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(propText)) {
|
||||
return propText;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: try to extract from text for simple patterns
|
||||
if (node.text && node.text.length > 0 && node.text.length < 50) {
|
||||
const text = node.text.trim();
|
||||
|
||||
// Handle simple function calls
|
||||
if (/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
// Handle member expressions like obj.method
|
||||
const memberMatch = text.match(/([a-zA-Z_][a-zA-Z0-9_]*)\s*$/);
|
||||
if (memberMatch) {
|
||||
return memberMatch[1];
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -677,7 +833,12 @@ export class CallProcessor {
|
|||
sameFileMatches: 0,
|
||||
heuristicMatches: 0,
|
||||
failed: 0,
|
||||
callTypes: {}
|
||||
callTypes: {},
|
||||
failuresByCategory: {
|
||||
externalLibraries: 0,
|
||||
pythonBuiltins: 0,
|
||||
actualFailures: 0
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -691,6 +852,18 @@ export class CallProcessor {
|
|||
console.log(` ✅ Same-file matches (Stage 2): ${this.stats.sameFileMatches} (${((this.stats.sameFileMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`);
|
||||
console.log(` 🎯 Heuristic matches (Stage 3): ${this.stats.heuristicMatches} (${((this.stats.heuristicMatches / this.stats.totalCalls) * 100).toFixed(1)}%)`);
|
||||
console.log(` ❌ Failed resolutions: ${this.stats.failed} (${((this.stats.failed / this.stats.totalCalls) * 100).toFixed(1)}%)`);
|
||||
|
||||
// Enhanced failure breakdown
|
||||
if (this.stats.failed > 0) {
|
||||
const { externalLibraries, pythonBuiltins, actualFailures } = this.stats.failuresByCategory;
|
||||
console.log(` 📦 External libraries (expected): ${externalLibraries} (${((externalLibraries / this.stats.failed) * 100).toFixed(1)}% of failures)`);
|
||||
console.log(` 🐍 Python built-ins (expected): ${pythonBuiltins} (${((pythonBuiltins / this.stats.failed) * 100).toFixed(1)}% of failures)`);
|
||||
console.log(` 🚨 Actual failures (unexpected): ${actualFailures} (${((actualFailures / this.stats.failed) * 100).toFixed(1)}% of failures)`);
|
||||
|
||||
const expectedFailures = externalLibraries + pythonBuiltins;
|
||||
console.log(` Real success rate (excluding expected failures): ${(((this.stats.totalCalls - actualFailures) / this.stats.totalCalls) * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
console.log(` Success rate: ${(((this.stats.totalCalls - this.stats.failed) / this.stats.totalCalls) * 100).toFixed(1)}%`);
|
||||
}
|
||||
|
||||
|
|
@ -710,6 +883,57 @@ export class CallProcessor {
|
|||
this.resetStats();
|
||||
}
|
||||
|
||||
/**
|
||||
* Categorize a failed call for statistics with detailed reason
|
||||
*/
|
||||
private categorizeFailureWithReason(call: CallInfo, reason: string): void {
|
||||
// Check if this is an expected failure (external library or built-in)
|
||||
if (this.shouldIgnoreCall(call.functionName, call.callerFile)) {
|
||||
// It's a call we expect to fail (external library or built-in)
|
||||
if (call.callerFile.endsWith('.py')) {
|
||||
this.stats.failuresByCategory.pythonBuiltins++;
|
||||
} else {
|
||||
this.stats.failuresByCategory.externalLibraries++;
|
||||
}
|
||||
} else {
|
||||
// It's a call to user code that we failed to resolve (unexpected)
|
||||
this.stats.failuresByCategory.actualFailures++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Diagnose why a specific call failed
|
||||
*/
|
||||
private diagnoseFailure(call: CallInfo): string {
|
||||
// Check if it's in import map but target not found
|
||||
const importInfo = this.importMap[call.callerFile]?.[call.functionName];
|
||||
if (importInfo) {
|
||||
const targetDefinitions = this.functionTrie.getAllDefinitions().filter(def =>
|
||||
def.filePath === importInfo.targetFile &&
|
||||
(def.functionName === importInfo.exportedName ||
|
||||
(importInfo.exportedName === 'default' && def.functionName === call.functionName))
|
||||
);
|
||||
|
||||
if (targetDefinitions.length === 0) {
|
||||
return `Imported from ${importInfo.targetFile} but definition not found`;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if function exists in same file
|
||||
const sameFileDefinitions = this.functionTrie.findInSameFile(call.callerFile, call.functionName);
|
||||
if (sameFileDefinitions.length === 0) {
|
||||
// Check if any similar functions exist
|
||||
const candidates = this.functionTrie.findEndingWith(call.functionName);
|
||||
if (candidates.length === 0) {
|
||||
return `No function named '${call.functionName}' found anywhere`;
|
||||
} else {
|
||||
return `Function '${call.functionName}' not in same file, ${candidates.length} candidates in other files`;
|
||||
}
|
||||
}
|
||||
|
||||
return 'Unknown failure reason';
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file is a source file that should contain function calls
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -102,10 +102,16 @@ export class ImportProcessor {
|
|||
ast: ParsedAST,
|
||||
graph: KnowledgeGraph
|
||||
): Promise<{ found: number; resolved: number }> {
|
||||
if (!ast.tree) return { found: 0, resolved: 0 };
|
||||
if (!ast.tree) {
|
||||
return { found: 0, resolved: 0 };
|
||||
}
|
||||
|
||||
|
||||
|
||||
const imports = this.extractImports(ast.tree.rootNode, filePath);
|
||||
|
||||
|
||||
|
||||
if (imports.length === 0) return { found: 0, resolved: 0 };
|
||||
|
||||
// Initialize import map for this file
|
||||
|
|
@ -227,17 +233,81 @@ export class ImportProcessor {
|
|||
filePath: string,
|
||||
imports: ImportInfo[]
|
||||
): void {
|
||||
|
||||
|
||||
if (node.type === 'import_statement') {
|
||||
const sourceNode = node.childForFieldName('source');
|
||||
if (!sourceNode) return;
|
||||
|
||||
|
||||
// Try different approaches to find the source
|
||||
let sourceNode = node.childForFieldName('source');
|
||||
if (!sourceNode) {
|
||||
// Try finding string literal directly
|
||||
for (let i = 0; i < node.childCount; i++) {
|
||||
const child = node.child(i);
|
||||
if (child && (child.type === 'string' || child.type === 'string_literal')) {
|
||||
sourceNode = child;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!sourceNode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sourcePath = sourceNode.text.replace(/['"]/g, '');
|
||||
const targetFile = this.resolveModulePath(sourcePath, filePath, 'javascript');
|
||||
|
||||
|
||||
|
||||
// Handle different import patterns
|
||||
const importClauseNode = node.childForFieldName('import_clause');
|
||||
let importClauseNode: Parser.SyntaxNode | null = node.childForFieldName('import_clause');
|
||||
|
||||
// CRITICAL FIX: If field-based approach fails, search by node type
|
||||
if (!importClauseNode) {
|
||||
for (let i = 0; i < node.childCount; i++) {
|
||||
const child = node.child(i);
|
||||
if (child?.type === 'import_clause') {
|
||||
importClauseNode = child;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (importClauseNode) {
|
||||
this.processJSImportClause(importClauseNode, filePath, targetFile, imports);
|
||||
} else {
|
||||
|
||||
// Handle simple imports like: import 'module'
|
||||
if (node.text.trim().startsWith('import') && !node.text.includes('{') && !node.text.includes('from')) {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: '_side_effect_',
|
||||
targetFile,
|
||||
exportedName: '*',
|
||||
importType: 'namespace'
|
||||
});
|
||||
|
||||
} else {
|
||||
// Try to extract import manually from text as fallback
|
||||
const importText = node.text.trim();
|
||||
const match = importText.match(/import\s+(.+?)\s+from\s+['"]([^'"]+)['"]/);;
|
||||
if (match) {
|
||||
const importPart = match[1].trim();
|
||||
|
||||
// Handle simple default import
|
||||
if (!importPart.includes('{') && !importPart.includes('*')) {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: importPart,
|
||||
targetFile,
|
||||
exportedName: 'default',
|
||||
importType: 'default'
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (node.type === 'variable_declaration') {
|
||||
// Handle CommonJS: const x = require('module')
|
||||
|
|
@ -262,13 +332,44 @@ export class ImportProcessor {
|
|||
targetFile: string,
|
||||
imports: ImportInfo[]
|
||||
): void {
|
||||
// Track what we've processed to ensure we don't miss anything
|
||||
let processedSomething = false;
|
||||
|
||||
for (let i = 0; i < importClauseNode.childCount; i++) {
|
||||
const child = importClauseNode.child(i);
|
||||
if (!child) continue;
|
||||
if (child.type === 'identifier') {
|
||||
// Default import - this is the most common case we're missing
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: child.text,
|
||||
targetFile,
|
||||
exportedName: 'default',
|
||||
importType: 'default'
|
||||
});
|
||||
|
||||
if (child.type === 'import_specifier') {
|
||||
// Named import: { name } or { name as alias }
|
||||
processedSomething = true;
|
||||
} else if (child.type === 'named_imports') {
|
||||
// Process named imports: { a, b, c }
|
||||
this.processNamedImportsNode(child, filePath, targetFile, imports);
|
||||
processedSomething = true;
|
||||
} else if (child.type === 'namespace_import') {
|
||||
// Namespace import: * as name
|
||||
const nameNode = child.childForFieldName('name');
|
||||
if (nameNode) {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: nameNode.text,
|
||||
targetFile,
|
||||
exportedName: '*',
|
||||
importType: 'namespace'
|
||||
});
|
||||
|
||||
processedSomething = true;
|
||||
}
|
||||
} else if (child.type === 'import_specifier') {
|
||||
// Direct import specifier (should be handled by named_imports, but just in case)
|
||||
const nameNode = child.childForFieldName('name') || child.child(0);
|
||||
const aliasNode = child.childForFieldName('alias');
|
||||
|
||||
if (nameNode) {
|
||||
|
|
@ -282,28 +383,98 @@ export class ImportProcessor {
|
|||
exportedName,
|
||||
importType: 'named'
|
||||
});
|
||||
|
||||
processedSomething = true;
|
||||
}
|
||||
} else if (child.type === 'namespace_import') {
|
||||
// Namespace import: * as name
|
||||
const nameNode = child.childForFieldName('name');
|
||||
if (nameNode) {
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: If structured processing didn't work, try text parsing
|
||||
if (!processedSomething) {
|
||||
const clauseText = importClauseNode.text.trim();
|
||||
|
||||
if (clauseText.startsWith('{') && clauseText.endsWith('}')) {
|
||||
// Named imports like { foo, bar }
|
||||
const namedImports = clauseText.slice(1, -1)
|
||||
.split(',')
|
||||
.map(s => s.trim())
|
||||
.filter(s => s.length > 0);
|
||||
|
||||
namedImports.forEach(importName => {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: nameNode.text,
|
||||
localName: importName,
|
||||
targetFile,
|
||||
exportedName: importName,
|
||||
importType: 'named'
|
||||
});
|
||||
});
|
||||
|
||||
} else if (clauseText.includes(' as ')) {
|
||||
// Namespace import like * as foo
|
||||
const namespaceMatch = clauseText.match(/\*\s+as\s+(\w+)/);
|
||||
if (namespaceMatch) {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: namespaceMatch[1],
|
||||
targetFile,
|
||||
exportedName: '*',
|
||||
importType: 'namespace'
|
||||
});
|
||||
|
||||
}
|
||||
} else if (child.type === 'identifier') {
|
||||
// Default import
|
||||
} else {
|
||||
// Simple default import
|
||||
const defaultName = clauseText.split(',')[0].trim(); // Handle mixed imports
|
||||
if (defaultName && !defaultName.includes('{') && !defaultName.includes('*')) {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: defaultName,
|
||||
targetFile,
|
||||
exportedName: 'default',
|
||||
importType: 'default'
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private processNamedImportsNode(
|
||||
namedImportsNode: Parser.SyntaxNode,
|
||||
filePath: string,
|
||||
targetFile: string,
|
||||
imports: ImportInfo[]
|
||||
): void {
|
||||
for (let j = 0; j < namedImportsNode.childCount; j++) {
|
||||
const namedChild = namedImportsNode.child(j);
|
||||
if (namedChild && namedChild.type === 'import_specifier') {
|
||||
|
||||
const nameNode = namedChild.childForFieldName('name') || namedChild.child(0);
|
||||
const aliasNode = namedChild.childForFieldName('alias');
|
||||
|
||||
if (nameNode) {
|
||||
const exportedName = nameNode.text;
|
||||
const localName = aliasNode ? aliasNode.text : exportedName;
|
||||
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName,
|
||||
targetFile,
|
||||
exportedName,
|
||||
importType: 'named'
|
||||
});
|
||||
|
||||
}
|
||||
} else if (namedChild && namedChild.type === 'identifier') {
|
||||
imports.push({
|
||||
importingFile: filePath,
|
||||
localName: child.text,
|
||||
localName: namedChild.text,
|
||||
targetFile,
|
||||
exportedName: 'default',
|
||||
importType: 'default'
|
||||
exportedName: namedChild.text,
|
||||
importType: 'named'
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
} from '../../lib/shared-utils.js';
|
||||
import { IGNORE_PATTERNS } from '../../config/language-config.js';
|
||||
import Parser from 'web-tree-sitter';
|
||||
import { TYPESCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries';
|
||||
import { TYPESCRIPT_QUERIES, JAVASCRIPT_QUERIES, PYTHON_QUERIES, JAVA_QUERIES } from './tree-sitter-queries';
|
||||
import { initTreeSitter, loadTypeScriptParser, loadPythonParser, loadJavaScriptParser } from '../tree-sitter/parser-loader.js';
|
||||
import { FunctionRegistryTrie, FunctionDefinition } from '../graph/trie.js';
|
||||
import { LRUCacheService } from '../../lib/lru-cache-service.js';
|
||||
|
|
@ -226,11 +226,19 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
|
||||
private async parseFile(graph: KnowledgeGraph, filePath: string, content: string): Promise<void> {
|
||||
const language = this.detectLanguage(filePath);
|
||||
|
||||
// Skip compiled/minified files for JavaScript
|
||||
if (language === 'javascript' && this.isCompiledOrMinified(content, filePath)) {
|
||||
console.log(`Skipping compiled/minified file: ${filePath}`);
|
||||
await this.parseGenericFile(graph, filePath, content);
|
||||
return;
|
||||
}
|
||||
|
||||
const contentHash = this.generateContentHash(content);
|
||||
const cacheKey = this.lruCache.generateFileCacheKey(filePath, contentHash);
|
||||
|
||||
// Check cache first
|
||||
const cachedResult = this.lruCache.getParsedFile(cacheKey);
|
||||
// Check cache first - TEMPORARILY DISABLED FOR DEBUGGING
|
||||
const cachedResult = null; // this.lruCache.getParsedFile(cacheKey);
|
||||
if (cachedResult) {
|
||||
console.log(`Cache hit for file: ${filePath}`);
|
||||
this.astMap.set(filePath, { tree: cachedResult.ast });
|
||||
|
|
@ -241,7 +249,7 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
const langParser = this.languageParsers.get(language);
|
||||
|
||||
if (!langParser || !this.parser) {
|
||||
console.warn(`No parser available for language: ${language}. Skipping file: ${filePath}`);
|
||||
// Skip file with reduced logging to avoid console spam
|
||||
await this.parseGenericFile(graph, filePath, content);
|
||||
return;
|
||||
}
|
||||
|
|
@ -262,8 +270,8 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
const queryCacheKey = this.lruCache.generateQueryCacheKey(language, queryString);
|
||||
let queryResults: Parser.QueryMatch[] = [];
|
||||
|
||||
// Check query cache
|
||||
const cachedQuery = this.lruCache.getQueryResult(queryCacheKey);
|
||||
// Check query cache - TEMPORARILY DISABLED FOR DEBUGGING
|
||||
const cachedQuery = null; // this.lruCache.getQueryResult(queryCacheKey);
|
||||
if (cachedQuery) {
|
||||
queryResults = cachedQuery.results;
|
||||
} else {
|
||||
|
|
@ -284,6 +292,11 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
const definition = this.extractDefinition(node, queryName, filePath);
|
||||
if (definition) {
|
||||
definitions.push(definition);
|
||||
|
||||
// Debug: Log what specific definitions we're finding
|
||||
if (language === 'python' && filePath.endsWith('.py')) {
|
||||
console.log(`🔍 DEBUG: Found ${queryName} -> ${definition.type}: "${definition.name}" in ${filePath.split('/').pop()}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -298,12 +311,39 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
fileSize: content.length
|
||||
});
|
||||
|
||||
// Debug: Log definition extraction results for Python files
|
||||
if (language === 'python' && filePath.endsWith('.py')) {
|
||||
console.log(`🔍 DEBUG: ${filePath.split('/').pop()} -> ${definitions.length} definitions extracted (content: ${content.length} chars, hash: ${contentHash.substring(0, 8)})`);
|
||||
|
||||
// Log definition types for debugging
|
||||
const definitionTypes = definitions.reduce((acc, def) => {
|
||||
acc[def.type] = (acc[def.type] || 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
if (definitions.length > 0) {
|
||||
console.log(`🔍 DEBUG: Definition types: ${JSON.stringify(definitionTypes)}`);
|
||||
}
|
||||
|
||||
if (definitions.length === 0 && content.length > 100) {
|
||||
console.log(`🚨 DEBUG: Large Python file with no definitions: ${filePath} (${content.length} chars)`);
|
||||
// Log first few lines to understand the content
|
||||
const firstLines = content.split('\n').slice(0, 3).join('\\n');
|
||||
console.log(`🔍 DEBUG: File starts with: ${firstLines}`);
|
||||
}
|
||||
}
|
||||
|
||||
await this.addDefinitionsToGraph(graph, filePath, definitions);
|
||||
}
|
||||
|
||||
private extractDefinition(node: Parser.SyntaxNode, queryName: string, filePath: string): ParsedDefinition | null {
|
||||
const nameNode = node.childForFieldName('name');
|
||||
const name = nameNode ? nameNode.text : 'anonymous';
|
||||
const name = nameNode ? nameNode.text : null;
|
||||
|
||||
// Skip anonymous definitions - they're usually from compiled/minified code
|
||||
if (!name || name === 'anonymous' || name.trim().length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
name,
|
||||
|
|
@ -319,16 +359,55 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
case 'classes': return 'class';
|
||||
case 'methods': return 'method';
|
||||
case 'functions':
|
||||
case 'arrowFunctions': return 'function';
|
||||
case 'arrowFunctions':
|
||||
case 'variableAssignments':
|
||||
case 'objectMethods': return 'function';
|
||||
case 'imports':
|
||||
case 'from_imports': return 'import';
|
||||
case 'exports':
|
||||
case 'defaultExports': return 'function'; // Exports usually export functions
|
||||
case 'interfaces': return 'interface';
|
||||
case 'types': return 'type';
|
||||
case 'decorators': return 'decorator';
|
||||
default: return 'variable';
|
||||
default:
|
||||
console.warn(`Unknown query type: ${queryName}, defaulting to 'function'`);
|
||||
return 'function'; // Better default than 'variable'
|
||||
}
|
||||
}
|
||||
|
||||
private isCompiledOrMinified(content: string, filePath: string): boolean {
|
||||
// Check file name patterns for known compiled files
|
||||
const fileName = filePath.split('/').pop()?.toLowerCase() || '';
|
||||
if (fileName.includes('.min.') ||
|
||||
fileName.includes('.bundle.') ||
|
||||
fileName.includes('tree-sitter.js') ||
|
||||
fileName.includes('kuzu_wasm_worker.js')) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check content characteristics for minified code
|
||||
const lines = content.split('\n');
|
||||
if (lines.length > 0) {
|
||||
const firstLine = lines[0];
|
||||
|
||||
// Very long first line (typical of minified code)
|
||||
if (firstLine.length > 500) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Contains typical minified patterns
|
||||
if (firstLine.includes('var Module=void 0!==Module?Module:{}') ||
|
||||
firstLine.includes('__webpack_require__') ||
|
||||
firstLine.includes('!function(') ||
|
||||
content.includes('/*! ') || // Webpack/build tool comments
|
||||
content.match(/^\s*!function\s*\(/)) { // IIFE patterns
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private detectLanguage(filePath: string): string {
|
||||
const extension = pathUtils.extname(filePath).toLowerCase();
|
||||
|
||||
|
|
@ -346,8 +425,9 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
private getQueriesForLanguage(language: string): Record<string, string> | null {
|
||||
switch (language) {
|
||||
case 'typescript':
|
||||
case 'javascript':
|
||||
return TYPESCRIPT_QUERIES;
|
||||
case 'javascript':
|
||||
return JAVASCRIPT_QUERIES; // Use separate JavaScript queries
|
||||
case 'python':
|
||||
return PYTHON_QUERIES;
|
||||
case 'java':
|
||||
|
|
@ -358,18 +438,31 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
}
|
||||
|
||||
private async parseGenericFile(graph: KnowledgeGraph, filePath: string, _content: string): Promise<void> {
|
||||
const fileNode: GraphNode = {
|
||||
id: generateId('file'),
|
||||
label: 'File' as NodeLabel,
|
||||
properties: {
|
||||
name: pathUtils.getFileName(filePath),
|
||||
path: filePath,
|
||||
size: _content.length,
|
||||
language: this.detectLanguage(filePath)
|
||||
}
|
||||
};
|
||||
// Find existing file node created by StructureProcessor
|
||||
let fileNode = graph.nodes.find(node =>
|
||||
node.label === 'File' &&
|
||||
(node.properties.filePath === filePath || node.properties.path === filePath)
|
||||
);
|
||||
|
||||
graph.addNode(fileNode);
|
||||
// If no existing file node found, create one (fallback)
|
||||
if (!fileNode) {
|
||||
fileNode = {
|
||||
id: generateId(`file_${filePath}`),
|
||||
label: 'File' as NodeLabel,
|
||||
properties: {
|
||||
name: pathUtils.getFileName(filePath),
|
||||
path: filePath,
|
||||
filePath: filePath,
|
||||
size: _content.length,
|
||||
language: this.detectLanguage(filePath)
|
||||
}
|
||||
};
|
||||
graph.addNode(fileNode);
|
||||
} else {
|
||||
// Update existing file node with additional properties
|
||||
fileNode.properties.size = _content.length;
|
||||
fileNode.properties.language = this.detectLanguage(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
private async addDefinitionsToGraph(
|
||||
|
|
@ -377,20 +470,30 @@ export class ParsingProcessor implements GraphProcessor<ParsingInput> {
|
|||
filePath: string,
|
||||
definitions: ParsedDefinition[]
|
||||
): Promise<void> {
|
||||
const fileNode: GraphNode = {
|
||||
id: generateId('file'),
|
||||
label: 'File' as NodeLabel,
|
||||
properties: {
|
||||
name: pathUtils.getFileName(filePath),
|
||||
path: filePath,
|
||||
language: this.detectLanguage(filePath)
|
||||
}
|
||||
};
|
||||
// Find existing file node created by StructureProcessor
|
||||
let fileNode = graph.nodes.find(node =>
|
||||
node.label === 'File' &&
|
||||
(node.properties.filePath === filePath || node.properties.path === filePath)
|
||||
);
|
||||
|
||||
graph.addNode(fileNode);
|
||||
// If no existing file node found, create one (fallback)
|
||||
if (!fileNode) {
|
||||
fileNode = {
|
||||
id: generateId(`file_${filePath}`),
|
||||
label: 'File' as NodeLabel,
|
||||
properties: {
|
||||
name: pathUtils.getFileName(filePath),
|
||||
path: filePath,
|
||||
filePath: filePath,
|
||||
language: this.detectLanguage(filePath)
|
||||
}
|
||||
};
|
||||
graph.addNode(fileNode);
|
||||
}
|
||||
|
||||
for (const def of definitions) {
|
||||
const nodeId = generateId(def.type);
|
||||
// Generate unique ID based on file path and definition name
|
||||
const nodeId = generateId(`${def.type}_${filePath}_${def.name}_${def.startLine}`);
|
||||
|
||||
if (this.duplicateDetector.checkAndMark(nodeId)) continue;
|
||||
|
||||
|
|
|
|||
|
|
@ -113,10 +113,10 @@ export class GraphPipeline {
|
|||
const filesWithoutDefinitions = fileNodes.filter(fileNode => {
|
||||
const hasDefinitions = graph.relationships.some(rel =>
|
||||
rel.source === fileNode.id &&
|
||||
rel.type === 'CONTAINS' &&
|
||||
rel.type === 'DEFINES' &&
|
||||
graph.nodes.some(targetNode =>
|
||||
targetNode.id === rel.target &&
|
||||
['Function', 'Class', 'Method'].includes(targetNode.label)
|
||||
['Function', 'Class', 'Method', 'Variable'].includes(targetNode.label)
|
||||
)
|
||||
);
|
||||
return !hasDefinitions;
|
||||
|
|
@ -181,7 +181,7 @@ export class GraphPipeline {
|
|||
const sourceFilesWithoutDefinitions = sourceFiles.filter(fileNode => {
|
||||
const hasDefinitions = graph.relationships.some(rel =>
|
||||
rel.source === fileNode.id &&
|
||||
rel.type === 'CONTAINS' &&
|
||||
rel.type === 'DEFINES' &&
|
||||
graph.nodes.some(n =>
|
||||
n.id === rel.target &&
|
||||
['Function', 'Class', 'Method', 'Variable'].includes(n.label)
|
||||
|
|
@ -205,7 +205,7 @@ export class GraphPipeline {
|
|||
const definitionsWithoutFiles = definitionNodes.filter(defNode => {
|
||||
const hasFileParent = graph.relationships.some(rel =>
|
||||
rel.target === defNode.id &&
|
||||
rel.type === 'CONTAINS' &&
|
||||
rel.type === 'DEFINES' &&
|
||||
graph.nodes.some(n => n.id === rel.source && n.label === 'File')
|
||||
);
|
||||
return !hasFileParent;
|
||||
|
|
|
|||
|
|
@ -67,10 +67,6 @@ export class StructureProcessor {
|
|||
const visibleDirectories = directories.filter(dir => !this.shouldHideDirectory(dir));
|
||||
const hiddenDirectoriesCount = directories.length - visibleDirectories.length;
|
||||
|
||||
if (hiddenDirectoriesCount > 0) {
|
||||
console.log(`StructureProcessor: Hiding ${hiddenDirectoriesCount} ignored directories from KG`);
|
||||
}
|
||||
|
||||
// Create directory nodes only for visible directories
|
||||
const directoryNodes = this.createDirectoryNodes(visibleDirectories);
|
||||
directoryNodes.forEach(node => graph.addNode(node));
|
||||
|
|
@ -79,10 +75,6 @@ export class StructureProcessor {
|
|||
const visibleFiles = files.filter(file => !this.shouldHideFile(file));
|
||||
const hiddenFilesCount = files.length - visibleFiles.length;
|
||||
|
||||
if (hiddenFilesCount > 0) {
|
||||
console.log(`StructureProcessor: Hiding ${hiddenFilesCount} files in ignored directories from KG`);
|
||||
}
|
||||
|
||||
// Create file nodes only for visible files
|
||||
const fileNodes = this.createFileNodes(visibleFiles);
|
||||
fileNodes.forEach(node => graph.addNode(node));
|
||||
|
|
@ -91,7 +83,7 @@ export class StructureProcessor {
|
|||
this.createContainsRelationships(graph, projectNode.id, visibleDirectories, visibleFiles);
|
||||
|
||||
const totalHidden = hiddenDirectoriesCount + hiddenFilesCount;
|
||||
console.log(`StructureProcessor: Created ${graph.nodes.length} nodes total (${totalHidden} items hidden)`);
|
||||
console.log(`StructureProcessor: Created ${graph.nodes.length} nodes total (${totalHidden} items filtered)`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -164,7 +156,7 @@ export class StructureProcessor {
|
|||
for (const dirPath of directoryPaths) {
|
||||
if (!dirPath) continue;
|
||||
|
||||
const id = generateId('folder');
|
||||
const id = generateId(`folder_${dirPath}`);
|
||||
this.nodeIdMap.set(dirPath, id);
|
||||
|
||||
const pathParts = dirPath.split('/');
|
||||
|
|
@ -196,7 +188,7 @@ export class StructureProcessor {
|
|||
for (const filePath of filePaths) {
|
||||
if (!filePath) continue;
|
||||
|
||||
const id = generateId('file');
|
||||
const id = generateId(`file_${filePath}`);
|
||||
this.nodeIdMap.set(filePath, id);
|
||||
|
||||
const fileName = filePath.split('/').pop() || filePath;
|
||||
|
|
|
|||
|
|
@ -26,6 +26,48 @@ export const TYPESCRIPT_QUERIES = {
|
|||
`,
|
||||
};
|
||||
|
||||
// JavaScript queries - similar to TypeScript but without TS-specific syntax
|
||||
export const JAVASCRIPT_QUERIES = {
|
||||
imports: `
|
||||
(import_statement) @import
|
||||
`,
|
||||
classes: `
|
||||
(class_declaration) @class
|
||||
`,
|
||||
methods: `
|
||||
(method_definition) @method
|
||||
`,
|
||||
functions: `
|
||||
(function_declaration) @function
|
||||
`,
|
||||
arrowFunctions: `
|
||||
(lexical_declaration
|
||||
(variable_declarator
|
||||
name: (identifier) @name
|
||||
value: (arrow_function))) @arrow_function
|
||||
`,
|
||||
exports: `
|
||||
(export_statement) @export
|
||||
`,
|
||||
defaultExports: `
|
||||
(export_statement
|
||||
(identifier) @default_export)
|
||||
`,
|
||||
variableAssignments: `
|
||||
(variable_declaration
|
||||
(variable_declarator
|
||||
name: (identifier) @name
|
||||
value: (function_expression))) @var_function
|
||||
`,
|
||||
objectMethods: `
|
||||
(assignment_expression
|
||||
left: (member_expression
|
||||
property: (property_identifier) @name)
|
||||
right: (function_expression)) @obj_method
|
||||
`,
|
||||
// Note: No interfaces or types for pure JavaScript
|
||||
};
|
||||
|
||||
export const PYTHON_QUERIES = {
|
||||
imports: `
|
||||
(import_statement) @import
|
||||
|
|
@ -39,8 +81,14 @@ export const PYTHON_QUERIES = {
|
|||
functions: `
|
||||
(function_definition) @function
|
||||
`,
|
||||
methods: `
|
||||
(class_definition
|
||||
body: (block
|
||||
(function_definition) @method))
|
||||
`,
|
||||
decorators: `
|
||||
(decorator) @decorator
|
||||
(decorated_definition
|
||||
(decorator) @decorator)
|
||||
`,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -477,7 +477,7 @@ export class GitHubService {
|
|||
|
||||
await this.collectPathsAndContent(owner, repo, '', allPaths, fileContents);
|
||||
|
||||
console.log(`GitHub: Discovered ${allPaths.length} total paths, ${fileContents.size} files with content`);
|
||||
console.log(`GitHub: Extracted ${allPaths.length} paths, ${fileContents.size} files`);
|
||||
|
||||
return {
|
||||
allPaths,
|
||||
|
|
@ -503,7 +503,7 @@ export class GitHubService {
|
|||
// Always try to get content for files, but skip unwanted files like .git files
|
||||
// Filtering will happen later in ParsingProcessor, but we can skip obvious files
|
||||
if (this.shouldSkipFileForContent(fullPath)) {
|
||||
console.log(`Skipping file content for: ${fullPath}`);
|
||||
// Reduced logging to avoid console spam
|
||||
} else {
|
||||
try {
|
||||
const content = await this.getFileContent(owner, repo, fullPath);
|
||||
|
|
|
|||
|
|
@ -153,35 +153,44 @@ export class IngestionService {
|
|||
|
||||
// Check if all paths start with the same top-level folder
|
||||
const potentialPrefix = pathParts[0] + '/';
|
||||
const allHaveSamePrefix = paths.every(path => path.startsWith(potentialPrefix));
|
||||
const pathsWithPrefix = paths.filter(path => path.startsWith(potentialPrefix));
|
||||
|
||||
if (!allHaveSamePrefix) {
|
||||
return structure; // No common prefix to remove
|
||||
}
|
||||
// If most paths (>80%) have the common prefix, normalize all paths
|
||||
if (pathsWithPrefix.length > paths.length * 0.8) {
|
||||
console.log(`Normalizing ZIP paths: removing common prefix "${potentialPrefix}" from ${pathsWithPrefix.length}/${paths.length} paths`);
|
||||
|
||||
console.log(`Normalizing ZIP paths: removing common prefix "${potentialPrefix}"`);
|
||||
// Remove the common prefix from all paths
|
||||
const normalizedPaths = paths.map(path => {
|
||||
if (path.startsWith(potentialPrefix)) {
|
||||
const withoutPrefix = path.substring(potentialPrefix.length);
|
||||
return withoutPrefix || path; // Keep original if normalization would result in empty string
|
||||
}
|
||||
// For paths without prefix, keep as-is but filter out the bare container name
|
||||
return path === pathParts[0] ? '' : path;
|
||||
}).filter(path => path.length > 0); // Remove empty paths
|
||||
|
||||
// Remove the common prefix from all paths
|
||||
const normalizedPaths = paths.map(path => {
|
||||
const withoutPrefix = path.substring(potentialPrefix.length);
|
||||
return withoutPrefix || path; // Keep original if normalization would result in empty string
|
||||
}).filter(path => path.length > 0); // Remove empty paths
|
||||
|
||||
// Normalize file contents map
|
||||
const normalizedContents = new Map<string, string>();
|
||||
for (const [originalPath, content] of structure.fileContents) {
|
||||
const normalizedPath = originalPath.startsWith(potentialPrefix)
|
||||
? originalPath.substring(potentialPrefix.length)
|
||||
: originalPath;
|
||||
|
||||
if (normalizedPath) {
|
||||
normalizedContents.set(normalizedPath, content);
|
||||
// Normalize file contents map
|
||||
const normalizedContents = new Map<string, string>();
|
||||
for (const [originalPath, content] of structure.fileContents) {
|
||||
let normalizedPath = originalPath;
|
||||
if (originalPath.startsWith(potentialPrefix)) {
|
||||
normalizedPath = originalPath.substring(potentialPrefix.length);
|
||||
} else if (originalPath === pathParts[0]) {
|
||||
// Skip the bare container directory
|
||||
continue;
|
||||
}
|
||||
|
||||
if (normalizedPath && normalizedPath.length > 0) {
|
||||
normalizedContents.set(normalizedPath, content);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
allPaths: normalizedPaths,
|
||||
fileContents: normalizedContents
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
allPaths: normalizedPaths,
|
||||
fileContents: normalizedContents
|
||||
};
|
||||
return structure; // No normalization if prefix isn't common enough
|
||||
}
|
||||
}
|
||||
|
|
@ -109,7 +109,7 @@ export class ZipService {
|
|||
|
||||
// Skip .git files and other unwanted files during content extraction
|
||||
if (this.shouldSkipFileForContent(normalizedPath)) {
|
||||
console.log(`Skipping file content for: ${normalizedPath}`);
|
||||
// Reduced logging - only log summary at the end
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -149,8 +149,7 @@ export class ZipService {
|
|||
// Wait for all file extractions to complete
|
||||
await Promise.all(filePromises);
|
||||
|
||||
console.log(`ZIP: Discovered ${allPaths.length} total paths, ${fileContents.size} files with content`);
|
||||
console.log(`ZIP: Total extracted size: ${totalExtractedSize} bytes`);
|
||||
console.log(`ZIP: Extracted ${allPaths.length} paths, ${fileContents.size} files (${totalExtractedSize} bytes)`);
|
||||
|
||||
return {
|
||||
allPaths: allPaths.sort(), // Sort for consistent ordering
|
||||
|
|
|
|||
344
src/tests/wasm-verification.test.ts
Normal file
344
src/tests/wasm-verification.test.ts
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
/**
|
||||
* TEMPORARY WASM VERIFICATION TESTS
|
||||
* These tests verify that Tree-sitter WASM files are loading and working correctly
|
||||
* DELETE THIS FILE after confirming everything works
|
||||
*/
|
||||
|
||||
import {
|
||||
initTreeSitter,
|
||||
loadPythonParser,
|
||||
loadJavaScriptParser,
|
||||
loadTypeScriptParser
|
||||
} from '../core/tree-sitter/parser-loader.js';
|
||||
import Parser from 'web-tree-sitter';
|
||||
import { PYTHON_QUERIES, TYPESCRIPT_QUERIES } from '../core/ingestion/tree-sitter-queries';
|
||||
|
||||
describe('🔍 WASM Verification Tests (TEMPORARY)', () => {
|
||||
let parser: Parser;
|
||||
|
||||
beforeAll(async () => {
|
||||
console.log('🧪 Starting WASM verification tests...');
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
console.log('✅ WASM verification tests completed');
|
||||
});
|
||||
|
||||
describe('Main Thread Tree-sitter Initialization', () => {
|
||||
it('should initialize Tree-sitter successfully', async () => {
|
||||
parser = await initTreeSitter();
|
||||
expect(parser).toBeDefined();
|
||||
expect(parser).toBeInstanceOf(Parser);
|
||||
console.log('✅ Tree-sitter main thread initialized successfully');
|
||||
});
|
||||
|
||||
it('should load Python WASM parser', async () => {
|
||||
const pythonLang = await loadPythonParser();
|
||||
expect(pythonLang).toBeDefined();
|
||||
expect(typeof pythonLang.query).toBe('function');
|
||||
console.log('✅ Python WASM parser loaded successfully');
|
||||
});
|
||||
|
||||
it('should load JavaScript WASM parser', async () => {
|
||||
const jsLang = await loadJavaScriptParser();
|
||||
expect(jsLang).toBeDefined();
|
||||
expect(typeof jsLang.query).toBe('function');
|
||||
console.log('✅ JavaScript WASM parser loaded successfully');
|
||||
});
|
||||
|
||||
it('should load TypeScript WASM parser', async () => {
|
||||
const tsLang = await loadTypeScriptParser();
|
||||
expect(tsLang).toBeDefined();
|
||||
expect(typeof tsLang.query).toBe('function');
|
||||
console.log('✅ TypeScript WASM parser loaded successfully');
|
||||
});
|
||||
});
|
||||
|
||||
describe('Python WASM Parsing Verification', () => {
|
||||
let pythonLang: Parser.Language;
|
||||
|
||||
beforeAll(async () => {
|
||||
pythonLang = await loadPythonParser();
|
||||
parser.setLanguage(pythonLang);
|
||||
});
|
||||
|
||||
it('should parse simple Python function', () => {
|
||||
const pythonCode = `
|
||||
def test_function(param1, param2):
|
||||
"""Test function docstring"""
|
||||
return param1 + param2
|
||||
|
||||
class TestClass:
|
||||
def method_one(self):
|
||||
pass
|
||||
|
||||
async def async_method(self):
|
||||
return "async result"
|
||||
|
||||
variable_assignment = "test value"
|
||||
`;
|
||||
|
||||
const tree = parser.parse(pythonCode);
|
||||
expect(tree).toBeDefined();
|
||||
expect(tree.rootNode).toBeDefined();
|
||||
expect(tree.rootNode.type).toBe('module');
|
||||
console.log('✅ Python code parsed successfully');
|
||||
console.log(` Root node type: ${tree.rootNode.type}`);
|
||||
console.log(` Child count: ${tree.rootNode.childCount}`);
|
||||
});
|
||||
|
||||
it('should execute Python queries successfully', () => {
|
||||
const pythonCode = `
|
||||
def hello_world():
|
||||
print("Hello, World!")
|
||||
|
||||
class Calculator:
|
||||
def add(self, a, b):
|
||||
return a + b
|
||||
|
||||
def subtract(self, a, b):
|
||||
return a - b
|
||||
|
||||
result = 42
|
||||
`;
|
||||
|
||||
const tree = parser.parse(pythonCode);
|
||||
|
||||
// Test each Python query
|
||||
for (const [queryName, queryString] of Object.entries(PYTHON_QUERIES)) {
|
||||
try {
|
||||
const query = pythonLang.query(queryString as string);
|
||||
const matches = query.matches(tree.rootNode);
|
||||
|
||||
console.log(` ✅ ${queryName} query executed: ${matches.length} matches`);
|
||||
|
||||
// Log some match details for verification
|
||||
if (matches.length > 0) {
|
||||
const firstMatch = matches[0];
|
||||
console.log(` First match captures: ${firstMatch.captures.length}`);
|
||||
if (firstMatch.captures.length > 0) {
|
||||
const firstCapture = firstMatch.captures[0];
|
||||
console.log(` First capture type: ${firstCapture.node.type}`);
|
||||
console.log(` First capture text: "${firstCapture.node.text.substring(0, 50)}..."`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(` ❌ ${queryName} query failed:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('should extract definitions correctly', () => {
|
||||
const pythonCode = `
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
def process_data(input_data):
|
||||
return input_data.upper()
|
||||
|
||||
class DataProcessor:
|
||||
def __init__(self):
|
||||
self.data = []
|
||||
|
||||
def add_item(self, item):
|
||||
self.data.append(item)
|
||||
|
||||
async def process_async(self):
|
||||
return await some_async_operation()
|
||||
|
||||
@decorator
|
||||
def decorated_function():
|
||||
pass
|
||||
|
||||
global_var = "test"
|
||||
`;
|
||||
|
||||
const tree = parser.parse(pythonCode);
|
||||
let totalDefinitions = 0;
|
||||
|
||||
for (const [queryName, queryString] of Object.entries(PYTHON_QUERIES)) {
|
||||
const query = pythonLang.query(queryString as string);
|
||||
const matches = query.matches(tree.rootNode);
|
||||
|
||||
for (const match of matches) {
|
||||
for (const capture of match.captures) {
|
||||
totalDefinitions++;
|
||||
const node = capture.node;
|
||||
console.log(` Found ${queryName}: "${node.text.split('\n')[0]}" at line ${node.startPosition.row + 1}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(totalDefinitions).toBeGreaterThan(0);
|
||||
console.log(`✅ Extracted ${totalDefinitions} total definitions`);
|
||||
});
|
||||
});
|
||||
|
||||
describe('TypeScript WASM Parsing Verification', () => {
|
||||
let typescriptLang: Parser.Language;
|
||||
|
||||
beforeAll(async () => {
|
||||
typescriptLang = await loadTypeScriptParser();
|
||||
parser.setLanguage(typescriptLang);
|
||||
});
|
||||
|
||||
it('should parse TypeScript code', () => {
|
||||
const tsCode = `
|
||||
import React from 'react';
|
||||
|
||||
interface User {
|
||||
id: number;
|
||||
name: string;
|
||||
}
|
||||
|
||||
class UserService {
|
||||
getUser(id: number): User {
|
||||
return { id, name: 'Test User' };
|
||||
}
|
||||
}
|
||||
|
||||
const createUser = (name: string): User => {
|
||||
return { id: Date.now(), name };
|
||||
};
|
||||
|
||||
function processUser(user: User): void {
|
||||
console.log(user.name);
|
||||
}
|
||||
`;
|
||||
|
||||
const tree = parser.parse(tsCode);
|
||||
expect(tree).toBeDefined();
|
||||
expect(tree.rootNode.type).toBe('program');
|
||||
console.log('✅ TypeScript code parsed successfully');
|
||||
});
|
||||
|
||||
it('should execute TypeScript queries successfully', () => {
|
||||
const tsCode = `
|
||||
class TestClass {
|
||||
method(): string {
|
||||
return "test";
|
||||
}
|
||||
}
|
||||
|
||||
function testFunction(): void {}
|
||||
|
||||
const arrowFunc = () => "result";
|
||||
|
||||
interface TestInterface {
|
||||
prop: string;
|
||||
}
|
||||
`;
|
||||
|
||||
const tree = parser.parse(tsCode);
|
||||
|
||||
for (const [queryName, queryString] of Object.entries(TYPESCRIPT_QUERIES)) {
|
||||
try {
|
||||
const query = typescriptLang.query(queryString as string);
|
||||
const matches = query.matches(tree.rootNode);
|
||||
console.log(` ✅ ${queryName} query executed: ${matches.length} matches`);
|
||||
} catch (error) {
|
||||
console.error(` ❌ ${queryName} query failed:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('Worker Thread WASM Verification', () => {
|
||||
it('should verify worker can load and use WASM', async () => {
|
||||
// Create a test worker to verify WASM loading
|
||||
const workerCode = `
|
||||
import Parser from 'web-tree-sitter';
|
||||
|
||||
async function testWorkerWasm() {
|
||||
try {
|
||||
// Test worker WASM initialization
|
||||
await Parser.init();
|
||||
const parser = new Parser();
|
||||
|
||||
// Test loading Python parser
|
||||
const pythonLang = await Parser.Language.load('/wasm/python/tree-sitter-python.wasm');
|
||||
parser.setLanguage(pythonLang);
|
||||
|
||||
// Test parsing
|
||||
const tree = parser.parse('def test(): pass');
|
||||
|
||||
return {
|
||||
success: true,
|
||||
rootNodeType: tree.rootNode.type,
|
||||
childCount: tree.rootNode.childCount
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
error: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
testWorkerWasm().then(result => {
|
||||
self.postMessage(result);
|
||||
});
|
||||
`;
|
||||
|
||||
const blob = new Blob([workerCode], { type: 'application/javascript' });
|
||||
const workerUrl = URL.createObjectURL(blob);
|
||||
|
||||
const workerResult = await new Promise((resolve, reject) => {
|
||||
const worker = new Worker(workerUrl, { type: 'module' });
|
||||
|
||||
worker.onmessage = (event) => {
|
||||
worker.terminate();
|
||||
URL.revokeObjectURL(workerUrl);
|
||||
resolve(event.data);
|
||||
};
|
||||
|
||||
worker.onerror = (error) => {
|
||||
worker.terminate();
|
||||
URL.revokeObjectURL(workerUrl);
|
||||
reject(error);
|
||||
};
|
||||
|
||||
// Timeout after 10 seconds
|
||||
setTimeout(() => {
|
||||
worker.terminate();
|
||||
URL.revokeObjectURL(workerUrl);
|
||||
reject(new Error('Worker test timeout'));
|
||||
}, 10000);
|
||||
});
|
||||
|
||||
console.log('🔍 Worker WASM test result:', workerResult);
|
||||
expect(workerResult).toHaveProperty('success');
|
||||
|
||||
if (!(workerResult as any).success) {
|
||||
throw new Error(`Worker WASM failed: ${(workerResult as any).error}`);
|
||||
}
|
||||
|
||||
console.log('✅ Worker WASM verification passed');
|
||||
}, 15000); // 15 second timeout for this test
|
||||
});
|
||||
|
||||
describe('WASM File Accessibility Check', () => {
|
||||
it('should verify WASM files are accessible', async () => {
|
||||
const wasmFiles = [
|
||||
'/wasm/python/tree-sitter-python.wasm',
|
||||
'/wasm/javascript/tree-sitter-javascript.wasm',
|
||||
'/wasm/typescript/tree-sitter-typescript.wasm',
|
||||
'/wasm/tree-sitter.wasm'
|
||||
];
|
||||
|
||||
for (const wasmPath of wasmFiles) {
|
||||
try {
|
||||
const response = await fetch(wasmPath);
|
||||
expect(response.ok).toBe(true);
|
||||
expect(response.headers.get('content-type')).toContain('wasm');
|
||||
console.log(`✅ ${wasmPath} is accessible (${response.status})`);
|
||||
} catch (error) {
|
||||
console.error(`❌ ${wasmPath} failed to load:`, error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -244,22 +244,43 @@ const FloatingSourceViewer: React.FC<FloatingSourceViewerProps> = ({
|
|||
rel.type === 'CONTAINS' && rel.target === nodeId
|
||||
);
|
||||
|
||||
if (containsRel) {
|
||||
const sourceNode = graph.nodes.find(n => n.id === containsRel.source);
|
||||
// If no CONTAINS relationship found, try DEFINES relationship (fallback)
|
||||
const definesRel = !containsRel ? graph.relationships.find(rel =>
|
||||
rel.type === 'DEFINES' && rel.target === nodeId
|
||||
) : null;
|
||||
|
||||
const fileRel = containsRel || definesRel;
|
||||
|
||||
if (fileRel) {
|
||||
const sourceNode = graph.nodes.find(n => n.id === fileRel.source);
|
||||
if (sourceNode && sourceNode.properties.filePath) {
|
||||
filePath = sourceNode.properties.filePath as string;
|
||||
}
|
||||
}
|
||||
|
||||
// If not found through relationships, search through file contents
|
||||
if (!filePath && fileContents) {
|
||||
// BUT ONLY for Function, Method, Class, and Variable nodes - NOT for Folder or File nodes
|
||||
if (!filePath && fileContents && ['Function', 'Method', 'Class', 'Variable'].includes(nodeType)) {
|
||||
console.log('FloatingSourceViewer - Searching file contents for node:', nodeName, 'of type:', nodeType);
|
||||
|
||||
// Use more specific search patterns instead of just checking if content includes nodeName
|
||||
const searchPatterns = [
|
||||
`def ${nodeName}(`, // Python function
|
||||
`function ${nodeName}(`, // JavaScript function
|
||||
`const ${nodeName} =`, // JavaScript const
|
||||
`class ${nodeName}`, // Class definition
|
||||
];
|
||||
|
||||
for (const [path, content] of fileContents) {
|
||||
// Skip .git files, node_modules, and other unwanted directories
|
||||
if (shouldSkipFileForSearch(path)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (content.includes(nodeName)) {
|
||||
// Check if any specific pattern matches instead of just nodeName
|
||||
const foundPattern = searchPatterns.find(pattern => content.includes(pattern));
|
||||
if (foundPattern) {
|
||||
console.log(`FloatingSourceViewer - Found pattern "${foundPattern}" in ${path}`);
|
||||
filePath = path;
|
||||
break;
|
||||
}
|
||||
|
|
@ -267,25 +288,73 @@ const FloatingSourceViewer: React.FC<FloatingSourceViewerProps> = ({
|
|||
}
|
||||
|
||||
if (!filePath) {
|
||||
// Special handling for Folder nodes
|
||||
if (nodeType === 'Folder') {
|
||||
const folderPath = node.properties.path as string || nodeName;
|
||||
const childFiles = graph?.nodes?.filter(n =>
|
||||
n.label === 'File' &&
|
||||
(n.properties.filePath as string || n.properties.path as string || '').startsWith(folderPath + '/')
|
||||
) || [];
|
||||
|
||||
const childFolders = graph?.nodes?.filter(n =>
|
||||
n.label === 'Folder' &&
|
||||
(n.properties.path as string || '').startsWith(folderPath + '/') &&
|
||||
(n.properties.path as string || '').split('/').length === folderPath.split('/').length + 1
|
||||
) || [];
|
||||
|
||||
let directoryContent = `# Directory: ${folderPath}\n`;
|
||||
directoryContent += `# This folder contains ${childFiles.length} files and ${childFolders.length} subfolders\n\n`;
|
||||
|
||||
if (childFolders.length > 0) {
|
||||
directoryContent += '## Subdirectories:\n';
|
||||
childFolders.forEach(folder => {
|
||||
const name = folder.properties.name as string || 'Unknown';
|
||||
directoryContent += `- 📁 ${name}\n`;
|
||||
});
|
||||
directoryContent += '\n';
|
||||
}
|
||||
|
||||
if (childFiles.length > 0) {
|
||||
directoryContent += '## Files:\n';
|
||||
childFiles.slice(0, 15).forEach(file => {
|
||||
const name = file.properties.name as string || 'Unknown';
|
||||
const ext = file.properties.extension as string || '';
|
||||
const icon = ext === '.py' ? '🐍' : ext === '.js' ? '📜' : ext === '.ts' ? '📘' : '📄';
|
||||
directoryContent += `- ${icon} ${name}\n`;
|
||||
});
|
||||
if (childFiles.length > 15) {
|
||||
directoryContent += `... and ${childFiles.length - 15} more files\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
fileName: folderPath.split('/').pop() || folderPath,
|
||||
filePath: 'folder-listing',
|
||||
content: directoryContent,
|
||||
nodeType,
|
||||
nodeName,
|
||||
language: 'markdown'
|
||||
};
|
||||
}
|
||||
|
||||
// Return mock content for nodes without file association
|
||||
const mockContent = `// ${nodeType}: ${nodeName}
|
||||
// This ${nodeType.toLowerCase()} is part of the knowledge graph
|
||||
// File path not available in the current context
|
||||
// This is a ${nodeType.toLowerCase()} definition in the knowledge graph
|
||||
|
||||
${nodeType === 'Function' ? `function ${nodeName}() {
|
||||
// Implementation details would be here
|
||||
return true;
|
||||
// Function implementation not available in current context
|
||||
// This may be an external library function or incomplete parsing
|
||||
}` : nodeType === 'Class' ? `class ${nodeName} {
|
||||
constructor() {
|
||||
// Constructor implementation
|
||||
}
|
||||
|
||||
// Class methods would be here
|
||||
}` : `// ${nodeType} definition for ${nodeName}`}`;
|
||||
// Class definition not available in current context
|
||||
// This may be an external library class or incomplete parsing
|
||||
}` : nodeType === 'Method' ? `${nodeName}() {
|
||||
// Method implementation not available in current context
|
||||
}` : `// ${nodeType} definition for ${nodeName}
|
||||
// Content not available in current context`}`;
|
||||
|
||||
return {
|
||||
fileName: `${nodeName}.${nodeType.toLowerCase()}`,
|
||||
filePath: 'graph-node',
|
||||
fileName: `${nodeName}`,
|
||||
filePath: 'virtual-node',
|
||||
content: mockContent,
|
||||
nodeType,
|
||||
nodeName,
|
||||
|
|
@ -293,20 +362,52 @@ ${nodeType === 'Function' ? `function ${nodeName}() {
|
|||
};
|
||||
}
|
||||
|
||||
const content = fileContents.get(filePath);
|
||||
if (!content) return null;
|
||||
// Try to find the file content using various path resolution strategies
|
||||
const nodeFilePath = node.properties.filePath as string ||
|
||||
node.properties.path as string ||
|
||||
node.properties.name as string;
|
||||
|
||||
if (nodeFilePath) {
|
||||
// Try exact match first
|
||||
let content = fileContents.get(nodeFilePath);
|
||||
|
||||
// If not found, try different path variations
|
||||
if (!content) {
|
||||
// Try relative paths starting from different roots
|
||||
const pathVariations = [
|
||||
nodeFilePath,
|
||||
nodeFilePath.replace(/^[./]*/, ''), // Remove leading ./ or /
|
||||
nodeFilePath.startsWith('/') ? nodeFilePath.substring(1) : `/${nodeFilePath}`, // Toggle leading slash
|
||||
`src/${nodeFilePath}`, // Try under src/
|
||||
nodeFilePath.replace(/\\/g, '/'), // Convert backslashes to forward slashes
|
||||
nodeFilePath.replace(/\//g, '\\') // Convert forward slashes to backslashes
|
||||
];
|
||||
|
||||
for (const variation of pathVariations) {
|
||||
content = fileContents.get(variation);
|
||||
if (content) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (content) {
|
||||
const extractedContent = extractRelevantContent(content, nodeName, nodeType);
|
||||
const language = nodeFilePath.split('.').pop() || 'text';
|
||||
|
||||
const extractedContent = extractRelevantContent(content, nodeName, nodeType);
|
||||
const language = filePath.split('.').pop() || 'text';
|
||||
|
||||
return {
|
||||
fileName: filePath.split('/').pop() || filePath,
|
||||
filePath,
|
||||
content: extractedContent || content.substring(0, 500) + '...',
|
||||
nodeType,
|
||||
nodeName,
|
||||
language
|
||||
};
|
||||
return {
|
||||
fileName: nodeFilePath.split('/').pop() || nodeFilePath,
|
||||
filePath: nodeFilePath,
|
||||
content: extractedContent || content.substring(0, 500) + '...',
|
||||
nodeType,
|
||||
nodeName,
|
||||
language
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// If no content found, return null to trigger the mock content generation above
|
||||
return null;
|
||||
}, [nodeId, graph, fileContents]);
|
||||
|
||||
// Dragging functionality
|
||||
|
|
|
|||
|
|
@ -423,17 +423,25 @@ def hasattr(obj: Any, name: str) -> bool:
|
|||
rel.type === 'CONTAINS' && rel.target === selectedNodeId
|
||||
);
|
||||
|
||||
if (containsRelationship) {
|
||||
const fileNode = graph.nodes.find(n => n.id === containsRelationship.source);
|
||||
// If no CONTAINS relationship found, try DEFINES relationship (fallback)
|
||||
const definesRelationship = !containsRelationship ? graph.relationships?.find(rel =>
|
||||
rel.type === 'DEFINES' && rel.target === selectedNodeId
|
||||
) : null;
|
||||
|
||||
const fileRelationship = containsRelationship || definesRelationship;
|
||||
|
||||
if (fileRelationship) {
|
||||
const fileNode = graph.nodes.find(n => n.id === fileRelationship.source);
|
||||
if (fileNode && fileNode.label === 'File') {
|
||||
filePath = fileNode.properties.path as string || fileNode.properties.filePath as string;
|
||||
console.log('SourceViewer - Found file through CONTAINS relationship:', filePath);
|
||||
console.log('SourceViewer - Found file through relationship:', filePath, 'via', fileRelationship.type);
|
||||
}
|
||||
}
|
||||
|
||||
// If still no file path, try reverse lookup by searching for the node name in file contents
|
||||
if (!filePath && fileContents) {
|
||||
console.log('SourceViewer - Searching file contents for node:', nodeName);
|
||||
// BUT ONLY for Function, Method, Class, and Variable nodes - NOT for Folder or File nodes
|
||||
if (!filePath && fileContents && ['Function', 'Method', 'Class', 'Variable'].includes(node.label)) {
|
||||
console.log('SourceViewer - Searching file contents for node:', nodeName, 'of type:', node.label);
|
||||
|
||||
// Try multiple search patterns for the function
|
||||
const searchPatterns = [
|
||||
|
|
@ -481,8 +489,9 @@ def hasattr(obj: Any, name: str) -> bool:
|
|||
}
|
||||
|
||||
// If still not found, try a more lenient search (case-insensitive)
|
||||
if (!filePath) {
|
||||
console.log('SourceViewer - Trying case-insensitive search for:', nodeName);
|
||||
// BUT ONLY for Function, Method, Class, and Variable nodes
|
||||
if (!filePath && ['Function', 'Method', 'Class', 'Variable'].includes(node.label)) {
|
||||
console.log('SourceViewer - Trying case-insensitive search for:', nodeName, 'of type:', node.label);
|
||||
for (const [path, content] of fileContents) {
|
||||
// Skip .git files, node_modules, and other unwanted directories
|
||||
if (shouldSkipFileForSearch(path)) {
|
||||
|
|
@ -510,10 +519,20 @@ def hasattr(obj: Any, name: str) -> bool:
|
|||
console.log('SourceViewer - Final node details:', {
|
||||
nodeId: selectedNodeId,
|
||||
nodeName,
|
||||
nodeLabel: node.label,
|
||||
nodeType: node.label,
|
||||
filePath,
|
||||
fileName,
|
||||
nodeLabel: node.label,
|
||||
nodeProperties: node.properties,
|
||||
// CRITICAL DEBUG: Show what type this node actually is
|
||||
actualNodeInfo: {
|
||||
isFolder: node.label === 'Folder',
|
||||
isFile: node.label === 'File',
|
||||
isFunction: node.label === 'Function',
|
||||
isClass: node.label === 'Class',
|
||||
nodeLabel: node.label,
|
||||
nodeName: nodeName
|
||||
},
|
||||
fileContentsSize: fileContents?.size || 0,
|
||||
// Add detailed relationship debugging
|
||||
allRelationships: graph?.relationships?.length || 0,
|
||||
|
|
@ -536,7 +555,55 @@ def hasattr(obj: Any, name: str) -> bool:
|
|||
// Try to get actual file content
|
||||
let content = '';
|
||||
|
||||
if (filePath && fileContents && fileContents.has(filePath)) {
|
||||
// Special handling for Folder nodes
|
||||
if (node.label === 'Folder') {
|
||||
const folderPath = filePath || nodeName || 'Unknown Folder';
|
||||
console.log('SourceViewer - Processing folder node:', {
|
||||
folderPath,
|
||||
nodeName,
|
||||
nodeProperties: node.properties
|
||||
});
|
||||
|
||||
const childFiles = graph?.nodes?.filter(n =>
|
||||
n.label === 'File' &&
|
||||
(n.properties.filePath as string || n.properties.path as string || '').startsWith(folderPath + '/')
|
||||
) || [];
|
||||
|
||||
const childFolders = graph?.nodes?.filter(n =>
|
||||
n.label === 'Folder' &&
|
||||
(n.properties.path as string || '').startsWith(folderPath + '/') &&
|
||||
(n.properties.path as string || '').split('/').length === folderPath.split('/').length + 1
|
||||
) || [];
|
||||
|
||||
content = `# Directory: ${folderPath}\n`;
|
||||
content += `# Node Type: ${node.label}\n`;
|
||||
content += `# This folder contains ${childFiles.length} files and ${childFolders.length} subfolders\n\n`;
|
||||
|
||||
if (childFolders.length > 0) {
|
||||
content += '## Subdirectories:\n';
|
||||
childFolders.forEach(folder => {
|
||||
const name = folder.properties.name as string || 'Unknown';
|
||||
content += `- 📁 ${name}\n`;
|
||||
});
|
||||
content += '\n';
|
||||
}
|
||||
|
||||
if (childFiles.length > 0) {
|
||||
content += '## Files:\n';
|
||||
childFiles.slice(0, 20).forEach(file => {
|
||||
const name = file.properties.name as string || 'Unknown';
|
||||
const ext = file.properties.extension as string || '';
|
||||
const icon = ext === '.py' ? '🐍' : ext === '.js' ? '📜' : ext === '.ts' ? '📘' : '📄';
|
||||
content += `- ${icon} ${name}\n`;
|
||||
});
|
||||
if (childFiles.length > 20) {
|
||||
content += `... and ${childFiles.length - 20} more files\n`;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('SourceViewer - Generated directory content for folder:', folderPath);
|
||||
}
|
||||
else if (filePath && fileContents && fileContents.has(filePath)) {
|
||||
content = fileContents.get(filePath)!;
|
||||
console.log('SourceViewer - Found file content for:', filePath);
|
||||
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ const GraphVisualization: React.FC<GraphVisualizationProps> = ({
|
|||
|
||||
return {
|
||||
id: node.id,
|
||||
label: node.properties.name as string || node.id,
|
||||
label: getNodeDisplayName(node),
|
||||
nodeType: node.label.toLowerCase(),
|
||||
properties: node.properties,
|
||||
color,
|
||||
|
|
@ -258,6 +258,50 @@ const GraphVisualization: React.FC<GraphVisualizationProps> = ({
|
|||
return { nodes, links };
|
||||
};
|
||||
|
||||
// Helper function to get proper display name for nodes
|
||||
const getNodeDisplayName = (node: GraphNode): string => {
|
||||
// Use the name property if available
|
||||
if (node.properties.name && typeof node.properties.name === 'string') {
|
||||
const name = node.properties.name;
|
||||
|
||||
// For file nodes, show just the filename without path
|
||||
if (node.label.toLowerCase() === 'file') {
|
||||
const fileName = name.split('/').pop() || name;
|
||||
return fileName;
|
||||
}
|
||||
|
||||
// For other nodes, use the name as-is
|
||||
return name;
|
||||
}
|
||||
|
||||
// Fallback to filePath for file nodes
|
||||
if (node.label.toLowerCase() === 'file' && node.properties.filePath) {
|
||||
const filePath = node.properties.filePath as string;
|
||||
const fileName = filePath.split('/').pop() || filePath;
|
||||
return fileName;
|
||||
}
|
||||
|
||||
// For function/method/class nodes, try common property names
|
||||
if (['function', 'method', 'class', 'interface'].includes(node.label.toLowerCase())) {
|
||||
const functionName = node.properties.functionName ||
|
||||
node.properties.methodName ||
|
||||
node.properties.className ||
|
||||
node.properties.interfaceName;
|
||||
if (functionName && typeof functionName === 'string') {
|
||||
return functionName;
|
||||
}
|
||||
}
|
||||
|
||||
// Last resort: use a cleaned version of the node ID
|
||||
let displayName = node.id;
|
||||
|
||||
// Remove common prefixes that might make it look like placeholder text
|
||||
displayName = displayName.replace(/^(file|function|method|class)_?/i, '');
|
||||
displayName = displayName.replace(/^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i, 'Unknown');
|
||||
|
||||
return displayName;
|
||||
};
|
||||
|
||||
// Helper function to adjust color brightness
|
||||
const adjustColorBrightness = (hex: string, percent: number): string => {
|
||||
const num = parseInt(hex.replace("#", ""), 16);
|
||||
|
|
|
|||
|
|
@ -70,8 +70,8 @@ export class IngestionWorker {
|
|||
fileContents: fileContentsMap
|
||||
});
|
||||
|
||||
// Clear file contents to free memory after processing
|
||||
fileContentsMap.clear();
|
||||
// Note: Keeping file contents available for UI components
|
||||
// fileContentsMap.clear(); // Commented out to preserve file contents for SourceViewer
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
|
|
|
|||
|
|
@ -71,8 +71,8 @@ export class KuzuIngestionWorker {
|
|||
fileContents: fileContentsMap
|
||||
});
|
||||
|
||||
// Clear file contents to free memory after processing
|
||||
fileContentsMap.clear();
|
||||
// Note: Keeping file contents available for UI components
|
||||
// fileContentsMap.clear(); // Commented out to preserve file contents for SourceViewer
|
||||
|
||||
const duration = Date.now() - startTime;
|
||||
|
||||
|
|
|
|||
753
test-export-functionality.html
Normal file
753
test-export-functionality.html
Normal file
|
|
@ -0,0 +1,753 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Export Functionality Test</title>
|
||||
<style>
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
max-width: 1200px;
|
||||
margin: 0 auto;
|
||||
padding: 20px;
|
||||
background-color: #f5f5f5;
|
||||
}
|
||||
.container {
|
||||
background: white;
|
||||
padding: 20px;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
}
|
||||
.test-section {
|
||||
margin: 20px 0;
|
||||
padding: 15px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 5px;
|
||||
background: #f9f9f9;
|
||||
}
|
||||
.success { border-color: #28a745; background: #d4edda; }
|
||||
.error { border-color: #dc3545; background: #f8d7da; }
|
||||
.warning { border-color: #ffc107; background: #fff3cd; }
|
||||
button {
|
||||
background: #007bff;
|
||||
color: white;
|
||||
border: none;
|
||||
padding: 10px 20px;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
margin: 5px;
|
||||
}
|
||||
button:hover { background: #0056b3; }
|
||||
button:disabled { background: #6c757d; cursor: not-allowed; }
|
||||
#output {
|
||||
white-space: pre-wrap;
|
||||
background: #f8f9fa;
|
||||
padding: 15px;
|
||||
border-radius: 4px;
|
||||
font-family: monospace;
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
margin-top: 10px;
|
||||
}
|
||||
.stats {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 10px;
|
||||
margin: 15px 0;
|
||||
}
|
||||
.stat-card {
|
||||
background: white;
|
||||
padding: 15px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #ddd;
|
||||
text-align: center;
|
||||
}
|
||||
.stat-number {
|
||||
font-size: 24px;
|
||||
font-weight: bold;
|
||||
color: #007bff;
|
||||
}
|
||||
.stat-label {
|
||||
font-size: 14px;
|
||||
color: #666;
|
||||
margin-top: 5px;
|
||||
}
|
||||
.download-link {
|
||||
display: inline-block;
|
||||
background: #28a745;
|
||||
color: white;
|
||||
padding: 8px 15px;
|
||||
text-decoration: none;
|
||||
border-radius: 4px;
|
||||
margin: 5px;
|
||||
}
|
||||
.download-link:hover {
|
||||
background: #1e7e34;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<h1>🧪 Export Functionality Test</h1>
|
||||
<p>This test verifies that the export functionality works correctly after our import resolution improvements.</p>
|
||||
|
||||
<div class="test-section">
|
||||
<h3>📊 Test Knowledge Graph</h3>
|
||||
<p>We'll create a sample knowledge graph with the types of nodes our import resolution system now properly handles.</p>
|
||||
<button onclick="createTestGraph()">Create Test Graph</button>
|
||||
<div id="graph-stats" class="stats"></div>
|
||||
</div>
|
||||
|
||||
<div class="test-section">
|
||||
<h3>🔍 Validation Tests</h3>
|
||||
<p>Test graph validation before export:</p>
|
||||
<button onclick="validateGraph()">Validate Graph</button>
|
||||
<button onclick="calculateSize()">Calculate Export Size</button>
|
||||
<div id="validation-output"></div>
|
||||
</div>
|
||||
|
||||
<div class="test-section">
|
||||
<h3>📥 Export Tests</h3>
|
||||
<p>Test different export formats:</p>
|
||||
<button onclick="testJSONExport()">Test JSON Export</button>
|
||||
<button onclick="testCSVExport()">Test CSV Export</button>
|
||||
<button onclick="testFilteredExport()">Test Filtered Export</button>
|
||||
<div id="export-links"></div>
|
||||
</div>
|
||||
|
||||
<div class="test-section">
|
||||
<h3>📤 Import Test</h3>
|
||||
<p>Test importing exported data:</p>
|
||||
<button onclick="testImport()">Test JSON Import</button>
|
||||
<div id="import-output"></div>
|
||||
</div>
|
||||
|
||||
<div class="test-section">
|
||||
<h3>🔧 Console Output</h3>
|
||||
<div id="output"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="module">
|
||||
// Mock the export functions (in real app these would be imported)
|
||||
window.testGraph = null;
|
||||
window.testFileContents = null;
|
||||
|
||||
// Create a comprehensive test graph
|
||||
window.createTestGraph = function() {
|
||||
const graph = {
|
||||
nodes: [
|
||||
// Project structure nodes
|
||||
{
|
||||
id: 'project_root',
|
||||
label: 'Project',
|
||||
properties: {
|
||||
name: 'GitNexus',
|
||||
description: 'Knowledge graph generation tool',
|
||||
version: '1.0.0'
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'src_folder',
|
||||
label: 'Folder',
|
||||
properties: {
|
||||
name: 'src',
|
||||
path: 'src',
|
||||
filePath: 'src'
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'services_folder',
|
||||
label: 'Folder',
|
||||
properties: {
|
||||
name: 'services',
|
||||
path: 'src/services',
|
||||
filePath: 'src/services'
|
||||
}
|
||||
},
|
||||
// File nodes (including our zip.ts)
|
||||
{
|
||||
id: 'zip_service_file',
|
||||
label: 'File',
|
||||
properties: {
|
||||
name: 'zip.ts',
|
||||
filePath: 'src/services/zip.ts',
|
||||
language: 'typescript',
|
||||
extension: '.ts',
|
||||
lineCount: 600
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'import_processor_file',
|
||||
label: 'File',
|
||||
properties: {
|
||||
name: 'import-processor.ts',
|
||||
filePath: 'src/core/ingestion/import-processor.ts',
|
||||
language: 'typescript',
|
||||
extension: '.ts',
|
||||
lineCount: 946
|
||||
}
|
||||
},
|
||||
// Class nodes
|
||||
{
|
||||
id: 'zip_service_class',
|
||||
label: 'Class',
|
||||
properties: {
|
||||
name: 'ZipService',
|
||||
filePath: 'src/services/zip.ts',
|
||||
startLine: 26,
|
||||
endLine: 599,
|
||||
type: 'class',
|
||||
qualifiedName: 'services.zip.ZipService'
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'import_processor_class',
|
||||
label: 'Class',
|
||||
properties: {
|
||||
name: 'ImportProcessor',
|
||||
filePath: 'src/core/ingestion/import-processor.ts',
|
||||
startLine: 50,
|
||||
endLine: 900,
|
||||
type: 'class',
|
||||
qualifiedName: 'core.ingestion.ImportProcessor'
|
||||
}
|
||||
},
|
||||
// Method nodes (reflecting our import resolution improvements)
|
||||
{
|
||||
id: 'extract_complete_structure',
|
||||
label: 'Method',
|
||||
properties: {
|
||||
name: 'extractCompleteStructure',
|
||||
filePath: 'src/services/zip.ts',
|
||||
startLine: 41,
|
||||
endLine: 120,
|
||||
type: 'method',
|
||||
qualifiedName: 'services.zip.ZipService.extractCompleteStructure'
|
||||
}
|
||||
},
|
||||
{
|
||||
id: 'extract_js_imports',
|
||||
label: 'Method',
|
||||
properties: {
|
||||
name: 'extractJSImports',
|
||||
filePath: 'src/core/ingestion/import-processor.ts',
|
||||
startLine: 210,
|
||||
endLine: 280,
|
||||
type: 'method',
|
||||
qualifiedName: 'core.ingestion.ImportProcessor.extractJSImports'
|
||||
}
|
||||
},
|
||||
// Function nodes
|
||||
{
|
||||
id: 'process_js_import_clause',
|
||||
label: 'Function',
|
||||
properties: {
|
||||
name: 'processJSImportClause',
|
||||
filePath: 'src/core/ingestion/import-processor.ts',
|
||||
startLine: 350,
|
||||
endLine: 450,
|
||||
type: 'function',
|
||||
qualifiedName: 'core.ingestion.processJSImportClause'
|
||||
}
|
||||
},
|
||||
// Import nodes (showing our improved import resolution)
|
||||
{
|
||||
id: 'jszip_import',
|
||||
label: 'Import',
|
||||
properties: {
|
||||
name: 'JSZip',
|
||||
filePath: 'src/services/zip.ts',
|
||||
importType: 'default',
|
||||
targetFile: 'node_modules/jszip/dist/jszip.min.js'
|
||||
}
|
||||
}
|
||||
],
|
||||
relationships: [
|
||||
// Containment relationships
|
||||
{
|
||||
id: 'rel_project_contains_src',
|
||||
type: 'CONTAINS',
|
||||
source: 'project_root',
|
||||
target: 'src_folder',
|
||||
properties: {}
|
||||
},
|
||||
{
|
||||
id: 'rel_src_contains_services',
|
||||
type: 'CONTAINS',
|
||||
source: 'src_folder',
|
||||
target: 'services_folder',
|
||||
properties: {}
|
||||
},
|
||||
{
|
||||
id: 'rel_services_contains_zip',
|
||||
type: 'CONTAINS',
|
||||
source: 'services_folder',
|
||||
target: 'zip_service_file',
|
||||
properties: {}
|
||||
},
|
||||
// Definition relationships
|
||||
{
|
||||
id: 'rel_zip_file_defines_class',
|
||||
type: 'DEFINES',
|
||||
source: 'zip_service_file',
|
||||
target: 'zip_service_class',
|
||||
properties: {}
|
||||
},
|
||||
{
|
||||
id: 'rel_zip_class_defines_method',
|
||||
type: 'DEFINES',
|
||||
source: 'zip_service_class',
|
||||
target: 'extract_complete_structure',
|
||||
properties: {}
|
||||
},
|
||||
{
|
||||
id: 'rel_import_processor_defines_method',
|
||||
type: 'DEFINES',
|
||||
source: 'import_processor_class',
|
||||
target: 'extract_js_imports',
|
||||
properties: {}
|
||||
},
|
||||
// Import relationships (our improved import resolution)
|
||||
{
|
||||
id: 'rel_zip_imports_jszip',
|
||||
type: 'IMPORTS',
|
||||
source: 'zip_service_file',
|
||||
target: 'jszip_import',
|
||||
properties: {
|
||||
importType: 'default',
|
||||
localName: 'JSZip',
|
||||
exportedName: 'default'
|
||||
}
|
||||
},
|
||||
// Call relationships
|
||||
{
|
||||
id: 'rel_extract_calls_process',
|
||||
type: 'CALLS',
|
||||
source: 'extract_js_imports',
|
||||
target: 'process_js_import_clause',
|
||||
properties: {
|
||||
callType: 'method'
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
// Create mock file contents
|
||||
const fileContents = new Map([
|
||||
['src/services/zip.ts', `import JSZip from 'jszip';
|
||||
|
||||
export class ZipService {
|
||||
public async extractCompleteStructure(file: File) {
|
||||
// Implementation with 415 imports resolved (100% success)
|
||||
console.log('🔍 ImportProcessor: Found 415 imports, resolved 415 (100.0%)');
|
||||
}
|
||||
}`],
|
||||
['src/core/ingestion/import-processor.ts', `export class ImportProcessor {
|
||||
private extractJSImports(node: Parser.SyntaxNode, filePath: string): void {
|
||||
// CRITICAL FIX: Field-based approach with type-based fallback
|
||||
let importClauseNode = node.childForFieldName('import_clause');
|
||||
|
||||
if (!importClauseNode) {
|
||||
for (let i = 0; i < node.childCount; i++) {
|
||||
const child = node.child(i);
|
||||
if (child?.type === 'import_clause') {
|
||||
importClauseNode = child;
|
||||
console.log('🔍 DEBUG: Found import_clause by type at index', i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`]
|
||||
]);
|
||||
|
||||
window.testGraph = graph;
|
||||
window.testFileContents = fileContents;
|
||||
|
||||
// Display stats
|
||||
updateGraphStats(graph);
|
||||
log('✅ Test graph created successfully');
|
||||
log(`📊 Nodes: ${graph.nodes.length}, Relationships: ${graph.relationships.length}`);
|
||||
log(`📁 File contents: ${fileContents.size} files`);
|
||||
|
||||
return graph;
|
||||
};
|
||||
|
||||
// Update graph statistics display
|
||||
function updateGraphStats(graph) {
|
||||
const nodesByType = graph.nodes.reduce((acc, node) => {
|
||||
acc[node.label] = (acc[node.label] || 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const relsByType = graph.relationships.reduce((acc, rel) => {
|
||||
acc[rel.type] = (acc[rel.type] || 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const statsContainer = document.getElementById('graph-stats');
|
||||
statsContainer.innerHTML = `
|
||||
<div class="stat-card">
|
||||
<div class="stat-number">${graph.nodes.length}</div>
|
||||
<div class="stat-label">Total Nodes</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="stat-number">${graph.relationships.length}</div>
|
||||
<div class="stat-label">Total Relationships</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="stat-number">${Object.keys(nodesByType).length}</div>
|
||||
<div class="stat-label">Node Types</div>
|
||||
</div>
|
||||
<div class="stat-card">
|
||||
<div class="stat-number">${Object.keys(relsByType).length}</div>
|
||||
<div class="stat-label">Relationship Types</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
log(`Node types: ${Object.entries(nodesByType).map(([k,v]) => `${k}:${v}`).join(', ')}`);
|
||||
log(`Relationship types: ${Object.entries(relsByType).map(([k,v]) => `${k}:${v}`).join(', ')}`);
|
||||
}
|
||||
|
||||
// Export validation functions (simplified versions for testing)
|
||||
window.validateGraph = function() {
|
||||
if (!window.testGraph) {
|
||||
log('❌ No test graph available. Create one first.');
|
||||
return;
|
||||
}
|
||||
|
||||
const graph = window.testGraph;
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
|
||||
// Check for duplicate node IDs
|
||||
const nodeIds = new Set(graph.nodes.map(n => n.id));
|
||||
if (nodeIds.size !== graph.nodes.length) {
|
||||
errors.push('Graph contains duplicate node IDs');
|
||||
}
|
||||
|
||||
// Check for invalid relationships
|
||||
graph.relationships.forEach((rel, index) => {
|
||||
if (!nodeIds.has(rel.source)) {
|
||||
errors.push(`Relationship ${index} has invalid source: ${rel.source}`);
|
||||
}
|
||||
if (!nodeIds.has(rel.target)) {
|
||||
errors.push(`Relationship ${index} has invalid target: ${rel.target}`);
|
||||
}
|
||||
});
|
||||
|
||||
// Check for empty nodes/relationships
|
||||
if (graph.nodes.length === 0) {
|
||||
warnings.push('Graph has no nodes');
|
||||
}
|
||||
if (graph.relationships.length === 0) {
|
||||
warnings.push('Graph has no relationships');
|
||||
}
|
||||
|
||||
const validationDiv = document.getElementById('validation-output');
|
||||
const isValid = errors.length === 0;
|
||||
|
||||
validationDiv.className = 'test-section ' + (isValid ? 'success' : 'error');
|
||||
validationDiv.innerHTML = `
|
||||
<h4>Validation Results</h4>
|
||||
<p><strong>Status:</strong> ${isValid ? '✅ Valid' : '❌ Invalid'}</p>
|
||||
${errors.length > 0 ? `<p><strong>Errors:</strong><br>${errors.join('<br>')}</p>` : ''}
|
||||
${warnings.length > 0 ? `<p><strong>Warnings:</strong><br>${warnings.join('<br>')}</p>` : ''}
|
||||
`;
|
||||
|
||||
log(isValid ? '✅ Graph validation passed' : '❌ Graph validation failed');
|
||||
errors.forEach(error => log(`❌ ${error}`));
|
||||
warnings.forEach(warning => log(`⚠️ ${warning}`));
|
||||
};
|
||||
|
||||
window.calculateSize = function() {
|
||||
if (!window.testGraph) {
|
||||
log('❌ No test graph available.');
|
||||
return;
|
||||
}
|
||||
|
||||
const graph = window.testGraph;
|
||||
const fileContents = window.testFileContents;
|
||||
|
||||
// Calculate sizes
|
||||
const graphOnly = JSON.stringify(graph);
|
||||
const withMetadata = JSON.stringify({
|
||||
metadata: {
|
||||
exportedAt: new Date().toISOString(),
|
||||
nodeCount: graph.nodes.length,
|
||||
relationshipCount: graph.relationships.length,
|
||||
fileCount: fileContents?.size || 0
|
||||
},
|
||||
graph
|
||||
});
|
||||
const withFiles = JSON.stringify({
|
||||
metadata: {
|
||||
exportedAt: new Date().toISOString(),
|
||||
nodeCount: graph.nodes.length,
|
||||
relationshipCount: graph.relationships.length,
|
||||
fileCount: fileContents?.size || 0
|
||||
},
|
||||
graph,
|
||||
fileContents: fileContents ? Object.fromEntries(fileContents) : {}
|
||||
});
|
||||
|
||||
function formatSize(bytes) {
|
||||
const sizes = ['Bytes', 'KB', 'MB'];
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
||||
return Math.round(bytes / Math.pow(1024, i) * 100) / 100 + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
log(`📏 Export sizes:`);
|
||||
log(` - Graph only: ${formatSize(graphOnly.length)}`);
|
||||
log(` - With metadata: ${formatSize(withMetadata.length)}`);
|
||||
log(` - With files: ${formatSize(withFiles.length)}`);
|
||||
};
|
||||
|
||||
// Export test functions
|
||||
window.testJSONExport = function() {
|
||||
if (!window.testGraph) {
|
||||
log('❌ No test graph available.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const graph = window.testGraph;
|
||||
const fileContents = window.testFileContents;
|
||||
|
||||
// Create export with metadata
|
||||
const exportData = {
|
||||
metadata: {
|
||||
exportedAt: new Date().toISOString(),
|
||||
version: '1.0.0',
|
||||
nodeCount: graph.nodes.length,
|
||||
relationshipCount: graph.relationships.length,
|
||||
fileCount: fileContents?.size || 0,
|
||||
processingDuration: 5000 // Mock duration
|
||||
},
|
||||
graph,
|
||||
fileContents: fileContents ? Object.fromEntries(fileContents) : {}
|
||||
};
|
||||
|
||||
const jsonContent = JSON.stringify(exportData, null, 2);
|
||||
|
||||
// Create download link
|
||||
const blob = new Blob([jsonContent], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const filename = `gitnexus-test-graph_${new Date().toISOString().split('T')[0]}.json`;
|
||||
|
||||
createDownloadLink(url, filename, 'JSON Export', '📄');
|
||||
|
||||
log('✅ JSON export test successful');
|
||||
log(`📁 Filename: ${filename}`);
|
||||
log(`📊 Export size: ${Math.round(blob.size / 1024)} KB`);
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ JSON export failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
window.testCSVExport = function() {
|
||||
if (!window.testGraph) {
|
||||
log('❌ No test graph available.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const graph = window.testGraph;
|
||||
|
||||
// Generate nodes CSV
|
||||
const nodeHeaders = ':ID,name,filePath,startLine,endLine,type,language,qualifiedName,:LABEL';
|
||||
const nodeRows = graph.nodes.map(node => {
|
||||
const props = node.properties;
|
||||
return [
|
||||
escapeCSV(node.id),
|
||||
escapeCSV(props.name || ''),
|
||||
escapeCSV(props.filePath || ''),
|
||||
props.startLine || '',
|
||||
props.endLine || '',
|
||||
escapeCSV(props.type || ''),
|
||||
escapeCSV(props.language || ''),
|
||||
escapeCSV(props.qualifiedName || ''),
|
||||
node.label
|
||||
].join(',');
|
||||
});
|
||||
const nodesCSV = [nodeHeaders, ...nodeRows].join('\n');
|
||||
|
||||
// Generate relationships CSV
|
||||
const relHeaders = ':START_ID,:TYPE,:END_ID,source,target';
|
||||
const relRows = graph.relationships.map(rel => {
|
||||
return [
|
||||
escapeCSV(rel.source),
|
||||
rel.type,
|
||||
escapeCSV(rel.target),
|
||||
escapeCSV(rel.source),
|
||||
escapeCSV(rel.target)
|
||||
].join(',');
|
||||
});
|
||||
const relsCSV = [relHeaders, ...relRows].join('\n');
|
||||
|
||||
// Create download links
|
||||
const timestamp = new Date().toISOString().split('T')[0];
|
||||
|
||||
const nodesBlob = new Blob([nodesCSV], { type: 'text/csv' });
|
||||
const nodesUrl = URL.createObjectURL(nodesBlob);
|
||||
createDownloadLink(nodesUrl, `gitnexus-test-graph_${timestamp}_nodes.csv`, 'Nodes CSV', '📊');
|
||||
|
||||
const relsBlob = new Blob([relsCSV], { type: 'text/csv' });
|
||||
const relsUrl = URL.createObjectURL(relsBlob);
|
||||
createDownloadLink(relsUrl, `gitnexus-test-graph_${timestamp}_relationships.csv`, 'Relationships CSV', '🔗');
|
||||
|
||||
log('✅ CSV export test successful');
|
||||
log(`📊 Nodes CSV: ${Math.round(nodesBlob.size / 1024)} KB`);
|
||||
log(`🔗 Relationships CSV: ${Math.round(relsBlob.size / 1024)} KB`);
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ CSV export failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
window.testFilteredExport = function() {
|
||||
if (!window.testGraph) {
|
||||
log('❌ No test graph available.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const graph = window.testGraph;
|
||||
|
||||
// Create filtered export (only TypeScript files and classes)
|
||||
const filteredNodes = graph.nodes.filter(node =>
|
||||
node.label === 'Class' ||
|
||||
node.label === 'Method' ||
|
||||
(node.properties.language === 'typescript')
|
||||
);
|
||||
|
||||
const filteredNodeIds = new Set(filteredNodes.map(n => n.id));
|
||||
const filteredRels = graph.relationships.filter(rel =>
|
||||
filteredNodeIds.has(rel.source) && filteredNodeIds.has(rel.target)
|
||||
);
|
||||
|
||||
const filteredGraph = {
|
||||
nodes: filteredNodes,
|
||||
relationships: filteredRels
|
||||
};
|
||||
|
||||
const exportData = {
|
||||
metadata: {
|
||||
exportedAt: new Date().toISOString(),
|
||||
version: '1.0.0',
|
||||
nodeCount: filteredGraph.nodes.length,
|
||||
relationshipCount: filteredGraph.relationships.length,
|
||||
filters: ['TypeScript files', 'Classes', 'Methods']
|
||||
},
|
||||
graph: filteredGraph
|
||||
};
|
||||
|
||||
const jsonContent = JSON.stringify(exportData, null, 2);
|
||||
const blob = new Blob([jsonContent], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const filename = `gitnexus-filtered-test_${new Date().toISOString().split('T')[0]}.json`;
|
||||
|
||||
createDownloadLink(url, filename, 'Filtered Export', '🔍');
|
||||
|
||||
log('✅ Filtered export test successful');
|
||||
log(`📊 Original: ${graph.nodes.length} nodes, ${graph.relationships.length} relationships`);
|
||||
log(`🔍 Filtered: ${filteredGraph.nodes.length} nodes, ${filteredGraph.relationships.length} relationships`);
|
||||
|
||||
} catch (error) {
|
||||
log(`❌ Filtered export failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
window.testImport = function() {
|
||||
// For this test, we'll simulate importing the JSON we just exported
|
||||
if (!window.testGraph) {
|
||||
log('❌ No test graph available. Export one first.');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const originalGraph = window.testGraph;
|
||||
const fileContents = window.testFileContents;
|
||||
|
||||
// Create mock export data
|
||||
const exportData = {
|
||||
metadata: {
|
||||
exportedAt: new Date().toISOString(),
|
||||
version: '1.0.0',
|
||||
nodeCount: originalGraph.nodes.length,
|
||||
relationshipCount: originalGraph.relationships.length,
|
||||
fileCount: fileContents?.size || 0
|
||||
},
|
||||
graph: originalGraph,
|
||||
fileContents: fileContents ? Object.fromEntries(fileContents) : {}
|
||||
};
|
||||
|
||||
// Simulate import process
|
||||
const jsonString = JSON.stringify(exportData);
|
||||
const parsed = JSON.parse(jsonString);
|
||||
|
||||
// Verify import
|
||||
const importedGraph = parsed.graph;
|
||||
const importedMetadata = parsed.metadata;
|
||||
const importedFiles = parsed.fileContents ? new Map(Object.entries(parsed.fileContents)) : null;
|
||||
|
||||
const importDiv = document.getElementById('import-output');
|
||||
importDiv.className = 'test-section success';
|
||||
importDiv.innerHTML = `
|
||||
<h4>✅ Import Test Results</h4>
|
||||
<p><strong>Metadata:</strong> ${importedMetadata ? '✅ Present' : '❌ Missing'}</p>
|
||||
<p><strong>Graph:</strong> ${importedGraph ? '✅ Present' : '❌ Missing'}</p>
|
||||
<p><strong>File Contents:</strong> ${importedFiles ? '✅ Present' : '❌ Missing'}</p>
|
||||
<p><strong>Nodes:</strong> ${importedGraph.nodes.length} (${originalGraph.nodes.length === importedGraph.nodes.length ? '✅ Match' : '❌ Mismatch'})</p>
|
||||
<p><strong>Relationships:</strong> ${importedGraph.relationships.length} (${originalGraph.relationships.length === importedGraph.relationships.length ? '✅ Match' : '❌ Mismatch'})</p>
|
||||
<p><strong>Files:</strong> ${importedFiles?.size || 0} (${fileContents?.size === importedFiles?.size ? '✅ Match' : '❌ Mismatch'})</p>
|
||||
`;
|
||||
|
||||
log('✅ Import test successful');
|
||||
log(`📊 Imported ${importedGraph.nodes.length} nodes, ${importedGraph.relationships.length} relationships`);
|
||||
log(`📁 Imported ${importedFiles?.size || 0} files`);
|
||||
|
||||
} catch (error) {
|
||||
const importDiv = document.getElementById('import-output');
|
||||
importDiv.className = 'test-section error';
|
||||
importDiv.innerHTML = `<h4>❌ Import Test Failed</h4><p>${error.message}</p>`;
|
||||
log(`❌ Import test failed: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
// Helper functions
|
||||
function createDownloadLink(url, filename, label, icon) {
|
||||
const linksDiv = document.getElementById('export-links');
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = filename;
|
||||
link.className = 'download-link';
|
||||
link.innerHTML = `${icon} Download ${label}`;
|
||||
linksDiv.appendChild(link);
|
||||
}
|
||||
|
||||
function escapeCSV(value) {
|
||||
if (!value) return '';
|
||||
if (value.includes(',') || value.includes('"') || value.includes('\n')) {
|
||||
return `"${value.replace(/"/g, '""')}"`;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function log(message) {
|
||||
const output = document.getElementById('output');
|
||||
const timestamp = new Date().toLocaleTimeString();
|
||||
output.textContent += `[${timestamp}] ${message}\n`;
|
||||
output.scrollTop = output.scrollHeight;
|
||||
console.log(message);
|
||||
}
|
||||
|
||||
// Initialize
|
||||
log('🚀 Export functionality test ready');
|
||||
log('👆 Click "Create Test Graph" to begin testing');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Add table
Reference in a new issue