From bb6de7b1ceb2a4b323d7f5c3b20fafee3a523db2 Mon Sep 17 00:00:00 2001 From: Dinh Huy Date: Sat, 30 May 2026 12:56:26 +0700 Subject: [PATCH] [~] Fix bugs and abstraction violation --- .../src/core/ingestion/language-provider.ts | 16 +++++++ gitnexus/src/core/ingestion/languages/php.ts | 2 + .../src/core/ingestion/languages/python.ts | 9 ++++ .../src/core/ingestion/parsing-processor.ts | 42 +++++++++---------- .../route-extractors/django-root-discovery.ts | 8 ++-- .../core/ingestion/route-extractors/django.ts | 29 +++++++++---- .../core/ingestion/workers/parse-worker.ts | 38 +++++------------ 7 files changed, 84 insertions(+), 60 deletions(-) diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index 058710b2b..e5d6fc96d 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -43,6 +43,8 @@ import type { ImportResolverFn } from './import-resolvers/types.js'; import type { NamedBindingExtractorFn } from './named-bindings/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; import type { NodeLabel } from 'gitnexus-shared'; +import type { ExtractedRoute } from './route-extractors/laravel.js'; +import type Parser from 'tree-sitter'; // ── Shared type aliases ──────────────────────────────────────────────────── /** Tree-sitter query captures: capture name → AST node (or undefined if not captured). */ @@ -301,6 +303,20 @@ interface LanguageProviderConfig { * When true, the worker extracts routes via the language's route extraction logic. * Default: undefined (no route files). */ readonly isRouteFile?: (filePath: string) => boolean; + /** Discover the root route file (e.g. Django root urls.py). + * If not provided, we extract from all route files matching `isRouteFile`. */ + readonly discoverRootRouteFile?: ( + files: Array<{ path: string; content: string }>, + contentMap?: Map, + ) => string | null; + /** Extract routes from a framework route file. + * Default: undefined (no route extraction). */ + readonly extractRoutes?: ( + tree: Parser.Tree, + filePath: string, + reader: (relativePath: string) => string | null, + parser?: Parser | null, + ) => ExtractedRoute[]; // ── Call-resolution DAG hooks ───────────────────────────────────── /** diff --git a/gitnexus/src/core/ingestion/languages/php.ts b/gitnexus/src/core/ingestion/languages/php.ts index caca85335..f0dad7f1f 100644 --- a/gitnexus/src/core/ingestion/languages/php.ts +++ b/gitnexus/src/core/ingestion/languages/php.ts @@ -38,6 +38,7 @@ import { phpVariableConfig } from '../variable-extractors/configs/php.js'; import { createCallExtractor } from '../call-extractors/generic.js'; import { phpCallConfig } from '../call-extractors/configs/php.js'; import { createHeritageExtractor } from '../heritage-extractors/generic.js'; +import { extractLaravelRoutes } from '../route-extractors/laravel.js'; const BUILT_INS: ReadonlySet = new Set([ 'echo', @@ -298,6 +299,7 @@ export const phpProvider = defineLanguage({ heritageExtractor: createHeritageExtractor(SupportedLanguages.PHP), descriptionExtractor: phpDescriptionExtractor, isRouteFile: isPhpRouteFile, + extractRoutes: (tree, filePath) => extractLaravelRoutes(tree, filePath), builtInNames: BUILT_INS, // ── RFC #909 Ring 3: scope-based resolution hooks ────────────────────── emitScopeCaptures: emitPhpScopeCaptures, diff --git a/gitnexus/src/core/ingestion/languages/python.ts b/gitnexus/src/core/ingestion/languages/python.ts index 35be5bf96..347f09e8c 100644 --- a/gitnexus/src/core/ingestion/languages/python.ts +++ b/gitnexus/src/core/ingestion/languages/python.ts @@ -45,6 +45,8 @@ import { pythonReceiverBinding, resolvePythonImportTarget, } from './python/index.js'; +import { extractDjangoRoutes, setDjangoParser } from '../route-extractors/django.js'; +import { discoverDjangoRootUrl } from '../route-extractors/django-root-discovery.js'; const BUILT_INS: ReadonlySet = new Set([ 'print', @@ -148,6 +150,13 @@ export const pythonProvider = defineLanguage({ descriptionExtractor: pythonDescriptionExtractor, builtInNames: BUILT_INS, isRouteFile: isDjangoRouteFile, + discoverRootRouteFile: (files, contentMap) => discoverDjangoRootUrl(files, contentMap), + extractRoutes: (tree, filePath, reader, parser) => { + if (parser) { + setDjangoParser(parser); + } + return extractDjangoRoutes(tree, filePath, reader); + }, labelOverride: pythonFunctionDefinitionLabel, // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index d0d2ab378..21351199c 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -60,13 +60,6 @@ import { getTreeSitterContentByteLength, TREE_SITTER_MAX_BUFFER, } from './constants.js'; -import { - extractDjangoRoutes, - setDjangoParser, - type DjangoFileReader, -} from './route-extractors/django.js'; -import { discoverDjangoRootUrl } from './route-extractors/django-root-discovery.js'; -import { extractLaravelRoutes } from './route-extractors/laravel.js'; import fs from 'node:fs'; import path from 'node:path'; @@ -385,10 +378,22 @@ const processParsingSequential = async ( const logSkipped = isVerboseIngestionEnabled(); const skippedByLang = logSkipped ? new Map() : null; - // Pre-compute file content map and discover Django root URL for route extraction + // Pre-compute file content map and discover root route files across all languages in this batch const fileContentMap = new Map(); for (const f of files) fileContentMap.set(f.path, f.content); - const djangoRootUrlFile = discoverDjangoRootUrl(files, fileContentMap); + + const rootRouteFiles = new Map(); + const languagesInBatch = new Set(); + for (const f of files) { + const lang = getLanguageFromFilename(f.path); + if (lang) languagesInBatch.add(lang); + } + for (const lang of languagesInBatch) { + const provider = getProvider(lang); + if (provider.discoverRootRouteFile) { + rootRouteFiles.set(lang, provider.discoverRootRouteFile(files, fileContentMap)); + } + } for (let i = 0; i < files.length; i++) { const file = files[i]; @@ -847,16 +852,14 @@ const processParsingSequential = async ( } }); - // ── Route extraction (Django / Laravel) ── + // ── Route extraction (Django / Laravel / generic) ── // Replicates the per-file route extraction from parse-worker.ts processFileGroup. const isRouteFile = provider.isRouteFile?.(file.path) ?? false; - if (isRouteFile) { - if (language === SupportedLanguages.Python) { - const isDjangoRoot = - djangoRootUrlFile !== null ? file.path === djangoRootUrlFile : isRouteFile; - if (!isDjangoRoot) continue; - setDjangoParser(parser); - const djangoReader: DjangoFileReader = (relativePath: string) => { + if (isRouteFile && provider.extractRoutes) { + const rootRouteFile = rootRouteFiles.get(language) ?? null; + const isRootRoute = rootRouteFile !== null ? file.path === rootRouteFile : isRouteFile; + if (isRootRoute) { + const reader = (relativePath: string) => { const cached = fileContentMap.get(relativePath); if (cached != null) return cached; try { @@ -865,10 +868,7 @@ const processParsingSequential = async ( return null; } }; - const extractedRoutes = extractDjangoRoutes(tree, file.path, djangoReader); - for (const r of extractedRoutes) outRoutes?.push(r); - } else { - const extractedRoutes = extractLaravelRoutes(tree, file.path); + const extractedRoutes = provider.extractRoutes(tree, file.path, reader, parser); for (const r of extractedRoutes) outRoutes?.push(r); } } diff --git a/gitnexus/src/core/ingestion/route-extractors/django-root-discovery.ts b/gitnexus/src/core/ingestion/route-extractors/django-root-discovery.ts index 6612d74be..db2f437fd 100644 --- a/gitnexus/src/core/ingestion/route-extractors/django-root-discovery.ts +++ b/gitnexus/src/core/ingestion/route-extractors/django-root-discovery.ts @@ -1,5 +1,3 @@ -import path from 'node:path'; - /** * Given a `manage.py` file content, extract the Django settings module. * e.g. `os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cmrMngt.settings')` @@ -138,8 +136,10 @@ export function discoverDjangoRootUrl( const basePaths: string[] = []; if (baseModule.startsWith('.')) { const resolved = resolveRelativeImport(resolvedSettingsPath!, baseModule); - if (resolved) basePaths.push(`${resolved.replace(/\./g, '/')}.py`); - basePaths.push(`${resolved.replace(/\./g, '/')}/__init__.py`); + if (resolved) { + basePaths.push(`${resolved.replace(/\./g, '/')}.py`); + basePaths.push(`${resolved.replace(/\./g, '/')}/__init__.py`); + } } else { basePaths.push(`${baseModule.replace(/\./g, '/')}.py`); basePaths.push(`${baseModule.replace(/\./g, '/')}/__init__.py`); diff --git a/gitnexus/src/core/ingestion/route-extractors/django.ts b/gitnexus/src/core/ingestion/route-extractors/django.ts index 49c874ae9..c83e21478 100644 --- a/gitnexus/src/core/ingestion/route-extractors/django.ts +++ b/gitnexus/src/core/ingestion/route-extractors/django.ts @@ -11,6 +11,7 @@ interface WalkFrame { node: SyntaxNode; routeCtx: DjangoRouteContext; currentFilePath: string; + depth: number; } const DJANGO_ROUTE_FUNCTIONS = new Set(['path', 're_path', 'url']); @@ -254,18 +255,23 @@ export function extractDjangoRoutes( const walkStack: WalkFrame[] = []; for (const listNode of listNodes) { - walkStack.push({ node: listNode, routeCtx: { prefix: null }, currentFilePath: filePath }); + walkStack.push({ + node: listNode, + routeCtx: { prefix: null }, + currentFilePath: filePath, + depth: 0, + }); } while (walkStack.length > 0) { - const { node, routeCtx, currentFilePath } = walkStack.pop()!; + const { node, routeCtx, currentFilePath, depth } = walkStack.pop()!; if (node.type === 'list') { const children = node.children ?? []; for (let i = children.length - 1; i >= 0; i--) { const child = children[i]; if (child.type === '[' || child.type === ']' || child.type === ',') continue; - walkStack.push({ node: child, routeCtx, currentFilePath }); + walkStack.push({ node: child, routeCtx, currentFilePath, depth }); } continue; } @@ -276,7 +282,7 @@ export function extractDjangoRoutes( if (!funcName) { for (const child of node.children ?? []) { if (child.type === 'call' || child.type === 'list') { - walkStack.push({ node: child, routeCtx, currentFilePath }); + walkStack.push({ node: child, routeCtx, currentFilePath, depth }); } } continue; @@ -291,7 +297,7 @@ export function extractDjangoRoutes( if (child.type === 'call' && getCallFuncName(child) === DJANGO_INCLUDE_FUNCTION) { hasIncludeChild = true; const modulePath = getIncludeModulePath(child); - if (modulePath && readFile && _djangoParser) { + if (modulePath && readFile && _djangoParser && depth < MAX_INCLUDE_DEPTH) { const resolved = resolveIncludedFile(modulePath, currentFilePath, readFile); if (resolved && !routeSet.has(resolved.filePath)) { routeSet.add(resolved.filePath); @@ -308,6 +314,7 @@ export function extractDjangoRoutes( node: childList, routeCtx: { prefix: childPrefix }, currentFilePath: resolved.filePath, + depth: depth + 1, }); } } @@ -322,7 +329,12 @@ export function extractDjangoRoutes( continue; } - if (funcName === DJANGO_INCLUDE_FUNCTION && readFile && _djangoParser) { + if ( + funcName === DJANGO_INCLUDE_FUNCTION && + readFile && + _djangoParser && + depth < MAX_INCLUDE_DEPTH + ) { const modulePath = getIncludeModulePath(node); if (modulePath) { const resolved = resolveIncludedFile(modulePath, currentFilePath, readFile); @@ -340,6 +352,7 @@ export function extractDjangoRoutes( node: childList, routeCtx, currentFilePath: resolved.filePath, + depth: depth + 1, }); } } @@ -349,7 +362,7 @@ export function extractDjangoRoutes( for (const child of node.children ?? []) { if (child.type === 'call' || child.type === 'list') { - walkStack.push({ node: child, routeCtx, currentFilePath }); + walkStack.push({ node: child, routeCtx, currentFilePath, depth }); } } continue; @@ -358,7 +371,7 @@ export function extractDjangoRoutes( for (const child of node.children ?? []) { if (child.type === '(' || child.type === ')' || child.type === ',') continue; if (child.type === 'call' || child.type === 'list') { - walkStack.push({ node: child, routeCtx, currentFilePath }); + walkStack.push({ node: child, routeCtx, currentFilePath, depth }); } } } diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 1bfc8ee87..5c7e32bcf 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -87,13 +87,7 @@ import { extractTemplateArguments, templateArgumentsIdTag } from '../utils/templ import type { LanguageProvider } from '../language-provider.js'; import type { ParsedFile } from 'gitnexus-shared'; import { extractParsedFile } from '../scope-extractor-bridge.js'; -import { extractLaravelRoutes, type ExtractedRoute } from '../route-extractors/laravel.js'; -import { - extractDjangoRoutes, - setDjangoParser, - type DjangoFileReader, -} from '../route-extractors/django.js'; -import { discoverDjangoRootUrl } from '../route-extractors/django-root-discovery.js'; +import type { ExtractedRoute } from '../route-extractors/laravel.js'; import { logger } from '../../logger.js'; export type { ExtractedRoute } from '../route-extractors/laravel.js'; @@ -986,12 +980,11 @@ const processFileGroup = ( const fileContentMap = new Map(); for (const f of files) fileContentMap.set(f.path, f.content); - // For Python, discover the Django root URL file so we only extract - // routes from it (includes cover the rest), avoiding duplicate entries. - let djangoRootUrlFile: string | null = null; - if (language === SupportedLanguages.Python) { - djangoRootUrlFile = discoverDjangoRootUrl(files, fileContentMap); - } + // Discover root route file if the language provider has discovery logic + const provider = getProvider(language); + const rootRouteFile = provider.discoverRootRouteFile + ? provider.discoverRootRouteFile(files, fileContentMap) + : null; let query: Parser.Query; try { @@ -2005,16 +1998,10 @@ const processFileGroup = ( // Extract framework routes via provider detection (e.g., Laravel routes.php) const isRouteFile = provider.isRouteFile?.(file.path) ?? false; - if (isRouteFile) { - if (language === SupportedLanguages.Python) { - // Only extract from the discovered Django root URL file; includes - // resolve the rest. Fall back to isRouteFile if discovery fails. - const isDjangoRoot = - djangoRootUrlFile !== null ? file.path === djangoRootUrlFile : isRouteFile; - if (!isDjangoRoot) continue; - - setDjangoParser(parser); - const djangoReader: DjangoFileReader = (relativePath: string) => { + if (isRouteFile && provider.extractRoutes) { + const isRootRoute = rootRouteFile !== null ? file.path === rootRouteFile : isRouteFile; + if (isRootRoute) { + const reader = (relativePath: string) => { const cached = fileContentMap.get(relativePath); if (cached != null) return cached; try { @@ -2023,10 +2010,7 @@ const processFileGroup = ( return null; } }; - const extractedRoutes = extractDjangoRoutes(tree, file.path, djangoReader); - for (const r of extractedRoutes) result.routes.push(r); - } else { - const extractedRoutes = extractLaravelRoutes(tree, file.path); + const extractedRoutes = provider.extractRoutes(tree, file.path, reader, parser); for (const r of extractedRoutes) result.routes.push(r); } }