mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-07 02:58:02 +00:00
[+] Add django route discovery to create cross-link for multi-repo
This commit is contained in:
parent
966ddb981e
commit
8469305966
6 changed files with 964 additions and 3 deletions
|
|
@ -11,6 +11,7 @@ import type { HttpDetection, HttpLanguagePlugin } from './types.js';
|
|||
/**
|
||||
* Python HTTP plugin. Handles:
|
||||
* - FastAPI `@app.get("/path")` provider decorators
|
||||
* - Django `path("route/", view)` provider calls
|
||||
* - `requests.get/post/...("url")` consumer calls
|
||||
* - Generic `requests.request("METHOD", "url")` consumer calls
|
||||
* - `httpx.AsyncClient` instances calling `.get/.post/...("url")`, including
|
||||
|
|
@ -48,6 +49,37 @@ const FASTAPI_PATTERNS = compilePatterns({
|
|||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Provider: Django path()/re_path()/url() ─────────────────────────
|
||||
const DJANGO_PATH_PATTERNS = compilePatterns({
|
||||
name: 'python-django-path',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call
|
||||
function: (identifier) @func (#match? @func "^(path|re_path)$")
|
||||
arguments: (argument_list . (string) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
const DJANGO_URL_PATTERNS = compilePatterns({
|
||||
name: 'python-django-url',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call
|
||||
function: (identifier) @func (#eq? @func "url")
|
||||
arguments: (argument_list . (string) @pattern . (identifier) @view))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: requests.get/post/... ──────────────────────────────────
|
||||
const REQUESTS_VERB_PATTERNS = compilePatterns({
|
||||
name: 'python-requests-verb',
|
||||
|
|
@ -473,6 +505,36 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
|
|||
});
|
||||
}
|
||||
|
||||
// Providers: Django path()/re_path()/url()
|
||||
for (const match of runCompiledPatterns(DJANGO_PATH_PATTERNS, tree)) {
|
||||
const pathNode = match.captures.path;
|
||||
if (!pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'django',
|
||||
method: '*',
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
for (const match of runCompiledPatterns(DJANGO_URL_PATTERNS, tree)) {
|
||||
const patternNode = match.captures.pattern;
|
||||
if (!patternNode) continue;
|
||||
const path = unquoteLiteral(patternNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'django',
|
||||
method: '*',
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumers: requests.<verb>
|
||||
for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.method;
|
||||
|
|
|
|||
|
|
@ -104,6 +104,16 @@ function normalizePythonStringLiteral(text: string): string | undefined {
|
|||
return raw.replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
/** Detect Django URL config files by naming convention. */
|
||||
function isDjangoRouteFile(filePath: string): boolean {
|
||||
return (
|
||||
filePath.endsWith('.py') &&
|
||||
(filePath.endsWith('/urls.py') ||
|
||||
filePath.endsWith('/urls/__init__.py') ||
|
||||
filePath === 'urls.py')
|
||||
);
|
||||
}
|
||||
|
||||
export const pythonProvider = defineLanguage({
|
||||
id: SupportedLanguages.Python,
|
||||
extensions: ['.py'],
|
||||
|
|
@ -137,6 +147,7 @@ export const pythonProvider = defineLanguage({
|
|||
heritageExtractor: createHeritageExtractor(SupportedLanguages.Python),
|
||||
descriptionExtractor: pythonDescriptionExtractor,
|
||||
builtInNames: BUILT_INS,
|
||||
isRouteFile: isDjangoRouteFile,
|
||||
labelOverride: pythonFunctionDefinitionLabel,
|
||||
|
||||
// ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ──────────
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
import path from 'node:path';
|
||||
|
||||
/**
|
||||
* Given a `manage.py` file content, extract the Django settings module.
|
||||
* e.g. `os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cmrMngt.settings')`
|
||||
* returns `'cmrMngt.settings'`
|
||||
*/
|
||||
function extractDjangoSettingsModule(manageContent: string): string | null {
|
||||
const m = manageContent.match(/DJANGO_SETTINGS_MODULE\s*['"]?[,= ]\s*['"]([^'"]+)['"]/);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a dotted Python module path, produce possible file paths.
|
||||
* e.g. `cmrMngt.settings` → `['cmrMngt/settings.py', 'cmrMngt/settings/__init__.py']`
|
||||
*/
|
||||
export function djangoModuleToFilePaths(modulePath: string): string[] {
|
||||
const base = modulePath.replace(/\./g, '/');
|
||||
return [`${base}.py`, `${base}/__init__.py`];
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a file, trying first from the given content map, then from disk.
|
||||
*/
|
||||
function tryReadFile(relativePath: string, contentMap: Map<string, string>): string | null {
|
||||
return contentMap.get(relativePath) ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract a module-level string assignment value from Python source.
|
||||
* e.g. `content` contains `ROOT_URLCONF = 'cmrMngt.urls'`
|
||||
* returns `'cmrMngt.urls'`
|
||||
*/
|
||||
function extractPythonStringAssignment(content: string, varName: string): string | null {
|
||||
const regex = new RegExp(`^${varName}\\s*=\\s*['"]([^'"]+)['"]`, 'm');
|
||||
const m = content.match(regex);
|
||||
return m ? m[1] : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract `from <module> import *` statements from Python source.
|
||||
* e.g. `from .settings_base import *` → `settings_base`
|
||||
* `from cmrMngt.settings_base import *` → `cmrMngt.settings_base`
|
||||
*/
|
||||
function extractStarImports(content: string): string[] {
|
||||
const modules: string[] = [];
|
||||
const regex = /^from\s+(\.?[\w.]+)\s+import\s+\*/gm;
|
||||
let m;
|
||||
while ((m = regex.exec(content)) !== null) {
|
||||
const moduleName = m[1];
|
||||
if (moduleName.startsWith('.')) {
|
||||
// Relative import — caller needs to resolve based on current module
|
||||
modules.push(moduleName);
|
||||
} else {
|
||||
modules.push(moduleName);
|
||||
}
|
||||
}
|
||||
return modules;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a relative Python import path.
|
||||
* `from .settings_base import *` in `cmrMngt/settings.py`
|
||||
* → `cmrMngt/settings_base.py`
|
||||
*/
|
||||
function resolveRelativeImport(currentModulePath: string, importPath: string): string | null {
|
||||
if (!importPath.startsWith('.')) return null;
|
||||
|
||||
const currentDir = currentModulePath.includes('/')
|
||||
? currentModulePath.substring(0, currentModulePath.lastIndexOf('/'))
|
||||
: '';
|
||||
|
||||
let relPath = importPath;
|
||||
let dir = currentDir;
|
||||
while (relPath.startsWith('.')) {
|
||||
if (relPath.startsWith('..')) {
|
||||
dir = dir.includes('/') ? dir.substring(0, dir.lastIndexOf('/')) : '';
|
||||
relPath = relPath.substring(2);
|
||||
} else {
|
||||
relPath = relPath.substring(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return dir ? `${dir}/${relPath}` : relPath;
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover the Django root URL file by following:
|
||||
* manage.py → DJANGO_SETTINGS_MODULE → settings → ROOT_URLCONF → urls.py
|
||||
*
|
||||
* @param files Array of file paths and their contents in the current batch.
|
||||
* @param contentMap Optional pre-built map of file path → content.
|
||||
* @returns The relative path to the root URL file, or null.
|
||||
*/
|
||||
export function discoverDjangoRootUrl(
|
||||
files: Array<{ path: string; content: string }>,
|
||||
contentMap?: Map<string, string>,
|
||||
): string | null {
|
||||
const map = contentMap ?? new Map<string, string>();
|
||||
for (const f of files) map.set(f.path, f.content);
|
||||
|
||||
const managePy = files.find((f) => f.path === 'manage.py' || f.path.endsWith('/manage.py'));
|
||||
if (!managePy) return null;
|
||||
|
||||
const settingsModule = extractDjangoSettingsModule(managePy.content);
|
||||
if (!settingsModule) return null;
|
||||
|
||||
// Find the settings file
|
||||
const settingsPaths = djangoModuleToFilePaths(settingsModule);
|
||||
let settingsContent: string | null = null;
|
||||
let resolvedSettingsPath: string | null = null;
|
||||
for (const sp of settingsPaths) {
|
||||
const c = tryReadFile(sp, map);
|
||||
if (c !== null) {
|
||||
settingsContent = c;
|
||||
resolvedSettingsPath = settingsModule.replace(/\./g, '/');
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!settingsContent) return null;
|
||||
|
||||
// Check ROOT_URLCONF in the main settings and any base settings (star imports)
|
||||
let rootUrlConf = extractPythonStringAssignment(settingsContent, 'ROOT_URLCONF');
|
||||
if (!rootUrlConf) {
|
||||
// Check star-imported base settings
|
||||
const starImports = extractStarImports(settingsContent);
|
||||
for (const imp of starImports) {
|
||||
let baseModule: string | null = null;
|
||||
if (imp.startsWith('.')) {
|
||||
const resolved = resolveRelativeImport(resolvedSettingsPath!, imp);
|
||||
if (resolved) baseModule = resolved;
|
||||
} else {
|
||||
baseModule = imp;
|
||||
}
|
||||
if (!baseModule) continue;
|
||||
|
||||
const basePaths: string[] = [];
|
||||
if (baseModule.startsWith('.')) {
|
||||
const resolved = resolveRelativeImport(resolvedSettingsPath!, baseModule);
|
||||
if (resolved) basePaths.push(`${resolved.replace(/\./g, '/')}.py`);
|
||||
basePaths.push(`${resolved.replace(/\./g, '/')}/__init__.py`);
|
||||
} else {
|
||||
basePaths.push(`${baseModule.replace(/\./g, '/')}.py`);
|
||||
basePaths.push(`${baseModule.replace(/\./g, '/')}/__init__.py`);
|
||||
}
|
||||
|
||||
for (const bp of basePaths) {
|
||||
const bc = tryReadFile(bp, map);
|
||||
if (bc) {
|
||||
rootUrlConf = extractPythonStringAssignment(bc, 'ROOT_URLCONF');
|
||||
if (rootUrlConf) break;
|
||||
}
|
||||
}
|
||||
if (rootUrlConf) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!rootUrlConf) return null;
|
||||
|
||||
// Convert ROOT_URLCONF module path to file path
|
||||
const urlPaths = djangoModuleToFilePaths(rootUrlConf);
|
||||
for (const up of urlPaths) {
|
||||
if (tryReadFile(up, map) !== null) return up;
|
||||
}
|
||||
|
||||
// Also try relative to the settings module's directory
|
||||
if (resolvedSettingsPath && resolvedSettingsPath.includes('/')) {
|
||||
const settingsDir = resolvedSettingsPath.substring(
|
||||
0,
|
||||
resolvedSettingsPath.lastIndexOf('/') + 1,
|
||||
);
|
||||
for (const up of urlPaths) {
|
||||
const tryPath = settingsDir + up;
|
||||
if (tryReadFile(tryPath, map) !== null) return tryPath;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
366
gitnexus/src/core/ingestion/route-extractors/django.ts
Normal file
366
gitnexus/src/core/ingestion/route-extractors/django.ts
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
import type Parser from 'tree-sitter';
|
||||
import { extractStringContent, type SyntaxNode } from '../utils/ast-helpers.js';
|
||||
import type { ExtractedRoute } from './laravel.js';
|
||||
|
||||
interface DjangoRouteContext {
|
||||
prefix: string | null;
|
||||
}
|
||||
|
||||
interface WalkFrame {
|
||||
node: SyntaxNode;
|
||||
routeCtx: DjangoRouteContext;
|
||||
currentFilePath: string;
|
||||
}
|
||||
|
||||
const DJANGO_ROUTE_FUNCTIONS = new Set(['path', 're_path', 'url']);
|
||||
const DJANGO_INCLUDE_FUNCTION = 'include';
|
||||
const MAX_INCLUDE_DEPTH = 8;
|
||||
|
||||
function modulePathToFilePath(modulePath: string): string {
|
||||
return modulePath.replace(/\./g, '/');
|
||||
}
|
||||
|
||||
export type DjangoFileReader = (relativePath: string) => string | null;
|
||||
|
||||
function extractStringArg(argsNode: SyntaxNode | null): string | null {
|
||||
if (!argsNode) return null;
|
||||
for (const child of argsNode.children ?? []) {
|
||||
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
|
||||
if (child.type === 'string') {
|
||||
return extractStringContent(child);
|
||||
}
|
||||
if (child.type === 'binary_operator') {
|
||||
let concat = '';
|
||||
for (const part of child.children ?? []) {
|
||||
if (part.type === 'string') {
|
||||
const s = extractStringContent(part);
|
||||
if (s !== null) concat += s;
|
||||
}
|
||||
}
|
||||
if (concat) return concat;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractViewTarget(argsNode: SyntaxNode | null): {
|
||||
viewName: string | null;
|
||||
viewCall: string | null;
|
||||
} {
|
||||
if (!argsNode) return { viewName: null, viewCall: null };
|
||||
const positionalArgs: SyntaxNode[] = [];
|
||||
for (const child of argsNode.children ?? []) {
|
||||
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
|
||||
positionalArgs.push(child);
|
||||
}
|
||||
const viewNode = positionalArgs[1];
|
||||
if (!viewNode) return { viewName: null, viewCall: null };
|
||||
if (viewNode.type === 'attribute') return { viewName: viewNode.text, viewCall: null };
|
||||
if (viewNode.type === 'call') return { viewName: null, viewCall: viewNode.text };
|
||||
if (viewNode.type === 'identifier') return { viewName: viewNode.text, viewCall: null };
|
||||
if (viewNode.type === 'string')
|
||||
return { viewName: extractStringContent(viewNode), viewCall: null };
|
||||
return { viewName: null, viewCall: null };
|
||||
}
|
||||
|
||||
function inferHttpMethod(viewName: string | null): string {
|
||||
if (!viewName) return '*';
|
||||
const lower = viewName.toLowerCase();
|
||||
const m = lower.match(/\.(get|post|put|patch|delete|head|options)(_|$)/);
|
||||
if (m) {
|
||||
return m[1].toUpperCase();
|
||||
}
|
||||
return '*';
|
||||
}
|
||||
|
||||
function findUrlpatternsLists(rootNode: SyntaxNode): SyntaxNode[] {
|
||||
const assignmentNodes: SyntaxNode[] = [];
|
||||
_collectAssignments(rootNode, assignmentNodes);
|
||||
const lists: SyntaxNode[] = [];
|
||||
for (const node of assignmentNodes) {
|
||||
const left = node.childForFieldName?.('left') ?? node.children?.[0] ?? null;
|
||||
if (left?.type === 'identifier' && left.text === 'urlpatterns') {
|
||||
const right = node.childForFieldName?.('right') ?? node.children?.[2] ?? null;
|
||||
if (right?.type === 'list') {
|
||||
lists.push(right);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lists;
|
||||
}
|
||||
|
||||
function _collectAssignments(node: SyntaxNode, out: SyntaxNode[]): void {
|
||||
if (node.type === 'assignment' || node.type === 'augmented_assignment') {
|
||||
out.push(node);
|
||||
}
|
||||
for (const child of node.children ?? []) {
|
||||
_collectAssignments(child, out);
|
||||
}
|
||||
}
|
||||
|
||||
function emitDjangoRoute(
|
||||
callNode: SyntaxNode,
|
||||
filePath: string,
|
||||
ctx: DjangoRouteContext,
|
||||
): ExtractedRoute {
|
||||
const argsNode = callNode.childForFieldName?.('arguments') ?? null;
|
||||
const routePath = extractStringArg(argsNode);
|
||||
|
||||
const { viewName, viewCall } = extractViewTarget(argsNode);
|
||||
const httpMethod = inferHttpMethod(viewName);
|
||||
|
||||
let routeName: string | null = null;
|
||||
if (argsNode) {
|
||||
for (let i = 0; i < argsNode.children.length; i++) {
|
||||
const child = argsNode.children[i];
|
||||
if (child.type === 'keyword_argument' && child.childForFieldName?.('name')?.text === 'name') {
|
||||
const valueNode = child.childForFieldName?.('value');
|
||||
if (valueNode?.type === 'string') {
|
||||
routeName = extractStringContent(valueNode);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
filePath,
|
||||
httpMethod,
|
||||
routePath,
|
||||
routeName,
|
||||
controllerName: viewName ?? viewCall,
|
||||
methodName: null,
|
||||
middleware: [],
|
||||
prefix: ctx.prefix,
|
||||
lineNumber: callNode.startPosition.row,
|
||||
};
|
||||
}
|
||||
|
||||
function getIncludeModulePath(callNode: SyntaxNode): string | null {
|
||||
const funcName =
|
||||
callNode.childForFieldName?.('function')?.text ??
|
||||
callNode.children?.find((c) => c.type === 'identifier')?.text;
|
||||
if (funcName !== DJANGO_INCLUDE_FUNCTION) return null;
|
||||
const argsNode = callNode.childForFieldName?.('arguments');
|
||||
if (!argsNode) return null;
|
||||
|
||||
const modulePath = extractStringArg(argsNode);
|
||||
if (modulePath) return modulePath;
|
||||
|
||||
for (const child of argsNode.children ?? []) {
|
||||
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
|
||||
if (child.type === 'tuple' || child.type === 'parenthesized_expression') {
|
||||
for (const inner of child.children ?? []) {
|
||||
if (inner.type === '(' || inner.type === ')' || inner.type === ',') continue;
|
||||
if (inner.type === 'string') return extractStringContent(inner);
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function makePrefix(parentPrefix: string | null, childPrefix: string | null): string | null {
|
||||
if (!childPrefix) return parentPrefix;
|
||||
if (!parentPrefix) return childPrefix;
|
||||
return `${parentPrefix}/${childPrefix}`.replace(/\/+/g, '/');
|
||||
}
|
||||
|
||||
function getCallFuncName(node: SyntaxNode): string | null {
|
||||
return (
|
||||
node.childForFieldName?.('function')?.text ??
|
||||
node.children?.find((c) => c.type === 'identifier')?.text ??
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
let _djangoParser: Parser | null = null;
|
||||
|
||||
export function setDjangoParser(p: Parser): void {
|
||||
_djangoParser = p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a Django dotted module path like `app.submodule.urls`,
|
||||
* try multiple path resolution strategies to find the file on disk.
|
||||
*
|
||||
* Strategies tried in order:
|
||||
* 1. Direct dot-to-slash: `module/path.py` and `module/path/__init__.py`
|
||||
* 2. Relative to the current file's directory
|
||||
* 3. Walk up the directory tree from the current file, trying each ancestor
|
||||
*/
|
||||
function resolveIncludedFile(
|
||||
modulePath: string,
|
||||
currentFilePath: string,
|
||||
readFile: DjangoFileReader,
|
||||
): { filePath: string; content: string } | null {
|
||||
const basePath = modulePathToFilePath(modulePath);
|
||||
|
||||
const candidates: string[] = [];
|
||||
|
||||
// Strategy 1: direct path (app/urls.py, app/urls/__init__.py)
|
||||
candidates.push(basePath + '.py');
|
||||
candidates.push(basePath + '/__init__.py');
|
||||
|
||||
// Strategy 2: relative to current file's directory
|
||||
if (currentFilePath.includes('/')) {
|
||||
const dir = currentFilePath.substring(0, currentFilePath.lastIndexOf('/') + 1);
|
||||
candidates.push(dir + basePath + '.py');
|
||||
candidates.push(dir + basePath + '/__init__.py');
|
||||
}
|
||||
|
||||
// Strategy 3: walk up from current file, trying each ancestor
|
||||
let parentDir = currentFilePath.includes('/')
|
||||
? currentFilePath.substring(0, currentFilePath.lastIndexOf('/'))
|
||||
: '';
|
||||
while (parentDir.length > 0) {
|
||||
const prefix = parentDir + '/';
|
||||
candidates.push(prefix + basePath + '.py');
|
||||
candidates.push(prefix + basePath + '/__init__.py');
|
||||
const nextSep = parentDir.lastIndexOf('/');
|
||||
if (nextSep < 0) break;
|
||||
parentDir = parentDir.substring(0, nextSep);
|
||||
}
|
||||
|
||||
// Strategy 4: bare path with just the last segment (e.g. 'urls.py' from 'app.urls')
|
||||
const segments = basePath.split('/');
|
||||
if (segments.length > 1) {
|
||||
const lastSegment = segments[segments.length - 1];
|
||||
candidates.push(lastSegment + '.py');
|
||||
candidates.push(lastSegment + '/__init__.py');
|
||||
}
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const content = readFile(candidate);
|
||||
if (content !== null) return { filePath: candidate, content };
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function extractDjangoRoutes(
|
||||
tree: Parser.Tree,
|
||||
filePath: string,
|
||||
readFile?: DjangoFileReader | null,
|
||||
_visited?: Set<string>,
|
||||
): ExtractedRoute[] {
|
||||
const routeSet = _visited ?? new Set<string>();
|
||||
if (routeSet.has(filePath)) return [];
|
||||
routeSet.add(filePath);
|
||||
|
||||
const listNodes = findUrlpatternsLists(tree.rootNode);
|
||||
if (listNodes.length === 0) return [];
|
||||
|
||||
const routes: ExtractedRoute[] = [];
|
||||
const walkStack: WalkFrame[] = [];
|
||||
|
||||
for (const listNode of listNodes) {
|
||||
walkStack.push({ node: listNode, routeCtx: { prefix: null }, currentFilePath: filePath });
|
||||
}
|
||||
|
||||
while (walkStack.length > 0) {
|
||||
const { node, routeCtx, currentFilePath } = walkStack.pop()!;
|
||||
|
||||
if (node.type === 'list') {
|
||||
const children = node.children ?? [];
|
||||
for (let i = children.length - 1; i >= 0; i--) {
|
||||
const child = children[i];
|
||||
if (child.type === '[' || child.type === ']' || child.type === ',') continue;
|
||||
walkStack.push({ node: child, routeCtx, currentFilePath });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (node.type === 'call') {
|
||||
const funcName = getCallFuncName(node);
|
||||
|
||||
if (!funcName) {
|
||||
for (const child of node.children ?? []) {
|
||||
if (child.type === 'call' || child.type === 'list') {
|
||||
walkStack.push({ node: child, routeCtx, currentFilePath });
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (DJANGO_ROUTE_FUNCTIONS.has(funcName)) {
|
||||
const argsNode = node.childForFieldName?.('arguments') ?? null;
|
||||
|
||||
let hasIncludeChild = false;
|
||||
if (argsNode) {
|
||||
for (const child of argsNode.children ?? []) {
|
||||
if (child.type === 'call' && getCallFuncName(child) === DJANGO_INCLUDE_FUNCTION) {
|
||||
hasIncludeChild = true;
|
||||
const modulePath = getIncludeModulePath(child);
|
||||
if (modulePath && readFile && _djangoParser) {
|
||||
const resolved = resolveIncludedFile(modulePath, currentFilePath, readFile);
|
||||
if (resolved && !routeSet.has(resolved.filePath)) {
|
||||
routeSet.add(resolved.filePath);
|
||||
let childTree: Parser.Tree;
|
||||
try {
|
||||
childTree = parseSourceSafe(_djangoParser, resolved.content);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const childLists = findUrlpatternsLists(childTree.rootNode);
|
||||
for (const childList of childLists) {
|
||||
const childPrefix = makePrefix(routeCtx.prefix, extractStringArg(argsNode));
|
||||
walkStack.push({
|
||||
node: childList,
|
||||
routeCtx: { prefix: childPrefix },
|
||||
currentFilePath: resolved.filePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!hasIncludeChild) {
|
||||
routes.push(emitDjangoRoute(node, currentFilePath, routeCtx));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (funcName === DJANGO_INCLUDE_FUNCTION && readFile && _djangoParser) {
|
||||
const modulePath = getIncludeModulePath(node);
|
||||
if (modulePath) {
|
||||
const resolved = resolveIncludedFile(modulePath, currentFilePath, readFile);
|
||||
if (resolved && !routeSet.has(resolved.filePath)) {
|
||||
routeSet.add(resolved.filePath);
|
||||
let childTree: Parser.Tree;
|
||||
try {
|
||||
childTree = parseSourceSafe(_djangoParser, resolved.content);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const childLists = findUrlpatternsLists(childTree.rootNode);
|
||||
for (const childList of childLists) {
|
||||
walkStack.push({
|
||||
node: childList,
|
||||
routeCtx,
|
||||
currentFilePath: resolved.filePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const child of node.children ?? []) {
|
||||
if (child.type === 'call' || child.type === 'list') {
|
||||
walkStack.push({ node: child, routeCtx, currentFilePath });
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const child of node.children ?? []) {
|
||||
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
|
||||
if (child.type === 'call' || child.type === 'list') {
|
||||
walkStack.push({ node: child, routeCtx, currentFilePath });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return routes;
|
||||
}
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
import { parentPort } from 'node:worker_threads';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import Parser from 'tree-sitter';
|
||||
import JavaScript from 'tree-sitter-javascript';
|
||||
import TypeScript from 'tree-sitter-typescript';
|
||||
|
|
@ -86,6 +88,12 @@ import type { LanguageProvider } from '../language-provider.js';
|
|||
import type { ParsedFile } from 'gitnexus-shared';
|
||||
import { extractParsedFile } from '../scope-extractor-bridge.js';
|
||||
import { extractLaravelRoutes, type ExtractedRoute } from '../route-extractors/laravel.js';
|
||||
import {
|
||||
extractDjangoRoutes,
|
||||
setDjangoParser,
|
||||
type DjangoFileReader,
|
||||
} from '../route-extractors/django.js';
|
||||
import { discoverDjangoRootUrl } from '../route-extractors/django-root-discovery.js';
|
||||
|
||||
import { logger } from '../../logger.js';
|
||||
export type { ExtractedRoute } from '../route-extractors/laravel.js';
|
||||
|
|
@ -951,6 +959,16 @@ const processFileGroup = (
|
|||
result: ParseWorkerResult,
|
||||
onFileProcessed?: () => void,
|
||||
): void => {
|
||||
const fileContentMap = new Map<string, string>();
|
||||
for (const f of files) fileContentMap.set(f.path, f.content);
|
||||
|
||||
// For Python, discover the Django root URL file so we only extract
|
||||
// routes from it (includes cover the rest), avoiding duplicate entries.
|
||||
let djangoRootUrlFile: string | null = null;
|
||||
if (language === SupportedLanguages.Python) {
|
||||
djangoRootUrlFile = discoverDjangoRootUrl(files, fileContentMap);
|
||||
}
|
||||
|
||||
let query: Parser.Query;
|
||||
try {
|
||||
const lang = parser.getLanguage();
|
||||
|
|
@ -1947,9 +1965,31 @@ const processFileGroup = (
|
|||
}
|
||||
|
||||
// Extract framework routes via provider detection (e.g., Laravel routes.php)
|
||||
if (provider.isRouteFile?.(file.path)) {
|
||||
const extractedRoutes = extractLaravelRoutes(tree, file.path);
|
||||
for (const r of extractedRoutes) result.routes.push(r);
|
||||
const isRouteFile = provider.isRouteFile?.(file.path) ?? false;
|
||||
if (isRouteFile) {
|
||||
if (language === SupportedLanguages.Python) {
|
||||
// Only extract from the discovered Django root URL file; includes
|
||||
// resolve the rest. Fall back to isRouteFile if discovery fails.
|
||||
const isDjangoRoot =
|
||||
djangoRootUrlFile !== null ? file.path === djangoRootUrlFile : isRouteFile;
|
||||
if (!isDjangoRoot) continue;
|
||||
|
||||
setDjangoParser(parser);
|
||||
const djangoReader: DjangoFileReader = (relativePath: string) => {
|
||||
const cached = fileContentMap.get(relativePath);
|
||||
if (cached != null) return cached;
|
||||
try {
|
||||
return fs.readFileSync(path.join(process.cwd(), relativePath), 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
const extractedRoutes = extractDjangoRoutes(tree, file.path, djangoReader);
|
||||
for (const r of extractedRoutes) result.routes.push(r);
|
||||
} else {
|
||||
const extractedRoutes = extractLaravelRoutes(tree, file.path);
|
||||
for (const r of extractedRoutes) result.routes.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
// Extract ORM queries (Prisma, Supabase)
|
||||
|
|
|
|||
302
gitnexus/test/unit/django-route-extraction.test.ts
Normal file
302
gitnexus/test/unit/django-route-extraction.test.ts
Normal file
|
|
@ -0,0 +1,302 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import Parser from 'tree-sitter';
|
||||
import Python from 'tree-sitter-python';
|
||||
import {
|
||||
extractDjangoRoutes,
|
||||
setDjangoParser,
|
||||
} from '../../src/core/ingestion/route-extractors/django.js';
|
||||
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(Python);
|
||||
|
||||
// Must be called once before extractDjangoRoutes can resolve includes
|
||||
setDjangoParser(parser);
|
||||
|
||||
const extract = (
|
||||
source: string,
|
||||
filePath = 'app/urls.py',
|
||||
readFile?: (path: string) => string | null,
|
||||
) =>
|
||||
extractDjangoRoutes(parser.parse(source), filePath, readFile).map((route) => ({
|
||||
httpMethod: route.httpMethod,
|
||||
routePath: route.routePath,
|
||||
routeName: route.routeName,
|
||||
controllerName: route.controllerName,
|
||||
prefix: route.prefix,
|
||||
filePath: route.filePath,
|
||||
}));
|
||||
|
||||
describe('Django route extraction', () => {
|
||||
it('extracts path() routes from urlpatterns', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('orders/', views.order_list),
|
||||
path('orders/<int:pk>/', views.order_detail),
|
||||
path('users/', views.user_list, name='user-list'),
|
||||
]
|
||||
`);
|
||||
expect(routes).toHaveLength(3);
|
||||
|
||||
expect(routes[0]).toMatchObject({ httpMethod: '*', routePath: 'orders/' });
|
||||
expect(routes[1]).toMatchObject({ httpMethod: '*', routePath: 'orders/<int:pk>/' });
|
||||
expect(routes[2]).toMatchObject({
|
||||
httpMethod: '*',
|
||||
routePath: 'users/',
|
||||
routeName: 'user-list',
|
||||
});
|
||||
});
|
||||
|
||||
it('extracts re_path() routes', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import re_path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
re_path(r'^articles/(?P<year>[0-9]{4})/$', views.year_archive),
|
||||
]
|
||||
`);
|
||||
expect(routes).toHaveLength(1);
|
||||
expect(routes[0]).toMatchObject({
|
||||
httpMethod: '*',
|
||||
routePath: '^articles/(?P<year>[0-9]{4})/$',
|
||||
});
|
||||
});
|
||||
|
||||
it('extracts legacy url() routes', () => {
|
||||
const routes = extract(`
|
||||
from django.conf.urls import url
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
url(r'^legacy/$', views.legacy_view),
|
||||
]
|
||||
`);
|
||||
expect(routes).toHaveLength(1);
|
||||
expect(routes[0]).toMatchObject({ httpMethod: '*', routePath: '^legacy/$' });
|
||||
});
|
||||
|
||||
it('handles str concatenation in path strings', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('api/' + 'v1/users/', views.user_list),
|
||||
]
|
||||
`);
|
||||
// Binary operator concatenation should produce the full path
|
||||
expect(routes).toHaveLength(1);
|
||||
if (routes.length > 0) {
|
||||
expect(routes[0].routePath).toContain('api/');
|
||||
expect(routes[0].routePath).toContain('v1/users/');
|
||||
}
|
||||
});
|
||||
|
||||
it('extracts from augmented assignment (urlpatterns += ...)', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('base/', views.base),
|
||||
]
|
||||
urlpatterns += [
|
||||
path('extra/', views.extra),
|
||||
]
|
||||
`);
|
||||
// Should find at least 'extra/' from the augmented assignment
|
||||
expect(routes.some((r) => r.routePath === 'extra/')).toBe(true);
|
||||
});
|
||||
|
||||
it('resolves include() to child url files via readFile', () => {
|
||||
const childContent = `
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('list/', views.item_list),
|
||||
path('<int:pk>/', views.item_detail),
|
||||
]
|
||||
`;
|
||||
const readFile = (path: string) => {
|
||||
if (path === 'items/urls.py' || path === 'app/items/urls.py') return childContent;
|
||||
return null;
|
||||
};
|
||||
|
||||
const routes = extract(
|
||||
`
|
||||
from django.urls import path, include
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('api/', include('items.urls')),
|
||||
path('health/', views.health),
|
||||
]
|
||||
`,
|
||||
'app/urls.py',
|
||||
readFile,
|
||||
);
|
||||
|
||||
// Should have: health/ and two routes from items/urls.py with prefix 'api/'
|
||||
const healthRoute = routes.find((r) => r.routePath === 'health/');
|
||||
expect(healthRoute).toBeDefined();
|
||||
expect(healthRoute?.filePath).toBe('app/urls.py');
|
||||
|
||||
const prefixedRoutes = routes.filter((r) => r.prefix === 'api/');
|
||||
expect(prefixedRoutes).toHaveLength(2);
|
||||
expect(prefixedRoutes.some((r) => r.routePath === 'list/')).toBe(true);
|
||||
expect(prefixedRoutes.some((r) => r.routePath === '<int:pk>/')).toBe(true);
|
||||
expect(prefixedRoutes.every((r) => r.filePath === 'items/urls.py')).toBe(true);
|
||||
});
|
||||
|
||||
it('resolves nested includes with accumulated prefixes', () => {
|
||||
const childContent = `
|
||||
from django.urls import path, include
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('v1/', include('v1.urls')),
|
||||
path('v2/', include('v2.urls')),
|
||||
]
|
||||
`;
|
||||
const grandchildContent = `
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('users/', views.user_list),
|
||||
]
|
||||
`;
|
||||
const readFile = (path: string) => {
|
||||
if (path === 'app/api/urls.py') return childContent;
|
||||
if (path === 'v1/urls.py' || path === 'app/v1/urls.py') return grandchildContent;
|
||||
if (path === 'v2/urls.py' || path === 'app/v2/urls.py') return grandchildContent;
|
||||
return null;
|
||||
};
|
||||
|
||||
const routes = extract(
|
||||
`
|
||||
from django.urls import path, include
|
||||
|
||||
urlpatterns = [
|
||||
path('api/', include('app.api.urls')),
|
||||
]
|
||||
`,
|
||||
'root/urls.py',
|
||||
readFile,
|
||||
);
|
||||
|
||||
// Should have deeply prefixed routes: api/v1/users/ and api/v2/users/
|
||||
const prefixedRoutes = routes.filter((r) => r.prefix != null);
|
||||
const hasApiV1Users = prefixedRoutes.some(
|
||||
(r) => r.prefix === 'api/v1/' && r.routePath === 'users/',
|
||||
);
|
||||
const hasApiV2Users = prefixedRoutes.some(
|
||||
(r) => r.prefix === 'api/v2/' && r.routePath === 'users/',
|
||||
);
|
||||
expect(hasApiV1Users).toBe(true);
|
||||
expect(hasApiV2Users).toBe(true);
|
||||
// Included routes should report their actual source file
|
||||
const v1Routes = routes.filter((r) => r.prefix === 'api/v1/');
|
||||
expect(v1Routes.every((r) => r.filePath === 'v1/urls.py')).toBe(true);
|
||||
const v2Routes = routes.filter((r) => r.prefix === 'api/v2/');
|
||||
expect(v2Routes.every((r) => r.filePath === 'v2/urls.py')).toBe(true);
|
||||
});
|
||||
|
||||
it('resolves views with attribute-style references (views.function)', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
from . import views
|
||||
|
||||
urlpatterns = [
|
||||
path('dashboard/', views.DashboardView.as_view()),
|
||||
path('report/', views.report_view),
|
||||
]
|
||||
`);
|
||||
expect(routes).toHaveLength(2);
|
||||
expect(routes[0]).toMatchObject({
|
||||
httpMethod: '*',
|
||||
routePath: 'dashboard/',
|
||||
controllerName: 'views.DashboardView.as_view()',
|
||||
});
|
||||
expect(routes[1]).toMatchObject({
|
||||
httpMethod: '*',
|
||||
routePath: 'report/',
|
||||
controllerName: 'views.report_view',
|
||||
});
|
||||
});
|
||||
|
||||
it('infers HTTP method from view name suffix', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = [
|
||||
path('users/', views.get_user),
|
||||
path('users/', views.post_user),
|
||||
path('users/', views.put_user),
|
||||
path('users/', views.patch_user),
|
||||
path('users/', views.delete_user),
|
||||
]
|
||||
`);
|
||||
const methods = routes.map((r) => r.httpMethod);
|
||||
expect(methods).toEqual(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);
|
||||
});
|
||||
|
||||
it('handles include with tuple namespace', () => {
|
||||
const childContent = `
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = [
|
||||
path('profile/', views.profile),
|
||||
]
|
||||
`;
|
||||
const readFile = (path: string) => {
|
||||
if (path === 'account/urls.py' || path === 'app/account/urls.py') return childContent;
|
||||
return null;
|
||||
};
|
||||
|
||||
const routes = extract(
|
||||
`
|
||||
from django.urls import path, include
|
||||
|
||||
urlpatterns = [
|
||||
path('account/', include(('account.urls', 'app_name'), namespace='account')),
|
||||
]
|
||||
`,
|
||||
'app/urls.py',
|
||||
readFile,
|
||||
);
|
||||
|
||||
const prefixedRoutes = routes.filter((r) => r.prefix === 'account/');
|
||||
expect(prefixedRoutes.length).toBeGreaterThanOrEqual(1);
|
||||
expect(prefixedRoutes.some((r) => r.filePath === 'account/urls.py')).toBe(true);
|
||||
});
|
||||
|
||||
it('does not crash on empty urlpatterns', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
|
||||
urlpatterns = []
|
||||
`);
|
||||
expect(routes).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('skips non-urlpatterns assignments', () => {
|
||||
const routes = extract(`
|
||||
from django.urls import path
|
||||
|
||||
OTHER_LIST = [
|
||||
path('not-a-route/', something),
|
||||
]
|
||||
|
||||
urlpatterns = [
|
||||
path('real/', views.real),
|
||||
]
|
||||
`);
|
||||
expect(routes).toHaveLength(1);
|
||||
expect(routes[0].routePath).toBe('real/');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue