scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-10 03:27:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'main' into docs/kilo-code-mcp

Browse source
This commit is contained in:
Tanishq Khatri 2026-06-22 11:18:37 +05:30 • committed by GitHub
commit f3e5aed7ff
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 1866 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

225
gitnexus/src/core/group/extractors/http-patterns/python.ts
View file

@ -11,6 +11,7 @@ import type { HttpDetection, HttpLanguagePlugin, RepoContext } from './types.js'
/**
* Python HTTP plugin. Handles:
* - FastAPI `@app.get("/path")` provider decorators
* - Django `path("route/", view)` provider calls
* - `requests.get/post/...("url")` consumer calls
* - Generic `requests.request("METHOD", "url")` consumer calls
* - `httpx.AsyncClient` instances calling `.get/.post/...("url")`, including
@ -52,6 +53,14 @@ const FASTAPI_APP_PATTERNS = compilePatterns({
],
} satisfies LanguagePatterns<Record<string, never>>);
// NOTE: Django providers are NOT extracted by this per-file source scan.
// A standalone scan of `path()`/`re_path()` calls cannot tell a route from an
// `include()` mount point, nor compose the include() prefix across files, so it
// emitted bogus fragments (e.g. `/api` for a mount and `/items` un-prefixed
// instead of the real `/api/items`). Django provider contracts come from the
// graph Route nodes, which the ingestion route extractor builds with the
// includes already composed.
const FASTAPI_ROUTER_PATTERNS = compilePatterns({
name: 'python-fastapi-router',
language: Python,
@ -184,7 +193,7 @@ const FROM_IMPORT_MODULE_PATTERNS = compilePatterns({
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: requests.get/post/... ──────────────────────────────────
// ─── Consumer: requests.get/post/...("literal") ──────────────────────
const REQUESTS_VERB_PATTERNS = compilePatterns({
name: 'python-requests-verb',
language: Python,
@ -202,6 +211,27 @@ const REQUESTS_VERB_PATTERNS = compilePatterns({
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: requests.get/post/...(url=VALUE) keyword ──────────────
const REQUESTS_KEYWORD_URL_PATTERNS = compilePatterns({
name: 'python-requests-keyword-url',
language: Python,
patterns: [
{
meta: {},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "requests")
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
arguments: (argument_list
(keyword_argument
name: (identifier) @kw (#eq? @kw "url")
value: (string) @path)))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: requests.request("METHOD", "url") ─────────────────────
const REQUESTS_GENERIC_PATTERNS = compilePatterns({
name: 'python-requests-generic',
@ -220,6 +250,101 @@ const REQUESTS_GENERIC_PATTERNS = compilePatterns({
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: wrapper classes with uri= or url= keyword argument ──────
// Common pattern: wrapper classes like RequestFetch that accept URL via
// named argument instead of positional argument:
// obj.fetch(uri="api/v1/camera/info/")
// obj.get(url="api/v1/camera/info/")
// obj.post(uri="api/v1/config/update/")
const WRAPPER_URI_PATTERNS = compilePatterns({
name: 'python-http-wrapper-uri',
language: Python,
patterns: [
{
meta: {},
// Match any method call where keyword argument is `uri` or `url`
query: `
(call
function: (attribute
object: (_) @client
attribute: (identifier) @method)
arguments: (argument_list
(keyword_argument
name: (identifier) @kw (#match? @kw "^(uri|url)$")
value: (string) @path)))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// Map wrapper method names to HTTP verbs
const WRAPPER_METHOD_TO_HTTP: Record<string, string> = {
get: 'GET',
post: 'POST',
put: 'PUT',
delete: 'DELETE',
patch: 'PATCH',
fetch: 'GET',
request: 'GET',
};
// ─── Variable-to-string propagation patterns ─────────────────────────
// Many repos assign URL paths to local variables then pass them as
// keyword arguments: uri = "api/v1/endpoint/"; obj.fetch(uri=uri, body)
// These patterns + buildLocalStringMap resolve the variable → literal chain.
// Track local string constants: uri = "api/v1/endpoint/"
const LOCAL_STRING_ASSIGNMENTS = compilePatterns({
name: 'python-local-string-assign',
language: Python,
patterns: [
{
meta: {},
query: `
(assignment
left: (identifier) @var_name
right: (string) @var_value)
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// Match method calls where uri=/url= value is a variable that was previously
// assigned a string literal
const WRAPPER_URI_VAR_PATTERNS = compilePatterns({
name: 'python-http-wrapper-uri-var',
language: Python,
patterns: [
{
meta: {},
query: `
(call
function: (attribute
object: (_) @client
attribute: (identifier) @method)
arguments: (argument_list
(keyword_argument
name: (identifier) @kw (#match? @kw "^(uri|url)$")
value: (identifier) @path_var)))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// Pre-scan: collect local string assignments (uri = "api/v1/endpoint/")
function buildLocalStringMap(tree: Parser.Tree): Map<string, string> {
const map = new Map<string, string>();
for (const match of runCompiledPatterns(LOCAL_STRING_ASSIGNMENTS, tree)) {
const varNode = match.captures.var_name;
const valNode = match.captures.var_value;
if (!varNode || !valNode) continue;
const val = unquoteLiteral(valNode.text);
if (val === null) continue;
map.set(varNode.text, val);
}
return map;
}
// ─── Consumer: httpx.AsyncClient assignments ────────────────────────
// Module-scope clients are only matched
// at module scope; calls inside functions require a function/class-local tracked
@ -822,6 +947,10 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
});
}
// Django providers come from the graph Route nodes (includes composed by
// the ingestion route extractor), not a per-file source scan — see the note
// at the top of this file.
// Providers: FastAPI @router.<verb>("/path") — must be joined
// with the prefix(es) declared at the include_router site. When
// no prefix is found we still emit the unprefixed path so this
@ -880,6 +1009,23 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
});
}
// Consumers: requests.<verb>(url="literal") keyword
for (const match of runCompiledPatterns(REQUESTS_KEYWORD_URL_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'python-requests',
method: methodNode.text.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// Consumers: requests.request("METHOD", "url")
for (const match of runCompiledPatterns(REQUESTS_GENERIC_PATTERNS, tree)) {
const methodNode = match.captures.http_method;
@ -937,6 +1083,83 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
});
}
// Consumers: wrapper classes with uri= or url= keyword argument
// obj.fetch(uri="api/v1/camera/info/")
// obj.post(url="api/v1/config/update/")
const seenUriDetections = new Set<string>(); // node byte ranges, to avoid duplicates
for (const match of runCompiledPatterns(WRAPPER_URI_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
// Deduplicate: the two pattern branches can match the same call. Key on
// node byte offsets, not line arithmetic (lineNum*1000+row can collide in
// files over 1000 lines, and miss a real dup when a node straddles a line).
const dedupKey = `${pathNode.startIndex}:${methodNode.startIndex}`;
if (seenUriDetections.has(dedupKey)) continue;
seenUriDetections.add(dedupKey);
const methodName = methodNode.text.toLowerCase();
// Map wrapper method name to HTTP verb (fetch, request → GET)
const httpMethod = WRAPPER_METHOD_TO_HTTP[methodName] ?? 'GET';
out.push({
role: 'consumer',
framework: 'python-http-wrapper',
method: httpMethod,
path,
name: null,
confidence: 0.65,
});
}
// Variable propagation: uri = "api/v1/endpoint/"; obj.fetch(uri=uri)
// Many repos assign URL paths to local vars then pass as keyword args.
const localStrings = buildLocalStringMap(tree);
const seenVarDetections = new Set<string>();
for (const match of runCompiledPatterns(WRAPPER_URI_VAR_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathVarNode = match.captures.path_var;
if (!methodNode || !pathVarNode) continue;
const dedupKey = `${pathVarNode.startPosition.row}:${methodNode.startPosition.row}`;
if (seenVarDetections.has(dedupKey)) continue;
seenVarDetections.add(dedupKey);
const resolved = localStrings.get(pathVarNode.text);
if (!resolved) continue;
const normalized = normalizeConsumerPath(resolved);
if (normalized === '/') continue;
const httpMethod = WRAPPER_METHOD_TO_HTTP[methodNode.text.toLowerCase()] ?? 'GET';
out.push({
role: 'consumer',
framework: 'python-http-wrapper',
method: httpMethod,
path: normalized,
name: null,
confidence: 0.6,
});
}
return out;
},
};
/** Normalize consumer path: strip host, template literals, numeric segments → {param} */
function normalizeConsumerPath(url: string): string {
let s = url.replace(/\$\{[^}]+\}/g, '{param}').trim();
if (/^https?:\/\//i.test(s)) {
try {
s = new URL(s).pathname;
} catch {
s = s.replace(/^https?:\/\/[^/]+/i, '');
}
}
if (!s.startsWith('/')) s = '/' + s;
const segments = s
.split('/')
.filter(Boolean)
.map((seg) => (/^\d+$/.test(seg) ? '{param}' : seg));
s = '/' + segments.join('/');
return s.replace(/\/+$/, '') || '/';
}

27
gitnexus/src/core/group/matching.ts
View file

@ -116,19 +116,34 @@ export function normalizeContractId(id: string): string {
function findMatchingKeys(contractId: string, index: Map<string, StoredContract[]>): string[] {
const normalized = normalizeContractId(contractId);
if (index.has(normalized)) return [normalized];
if (normalized.startsWith('http::*::')) {
const pathPart = normalized.substring('http::*::'.length);
if (normalized.startsWith('http::')) {
const rest = normalized.substring('http::'.length);
const sepIdx = rest.indexOf('::');
const method = sepIdx >= 0 ? rest.substring(0, sepIdx) : '';
const pathPart = sepIdx >= 0 ? rest.substring(sepIdx + 2) : rest;
const matches: string[] = [];
for (const key of index.keys()) {
if (key.startsWith('http::') && key.endsWith(`::${pathPart}`)) {
matches.push(key);
if (method === '*') {
// Wildcard consumer: match a provider of any method on this path.
for (const key of index.keys()) {
if (key.startsWith('http::') && key.endsWith(`::${pathPart}`)) {
matches.push(key);
}
}
return matches;
}
// Specific consumer: match an exact-method provider OR a method-agnostic
// (`*`) provider on the same path — symmetric to the wildcard-consumer case,
// so a `POST /x` consumer still matches a method-agnostic (e.g. Django)
// provider for `/x`.
if (index.has(normalized)) matches.push(normalized);
const wildcardKey = `http::*::${pathPart}`;
if (index.has(wildcardKey)) matches.push(wildcardKey);
return matches;
}
if (index.has(normalized)) return [normalized];
if (normalized.startsWith('thrift::')) {
const rest = normalized.substring('thrift::'.length);
const slashIdx = rest.indexOf('/');

31
gitnexus/src/core/ingestion/language-provider.ts
View file

@ -37,6 +37,7 @@ import type { ImportResolverFn } from './import-resolvers/types.js';
import type { SyntaxNode } from './utils/ast-helpers.js';
import type { CfgVisitor } from './cfg/types.js';
import type { NodeLabel } from 'gitnexus-shared';
import type { ExtractedRoute } from './route-extractors/laravel.js';
import type Parser from 'tree-sitter';
import type { ExtractedDecoratorRoute } from './workers/parse-worker.js';
@ -240,10 +241,36 @@ interface LanguageProviderConfig {
nodeName: string,
captureMap: CaptureMap,
) => string | undefined;
/** Detect if a file contains framework route definitions (e.g., Laravel routes.php).
* When true, the worker extracts routes via the language's route extraction logic.
/** Detect if a file contains single-file framework route definitions
* (e.g., Laravel `routes/*.php`). When true, the parse worker extracts
* routes from that file in isolation via the worker's route logic.
* Default: undefined (no route files). */
readonly isRouteFile?: (filePath: string) => boolean;
/** Discover the root route file(s) for a whole-repo, cross-file routing
* framework (e.g. Django: manage.py → settings → ROOT_URLCONF → root urls.py).
* Runs once on the main thread after all files are scanned. `reader` resolves
* arbitrary repo-relative paths (in-memory map, then disk) so discovery never
* depends on which parse chunk a file landed in. Returns one repo-relative
* path per discoverable project (empty when the framework is absent) — a
* monorepo with several projects yields each project's root.
* Pairs with `extractRoutes`; languages with this hook are skipped by the
* worker's single-file `isRouteFile` path. */
readonly discoverRootRouteFiles?: (
files: Array<{ path: string; content?: string }>,
contentMap?: Map<string, string>,
reader?: (relativePath: string) => string | null,
) => string[];
/** Extract routes from a root route file, following cross-file includes via
* `reader`. Runs on the main thread (never in the worker, which has no
* filesystem access). `parser` is a tree-sitter parser preloaded with this
* language's grammar, available for re-parsing included files.
* Default: undefined (no route extraction). */
readonly extractRoutes?: (
tree: Parser.Tree,
filePath: string,
reader: (relativePath: string) => string | null,
parser?: Parser | null,
) => ExtractedRoute[];
/**
* Extract decorator-style route annotations from a parsed file.

11
gitnexus/src/core/ingestion/languages/python.ts
View file

@ -42,6 +42,8 @@ import {
pythonReceiverBinding,
resolvePythonImportTarget,
} from './python/index.js';
import { extractDjangoRoutes } from '../route-extractors/django.js';
import { discoverDjangoRootUrls } from '../route-extractors/django-root-discovery.js';
const BUILT_INS: ReadonlySet<string> = new Set([
'print',
@ -131,6 +133,15 @@ export const pythonProvider = defineLanguage({
classExtractor: createClassExtractor(pythonClassConfig),
descriptionExtractor: pythonDescriptionExtractor,
builtInNames: BUILT_INS,
// Django routing is whole-repo and cross-file (manage.py → settings →
// ROOT_URLCONF → root urls.py, then include()s across files), so it runs as
// a main-thread pass (see parse-impl's cross-file route extraction) rather
// than the worker's single-file `isRouteFile` path. `reader` lets discovery
// and extraction resolve any repo-relative file regardless of parse chunking.
discoverRootRouteFiles: (files, contentMap, reader) =>
discoverDjangoRootUrls(files, contentMap, reader),
extractRoutes: (tree, filePath, reader, parser) =>
parser ? extractDjangoRoutes(tree, filePath, parser, reader) : [],
labelOverride: pythonFunctionDefinitionLabel,
// ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ──────────

136
gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts
View file

@ -50,7 +50,14 @@ import {
SupportedLanguages,
} from 'gitnexus-shared';
import { readFileContents } from '../filesystem-walker.js';
import { isLanguageAvailable, isGrammarRuntimeSkipped } from '../../tree-sitter/parser-loader.js';
import {
isLanguageAvailable,
isGrammarRuntimeSkipped,
createParserForLanguage,
} from '../../tree-sitter/parser-loader.js';
import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
import { getProvider, providers } from '../languages/index.js';
import type Parser from 'tree-sitter';
import {
createWorkerPool,
workerPoolDisabledByEnv,
@ -150,6 +157,122 @@ function resolveChunkByteBudget(options?: PipelineOptions, effectivePoolSize = 1
type ScannedFile = { path: string; size: number };
type ProgressFn = (progress: PipelineProgress) => void;
/**
* Whole-repo, cross-file route extraction (main thread).
*
* Some frameworks define their route table from a single root file that pulls
* in other files across the repo — e.g. Django follows
* `manage.py → DJANGO_SETTINGS_MODULE → ROOT_URLCONF → root urls.py`, then walks
* `include()` chains across many files. Unlike single-file route files (Laravel
* `routes/*.php`), which the parse worker extracts in isolation, these need a
* whole-repo view and on-demand cross-file reads — neither of which the
* filesystem-free worker can provide, and which a per-chunk worker view gets
* wrong whenever the root file and its includes land in different chunks.
*
* So it runs here, once, after every file is scanned — mirroring the FastAPI
* router-include join further below. The pass is language-agnostic: any
* {@link LanguageProvider} exposing both `discoverRootRouteFile` and
* `extractRoutes` participates (today only Python/Django). For repos without
* such a framework the cost is a path scan plus one `manage.py`-style miss.
*/
export async function extractCrossFileRoutes(
allPaths: string[],
repoPath: string,
): Promise<ExtractedRoute[]> {
const out: ExtractedRoute[] = [];
// Languages whose provider implements the cross-file route hooks. Route
// results are intentionally NOT persisted across analyze runs, so a repo
// using such a framework (e.g. Django) re-derives its routes on every run;
// a repo without one does effectively nothing here. Cross-run route caching
// is a deliberate follow-up — see #1836.
const routeCapableLangs = new Set<SupportedLanguages>();
for (const provider of Object.values(providers)) {
if (provider.discoverRootRouteFiles && provider.extractRoutes) {
routeCapableLangs.add(provider.id);
}
}
if (routeCapableLangs.size === 0) return out;
// Bucket only the paths whose language can contribute routes, so a non-
// framework repo never pays to bucket the languages it doesn't use here.
const pathsByLang = new Map<SupportedLanguages, string[]>();
for (const p of allPaths) {
const lang = getLanguageFromFilename(p);
if (!lang || !routeCapableLangs.has(lang)) continue;
let bucket = pathsByLang.get(lang);
if (!bucket) {
bucket = [];
pathsByLang.set(lang, bucket);
}
bucket.push(p);
}
for (const [lang, langPaths] of pathsByLang) {
if (!isLanguageAvailable(lang)) continue;
const provider = getProvider(lang);
if (!provider.discoverRootRouteFiles || !provider.extractRoutes) continue;
// Disk-backed reader keyed on repo-relative paths. Discovery and the
// include() walk read through this; nothing is pre-loaded, so a repo that
// lacks the framework pays only the reads its own discovery probes trigger.
const readCache = new Map<string, string | null>();
const reader = (relativePath: string): string | null => {
const cached = readCache.get(relativePath);
if (cached !== undefined) return cached;
let content: string | null = null;
try {
content = fs.readFileSync(path.join(repoPath, relativePath), 'utf-8');
} catch {
content = null;
}
readCache.set(relativePath, content);
return content;
};
// One root route file per discoverable project (a monorepo can have several).
const rootPaths = provider.discoverRootRouteFiles(
langPaths.map((p) => ({ path: p })),
undefined,
reader,
);
if (rootPaths.length === 0) continue;
// One parser per language — the grammar is language-scoped, so it is reused
// for every project root and every include() re-parse.
let parser: Parser;
try {
parser = await createParserForLanguage(lang, rootPaths[0]);
} catch {
continue; // grammar unavailable — skip the language, mirrors worker safety net
}
for (const rootPath of rootPaths) {
const rootContent = reader(rootPath);
if (rootContent === null) continue; // skip this root only, not the language
let rootTree: Parser.Tree;
try {
rootTree = parseSourceSafe(parser, rootContent);
} catch {
logger.warn(`Skipping unparseable root route file: ${rootPath}`);
continue; // skip this root only
}
// Isolate a misbehaving provider: a throw here must not abort the whole
// analyze (mirrors the worker's per-file isolation). Skip this root, warn.
try {
const routes = provider.extractRoutes(rootTree, rootPath, reader, parser);
for (const r of routes) out.push(r);
} catch (err) {
logger.warn({ err }, `Cross-file route extraction failed for ${rootPath}`);
}
}
}
return out;
}
/**
* Handle a worker-pool startup failure by FAILING FAST with the captured cause
* (#1741). The pool self-heals *transient* worker crashes on its own — a
@ -932,6 +1055,17 @@ export async function runChunkedParseAndResolve(
for (const [fp, exports] of graphExports) exportedTypeMap.set(fp, exports);
logHeapProbe('post-buildExportedTypeMapFromGraph');
}
// Whole-repo, cross-file route extraction (e.g. Django) runs on the main
// thread — the worker has no filesystem access and can't follow `include()`
// chains across files. Merge its routes in before `processRoutesFromExtracted`
// and the routes phase consume `allExtractedRoutes`.
const crossFileRoutes = await extractCrossFileRoutes(allPaths, repoPath);
if (crossFileRoutes.length > 0) {
for (const r of crossFileRoutes) allExtractedRoutes.push(r);
if (deferredProfile) {
logDeferredProfile(`cross-file routes: +${crossFileRoutes.length}`);
}
}
if (allExtractedRoutes.length > 0) {
const tRoutes = startTimer(deferredProfile);
await processRoutesFromExtracted(graph, allExtractedRoutes, model, (current, total) => {

4
gitnexus/src/core/ingestion/pipeline-phases/routes.ts
View file

@ -168,6 +168,10 @@ const VALID_HTTP_METHODS = new Set([
export function normalizeRouteMethod(raw: string | null | undefined): string | undefined {
if (typeof raw !== 'string') return undefined;
const verb = raw.trim().toUpperCase();
// '*' marks a method-agnostic route (e.g. a Django function view handles any
// verb). Preserve it so the contract layer emits a wildcard provider that
// matches consumers of any method, instead of silently narrowing to GET.
if (verb === '*') return '*';
return VALID_HTTP_METHODS.has(verb) ? verb : undefined;
}

238
gitnexus/src/core/ingestion/route-extractors/django-root-discovery.ts Normal file
View file

@ -0,0 +1,238 @@
import type { DjangoFileReader } from './django.js';
/**
* Given a `manage.py` file content, extract the Django settings module.
* e.g. `os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'cmrMngt.settings')`
* returns `'cmrMngt.settings'`
*/
function extractDjangoSettingsModule(manageContent: string): string | null {
const m = manageContent.match(/DJANGO_SETTINGS_MODULE\s*['"]?[,= ]\s*['"]([^'"]+)['"]/);
return m ? m[1] : null;
}
/**
* Given a dotted Python module path, produce possible file paths.
* e.g. `cmrMngt.settings` → `['cmrMngt/settings.py', 'cmrMngt/settings/__init__.py']`
*/
export function djangoModuleToFilePaths(modulePath: string): string[] {
const base = modulePath.replace(/\./g, '/');
return [`${base}.py`, `${base}/__init__.py`];
}
/**
* Read a file, trying first the in-memory content map, then the optional
* reader (typically a disk-backed reader on the main thread). The map keeps
* already-loaded content cheap; the reader lets discovery reach files that
* were never pre-loaded — critical because the relevant files (manage.py,
* settings, the root urls.py) can be scattered across parse chunks.
*/
function tryReadFile(
relativePath: string,
contentMap: Map<string, string>,
reader?: DjangoFileReader,
): string | null {
return contentMap.get(relativePath) ?? reader?.(relativePath) ?? null;
}
/**
* Extract a module-level string assignment value from Python source.
* e.g. `content` contains `ROOT_URLCONF = 'cmrMngt.urls'`
* returns `'cmrMngt.urls'`
*/
function extractPythonStringAssignment(content: string, varName: string): string | null {
const regex = new RegExp(`^${varName}\\s*=\\s*['"]([^'"]+)['"]`, 'm');
const m = content.match(regex);
return m ? m[1] : null;
}
/**
* Extract `from <module> import *` statements from Python source.
* e.g. `from .settings_base import *` → `settings_base`
* `from cmrMngt.settings_base import *` → `cmrMngt.settings_base`
*/
function extractStarImports(content: string): string[] {
const modules: string[] = [];
const regex = /^from\s+(\.?[\w.]+)\s+import\s+\*/gm;
let m;
while ((m = regex.exec(content)) !== null) {
// Relative (leading-dot) and absolute module names are both pushed verbatim;
// the caller resolves relative ones against the current module path.
modules.push(m[1]);
}
return modules;
}
/**
* Resolve a relative Python import path.
* `from .settings_base import *` in `cmrMngt/settings.py`
* → `cmrMngt/settings_base.py`
*/
function resolveRelativeImport(currentModulePath: string, importPath: string): string | null {
if (!importPath.startsWith('.')) return null;
const currentDir = currentModulePath.includes('/')
? currentModulePath.substring(0, currentModulePath.lastIndexOf('/'))
: '';
let relPath = importPath;
let dir = currentDir;
while (relPath.startsWith('.')) {
if (relPath.startsWith('..')) {
dir = dir.includes('/') ? dir.substring(0, dir.lastIndexOf('/')) : '';
relPath = relPath.substring(2);
} else {
relPath = relPath.substring(1);
break;
}
}
return dir ? `${dir}/${relPath}` : relPath;
}
/**
* Resolve the root URL file for a SINGLE Django project rooted at `managePyPath`.
*
* Module paths in `manage.py`/`settings.py` are written relative to the
* project directory (the one containing `manage.py`), NOT the repo root — so a
* project under `backend/` declares `myproj.settings`, with the file living at
* `backend/myproj/settings.py`. We therefore resolve every candidate against
* the project directory first and the repo root second. `resolvedSettingsPath`
* is kept project-dir-aware so the downstream star-import and urls-dir
* resolution anchor correctly.
*/
function resolveDjangoProjectRoot(
managePyPath: string,
manageContent: string,
map: Map<string, string>,
reader?: DjangoFileReader,
): string | null {
const projectDir = managePyPath.includes('/')
? managePyPath.substring(0, managePyPath.lastIndexOf('/'))
: '';
// Try the project dir first (the common subdir case), then the repo root.
const bases = projectDir ? [`${projectDir}/`, ''] : [''];
const settingsModule = extractDjangoSettingsModule(manageContent);
if (!settingsModule) return null;
const settingsSlash = settingsModule.replace(/\./g, '/');
// Find the settings file, recording which base it resolved under so relative
// imports and the urls-dir fallback stay anchored to the right directory.
let settingsContent: string | null = null;
let resolvedSettingsPath: string | null = null;
for (const base of bases) {
for (const sp of djangoModuleToFilePaths(settingsModule)) {
const c = tryReadFile(base + sp, map, reader);
if (c !== null) {
settingsContent = c;
resolvedSettingsPath = base + settingsSlash;
break;
}
}
if (settingsContent) break;
}
if (!settingsContent || resolvedSettingsPath === null) return null;
// Check ROOT_URLCONF in the main settings and any base settings (star imports)
let rootUrlConf = extractPythonStringAssignment(settingsContent, 'ROOT_URLCONF');
if (!rootUrlConf) {
// Check star-imported base settings
const starImports = extractStarImports(settingsContent);
for (const imp of starImports) {
let baseModule: string | null = null;
if (imp.startsWith('.')) {
const resolved = resolveRelativeImport(resolvedSettingsPath, imp);
if (resolved) baseModule = resolved;
} else {
baseModule = imp;
}
if (!baseModule) continue;
// `baseModule` is always a slash-path here: a relative import is resolved
// by `resolveRelativeImport` (which never returns a leading-dot path), and
// an absolute import is the bare module name. So there is no remaining
// dot-prefixed case to handle.
const basePaths: string[] = [];
const baseSlash = baseModule.replace(/\./g, '/');
// A relative import (`imp` started with `.`) is already anchored under
// the project dir via `resolvedSettingsPath`; an absolute module name
// may live under the project dir OR the repo root.
const candidateBases = imp.startsWith('.') ? [''] : bases;
for (const cb of candidateBases) {
basePaths.push(`${cb}${baseSlash}.py`);
basePaths.push(`${cb}${baseSlash}/__init__.py`);
}
for (const bp of basePaths) {
const bc = tryReadFile(bp, map, reader);
if (bc) {
rootUrlConf = extractPythonStringAssignment(bc, 'ROOT_URLCONF');
if (rootUrlConf) break;
}
}
if (rootUrlConf) break;
}
}
if (!rootUrlConf) return null;
// Convert ROOT_URLCONF module path to a file path, trying project dir then root.
const urlPaths = djangoModuleToFilePaths(rootUrlConf);
for (const base of bases) {
for (const up of urlPaths) {
if (tryReadFile(base + up, map, reader) !== null) return base + up;
}
}
// Also try relative to the settings module's directory (project-dir-aware).
if (resolvedSettingsPath.includes('/')) {
const settingsDir = resolvedSettingsPath.substring(
0,
resolvedSettingsPath.lastIndexOf('/') + 1,
);
for (const up of urlPaths) {
const tryPath = settingsDir + up;
if (tryReadFile(tryPath, map, reader) !== null) return tryPath;
}
}
return null;
}
/**
* Discover the Django root URL file(s) by following, for EVERY `manage.py` in
* the file set:
* manage.py → DJANGO_SETTINGS_MODULE → settings → ROOT_URLCONF → urls.py
*
* Returns one root urls path per discoverable Django project, so a monorepo
* with several `manage.py` files (e.g. `serviceA/manage.py`, `serviceB/manage.py`)
* yields every project's routes rather than only the first.
*
* @param files Array of file paths (content optional — when absent, `reader`
* resolves it on demand).
* @param contentMap Optional pre-built map of file path → content.
* @param reader Optional disk-backed reader for files not present in the map.
* @returns De-duplicated relative paths to each project's root URL file (empty if none).
*/
export function discoverDjangoRootUrls(
files: Array<{ path: string; content?: string }>,
contentMap?: Map<string, string>,
reader?: DjangoFileReader,
): string[] {
const map = contentMap ?? new Map<string, string>();
for (const f of files) if (f.content != null) map.set(f.path, f.content);
const roots: string[] = [];
const seen = new Set<string>();
for (const f of files) {
if (f.path !== 'manage.py' && !f.path.endsWith('/manage.py')) continue;
const manageContent = f.content ?? tryReadFile(f.path, map, reader);
if (!manageContent) continue;
const root = resolveDjangoProjectRoot(f.path, manageContent, map, reader);
if (root !== null && !seen.has(root)) {
seen.add(root);
roots.push(root);
}
}
return roots;
}

481
gitnexus/src/core/ingestion/route-extractors/django.ts Normal file
View file

@ -0,0 +1,481 @@
import type Parser from 'tree-sitter';
import { parseSourceSafe } from '../../tree-sitter/safe-parse.js';
import { extractStringContent, type SyntaxNode } from '../utils/ast-helpers.js';
import type { ExtractedRoute } from './laravel.js';
import { logger } from '../../logger.js';
interface DjangoRouteContext {
prefix: string | null;
}
interface WalkFrame {
node: SyntaxNode;
routeCtx: DjangoRouteContext;
currentFilePath: string;
depth: number;
}
const DJANGO_ROUTE_FUNCTIONS = new Set(['path', 're_path', 'url']);
const DJANGO_INCLUDE_FUNCTION = 'include';
const MAX_INCLUDE_DEPTH = 8;
// Wrapper calls whose first list/tuple argument is itself a urlpatterns list
// (e.g. DRF's `format_suffix_patterns([...])`, `i18n_patterns(...)`,
// `staticfiles_urlpatterns()`). We descend into their list-typed arguments.
const URLPATTERNS_WRAPPER_FUNCTIONS = new Set([
'format_suffix_patterns',
'staticfiles_urlpatterns',
'i18n_patterns',
]);
function modulePathToFilePath(modulePath: string): string {
return modulePath.replace(/\./g, '/');
}
export type DjangoFileReader = (relativePath: string) => string | null;
function extractStringArg(argsNode: SyntaxNode | null): string | null {
if (!argsNode) return null;
for (const child of argsNode.children ?? []) {
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
if (child.type === 'string') {
return extractStringContent(child);
}
if (child.type === 'binary_operator') {
let concat = '';
for (const part of child.children ?? []) {
if (part.type === 'string') {
const s = extractStringContent(part);
if (s !== null) concat += s;
}
}
if (concat) return concat;
}
}
return null;
}
function extractViewTarget(argsNode: SyntaxNode | null): {
viewName: string | null;
viewCall: string | null;
} {
if (!argsNode) return { viewName: null, viewCall: null };
const positionalArgs: SyntaxNode[] = [];
for (const child of argsNode.children ?? []) {
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
positionalArgs.push(child);
}
const viewNode = positionalArgs[1];
if (!viewNode) return { viewName: null, viewCall: null };
if (viewNode.type === 'attribute') return { viewName: viewNode.text, viewCall: null };
if (viewNode.type === 'call') return { viewName: null, viewCall: viewNode.text };
if (viewNode.type === 'identifier') return { viewName: viewNode.text, viewCall: null };
if (viewNode.type === 'string')
return { viewName: extractStringContent(viewNode), viewCall: null };
return { viewName: null, viewCall: null };
}
function inferHttpMethod(viewName: string | null): string {
if (!viewName) return '*';
const lower = viewName.toLowerCase();
const m = lower.match(/\.(get|post|put|patch|delete|head|options)(_|$)/);
if (m) {
return m[1].toUpperCase();
}
return '*';
}
/**
* Collect the list/tuple container node(s) that hold route entries from a
* `urlpatterns` right-hand side. Handles the common non-literal shapes:
* - `urlpatterns = [...]` → the list
* - `urlpatterns = (...)` → the tuple
* - `urlpatterns = a + b` → both operands (concatenation)
* - `urlpatterns = wrapper([...])` → the wrapper's list argument
* Inherently-dynamic shapes (`router.urls`, comprehensions, bare names) yield
* nothing — they cannot be resolved statically.
*/
function collectUrlpatternContainers(node: SyntaxNode, out: SyntaxNode[]): void {
switch (node.type) {
case 'list':
case 'tuple':
out.push(node);
return;
case 'binary_operator':
// `a + b` concatenation — descend into both operands.
for (const child of node.children ?? []) collectUrlpatternContainers(child, out);
return;
case 'call': {
const fn = getCallFuncName(node);
if (fn && URLPATTERNS_WRAPPER_FUNCTIONS.has(fn)) {
const args = node.childForFieldName?.('arguments') ?? null;
for (const child of args?.children ?? []) collectUrlpatternContainers(child, out);
}
return;
}
default:
return;
}
}
function findUrlpatternsLists(rootNode: SyntaxNode): SyntaxNode[] {
const assignmentNodes: SyntaxNode[] = [];
_collectAssignments(rootNode, assignmentNodes);
const lists: SyntaxNode[] = [];
for (const node of assignmentNodes) {
const left = node.childForFieldName?.('left') ?? node.children?.[0] ?? null;
if (left?.type === 'identifier' && left.text === 'urlpatterns') {
const right = node.childForFieldName?.('right') ?? node.children?.[2] ?? null;
if (!right) continue;
const before = lists.length;
collectUrlpatternContainers(right, lists);
if (lists.length === before && (right.type === 'attribute' || right.type === 'identifier')) {
// Dynamically-built urlpatterns (e.g. DRF `router.urls`) can't be
// resolved statically — surface it so the silent-zero case is visible.
logger.debug(`Django: skipping non-static urlpatterns (${right.type}: ${right.text})`);
}
}
}
return lists;
}
function _collectAssignments(node: SyntaxNode, out: SyntaxNode[]): void {
if (node.type === 'assignment' || node.type === 'augmented_assignment') {
out.push(node);
}
for (const child of node.children ?? []) {
_collectAssignments(child, out);
}
}
function emitDjangoRoute(
callNode: SyntaxNode,
filePath: string,
ctx: DjangoRouteContext,
): ExtractedRoute {
const argsNode = callNode.childForFieldName?.('arguments') ?? null;
const routePath = extractStringArg(argsNode);
const { viewName, viewCall } = extractViewTarget(argsNode);
const httpMethod = inferHttpMethod(viewName);
let routeName: string | null = null;
if (argsNode) {
for (let i = 0; i < argsNode.children.length; i++) {
const child = argsNode.children[i];
if (child.type === 'keyword_argument' && child.childForFieldName?.('name')?.text === 'name') {
const valueNode = child.childForFieldName?.('value');
if (valueNode?.type === 'string') {
routeName = extractStringContent(valueNode);
}
}
}
}
return {
filePath,
httpMethod,
routePath,
routeName,
controllerName: viewName ?? viewCall,
methodName: null,
middleware: [],
prefix: ctx.prefix,
lineNumber: callNode.startPosition.row,
};
}
function getIncludeModulePath(callNode: SyntaxNode): string | null {
const funcName =
callNode.childForFieldName?.('function')?.text ??
callNode.children?.find((c) => c.type === 'identifier')?.text;
if (funcName !== DJANGO_INCLUDE_FUNCTION) return null;
const argsNode = callNode.childForFieldName?.('arguments');
if (!argsNode) return null;
const modulePath = extractStringArg(argsNode);
if (modulePath) return modulePath;
for (const child of argsNode.children ?? []) {
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
if (child.type === 'tuple' || child.type === 'parenthesized_expression') {
for (const inner of child.children ?? []) {
if (inner.type === '(' || inner.type === ')' || inner.type === ',') continue;
if (inner.type === 'string') return extractStringContent(inner);
}
}
}
return null;
}
function makePrefix(parentPrefix: string | null, childPrefix: string | null): string | null {
if (!childPrefix) return parentPrefix;
if (!parentPrefix) return childPrefix;
return `${parentPrefix}/${childPrefix}`.replace(/\/+/g, '/');
}
/**
* Recursion-guard key. A urlconf file is walked once *per accumulated prefix*,
* so a urlconf `include()`d under two different prefixes (a "diamond" — e.g.
* the same app mounted at `/v1/` and `/v2/`) emits routes for both mounts,
* while a genuine cycle (same file + same prefix) still terminates. `null` and
* `''` collapse to the same key so a no-prefix re-entry is treated as a cycle.
*/
function includeVisitKey(filePath: string, prefix: string | null): string {
return `${filePath}\u0000${prefix ?? ''}`;
}
function getCallFuncName(node: SyntaxNode): string | null {
return (
node.childForFieldName?.('function')?.text ??
node.children?.find((c) => c.type === 'identifier')?.text ??
null
);
}
/**
* Find the Django project root for `startFilePath` — the nearest ancestor
* directory that contains a `manage.py`. Django resolves `include('app.urls')`
* as an absolute import from this directory (it is the entry on `sys.path`), so
* knowing it lets us resolve includes unambiguously even when an unrelated
* `app/urls.py` exists at the repo root (the monorepo wrong-app hazard).
* Returns the project-root directory (possibly `''` for a repo-root project),
* or `null` when no `manage.py` ancestor is readable.
*/
function findDjangoProjectRoot(
startFilePath: string,
readFile: DjangoFileReader | null | undefined,
): string | null {
if (!readFile) return null;
let dir = startFilePath.includes('/')
? startFilePath.substring(0, startFilePath.lastIndexOf('/'))
: '';
for (;;) {
const candidate = dir ? `${dir}/manage.py` : 'manage.py';
if (readFile(candidate) !== null) return dir;
if (!dir) return null;
const sep = dir.lastIndexOf('/');
dir = sep < 0 ? '' : dir.substring(0, sep);
}
}
/**
* Given a Django dotted module path like `app.submodule.urls`,
* try multiple path resolution strategies to find the file on disk.
*
* Strategies tried in order:
* 0. Anchored at the Django project root (manage.py dir) — the authoritative
* resolution for absolute module paths, when the project root is known
* 1. Direct dot-to-slash: `module/path.py` and `module/path/__init__.py`
* 2. Relative to the current file's directory
* 3. Walk up the directory tree from the current file, trying each ancestor
*/
function resolveIncludedFile(
modulePath: string,
currentFilePath: string,
readFile: DjangoFileReader,
projectRoot: string | null,
): { filePath: string; content: string } | null {
const basePath = modulePathToFilePath(modulePath);
const candidates: string[] = [];
// Strategy 0: anchored at the project root (sys.path entry). Tried first so
// `include('app.urls')` from backend/ resolves to backend/app/urls.py rather
// than a same-named app at the repo root.
if (projectRoot !== null) {
const anchored = projectRoot ? `${projectRoot}/${basePath}` : basePath;
candidates.push(anchored + '.py');
candidates.push(anchored + '/__init__.py');
}
// Strategy 1: direct path (app/urls.py, app/urls/__init__.py)
candidates.push(basePath + '.py');
candidates.push(basePath + '/__init__.py');
// Strategy 2: relative to current file's directory
if (currentFilePath.includes('/')) {
const dir = currentFilePath.substring(0, currentFilePath.lastIndexOf('/') + 1);
candidates.push(dir + basePath + '.py');
candidates.push(dir + basePath + '/__init__.py');
}
// Strategy 3: walk up from current file, trying each ancestor
let parentDir = currentFilePath.includes('/')
? currentFilePath.substring(0, currentFilePath.lastIndexOf('/'))
: '';
while (parentDir.length > 0) {
const prefix = parentDir + '/';
candidates.push(prefix + basePath + '.py');
candidates.push(prefix + basePath + '/__init__.py');
const nextSep = parentDir.lastIndexOf('/');
if (nextSep < 0) break;
parentDir = parentDir.substring(0, nextSep);
}
// Strategy 4: bare path with just the last segment (e.g. 'urls.py' from 'app.urls')
const segments = basePath.split('/');
if (segments.length > 1) {
const lastSegment = segments[segments.length - 1];
candidates.push(lastSegment + '.py');
candidates.push(lastSegment + '/__init__.py');
}
for (const candidate of candidates) {
const content = readFile(candidate);
if (content !== null) return { filePath: candidate, content };
}
return null;
}
export function extractDjangoRoutes(
tree: Parser.Tree,
filePath: string,
parser: Parser,
readFile?: DjangoFileReader | null,
_visited?: Set<string>,
): ExtractedRoute[] {
const routeSet = _visited ?? new Set<string>();
const entryKey = includeVisitKey(filePath, null);
if (routeSet.has(entryKey)) return [];
routeSet.add(entryKey);
// Resolve the project root once (constant across the whole walk) so absolute
// include() module paths anchor correctly even in a monorepo.
const projectRoot = findDjangoProjectRoot(filePath, readFile);
const listNodes = findUrlpatternsLists(tree.rootNode);
if (listNodes.length === 0) return [];
const routes: ExtractedRoute[] = [];
const walkStack: WalkFrame[] = [];
for (const listNode of listNodes) {
walkStack.push({
node: listNode,
routeCtx: { prefix: null },
currentFilePath: filePath,
depth: 0,
});
}
while (walkStack.length > 0) {
const { node, routeCtx, currentFilePath, depth } = walkStack.pop()!;
if (node.type === 'list') {
const children = node.children ?? [];
for (let i = children.length - 1; i >= 0; i--) {
const child = children[i];
if (child.type === '[' || child.type === ']' || child.type === ',') continue;
walkStack.push({ node: child, routeCtx, currentFilePath, depth });
}
continue;
}
if (node.type === 'call') {
const funcName = getCallFuncName(node);
if (!funcName) {
for (const child of node.children ?? []) {
if (child.type === 'call' || child.type === 'list') {
walkStack.push({ node: child, routeCtx, currentFilePath, depth });
}
}
continue;
}
if (DJANGO_ROUTE_FUNCTIONS.has(funcName)) {
const argsNode = node.childForFieldName?.('arguments') ?? null;
let hasIncludeChild = false;
if (argsNode) {
for (const child of argsNode.children ?? []) {
if (child.type === 'call' && getCallFuncName(child) === DJANGO_INCLUDE_FUNCTION) {
hasIncludeChild = true;
const modulePath = getIncludeModulePath(child);
if (modulePath && readFile && depth < MAX_INCLUDE_DEPTH) {
const resolved = resolveIncludedFile(
modulePath,
currentFilePath,
readFile,
projectRoot,
);
// Key the guard on (file, accumulated prefix) so the same
// urlconf mounted under another prefix elsewhere is still walked.
const childPrefix = makePrefix(routeCtx.prefix, extractStringArg(argsNode));
if (resolved && !routeSet.has(includeVisitKey(resolved.filePath, childPrefix))) {
routeSet.add(includeVisitKey(resolved.filePath, childPrefix));
let childTree: Parser.Tree;
try {
childTree = parseSourceSafe(parser, resolved.content);
} catch {
continue;
}
const childLists = findUrlpatternsLists(childTree.rootNode);
for (const childList of childLists) {
walkStack.push({
node: childList,
routeCtx: { prefix: childPrefix },
currentFilePath: resolved.filePath,
depth: depth + 1,
});
}
}
}
}
}
}
if (!hasIncludeChild) {
routes.push(emitDjangoRoute(node, currentFilePath, routeCtx));
}
continue;
}
if (funcName === DJANGO_INCLUDE_FUNCTION && readFile && depth < MAX_INCLUDE_DEPTH) {
const modulePath = getIncludeModulePath(node);
if (modulePath) {
const resolved = resolveIncludedFile(modulePath, currentFilePath, readFile, projectRoot);
// Bare include() inherits the current prefix; key the guard on it so a
// shared urlconf reached under two prefixes is walked once per prefix.
if (resolved && !routeSet.has(includeVisitKey(resolved.filePath, routeCtx.prefix))) {
routeSet.add(includeVisitKey(resolved.filePath, routeCtx.prefix));
let childTree: Parser.Tree;
try {
childTree = parseSourceSafe(parser, resolved.content);
} catch {
continue;
}
const childLists = findUrlpatternsLists(childTree.rootNode);
for (const childList of childLists) {
walkStack.push({
node: childList,
routeCtx,
currentFilePath: resolved.filePath,
depth: depth + 1,
});
}
}
}
continue;
}
for (const child of node.children ?? []) {
if (child.type === 'call' || child.type === 'list') {
walkStack.push({ node: child, routeCtx, currentFilePath, depth });
}
}
continue;
}
for (const child of node.children ?? []) {
if (child.type === '(' || child.type === ')' || child.type === ',') continue;
if (child.type === 'call' || child.type === 'list') {
walkStack.push({ node: child, routeCtx, currentFilePath, depth });
}
}
}
return routes;
}

9
gitnexus/test/fixtures/django-subdir-app/backend/app/urls.py vendored Normal file
View file

@ -0,0 +1,9 @@
"""App URL conf, included under the `api/` prefix from the root urls.py."""
from django.urls import path
from . import views
urlpatterns = [
path('items/', views.item_list, name='item-list'),
path('items/<int:pk>/', views.item_detail, name='item-detail'),
]

10
gitnexus/test/fixtures/django-subdir-app/backend/app/views.py vendored Normal file
View file

@ -0,0 +1,10 @@
"""App views referenced by app/urls.py."""
from django.http import JsonResponse
def item_list(request):
return JsonResponse({'items': []})
def item_detail(request, pk):
return JsonResponse({'id': pk})

15
gitnexus/test/fixtures/django-subdir-app/backend/manage.py vendored Normal file
View file

@ -0,0 +1,15 @@
#!/usr/bin/env python
"""Django management entry point for a project that lives under backend/."""
import os
import sys
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myproj.settings')
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

6
gitnexus/test/fixtures/django-subdir-app/backend/myproj/settings.py vendored Normal file
View file

@ -0,0 +1,6 @@
"""Settings for the subdir Django project. Module path is `myproj.settings`,
resolved relative to manage.py's directory (backend/), not the repo root."""
DEBUG = True
ROOT_URLCONF = 'myproj.urls'
INSTALLED_APPS = ['app']

9
gitnexus/test/fixtures/django-subdir-app/backend/myproj/urls.py vendored Normal file
View file

@ -0,0 +1,9 @@
"""Root URL conf — a root-level route plus an include() into the app."""
from django.urls import path, include
from . import views
urlpatterns = [
path('health/', views.health, name='health'),
path('api/', include('app.urls')),
]

6
gitnexus/test/fixtures/django-subdir-app/backend/myproj/views.py vendored Normal file
View file

@ -0,0 +1,6 @@
"""Root-level views."""
from django.http import JsonResponse
def health(request):
return JsonResponse({'status': 'ok'})

47
gitnexus/test/integration/django-route-extraction-e2e.test.ts Normal file
View file

@ -0,0 +1,47 @@
/**
* End-to-end coverage of the main-thread Django cross-file route pass.
*
* Unit tests pin `extractDjangoRoutes` and `discoverDjangoRootUrls` in
* isolation; this file runs the whole pipeline (`runPipelineFromRepo`) against
* an on-disk fixture so the orchestration glue — discovery → parse →
* `extractRoutes` → `allExtractedRoutes` → `Route` graph nodes — is actually
* exercised. The fixture deliberately places the Django project under a
* `backend/` subdirectory so this test also guards the subdir-discovery fix
* (#1836 R1): if discovery resolved the settings module from the repo root
* instead of the manage.py directory, no `Route` nodes would appear at all.
*/
import { describe, it, expect, beforeAll } from 'vitest';
import path from 'node:path';
import { runPipelineFromRepo } from '../../src/core/ingestion/pipeline.js';
import type { PipelineResult } from '../../types/pipeline.js';
const FIXTURE = path.resolve(__dirname, '..', 'fixtures', 'django-subdir-app');
describe('Django cross-file route extraction — ingestion pipeline', () => {
let result: PipelineResult;
beforeAll(async () => {
result = await runPipelineFromRepo(FIXTURE, () => {}, {});
}, 60_000);
const routeNames = (): string[] => {
const nodes: Array<{ label: string; name: string }> = [];
result.graph.forEachNode((n) => {
nodes.push({ label: String(n.label), name: String(n.properties.name) });
});
return nodes
.filter((n) => n.label === 'Route')
.map((n) => n.name)
.sort();
};
it('discovers the subdir project root and emits prefixed Route nodes across the include()', () => {
const names = routeNames();
// Root-level route — proves backend/manage.py → root urls.py discovery.
expect(names).toContain('/health');
// Included app routes inherit the parent `api/` prefix — proves the
// cross-file include() walk and prefix accumulation end-to-end.
expect(names).toContain('/api/items');
expect(names).toContain('/api/items/<int:pk>');
});
});

52
gitnexus/test/unit/cross-file-routes.test.ts Normal file
View file

@ -0,0 +1,52 @@
import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import type { ExtractedRoute } from '../../src/core/ingestion/workers/parse-worker.js';
// Hoisted so the (also-hoisted) vi.mock factory below can reference it.
const { extractRoutesSpy } = vi.hoisted(() => ({
extractRoutesSpy: vi.fn((): ExtractedRoute[] => {
throw new Error('boom from a misbehaving provider');
}),
}));
// Override only the route hooks on the real Python provider so the main-thread
// pass reaches extractRoutes and the throw exercises the in-loop guard. Every
// other dependency (parser, parse, fs reader) stays real.
vi.mock('../../src/core/ingestion/languages/index.js', async (importOriginal) => {
const actual =
await importOriginal<typeof import('../../src/core/ingestion/languages/index.js')>();
return {
...actual,
getProvider: (lang: Parameters<typeof actual.getProvider>[0]) => ({
...actual.getProvider(lang),
discoverRootRouteFiles: () => ['proj/urls.py'],
extractRoutes: extractRoutesSpy,
}),
};
});
import { extractCrossFileRoutes } from '../../src/core/ingestion/pipeline-phases/parse-impl.js';
let repoDir: string;
beforeAll(() => {
repoDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-xfr-'));
fs.mkdirSync(path.join(repoDir, 'proj'), { recursive: true });
fs.writeFileSync(path.join(repoDir, 'proj/urls.py'), 'urlpatterns = []\n');
});
afterAll(() => {
fs.rmSync(repoDir, { recursive: true, force: true });
});
describe('extractCrossFileRoutes', () => {
it('isolates a throwing provider.extractRoutes instead of aborting the run', async () => {
const routes = await extractCrossFileRoutes(['proj/urls.py'], repoDir);
// The guard caught the throw (provider was actually reached) and the run
// produced no routes rather than propagating the error.
expect(extractRoutesSpy).toHaveBeenCalledTimes(1);
expect(routes).toEqual([]);
});
});

117
gitnexus/test/unit/django-root-discovery.test.ts Normal file
View file

@ -0,0 +1,117 @@
import { describe, it, expect } from 'vitest';
import { discoverDjangoRootUrls } from '../../src/core/ingestion/route-extractors/django-root-discovery.js';
/** Build a disk-style reader from a path → content record. */
const makeReader = (fsMap: Record<string, string>) => (relativePath: string) =>
Object.prototype.hasOwnProperty.call(fsMap, relativePath) ? fsMap[relativePath] : null;
/** A `manage.py` body pointing at the given dotted settings module. */
const manageFor = (settingsModule: string) =>
`#!/usr/bin/env python\nimport os\ndef main():\n os.environ.setdefault('DJANGO_SETTINGS_MODULE', '${settingsModule}')\n`;
const MANAGE_PY = manageFor('myproj.settings');
describe('discoverDjangoRootUrls', () => {
it('discovers the root urls.py from content-bearing files (no reader)', () => {
const files = [
{ path: 'manage.py', content: MANAGE_PY },
{ path: 'myproj/settings.py', content: `ROOT_URLCONF = 'myproj.urls'\n` },
{ path: 'myproj/urls.py', content: `urlpatterns = []\n` },
];
expect(discoverDjangoRootUrls(files)).toEqual(['myproj/urls.py']);
});
it('discovers the root urls.py via the reader fallback when files carry no content', () => {
const fsMap: Record<string, string> = {
'manage.py': MANAGE_PY,
'myproj/settings.py': `ROOT_URLCONF = 'myproj.urls'\n`,
'myproj/urls.py': `urlpatterns = []\n`,
};
// Only paths are passed (the main-thread pass does this); content is resolved on demand.
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual(['myproj/urls.py']);
});
it('follows ROOT_URLCONF through a star-imported base settings module via the reader', () => {
const fsMap: Record<string, string> = {
'manage.py': MANAGE_PY,
'myproj/settings.py': `from .base import *\n`,
'myproj/base.py': `DEBUG = True\nROOT_URLCONF = 'myproj.urls'\n`,
'myproj/urls.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual(['myproj/urls.py']);
});
it('resolves a urls package directory module (urls/__init__.py)', () => {
const fsMap: Record<string, string> = {
'manage.py': MANAGE_PY,
'myproj/settings.py': `ROOT_URLCONF = 'myproj.urls'\n`,
'myproj/urls/__init__.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual([
'myproj/urls/__init__.py',
]);
});
it('discovers a Django project located in a subdirectory (settings resolved relative to manage.py)', () => {
const fsMap: Record<string, string> = {
'backend/manage.py': MANAGE_PY,
'backend/myproj/settings.py': `ROOT_URLCONF = 'myproj.urls'\n`,
'backend/myproj/urls.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual([
'backend/myproj/urls.py',
]);
});
it('follows star-imported base settings for a subdirectory project', () => {
const fsMap: Record<string, string> = {
'backend/manage.py': MANAGE_PY,
'backend/myproj/settings.py': `from .base import *\n`,
'backend/myproj/base.py': `ROOT_URLCONF = 'myproj.urls'\n`,
'backend/myproj/urls.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual([
'backend/myproj/urls.py',
]);
});
it('discovers every project in a monorepo with multiple manage.py files', () => {
const fsMap: Record<string, string> = {
'serviceA/manage.py': manageFor('svca.settings'),
'serviceA/svca/settings.py': `ROOT_URLCONF = 'svca.urls'\n`,
'serviceA/svca/urls.py': `urlpatterns = []\n`,
'serviceB/manage.py': manageFor('svcb.settings'),
'serviceB/svcb/settings.py': `ROOT_URLCONF = 'svcb.urls'\n`,
'serviceB/svcb/urls.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual([
'serviceA/svca/urls.py',
'serviceB/svcb/urls.py',
]);
});
it('returns an empty array when there is no manage.py', () => {
const fsMap: Record<string, string> = {
'myproj/settings.py': `ROOT_URLCONF = 'myproj.urls'\n`,
'myproj/urls.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual([]);
});
it('returns an empty array when ROOT_URLCONF cannot be found in settings', () => {
const fsMap: Record<string, string> = {
'manage.py': MANAGE_PY,
'myproj/settings.py': `DEBUG = True\n`,
'myproj/urls.py': `urlpatterns = []\n`,
};
const files = Object.keys(fsMap).map((path) => ({ path }));
expect(discoverDjangoRootUrls(files, undefined, makeReader(fsMap))).toEqual([]);
});
});

436
gitnexus/test/unit/django-route-extraction.test.ts Normal file
View file

@ -0,0 +1,436 @@
import { describe, expect, it } from 'vitest';
import Parser from 'tree-sitter';
import Python from 'tree-sitter-python';
import { extractDjangoRoutes } from '../../src/core/ingestion/route-extractors/django.js';
const parser = new Parser();
parser.setLanguage(Python);
const extract = (
source: string,
filePath = 'app/urls.py',
readFile?: (path: string) => string | null,
) =>
extractDjangoRoutes(parser.parse(source), filePath, parser, readFile).map((route) => ({
httpMethod: route.httpMethod,
routePath: route.routePath,
routeName: route.routeName,
controllerName: route.controllerName,
prefix: route.prefix,
filePath: route.filePath,
}));
describe('Django route extraction', () => {
it('extracts path() routes from urlpatterns', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = [
path('orders/', views.order_list),
path('orders/<int:pk>/', views.order_detail),
path('users/', views.user_list, name='user-list'),
]
`);
expect(routes).toHaveLength(3);
expect(routes[0]).toMatchObject({ httpMethod: '*', routePath: 'orders/' });
expect(routes[1]).toMatchObject({ httpMethod: '*', routePath: 'orders/<int:pk>/' });
expect(routes[2]).toMatchObject({
httpMethod: '*',
routePath: 'users/',
routeName: 'user-list',
});
});
it('extracts re_path() routes', () => {
const routes = extract(`
from django.urls import re_path
from . import views
urlpatterns = [
re_path(r'^articles/(?P<year>[0-9]{4})/$', views.year_archive),
]
`);
expect(routes).toHaveLength(1);
expect(routes[0]).toMatchObject({
httpMethod: '*',
routePath: '^articles/(?P<year>[0-9]{4})/$',
});
});
it('extracts legacy url() routes', () => {
const routes = extract(`
from django.conf.urls import url
from . import views
urlpatterns = [
url(r'^legacy/$', views.legacy_view),
]
`);
expect(routes).toHaveLength(1);
expect(routes[0]).toMatchObject({ httpMethod: '*', routePath: '^legacy/$' });
});
it('handles str concatenation in path strings', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = [
path('api/' + 'v1/users/', views.user_list),
]
`);
// Binary operator concatenation should produce the full path
expect(routes).toHaveLength(1);
if (routes.length > 0) {
expect(routes[0].routePath).toContain('api/');
expect(routes[0].routePath).toContain('v1/users/');
}
});
it('extracts from augmented assignment (urlpatterns += ...)', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = [
path('base/', views.base),
]
urlpatterns += [
path('extra/', views.extra),
]
`);
// Should find at least 'extra/' from the augmented assignment
expect(routes.some((r) => r.routePath === 'extra/')).toBe(true);
});
it('resolves include() to child url files via readFile', () => {
const childContent = `
from django.urls import path
from . import views
urlpatterns = [
path('list/', views.item_list),
path('<int:pk>/', views.item_detail),
]
`;
const readFile = (path: string) => {
if (path === 'items/urls.py' || path === 'app/items/urls.py') return childContent;
return null;
};
const routes = extract(
`
from django.urls import path, include
from . import views
urlpatterns = [
path('api/', include('items.urls')),
path('health/', views.health),
]
`,
'app/urls.py',
readFile,
);
// Should have: health/ and two routes from items/urls.py with prefix 'api/'
const healthRoute = routes.find((r) => r.routePath === 'health/');
expect(healthRoute).toBeDefined();
expect(healthRoute?.filePath).toBe('app/urls.py');
const prefixedRoutes = routes.filter((r) => r.prefix === 'api/');
expect(prefixedRoutes).toHaveLength(2);
expect(prefixedRoutes.some((r) => r.routePath === 'list/')).toBe(true);
expect(prefixedRoutes.some((r) => r.routePath === '<int:pk>/')).toBe(true);
expect(prefixedRoutes.every((r) => r.filePath === 'items/urls.py')).toBe(true);
});
it('resolves nested includes with accumulated prefixes', () => {
const childContent = `
from django.urls import path, include
from . import views
urlpatterns = [
path('v1/', include('v1.urls')),
path('v2/', include('v2.urls')),
]
`;
const grandchildContent = `
from django.urls import path
from . import views
urlpatterns = [
path('users/', views.user_list),
]
`;
const readFile = (path: string) => {
if (path === 'app/api/urls.py') return childContent;
if (path === 'v1/urls.py' || path === 'app/v1/urls.py') return grandchildContent;
if (path === 'v2/urls.py' || path === 'app/v2/urls.py') return grandchildContent;
return null;
};
const routes = extract(
`
from django.urls import path, include
urlpatterns = [
path('api/', include('app.api.urls')),
]
`,
'root/urls.py',
readFile,
);
// Should have deeply prefixed routes: api/v1/users/ and api/v2/users/
const prefixedRoutes = routes.filter((r) => r.prefix != null);
const hasApiV1Users = prefixedRoutes.some(
(r) => r.prefix === 'api/v1/' && r.routePath === 'users/',
);
const hasApiV2Users = prefixedRoutes.some(
(r) => r.prefix === 'api/v2/' && r.routePath === 'users/',
);
expect(hasApiV1Users).toBe(true);
expect(hasApiV2Users).toBe(true);
// Included routes should report their actual source file
const v1Routes = routes.filter((r) => r.prefix === 'api/v1/');
expect(v1Routes.every((r) => r.filePath === 'v1/urls.py')).toBe(true);
const v2Routes = routes.filter((r) => r.prefix === 'api/v2/');
expect(v2Routes.every((r) => r.filePath === 'v2/urls.py')).toBe(true);
});
it('emits both mounts when one urlconf is included under two prefixes (diamond)', () => {
const common = `
from django.urls import path
from . import views
urlpatterns = [
path('ping/', views.ping),
]
`;
const readFile = (path: string) =>
path === 'common/urls.py' || path === 'app/common/urls.py' ? common : null;
const routes = extract(
`
from django.urls import path, include
urlpatterns = [
path('v1/', include('common.urls')),
path('v2/', include('common.urls')),
]
`,
'app/urls.py',
readFile,
);
const pings = routes.filter((r) => r.routePath === 'ping/');
expect(pings).toHaveLength(2);
expect(pings.some((r) => r.prefix === 'v1/')).toBe(true);
expect(pings.some((r) => r.prefix === 'v2/')).toBe(true);
});
it('terminates on a self-referential include() cycle without re-emitting routes', () => {
const entrySource = `
from django.urls import path, include
from . import views
urlpatterns = [
path('home/', views.home),
include('app.urls'),
]
`;
// `app.urls` resolves back to the entry file — a cycle the guard must break.
const readFile = (path: string) => (path === 'app/urls.py' ? entrySource : null);
const routes = extract(entrySource, 'app/urls.py', readFile);
// The cycle is bounded (no hang) and home/ is emitted exactly once.
expect(routes.filter((r) => r.routePath === 'home/')).toHaveLength(1);
});
it('resolves include() to the project-local app, not a same-named app at the repo root (monorepo)', () => {
const rootApp = `
from django.urls import path
from . import views
urlpatterns = [path('wrong/', views.wrong)]
`;
const backendApp = `
from django.urls import path
from . import views
urlpatterns = [path('right/', views.right)]
`;
// A monorepo with a repo-root app/ AND a backend/ Django project that also
// has an app/. The manage.py at backend/ pins the project root.
const fsMap: Record<string, string> = {
'backend/manage.py': "DJANGO_SETTINGS_MODULE = 'myproj.settings'\n",
'app/urls.py': rootApp,
'backend/app/urls.py': backendApp,
};
const readFile = (p: string) =>
Object.prototype.hasOwnProperty.call(fsMap, p) ? fsMap[p] : null;
const routes = extract(
`
from django.urls import path, include
urlpatterns = [path('api/', include('app.urls'))]
`,
'backend/myproj/urls.py',
readFile,
);
// The include resolves to backend/app/urls.py (project-local), not the
// repo-root app/urls.py.
expect(routes.map((r) => r.routePath)).toEqual(['right/']);
expect(routes.map((r) => r.filePath)).toEqual(['backend/app/urls.py']);
});
it('resolves views with attribute-style references (views.function)', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = [
path('dashboard/', views.DashboardView.as_view()),
path('report/', views.report_view),
]
`);
expect(routes).toHaveLength(2);
expect(routes[0]).toMatchObject({
httpMethod: '*',
routePath: 'dashboard/',
controllerName: 'views.DashboardView.as_view()',
});
expect(routes[1]).toMatchObject({
httpMethod: '*',
routePath: 'report/',
controllerName: 'views.report_view',
});
});
it('infers HTTP method from view name suffix', () => {
const routes = extract(`
from django.urls import path
urlpatterns = [
path('users/', views.get_user),
path('users/', views.post_user),
path('users/', views.put_user),
path('users/', views.patch_user),
path('users/', views.delete_user),
]
`);
const methods = routes.map((r) => r.httpMethod);
expect(methods).toEqual(['GET', 'POST', 'PUT', 'PATCH', 'DELETE']);
});
it('handles include with tuple namespace', () => {
const childContent = `
from django.urls import path
urlpatterns = [
path('profile/', views.profile),
]
`;
const readFile = (path: string) => {
if (path === 'account/urls.py' || path === 'app/account/urls.py') return childContent;
return null;
};
const routes = extract(
`
from django.urls import path, include
urlpatterns = [
path('account/', include(('account.urls', 'app_name'), namespace='account')),
]
`,
'app/urls.py',
readFile,
);
const prefixedRoutes = routes.filter((r) => r.prefix === 'account/');
expect(prefixedRoutes.length).toBeGreaterThanOrEqual(1);
expect(prefixedRoutes.some((r) => r.filePath === 'account/urls.py')).toBe(true);
});
it('does not crash on empty urlpatterns', () => {
const routes = extract(`
from django.urls import path
urlpatterns = []
`);
expect(routes).toHaveLength(0);
});
it('skips non-urlpatterns assignments', () => {
const routes = extract(`
from django.urls import path
OTHER_LIST = [
path('not-a-route/', something),
]
urlpatterns = [
path('real/', views.real),
]
`);
expect(routes).toHaveLength(1);
expect(routes[0].routePath).toBe('real/');
});
it('extracts routes from list concatenation (urlpatterns = a + b)', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = [path('a/', views.a)] + [path('b/', views.b)]
`);
expect(routes.map((r) => r.routePath).sort()).toEqual(['a/', 'b/']);
});
it('extracts routes wrapped in format_suffix_patterns()', () => {
const routes = extract(`
from rest_framework.urlpatterns import format_suffix_patterns
from django.urls import path
from . import views
urlpatterns = format_suffix_patterns([path('a/', views.a)])
`);
expect(routes).toHaveLength(1);
expect(routes[0].routePath).toBe('a/');
});
it('extracts routes from a tuple urlpatterns', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = (path('a/', views.a),)
`);
expect(routes).toHaveLength(1);
expect(routes[0].routePath).toBe('a/');
});
it('combines a base list with an augmented concatenation (urlpatterns += a + b)', () => {
const routes = extract(`
from django.urls import path
from . import views
urlpatterns = [path('base/', views.base)]
urlpatterns += [path('x/', views.x)] + [path('y/', views.y)]
`);
expect(routes.map((r) => r.routePath).sort()).toEqual(['base/', 'x/', 'y/']);
});
it('returns no routes (without throwing) for dynamic urlpatterns like router.urls', () => {
const routes = extract(`
from rest_framework import routers
router = routers.DefaultRouter()
urlpatterns = router.urls
`);
expect(routes).toEqual([]);
});
});

16
gitnexus/test/unit/group/matching.test.ts
View file

@ -100,6 +100,22 @@ describe('runExactMatch', () => {
expect(matched).toHaveLength(2);
});
it('matches a specific-method consumer to a method-agnostic (wildcard) provider', () => {
// A Django function view is method-agnostic (provider method '*'); a POST
// consumer on the same path must still match it.
const contracts: StoredContract[] = [
makeContract('http::*::/api/items', 'provider', 'backend'),
makeContract('http::POST::/api/items', 'consumer', 'frontend'),
];
const { matched, unmatched } = runExactMatch(contracts);
expect(matched).toHaveLength(1);
expect(matched[0].from.repo).toBe('frontend');
expect(matched[0].to.repo).toBe('backend');
expect(unmatched).toHaveLength(0);
});
it('reports unmatched contracts', () => {
const contracts: StoredContract[] = [
makeContract('http::GET::/api/users', 'provider', 'backend'),