From 97c1f85e873816c32c9ea40a5f9372edbc094087 Mon Sep 17 00:00:00 2001 From: azizur100389 Date: Thu, 28 May 2026 17:18:19 +0100 Subject: [PATCH 1/2] refactor(cpp): Use function-type ADL entities (#1822) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cpp): use function-type ADL entities * test(hooks): stabilize concurrency burst reporting * Fix C++ return type capture subtag handling * Harden C++ function-type ADL extraction --------- Co-authored-by: Gergő Magyar --- .../src/core/ingestion/languages/cpp/adl.ts | 180 +++++++++++++----- .../core/ingestion/languages/cpp/captures.ts | 32 ++++ .../src/core/ingestion/scope-extractor.ts | 1 + .../app.cpp | 7 + .../cpp-adl-free-func-ref-return-strict/lib.h | 11 ++ .../cpp-adl-free-func-ref-strict/app.cpp | 7 + .../cpp-adl-free-func-ref-strict/lib.h | 11 ++ .../test/integration/resolvers/cpp.test.ts | 84 ++++---- .../test/integration/resolvers/helpers.ts | 5 + 9 files changed, 260 insertions(+), 78 deletions(-) create mode 100644 gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/app.cpp create mode 100644 gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/lib.h create mode 100644 gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/app.cpp create mode 100644 gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/lib.h diff --git a/gitnexus/src/core/ingestion/languages/cpp/adl.ts b/gitnexus/src/core/ingestion/languages/cpp/adl.ts index 0bcda9322..9c23c73cc 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/adl.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/adl.ts @@ -24,22 +24,18 @@ * V2 additionally walks class ancestors (via MRO), so base-class enclosing * namespaces also contribute associated namespaces. * - * **GitNexus approximation (not strict ISO C++ ADL):** passing a qualified - * function reference like `utils::worker` contributes `utils` to the associated - * set, enabling resolution of unqualified calls like `with_callback(utils::worker)` - * to `utils::with_callback`. Under ISO C++ `[basic.lookup.argdep]`, associated - * entities for function-type arguments come from the **parameter types and return - * type** of each function in the overload set — NOT the function's enclosing - * namespace. For `void worker()`, the standard-compliant associated set is empty. - * GitNexus instead contributes the enclosing namespace of any Function/Method - * def whose simple name matches, because it enables the dominant real-world ADL - * pattern at reasonable precision cost. + * Function-reference arguments follow ISO C++ `[basic.lookup.argdep]`: + * associated entities come from the parameter types and return type of each + * referenced function in the overload set, not from the function's enclosing + * namespace. For `void worker()`, the associated set is empty. For + * `void worker(api::Token)` or `api::Token make_token()`, `api` is associated + * through `Token`. * - * For qualified refs (e.g. `utils::worker`) the namespace is confirmed via a - * workspace lookup (only contributed when a Function/Method named `worker` exists - * in `utils`). For unqualified refs the workspace is searched for any Function - * def with that simple name. Locally-declared function-pointer variables - * (e.g. `void (*g)()`) and function parameters are excluded from this path. + * For qualified refs (e.g. `utils::worker`) the workspace lookup is restricted + * to functions/methods named `worker` in `utils`; for unqualified refs the + * workspace is searched for matching functions/methods by simple name. Locally + * declared function-pointer variables and function parameters are excluded + * from this path. * * ADL candidates are merged with ordinary unqualified-lookup candidates * in the free-call fallback before overload narrowing. @@ -70,6 +66,7 @@ import type { ParsedFile, ScopeId, SymbolDefinition } from 'gitnexus-shared'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { normalizeCppParamType } from './arity-metadata.js'; import { isCppInlineNamespaceScope } from './inline-namespaces.js'; /** @@ -97,11 +94,8 @@ export interface CppAdlArgInfo { /** When set, the arg is a potential free-function reference (not a locally- * declared function-pointer variable or function parameter). Contains the * identifier text as written in source (e.g. `"utils::worker"` or - * `"worker"`). GitNexus approximation: the function's enclosing namespace - * is contributed to the ADL associated set. For qualified refs a workspace - * lookup confirms a Function/Method with that simple name exists in the - * namespace before contributing; for unqualified refs every namespace - * containing a matching Function/Method def is contributed. */ + * `"worker"`). Resolution contributes associated namespaces from each + * referenced Function/Method def's parameter and return types. */ readonly functionRefText?: string; } @@ -207,7 +201,12 @@ export function pickCppAdlCandidates( for (const arg of args) { collectAssociatedNamespacesForAdlArg(arg, scopes, associatedNamespaces); if (arg.functionRefText !== undefined) { - collectFunctionRefNamespaces(arg.functionRefText, parsedFiles, associatedNamespaces); + collectFunctionTypeAssociatedNamespaces( + arg.functionRefText, + scopes, + parsedFiles, + associatedNamespaces, + ); } } if (associatedNamespaces.size === 0) return undefined; @@ -472,23 +471,12 @@ function findCppClassDefBySimpleName( } /** - * Contribute associated namespaces for a function-reference argument. - * - * - **Qualified refs** (`utils::worker`, `outer::inner::fn`): the namespace - * is extracted from the qualifier text (converting `::` to `.` for dot-joined - * QName matching). A workspace lookup then **verifies** that a Function or - * Method def named `worker` (the simple name after the last `::`) actually - * exists in the extracted namespace. This prevents false positives from - * namespace-qualified variables, enum values, and static data members, which - * also produce `qualified_identifier` AST nodes in tree-sitter-cpp (the - * AST node type alone does not distinguish functions from non-function names). - * - **Unqualified refs** (`worker`): the workspace is searched for any - * Function/Method def whose simple name matches. Every distinct enclosing - * namespace found is added — overloads across the same namespace produce - * a single entry; GitNexus does not select a specific overload at this stage. + * Contribute associated namespaces for a function-reference argument by walking + * the referenced overload set's parameter and return types. */ -function collectFunctionRefNamespaces( +function collectFunctionTypeAssociatedNamespaces( refText: string, + scopes: ScopeResolutionIndexes, parsedFiles: readonly ParsedFile[], out: Set, ): void { @@ -511,30 +499,130 @@ function collectFunctionRefNamespaces( for (const def of scope.ownedDefs) { if (def.type !== 'Function' && def.type !== 'Method') continue; const simple = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; - if (simple === simpleName) { - out.add(nsText); - return; // Namespace confirmed; no need to scan further files. - } + if (simple === simpleName) collectAssociatedNamespacesForFunctionDef(def, scopes, out); } } } return; } - // Unqualified: search all namespace scopes for a Function def with this - // simple name and contribute its enclosing namespace. + // Unqualified function references are approximated workspace-wide, matching + // the previous V1 lookup scope. The stricter part of this PR is what each + // overload contributes: only namespaces from parameter/return types, never + // the function's own enclosing namespace. for (const parsed of parsedFiles) { - const scopesById = new Map(); - for (const sc of parsed.scopes) scopesById.set(sc.id, sc); for (const scope of parsed.scopes) { if (scope.kind !== 'Namespace') continue; for (const def of scope.ownedDefs) { if (def.type !== 'Function' && def.type !== 'Method') continue; const simple = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; if (simple !== refText) continue; - const nsQName = computeNamespaceQName(scope, scopesById); - if (nsQName !== '') out.add(nsQName); + collectAssociatedNamespacesForFunctionDef(def, scopes, out); } } } } + +function collectAssociatedNamespacesForFunctionDef( + def: SymbolDefinition, + scopes: ScopeResolutionIndexes, + out: Set, +): void { + const parameterTypes = def.parameterTypeClasses?.map((typeClass) => typeClass.base); + for (const paramType of parameterTypes ?? def.parameterTypes ?? []) { + collectAssociatedNamespacesForFunctionTypeText(paramType, scopes, out); + } + if (def.returnType !== undefined) { + collectAssociatedNamespacesForFunctionTypeText(def.returnType, scopes, out); + } +} + +function collectAssociatedNamespacesForFunctionTypeText( + typeText: string, + scopes: ScopeResolutionIndexes, + out: Set, +): void { + for (const token of extractCppTypeNameTokens(typeText)) { + if (isIgnoredCppAdlNamespace(token.namespaceName)) continue; + addAssociatedNamespaceForClassName(token.simpleName, scopes, out); + if (token.namespaceName !== '') out.add(token.namespaceName); + } +} + +function extractCppTypeNameTokens(typeText: string): readonly { + readonly simpleName: string; + readonly namespaceName: string; +}[] { + const cleaned = normalizeCppParamType(typeText); + if (cleaned === '' || isPrimitiveCppAdlType(cleaned)) return []; + const out: { simpleName: string; namespaceName: string }[] = []; + const seen = new Set(); + const tokenSource = typeText.includes('<') ? `${cleaned} ${typeText}` : cleaned; + for (const rawToken of tokenSource.match(/[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*/g) ?? []) { + if (isPrimitiveCppAdlType(rawToken)) continue; + const segments = rawToken.split('::').filter((part) => part.length > 0); + const simpleName = segments.at(-1) ?? ''; + if (simpleName === '' || isPrimitiveCppAdlType(simpleName)) continue; + const namespaceName = segments.length > 1 ? segments.slice(0, -1).join('.') : ''; + const key = `${namespaceName}\0${simpleName}`; + if (seen.has(key)) continue; + seen.add(key); + out.push({ + simpleName, + namespaceName, + }); + } + return out; +} + +const CPP_ADL_PRIMITIVE_OR_KEYWORD_TYPES = new Set([ + 'alignas', + 'alignof', + 'auto', + 'bool', + 'char', + 'char8_t', + 'char16_t', + 'char32_t', + 'class', + 'const', + 'consteval', + 'constexpr', + 'constinit', + 'decltype', + 'double', + 'enum', + 'explicit', + 'extern', + 'float', + 'inline', + 'int', + 'long', + 'mutable', + 'noexcept', + 'null', + 'register', + 'short', + 'signed', + 'static', + 'string', + 'struct', + 'template', + 'thread_local', + 'typename', + 'union', + 'unknown', + 'unsigned', + 'void', + 'volatile', + 'wchar_t', + '...', +]); + +function isPrimitiveCppAdlType(typeText: string): boolean { + return CPP_ADL_PRIMITIVE_OR_KEYWORD_TYPES.has(typeText); +} + +function isIgnoredCppAdlNamespace(namespaceName: string): boolean { + return namespaceName === 'std' || namespaceName.startsWith('std.'); +} diff --git a/gitnexus/src/core/ingestion/languages/cpp/captures.ts b/gitnexus/src/core/ingestion/languages/cpp/captures.ts index 354f6ce4c..86384a759 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/captures.ts @@ -126,6 +126,14 @@ export function emitCppScopeCaptures( JSON.stringify(arity.parameterTypeClasses), ); } + const returnType = extractCppDeclarationReturnType(fnNode); + if (returnType !== undefined) { + grouped['@declaration.return-type'] = syntheticCapture( + '@declaration.return-type', + fnNode, + returnType, + ); + } if (hasExplicitSpecifier(fnNode)) { grouped['@declaration.is-explicit'] = syntheticCapture( '@declaration.is-explicit', @@ -417,6 +425,30 @@ export function emitCppScopeCaptures( return out; } +function extractCppDeclarationReturnType(fnNode: SyntaxNode): string | undefined { + const typeNode = fnNode.childForFieldName('type'); + if (typeNode === null) return undefined; + const funcDeclarator = findFunctionDeclarator(fnNode); + if (funcDeclarator !== null && isCppUnsupportedReturnTypeDeclarator(funcDeclarator)) { + return undefined; + } + const typeText = typeNode.text.trim(); + if (typeText !== 'auto') return typeText.length > 0 ? typeText : undefined; + if (funcDeclarator === null) return typeText; + for (let i = 0; i < funcDeclarator.namedChildCount; i++) { + const child = funcDeclarator.namedChild(i); + if (child?.type !== 'trailing_return_type') continue; + const typeDesc = child.firstNamedChild; + return typeDesc?.text.trim() || typeText; + } + return typeText; +} + +function isCppUnsupportedReturnTypeDeclarator(funcDeclarator: SyntaxNode): boolean { + const text = funcDeclarator.text; + return /\boperator\b/.test(text) || /(^|[(:\s])~\s*[A-Za-z_]\w*/.test(text); +} + /** * Walk every C++ class/struct base clause and emit `@reference.inherits` * captures for each base so scope resolution can resolve them into EXTENDS diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index 963cf4862..973d4ac76 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -1087,6 +1087,7 @@ const KNOWN_SUB_TAGS: ReadonlySet = new Set([ '@declaration.required-parameter-count', '@declaration.parameter-types', '@declaration.parameter-type-classes', + '@declaration.return-type', '@declaration.template-constraints', '@declaration.is-explicit', ]); diff --git a/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/app.cpp b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/app.cpp new file mode 100644 index 000000000..d8d768ded --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/app.cpp @@ -0,0 +1,7 @@ +#include "lib.h" + +namespace caller { + void run() { + run_callback(utils::make_token); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/lib.h b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/lib.h new file mode 100644 index 000000000..857d9cc42 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-return-strict/lib.h @@ -0,0 +1,11 @@ +#pragma once + +namespace api { + struct Token { + friend void run_callback(Token t) {} + }; +} + +namespace utils { + api::Token make_token(); +} diff --git a/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/app.cpp b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/app.cpp new file mode 100644 index 000000000..6c3eb0786 --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/app.cpp @@ -0,0 +1,7 @@ +#include "lib.h" + +namespace caller { + void run() { + run_callback(utils::worker); + } +} diff --git a/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/lib.h b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/lib.h new file mode 100644 index 000000000..9463986aa --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/cpp-adl-free-func-ref-strict/lib.h @@ -0,0 +1,11 @@ +#pragma once + +namespace api { + struct Token { + friend void run_callback(Token t) {} + }; +} + +namespace utils { + void worker(api::Token token); +} diff --git a/gitnexus/test/integration/resolvers/cpp.test.ts b/gitnexus/test/integration/resolvers/cpp.test.ts index babdaf569..a4a9a9fa8 100644 --- a/gitnexus/test/integration/resolvers/cpp.test.ts +++ b/gitnexus/test/integration/resolvers/cpp.test.ts @@ -2963,41 +2963,66 @@ describe('C++ ADL — block-scope function declaration suppresses ADL', () => { }); // --------------------------------------------------------------------------- -// ADL V2 — free-function reference args contribute their namespace. +// ADL V2 - strict function-type associated entities. // -// GitNexus approximation (not strict ISO C++ ADL): when a qualified_identifier -// like `utils::worker` is passed as an argument, GitNexus contributes the -// enclosing namespace (`utils`) to the associated set, provided a Function or -// Method named `worker` is found in the `utils` namespace at resolution time. -// Under ISO C++ [basic.lookup.argdep] the associated entities for a function-type -// argument come from the parameter types and return type of the overload set — -// NOT the function's enclosing namespace. For `void worker()`, the standard- -// compliant associated set is empty. The approximation captures the dominant -// real-world pattern (pass a utility function → find its sibling) at the cost -// of potential false positives when an unrelated function with the same simple -// name exists in the same namespace (bounded by the workspace-function lookup). +// Function-reference arguments follow strict ISO C++ ADL: GitNexus walks the +// referenced overload set's parameter and return types instead of contributing +// the referenced function's enclosing namespace. +// For `void worker()`, the associated set is empty; for `void worker(api::Token)` +// or `api::Token make_token()`, `api` is associated through `Token`. // --------------------------------------------------------------------------- -describe('C++ ADL — qualified free-function reference contributes its namespace', () => { +describe('C++ ADL - free-function reference does not contribute its namespace', () => { let result: PipelineResult; beforeAll(async () => { result = await runPipelineFromRepo(path.join(FIXTURES, 'cpp-adl-free-func-ref'), () => {}); }, 60000); - it('with_callback(utils::worker) resolves to utils::with_callback via ADL', () => { + it('with_callback(utils::worker) emits zero CALLS edges when worker has no class parameter or return type', () => { const calls = getRelationships(result, 'CALLS'); const cbCalls = calls.filter((c) => c.source === 'run' && c.target === 'with_callback'); - // Ordinary lookup inside caller::run finds nothing (no `using`, no local - // declaration). utils::worker is a qualified_identifier argument, so ADL - // contributes `utils` to the associated-namespace set. utils::with_callback - // is then discovered as the sole candidate. - expect(cbCalls.length).toBe(1); - expect(cbCalls[0].targetFilePath).toContain('utils.h'); + expect(cbCalls.length).toBe(0); }); }); -describe('C++ ADL — overloaded free-function reference does not crash', () => { +describe('C++ ADL - free-function reference contributes parameter-type associated namespace', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'cpp-adl-free-func-ref-strict'), + () => {}, + ); + }, 60000); + + it('run_callback(utils::worker) resolves hidden friend through worker(api::Token)', () => { + const calls = getRelationships(result, 'CALLS'); + const cbCalls = calls.filter((c) => c.source === 'run' && c.target === 'run_callback'); + expect(cbCalls.length).toBe(1); + expect(cbCalls[0].targetFilePath).toContain('lib.h'); + }); +}); + +describe('C++ ADL - free-function reference contributes return-type associated namespace', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo( + path.join(FIXTURES, 'cpp-adl-free-func-ref-return-strict'), + () => {}, + ); + }, 60000); + + it('run_callback(utils::make_token) resolves hidden friend through api::Token return type', () => { + const calls = getRelationships(result, 'CALLS'); + const cbCalls = calls.filter((c) => c.source === 'run' && c.target === 'run_callback'); + expect(cbCalls.length).toBe(1); + expect(cbCalls[0].targetFilePath).toContain('lib.h'); + }); +}); + +describe('C++ ADL - overloaded free-function reference stays strict', () => { let result: PipelineResult; beforeAll(async () => { @@ -3007,15 +3032,10 @@ describe('C++ ADL — overloaded free-function reference does not crash', () => ); }, 60000); - it('with_callback(utils::worker) with overloaded utils::worker still resolves utils::with_callback via ADL', () => { + it('with_callback(utils::worker) with overloaded utils::worker still emits zero CALLS edges', () => { const calls = getRelationships(result, 'CALLS'); const cbCalls = calls.filter((c) => c.source === 'run' && c.target === 'with_callback'); - // utils::worker has two overloads (worker() and worker(int)). V1 - // simplification: contribute the namespace if ANY overload exists in the - // workspace, regardless of which one would be selected. The namespace - // `utils` is still added, and utils::with_callback is discovered. - expect(cbCalls.length).toBe(1); - expect(cbCalls[0].targetFilePath).toContain('utils.h'); + expect(cbCalls.length).toBe(0); }); }); @@ -3035,10 +3055,10 @@ describe('C++ ADL — namespace-qualified variable arg does NOT contribute names // data::value is a namespace-qualified integer variable. tree-sitter-cpp // produces a qualified_identifier AST node regardless of whether `value` // denotes a function, variable, enum, or static member. The GitNexus guard - // in collectFunctionRefNamespaces verifies that a Function/Method named - // `value` exists in the `data` namespace before contributing it. Since - // `data::value` is an int variable, `data` is never added to the associated - // set, so data::process is never found as an ADL candidate. + // in collectFunctionTypeAssociatedNamespaces verifies that a Function/Method + // named `value` exists in the `data` namespace before walking any function + // type. Since `data::value` is an int variable, no function type is walked, + // so data::process is never found as an ADL candidate. expect(processCalls.length).toBe(0); }); }); diff --git a/gitnexus/test/integration/resolvers/helpers.ts b/gitnexus/test/integration/resolvers/helpers.ts index bd17483ba..f91349d91 100644 --- a/gitnexus/test/integration/resolvers/helpers.ts +++ b/gitnexus/test/integration/resolvers/helpers.ts @@ -363,6 +363,11 @@ const LEGACY_RESOLVER_PARITY_EXPECTED_FAILURES: Readonly Date: Fri, 29 May 2026 02:04:19 +0800 Subject: [PATCH 2/2] feat(ingestion): resolve FastAPI include_router(prefix=...) cross-file routes (#1877) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(ingestion): resolve FastAPI include_router(prefix=...) cross-file routes FastAPI sub-route files declare paths via @router. while the entry file mounts the router with app.include_router(, prefix='/x'). Previously both the ingestion-layer Route graph nodes and the group-layer ExtractedContract URLs lost the cross-file prefix, breaking provider <-> consumer matching. Ingestion layer: - parse-worker emits routerIncludes / routerImports + decoratorReceiver - parsing-processor / parse-impl thread the new fields and aggregate prefixesByModule across chunks; decorator routes whose receiver is 'router' are duplicated once per matching prefix - routes.ts joins prefix via normalizeExtractedRoutePath Group layer: - HttpLanguagePlugin gains an optional prepareRepo() pre-pass and a repoContext arg to scan(); python.ts builds prefixesByModule and falls back to the bare path when no entry matches - http-route-extractor caches one repoContext per plugin Tests: - 3 new http-route-extractor cases (attr / named-import / no-prefix) - ParseWorkerResult literals in 3 test files updated to the new shape Co-Authored-By: Claude Opus 4.7 * fix(ingestion,group): address PR #1877 review — relative imports, cross-package collisions, host names, ingestion tests Follow-ups to the FastAPI `include_router(prefix=...)` cross-file fix based on PR #1877's automated production-readiness review. Three correctness gaps and one test coverage gap addressed: 1. Relative-import support in the worker regex (FINDING 2) `FROM_IMPORT_ROUTER_RE` now accepts module paths starting with a `.` (e.g. `from .calls import router as calls_router`). The previous `[A-Za-z_][\w.]*` rejected leading dots and silently dropped every relative-import Shape-B include — a real pattern from the PR description's own motivating example. The matching helpers now strip leading dots before keying so absolute and relative imports collapse to the same module key. 2. Cross-package same-name module collisions (FINDING 3) Two-tier module keying replaces the previous basename-only key: • short key — `users` (file basename without `.py`) • long key — `api/users` (parent dir + stem) `prefixesByLongKey` is consulted first and only falls back to `prefixesByShortKey` when no long-key match is available. Both the ingestion pipeline (parse-impl.ts) and the group extractor (http-patterns/python.ts) carry the same scheme so the graph nodes and HTTP contracts agree on which prefix applies. New protocol field `ExtractedRouterModuleAlias` (parse-worker → parsing-processor → parse-impl) lets Shape-A `.include_router(.router, prefix='/x')` calls promote to a long key when the same file imports `` via `from import `. Without this, `api/users.py` and `admin/users.py` collided on the basename `users` and the admin file's routes inherited the `/users` prefix that was only meant for `api/users.py`. 3. Non-`app` host variable names (FINDING 4) The group-layer `INCLUDE_ROUTER_*_PATTERNS` queries pinned the host identifier to the literal `"app"` and dropped every `application = FastAPI()` / `api = FastAPI()` pattern — the constraint was redundant given that the call shape (`include_router` invoked with a router argument and a `prefix=` keyword) is already specific enough. The pin is removed; the ingestion regex was already unrestricted. 4. Ingestion-layer regression tests (FINDING 1) The previous PR added group-layer tests (`http-route-extractor.test.ts`) but zero in-tree tests for the ingestion path. Two new suites pin the worker → parse-impl → routes flow: - `test/unit/fastapi-router-bindings.test.ts` (23 cases): `extractFastAPIRouterBindings()` is split into a stand-alone module so it can be unit-tested without booting a worker thread, then pinned for regex shape, two-tier key emission, relative-import support, and negative cases. - `test/integration/fastapi-prefix-pipeline.test.ts` (5 cases) plus `test/fixtures/fastapi-prefix-app/` — runs the full `runPipelineFromRepo()` against a realistic multi-package fixture (containing both `api/users.py` and `admin/users.py`) and inspects the resulting `Route` graph nodes for cross-file prefix joining and absence of cross-package bleed. Verification - `npx tsc --noEmit`: pass - PR-touched test suites (6 files / 117 cases): all green - `npx prettier --check`: pass on touched files - `npx eslint`: 0 errors on touched files Cache / compatibility The new `routerModuleAliases?` field on `ParseWorkerResult` and `routerModuleAliases` on `WorkerExtractedData` are optional / guarded with `?? []`, so historical parse-cache entries continue to load without forced re-scan. Refs PR #1877. * refactor(ingestion): move fastapi-router-bindings out of workers/ — pure module, not a worker Addresses @magyargergo's `CHANGES_REQUESTED` review on PR #1877: > Sorry I just found that we are introducing a new worker in the PR. `gitnexus/src/core/ingestion/workers/fastapi-router-bindings.ts` was a **pure-function module** — it never imported `worker_threads` or `parentPort`, never spawned a worker, and was never registered as a worker entry. It was placed in `workers/` purely because it was split out of `workers/parse-worker.ts` to make its functions unit-testable without booting a worker thread (parse-worker is itself the worker entry and cannot be loaded from the main thread). To remove the misleading directory placement: • The implementation moves to `gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts`, alongside the other framework-specific route extractors (`expo`, `nextjs`, `php`, `laravel`, `middleware`, `response-shapes`). • `workers/parse-worker.ts` keeps a thin re-export so the worker entry can keep using `extractFastAPIRouterBindings` directly. The re-export now carries an explicit comment stating that the imported file is **not** a worker and that the `workers/` directory deliberately hosts only true worker entries (`parse-worker.ts`, `worker-pool.ts`, `quarantine.ts`). • The new file's leading docstring opens with "NOT A WORKER" and explains why it exists where it does. • The unit test (`test/unit/fastapi-router-bindings.test.ts`) is updated to import from the new path. No behaviour change. The function body, signatures, and exported types are identical. Verification • `npx tsc --noEmit`: pass • `npx tsc` (dist rebuild): pass • `test/unit/fastapi-router-bindings.test.ts` (23 cases): all green • `test/integration/fastapi-prefix-pipeline.test.ts` (5 cases): all green • `test/unit/group/http-route-extractor.test.ts` (63 cases): all green • `npx prettier --check` on touched files: pass • `npx eslint` on touched files: 0 errors Refs PR #1877. * refactor(ingestion): drop parse-worker re-exports; consumers import router types directly from route-extractors Addresses @magyargergo's two remaining review comments on PR #1877: 1. **`gitnexus/src/core/ingestion/workers/parse-worker.ts:247`** — "Can you please remove them and update the call sites?" The `export type { ExtractedRouterInclude, ExtractedRouterImport, ExtractedRouterModuleAlias } from '../route-extractors/...'` block in parse-worker.ts is gone. The remaining `import type {…}` is purely local — used only to type the corresponding fields on `ParseWorkerResult` below — and the leading comment now says so explicitly ("this file does NOT re-export them"). The `extractFastAPIRouterBindings` symbol is also no longer re-exported from parse-worker.ts; it's still imported here so the worker entry can call it per file, but downstream consumers must reach it via `route-extractors/fastapi-router-bindings` directly. Call sites updated: - `gitnexus/src/core/ingestion/parsing-processor.ts` - `gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts` Both files now `import type { ExtractedRouterInclude, ExtractedRouterImport, ExtractedRouterModuleAlias }` directly from `route-extractors/fastapi-router-bindings.js`. The worker types they still need (`ParseWorkerResult`, `ExtractedToolDef`, etc.) keep coming from `workers/parse-worker.js`. The unit + integration tests already imported from the new path, so no test changes were required. 2. **`gitnexus/src/core/ingestion/parsing-processor.ts:168`** — suggested simplification: for (const item of result.routerIncludes ?? []) allRouterIncludes.push(item); for (const item of result.routerImports ?? []) allRouterImports.push(item); for (const item of result.routerModuleAliases ?? []) allRouterModuleAliases.push(item); Applied verbatim. Replaces the previous `if (result.…) for …` guards. The cache-compat semantics are unchanged — historical parse-cache entries that lack these fields still load cleanly, the new form just spells the fallback inline. No behavior change, no tests touched, no public API change. Verification • `npx tsc --noEmit`: pass • `npx tsc` (dist rebuild): pass • PR-touched test suites (6 files / 117 cases): all green • `npx prettier --check` on touched files: pass • `npx eslint` on touched files: 0 errors Refs PR #1877. * refactor(ingestion): hoist fastapi-router-bindings type imports to top of parse-worker.ts Move the `import type { ExtractedRouterInclude, ExtractedRouterImport, ExtractedRouterModuleAlias }` block to the top of the file with the other type imports, and drop the comment that previously sat next to ExtractedDecoratorRoute. --------- Co-authored-by: henry Co-authored-by: Claude Opus 4.7 Co-authored-by: Gergő Magyar --- .../group/extractors/http-patterns/python.ts | 402 +++++++++++++++++- .../group/extractors/http-patterns/types.ts | 35 +- .../group/extractors/http-route-extractor.ts | 62 ++- .../src/core/ingestion/parsing-processor.ts | 20 + .../ingestion/pipeline-phases/parse-impl.ts | 168 ++++++++ .../core/ingestion/pipeline-phases/routes.ts | 4 +- .../fastapi-router-bindings.ts | 275 ++++++++++++ .../core/ingestion/workers/parse-worker.ts | 73 ++++ .../fastapi-prefix-app/admin/users.py | 14 + .../fixtures/fastapi-prefix-app/api/calls.py | 8 + .../fixtures/fastapi-prefix-app/api/users.py | 13 + .../test/fixtures/fastapi-prefix-app/main.py | 13 + .../fixtures/fastapi-prefix-app/relative.py | 8 + .../fastapi-prefix-pipeline.test.ts | 125 ++++++ .../parse-impl-quarantine-cache-skip.test.ts | 2 + .../test/unit/fastapi-router-bindings.test.ts | 287 +++++++++++++ .../unit/group/http-route-extractor.test.ts | 78 ++++ .../test/unit/incremental-parse-cache.test.ts | 2 + .../unit/parse-impl-worker-lazy-cache.test.ts | 4 +- 19 files changed, 1566 insertions(+), 27 deletions(-) create mode 100644 gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts create mode 100644 gitnexus/test/fixtures/fastapi-prefix-app/admin/users.py create mode 100644 gitnexus/test/fixtures/fastapi-prefix-app/api/calls.py create mode 100644 gitnexus/test/fixtures/fastapi-prefix-app/api/users.py create mode 100644 gitnexus/test/fixtures/fastapi-prefix-app/main.py create mode 100644 gitnexus/test/fixtures/fastapi-prefix-app/relative.py create mode 100644 gitnexus/test/integration/fastapi-prefix-pipeline.test.ts create mode 100644 gitnexus/test/unit/fastapi-router-bindings.test.ts diff --git a/gitnexus/src/core/group/extractors/http-patterns/python.ts b/gitnexus/src/core/group/extractors/http-patterns/python.ts index 1667d0de9..5dc314d35 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/python.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/python.ts @@ -6,7 +6,7 @@ import { unquoteLiteral, type LanguagePatterns, } from '../tree-sitter-scanner.js'; -import type { HttpDetection, HttpLanguagePlugin } from './types.js'; +import type { HttpDetection, HttpLanguagePlugin, RepoContext } from './types.js'; /** * Python HTTP plugin. Handles: @@ -29,9 +29,13 @@ const FASTAPI_VERBS: Record = { patch: 'PATCH', }; -// ─── Provider: FastAPI @app.get/... ────────────────────────────────── -const FASTAPI_PATTERNS = compilePatterns({ - name: 'python-fastapi', +// ─── Provider: FastAPI @app. / @router. ────────────────── +// Two separate patterns so we can tag detections by decorator object. +// Only `@router.*` detections participate in `include_router(prefix=)` +// path-prefix joining (see `PythonRepoContext` + `joinPrefix`); `@app.*` +// routes already carry their final path verbatim. +const FASTAPI_APP_PATTERNS = compilePatterns({ + name: 'python-fastapi-app', language: Python, patterns: [ { @@ -48,6 +52,138 @@ const FASTAPI_PATTERNS = compilePatterns({ ], } satisfies LanguagePatterns>); +const FASTAPI_ROUTER_PATTERNS = compilePatterns({ + name: 'python-fastapi-router', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (decorator + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "router") + attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$")) + arguments: (argument_list . (string) @path))) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── include_router(, prefix='/x') across the repo ──────── +// Two shapes are common: +// app.include_router(assistant.router, prefix='/ai') +// app.include_router(my_router, prefix='/ai') +// The first names the originating module via `.router`; the second +// references a name imported into the host file. We capture both. +const INCLUDE_ROUTER_ATTR_PATTERNS = compilePatterns({ + name: 'python-fastapi-include-router-attr', + language: Python, + patterns: [ + { + meta: {}, + // Match any `.include_router(.router, ..., prefix='/x')` + // call. We deliberately do NOT pin `` to the literal name `app` + // — production code routinely uses `api`, `application`, `asgi_app`, + // etc. The shape (`include_router` invoked with a router argument and + // a `prefix=` keyword) is specific enough on its own; restricting the + // host produces false negatives without removing meaningful false + // positives. + query: ` + (call + function: (attribute + attribute: (identifier) @incl (#eq? @incl "include_router")) + arguments: (argument_list + (attribute + object: (identifier) @router_module + attribute: (identifier) @router_attr (#eq? @router_attr "router")) + (keyword_argument + name: (identifier) @kw (#eq? @kw "prefix") + value: (string) @prefix))) + `, + }, + ], +} satisfies LanguagePatterns>); + +const INCLUDE_ROUTER_NAME_PATTERNS = compilePatterns({ + name: 'python-fastapi-include-router-name', + language: Python, + patterns: [ + { + meta: {}, + // Same `` rationale as INCLUDE_ROUTER_ATTR_PATTERNS — see above. + query: ` + (call + function: (attribute + attribute: (identifier) @incl (#eq? @incl "include_router")) + arguments: (argument_list + (identifier) @router_name + (keyword_argument + name: (identifier) @kw (#eq? @kw "prefix") + value: (string) @prefix))) + `, + }, + ], +} satisfies LanguagePatterns>); + +// `from .api.assistant import router` style — used together with +// INCLUDE_ROUTER_NAME so we can map a local name back to its module +// path, then back to the file the router was declared in. +const FROM_IMPORT_ROUTER_PATTERNS = compilePatterns({ + name: 'python-fastapi-from-import-router', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (dotted_name (identifier) @imported (#eq? @imported "router"))) + `, + }, + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (aliased_import + name: (dotted_name (identifier) @imported (#eq? @imported "router")) + alias: (identifier) @alias)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// `from api import users` / `from api import users as u` — module-level +// imports where the imported name is itself the module that owns +// `.router`. Lets Shape A (`.include_router(.router, …)`) +// look up the full package path of `` and pin the prefix onto the +// exact file (`api/users.py`) rather than every file basenamed `users.py`. +const FROM_IMPORT_MODULE_PATTERNS = compilePatterns({ + name: 'python-fastapi-from-import-module', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (dotted_name (identifier) @imported)) + `, + }, + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (aliased_import + name: (dotted_name (identifier) @imported) + alias: (identifier) @alias)) + `, + }, + ], +} satisfies LanguagePatterns>); + // ─── Consumer: requests.get/post/... ────────────────────────────────── const REQUESTS_VERB_PATTERNS = compilePatterns({ name: 'python-requests-verb', @@ -447,15 +583,226 @@ const HTTPX_ASYNC_CLIENT_GENERIC_PATTERNS = compilePatterns({ ], } satisfies LanguagePatterns>); +// ─── prepareRepo: build router-module → prefix list map ───────────── +// +// FastAPI splits route declarations across files: handler decorators +// live in `api/.py` while `app.include_router(.router, +// prefix='/ai')` lives in `main.py`. A per-file plugin scan therefore +// can't see the prefix that ought to be applied. We resolve this by +// running a one-shot pre-pass over the repo: for every file that +// hosts an `app.include_router(...)` we record the module the router +// came from (either via `module.router` attribute access, or via a +// local name resolved through a `from import router` import) +// together with the prefix string. At scan time the python plugin +// looks up the current file's module key in this map and joins each +// prefix with each `@router.` decorator's path. +// +// Multiple prefixes for the same module are kept and emitted as +// separate detections — this matches FastAPI's behaviour when one +// router is mounted under several prefixes. +// +// Module keying is two-tiered to avoid prefix bleed between same-named +// files in different packages (e.g. `api/users.py` vs `admin/users.py`): +// • short key — file basename without `.py` (`users`) +// • long key — `/` (`api/users`) +// The pre-pass records prefixes against the long key whenever the import +// site supplies enough context (`from api.users import router as ...` → +// long key `api/users`); otherwise it falls back to the short key. +// At scan time the file's own long key is consulted first; only when no +// long-key entry targets this file do we look up the short key. This +// preserves the previous coarse-grained behaviour where context is +// missing while delivering precision wherever the import statement +// gives us a multi-segment module path. +interface PythonRepoContext { + /** `/` → set of prefixes (precise, package-aware) */ + prefixesByLongKey: Map>; + /** stem only → set of prefixes (basename fallback, may collide) */ + prefixesByShortKey: Map>; +} + +/** Strip `.py` and return the bare basename (e.g. `api/users.py` → `users`). */ +function fileShortKey(rel: string): string { + const slash = rel.lastIndexOf('/'); + const file = slash >= 0 ? rel.slice(slash + 1) : rel; + return file.endsWith('.py') ? file.slice(0, -3) : file; +} + +/** + * Long key for a `.py` file: parent directory + stem, joined with `/`. + * Files at the repo root return the empty string (no parent), in which + * case callers should fall back to the short key. + */ +function fileLongKey(rel: string): string { + const noExt = rel.endsWith('.py') ? rel.slice(0, -3) : rel; + const lastSlash = noExt.lastIndexOf('/'); + if (lastSlash < 0) return ''; + const beforeLast = noExt.slice(0, lastSlash); + const stem = noExt.slice(lastSlash + 1); + const prevSlash = beforeLast.lastIndexOf('/'); + const parent = prevSlash >= 0 ? beforeLast.slice(prevSlash + 1) : beforeLast; + return `${parent}/${stem}`; +} + +/** Last `.`-separated segment of a (possibly relative) module path. */ +function lastSegmentOfDotted(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const dot = stripped.lastIndexOf('.'); + return dot >= 0 ? stripped.slice(dot + 1) : stripped; +} + +/** + * Last two `.`-separated segments of a (possibly relative) module path + * joined with `/`, e.g. `api.users` → `api/users`. Single-segment paths + * and pure-dot inputs return the empty string; callers should fall back + * to the short key in that case. + */ +function lastTwoSegmentsAsLongKey(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const last = stripped.lastIndexOf('.'); + if (last <= 0) return ''; + const beforeLast = stripped.slice(0, last); + const stem = stripped.slice(last + 1); + const prev = beforeLast.lastIndexOf('.'); + const parent = prev >= 0 ? beforeLast.slice(prev + 1) : beforeLast; + return `${parent}/${stem}`; +} + +function recordPrefix(target: Map>, key: string, prefix: string): void { + const set = target.get(key) ?? new Set(); + set.add(prefix); + target.set(key, set); +} + +function buildPythonRepoContext( + files: string[], + parser: Parser, + readFile: (rel: string) => string | null, + parseSource: (parser: Parser, src: string) => Parser.Tree | null, +): PythonRepoContext { + const prefixesByLongKey = new Map>(); + const prefixesByShortKey = new Map>(); + + // Pre-pass over .py files. We deliberately run this even on files + // that don't contain `include_router` — the cost of an extra parse + // is bounded by the file count, and detecting `include_router` + // beforehand would require its own grep/scan. + for (const rel of files) { + if (!rel.endsWith('.py')) continue; + const src = readFile(rel); + if (!src) continue; + if (!src.includes('include_router')) continue; + parser.setLanguage(Python); + const tree = parseSource(parser, src); + if (!tree) continue; + + // Local name → (short, long) map for the current file, populated + // from `from import router [as ]` statements. The + // alias (or 'router' when there is no alias) is the local name + // we'll later see passed to `.include_router`. + interface LocalImport { + moduleShort: string; + moduleLong: string; + } + const localNameToModule = new Map(); + for (const m of runCompiledPatterns(FROM_IMPORT_ROUTER_PATTERNS, tree)) { + const moduleNode = m.captures.module; + const aliasNode = m.captures.alias; + const importedNode = m.captures.imported; + if (!moduleNode || !importedNode) continue; + const localName = aliasNode?.text ?? importedNode.text; + const moduleShort = lastSegmentOfDotted(moduleNode.text); + if (!moduleShort) continue; + const moduleLong = lastTwoSegmentsAsLongKey(moduleNode.text); + localNameToModule.set(localName, { moduleShort, moduleLong }); + } + + // Module-alias map: name imported from a multi-segment package → + // long key. Lets Shape A look up the precise file for `.router` + // even when `` collides with another package's basename. + const localNameToModuleAlias = new Map(); + for (const m of runCompiledPatterns(FROM_IMPORT_MODULE_PATTERNS, tree)) { + const moduleNode = m.captures.module; + const importedNode = m.captures.imported; + const aliasNode = m.captures.alias; + if (!moduleNode || !importedNode) continue; + // Skip the `router` shape — already handled by FROM_IMPORT_ROUTER_PATTERNS + // above and stored under its router-aware semantics. + if (importedNode.text === 'router') continue; + const moduleLong = lastTwoSegmentsAsLongKey(`${moduleNode.text}.${importedNode.text}`); + if (!moduleLong) continue; + const localName = aliasNode?.text ?? importedNode.text; + localNameToModuleAlias.set(localName, moduleLong); + } + + // Shape A: `.include_router(.router, prefix='/x')`. + // The call site gives us only a short module name. We promote to a + // long key when the same file imports `` via either + // `from import ` (recorded in `localNameToModuleAlias` + // — the typical pattern) or, less commonly, a router-aware import + // statement. Only fall back to the basename short key when neither + // alias is available. + for (const m of runCompiledPatterns(INCLUDE_ROUTER_ATTR_PATTERNS, tree)) { + const modNode = m.captures.router_module; + const prefixNode = m.captures.prefix; + if (!modNode || !prefixNode) continue; + const prefix = unquoteLiteral(prefixNode.text); + if (prefix === null) continue; + const moduleShort = modNode.text; + const aliasLong = localNameToModuleAlias.get(moduleShort); + const sameFileImport = localNameToModule.get(moduleShort); + const longKey = aliasLong ?? sameFileImport?.moduleLong; + if (longKey) { + recordPrefix(prefixesByLongKey, longKey, prefix); + } else { + recordPrefix(prefixesByShortKey, moduleShort, prefix); + } + } + + // Shape B: `.include_router(my_router, prefix='/x')` — resolve + // `my_router` via the import map built above. Whenever the import + // statement supplied a multi-segment module path the long key is + // recorded, eliminating cross-package collisions. + for (const m of runCompiledPatterns(INCLUDE_ROUTER_NAME_PATTERNS, tree)) { + const nameNode = m.captures.router_name; + const prefixNode = m.captures.prefix; + if (!nameNode || !prefixNode) continue; + const localImp = localNameToModule.get(nameNode.text); + if (!localImp) continue; + const prefix = unquoteLiteral(prefixNode.text); + if (prefix === null) continue; + if (localImp.moduleLong) { + recordPrefix(prefixesByLongKey, localImp.moduleLong, prefix); + } else { + recordPrefix(prefixesByShortKey, localImp.moduleShort, prefix); + } + } + } + + return { prefixesByLongKey, prefixesByShortKey }; +} + +function joinPrefix(prefix: string, route: string): string { + // Mirror FastAPI's path joining: trim trailing slash off prefix, + // ensure exactly one leading slash on the result. + const p = prefix.replace(/\/+$/, ''); + const r = route.startsWith('/') ? route : `/${route}`; + return `${p}${r}`; +} export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'python-http', language: Python, - scan(tree) { + prepareRepo({ files, parser, readFile, parseSource }): RepoContext { + return buildPythonRepoContext(files, parser, readFile, parseSource); + }, + scan(tree, repoContext, fileRel) { const out: HttpDetection[] = []; const httpxAsyncClients = collectHttpxAsyncClients(tree); + const ctx = repoContext as PythonRepoContext | undefined; - // Providers: FastAPI - for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) { + // Providers: FastAPI @app.("/path") — already absolute path. + for (const match of runCompiledPatterns(FASTAPI_APP_PATTERNS, tree)) { const methodNode = match.captures.method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; @@ -473,6 +820,47 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { }); } + // Providers: FastAPI @router.("/path") — must be joined + // with the prefix(es) declared at the include_router site. When + // no prefix is found we still emit the unprefixed path so this + // change is strictly additive vs. the prior @app-only behaviour; + // when the same router is mounted under multiple prefixes we emit + // one detection per prefix. + for (const match of runCompiledPatterns(FASTAPI_ROUTER_PATTERNS, tree)) { + const methodNode = match.captures.method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const httpMethod = FASTAPI_VERBS[methodNode.text]; + if (!httpMethod) continue; + const rawPath = unquoteLiteral(pathNode.text); + if (rawPath === null) continue; + + // Long key first (precise, package-aware), short key as fallback. + // Mirrors the ingestion-side resolution in parse-impl.ts so the + // graph nodes and group contracts agree on which prefix applies. + const longKey = fileRel ? fileLongKey(fileRel) : ''; + const longPrefixes = longKey ? ctx?.prefixesByLongKey.get(longKey) : undefined; + const shortKey = fileRel ? fileShortKey(fileRel) : ''; + const shortPrefixes = + longPrefixes || !shortKey ? undefined : ctx?.prefixesByShortKey.get(shortKey); + const prefixSet = longPrefixes ?? shortPrefixes; + const paths = + prefixSet && prefixSet.size > 0 + ? [...prefixSet].map((p) => joinPrefix(p, rawPath)) + : [rawPath]; + + for (const p of paths) { + out.push({ + role: 'provider', + framework: 'fastapi', + method: httpMethod, + path: p, + name: null, + confidence: 0.8, + }); + } + } + // Consumers: requests. for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) { const methodNode = match.captures.method; diff --git a/gitnexus/src/core/group/extractors/http-patterns/types.ts b/gitnexus/src/core/group/extractors/http-patterns/types.ts index 6df0ede28..e1c85369d 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/types.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/types.ts @@ -51,15 +51,48 @@ export interface HttpDetection { * `LanguagePatterns.language` in `tree-sitter-scanner.ts` — the * grammar modules export different shapes. */ +/** + * Per-repo state a plugin can build during a `prepareRepo` pass before + * any per-file `scan` is invoked. The orchestrator threads this opaque + * value back into each `scan` call so plugins can resolve cross-file + * facts (e.g. FastAPI `app.include_router(prefix=...)` mappings live + * in `main.py` but apply to handlers declared in `api/*.py`). + * + * Plugins that have no cross-file state can omit `prepareRepo` and + * receive `undefined`. + */ +export type RepoContext = unknown; + export interface HttpLanguagePlugin { /** Human-readable plugin name for diagnostics. */ name: string; /** tree-sitter grammar object (passed to the shared parser). */ language: unknown; + /** + * Optional pre-pass: walk the relevant files in the repo and produce + * an opaque context that `scan` can use to resolve cross-file facts. + * Implementations must not throw — return undefined on any error so + * the orchestrator falls back to context-less scanning. + */ + prepareRepo?(args: { + repoPath: string; + files: string[]; + parser: Parser; + readFile: (rel: string) => string | null; + parseSource: (parser: Parser, src: string) => Parser.Tree | null; + }): RepoContext | undefined; /** * Scan a parsed tree and return zero or more HTTP detections. Plugins * must not throw — they should swallow per-match errors so a single * malformed construct does not abort the whole file. + * + * `repoContext` is whatever the plugin's `prepareRepo` produced (or + * `undefined` if there is no `prepareRepo`). + * + * `fileRel` is the repo-relative path of the file being scanned; + * plugins that resolve cross-file facts (e.g. FastAPI router prefix + * joining) need it to key into `repoContext`. Optional so existing + * single-file plugins can keep their unary `scan(tree)` shape. */ - scan(tree: Parser.Tree): HttpDetection[]; + scan(tree: Parser.Tree, repoContext?: RepoContext, fileRel?: string): HttpDetection[]; } diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index 54aeb9150..37237242f 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -160,7 +160,36 @@ export class HttpRouteExtractor implements ContractExtractor { // both graph-assisted enrichment and source-scan emission. const parser = new Parser(); const cachedDetections = new Map(); - const getDetections = (rel: string): HttpDetection[] => { + + // Per-plugin cross-file context (e.g. Python's FastAPI router → + // include_router(prefix=...) map). Built lazily on first + // `getDetections` call for a file the plugin handles, scoped to the + // file list returned by `getScannedFiles`. Stored by plugin name so + // a repo with multiple languages keeps each plugin's context + // independent. + const repoContextByPlugin = new Map(); + const ensureRepoContext = async ( + plugin: ReturnType, + ): Promise => { + if (!plugin || typeof plugin.prepareRepo !== 'function') return undefined; + if (repoContextByPlugin.has(plugin.name)) return repoContextByPlugin.get(plugin.name); + try { + const ctx = plugin.prepareRepo({ + repoPath, + files: await getScannedFiles(), + parser, + readFile: (rel) => readSafe(repoPath, rel), + parseSource: (p, src) => parseSourceSafe(p, src), + }); + repoContextByPlugin.set(plugin.name, ctx); + return ctx; + } catch { + repoContextByPlugin.set(plugin.name, undefined); + return undefined; + } + }; + + const getDetections = async (rel: string): Promise => { const cached = cachedDetections.get(rel); if (cached) return cached; const plugin = getPluginForFile(rel); @@ -168,6 +197,7 @@ export class HttpRouteExtractor implements ContractExtractor { cachedDetections.set(rel, []); return []; } + const repoContext = await ensureRepoContext(plugin); const content = readSafe(repoPath, rel); if (!content) { cachedDetections.set(rel, []); @@ -176,7 +206,7 @@ export class HttpRouteExtractor implements ContractExtractor { try { parser.setLanguage(plugin.language); const tree = parseSourceSafe(parser, content); - const detections = plugin.scan(tree); + const detections = plugin.scan(tree, repoContext, rel); cachedDetections.set(rel, detections); return detections; } catch { @@ -200,14 +230,14 @@ export class HttpRouteExtractor implements ContractExtractor { // by graph edges; the glob and per-file parse results are cached above. const providers = this.mergeGraphAndSourceContracts( graphProviders, - this.extractProvidersSourceScan(await getScannedFiles(), getDetections), + await this.extractProvidersSourceScan(await getScannedFiles(), getDetections), ); const graphConsumers = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : []; const consumers = this.mergeGraphAndSourceContracts( graphConsumers, - this.extractConsumersSourceScan(await getScannedFiles(), getDetections), + await this.extractConsumersSourceScan(await getScannedFiles(), getDetections), ); return [...providers, ...consumers]; @@ -232,7 +262,7 @@ export class HttpRouteExtractor implements ContractExtractor { private async extractProvidersGraph( db: CypherExecutor, - getDetections: (rel: string) => HttpDetection[], + getDetections: (rel: string) => Promise, ): Promise { const out: ExtractedContract[] = []; let rows: Record[]; @@ -254,7 +284,7 @@ export class HttpRouteExtractor implements ContractExtractor { // helpers — tree-sitter gives both pieces of information // structurally. Always run the lookup: even when method is set by // `methodFromRouteReason`, we still need the handler name. - const detections = filePath ? getDetections(filePath) : []; + const detections = filePath ? await getDetections(filePath) : []; const providerDetections = detections.filter((d) => d.role === 'provider'); let handlerName: string | null = null; const normalizedRoute = normalizeHttpPath(routePath); @@ -331,13 +361,13 @@ export class HttpRouteExtractor implements ContractExtractor { // ─── Source-scan providers ───────────────────────────────────────── - private extractProvidersSourceScan( + private async extractProvidersSourceScan( files: string[], - getDetections: (rel: string) => HttpDetection[], - ): ExtractedContract[] { + getDetections: (rel: string) => Promise, + ): Promise { const out: ExtractedContract[] = []; for (const rel of files) { - const detections = getDetections(rel); + const detections = await getDetections(rel); for (const d of detections) { if (d.role !== 'provider') continue; const pathNorm = normalizeHttpPath(d.path); @@ -366,7 +396,7 @@ export class HttpRouteExtractor implements ContractExtractor { private async extractConsumersGraph( db: CypherExecutor, - getDetections: (rel: string) => HttpDetection[], + getDetections: (rel: string) => Promise, ): Promise { const out: ExtractedContract[] = []; let rows: Record[]; @@ -382,7 +412,7 @@ export class HttpRouteExtractor implements ContractExtractor { let method = 'GET'; // Prefer the plugin's detected method if we can find a matching // fetch/axios call in the same file. - const detections = filePath ? getDetections(filePath) : []; + const detections = filePath ? await getDetections(filePath) : []; // Symmetric to the provider path: if multiple consumer calls in // the same file share the same normalized path (e.g. a GET // fetch AND a POST fetch to `/api/orders`), `.find()` silently @@ -436,13 +466,13 @@ export class HttpRouteExtractor implements ContractExtractor { // ─── Source-scan consumers ───────────────────────────────────────── - private extractConsumersSourceScan( + private async extractConsumersSourceScan( files: string[], - getDetections: (rel: string) => HttpDetection[], - ): ExtractedContract[] { + getDetections: (rel: string) => Promise, + ): Promise { const out: ExtractedContract[] = []; for (const rel of files) { - const detections = getDetections(rel); + const detections = await getDetections(rel); for (const d of detections) { if (d.role !== 'consumer') continue; const pathNorm = normalizeConsumerPath(d.path); diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 39d16461a..6b69f75c3 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -55,6 +55,11 @@ import type { ExtractedORMQuery, FetchWrapperDef, } from './workers/parse-worker.js'; +import type { + ExtractedRouterImport, + ExtractedRouterInclude, + ExtractedRouterModuleAlias, +} from './route-extractors/fastapi-router-bindings.js'; import { getTreeSitterBufferSize, getTreeSitterContentByteLength, @@ -72,6 +77,9 @@ export interface WorkerExtractedData { fetchCalls: ExtractedFetchCall[]; fetchWrapperDefs: FetchWrapperDef[]; decoratorRoutes: ExtractedDecoratorRoute[]; + routerIncludes: ExtractedRouterInclude[]; + routerImports: ExtractedRouterImport[]; + routerModuleAliases: ExtractedRouterModuleAlias[]; toolDefs: ExtractedToolDef[]; ormQueries: ExtractedORMQuery[]; constructorBindings: FileConstructorBindings[]; @@ -114,6 +122,9 @@ export const mergeChunkResults = ( const allFetchCalls: ExtractedFetchCall[] = []; const allFetchWrapperDefs: FetchWrapperDef[] = []; const allDecoratorRoutes: ExtractedDecoratorRoute[] = []; + const allRouterIncludes: ExtractedRouterInclude[] = []; + const allRouterImports: ExtractedRouterImport[] = []; + const allRouterModuleAliases: ExtractedRouterModuleAlias[] = []; const allToolDefs: ExtractedToolDef[] = []; const allORMQueries: ExtractedORMQuery[] = []; const allConstructorBindings: FileConstructorBindings[] = []; @@ -152,6 +163,9 @@ export const mergeChunkResults = ( for (const item of result.fetchCalls) allFetchCalls.push(item); for (const item of result.fetchWrapperDefs ?? []) allFetchWrapperDefs.push(item); for (const item of result.decoratorRoutes) allDecoratorRoutes.push(item); + for (const item of result.routerIncludes ?? []) allRouterIncludes.push(item); + for (const item of result.routerImports ?? []) allRouterImports.push(item); + for (const item of result.routerModuleAliases ?? []) allRouterModuleAliases.push(item); for (const item of result.toolDefs) allToolDefs.push(item); if (result.ormQueries) for (const item of result.ormQueries) allORMQueries.push(item); for (const item of result.constructorBindings) allConstructorBindings.push(item); @@ -169,6 +183,9 @@ export const mergeChunkResults = ( fetchCalls: allFetchCalls, fetchWrapperDefs: allFetchWrapperDefs, decoratorRoutes: allDecoratorRoutes, + routerIncludes: allRouterIncludes, + routerImports: allRouterImports, + routerModuleAliases: allRouterModuleAliases, toolDefs: allToolDefs, ormQueries: allORMQueries, constructorBindings: allConstructorBindings, @@ -210,6 +227,9 @@ const processParsingWithWorkers = async ( fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], + routerIncludes: [], + routerImports: [], + routerModuleAliases: [], toolDefs: [], ormQueries: [], constructorBindings: [], diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index a37d040e6..dc8a8ab00 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -63,6 +63,11 @@ import type { FileConstructorBindings, FetchWrapperDef, } from '../workers/parse-worker.js'; +import type { + ExtractedRouterImport, + ExtractedRouterInclude, + ExtractedRouterModuleAlias, +} from '../route-extractors/fastapi-router-bindings.js'; import type { ExtractedHeritage } from '../model/heritage-map.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import type { PipelineOptions } from '../pipeline.js'; @@ -357,6 +362,9 @@ export async function runChunkedParseAndResolve( const allFetchWrapperDefs: FetchWrapperDef[] = []; const allExtractedRoutes: ExtractedRoute[] = []; const allDecoratorRoutes: ExtractedDecoratorRoute[] = []; + const allRouterIncludes: ExtractedRouterInclude[] = []; + const allRouterImports: ExtractedRouterImport[] = []; + const allRouterModuleAliases: ExtractedRouterModuleAlias[] = []; const allToolDefs: ExtractedToolDef[] = []; const allORMQueries: ExtractedORMQuery[] = []; const deferredWorkerCalls: ExtractedCall[] = []; @@ -675,6 +683,15 @@ export async function runChunkedParseAndResolve( if (chunkWorkerData.decoratorRoutes?.length) { for (const item of chunkWorkerData.decoratorRoutes) allDecoratorRoutes.push(item); } + if (chunkWorkerData.routerIncludes?.length) { + for (const item of chunkWorkerData.routerIncludes) allRouterIncludes.push(item); + } + if (chunkWorkerData.routerImports?.length) { + for (const item of chunkWorkerData.routerImports) allRouterImports.push(item); + } + if (chunkWorkerData.routerModuleAliases?.length) { + for (const item of chunkWorkerData.routerModuleAliases) allRouterModuleAliases.push(item); + } if (chunkWorkerData.toolDefs?.length) { for (const item of chunkWorkerData.toolDefs) allToolDefs.push(item); } @@ -1085,6 +1102,157 @@ export async function runChunkedParseAndResolve( importCtx.index = EMPTY_INDEX; importCtx.normalizedFileList = []; + // FastAPI router-prefix resolution (cross-file). + // + // Workers emit two kinds of records per Python file: + // • `routerIncludes` — every `app.include_router(, prefix='/x')` + // site, where `routerExpr` is either `.router` (Shape A) or a + // bare local name (Shape B). + // • `routerImports` — every `from import router [as ]`, + // mapping a local name to a module key (the basename of the source + // module). These let us resolve Shape-B router includes back to the + // module that defines the router. + // + // We build `module-basename → Set` and then walk + // `allDecoratorRoutes`: any decorator route emitted from a `router.` + // decorator inherits its file-basename's prefix. When a router is mounted + // under multiple prefixes we duplicate the route entry, mirroring FastAPI's + // runtime behaviour. + if (allRouterIncludes.length > 0 && allDecoratorRoutes.length > 0) { + // Group `routerImports` by file so we can resolve Shape-B locals against + // imports declared in the SAME file as the include_router call. We carry + // both the short module key (file basename) and, when available, the long + // key (`/`) so cross-package same-name modules don't blur + // their prefixes together. `routerModuleAliases` lifts the same long-key + // information for Shape-A includes whose receiving module was imported + // via `from import `. + interface LocalImport { + moduleKey: string; + moduleKeyLong: string | undefined; + } + const importsByFile = new Map>(); + for (const imp of allRouterImports) { + let m = importsByFile.get(imp.filePath); + if (!m) { + m = new Map(); + importsByFile.set(imp.filePath, m); + } + m.set(imp.localName, { + moduleKey: imp.moduleKey, + moduleKeyLong: imp.moduleKeyLong, + }); + } + // Module-alias map keyed by file: `localName` (the imported module + // identifier in this file) → long key. Shape-A receivers like + // `users.router` are matched against this map; the long key, when + // present, scopes the prefix to the precise source file. + const moduleAliasesByFile = new Map>(); + for (const alias of allRouterModuleAliases) { + let m = moduleAliasesByFile.get(alias.filePath); + if (!m) { + m = new Map(); + moduleAliasesByFile.set(alias.filePath, m); + } + m.set(alias.localName, alias.moduleKeyLong); + } + + // Two parallel maps: long-key (precise) and short-key (basename + // fallback). Long-key entries are preferred when the file's own long + // key matches; short-key entries match any file with that basename and + // remain the fallback when no long key is known (e.g. Shape A includes + // without a corresponding import statement). + const prefixesByLongKey = new Map>(); + const prefixesByShortKey = new Map>(); + + const recordPrefix = (target: Map>, key: string, prefix: string): void => { + let set = target.get(key); + if (!set) { + set = new Set(); + target.set(key, set); + } + set.add(prefix); + }; + + for (const inc of allRouterIncludes) { + // Shape A: `.router`. The worker emits `routerExpr` already + // including `.router`, so split it back. We only know a short module + // key here — the call site doesn't carry the dotted package path. If + // the same file imports `` via `from import ` + // (recorded in `allRouterModuleAliases`) we promote to a long key. + const dotIdx = inc.routerExpr.indexOf('.router'); + if (dotIdx > 0) { + const moduleShort = inc.routerExpr.slice(0, dotIdx); + const aliasLong = moduleAliasesByFile.get(inc.filePath)?.get(moduleShort); + if (aliasLong) { + recordPrefix(prefixesByLongKey, aliasLong, inc.prefix); + } else { + recordPrefix(prefixesByShortKey, moduleShort, inc.prefix); + } + continue; + } + + // Shape B: bare local name. Resolve through this file's imports. The + // import line gives us a long key whenever the module path was multi- + // segment, so cross-package collisions are eliminated for Shape B. + const localImp = importsByFile.get(inc.filePath)?.get(inc.routerExpr); + if (!localImp) continue; + if (localImp.moduleKeyLong) { + recordPrefix(prefixesByLongKey, localImp.moduleKeyLong, inc.prefix); + } else { + recordPrefix(prefixesByShortKey, localImp.moduleKey, inc.prefix); + } + } + + if (prefixesByLongKey.size > 0 || prefixesByShortKey.size > 0) { + const fileLongKey = (rel: string): string => { + // Strip `.py`, then take the last two path segments. `api/users.py` + // → `api/users`. Files at the repo root return the empty string, + // which can never match a long-key entry (those always include a + // parent directory) and so fall through to the short-key lookup. + const noExt = rel.endsWith('.py') ? rel.slice(0, -3) : rel; + const lastSlash = noExt.lastIndexOf('/'); + if (lastSlash < 0) return ''; + const beforeLast = noExt.slice(0, lastSlash); + const stem = noExt.slice(lastSlash + 1); + const prevSlash = beforeLast.lastIndexOf('/'); + const parent = prevSlash >= 0 ? beforeLast.slice(prevSlash + 1) : beforeLast; + return `${parent}/${stem}`; + }; + + const fileShortKey = (rel: string): string => { + const slash = rel.lastIndexOf('/'); + const file = slash >= 0 ? rel.slice(slash + 1) : rel; + return file.endsWith('.py') ? file.slice(0, -3) : file; + }; + + const expanded: ExtractedDecoratorRoute[] = []; + for (const dr of allDecoratorRoutes) { + if (dr.decoratorReceiver !== 'router' || !dr.filePath.endsWith('.py')) { + expanded.push(dr); + continue; + } + // Long-key lookup first; only fall back to the short key when no + // long-key prefix targets this file. This avoids prefix leakage + // between e.g. `api/users.py` and `admin/users.py`. + const longKey = fileLongKey(dr.filePath); + const longPrefixes = longKey ? prefixesByLongKey.get(longKey) : undefined; + const shortPrefixes = longPrefixes + ? undefined + : prefixesByShortKey.get(fileShortKey(dr.filePath)); + const prefixes = longPrefixes ?? shortPrefixes; + if (!prefixes || prefixes.size === 0) { + expanded.push(dr); + continue; + } + for (const prefix of prefixes) { + expanded.push({ ...dr, prefix }); + } + } + allDecoratorRoutes.length = 0; + for (const dr of expanded) allDecoratorRoutes.push(dr); + } + } + return { exportedTypeMap, allFetchCalls, diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index a87b9576d..8c0a67ac5 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -198,7 +198,6 @@ export const routesPhase: PipelinePhase = { } } - const ensureSlash = (path: string) => (path.startsWith('/') ? path : '/' + path); let duplicateRoutes = 0; const namedRouteRegistry = new Map(); const addRoute = (url: string, entry: RouteEntry) => { @@ -220,7 +219,8 @@ export const routesPhase: PipelinePhase = { } } for (const dr of allDecoratorRoutes) { - addRoute(ensureSlash(dr.routePath), { + const url = normalizeExtractedRoutePath(dr.routePath, dr.prefix ?? null); + addRoute(url, { filePath: dr.filePath, source: `decorator-${dr.decoratorName}`, }); diff --git a/gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts b/gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts new file mode 100644 index 000000000..34e2ded04 --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts @@ -0,0 +1,275 @@ +/** + * FastAPI router-prefix detection — pure functions, no worker thread. + * + * NOT A WORKER. This module exports plain synchronous functions; it + * does not import `worker_threads`, does not call `parentPort`, and + * is not a new worker entry point. It lives next to the other route + * extractors (expo, nextjs, php, laravel) for that reason. + * + * The implementation was historically inlined in `workers/parse-worker.ts`, + * but parse-worker.ts is itself the worker entry point and cannot be + * loaded from the main thread (see the same constraint used by + * `test/unit/call-attribution-issue-1166.test.ts`). Splitting the pure + * extraction here lets unit tests import the function directly without + * booting a worker, satisfying DoD §2.7. + * + * Worker phase is per-file, so the heavy cross-file resolution lives in + * `pipeline-phases/parse-impl.ts`. Here we only extract two raw record + * kinds and let the pipeline aggregate them across files: + * + * • {@link ExtractedRouterInclude} — every + * `.include_router(, prefix='/x')` site, where + * `` is either `.router` (Shape A) or a bare + * local name (Shape B). `` is intentionally unconstrained: + * production code uses `app`, `api`, `application`, `asgi_app`, + * etc., and the call shape (`include_router` invoked with a + * `prefix=` keyword) is specific enough on its own. + * + * • {@link ExtractedRouterImport} — every + * `from import router [as ]`, captured for both + * absolute and relative module paths (`from .calls import …`). + * parse-impl uses the imports to resolve Shape-B local names back + * to the file that declares the router. + * + * Module keying is two-tiered to avoid prefix bleed between same-named + * files in different packages (e.g. `api/users.py` vs `admin/users.py`): + * + * • short key — basename without `.py` (`users`) + * • long key — `/` (`api/users`) + * + * Imports always carry the short key and, when the module path was + * multi-segment, also the long key. parse-impl matches against the + * long key first and falls back to the short key, so cross-package + * collisions are eliminated for Shape B and minimised for Shape A. + * + * The functions in this module are pure (no Worker / parentPort + * dependency) so they can be unit-tested directly without booting a + * worker thread. + */ + +/** + * One `.include_router(, prefix='/x')` site. + * + * `routerExpr` is the raw text of the first argument — either + * `.router` (Shape A) or a bare local name (Shape B). + * parse-impl resolves Shape B against {@link ExtractedRouterImport} + * records emitted by the same file. + */ +export interface ExtractedRouterInclude { + filePath: string; + routerExpr: string; + prefix: string; + lineNumber: number; +} + +/** + * One `from import router [as ]` discovered in a + * Python file. + * + * `moduleKey` is the short key (last `.`-segment of the module path, + * e.g. `api.users` → `users`). `moduleKeyLong` is the long key (last + * two segments joined with `/`, e.g. `api/users`); it is the empty + * string / undefined when the import is single-segment (e.g. + * `from users import router`) or pure-dots (e.g. `from . import + * router`). The long key, when present, gives parse-impl a precise + * way to bind a Shape-B `include_router` call to exactly one Python + * file even when other packages contain a same-named module. + */ +export interface ExtractedRouterImport { + filePath: string; + localName: string; + moduleKey: string; + moduleKeyLong?: string; +} + +/** + * One `from import ` discovered in a Python file + * where `` is later used as a Shape-A include receiver + * (`.include_router(.router, prefix='/x')`). Without + * this record parse-impl would have to fall back to the short key + * ``, which collides between e.g. `api/users.py` and + * `admin/users.py`. The record carries the long key + * (`/`) so parse-impl can pin the prefix onto the + * exact source file. + * + * Only emitted when the import path was multi-segment (a single + * `from users import users` would yield no long key). All fields + * carry the same module-key semantics as + * {@link ExtractedRouterImport}. + */ +export interface ExtractedRouterModuleAlias { + filePath: string; + /** Local name in the importing file (== imported name or its alias). */ + localName: string; + /** Long key (`/`) — non-empty for every emitted record. */ + moduleKeyLong: string; +} + +// `.include_router(.router, ..., prefix='/x')` (Shape A). +// `` is left unrestricted — common production names include +// `app`, `api`, `application`, `asgi_app`. Pinning to the literal +// `app` would silently drop these. +const INCLUDE_ROUTER_ATTR_RE = + /\b(?:[A-Za-z_][\w.]*)\.include_router\s*\(\s*([A-Za-z_][\w]*)\.router\b[^)]*?\bprefix\s*=\s*(['"])([^'"]*)\2/g; + +// `.include_router(, ..., prefix='/x')` (Shape B). +const INCLUDE_ROUTER_NAME_RE = + /\b(?:[A-Za-z_][\w.]*)\.include_router\s*\(\s*([A-Za-z_][\w]*)\b[^)]*?\bprefix\s*=\s*(['"])([^'"]*)\2/g; + +// Module path: a sequence of dots (`.`, `..`, `...`) for "current +// package" imports, OR an optional leading-dot prefix followed by a +// dotted identifier (`api.users`, `.api.users`, `..siblings.users`). +// The latter is the common case and the only one we can map back to +// a module stem. +const FROM_IMPORT_ROUTER_RE = /^\s*from\s+(\.+|\.*[A-Za-z_][\w.]*)\s+import\s+([^#\n]+)/gm; + +/** + * Last `.`-separated segment of a (possibly relative) Python module + * path. Strips any leading dots first so `from .api.assistant import + * …` and `from api.assistant import …` both yield `assistant`. + * Pure-dot inputs (`.`, `..`) have no segment and return the empty + * string; callers should skip empty results. + */ +export function lastDottedSegment(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const dot = stripped.lastIndexOf('.'); + return dot >= 0 ? stripped.slice(dot + 1) : stripped; +} + +/** + * Last two `.`-separated segments of a (possibly relative) module + * path joined with `/`, e.g. `api.users` → `api/users`. Mirrors the + * long-key shape used for files (`api/users.py` → `api/users`). + * Returns the empty string when no parent segment is available + * (single-segment imports or pure dots); callers should fall back + * to the short key in that case. + */ +export function lastTwoSegmentsAsPath(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const last = stripped.lastIndexOf('.'); + if (last <= 0) return ''; + const beforeLast = stripped.slice(0, last); + const stem = stripped.slice(last + 1); + const prev = beforeLast.lastIndexOf('.'); + const parent = prev >= 0 ? beforeLast.slice(prev + 1) : beforeLast; + return `${parent}/${stem}`; +} + +/** + * Scan a single Python file's source text for FastAPI router + * `include_router` sites and `from import router` imports, + * appending raw records to the supplied collectors. + * + * `outModuleAliases` is optional: when supplied, every multi-segment + * `from import ` (other than `router` itself) is recorded + * as a module alias so parse-impl can pin Shape-A + * `.include_router(...)` calls onto the exact module file. When + * omitted, the function preserves the pre-existing behaviour and + * skips the alias collection — this keeps the function signature + * back-compat with older callers (and the parse-cache replay path). + */ +export function extractFastAPIRouterBindings( + filePath: string, + content: string, + outIncludes: ExtractedRouterInclude[], + outImports: ExtractedRouterImport[], + outModuleAliases?: ExtractedRouterModuleAlias[], +): void { + if (!content.includes('include_router') && !content.includes('router')) return; + + // `from import router [as ]`. We capture every name + // in the import list. `router` (with or without an `as` alias) maps + // to outImports; every other name lands in outModuleAliases when a + // long key is available, so Shape-A `.router` includes can be + // pinned to the exact module file. + if (content.includes(' import ')) { + FROM_IMPORT_ROUTER_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = FROM_IMPORT_ROUTER_RE.exec(content)) !== null) { + const moduleText = m[1]; + const importList = m[2]; + const moduleShort = lastDottedSegment(moduleText); + if (!moduleShort) continue; + // Long key for the imported MODULE itself (used by router + // imports — `from api.users import router` sets + // `moduleKeyLong = api/users`). + const moduleLong = lastTwoSegmentsAsPath(moduleText); + // Strip surrounding parens / trailing whitespace; split on + // commas. (Multiline import groups already have their newlines + // present in the captured list.) + const cleaned = importList.replace(/[()]/g, '').trim(); + for (const rawPart of cleaned.split(',')) { + const part = rawPart.trim(); + if (!part) continue; + + // `router` or `router as foo` → ExtractedRouterImport. + const routerAlias = /^router(?:\s+as\s+([A-Za-z_]\w*))?$/.exec(part); + if (routerAlias) { + const localName = routerAlias[1] ?? 'router'; + outImports.push({ + filePath, + localName, + moduleKey: moduleShort, + ...(moduleLong ? { moduleKeyLong: moduleLong } : {}), + }); + continue; + } + + // Any other `` or ` as ` — recorded as a + // module alias so parse-impl can pin Shape-A includes. The + // long key here is computed against the IMPORTED MODULE PATH + // (`.`), not the package path that `` + // was imported FROM. `from api import users` therefore yields + // `api/users`, the same long key as the file it points at. + if (!outModuleAliases) continue; + const otherAlias = /^([A-Za-z_]\w*)(?:\s+as\s+([A-Za-z_]\w*))?$/.exec(part); + if (!otherAlias) continue; + const importedName = otherAlias[1]; + const localName = otherAlias[2] ?? importedName; + const aliasLong = lastTwoSegmentsAsPath(`${moduleText}.${importedName}`); + if (!aliasLong) continue; + outModuleAliases.push({ + filePath, + localName, + moduleKeyLong: aliasLong, + }); + } + } + } + + if (!content.includes('include_router')) return; + + // Shape A: `.include_router(.router, prefix='/x')`. + INCLUDE_ROUTER_ATTR_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = INCLUDE_ROUTER_ATTR_RE.exec(content)) !== null) { + outIncludes.push({ + filePath, + routerExpr: `${m[1]}.router`, + prefix: m[3], + lineNumber: content.substring(0, m.index).split('\n').length, + }); + } + + // Shape B: `.include_router(my_router, prefix='/x')`. + // Resolution to a module key happens in parse-impl using + // outImports from the same file. + INCLUDE_ROUTER_NAME_RE.lastIndex = 0; + while ((m = INCLUDE_ROUTER_NAME_RE.exec(content)) !== null) { + // Skip cases that already matched Shape A — INCLUDE_ROUTER_NAME_RE + // is intentionally permissive and would re-capture `.router` + // as the bare name `mod`. Discriminate by re-checking the + // immediate source around the captured argument position. + const argStart = m.index + m[0].indexOf(m[1]); + const dotProbe = content.slice(argStart + m[1].length, argStart + m[1].length + 8); + if (/^\s*\.\s*router/.test(dotProbe)) continue; + outIncludes.push({ + filePath, + routerExpr: m[1], + prefix: m[3], + lineNumber: content.substring(0, m.index).split('\n').length, + }); + } +} diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index f092d4a1a..6ebc10782 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -23,6 +23,11 @@ import { import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; import type { SymbolTableReader } from '../model/symbol-table.js'; import type { ExtractedHeritage } from '../model/heritage-map.js'; +import type { + ExtractedRouterInclude, + ExtractedRouterImport, + ExtractedRouterModuleAlias, +} from '../route-extractors/fastapi-router-bindings.js'; /** Language grammar type accepted by Parser.setLanguage(). */ type TreeSitterLanguage = Parameters[0]; @@ -209,6 +214,19 @@ export interface ExtractedDecoratorRoute { httpMethod: string; decoratorName: string; lineNumber: number; + /** + * Decorator receiver identifier (e.g. `router` for `@router.get(...)`, + * `app` for `@app.get(...)`). Used by parse-impl to decide which routes + * participate in `include_router(prefix=...)` joining. + */ + decoratorReceiver?: string; + /** + * FastAPI `app.include_router(prefix='/x')` prefix that applies to + * this route. Filled by parse-impl after cross-file aggregation; the + * routes phase joins it via `normalizeExtractedRoutePath`. `null` / + * absent ⇒ no prefix applies. + */ + prefix?: string | null; } export interface ExtractedToolDef { @@ -275,6 +293,18 @@ export interface ParseWorkerResult { fetchCalls: ExtractedFetchCall[]; fetchWrapperDefs: FetchWrapperDef[]; decoratorRoutes: ExtractedDecoratorRoute[]; + routerIncludes: ExtractedRouterInclude[]; + routerImports: ExtractedRouterImport[]; + /** + * Optional. `from import ` records from Python files + * where `` is later used as a Shape-A include receiver + * (`.include_router(.router, prefix='/x')`). parse-impl + * uses these to promote Shape-A short-key entries to long keys, so + * same-named modules in different packages don't share prefixes. + * Optional for cache backward compatibility (older cache entries + * predate the field; consumers must guard with `if (… ?? [])`). + */ + routerModuleAliases?: ExtractedRouterModuleAlias[]; toolDefs: ExtractedToolDef[]; ormQueries: ExtractedORMQuery[]; constructorBindings: FileConstructorBindings[]; @@ -740,6 +770,9 @@ const processBatch = ( fetchCalls: [], fetchWrapperDefs: [], decoratorRoutes: [], + routerIncludes: [], + routerImports: [], + routerModuleAliases: [], toolDefs: [], ormQueries: [], constructorBindings: [], @@ -968,6 +1001,18 @@ export function extractORMQueries( } } +// ============================================================================ +// FastAPI router prefix detection (Python) +// ============================================================================ +// +// The extraction lives in `../route-extractors/fastapi-router-bindings` +// (a pure-function module — NOT a worker, no `worker_threads`, no +// `parentPort`). It's imported here only so the worker entry can call it +// per file; this module does not re-export it. Downstream consumers +// import the function and its types directly from `route-extractors/`. + +import { extractFastAPIRouterBindings } from '../route-extractors/fastapi-router-bindings.js'; + const processFileGroup = ( files: ParseWorkerInput[], language: SupportedLanguages, @@ -1200,6 +1245,7 @@ const processFileGroup = ( if (captureMap['decorator'] && captureMap['decorator.name']) { const decoratorName = captureMap['decorator.name'].text; const decoratorArg = captureMap['decorator.arg']?.text; + const decoratorReceiver = captureMap['decorator.receiver']?.text; const decoratorNode = captureMap['decorator']; // Store by the decorator's end line — the definition follows immediately after fileDecorators.set(decoratorNode.endPosition.row, { @@ -1219,6 +1265,7 @@ const processFileGroup = ( httpMethod, decoratorName, lineNumber: decoratorNode.startPosition.row + lineOffset, + ...(decoratorReceiver ? { decoratorReceiver } : {}), }); } // MCP/RPC tool detection: @mcp.tool(), @app.tool(), @server.tool() @@ -1994,6 +2041,20 @@ const processFileGroup = ( // Extract ORM queries (Prisma, Supabase) extractORMQueries(file.path, parseContent, result.ormQueries); + // Extract FastAPI include_router(prefix=...) and `from import router` + // sites. parse-impl aggregates these into a per-module prefix map and + // injects the resolved prefix onto each ExtractedDecoratorRoute that + // came from a `@router.` decorator. Python-only. + if (language === SupportedLanguages.Python) { + extractFastAPIRouterBindings( + file.path, + parseContent, + result.routerIncludes, + result.routerImports, + (result.routerModuleAliases ??= []), + ); + } + // Vue: emit CALLS edges for components used in