diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 5f6ccf404..8ab8342d6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -48,7 +48,7 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: languages: ${{ matrix.language }} queries: security-and-quality @@ -69,6 +69,6 @@ jobs: - '**/test/fixtures/**' - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: category: '/language:${{ matrix.language }}' diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 5d564db79..5c24cfa29 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -53,6 +53,6 @@ jobs: retention-days: 5 - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/upload-sarif@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: sarif_file: results.sarif diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 6ac2c4d08..76f995c2a 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -76,7 +76,7 @@ jobs: exit-code: '0' - name: Upload to Security tab - uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/upload-sarif@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: sarif_file: trivy-${{ matrix.image.name }}.sarif category: trivy-${{ matrix.image.name }} diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index 42effcdf7..8b121b7c1 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -76,7 +76,7 @@ jobs: continue-on-error: true - name: Upload SARIF - uses: github/codeql-action/upload-sarif@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4.35.4 + uses: github/codeql-action/upload-sarif@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4.35.5 with: sarif_file: zizmor.sarif category: zizmor diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index f31a7ef5e..b9719e46b 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -11,12 +11,12 @@ "@langchain/anthropic": "^1.3.29", "@langchain/core": "^1.1.44", "@langchain/google-genai": "^2.1.30", - "@langchain/langgraph": "^1.2.9", + "@langchain/langgraph": "^1.3.2", "@langchain/ollama": "^1.2.6", "@langchain/openai": "^1.4.5", "@sigma/edge-curve": "^3.1.0", "@tailwindcss/vite": "^4.3.0", - "axios": "^1.16.0", + "axios": "^1.16.1", "d3": "^7.9.0", "dompurify": "^3.4.3", "gitnexus-shared": "file:../gitnexus-shared", @@ -48,12 +48,12 @@ }, "devDependencies": { "@babel/types": "^7.29.0", - "@playwright/test": "^1.58.2", + "@playwright/test": "^1.60.0", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/dompurify": "^3.2.0", - "@types/node": "^25.6.0", + "@types/node": "^25.9.1", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@types/react-syntax-highlighter": "^15.5.13", @@ -1393,13 +1393,14 @@ } }, "node_modules/@langchain/langgraph": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.2.9.tgz", - "integrity": "sha512-3c7BtGycHC2v9p6w/Hv8L7kEl1YnZYOQTDJtmAp3knk6JOedO7d2bYP3y0SRyhv5orUEGf/KGvx8ZsB/ideP7g==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.3.2.tgz", + "integrity": "sha512-SL7Ktsr681R7da+1b2MVOWEbaCoFJOXEJPTGOjg4JIG4C7quWbTYC8DzxhcCxte6D/8cGp0rYDBnbKLXEpNqlA==", "license": "MIT", "dependencies": { - "@langchain/langgraph-checkpoint": "^1.0.1", - "@langchain/langgraph-sdk": "~1.8.9", + "@langchain/langgraph-checkpoint": "^1.0.2", + "@langchain/langgraph-sdk": "~1.9.4", + "@langchain/protocol": "^0.0.15", "@standard-schema/spec": "1.1.0", "uuid": "^10.0.0" }, @@ -1407,7 +1408,7 @@ "node": ">=18" }, "peerDependencies": { - "@langchain/core": "^1.1.40", + "@langchain/core": "^1.1.44", "zod": "^3.25.32 || ^4.2.0", "zod-to-json-schema": "^3.x" }, @@ -1418,9 +1419,9 @@ } }, "node_modules/@langchain/langgraph-checkpoint": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.0.1.tgz", - "integrity": "sha512-HM0cJLRpIsSlWBQ/xuDC67l52SqZ62Bh2Y61DX+Xorqwoh5e1KxYvfCD7GnSTbWWhjBOutvnR0vPhu4orFkZfw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.0.2.tgz", + "integrity": "sha512-F4E5Tr0nt8FGghgdscJtHw+ABzChOHeI80R7Y1pjIHdiJom6c2ieo76vL+FWiny80JmoGqhrVAEIWrw0cXKPxg==", "license": "MIT", "dependencies": { "uuid": "^10.0.0" @@ -1429,7 +1430,7 @@ "node": ">=18" }, "peerDependencies": { - "@langchain/core": "^1.0.1" + "@langchain/core": "^1.1.44" } }, "node_modules/@langchain/langgraph-checkpoint/node_modules/uuid": { @@ -1446,27 +1447,25 @@ } }, "node_modules/@langchain/langgraph-sdk": { - "version": "1.8.10", - "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.8.10.tgz", - "integrity": "sha512-wrB3rkRw5KAmsqezwvKP3midT4qJrV6Hj9XJMYo+cbvXC4HYpSAmyY/VriSyeTFRbLG/OP/pY2Yz+9Z54nSaXQ==", + "version": "1.9.9", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.9.tgz", + "integrity": "sha512-aiWHbmqxWj5sAMwFsaB3eSGQvKpMbUKTlt9zbAC0T7IiFqDYUWi9gJUGsTdvJutAfB3P/NzC4s8ETUtUQEUlYg==", "license": "MIT", "dependencies": { + "@langchain/protocol": "^0.0.15", "@types/json-schema": "^7.0.15", "p-queue": "^9.0.1", "p-retry": "^7.1.1", "uuid": "^13.0.0" }, "peerDependencies": { - "@langchain/core": "^1.1.16", + "@langchain/core": "^1.1.44", "react": "^18 || ^19", "react-dom": "^18 || ^19", "svelte": "^4.0.0 || ^5.0.0", "vue": "^3.0.0" }, "peerDependenciesMeta": { - "@langchain/core": { - "optional": true - }, "react": { "optional": true }, @@ -1488,9 +1487,9 @@ "license": "MIT" }, "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { - "version": "9.2.0", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.2.0.tgz", - "integrity": "sha512-dWgLE8AH0HjQ9fe74pUkKkvzzYT18Inp4zra3lKHnnwqGvcfcUBrvF2EAVX+envufDNBOzpPq/IBUONDbI7+3g==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.0.tgz", + "integrity": "sha512-7NED7xhQ74Ngp4JP/2e0VZHp7vSWfJfqeiR92jPgxsz6m0Se4P03YoTKa9dDXyZ3r6P616gUXttrB6nnHYKang==", "license": "MIT", "dependencies": { "eventemitter3": "^5.0.4", @@ -1516,9 +1515,9 @@ } }, "node_modules/@langchain/langgraph-sdk/node_modules/uuid": { - "version": "13.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.1.tgz", - "integrity": "sha512-9ezox2roIft6ExBVTVqibSd5dc5/47Sw/uY6b4SjQUT2TzQ0tltNquWA46y4xPQmdZYqvnio22SgWd41M86+jw==", + "version": "13.0.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-13.0.2.tgz", + "integrity": "sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==", "funding": [ "https://github.com/sponsors/broofa", "https://github.com/sponsors/ctavan" @@ -1587,6 +1586,12 @@ "@langchain/core": "^1.1.42" } }, + "node_modules/@langchain/protocol": { + "version": "0.0.15", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.15.tgz", + "integrity": "sha512-MllvbpMjqHevUm+v94M422mH7XKN+wGCvJRBVROTWBotEDOATYB4Ktk2UheYP859y9o2LlhtPek5t1T9eyfAbQ==", + "license": "MIT" + }, "node_modules/@mapbox/node-pre-gyp": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-2.0.3.tgz", @@ -1684,13 +1689,13 @@ } }, "node_modules/@playwright/test": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.58.2.tgz", - "integrity": "sha512-akea+6bHYBBfA9uQqSYmlJXn61cTa+jbO87xVLCWbTqbWadRVmhxlXATaOjOgcBaWU4ePo0wB41KMFv3o35IXA==", + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.60.0.tgz", + "integrity": "sha512-O71yZIbAh/PxDMNGns37GHBIfrVkEVyn+AXyIa5dOTfb4/xNvRWV+Vv/NMbNCtODB/pO7vLlF2OTmMVLhmr7Ag==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright": "1.58.2" + "playwright": "1.60.0" }, "bin": { "playwright": "cli.js" @@ -2854,13 +2859,13 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.6.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", - "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", + "version": "25.9.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz", + "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", "devOptional": true, "license": "MIT", "dependencies": { - "undici-types": "~7.19.0" + "undici-types": ">=7.24.0 <7.24.7" } }, "node_modules/@types/prismjs": { @@ -3414,16 +3419,42 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.0.tgz", - "integrity": "sha512-6hp5CwvTPlN2A31g5dxnwAX0orzM7pmCRDLnZSX772mv8WDqICwFjowHuPs04Mc8deIld1+ejhtaMn5vp6b+1w==", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.1.tgz", + "integrity": "sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.5", + "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, + "node_modules/axios/node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "license": "MIT", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/axios/node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "license": "MIT", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/bail": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", @@ -5444,9 +5475,9 @@ } }, "node_modules/is-network-error": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.1.tgz", - "integrity": "sha512-6QCxa49rQbmUWLfk0nuGqzql9U8uaV2H6279bRErPBHe/109hCzsLUBUHfbEtvLIHBd6hyXbgedBSHevm43Edw==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", "license": "MIT", "engines": { "node": ">=16" @@ -7572,13 +7603,13 @@ } }, "node_modules/playwright": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.58.2.tgz", - "integrity": "sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A==", + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.60.0.tgz", + "integrity": "sha512-hheHdokM8cdqCb0lcE3s+zT4t4W+vvjpGxsZlDnikarzx8tSzMebh3UiFtgqwFwnTnjYQcsyMF8ei2mCO/tpeA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "playwright-core": "1.58.2" + "playwright-core": "1.60.0" }, "bin": { "playwright": "cli.js" @@ -7591,9 +7622,9 @@ } }, "node_modules/playwright-core": { - "version": "1.58.2", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.58.2.tgz", - "integrity": "sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg==", + "version": "1.60.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz", + "integrity": "sha512-9bW6zvX/m0lEbgTKJ6YppOKx8H3VOPBMOCFh2irXFOT4BbHgrx5hPjwJYLT40Lu+4qtD36qKc/Hn56StUW57IA==", "dev": true, "license": "Apache-2.0", "bin": { @@ -8567,9 +8598,9 @@ } }, "node_modules/undici-types": { - "version": "7.19.2", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", - "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", "devOptional": true, "license": "MIT" }, diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 318624e3e..f6943cbf2 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -21,12 +21,12 @@ "@langchain/anthropic": "^1.3.29", "@langchain/core": "^1.1.44", "@langchain/google-genai": "^2.1.30", - "@langchain/langgraph": "^1.2.9", + "@langchain/langgraph": "^1.3.2", "@langchain/ollama": "^1.2.6", "@langchain/openai": "^1.4.5", "@sigma/edge-curve": "^3.1.0", "@tailwindcss/vite": "^4.3.0", - "axios": "^1.16.0", + "axios": "^1.16.1", "d3": "^7.9.0", "dompurify": "^3.4.3", "gitnexus-shared": "file:../gitnexus-shared", @@ -58,12 +58,12 @@ }, "devDependencies": { "@babel/types": "^7.29.0", - "@playwright/test": "^1.58.2", + "@playwright/test": "^1.60.0", "@testing-library/jest-dom": "^6.9.1", "@testing-library/react": "^16.3.2", "@testing-library/user-event": "^14.6.1", "@types/dompurify": "^3.2.0", - "@types/node": "^25.6.0", + "@types/node": "^25.9.1", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@types/react-syntax-highlighter": "^15.5.13", diff --git a/gitnexus/src/core/group/extractors/http-patterns/python.ts b/gitnexus/src/core/group/extractors/http-patterns/python.ts index 1667d0de9..5dc314d35 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/python.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/python.ts @@ -6,7 +6,7 @@ import { unquoteLiteral, type LanguagePatterns, } from '../tree-sitter-scanner.js'; -import type { HttpDetection, HttpLanguagePlugin } from './types.js'; +import type { HttpDetection, HttpLanguagePlugin, RepoContext } from './types.js'; /** * Python HTTP plugin. Handles: @@ -29,9 +29,13 @@ const FASTAPI_VERBS: Record = { patch: 'PATCH', }; -// ─── Provider: FastAPI @app.get/... ────────────────────────────────── -const FASTAPI_PATTERNS = compilePatterns({ - name: 'python-fastapi', +// ─── Provider: FastAPI @app. / @router. ────────────────── +// Two separate patterns so we can tag detections by decorator object. +// Only `@router.*` detections participate in `include_router(prefix=)` +// path-prefix joining (see `PythonRepoContext` + `joinPrefix`); `@app.*` +// routes already carry their final path verbatim. +const FASTAPI_APP_PATTERNS = compilePatterns({ + name: 'python-fastapi-app', language: Python, patterns: [ { @@ -48,6 +52,138 @@ const FASTAPI_PATTERNS = compilePatterns({ ], } satisfies LanguagePatterns>); +const FASTAPI_ROUTER_PATTERNS = compilePatterns({ + name: 'python-fastapi-router', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (decorator + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "router") + attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$")) + arguments: (argument_list . (string) @path))) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── include_router(, prefix='/x') across the repo ──────── +// Two shapes are common: +// app.include_router(assistant.router, prefix='/ai') +// app.include_router(my_router, prefix='/ai') +// The first names the originating module via `.router`; the second +// references a name imported into the host file. We capture both. +const INCLUDE_ROUTER_ATTR_PATTERNS = compilePatterns({ + name: 'python-fastapi-include-router-attr', + language: Python, + patterns: [ + { + meta: {}, + // Match any `.include_router(.router, ..., prefix='/x')` + // call. We deliberately do NOT pin `` to the literal name `app` + // — production code routinely uses `api`, `application`, `asgi_app`, + // etc. The shape (`include_router` invoked with a router argument and + // a `prefix=` keyword) is specific enough on its own; restricting the + // host produces false negatives without removing meaningful false + // positives. + query: ` + (call + function: (attribute + attribute: (identifier) @incl (#eq? @incl "include_router")) + arguments: (argument_list + (attribute + object: (identifier) @router_module + attribute: (identifier) @router_attr (#eq? @router_attr "router")) + (keyword_argument + name: (identifier) @kw (#eq? @kw "prefix") + value: (string) @prefix))) + `, + }, + ], +} satisfies LanguagePatterns>); + +const INCLUDE_ROUTER_NAME_PATTERNS = compilePatterns({ + name: 'python-fastapi-include-router-name', + language: Python, + patterns: [ + { + meta: {}, + // Same `` rationale as INCLUDE_ROUTER_ATTR_PATTERNS — see above. + query: ` + (call + function: (attribute + attribute: (identifier) @incl (#eq? @incl "include_router")) + arguments: (argument_list + (identifier) @router_name + (keyword_argument + name: (identifier) @kw (#eq? @kw "prefix") + value: (string) @prefix))) + `, + }, + ], +} satisfies LanguagePatterns>); + +// `from .api.assistant import router` style — used together with +// INCLUDE_ROUTER_NAME so we can map a local name back to its module +// path, then back to the file the router was declared in. +const FROM_IMPORT_ROUTER_PATTERNS = compilePatterns({ + name: 'python-fastapi-from-import-router', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (dotted_name (identifier) @imported (#eq? @imported "router"))) + `, + }, + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (aliased_import + name: (dotted_name (identifier) @imported (#eq? @imported "router")) + alias: (identifier) @alias)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// `from api import users` / `from api import users as u` — module-level +// imports where the imported name is itself the module that owns +// `.router`. Lets Shape A (`.include_router(.router, …)`) +// look up the full package path of `` and pin the prefix onto the +// exact file (`api/users.py`) rather than every file basenamed `users.py`. +const FROM_IMPORT_MODULE_PATTERNS = compilePatterns({ + name: 'python-fastapi-from-import-module', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (dotted_name (identifier) @imported)) + `, + }, + { + meta: {}, + query: ` + (import_from_statement + module_name: (_) @module + name: (aliased_import + name: (dotted_name (identifier) @imported) + alias: (identifier) @alias)) + `, + }, + ], +} satisfies LanguagePatterns>); + // ─── Consumer: requests.get/post/... ────────────────────────────────── const REQUESTS_VERB_PATTERNS = compilePatterns({ name: 'python-requests-verb', @@ -447,15 +583,226 @@ const HTTPX_ASYNC_CLIENT_GENERIC_PATTERNS = compilePatterns({ ], } satisfies LanguagePatterns>); +// ─── prepareRepo: build router-module → prefix list map ───────────── +// +// FastAPI splits route declarations across files: handler decorators +// live in `api/.py` while `app.include_router(.router, +// prefix='/ai')` lives in `main.py`. A per-file plugin scan therefore +// can't see the prefix that ought to be applied. We resolve this by +// running a one-shot pre-pass over the repo: for every file that +// hosts an `app.include_router(...)` we record the module the router +// came from (either via `module.router` attribute access, or via a +// local name resolved through a `from import router` import) +// together with the prefix string. At scan time the python plugin +// looks up the current file's module key in this map and joins each +// prefix with each `@router.` decorator's path. +// +// Multiple prefixes for the same module are kept and emitted as +// separate detections — this matches FastAPI's behaviour when one +// router is mounted under several prefixes. +// +// Module keying is two-tiered to avoid prefix bleed between same-named +// files in different packages (e.g. `api/users.py` vs `admin/users.py`): +// • short key — file basename without `.py` (`users`) +// • long key — `/` (`api/users`) +// The pre-pass records prefixes against the long key whenever the import +// site supplies enough context (`from api.users import router as ...` → +// long key `api/users`); otherwise it falls back to the short key. +// At scan time the file's own long key is consulted first; only when no +// long-key entry targets this file do we look up the short key. This +// preserves the previous coarse-grained behaviour where context is +// missing while delivering precision wherever the import statement +// gives us a multi-segment module path. +interface PythonRepoContext { + /** `/` → set of prefixes (precise, package-aware) */ + prefixesByLongKey: Map>; + /** stem only → set of prefixes (basename fallback, may collide) */ + prefixesByShortKey: Map>; +} + +/** Strip `.py` and return the bare basename (e.g. `api/users.py` → `users`). */ +function fileShortKey(rel: string): string { + const slash = rel.lastIndexOf('/'); + const file = slash >= 0 ? rel.slice(slash + 1) : rel; + return file.endsWith('.py') ? file.slice(0, -3) : file; +} + +/** + * Long key for a `.py` file: parent directory + stem, joined with `/`. + * Files at the repo root return the empty string (no parent), in which + * case callers should fall back to the short key. + */ +function fileLongKey(rel: string): string { + const noExt = rel.endsWith('.py') ? rel.slice(0, -3) : rel; + const lastSlash = noExt.lastIndexOf('/'); + if (lastSlash < 0) return ''; + const beforeLast = noExt.slice(0, lastSlash); + const stem = noExt.slice(lastSlash + 1); + const prevSlash = beforeLast.lastIndexOf('/'); + const parent = prevSlash >= 0 ? beforeLast.slice(prevSlash + 1) : beforeLast; + return `${parent}/${stem}`; +} + +/** Last `.`-separated segment of a (possibly relative) module path. */ +function lastSegmentOfDotted(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const dot = stripped.lastIndexOf('.'); + return dot >= 0 ? stripped.slice(dot + 1) : stripped; +} + +/** + * Last two `.`-separated segments of a (possibly relative) module path + * joined with `/`, e.g. `api.users` → `api/users`. Single-segment paths + * and pure-dot inputs return the empty string; callers should fall back + * to the short key in that case. + */ +function lastTwoSegmentsAsLongKey(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const last = stripped.lastIndexOf('.'); + if (last <= 0) return ''; + const beforeLast = stripped.slice(0, last); + const stem = stripped.slice(last + 1); + const prev = beforeLast.lastIndexOf('.'); + const parent = prev >= 0 ? beforeLast.slice(prev + 1) : beforeLast; + return `${parent}/${stem}`; +} + +function recordPrefix(target: Map>, key: string, prefix: string): void { + const set = target.get(key) ?? new Set(); + set.add(prefix); + target.set(key, set); +} + +function buildPythonRepoContext( + files: string[], + parser: Parser, + readFile: (rel: string) => string | null, + parseSource: (parser: Parser, src: string) => Parser.Tree | null, +): PythonRepoContext { + const prefixesByLongKey = new Map>(); + const prefixesByShortKey = new Map>(); + + // Pre-pass over .py files. We deliberately run this even on files + // that don't contain `include_router` — the cost of an extra parse + // is bounded by the file count, and detecting `include_router` + // beforehand would require its own grep/scan. + for (const rel of files) { + if (!rel.endsWith('.py')) continue; + const src = readFile(rel); + if (!src) continue; + if (!src.includes('include_router')) continue; + parser.setLanguage(Python); + const tree = parseSource(parser, src); + if (!tree) continue; + + // Local name → (short, long) map for the current file, populated + // from `from import router [as ]` statements. The + // alias (or 'router' when there is no alias) is the local name + // we'll later see passed to `.include_router`. + interface LocalImport { + moduleShort: string; + moduleLong: string; + } + const localNameToModule = new Map(); + for (const m of runCompiledPatterns(FROM_IMPORT_ROUTER_PATTERNS, tree)) { + const moduleNode = m.captures.module; + const aliasNode = m.captures.alias; + const importedNode = m.captures.imported; + if (!moduleNode || !importedNode) continue; + const localName = aliasNode?.text ?? importedNode.text; + const moduleShort = lastSegmentOfDotted(moduleNode.text); + if (!moduleShort) continue; + const moduleLong = lastTwoSegmentsAsLongKey(moduleNode.text); + localNameToModule.set(localName, { moduleShort, moduleLong }); + } + + // Module-alias map: name imported from a multi-segment package → + // long key. Lets Shape A look up the precise file for `.router` + // even when `` collides with another package's basename. + const localNameToModuleAlias = new Map(); + for (const m of runCompiledPatterns(FROM_IMPORT_MODULE_PATTERNS, tree)) { + const moduleNode = m.captures.module; + const importedNode = m.captures.imported; + const aliasNode = m.captures.alias; + if (!moduleNode || !importedNode) continue; + // Skip the `router` shape — already handled by FROM_IMPORT_ROUTER_PATTERNS + // above and stored under its router-aware semantics. + if (importedNode.text === 'router') continue; + const moduleLong = lastTwoSegmentsAsLongKey(`${moduleNode.text}.${importedNode.text}`); + if (!moduleLong) continue; + const localName = aliasNode?.text ?? importedNode.text; + localNameToModuleAlias.set(localName, moduleLong); + } + + // Shape A: `.include_router(.router, prefix='/x')`. + // The call site gives us only a short module name. We promote to a + // long key when the same file imports `` via either + // `from import ` (recorded in `localNameToModuleAlias` + // — the typical pattern) or, less commonly, a router-aware import + // statement. Only fall back to the basename short key when neither + // alias is available. + for (const m of runCompiledPatterns(INCLUDE_ROUTER_ATTR_PATTERNS, tree)) { + const modNode = m.captures.router_module; + const prefixNode = m.captures.prefix; + if (!modNode || !prefixNode) continue; + const prefix = unquoteLiteral(prefixNode.text); + if (prefix === null) continue; + const moduleShort = modNode.text; + const aliasLong = localNameToModuleAlias.get(moduleShort); + const sameFileImport = localNameToModule.get(moduleShort); + const longKey = aliasLong ?? sameFileImport?.moduleLong; + if (longKey) { + recordPrefix(prefixesByLongKey, longKey, prefix); + } else { + recordPrefix(prefixesByShortKey, moduleShort, prefix); + } + } + + // Shape B: `.include_router(my_router, prefix='/x')` — resolve + // `my_router` via the import map built above. Whenever the import + // statement supplied a multi-segment module path the long key is + // recorded, eliminating cross-package collisions. + for (const m of runCompiledPatterns(INCLUDE_ROUTER_NAME_PATTERNS, tree)) { + const nameNode = m.captures.router_name; + const prefixNode = m.captures.prefix; + if (!nameNode || !prefixNode) continue; + const localImp = localNameToModule.get(nameNode.text); + if (!localImp) continue; + const prefix = unquoteLiteral(prefixNode.text); + if (prefix === null) continue; + if (localImp.moduleLong) { + recordPrefix(prefixesByLongKey, localImp.moduleLong, prefix); + } else { + recordPrefix(prefixesByShortKey, localImp.moduleShort, prefix); + } + } + } + + return { prefixesByLongKey, prefixesByShortKey }; +} + +function joinPrefix(prefix: string, route: string): string { + // Mirror FastAPI's path joining: trim trailing slash off prefix, + // ensure exactly one leading slash on the result. + const p = prefix.replace(/\/+$/, ''); + const r = route.startsWith('/') ? route : `/${route}`; + return `${p}${r}`; +} export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { name: 'python-http', language: Python, - scan(tree) { + prepareRepo({ files, parser, readFile, parseSource }): RepoContext { + return buildPythonRepoContext(files, parser, readFile, parseSource); + }, + scan(tree, repoContext, fileRel) { const out: HttpDetection[] = []; const httpxAsyncClients = collectHttpxAsyncClients(tree); + const ctx = repoContext as PythonRepoContext | undefined; - // Providers: FastAPI - for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) { + // Providers: FastAPI @app.("/path") — already absolute path. + for (const match of runCompiledPatterns(FASTAPI_APP_PATTERNS, tree)) { const methodNode = match.captures.method; const pathNode = match.captures.path; if (!methodNode || !pathNode) continue; @@ -473,6 +820,47 @@ export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { }); } + // Providers: FastAPI @router.("/path") — must be joined + // with the prefix(es) declared at the include_router site. When + // no prefix is found we still emit the unprefixed path so this + // change is strictly additive vs. the prior @app-only behaviour; + // when the same router is mounted under multiple prefixes we emit + // one detection per prefix. + for (const match of runCompiledPatterns(FASTAPI_ROUTER_PATTERNS, tree)) { + const methodNode = match.captures.method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const httpMethod = FASTAPI_VERBS[methodNode.text]; + if (!httpMethod) continue; + const rawPath = unquoteLiteral(pathNode.text); + if (rawPath === null) continue; + + // Long key first (precise, package-aware), short key as fallback. + // Mirrors the ingestion-side resolution in parse-impl.ts so the + // graph nodes and group contracts agree on which prefix applies. + const longKey = fileRel ? fileLongKey(fileRel) : ''; + const longPrefixes = longKey ? ctx?.prefixesByLongKey.get(longKey) : undefined; + const shortKey = fileRel ? fileShortKey(fileRel) : ''; + const shortPrefixes = + longPrefixes || !shortKey ? undefined : ctx?.prefixesByShortKey.get(shortKey); + const prefixSet = longPrefixes ?? shortPrefixes; + const paths = + prefixSet && prefixSet.size > 0 + ? [...prefixSet].map((p) => joinPrefix(p, rawPath)) + : [rawPath]; + + for (const p of paths) { + out.push({ + role: 'provider', + framework: 'fastapi', + method: httpMethod, + path: p, + name: null, + confidence: 0.8, + }); + } + } + // Consumers: requests. for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) { const methodNode = match.captures.method; diff --git a/gitnexus/src/core/group/extractors/http-patterns/types.ts b/gitnexus/src/core/group/extractors/http-patterns/types.ts index 6df0ede28..e1c85369d 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/types.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/types.ts @@ -51,15 +51,48 @@ export interface HttpDetection { * `LanguagePatterns.language` in `tree-sitter-scanner.ts` — the * grammar modules export different shapes. */ +/** + * Per-repo state a plugin can build during a `prepareRepo` pass before + * any per-file `scan` is invoked. The orchestrator threads this opaque + * value back into each `scan` call so plugins can resolve cross-file + * facts (e.g. FastAPI `app.include_router(prefix=...)` mappings live + * in `main.py` but apply to handlers declared in `api/*.py`). + * + * Plugins that have no cross-file state can omit `prepareRepo` and + * receive `undefined`. + */ +export type RepoContext = unknown; + export interface HttpLanguagePlugin { /** Human-readable plugin name for diagnostics. */ name: string; /** tree-sitter grammar object (passed to the shared parser). */ language: unknown; + /** + * Optional pre-pass: walk the relevant files in the repo and produce + * an opaque context that `scan` can use to resolve cross-file facts. + * Implementations must not throw — return undefined on any error so + * the orchestrator falls back to context-less scanning. + */ + prepareRepo?(args: { + repoPath: string; + files: string[]; + parser: Parser; + readFile: (rel: string) => string | null; + parseSource: (parser: Parser, src: string) => Parser.Tree | null; + }): RepoContext | undefined; /** * Scan a parsed tree and return zero or more HTTP detections. Plugins * must not throw — they should swallow per-match errors so a single * malformed construct does not abort the whole file. + * + * `repoContext` is whatever the plugin's `prepareRepo` produced (or + * `undefined` if there is no `prepareRepo`). + * + * `fileRel` is the repo-relative path of the file being scanned; + * plugins that resolve cross-file facts (e.g. FastAPI router prefix + * joining) need it to key into `repoContext`. Optional so existing + * single-file plugins can keep their unary `scan(tree)` shape. */ - scan(tree: Parser.Tree): HttpDetection[]; + scan(tree: Parser.Tree, repoContext?: RepoContext, fileRel?: string): HttpDetection[]; } diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index 54aeb9150..37237242f 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -160,7 +160,36 @@ export class HttpRouteExtractor implements ContractExtractor { // both graph-assisted enrichment and source-scan emission. const parser = new Parser(); const cachedDetections = new Map(); - const getDetections = (rel: string): HttpDetection[] => { + + // Per-plugin cross-file context (e.g. Python's FastAPI router → + // include_router(prefix=...) map). Built lazily on first + // `getDetections` call for a file the plugin handles, scoped to the + // file list returned by `getScannedFiles`. Stored by plugin name so + // a repo with multiple languages keeps each plugin's context + // independent. + const repoContextByPlugin = new Map(); + const ensureRepoContext = async ( + plugin: ReturnType, + ): Promise => { + if (!plugin || typeof plugin.prepareRepo !== 'function') return undefined; + if (repoContextByPlugin.has(plugin.name)) return repoContextByPlugin.get(plugin.name); + try { + const ctx = plugin.prepareRepo({ + repoPath, + files: await getScannedFiles(), + parser, + readFile: (rel) => readSafe(repoPath, rel), + parseSource: (p, src) => parseSourceSafe(p, src), + }); + repoContextByPlugin.set(plugin.name, ctx); + return ctx; + } catch { + repoContextByPlugin.set(plugin.name, undefined); + return undefined; + } + }; + + const getDetections = async (rel: string): Promise => { const cached = cachedDetections.get(rel); if (cached) return cached; const plugin = getPluginForFile(rel); @@ -168,6 +197,7 @@ export class HttpRouteExtractor implements ContractExtractor { cachedDetections.set(rel, []); return []; } + const repoContext = await ensureRepoContext(plugin); const content = readSafe(repoPath, rel); if (!content) { cachedDetections.set(rel, []); @@ -176,7 +206,7 @@ export class HttpRouteExtractor implements ContractExtractor { try { parser.setLanguage(plugin.language); const tree = parseSourceSafe(parser, content); - const detections = plugin.scan(tree); + const detections = plugin.scan(tree, repoContext, rel); cachedDetections.set(rel, detections); return detections; } catch { @@ -200,14 +230,14 @@ export class HttpRouteExtractor implements ContractExtractor { // by graph edges; the glob and per-file parse results are cached above. const providers = this.mergeGraphAndSourceContracts( graphProviders, - this.extractProvidersSourceScan(await getScannedFiles(), getDetections), + await this.extractProvidersSourceScan(await getScannedFiles(), getDetections), ); const graphConsumers = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : []; const consumers = this.mergeGraphAndSourceContracts( graphConsumers, - this.extractConsumersSourceScan(await getScannedFiles(), getDetections), + await this.extractConsumersSourceScan(await getScannedFiles(), getDetections), ); return [...providers, ...consumers]; @@ -232,7 +262,7 @@ export class HttpRouteExtractor implements ContractExtractor { private async extractProvidersGraph( db: CypherExecutor, - getDetections: (rel: string) => HttpDetection[], + getDetections: (rel: string) => Promise, ): Promise { const out: ExtractedContract[] = []; let rows: Record[]; @@ -254,7 +284,7 @@ export class HttpRouteExtractor implements ContractExtractor { // helpers — tree-sitter gives both pieces of information // structurally. Always run the lookup: even when method is set by // `methodFromRouteReason`, we still need the handler name. - const detections = filePath ? getDetections(filePath) : []; + const detections = filePath ? await getDetections(filePath) : []; const providerDetections = detections.filter((d) => d.role === 'provider'); let handlerName: string | null = null; const normalizedRoute = normalizeHttpPath(routePath); @@ -331,13 +361,13 @@ export class HttpRouteExtractor implements ContractExtractor { // ─── Source-scan providers ───────────────────────────────────────── - private extractProvidersSourceScan( + private async extractProvidersSourceScan( files: string[], - getDetections: (rel: string) => HttpDetection[], - ): ExtractedContract[] { + getDetections: (rel: string) => Promise, + ): Promise { const out: ExtractedContract[] = []; for (const rel of files) { - const detections = getDetections(rel); + const detections = await getDetections(rel); for (const d of detections) { if (d.role !== 'provider') continue; const pathNorm = normalizeHttpPath(d.path); @@ -366,7 +396,7 @@ export class HttpRouteExtractor implements ContractExtractor { private async extractConsumersGraph( db: CypherExecutor, - getDetections: (rel: string) => HttpDetection[], + getDetections: (rel: string) => Promise, ): Promise { const out: ExtractedContract[] = []; let rows: Record[]; @@ -382,7 +412,7 @@ export class HttpRouteExtractor implements ContractExtractor { let method = 'GET'; // Prefer the plugin's detected method if we can find a matching // fetch/axios call in the same file. - const detections = filePath ? getDetections(filePath) : []; + const detections = filePath ? await getDetections(filePath) : []; // Symmetric to the provider path: if multiple consumer calls in // the same file share the same normalized path (e.g. a GET // fetch AND a POST fetch to `/api/orders`), `.find()` silently @@ -436,13 +466,13 @@ export class HttpRouteExtractor implements ContractExtractor { // ─── Source-scan consumers ───────────────────────────────────────── - private extractConsumersSourceScan( + private async extractConsumersSourceScan( files: string[], - getDetections: (rel: string) => HttpDetection[], - ): ExtractedContract[] { + getDetections: (rel: string) => Promise, + ): Promise { const out: ExtractedContract[] = []; for (const rel of files) { - const detections = getDetections(rel); + const detections = await getDetections(rel); for (const d of detections) { if (d.role !== 'consumer') continue; const pathNorm = normalizeConsumerPath(d.path); diff --git a/gitnexus/src/core/ingestion/languages/cpp/adl.ts b/gitnexus/src/core/ingestion/languages/cpp/adl.ts index 0bcda9322..9c23c73cc 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/adl.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/adl.ts @@ -24,22 +24,18 @@ * V2 additionally walks class ancestors (via MRO), so base-class enclosing * namespaces also contribute associated namespaces. * - * **GitNexus approximation (not strict ISO C++ ADL):** passing a qualified - * function reference like `utils::worker` contributes `utils` to the associated - * set, enabling resolution of unqualified calls like `with_callback(utils::worker)` - * to `utils::with_callback`. Under ISO C++ `[basic.lookup.argdep]`, associated - * entities for function-type arguments come from the **parameter types and return - * type** of each function in the overload set — NOT the function's enclosing - * namespace. For `void worker()`, the standard-compliant associated set is empty. - * GitNexus instead contributes the enclosing namespace of any Function/Method - * def whose simple name matches, because it enables the dominant real-world ADL - * pattern at reasonable precision cost. + * Function-reference arguments follow ISO C++ `[basic.lookup.argdep]`: + * associated entities come from the parameter types and return type of each + * referenced function in the overload set, not from the function's enclosing + * namespace. For `void worker()`, the associated set is empty. For + * `void worker(api::Token)` or `api::Token make_token()`, `api` is associated + * through `Token`. * - * For qualified refs (e.g. `utils::worker`) the namespace is confirmed via a - * workspace lookup (only contributed when a Function/Method named `worker` exists - * in `utils`). For unqualified refs the workspace is searched for any Function - * def with that simple name. Locally-declared function-pointer variables - * (e.g. `void (*g)()`) and function parameters are excluded from this path. + * For qualified refs (e.g. `utils::worker`) the workspace lookup is restricted + * to functions/methods named `worker` in `utils`; for unqualified refs the + * workspace is searched for matching functions/methods by simple name. Locally + * declared function-pointer variables and function parameters are excluded + * from this path. * * ADL candidates are merged with ordinary unqualified-lookup candidates * in the free-call fallback before overload narrowing. @@ -70,6 +66,7 @@ import type { ParsedFile, ScopeId, SymbolDefinition } from 'gitnexus-shared'; import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import { normalizeCppParamType } from './arity-metadata.js'; import { isCppInlineNamespaceScope } from './inline-namespaces.js'; /** @@ -97,11 +94,8 @@ export interface CppAdlArgInfo { /** When set, the arg is a potential free-function reference (not a locally- * declared function-pointer variable or function parameter). Contains the * identifier text as written in source (e.g. `"utils::worker"` or - * `"worker"`). GitNexus approximation: the function's enclosing namespace - * is contributed to the ADL associated set. For qualified refs a workspace - * lookup confirms a Function/Method with that simple name exists in the - * namespace before contributing; for unqualified refs every namespace - * containing a matching Function/Method def is contributed. */ + * `"worker"`). Resolution contributes associated namespaces from each + * referenced Function/Method def's parameter and return types. */ readonly functionRefText?: string; } @@ -207,7 +201,12 @@ export function pickCppAdlCandidates( for (const arg of args) { collectAssociatedNamespacesForAdlArg(arg, scopes, associatedNamespaces); if (arg.functionRefText !== undefined) { - collectFunctionRefNamespaces(arg.functionRefText, parsedFiles, associatedNamespaces); + collectFunctionTypeAssociatedNamespaces( + arg.functionRefText, + scopes, + parsedFiles, + associatedNamespaces, + ); } } if (associatedNamespaces.size === 0) return undefined; @@ -472,23 +471,12 @@ function findCppClassDefBySimpleName( } /** - * Contribute associated namespaces for a function-reference argument. - * - * - **Qualified refs** (`utils::worker`, `outer::inner::fn`): the namespace - * is extracted from the qualifier text (converting `::` to `.` for dot-joined - * QName matching). A workspace lookup then **verifies** that a Function or - * Method def named `worker` (the simple name after the last `::`) actually - * exists in the extracted namespace. This prevents false positives from - * namespace-qualified variables, enum values, and static data members, which - * also produce `qualified_identifier` AST nodes in tree-sitter-cpp (the - * AST node type alone does not distinguish functions from non-function names). - * - **Unqualified refs** (`worker`): the workspace is searched for any - * Function/Method def whose simple name matches. Every distinct enclosing - * namespace found is added — overloads across the same namespace produce - * a single entry; GitNexus does not select a specific overload at this stage. + * Contribute associated namespaces for a function-reference argument by walking + * the referenced overload set's parameter and return types. */ -function collectFunctionRefNamespaces( +function collectFunctionTypeAssociatedNamespaces( refText: string, + scopes: ScopeResolutionIndexes, parsedFiles: readonly ParsedFile[], out: Set, ): void { @@ -511,30 +499,130 @@ function collectFunctionRefNamespaces( for (const def of scope.ownedDefs) { if (def.type !== 'Function' && def.type !== 'Method') continue; const simple = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; - if (simple === simpleName) { - out.add(nsText); - return; // Namespace confirmed; no need to scan further files. - } + if (simple === simpleName) collectAssociatedNamespacesForFunctionDef(def, scopes, out); } } } return; } - // Unqualified: search all namespace scopes for a Function def with this - // simple name and contribute its enclosing namespace. + // Unqualified function references are approximated workspace-wide, matching + // the previous V1 lookup scope. The stricter part of this PR is what each + // overload contributes: only namespaces from parameter/return types, never + // the function's own enclosing namespace. for (const parsed of parsedFiles) { - const scopesById = new Map(); - for (const sc of parsed.scopes) scopesById.set(sc.id, sc); for (const scope of parsed.scopes) { if (scope.kind !== 'Namespace') continue; for (const def of scope.ownedDefs) { if (def.type !== 'Function' && def.type !== 'Method') continue; const simple = def.qualifiedName?.split('.').pop() ?? def.qualifiedName ?? ''; if (simple !== refText) continue; - const nsQName = computeNamespaceQName(scope, scopesById); - if (nsQName !== '') out.add(nsQName); + collectAssociatedNamespacesForFunctionDef(def, scopes, out); } } } } + +function collectAssociatedNamespacesForFunctionDef( + def: SymbolDefinition, + scopes: ScopeResolutionIndexes, + out: Set, +): void { + const parameterTypes = def.parameterTypeClasses?.map((typeClass) => typeClass.base); + for (const paramType of parameterTypes ?? def.parameterTypes ?? []) { + collectAssociatedNamespacesForFunctionTypeText(paramType, scopes, out); + } + if (def.returnType !== undefined) { + collectAssociatedNamespacesForFunctionTypeText(def.returnType, scopes, out); + } +} + +function collectAssociatedNamespacesForFunctionTypeText( + typeText: string, + scopes: ScopeResolutionIndexes, + out: Set, +): void { + for (const token of extractCppTypeNameTokens(typeText)) { + if (isIgnoredCppAdlNamespace(token.namespaceName)) continue; + addAssociatedNamespaceForClassName(token.simpleName, scopes, out); + if (token.namespaceName !== '') out.add(token.namespaceName); + } +} + +function extractCppTypeNameTokens(typeText: string): readonly { + readonly simpleName: string; + readonly namespaceName: string; +}[] { + const cleaned = normalizeCppParamType(typeText); + if (cleaned === '' || isPrimitiveCppAdlType(cleaned)) return []; + const out: { simpleName: string; namespaceName: string }[] = []; + const seen = new Set(); + const tokenSource = typeText.includes('<') ? `${cleaned} ${typeText}` : cleaned; + for (const rawToken of tokenSource.match(/[A-Za-z_]\w*(?:::[A-Za-z_]\w*)*/g) ?? []) { + if (isPrimitiveCppAdlType(rawToken)) continue; + const segments = rawToken.split('::').filter((part) => part.length > 0); + const simpleName = segments.at(-1) ?? ''; + if (simpleName === '' || isPrimitiveCppAdlType(simpleName)) continue; + const namespaceName = segments.length > 1 ? segments.slice(0, -1).join('.') : ''; + const key = `${namespaceName}\0${simpleName}`; + if (seen.has(key)) continue; + seen.add(key); + out.push({ + simpleName, + namespaceName, + }); + } + return out; +} + +const CPP_ADL_PRIMITIVE_OR_KEYWORD_TYPES = new Set([ + 'alignas', + 'alignof', + 'auto', + 'bool', + 'char', + 'char8_t', + 'char16_t', + 'char32_t', + 'class', + 'const', + 'consteval', + 'constexpr', + 'constinit', + 'decltype', + 'double', + 'enum', + 'explicit', + 'extern', + 'float', + 'inline', + 'int', + 'long', + 'mutable', + 'noexcept', + 'null', + 'register', + 'short', + 'signed', + 'static', + 'string', + 'struct', + 'template', + 'thread_local', + 'typename', + 'union', + 'unknown', + 'unsigned', + 'void', + 'volatile', + 'wchar_t', + '...', +]); + +function isPrimitiveCppAdlType(typeText: string): boolean { + return CPP_ADL_PRIMITIVE_OR_KEYWORD_TYPES.has(typeText); +} + +function isIgnoredCppAdlNamespace(namespaceName: string): boolean { + return namespaceName === 'std' || namespaceName.startsWith('std.'); +} diff --git a/gitnexus/src/core/ingestion/languages/cpp/captures.ts b/gitnexus/src/core/ingestion/languages/cpp/captures.ts index 354f6ce4c..86384a759 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/captures.ts @@ -126,6 +126,14 @@ export function emitCppScopeCaptures( JSON.stringify(arity.parameterTypeClasses), ); } + const returnType = extractCppDeclarationReturnType(fnNode); + if (returnType !== undefined) { + grouped['@declaration.return-type'] = syntheticCapture( + '@declaration.return-type', + fnNode, + returnType, + ); + } if (hasExplicitSpecifier(fnNode)) { grouped['@declaration.is-explicit'] = syntheticCapture( '@declaration.is-explicit', @@ -417,6 +425,30 @@ export function emitCppScopeCaptures( return out; } +function extractCppDeclarationReturnType(fnNode: SyntaxNode): string | undefined { + const typeNode = fnNode.childForFieldName('type'); + if (typeNode === null) return undefined; + const funcDeclarator = findFunctionDeclarator(fnNode); + if (funcDeclarator !== null && isCppUnsupportedReturnTypeDeclarator(funcDeclarator)) { + return undefined; + } + const typeText = typeNode.text.trim(); + if (typeText !== 'auto') return typeText.length > 0 ? typeText : undefined; + if (funcDeclarator === null) return typeText; + for (let i = 0; i < funcDeclarator.namedChildCount; i++) { + const child = funcDeclarator.namedChild(i); + if (child?.type !== 'trailing_return_type') continue; + const typeDesc = child.firstNamedChild; + return typeDesc?.text.trim() || typeText; + } + return typeText; +} + +function isCppUnsupportedReturnTypeDeclarator(funcDeclarator: SyntaxNode): boolean { + const text = funcDeclarator.text; + return /\boperator\b/.test(text) || /(^|[(:\s])~\s*[A-Za-z_]\w*/.test(text); +} + /** * Walk every C++ class/struct base clause and emit `@reference.inherits` * captures for each base so scope resolution can resolve them into EXTENDS diff --git a/gitnexus/src/core/ingestion/languages/python/captures.ts b/gitnexus/src/core/ingestion/languages/python/captures.ts index bc4911546..3c20101db 100644 --- a/gitnexus/src/core/ingestion/languages/python/captures.ts +++ b/gitnexus/src/core/ingestion/languages/python/captures.ts @@ -21,6 +21,7 @@ import { findNodeAtRange, nodeToCapture, syntheticCapture } from '../../utils/as import { splitImportStatement } from './import-decomposer.js'; import { getPythonParser, getPythonScopeQuery } from './query.js'; import { synthesizeReceiverTypeBinding } from './receiver-binding.js'; +import { synthesizeDependsReferences } from './depends-references.js'; import { computePythonArityMetadata } from './arity-metadata.js'; import { recordCacheHit, recordCacheMiss } from './cache-stats.js'; import { getTreeSitterBufferSize } from '../../constants.js'; @@ -98,6 +99,7 @@ export function emitPythonScopeCaptures( if (fnNode !== null) { const synth = synthesizeReceiverTypeBinding(fnNode); if (synth !== null) out.push(synth); + for (const depRef of synthesizeDependsReferences(fnNode)) out.push(depRef); } continue; } diff --git a/gitnexus/src/core/ingestion/languages/python/depends-references.ts b/gitnexus/src/core/ingestion/languages/python/depends-references.ts new file mode 100644 index 000000000..333c4f7e2 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/python/depends-references.ts @@ -0,0 +1,72 @@ +/** + * Synthesize `@reference.call.free` captures for FastAPI `Depends(callable)` + * parameter defaults. + * + * `Depends(get_db)` passes `get_db` as a callable that the DI framework + * calls on every request. The route handler is functionally a caller of + * the dependency — impact analysis needs that edge. + * + * Tree-sitter can't express "the first argument of a call named Depends + * inside a parameter default" in a single static query, so we synthesize + * reference captures in code, mirroring the receiver-binding pattern. + */ + +import type { CaptureMatch } from 'gitnexus-shared'; +import { nodeToCapture, type SyntaxNode } from '../../utils/ast-helpers.js'; + +/** + * Inspect a `function_definition` node's parameters for `Depends(callable)` + * defaults. Returns one `@reference.call.free` CaptureMatch per dependency. + */ +export function synthesizeDependsReferences(fnNode: SyntaxNode): readonly CaptureMatch[] { + const params = fnNode.childForFieldName('parameters'); + if (params === null) return []; + + const results: CaptureMatch[] = []; + + for (let i = 0; i < params.namedChildCount; i++) { + const param = params.namedChild(i); + if (param === null) continue; + + if (param.type !== 'typed_default_parameter' && param.type !== 'default_parameter') { + continue; + } + + const defaultValue = param.childForFieldName('value') ?? param.childForFieldName('default'); + if (defaultValue === null) continue; + + const callNode = defaultValue.type === 'call' ? defaultValue : null; + if (callNode === null) continue; + + const fnIdent = callNode.childForFieldName('function'); + if (fnIdent === null || fnIdent.type !== 'identifier' || fnIdent.text !== 'Depends') continue; + + const args = callNode.childForFieldName('arguments'); + if (args === null || args.namedChildCount === 0) continue; + + const firstArg = args.namedChild(0); + if (firstArg === null) continue; + + if (firstArg.type === 'identifier') { + results.push({ + '@reference.call.free': nodeToCapture('@reference.call.free', firstArg), + '@reference.name': nodeToCapture('@reference.name', firstArg), + }); + continue; + } + + if (firstArg.type === 'attribute') { + const attrName = firstArg.childForFieldName('attribute'); + const obj = firstArg.childForFieldName('object'); + if (attrName !== null && obj !== null) { + results.push({ + '@reference.call.member': nodeToCapture('@reference.call.member', attrName), + '@reference.name': nodeToCapture('@reference.name', attrName), + '@reference.receiver': nodeToCapture('@reference.receiver', obj), + }); + } + } + } + + return results; +} diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 6465a9782..6b69f75c3 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -53,7 +53,13 @@ import type { FileConstructorBindings, FileScopeBindings, ExtractedORMQuery, + FetchWrapperDef, } from './workers/parse-worker.js'; +import type { + ExtractedRouterImport, + ExtractedRouterInclude, + ExtractedRouterModuleAlias, +} from './route-extractors/fastapi-router-bindings.js'; import { getTreeSitterBufferSize, getTreeSitterContentByteLength, @@ -69,7 +75,11 @@ export interface WorkerExtractedData { heritage: ExtractedHeritage[]; routes: ExtractedRoute[]; fetchCalls: ExtractedFetchCall[]; + fetchWrapperDefs: FetchWrapperDef[]; decoratorRoutes: ExtractedDecoratorRoute[]; + routerIncludes: ExtractedRouterInclude[]; + routerImports: ExtractedRouterImport[]; + routerModuleAliases: ExtractedRouterModuleAlias[]; toolDefs: ExtractedToolDef[]; ormQueries: ExtractedORMQuery[]; constructorBindings: FileConstructorBindings[]; @@ -110,7 +120,11 @@ export const mergeChunkResults = ( const allHeritage: ExtractedHeritage[] = []; const allRoutes: ExtractedRoute[] = []; const allFetchCalls: ExtractedFetchCall[] = []; + const allFetchWrapperDefs: FetchWrapperDef[] = []; const allDecoratorRoutes: ExtractedDecoratorRoute[] = []; + const allRouterIncludes: ExtractedRouterInclude[] = []; + const allRouterImports: ExtractedRouterImport[] = []; + const allRouterModuleAliases: ExtractedRouterModuleAlias[] = []; const allToolDefs: ExtractedToolDef[] = []; const allORMQueries: ExtractedORMQuery[] = []; const allConstructorBindings: FileConstructorBindings[] = []; @@ -147,7 +161,11 @@ export const mergeChunkResults = ( for (const item of result.heritage) allHeritage.push(item); for (const item of result.routes) allRoutes.push(item); for (const item of result.fetchCalls) allFetchCalls.push(item); + for (const item of result.fetchWrapperDefs ?? []) allFetchWrapperDefs.push(item); for (const item of result.decoratorRoutes) allDecoratorRoutes.push(item); + for (const item of result.routerIncludes ?? []) allRouterIncludes.push(item); + for (const item of result.routerImports ?? []) allRouterImports.push(item); + for (const item of result.routerModuleAliases ?? []) allRouterModuleAliases.push(item); for (const item of result.toolDefs) allToolDefs.push(item); if (result.ormQueries) for (const item of result.ormQueries) allORMQueries.push(item); for (const item of result.constructorBindings) allConstructorBindings.push(item); @@ -163,7 +181,11 @@ export const mergeChunkResults = ( heritage: allHeritage, routes: allRoutes, fetchCalls: allFetchCalls, + fetchWrapperDefs: allFetchWrapperDefs, decoratorRoutes: allDecoratorRoutes, + routerIncludes: allRouterIncludes, + routerImports: allRouterImports, + routerModuleAliases: allRouterModuleAliases, toolDefs: allToolDefs, ormQueries: allORMQueries, constructorBindings: allConstructorBindings, @@ -203,7 +225,11 @@ const processParsingWithWorkers = async ( heritage: [], routes: [], fetchCalls: [], + fetchWrapperDefs: [], decoratorRoutes: [], + routerIncludes: [], + routerImports: [], + routerModuleAliases: [], toolDefs: [], ormQueries: [], constructorBindings: [], diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index 4d06b8d58..dc8a8ab00 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -61,7 +61,13 @@ import type { ExtractedRoute, ExtractedToolDef, FileConstructorBindings, + FetchWrapperDef, } from '../workers/parse-worker.js'; +import type { + ExtractedRouterImport, + ExtractedRouterInclude, + ExtractedRouterModuleAlias, +} from '../route-extractors/fastapi-router-bindings.js'; import type { ExtractedHeritage } from '../model/heritage-map.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import type { PipelineOptions } from '../pipeline.js'; @@ -141,6 +147,7 @@ export async function runChunkedParseAndResolve( ): Promise<{ exportedTypeMap: ExportedTypeMap; allFetchCalls: ExtractedFetchCall[]; + allFetchWrapperDefs: FetchWrapperDef[]; allExtractedRoutes: ExtractedRoute[]; allDecoratorRoutes: ExtractedDecoratorRoute[]; allToolDefs: ExtractedToolDef[]; @@ -352,8 +359,12 @@ export async function runChunkedParseAndResolve( // it, and later wildcard chunks re-run it themselves. let hasSynthesized = false; const allFetchCalls: ExtractedFetchCall[] = []; + const allFetchWrapperDefs: FetchWrapperDef[] = []; const allExtractedRoutes: ExtractedRoute[] = []; const allDecoratorRoutes: ExtractedDecoratorRoute[] = []; + const allRouterIncludes: ExtractedRouterInclude[] = []; + const allRouterImports: ExtractedRouterImport[] = []; + const allRouterModuleAliases: ExtractedRouterModuleAlias[] = []; const allToolDefs: ExtractedToolDef[] = []; const allORMQueries: ExtractedORMQuery[] = []; const deferredWorkerCalls: ExtractedCall[] = []; @@ -663,12 +674,24 @@ export async function runChunkedParseAndResolve( if (chunkWorkerData.fetchCalls?.length) { for (const item of chunkWorkerData.fetchCalls) allFetchCalls.push(item); } + if (chunkWorkerData.fetchWrapperDefs?.length) { + for (const item of chunkWorkerData.fetchWrapperDefs) allFetchWrapperDefs.push(item); + } if (chunkWorkerData.routes?.length) { for (const item of chunkWorkerData.routes) allExtractedRoutes.push(item); } if (chunkWorkerData.decoratorRoutes?.length) { for (const item of chunkWorkerData.decoratorRoutes) allDecoratorRoutes.push(item); } + if (chunkWorkerData.routerIncludes?.length) { + for (const item of chunkWorkerData.routerIncludes) allRouterIncludes.push(item); + } + if (chunkWorkerData.routerImports?.length) { + for (const item of chunkWorkerData.routerImports) allRouterImports.push(item); + } + if (chunkWorkerData.routerModuleAliases?.length) { + for (const item of chunkWorkerData.routerModuleAliases) allRouterModuleAliases.push(item); + } if (chunkWorkerData.toolDefs?.length) { for (const item of chunkWorkerData.toolDefs) allToolDefs.push(item); } @@ -1079,9 +1102,161 @@ export async function runChunkedParseAndResolve( importCtx.index = EMPTY_INDEX; importCtx.normalizedFileList = []; + // FastAPI router-prefix resolution (cross-file). + // + // Workers emit two kinds of records per Python file: + // • `routerIncludes` — every `app.include_router(, prefix='/x')` + // site, where `routerExpr` is either `.router` (Shape A) or a + // bare local name (Shape B). + // • `routerImports` — every `from import router [as ]`, + // mapping a local name to a module key (the basename of the source + // module). These let us resolve Shape-B router includes back to the + // module that defines the router. + // + // We build `module-basename → Set` and then walk + // `allDecoratorRoutes`: any decorator route emitted from a `router.` + // decorator inherits its file-basename's prefix. When a router is mounted + // under multiple prefixes we duplicate the route entry, mirroring FastAPI's + // runtime behaviour. + if (allRouterIncludes.length > 0 && allDecoratorRoutes.length > 0) { + // Group `routerImports` by file so we can resolve Shape-B locals against + // imports declared in the SAME file as the include_router call. We carry + // both the short module key (file basename) and, when available, the long + // key (`/`) so cross-package same-name modules don't blur + // their prefixes together. `routerModuleAliases` lifts the same long-key + // information for Shape-A includes whose receiving module was imported + // via `from import `. + interface LocalImport { + moduleKey: string; + moduleKeyLong: string | undefined; + } + const importsByFile = new Map>(); + for (const imp of allRouterImports) { + let m = importsByFile.get(imp.filePath); + if (!m) { + m = new Map(); + importsByFile.set(imp.filePath, m); + } + m.set(imp.localName, { + moduleKey: imp.moduleKey, + moduleKeyLong: imp.moduleKeyLong, + }); + } + // Module-alias map keyed by file: `localName` (the imported module + // identifier in this file) → long key. Shape-A receivers like + // `users.router` are matched against this map; the long key, when + // present, scopes the prefix to the precise source file. + const moduleAliasesByFile = new Map>(); + for (const alias of allRouterModuleAliases) { + let m = moduleAliasesByFile.get(alias.filePath); + if (!m) { + m = new Map(); + moduleAliasesByFile.set(alias.filePath, m); + } + m.set(alias.localName, alias.moduleKeyLong); + } + + // Two parallel maps: long-key (precise) and short-key (basename + // fallback). Long-key entries are preferred when the file's own long + // key matches; short-key entries match any file with that basename and + // remain the fallback when no long key is known (e.g. Shape A includes + // without a corresponding import statement). + const prefixesByLongKey = new Map>(); + const prefixesByShortKey = new Map>(); + + const recordPrefix = (target: Map>, key: string, prefix: string): void => { + let set = target.get(key); + if (!set) { + set = new Set(); + target.set(key, set); + } + set.add(prefix); + }; + + for (const inc of allRouterIncludes) { + // Shape A: `.router`. The worker emits `routerExpr` already + // including `.router`, so split it back. We only know a short module + // key here — the call site doesn't carry the dotted package path. If + // the same file imports `` via `from import ` + // (recorded in `allRouterModuleAliases`) we promote to a long key. + const dotIdx = inc.routerExpr.indexOf('.router'); + if (dotIdx > 0) { + const moduleShort = inc.routerExpr.slice(0, dotIdx); + const aliasLong = moduleAliasesByFile.get(inc.filePath)?.get(moduleShort); + if (aliasLong) { + recordPrefix(prefixesByLongKey, aliasLong, inc.prefix); + } else { + recordPrefix(prefixesByShortKey, moduleShort, inc.prefix); + } + continue; + } + + // Shape B: bare local name. Resolve through this file's imports. The + // import line gives us a long key whenever the module path was multi- + // segment, so cross-package collisions are eliminated for Shape B. + const localImp = importsByFile.get(inc.filePath)?.get(inc.routerExpr); + if (!localImp) continue; + if (localImp.moduleKeyLong) { + recordPrefix(prefixesByLongKey, localImp.moduleKeyLong, inc.prefix); + } else { + recordPrefix(prefixesByShortKey, localImp.moduleKey, inc.prefix); + } + } + + if (prefixesByLongKey.size > 0 || prefixesByShortKey.size > 0) { + const fileLongKey = (rel: string): string => { + // Strip `.py`, then take the last two path segments. `api/users.py` + // → `api/users`. Files at the repo root return the empty string, + // which can never match a long-key entry (those always include a + // parent directory) and so fall through to the short-key lookup. + const noExt = rel.endsWith('.py') ? rel.slice(0, -3) : rel; + const lastSlash = noExt.lastIndexOf('/'); + if (lastSlash < 0) return ''; + const beforeLast = noExt.slice(0, lastSlash); + const stem = noExt.slice(lastSlash + 1); + const prevSlash = beforeLast.lastIndexOf('/'); + const parent = prevSlash >= 0 ? beforeLast.slice(prevSlash + 1) : beforeLast; + return `${parent}/${stem}`; + }; + + const fileShortKey = (rel: string): string => { + const slash = rel.lastIndexOf('/'); + const file = slash >= 0 ? rel.slice(slash + 1) : rel; + return file.endsWith('.py') ? file.slice(0, -3) : file; + }; + + const expanded: ExtractedDecoratorRoute[] = []; + for (const dr of allDecoratorRoutes) { + if (dr.decoratorReceiver !== 'router' || !dr.filePath.endsWith('.py')) { + expanded.push(dr); + continue; + } + // Long-key lookup first; only fall back to the short key when no + // long-key prefix targets this file. This avoids prefix leakage + // between e.g. `api/users.py` and `admin/users.py`. + const longKey = fileLongKey(dr.filePath); + const longPrefixes = longKey ? prefixesByLongKey.get(longKey) : undefined; + const shortPrefixes = longPrefixes + ? undefined + : prefixesByShortKey.get(fileShortKey(dr.filePath)); + const prefixes = longPrefixes ?? shortPrefixes; + if (!prefixes || prefixes.size === 0) { + expanded.push(dr); + continue; + } + for (const prefix of prefixes) { + expanded.push({ ...dr, prefix }); + } + } + allDecoratorRoutes.length = 0; + for (const dr of expanded) allDecoratorRoutes.push(dr); + } + } + return { exportedTypeMap, allFetchCalls, + allFetchWrapperDefs, allExtractedRoutes, allDecoratorRoutes, allToolDefs, diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts index a3fa81be7..280731e68 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse.ts @@ -27,6 +27,7 @@ import type { ExtractedDecoratorRoute, ExtractedToolDef, ExtractedORMQuery, + FetchWrapperDef, } from '../workers/parse-worker.js'; import type { createResolutionContext } from '../model/resolution-context.js'; import { runChunkedParseAndResolve } from './parse-impl.js'; @@ -45,6 +46,7 @@ export interface ParseOutput { */ readonly exportedTypeMap: ReadonlyMap>; readonly allFetchCalls: readonly ExtractedFetchCall[]; + readonly allFetchWrapperDefs: readonly FetchWrapperDef[]; readonly allExtractedRoutes: readonly ExtractedRoute[]; readonly allDecoratorRoutes: readonly ExtractedDecoratorRoute[]; readonly allToolDefs: readonly ExtractedToolDef[]; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index a1ee73eea..8c0a67ac5 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -131,6 +131,10 @@ export function normalizeExtractedRoutePath(routePath: string, prefix: string | return joined.replace(/\/+/g, '/') || '/'; } +function escapeRegex(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + export const routesPhase: PipelinePhase = { name: 'routes', deps: ['parse'], @@ -142,6 +146,7 @@ export const routesPhase: PipelinePhase = { const { allPaths, allFetchCalls: parseFetchCalls, + allFetchWrapperDefs, allExtractedRoutes, allDecoratorRoutes, } = getPhaseOutput(deps, 'parse'); @@ -193,7 +198,6 @@ export const routesPhase: PipelinePhase = { } } - const ensureSlash = (path: string) => (path.startsWith('/') ? path : '/' + path); let duplicateRoutes = 0; const namedRouteRegistry = new Map(); const addRoute = (url: string, entry: RouteEntry) => { @@ -215,7 +219,8 @@ export const routesPhase: PipelinePhase = { } } for (const dr of allDecoratorRoutes) { - addRoute(ensureSlash(dr.routePath), { + const url = normalizeExtractedRoutePath(dr.routePath, dr.prefix ?? null); + addRoute(url, { filePath: dr.filePath, source: `decorator-${dr.decoratorName}`, }); @@ -357,6 +362,35 @@ export const routesPhase: PipelinePhase = { } } + // ── Cross-file fetch wrapper consumer extraction ── + // When the parse phase discovered functions that internally call fetch(), + // scan JS/TS consumer files for calls to those wrapper functions with + // URL-like string arguments and add them to allFetchCalls so + // processNextjsFetchRoutes can create FETCHES edges. + if (allFetchWrapperDefs && allFetchWrapperDefs.length > 0 && routeRegistry.size > 0) { + const wrapperNames = new Set(allFetchWrapperDefs.map((d) => d.functionName)); + const jsFiles = allPaths.filter((p) => /\.[jt]sx?$/.test(p)); + if (jsFiles.length > 0 && wrapperNames.size > 0) { + const jsContents = await readFileContents(ctx.repoPath, jsFiles); + for (const [filePath, content] of jsContents) { + for (const name of wrapperNames) { + const regex = new RegExp( + `\\b${escapeRegex(name)}\\s*\\(\\s*['"\`](/[^'"\`\\s)]+)['"\`]`, + 'g', + ); + let match; + while ((match = regex.exec(content)) !== null) { + allFetchCalls.push({ + filePath, + fetchURL: match[1], + lineNumber: content.substring(0, match.index).split('\n').length, + }); + } + } + } + } + } + if (routeRegistry.size > 0 && allFetchCalls.length > 0) { const routeURLToFile = new Map(); for (const [url, entry] of routeRegistry) routeURLToFile.set(url, entry.filePath); diff --git a/gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts b/gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts new file mode 100644 index 000000000..34e2ded04 --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/fastapi-router-bindings.ts @@ -0,0 +1,275 @@ +/** + * FastAPI router-prefix detection — pure functions, no worker thread. + * + * NOT A WORKER. This module exports plain synchronous functions; it + * does not import `worker_threads`, does not call `parentPort`, and + * is not a new worker entry point. It lives next to the other route + * extractors (expo, nextjs, php, laravel) for that reason. + * + * The implementation was historically inlined in `workers/parse-worker.ts`, + * but parse-worker.ts is itself the worker entry point and cannot be + * loaded from the main thread (see the same constraint used by + * `test/unit/call-attribution-issue-1166.test.ts`). Splitting the pure + * extraction here lets unit tests import the function directly without + * booting a worker, satisfying DoD §2.7. + * + * Worker phase is per-file, so the heavy cross-file resolution lives in + * `pipeline-phases/parse-impl.ts`. Here we only extract two raw record + * kinds and let the pipeline aggregate them across files: + * + * • {@link ExtractedRouterInclude} — every + * `.include_router(, prefix='/x')` site, where + * `` is either `.router` (Shape A) or a bare + * local name (Shape B). `` is intentionally unconstrained: + * production code uses `app`, `api`, `application`, `asgi_app`, + * etc., and the call shape (`include_router` invoked with a + * `prefix=` keyword) is specific enough on its own. + * + * • {@link ExtractedRouterImport} — every + * `from import router [as ]`, captured for both + * absolute and relative module paths (`from .calls import …`). + * parse-impl uses the imports to resolve Shape-B local names back + * to the file that declares the router. + * + * Module keying is two-tiered to avoid prefix bleed between same-named + * files in different packages (e.g. `api/users.py` vs `admin/users.py`): + * + * • short key — basename without `.py` (`users`) + * • long key — `/` (`api/users`) + * + * Imports always carry the short key and, when the module path was + * multi-segment, also the long key. parse-impl matches against the + * long key first and falls back to the short key, so cross-package + * collisions are eliminated for Shape B and minimised for Shape A. + * + * The functions in this module are pure (no Worker / parentPort + * dependency) so they can be unit-tested directly without booting a + * worker thread. + */ + +/** + * One `.include_router(, prefix='/x')` site. + * + * `routerExpr` is the raw text of the first argument — either + * `.router` (Shape A) or a bare local name (Shape B). + * parse-impl resolves Shape B against {@link ExtractedRouterImport} + * records emitted by the same file. + */ +export interface ExtractedRouterInclude { + filePath: string; + routerExpr: string; + prefix: string; + lineNumber: number; +} + +/** + * One `from import router [as ]` discovered in a + * Python file. + * + * `moduleKey` is the short key (last `.`-segment of the module path, + * e.g. `api.users` → `users`). `moduleKeyLong` is the long key (last + * two segments joined with `/`, e.g. `api/users`); it is the empty + * string / undefined when the import is single-segment (e.g. + * `from users import router`) or pure-dots (e.g. `from . import + * router`). The long key, when present, gives parse-impl a precise + * way to bind a Shape-B `include_router` call to exactly one Python + * file even when other packages contain a same-named module. + */ +export interface ExtractedRouterImport { + filePath: string; + localName: string; + moduleKey: string; + moduleKeyLong?: string; +} + +/** + * One `from import ` discovered in a Python file + * where `` is later used as a Shape-A include receiver + * (`.include_router(.router, prefix='/x')`). Without + * this record parse-impl would have to fall back to the short key + * ``, which collides between e.g. `api/users.py` and + * `admin/users.py`. The record carries the long key + * (`/`) so parse-impl can pin the prefix onto the + * exact source file. + * + * Only emitted when the import path was multi-segment (a single + * `from users import users` would yield no long key). All fields + * carry the same module-key semantics as + * {@link ExtractedRouterImport}. + */ +export interface ExtractedRouterModuleAlias { + filePath: string; + /** Local name in the importing file (== imported name or its alias). */ + localName: string; + /** Long key (`/`) — non-empty for every emitted record. */ + moduleKeyLong: string; +} + +// `.include_router(.router, ..., prefix='/x')` (Shape A). +// `` is left unrestricted — common production names include +// `app`, `api`, `application`, `asgi_app`. Pinning to the literal +// `app` would silently drop these. +const INCLUDE_ROUTER_ATTR_RE = + /\b(?:[A-Za-z_][\w.]*)\.include_router\s*\(\s*([A-Za-z_][\w]*)\.router\b[^)]*?\bprefix\s*=\s*(['"])([^'"]*)\2/g; + +// `.include_router(, ..., prefix='/x')` (Shape B). +const INCLUDE_ROUTER_NAME_RE = + /\b(?:[A-Za-z_][\w.]*)\.include_router\s*\(\s*([A-Za-z_][\w]*)\b[^)]*?\bprefix\s*=\s*(['"])([^'"]*)\2/g; + +// Module path: a sequence of dots (`.`, `..`, `...`) for "current +// package" imports, OR an optional leading-dot prefix followed by a +// dotted identifier (`api.users`, `.api.users`, `..siblings.users`). +// The latter is the common case and the only one we can map back to +// a module stem. +const FROM_IMPORT_ROUTER_RE = /^\s*from\s+(\.+|\.*[A-Za-z_][\w.]*)\s+import\s+([^#\n]+)/gm; + +/** + * Last `.`-separated segment of a (possibly relative) Python module + * path. Strips any leading dots first so `from .api.assistant import + * …` and `from api.assistant import …` both yield `assistant`. + * Pure-dot inputs (`.`, `..`) have no segment and return the empty + * string; callers should skip empty results. + */ +export function lastDottedSegment(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const dot = stripped.lastIndexOf('.'); + return dot >= 0 ? stripped.slice(dot + 1) : stripped; +} + +/** + * Last two `.`-separated segments of a (possibly relative) module + * path joined with `/`, e.g. `api.users` → `api/users`. Mirrors the + * long-key shape used for files (`api/users.py` → `api/users`). + * Returns the empty string when no parent segment is available + * (single-segment imports or pure dots); callers should fall back + * to the short key in that case. + */ +export function lastTwoSegmentsAsPath(text: string): string { + const stripped = text.replace(/^\.+/, ''); + if (!stripped) return ''; + const last = stripped.lastIndexOf('.'); + if (last <= 0) return ''; + const beforeLast = stripped.slice(0, last); + const stem = stripped.slice(last + 1); + const prev = beforeLast.lastIndexOf('.'); + const parent = prev >= 0 ? beforeLast.slice(prev + 1) : beforeLast; + return `${parent}/${stem}`; +} + +/** + * Scan a single Python file's source text for FastAPI router + * `include_router` sites and `from import router` imports, + * appending raw records to the supplied collectors. + * + * `outModuleAliases` is optional: when supplied, every multi-segment + * `from import ` (other than `router` itself) is recorded + * as a module alias so parse-impl can pin Shape-A + * `.include_router(...)` calls onto the exact module file. When + * omitted, the function preserves the pre-existing behaviour and + * skips the alias collection — this keeps the function signature + * back-compat with older callers (and the parse-cache replay path). + */ +export function extractFastAPIRouterBindings( + filePath: string, + content: string, + outIncludes: ExtractedRouterInclude[], + outImports: ExtractedRouterImport[], + outModuleAliases?: ExtractedRouterModuleAlias[], +): void { + if (!content.includes('include_router') && !content.includes('router')) return; + + // `from import router [as ]`. We capture every name + // in the import list. `router` (with or without an `as` alias) maps + // to outImports; every other name lands in outModuleAliases when a + // long key is available, so Shape-A `.router` includes can be + // pinned to the exact module file. + if (content.includes(' import ')) { + FROM_IMPORT_ROUTER_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = FROM_IMPORT_ROUTER_RE.exec(content)) !== null) { + const moduleText = m[1]; + const importList = m[2]; + const moduleShort = lastDottedSegment(moduleText); + if (!moduleShort) continue; + // Long key for the imported MODULE itself (used by router + // imports — `from api.users import router` sets + // `moduleKeyLong = api/users`). + const moduleLong = lastTwoSegmentsAsPath(moduleText); + // Strip surrounding parens / trailing whitespace; split on + // commas. (Multiline import groups already have their newlines + // present in the captured list.) + const cleaned = importList.replace(/[()]/g, '').trim(); + for (const rawPart of cleaned.split(',')) { + const part = rawPart.trim(); + if (!part) continue; + + // `router` or `router as foo` → ExtractedRouterImport. + const routerAlias = /^router(?:\s+as\s+([A-Za-z_]\w*))?$/.exec(part); + if (routerAlias) { + const localName = routerAlias[1] ?? 'router'; + outImports.push({ + filePath, + localName, + moduleKey: moduleShort, + ...(moduleLong ? { moduleKeyLong: moduleLong } : {}), + }); + continue; + } + + // Any other `` or ` as ` — recorded as a + // module alias so parse-impl can pin Shape-A includes. The + // long key here is computed against the IMPORTED MODULE PATH + // (`.`), not the package path that `` + // was imported FROM. `from api import users` therefore yields + // `api/users`, the same long key as the file it points at. + if (!outModuleAliases) continue; + const otherAlias = /^([A-Za-z_]\w*)(?:\s+as\s+([A-Za-z_]\w*))?$/.exec(part); + if (!otherAlias) continue; + const importedName = otherAlias[1]; + const localName = otherAlias[2] ?? importedName; + const aliasLong = lastTwoSegmentsAsPath(`${moduleText}.${importedName}`); + if (!aliasLong) continue; + outModuleAliases.push({ + filePath, + localName, + moduleKeyLong: aliasLong, + }); + } + } + } + + if (!content.includes('include_router')) return; + + // Shape A: `.include_router(.router, prefix='/x')`. + INCLUDE_ROUTER_ATTR_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = INCLUDE_ROUTER_ATTR_RE.exec(content)) !== null) { + outIncludes.push({ + filePath, + routerExpr: `${m[1]}.router`, + prefix: m[3], + lineNumber: content.substring(0, m.index).split('\n').length, + }); + } + + // Shape B: `.include_router(my_router, prefix='/x')`. + // Resolution to a module key happens in parse-impl using + // outImports from the same file. + INCLUDE_ROUTER_NAME_RE.lastIndex = 0; + while ((m = INCLUDE_ROUTER_NAME_RE.exec(content)) !== null) { + // Skip cases that already matched Shape A — INCLUDE_ROUTER_NAME_RE + // is intentionally permissive and would re-capture `.router` + // as the bare name `mod`. Discriminate by re-checking the + // immediate source around the captured argument position. + const argStart = m.index + m[0].indexOf(m[1]); + const dotProbe = content.slice(argStart + m[1].length, argStart + m[1].length + 8); + if (/^\s*\.\s*router/.test(dotProbe)) continue; + outIncludes.push({ + filePath, + routerExpr: m[1], + prefix: m[3], + lineNumber: content.substring(0, m.index).split('\n').length, + }); + } +} diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index 963cf4862..973d4ac76 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -1087,6 +1087,7 @@ const KNOWN_SUB_TAGS: ReadonlySet = new Set([ '@declaration.required-parameter-count', '@declaration.parameter-types', '@declaration.parameter-type-classes', + '@declaration.return-type', '@declaration.template-constraints', '@declaration.is-explicit', ]); diff --git a/gitnexus/src/core/ingestion/tree-sitter-queries.ts b/gitnexus/src/core/ingestion/tree-sitter-queries.ts index f67823c8c..f27fcc8ce 100644 --- a/gitnexus/src/core/ingestion/tree-sitter-queries.ts +++ b/gitnexus/src/core/ingestion/tree-sitter-queries.ts @@ -241,6 +241,12 @@ export const TYPESCRIPT_QUERIES = ` [(string (string_fragment) @route.url) (template_string) @route.template_url])) @route.fetch +; Custom fetch wrappers: apiFetch('/path'), fetchJSON('/api/data'), httpGet('/users'), etc. +(call_expression + function: (identifier) @_wrapper_fn (#match? @_wrapper_fn "^(api(Fetch|Get|Post|Put|Delete|Patch|Request)|fetch(API|JSON|Data|Endpoint|Resource|Url)|http(Fetch|Get|Post|Put|Delete|Patch|Request))$") + arguments: (arguments + (string (string_fragment) @route.url))) @route.fetch + ; axios.get/post/put/delete/patch('/path'), $.get/post/ajax({url:'/path'}) (call_expression function: (member_expression @@ -434,6 +440,12 @@ export const JAVASCRIPT_QUERIES = ` [(string (string_fragment) @route.url) (template_string) @route.template_url])) @route.fetch +; Custom fetch wrappers: apiFetch('/path'), fetchJSON('/api/data'), httpGet('/users'), etc. +(call_expression + function: (identifier) @_wrapper_fn (#match? @_wrapper_fn "^(api(Fetch|Get|Post|Put|Delete|Patch|Request)|fetch(API|JSON|Data|Endpoint|Resource|Url)|http(Fetch|Get|Post|Put|Delete|Patch|Request))$") + arguments: (arguments + (string (string_fragment) @route.url))) @route.fetch + ; axios.get/post, $.get/post/ajax (call_expression function: (member_expression diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 69e28b91f..6ebc10782 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -23,6 +23,11 @@ import { import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; import type { SymbolTableReader } from '../model/symbol-table.js'; import type { ExtractedHeritage } from '../model/heritage-map.js'; +import type { + ExtractedRouterInclude, + ExtractedRouterImport, + ExtractedRouterModuleAlias, +} from '../route-extractors/fastapi-router-bindings.js'; /** Language grammar type accepted by Parser.setLanguage(). */ type TreeSitterLanguage = Parameters[0]; @@ -198,12 +203,30 @@ export interface ExtractedFetchCall { lineNumber: number; } +export interface FetchWrapperDef { + filePath: string; + functionName: string; +} + export interface ExtractedDecoratorRoute { filePath: string; routePath: string; httpMethod: string; decoratorName: string; lineNumber: number; + /** + * Decorator receiver identifier (e.g. `router` for `@router.get(...)`, + * `app` for `@app.get(...)`). Used by parse-impl to decide which routes + * participate in `include_router(prefix=...)` joining. + */ + decoratorReceiver?: string; + /** + * FastAPI `app.include_router(prefix='/x')` prefix that applies to + * this route. Filled by parse-impl after cross-file aggregation; the + * routes phase joins it via `normalizeExtractedRoutePath`. `null` / + * absent ⇒ no prefix applies. + */ + prefix?: string | null; } export interface ExtractedToolDef { @@ -268,7 +291,20 @@ export interface ParseWorkerResult { heritage: ExtractedHeritage[]; routes: ExtractedRoute[]; fetchCalls: ExtractedFetchCall[]; + fetchWrapperDefs: FetchWrapperDef[]; decoratorRoutes: ExtractedDecoratorRoute[]; + routerIncludes: ExtractedRouterInclude[]; + routerImports: ExtractedRouterImport[]; + /** + * Optional. `from import ` records from Python files + * where `` is later used as a Shape-A include receiver + * (`.include_router(.router, prefix='/x')`). parse-impl + * uses these to promote Shape-A short-key entries to long keys, so + * same-named modules in different packages don't share prefixes. + * Optional for cache backward compatibility (older cache entries + * predate the field; consumers must guard with `if (… ?? [])`). + */ + routerModuleAliases?: ExtractedRouterModuleAlias[]; toolDefs: ExtractedToolDef[]; ormQueries: ExtractedORMQuery[]; constructorBindings: FileConstructorBindings[]; @@ -732,7 +768,11 @@ const processBatch = ( heritage: [], routes: [], fetchCalls: [], + fetchWrapperDefs: [], decoratorRoutes: [], + routerIncludes: [], + routerImports: [], + routerModuleAliases: [], toolDefs: [], ormQueries: [], constructorBindings: [], @@ -842,6 +882,23 @@ const EXPRESS_ROUTE_METHODS = new Set([ 'route', ]); +/** + * Walk a tree-sitter AST subtree looking for a call to the global `fetch()` function. + * Returns `true` if found within `maxDepth` levels of nesting — keeps the check + * lightweight so it doesn't slow down parse-worker on large function bodies. + */ +const checkForFetchCall = (node: SyntaxNode, depth = 0, maxDepth = 5): boolean => { + if (depth > maxDepth) return false; + if (node.type === 'call_expression') { + const fn = node.childForFieldName('function'); + if (fn?.type === 'identifier' && fn.text === 'fetch') return true; + } + for (let i = 0; i < node.childCount; i++) { + if (checkForFetchCall(node.child(i)!, depth + 1, maxDepth)) return true; + } + return false; +}; + // HTTP client methods that are ONLY used by clients, not Express route registration. // Methods like get/post/put/delete/patch overlap with Express — those are captured by // the express_route handler as route definitions, not consumers. The fetch() global @@ -944,6 +1001,18 @@ export function extractORMQueries( } } +// ============================================================================ +// FastAPI router prefix detection (Python) +// ============================================================================ +// +// The extraction lives in `../route-extractors/fastapi-router-bindings` +// (a pure-function module — NOT a worker, no `worker_threads`, no +// `parentPort`). It's imported here only so the worker entry can call it +// per file; this module does not re-export it. Downstream consumers +// import the function and its types directly from `route-extractors/`. + +import { extractFastAPIRouterBindings } from '../route-extractors/fastapi-router-bindings.js'; + const processFileGroup = ( files: ParseWorkerInput[], language: SupportedLanguages, @@ -1176,6 +1245,7 @@ const processFileGroup = ( if (captureMap['decorator'] && captureMap['decorator.name']) { const decoratorName = captureMap['decorator.name'].text; const decoratorArg = captureMap['decorator.arg']?.text; + const decoratorReceiver = captureMap['decorator.receiver']?.text; const decoratorNode = captureMap['decorator']; // Store by the decorator's end line — the definition follows immediately after fileDecorators.set(decoratorNode.endPosition.row, { @@ -1195,6 +1265,7 @@ const processFileGroup = ( httpMethod, decoratorName, lineNumber: decoratorNode.startPosition.row + lineOffset, + ...(decoratorReceiver ? { decoratorReceiver } : {}), }); } // MCP/RPC tool detection: @mcp.tool(), @app.tool(), @server.tool() @@ -1944,6 +2015,21 @@ const processFileGroup = ( : '', }); } + + // ── Fetch wrapper detection: record functions that call fetch() internally ── + if ( + nodeLabel === 'Function' && + definitionNode && + nameNode && + (language === SupportedLanguages.TypeScript || language === SupportedLanguages.JavaScript) + ) { + if (checkForFetchCall(definitionNode)) { + result.fetchWrapperDefs.push({ + filePath: file.path, + functionName: nameNode.text, + }); + } + } } // Extract framework routes via provider detection (e.g., Laravel routes.php) @@ -1955,6 +2041,20 @@ const processFileGroup = ( // Extract ORM queries (Prisma, Supabase) extractORMQueries(file.path, parseContent, result.ormQueries); + // Extract FastAPI include_router(prefix=...) and `from import router` + // sites. parse-impl aggregates these into a per-module prefix map and + // injects the resolved prefix onto each ExtractedDecoratorRoute that + // came from a `@router.` decorator. Python-only. + if (language === SupportedLanguages.Python) { + extractFastAPIRouterBindings( + file.path, + parseContent, + result.routerIncludes, + result.routerImports, + (result.routerModuleAliases ??= []), + ); + } + // Vue: emit CALLS edges for components used in