diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 719473def..9d95ede9f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.3.3", + "version": "1.6.7", "source": "./gitnexus-claude-plugin", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase." } diff --git a/.claude/skills/gitnexus/gitnexus-debugging/SKILL.md b/.claude/skills/gitnexus/gitnexus-debugging/SKILL.md index 937b5e2a4..9834f94b7 100644 --- a/.claude/skills/gitnexus/gitnexus-debugging/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-debugging/SKILL.md @@ -16,10 +16,10 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ## Workflow ``` -1. gitnexus_query({query: ""}) → Find related execution flows -2. gitnexus_context({name: ""}) → See callers/callees/processes +1. query({query: ""}) → Find related execution flows +2. context({name: ""}) → See callers/callees/processes 3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow -4. gitnexus_cypher({query: "MATCH path..."}) → Custom traces if needed +4. cypher({query: "MATCH path..."}) → Custom traces if needed ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. @@ -28,11 +28,11 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ``` - [ ] Understand the symptom (error message, unexpected behavior) -- [ ] gitnexus_query for error text or related code +- [ ] query for error text or related code - [ ] Identify the suspect function from returned processes -- [ ] gitnexus_context to see callers and callees +- [ ] context to see callers and callees - [ ] Trace execution flow via process resource if applicable -- [ ] gitnexus_cypher for custom call chain traces if needed +- [ ] cypher for custom call chain traces if needed - [ ] Read source files to confirm root cause ``` @@ -40,7 +40,7 @@ description: "Use when the user is debugging a bug, tracing an error, or asking | Symptom | GitNexus Approach | | -------------------- | ---------------------------------------------------------- | -| Error message | `gitnexus_query` for error text → `context` on throw sites | +| Error message | `query` for error text → `context` on throw sites | | Wrong return value | `context` on the function → trace callees for data flow | | Intermittent failure | `context` → look for external calls, async deps | | Performance issue | `context` → find symbols with many callers (hot paths) | @@ -48,24 +48,24 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ## Tools -**gitnexus_query** — find code related to error: +**query** — find code related to error: ``` -gitnexus_query({query: "payment validation error"}) +query({query: "payment validation error"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError, PaymentException ``` -**gitnexus_context** — full context for a suspect: +**context** — full context for a suspect: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates (external API!) → Processes: CheckoutFlow (step 3/7) ``` -**gitnexus_cypher** — custom call chain traces: +**cypher** — custom call chain traces: ```cypher MATCH path = (a)-[:CodeRelation {type: 'CALLS'}*1..2]->(b:Function {name: "validatePayment"}) @@ -75,11 +75,11 @@ RETURN [n IN nodes(path) | n.name] AS chain ## Example: "Payment endpoint returns 500 intermittently" ``` -1. gitnexus_query({query: "payment error handling"}) +1. query({query: "payment error handling"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError -2. gitnexus_context({name: "validatePayment"}) +2. context({name: "validatePayment"}) → Outgoing calls: verifyCard, fetchRates (external API!) 3. READ gitnexus://repo/my-app/process/CheckoutFlow diff --git a/.claude/skills/gitnexus/gitnexus-exploring/SKILL.md b/.claude/skills/gitnexus/gitnexus-exploring/SKILL.md index 2dcf7b578..ccf684c28 100644 --- a/.claude/skills/gitnexus/gitnexus-exploring/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-exploring/SKILL.md @@ -18,8 +18,8 @@ description: "Use when the user asks how code works, wants to understand archite ``` 1. READ gitnexus://repos → Discover indexed repos 2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. gitnexus_query({query: ""}) → Find related execution flows -4. gitnexus_context({name: ""}) → Deep dive on specific symbol +3. query({query: ""}) → Find related execution flows +4. context({name: ""}) → Deep dive on specific symbol 5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow ``` @@ -29,9 +29,9 @@ description: "Use when the user asks how code works, wants to understand archite ``` - [ ] READ gitnexus://repo/{name}/context -- [ ] gitnexus_query for the concept you want to understand +- [ ] query for the concept you want to understand - [ ] Review returned processes (execution flows) -- [ ] gitnexus_context on key symbols for callers/callees +- [ ] context on key symbols for callers/callees - [ ] READ process resource for full execution traces - [ ] Read source files for implementation details ``` @@ -47,18 +47,18 @@ description: "Use when the user asks how code works, wants to understand archite ## Tools -**gitnexus_query** — find execution flows related to a concept: +**query** — find execution flows related to a concept: ``` -gitnexus_query({query: "payment processing"}) +query({query: "payment processing"}) → Processes: CheckoutFlow, RefundFlow, WebhookHandler → Symbols grouped by flow with file locations ``` -**gitnexus_context** — 360-degree view of a symbol: +**context** — 360-degree view of a symbol: ``` -gitnexus_context({name: "validateUser"}) +context({name: "validateUser"}) → Incoming calls: loginHandler, apiMiddleware → Outgoing calls: checkToken, getUserById → Processes: LoginFlow (step 2/5), TokenRefresh (step 1/3) @@ -68,10 +68,10 @@ gitnexus_context({name: "validateUser"}) ``` 1. READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. gitnexus_query({query: "payment processing"}) +2. query({query: "payment processing"}) → CheckoutFlow: processPayment → validateCard → chargeStripe → RefundFlow: initiateRefund → calculateRefund → processRefund -3. gitnexus_context({name: "processPayment"}) +3. context({name: "processPayment"}) → Incoming: checkoutHandler, webhookHandler → Outgoing: validateCard, chargeStripe, saveTransaction 4. Read src/payments/processor.ts for implementation details diff --git a/.claude/skills/gitnexus/gitnexus-guide/SKILL.md b/.claude/skills/gitnexus/gitnexus-guide/SKILL.md index b81900b5e..cacc4e886 100644 --- a/.claude/skills/gitnexus/gitnexus-guide/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-guide/SKILL.md @@ -38,7 +38,38 @@ For any task involving code understanding, debugging, impact analysis, or refact | `detect_changes` | Git-diff impact — what do your current changes affect | | `rename` | Multi-file coordinated rename with confidence-tagged edits | | `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | +| `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | + +### Paginating `list_repos` + +`list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: + +```jsonc +{ + "repositories": [ + { "name": "...", "path": "...", "indexedAt": "...", "lastCommit": "...", "stats": { } } + ], + "pagination": { + "total": 437, + "limit": 50, + "offset": 0, + "returned": 50, + "hasMore": true, + "nextOffset": 50 + } +} +``` + +To enumerate **every** repository, keep calling with `offset` set to `pagination.nextOffset` until `hasMore` is `false`: + +```text +list_repos {} → repos 1–50, nextOffset 50, hasMore true +list_repos { offset: 50 } → repos 51–100, nextOffset 100, hasMore true +… +list_repos { offset: 400 } → repos 401–437, hasMore false (done) +``` + +Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. ## Resources Reference diff --git a/.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md b/.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md index 7206ca506..45eb7ce87 100644 --- a/.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md @@ -17,9 +17,9 @@ description: "Use when the user wants to know what will break if they change som ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) → What depends on this 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. gitnexus_detect_changes() → Map current git changes to affected flows +3. detect_changes() → Map current git changes to affected flows 4. Assess risk and report to user ``` @@ -28,11 +28,11 @@ description: "Use when the user wants to know what will break if they change som ## Checklist ``` -- [ ] gitnexus_impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] gitnexus_detect_changes() for pre-commit check +- [ ] detect_changes() for pre-commit check - [ ] Assess risk level and report to user ``` @@ -55,10 +55,10 @@ description: "Use when the user wants to know what will break if they change som ## Tools -**gitnexus_impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius: ``` -gitnexus_impact({ +impact({ target: "validateUser", direction: "upstream", minConfidence: 0.8, @@ -73,10 +73,10 @@ gitnexus_impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**gitnexus_detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis: ``` -gitnexus_detect_changes({scope: "staged"}) +detect_changes({scope: "staged"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -86,7 +86,7 @@ gitnexus_detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. gitnexus_impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/.claude/skills/gitnexus/gitnexus-pr-review/SKILL.md b/.claude/skills/gitnexus/gitnexus-pr-review/SKILL.md index 319c063f9..9f1d362e5 100644 --- a/.claude/skills/gitnexus/gitnexus-pr-review/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-pr-review/SKILL.md @@ -18,10 +18,10 @@ description: "Use when the user wants to review a pull request, understand what ``` 1. gh pr diff → Get the raw diff -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows +2. detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows 3. For each changed symbol: - gitnexus_impact({target: "", direction: "upstream"}) → Blast radius per change -4. gitnexus_context({name: ""}) → Understand callers/callees + impact({target: "", direction: "upstream"}) → Blast radius per change +4. context({name: ""}) → Understand callers/callees 5. READ gitnexus://repo/{name}/processes → Check affected execution flows 6. Summarize findings with risk assessment ``` @@ -32,10 +32,10 @@ description: "Use when the user wants to review a pull request, understand what ``` - [ ] Fetch PR diff (gh pr diff or git diff base...head) -- [ ] gitnexus_detect_changes to map changes to affected execution flows -- [ ] gitnexus_impact on each non-trivial changed symbol +- [ ] detect_changes to map changes to affected execution flows +- [ ] impact on each non-trivial changed symbol - [ ] Review d=1 items (WILL BREAK) — are callers updated? -- [ ] gitnexus_context on key changed symbols to understand full picture +- [ ] context on key changed symbols to understand full picture - [ ] Check if affected processes have test coverage - [ ] Assess overall risk level - [ ] Write review summary with findings @@ -63,20 +63,20 @@ description: "Use when the user wants to review a pull request, understand what ## Tools -**gitnexus_detect_changes** — map PR diff to affected execution flows: +**detect_changes** — map PR diff to affected execution flows: ``` -gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +detect_changes({scope: "compare", base_ref: "main"}) → Changed: 8 symbols in 4 files → Affected processes: CheckoutFlow, RefundFlow, WebhookHandler → Risk: MEDIUM ``` -**gitnexus_impact** — blast radius per changed symbol: +**impact** — blast radius per changed symbol: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream"}) +impact({target: "validatePayment", direction: "upstream"}) → d=1 (WILL BREAK): - processCheckout (src/checkout.ts:42) [CALLS, 100%] @@ -86,20 +86,20 @@ gitnexus_impact({target: "validatePayment", direction: "upstream"}) - checkoutRouter (src/routes/checkout.ts:22) [CALLS, 95%] ``` -**gitnexus_impact with tests** — check test coverage: +**impact with tests** — check test coverage: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream", includeTests: true}) +impact({target: "validatePayment", direction: "upstream", includeTests: true}) → Tests that cover this symbol: - validatePayment.test.ts [direct] - checkout.integration.test.ts [via processCheckout] ``` -**gitnexus_context** — understand a changed symbol's role: +**context** — understand a changed symbol's role: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates @@ -112,20 +112,20 @@ gitnexus_context({name: "validatePayment"}) 1. gh pr diff 42 > /tmp/pr42.diff → 4 files changed: payments.ts, checkout.ts, types.ts, utils.ts -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +2. detect_changes({scope: "compare", base_ref: "main"}) → Changed symbols: validatePayment, PaymentInput, formatAmount → Affected processes: CheckoutFlow, RefundFlow → Risk: MEDIUM -3. gitnexus_impact({target: "validatePayment", direction: "upstream"}) +3. impact({target: "validatePayment", direction: "upstream"}) → d=1: processCheckout, webhookHandler (WILL BREAK) → webhookHandler is NOT in the PR diff — potential breakage! -4. gitnexus_impact({target: "PaymentInput", direction: "upstream"}) +4. impact({target: "PaymentInput", direction: "upstream"}) → d=1: validatePayment (in PR), createPayment (NOT in PR) → createPayment uses the old PaymentInput shape — breaking change! -5. gitnexus_context({name: "formatAmount"}) +5. context({name: "formatAmount"}) → Called by 12 functions — but change is backwards-compatible (added optional param) 6. Review summary: diff --git a/.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md b/.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md index c749eb384..e13c04e14 100644 --- a/.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md @@ -16,9 +16,9 @@ description: "Use when the user wants to rename, extract, split, move, or restru ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → Map all dependents -2. gitnexus_query({query: "X"}) → Find execution flows involving X -3. gitnexus_context({name: "X"}) → See all incoming/outgoing refs +1. impact({target: "X", direction: "upstream"}) → Map all dependents +2. query({query: "X"}) → Find execution flows involving X +3. context({name: "X"}) → See all incoming/outgoing refs 4. Plan update order: interfaces → implementations → callers → tests ``` @@ -29,65 +29,65 @@ description: "Use when the user wants to rename, extract, split, move, or restru ### Rename Symbol ``` -- [ ] gitnexus_rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits +- [ ] rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits - [ ] Review graph edits (high confidence) and ast_search edits (review carefully) -- [ ] If satisfied: gitnexus_rename({..., dry_run: false}) — apply edits -- [ ] gitnexus_detect_changes() — verify only expected files changed +- [ ] If satisfied: rename({..., dry_run: false}) — apply edits +- [ ] detect_changes() — verify only expected files changed - [ ] Run tests for affected processes ``` ### Extract Module ``` -- [ ] gitnexus_context({name: target}) — see all incoming/outgoing refs -- [ ] gitnexus_impact({target, direction: "upstream"}) — find all external callers +- [ ] context({name: target}) — see all incoming/outgoing refs +- [ ] impact({target, direction: "upstream"}) — find all external callers - [ ] Define new module interface - [ ] Extract code, update imports -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ### Split Function/Service ``` -- [ ] gitnexus_context({name: target}) — understand all callees +- [ ] context({name: target}) — understand all callees - [ ] Group callees by responsibility -- [ ] gitnexus_impact({target, direction: "upstream"}) — map callers to update +- [ ] impact({target, direction: "upstream"}) — map callers to update - [ ] Create new functions/services - [ ] Update callers -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ## Tools -**gitnexus_rename** — automated multi-file rename: +**rename** — automated multi-file rename: ``` -gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits across 8 files → 10 graph edits (high confidence), 2 ast_search edits (review) → Changes: [{file_path, edits: [{line, old_text, new_text, confidence}]}] ``` -**gitnexus_impact** — map all dependents first: +**impact** — map all dependents first: ``` -gitnexus_impact({target: "validateUser", direction: "upstream"}) +impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware, testUtils → Affected Processes: LoginFlow, TokenRefresh ``` -**gitnexus_detect_changes** — verify your changes after refactoring: +**detect_changes** — verify your changes after refactoring: ``` -gitnexus_detect_changes({scope: "all"}) +detect_changes({scope: "all"}) → Changed: 8 files, 12 symbols → Affected processes: LoginFlow, TokenRefresh → Risk: MEDIUM ``` -**gitnexus_cypher** — custom reference queries: +**cypher** — custom reference queries: ```cypher MATCH (caller)-[:CodeRelation {type: 'CALLS'}]->(f:Function {name: "validateUser"}) @@ -98,24 +98,24 @@ RETURN caller.name, caller.filePath ORDER BY caller.filePath | Risk Factor | Mitigation | | ------------------- | ----------------------------------------- | -| Many callers (>5) | Use gitnexus_rename for automated updates | +| Many callers (>5) | Use rename for automated updates | | Cross-area refs | Use detect_changes after to verify scope | -| String/dynamic refs | gitnexus_query to find them | +| String/dynamic refs | query to find them | | External/public API | Version and deprecate properly | ## Example: Rename `validateUser` to `authenticateUser` ``` -1. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +1. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits: 10 graph (safe), 2 ast_search (review) → Files: validator.ts, login.ts, middleware.ts, config.json... 2. Review ast_search edits (config.json: dynamic reference!) -3. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) +3. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) → Applied 12 edits across 8 files -4. gitnexus_detect_changes({scope: "all"}) +4. detect_changes({scope: "all"}) → Affected: LoginFlow, TokenRefresh → Risk: MEDIUM — run tests for these flows ``` diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 8817c1773..103a7fa44 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -310,8 +310,8 @@ VS Code's Ports panel shows forwarded ports once their listener starts. - **LadybugDB integration tests may fail in containers** (file-locking, `AGENTS.md` § Testing). Default to `npm run test:unit` inside the container; run integration tests on the host. Tracking issue: documented as a known limitation. - **Single-writer LadybugDB constraint** (`GUARDRAILS.md` § LadybugDB lock). Don't run `gitnexus analyze` on the host and inside the container against the same `.gitnexus/` directory simultaneously — the second writer will get `database busy`. -- **Native grammar builds add ~30s to first install.** Tree-sitter Dart/Proto/Swift grammars build during `gitnexus`'s `postinstall`. To skip them (loses parsing for those three languages), set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` in your shell or add it to `remoteEnv` and rebuild. -- **`tree-sitter-kotlin` warnings on install** are expected (per `AGENTS.md`). Ignore them. +- **Native grammar builds add ~30s to first install.** Tree-sitter Dart/Proto/Swift/Kotlin are all vendored uniformly: `node-gyp-build` picks a committed GitNexus-built prebuilt `.node` at install time (no compile), and only falls back to compiling from the vendored source during `postinstall` if no prebuild matches the host (then a toolchain is needed). Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` (in your shell or `remoteEnv`, then rebuild) to skip all four; each loses parsing for the affected language(s), and the install still succeeds. +- **`tree-sitter-kotlin`/`tree-sitter-swift` warnings on install** only appear when no prebuild matches the platform-arch (per `AGENTS.md`); they are non-fatal — parsing for that language is simply unavailable. - **`.mcp.json` works inside the container**: `npx -y gitnexus@latest mcp` resolves cleanly because npm registry is reachable and the workspace bind mount exposes the same `.mcp.json` the host sees. - **Husky pre-commit fires inside the container** without extra setup. The root `npm install` (run automatically in `postCreateCommand`) installs the hook via `package.json` `prepare`. diff --git a/.github/scripts/update-vendored-grammars.mjs b/.github/scripts/update-vendored-grammars.mjs new file mode 100644 index 000000000..957200e77 --- /dev/null +++ b/.github/scripts/update-vendored-grammars.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +/** + * Vendored tree-sitter grammar update monitor. + * + * Checks each vendored grammar against its upstream source-of-origin and, for an + * available AND ABI-compatible update, re-vendors the grammar source in place so + * a PR can be opened. The version bump in vendor//package.json then triggers + * .github/workflows/build-tree-sitter-prebuilds.yml, which cross-builds + ABI- + * validates the prebuilds — so even an imperfect re-vendor can never silently + * ship: its PR's CI goes red. + * + * ABI awareness is load-bearing. Every grammar is pinned to tree-sitter@0.21.1 + * (LANGUAGE_VERSION 13–14, the #1922 gate). Most upstream grammar releases target + * a newer tree-sitter, so a blind "bump to latest" would pull an ABI-incompatible + * parser and open doomed PRs. This monitor fetches the candidate source, reads its + * parser.c `#define LANGUAGE_VERSION`, and only re-vendors when it is 13 or 14; + * incompatible updates are reported (and surfaced as a workflow notice), not + * applied. + * + * Usage: + * node update-vendored-grammars.mjs # detect only → JSON report on stdout + * node update-vendored-grammars.mjs --apply X # re-vendor grammar X in place + * + * tree-sitter-c is MONITORED but report-only (`hold`): it is ABI-pinned at 0.21.4 + * (#1242/#858) and must not auto-bump without a tree-sitter runtime upgrade, so an + * available c update is detected + reported but never auto-applied — even if it is + * ABI-13/14. A maintainer re-vendors it deliberately. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const VENDOR = path.join(REPO_ROOT, 'gitnexus', 'vendor'); + +const COMPATIBLE_ABI = new Set([13, 14]); // tree-sitter@0.21.1 LANGUAGE_VERSION range + +// Source-of-origin per grammar. npm grammars resolve `latest` via the registry; +// github grammars (no usable npm release) track the default branch HEAD. A `hold` +// reason makes a grammar report-only: updates are detected + surfaced but never +// auto-applied (c is ABI-pinned and must not move without a runtime upgrade). +const GRAMMARS = { + c: { + name: 'tree-sitter-c', + npm: 'tree-sitter-c', + hold: 'ABI-pinned at 0.21.4 (#1242/#858) — needs a tree-sitter runtime upgrade before bumping', + }, + swift: { name: 'tree-sitter-swift', npm: 'tree-sitter-swift' }, + kotlin: { name: 'tree-sitter-kotlin', npm: 'tree-sitter-kotlin' }, + dart: { name: 'tree-sitter-dart', github: 'UserNobody14/tree-sitter-dart' }, + proto: { name: 'tree-sitter-proto', github: 'coder3101/tree-sitter-proto' }, +}; + +const sh = (cmd, args, opts = {}) => + execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }).trim(); + +const clean = (v) => + String(v || '') + .replace(/^[v^~]/, '') + .trim(); + +function vendoredVersion(g) { + const p = path.join(VENDOR, g.name, 'package.json'); + return clean(JSON.parse(fs.readFileSync(p, 'utf8')).version); +} + +/** Resolve the upstream candidate: { version, ref, kind }. */ +function resolveUpstream(g) { + if (g.npm) { + const version = clean(sh('npm', ['view', g.npm, 'version'])); + return { version, ref: version, kind: 'npm' }; + } + // github: no reliable release tags here, so track the default branch HEAD sha. + const meta = JSON.parse(sh('gh', ['api', `repos/${g.github}`])); + const branch = meta.default_branch; + const sha = JSON.parse(sh('gh', ['api', `repos/${g.github}/commits/${branch}`])).sha; + // Version key: "-g" — safeRef-compatible (no `+`, + // which the build workflow's ref validator rejects) and changes on every commit. + let base = '0.0.0'; + try { + const pkg = JSON.parse( + Buffer.from( + JSON.parse(sh('gh', ['api', `repos/${g.github}/contents/package.json?ref=${sha}`])).content, + 'base64', + ).toString('utf8'), + ); + if (pkg.version) base = clean(pkg.version); + } catch { + /* no upstream package.json — base stays 0.0.0 */ + } + return { version: `${base}-g${sha.slice(0, 7)}`, ref: sha, kind: 'github' }; +} + +/** Fetch the candidate source into a temp dir; return the package root. */ +function fetchSource(g, ref) { + const work = fs.mkdtempSync( + path.join(os.tmpdir(), `revendor-${Object.keys(GRAMMARS).find((k) => GRAMMARS[k] === g)}-`), + ); + if (g.npm) { + sh('npm', ['pack', `${g.npm}@${ref}`, '--silent'], { cwd: work }); + const tgz = fs.readdirSync(work).find((f) => f.endsWith('.tgz')); + sh('tar', ['xzf', tgz], { cwd: work }); + return path.join(work, 'package'); + } + // github tarball at the resolved sha. Download + extract WITHOUT a shell + // (no `bash -c`/redirect): `gh api` writes the binary tarball to stdout, which + // we capture as a Buffer and write to a fixed path, then extract with execFile. + // Avoids the shell-command-injection surface CodeQL flags when an API-derived + // ref is interpolated into a `bash -c` string. + const tgz = path.join(work, 'src.tgz'); + fs.writeFileSync( + tgz, + execFileSync('gh', ['api', `repos/${g.github}/tarball/${ref}`], { + maxBuffer: 512 * 1024 * 1024, + }), + ); + sh('tar', ['xzf', tgz], { cwd: work }); + const dir = fs.readdirSync(work).find((f) => fs.statSync(path.join(work, f)).isDirectory()); + return path.join(work, dir); +} + +/** Read parser.c's LANGUAGE_VERSION (ABI). Prefer the ABI-14 default parser.c. */ +function readAbi(srcRoot) { + const candidates = ['src/parser.c', 'parser.c']; + for (const rel of candidates) { + const p = path.join(srcRoot, rel); + if (!fs.existsSync(p)) continue; + // Read only the head — the #define is near the top. + const head = fs.readFileSync(p, 'utf8').slice(0, 4000); + const m = head.match(/#define\s+LANGUAGE_VERSION\s+(\d+)/); + if (m) return Number(m[1]); + } + return null; // unknown (e.g. parser.c only generated at build time) +} + +function detect() { + const report = []; + for (const [key, g] of Object.entries(GRAMMARS)) { + const have = vendoredVersion(g); + let up; + try { + up = resolveUpstream(g); + } catch (err) { + report.push({ grammar: key, error: String(err.message || err) }); + continue; + } + const newer = up.kind === 'npm' ? up.version !== have : !have || up.ref.slice(0, 7) !== have; + let abi = null; + if (newer) { + try { + abi = readAbi(fetchSource(g, up.ref)); + } catch { + /* fetch/abi best-effort; null = unknown */ + } + } + report.push({ + grammar: key, + vendored: have, + upstream: up.version, + ref: up.ref, + kind: up.kind, + update: newer, + abi, + abiCompatible: abi == null ? null : COMPATIBLE_ABI.has(abi), + hold: g.hold || null, + // Auto-appliable only when there's an update, the ABI is known-compatible, + // AND the grammar is not on a policy hold (c). + applicable: newer && abi != null && COMPATIBLE_ABI.has(abi) && !g.hold, + }); + } + return report; +} + +const copyFile = (srcRoot, dest, rel) => { + const from = path.join(srcRoot, rel); + if (!fs.existsSync(from)) return false; + const to = path.join(dest, rel); + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.copyFileSync(from, to); + return true; +}; + +/** + * Re-vendor one grammar in place from its ABI-compatible upstream candidate. + * Copies ONLY the generated source-build + runtime files; deliberately KEEPS the + * GitNexus-hardened binding.gyp (Windows cflags, target_name), README (vendor + * notice), LICENSE, and prebuilds/ (the build workflow refreshes those). Bumps the + * stripped vendor package.json version + provenance — never re-introduces + * scripts/dependencies (#836/#1728). Returns the new version. + */ +function apply(key) { + const g = GRAMMARS[key]; + if (!g) { + console.error(`unknown grammar '${key}'`); + process.exit(2); + } + if (g.hold) { + console.error( + `${key}: report-only (${g.hold}); not auto-applied. Re-vendor manually if intended.`, + ); + process.exit(3); + } + const have = vendoredVersion(g); + const up = resolveUpstream(g); + const newer = up.kind === 'npm' ? up.version !== have : !have || up.version !== have; + if (!newer) { + console.error(`${key}: already current (${have}); nothing to apply.`); + process.exit(0); + } + const srcRoot = fetchSource(g, up.ref); + const abi = readAbi(srcRoot); + if (abi == null || !COMPATIBLE_ABI.has(abi)) { + console.error( + `${key}: candidate ${up.version} is ABI ${abi ?? 'unknown'} — not tree-sitter@0.21.1 ` + + `compatible (need 13/14); refusing to re-vendor. Handle manually.`, + ); + process.exit(3); + } + + const dest = path.join(VENDOR, g.name); + // The source-build inputs + runtime entrypoints that change between versions. + // binding.gyp / README / LICENSE / prebuilds are intentionally NOT touched. + for (const rel of [ + 'src/parser.c', + 'src/scanner.c', + 'src/node-types.json', + 'src/tree_sitter/alloc.h', + 'src/tree_sitter/array.h', + 'src/tree_sitter/parser.h', + 'bindings/node/binding.cc', + 'bindings/node/index.js', + 'bindings/node/index.d.ts', + ]) { + copyFile(srcRoot, dest, rel); + } + + const pkgPath = path.join(dest, 'package.json'); + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')); + pkg.version = up.version; + pkg._vendoredBy = + `gitnexus - re-vendored from ${g.npm ? `npm ${g.npm}@${up.version}` : `${g.github}@${up.ref}`} ` + + `by grammar-update-monitor on ABI ${abi}. Source-build inputs (parser.c/scanner.c/src/) refreshed; ` + + `the GitNexus-hardened binding.gyp + vendor README + prebuilds are preserved (prebuilds are ` + + `rebuilt by build-tree-sitter-prebuilds.yml on this version change). No scripts/dependencies here ` + + `(#836/#1728).`; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n'); + + console.log(`${key}: re-vendored ${g.name} → ${up.version} (ABI ${abi}).`); + return up.version; +} + +// Run the CLI only when invoked directly (not when imported by a test) — detect() +// makes live network calls, so importing must be side-effect-free. +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (isMain) { + if (process.argv[2] === '--apply') { + apply(process.argv[3]); + } else { + process.stdout.write(JSON.stringify(detect(), null, 2) + '\n'); + } +} + +export { detect, apply, resolveUpstream, readAbi, vendoredVersion, GRAMMARS, COMPATIBLE_ABI }; diff --git a/.github/workflows/build-tree-sitter-prebuilds.yml b/.github/workflows/build-tree-sitter-prebuilds.yml new file mode 100644 index 000000000..0acfca61a --- /dev/null +++ b/.github/workflows/build-tree-sitter-prebuilds.yml @@ -0,0 +1,529 @@ +name: Build tree-sitter prebuilds + +# Cross-builds the native tree-sitter prebuilds GitNexus vendors itself, so that +# grammars whose upstream packages ship SOURCE ONLY (no usable prebuilds/) never +# require a C/C++ toolchain at a user's install. This is the "no operational +# risk for any tree-sitter grammar" pipeline. +# +# Grammars covered here (the at-risk set — everything else already ships 6 +# upstream prebuilds AND stays dependency-review-tracked, so it is left alone). +# All five are vendored under gitnexus/vendor/; `kind` (below) only picks where +# the build job fetches the C source to compile: +# - tree-sitter-c (vendored prebuild-only; built from the published npm +# package — closes upstream's 4/6 ARM gap #2116 for a +# REQUIRED grammar) +# - tree-sitter-dart (vendored source; built from gitnexus/vendor/) +# - tree-sitter-proto (vendored source; built from gitnexus/vendor/) +# - tree-sitter-kotlin (vendored source; built from the published npm package — +# upstream ships source only) +# - tree-sitter-swift (vendored source; built from gitnexus/vendor/ — its +# prebuilds were originally upstream-shipped, now +# GitNexus-cross-built like the rest for uniformity) +# +# Output: gitnexus/vendor//prebuilds//.node for +# all 6 targets ({linux,darwin,win32}-{x64,arm64}). tree-sitter grammars are +# N-API, so one ABI-stable .node per platform-arch works across all Node majors. +# +# COST DISCIPLINE — this is a HEAVY native matrix (up to 3 grammars x 6 runners, +# incl. macOS + arm64). It is DELIBERATELY NOT wired into normal PR/push CI. It +# runs only: +# 1. on manual dispatch (workflow_dispatch); or +# 2. when a covered grammar's recorded version actually CHANGES — the `guard` +# job is the real gate (it diffs the recorded version vs the PR base); the +# `paths:` filter below only makes ordinary code PRs cost ZERO matrix time. +# Net effect: an ordinary code PR triggers nothing; bumping one grammar costs +# exactly one matrix run for that grammar, which opens a PR committing its rebuilt +# binaries. +# +# Concurrency convention: see CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention". +# +# NOTE: every action below is pinned to a release commit SHA (with the matching +# `# vX.Y.Z` tag comment verified against the GitHub API). If a future bump adds +# a new action, pin its real release SHA and allowlist it in .github/zizmor.yml / +# Scorecard before merge. + +on: + workflow_dispatch: + inputs: + grammars: + description: 'Comma-separated grammar shortnames to build (c,dart,proto,kotlin,swift), or "all".' + required: false + type: string + default: 'all' + ref: + description: 'Upstream version/tag/sha override (only honored when exactly one grammar is selected).' + required: false + type: string + default: '' + force: + description: 'Build even if the recorded version is unchanged (re-cut a broken prebuild).' + required: false + type: boolean + default: false + open_pr: + description: 'Open a PR with the rebuilt prebuilds (false = artifacts only).' + required: false + type: boolean + default: true + pull_request: + branches: [main] + paths: + # Vendored grammars: their version lives in the vendor snapshot package.json. + - 'gitnexus/vendor/tree-sitter-c/package.json' + - 'gitnexus/vendor/tree-sitter-dart/package.json' + - 'gitnexus/vendor/tree-sitter-proto/package.json' + - 'gitnexus/vendor/tree-sitter-kotlin/package.json' + - 'gitnexus/vendor/tree-sitter-swift/package.json' + # Transition window: kotlin's pin still lives here until it is vendored. + - 'gitnexus/package.json' + # Self-test: re-run the guard (normally a no-op) when the recipe changes. + - '.github/workflows/build-tree-sitter-prebuilds.yml' + +# Least privilege by default; only `aggregate` opts up. +permissions: + contents: read + +# One slot per ref. Collapse PR re-pushes, but never cancel a manual re-cut. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # ── Gate: decide which grammars (if any) need a native rebuild, and emit the + # {grammar x platform-arch} matrix the build job consumes. ─────────────── + guard: + name: Decide what to build + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + any: ${{ steps.decide.outputs.any }} + matrix: ${{ steps.decide.outputs.matrix }} + release_app: ${{ steps.relapp.outputs.configured }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 # need base history to diff recorded versions + persist-credentials: false + + - name: Decide + id: decide + env: + EVENT: ${{ github.event_name }} + # Untrusted dispatch inputs — read via env only, validated in JS. + INPUT_GRAMMARS: ${{ inputs.grammars }} + INPUT_REF: ${{ inputs.ref }} + FORCE: ${{ github.event_name == 'workflow_dispatch' && inputs.force || 'false' }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + node --input-type=module - <<'NODE' + import { execSync } from 'node:child_process'; + import fs from 'node:fs'; + import { appendFileSync } from 'node:fs'; + + // Registry of the at-risk grammars this workflow owns. `kind` drives + // how the build job resolves source: 'npm' pulls the published package; + // 'vendored' builds from gitnexus/vendor/ (which carries the C + // source + binding.gyp). Extend this list to cover a new grammar. + const REGISTRY = { + // c is vendored prebuild-only but BUILT from the published npm + // package (kind 'npm'), held at 0.21.4 — it closes upstream's 4/6 + // ARM gap (#2116) for a REQUIRED grammar that otherwise hard-fails + // install on toolchain-less ARM. + c: { name: 'tree-sitter-c', kind: 'npm' }, + dart: { name: 'tree-sitter-dart', kind: 'vendored' }, + proto: { name: 'tree-sitter-proto', kind: 'vendored' }, + kotlin: { name: 'tree-sitter-kotlin', kind: 'npm' }, + // swift is vendored WITH its source (parser.c/scanner.c/binding.gyp), + // so it builds from gitnexus/vendor/ like dart/proto. Its prebuilds + // were originally upstream-shipped; rebuilding them here unifies it. + swift: { name: 'tree-sitter-swift', kind: 'vendored' }, + }; + const PLATFORMS = [ + { platform_arch: 'linux-x64', os: 'ubuntu-24.04' }, + { platform_arch: 'linux-arm64', os: 'ubuntu-24.04-arm' }, + { platform_arch: 'darwin-arm64', os: 'macos-15' }, + { platform_arch: 'darwin-x64', os: 'macos-15-intel' }, // macos-13 retired Dec-2025; Intel EOL ~Aug-2027 + { platform_arch: 'win32-x64', os: 'windows-2022' }, + { platform_arch: 'win32-arm64', os: 'windows-11-arm' }, + ]; + + const clean = (v) => (v || '').replace(/^[\^~]/, '').trim(); + const json = (p) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; } }; + + // Durable version key for a grammar at a checkout root. Prefer the + // vendor snapshot (the post-vendor source of truth); fall back to the + // optionalDependencies pin during the transition window. (A guard keyed + // on the node_modules lock entry would self-disable once a grammar is + // vendored, because that entry is deleted.) + function recordedVersion(root, name) { + const v = json(`${root}/gitnexus/vendor/${name}/package.json`); + if (v && v.version) return clean(v.version); + const pkg = json(`${root}/gitnexus/package.json`); + const od = pkg && (pkg.optionalDependencies || {}); + const d = pkg && (pkg.dependencies || {}); + return clean((od && od[name]) || (d && d[name]) || ''); + } + + const event = process.env.EVENT; + const force = process.env.FORCE === 'true'; + + // Select which grammar shortnames are in play. + let selected; + if (event === 'workflow_dispatch') { + const raw = (process.env.INPUT_GRAMMARS || 'all').trim(); + selected = raw === 'all' ? Object.keys(REGISTRY) + : raw.split(',').map((s) => s.trim()).filter(Boolean); + for (const s of selected) if (!REGISTRY[s]) throw new Error(`unknown grammar '${s}'`); + } else { + selected = Object.keys(REGISTRY); + } + + // Resolve the base-ref recorded versions (pull_request only) so we can + // diff. On dispatch, base is irrelevant (manual intent / force wins). + const baseRoot = `${process.env.RUNNER_TEMP}/base`; + if (event === 'pull_request') { + const baseSha = process.env.BASE_SHA; + for (const s of selected) { + const name = REGISTRY[s].name; + for (const rel of [`gitnexus/vendor/${name}/package.json`, `gitnexus/package.json`]) { + const dst = `${baseRoot}/${rel}`; + fs.mkdirSync(dst.slice(0, dst.lastIndexOf('/')), { recursive: true }); + try { + const buf = execSync(`git show ${baseSha}:${rel}`, { stdio: ['ignore', 'pipe', 'ignore'] }); + fs.writeFileSync(dst, buf); + } catch { /* file absent at base — fine */ } + } + } + } + + // The single-ref override is only meaningful for a one-grammar dispatch. + const refOverride = clean(process.env.INPUT_REF); + if (refOverride && !(event === 'workflow_dispatch' && selected.length === 1)) { + throw new Error('ref override requires exactly one grammar selected'); + } + const safeRef = (r) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(r); + + const include = []; + const built = []; + for (const short of selected) { + const { name, kind } = REGISTRY[short]; + const head = recordedVersion('.', name); + const ref = refOverride || head; + if (!ref) { console.log(`skip ${short}: no recorded version`); continue; } + if (!safeRef(ref)) throw new Error(`unsafe ref for ${short}: '${ref}'`); + + let build = false; + if (event === 'workflow_dispatch') { + build = true; // manual intent (force toggles only the unchanged-guard, which is bypassed here) + } else { + const base = recordedVersion(baseRoot, name); + build = !!head && head !== base; + console.log(`${short}: head='${head || ''}' base='${base || ''}' -> ${build ? 'BUILD' : 'skip'}`); + } + if (force) build = true; + if (!build) continue; + built.push(short); + for (const p of PLATFORMS) include.push({ grammar: short, name, kind, ref, ...p }); + } + + const out = process.env.GITHUB_OUTPUT; + appendFileSync(out, `any=${include.length > 0}\n`); + appendFileSync(out, `matrix=${JSON.stringify({ include })}\n`); + if (include.length === 0) { + console.log('::notice::No covered grammar version changed — skipping native matrix.'); + } else { + console.log(`Building: ${built.join(', ')} (${include.length} jobs)`); + } + NODE + + # The aggregate job opens a PR via a GitHub App token; without the App + # secrets it would hard-fail AFTER a full native build. Surface their + # presence as a guard output so aggregate skips cleanly (the build job's + # artifacts still upload). secrets aren't available in a job-level `if:`, + # so we compute the boolean here (a step CAN read secrets) and gate on it. + - name: Check release App secret + id: relapp + env: + HAS_APP: ${{ secrets.RELEASE_APP_ID != '' && secrets.RELEASE_APP_PRIVATE_KEY != '' }} + run: | + set -euo pipefail + echo "configured=$HAS_APP" >> "$GITHUB_OUTPUT" + if [ "$HAS_APP" != "true" ]; then + echo "::notice::Release GitHub App secrets (RELEASE_APP_ID / RELEASE_APP_PRIVATE_KEY) are not configured — prebuilds will build and upload as artifacts, but the auto-PR is skipped. Provision the App, or run with open_pr=false to suppress this notice." + fi + + # ── Build one native prebuild per (grammar, platform-arch). No cross-compile. ─ + build: + name: ${{ matrix.grammar }} ${{ matrix.platform_arch }} + needs: guard + if: needs.guard.outputs.any == 'true' + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.guard.outputs.matrix) }} + runs-on: ${{ matrix.os }} + # 45 (not 30) for headroom: the kotlin parser.c is ~23 MB and swift's ~18 MB, + # and compiling them under emulation on the arm runners is slow. + timeout-minutes: 45 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false # this job uploads artifacts (artipacked) + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + + - name: Ensure Python (arm64 Windows only) + if: matrix.platform_arch == 'win32-arm64' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Build prebuild + id: build + shell: bash + env: + GRAMMAR: ${{ matrix.grammar }} + NAME: ${{ matrix.name }} + KIND: ${{ matrix.kind }} + REF: ${{ matrix.ref }} + PLATFORM_ARCH: ${{ matrix.platform_arch }} + run: | + set -euo pipefail + work="$RUNNER_TEMP/ts-build" + rm -rf "$work"; mkdir -p "$work"; cd "$work" + npm init -y >/dev/null + + # node-addon-api must match what the grammar's binding.cc expects. + # GitNexus hoists ^8 for the vendored grammars; npm grammars declare + # their own (do NOT pin it for npm grammars — let the dep resolve it). + if [ "$KIND" = "vendored" ]; then + # Build from the vendored C source (carries parser.c + binding.gyp). + srcdir="$work/$NAME" + cp -R "$GITHUB_WORKSPACE/gitnexus/vendor/$NAME" "$srcdir" + rm -rf "$srcdir/prebuilds" "$srcdir/build" "$srcdir/node_modules" + npm install --no-audit --no-fund --ignore-scripts \ + prebuildify@^6 node-gyp@^11 node-addon-api@^8 + pkgdir="$srcdir" + export npm_config_node_gyp="$work/node_modules/node-gyp/bin/node-gyp.js" + else + # Pull the published source-only package. + npm install --no-audit --no-fund --ignore-scripts \ + "$NAME@${REF}" prebuildify@^6 node-gyp@^11 + pkgdir="$work/node_modules/$NAME" + fi + + test -f "$pkgdir/binding.gyp" || { echo "::error::no binding.gyp for $NAME@$REF"; exit 1; } + + # Drop any prebuilds the package shipped in its own tarball before we + # build. The tree-sitter-org npm grammars (e.g. tree-sitter-c) bundle + # prebuilds/ for all 6 tuples; left in place, the `find ... -print -quit` + # below would pick a non-host tuple (e.g. win32-x64 on a linux runner) + # and the assertion would wrongly fail. prebuildify rebuilds THIS host's + # tuple from the source the tarball also ships. (Vendored grammars are + # already cleaned above; this also covers the npm branch.) + rm -rf "$pkgdir/prebuilds" + + # N-API, stripped, single ABI-stable binary for THIS host's arch. No + # `-t `: an N-API prebuild is Node-version-agnostic, and + # prebuildify parses a bare `-t 22` as the NUMBER 22 and crashes + # (`v.indexOf is not a function`). prebuildify emits + # prebuilds/-/.node. + ( cd "$pkgdir" && npx --no-install prebuildify --napi --strip ) + + out=$(find "$pkgdir/prebuilds" -name '*.node' -print -quit) + test -n "$out" || { echo "::error::prebuildify produced no .node"; exit 1; } + produced=$(basename "$(dirname "$out")") + [ "$produced" = "$PLATFORM_ARCH" ] || { echo "::error::built $produced, expected $PLATFORM_ARCH"; exit 1; } + + stage="$RUNNER_TEMP/stage/$GRAMMAR/$PLATFORM_ARCH"; mkdir -p "$stage" + cp "$out" "$stage/$NAME.node" + echo "stage=$stage" >> "$GITHUB_OUTPUT" + + - name: Validate the .node loads and parses on this arch + shell: bash + env: + GRAMMAR: ${{ matrix.grammar }} + NAME: ${{ matrix.name }} + PLATFORM_ARCH: ${{ matrix.platform_arch }} + EXPECT_ARCH: ${{ contains(matrix.platform_arch, 'arm64') && 'arm64' || 'x64' }} + run: | + set -euo pipefail + probe="$RUNNER_TEMP/probe"; rm -rf "$probe" + mkdir -p "$probe/prebuilds/$PLATFORM_ARCH" + cp "$RUNNER_TEMP/stage/$GRAMMAR/$PLATFORM_ARCH/$NAME.node" \ + "$probe/prebuilds/$PLATFORM_ARCH/$NAME.node" + cd "$probe" + # Pin tree-sitter to the repo's exact runtime peer so an ABI mismatch + # fails HERE, not in a user's install (mirrors the #1922 ABI gate). + # NOT --ignore-scripts: tree-sitter@0.21.1's tarball ships prebuilds for + # the common tuples but NOT linux-arm64 / win32-arm64, so on the arm64 + # runners node-gyp-build must source-build the runtime — give it node-gyp + # + node-addon-api to do so. Where tree-sitter ships a prebuild (x64, + # darwin-arm64) node-gyp-build uses it and nothing compiles. The grammar + # .node we built is still loaded as a prebuild; only the runtime peer may + # compile. The grammar-vs-runtime ABI check still fires at setLanguage. + npm install --no-audit --no-fund \ + node-gyp-build@^4 node-gyp@^11 node-addon-api@^8 tree-sitter@0.21.1 + # The node script is single-quoted on purpose — its ${...} are JS + # template literals read from the environment, not shell expansions. + # shellcheck disable=SC2016 + GRAMMAR="$GRAMMAR" EXPECT_ARCH="$EXPECT_ARCH" node -e ' + const expect = process.env.EXPECT_ARCH; + // Catch an emulated x64 Node silently mis-passing on an arm64 runner. + if (process.arch !== expect) throw new Error(`runner arch ${process.arch} != ${expect}`); + const snippets = { + c: "int main(void) { return 0; }", + dart: "void main() { print(\"hi\"); }", + proto: "syntax = \"proto3\";\nmessage M { int32 id = 1; }", + kotlin: "fun main() { println(\"hi\") }", + swift: "func greet() { print(\"hi\") }", + }; + const lang = require("node-gyp-build")(process.cwd()); + const Parser = require("tree-sitter"); + const p = new Parser(); p.setLanguage(lang); + const tree = p.parse(snippets[process.env.GRAMMAR]); + if (!tree || !tree.rootNode || tree.rootNode.hasError) { + throw new Error("parse failed/error: " + (tree && tree.rootNode && tree.rootNode.type)); + } + console.log("OK", process.env.GRAMMAR, process.platform + "-" + process.arch, tree.rootNode.type); + ' + + - name: Upload prebuild artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ts-prebuild-${{ matrix.grammar }}-${{ matrix.platform_arch }} + path: ${{ steps.build.outputs.stage }}/${{ matrix.name }}.node + if-no-files-found: error + retention-days: 7 + + # ── Aggregate every grammar's six prebuilds, assert completeness, open a PR. ─ + aggregate: + name: Vendor prebuilds + open PR + needs: [guard, build] + # Open the prebuild PR on a non-fork pull_request that bumped a grammar + # version (the documented version-change -> prebuild-PR flow), or on a manual + # dispatch with open_pr=true. Event-gating is explicit so we never rely on + # GHA coercing a null `inputs.open_pr` on pull_request events (Codex F4): + # `inputs.open_pr` is null off-dispatch, and `null != false` is direction- + # ambiguous, so `open_pr` is only consulted on workflow_dispatch. + if: >- + needs.guard.outputs.any == 'true' && + needs.guard.outputs.release_app == 'true' && + github.event.pull_request.head.repo.fork != true && + (github.event_name == 'pull_request' || inputs.open_pr == true) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read # actual writes use a short-lived App token below + id-token: write # SLSA provenance attestation + attestations: write + steps: + - name: Mint GitHub App token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ steps.app-token.outputs.token }} + persist-credentials: false + + - name: Download all prebuild artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ${{ runner.temp }}/dl + pattern: ts-prebuild-* + + - name: Place prebuilds, assert each built grammar has all 6, write SHA256SUMS + id: place + shell: bash + env: + MATRIX: ${{ needs.guard.outputs.matrix }} + DL: ${{ runner.temp }}/dl + run: | + set -euo pipefail + node --input-type=module - <<'NODE' + import fs from 'node:fs'; + import { execSync } from 'node:child_process'; + const include = JSON.parse(process.env.MATRIX).include; + const dl = process.env.DL; + const byGrammar = {}; + for (const e of include) (byGrammar[e.grammar] ||= { name: e.name, archs: [] }).archs.push(e.platform_arch); + const PLATFORMS = ['linux-x64','linux-arm64','darwin-arm64','darwin-x64','win32-x64','win32-arm64']; + const changed = []; + for (const [grammar, { name }] of Object.entries(byGrammar)) { + const dest = `gitnexus/vendor/${name}/prebuilds`; + // A vendored grammar with 5/6 prebuilds silently breaks node-gyp-build + // on the 6th platform — refuse a partial result. + for (const pa of PLATFORMS) { + const art = `${dl}/ts-prebuild-${grammar}-${pa}/${name}.node`; + if (!fs.existsSync(art)) throw new Error(`missing ${grammar} prebuild for ${pa}`); + fs.mkdirSync(`${dest}/${pa}`, { recursive: true }); + fs.copyFileSync(art, `${dest}/${pa}/${name}.node`); + } + execSync(`cd ${dest} && find . -name '*.node' | sort | xargs sha256sum > SHA256SUMS`); + changed.push(name); + } + fs.appendFileSync(process.env.GITHUB_OUTPUT, `grammars=${changed.join(',')}\n`); + console.log('Vendored prebuilds for:', changed.join(', ')); + NODE + + - name: Attest build provenance (SLSA) + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-path: 'gitnexus/vendor/tree-sitter-*/prebuilds/**/*.node' + + - name: Create or update PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GRAMMARS: ${{ steps.place.outputs.grammars }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const { execSync } = require('node:child_process'); + const run = (c) => execSync(c, { stdio: ['ignore', 'pipe', 'inherit'] }).toString().trim(); + const grammars = process.env.GRAMMARS; + const slug = grammars.replace(/[^a-z0-9]+/gi, '-'); + const branch = `chore/vendor-ts-prebuilds-${slug}-${context.runId}`; + + run('git add gitnexus/vendor/tree-sitter-*/prebuilds'); + if (!run('git status --porcelain -- gitnexus/vendor/tree-sitter-*/prebuilds')) { + core.notice('Prebuilds byte-identical to vendor; nothing to commit.'); + return; + } + run('git config user.name "gitnexus-release-bot[bot]"'); + run('git config user.email "gitnexus-release-bot[bot]@users.noreply.github.com"'); + run(`git checkout -b "${branch}"`); + run(`git commit -m "chore(vendor): rebuild native prebuilds (${grammars})\n\nBuilt by ${process.env.RUN_URL}"`); + const { owner, repo } = context.repo; + const remote = `https://x-access-token:${process.env.GH_TOKEN}@github.com/${owner}/${repo}.git`; + // Plain --force, not --force-with-lease: the branch is ephemeral and + // unique per run (keyed by context.runId), written ONLY by this job, so + // there is no concurrent writer to protect against. --force-with-lease + // would compare against a remote-tracking ref this fresh checkout never + // fetched, so re-running the SAME run (branch already pushed by attempt + // 1) fails with "stale info" instead of overwriting. + run(`git push --force "${remote}" "HEAD:${branch}"`); + const body = [ + `Rebuilt the vendored native prebuilds for: **${grammars}**.`, + '', + `Builder run: ${process.env.RUN_URL}`, + 'Each `.node` was `require()`-loaded + parsed a real snippet on its target', + 'platform-arch before upload. SLSA build-provenance attested; `SHA256SUMS`', + 'committed alongside each grammar.', + ].join('\n'); + const { data: pr } = await github.rest.pulls.create({ + owner, repo, head: branch, base: 'main', + title: `chore(vendor): tree-sitter prebuilds (${grammars})`, body, + }); + core.info(`Opened PR #${pr.number}`); diff --git a/.github/workflows/ci-devcontainer.yml b/.github/workflows/ci-devcontainer.yml index 96849b17a..d2d79f4be 100644 --- a/.github/workflows/ci-devcontainer.yml +++ b/.github/workflows/ci-devcontainer.yml @@ -36,7 +36,7 @@ jobs: # persist-credentials: false — this job only reads (tests and syntax # checks) and never pushes. The setting keeps GITHUB_TOKEN out of # .git/config, which zizmor flags as the "artipacked" issue. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 @@ -57,7 +57,7 @@ jobs: # persist-credentials: false — this is a read-only build smoke that # never pushes. The setting keeps GITHUB_TOKEN out of .git/config, # which zizmor flags as the "artipacked" issue. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 96b0a4b26..65c2e6248 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -14,7 +14,7 @@ jobs: outputs: web_changed: ${{ steps.filter.outputs.web }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v3 id: filter with: @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure e2e GitNexus home run: echo "GITNEXUS_HOME=${RUNNER_TEMP}/gitnexus-home" >> "$GITHUB_ENV" diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index a81876d9d..7ed4a345f 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus - run: npx tsc --noEmit working-directory: gitnexus @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus-web - run: npx tsc -b --noEmit working-directory: gitnexus-web @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Validate workflow concurrency convention shell: bash run: | diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 03c933ad0..5200d3758 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -125,7 +125,7 @@ jobs: - name: Checkout (for vitest config) if: steps.meta.outputs.skip != 'true' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: sparse-checkout: gitnexus/vitest.config.ts sparse-checkout-cone-mode: false diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b2341db36..ee0d92332 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -16,7 +16,7 @@ jobs: # test-reports artifact (if: always()). The default-persisted token in # .git/config must not be capturable through that upload (zizmor # credential-persistence / artipacked audit). The job never pushes. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -80,7 +80,7 @@ jobs: steps: # persist-credentials: false — runs tests only, never pushes (zizmor # credential-persistence / artipacked audit). - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -94,8 +94,9 @@ jobs: # 1. Static, offline: assert every grammar's compiled ABI loads on the # pinned runtime (check-tree-sitter-upgrade-readiness.py --assert-current). # 2. Dynamic: run the parser-loader ABI load-smoke on the OS matrix so an - # ABI-incompatible prebuilt (esp. the binary-only Swift vendor, which the - # static check can't introspect) fails on the platform it ships to. + # ABI-incompatible committed vendor prebuilt (e.g. Swift's — the static + # check introspects source, not the shipped .node) fails on the platform + # it ships to. abi-assert: name: tree-sitter ABI (${{ matrix.os }}) strategy: @@ -105,7 +106,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 20 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus with: build: 'true' @@ -137,7 +138,7 @@ jobs: # from a tarball and never pushes back; the token in .git/config would # be at risk of leaking through any future artifact-upload step # (zizmor artipacked audit). Disable upfront. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -245,7 +246,7 @@ jobs: # and never pushes; the default-persisted token in .git/config would be at # risk of leaking through an artifact upload (zizmor credential-persistence # / artipacked audit). Mirrors the packaged-install-smoke job below. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -265,6 +266,16 @@ jobs: run: node --import tsx bench/scope-capture/measure.mjs --check working-directory: gitnexus + - name: CFG construction time / disk / memory guards (#2081 M1) + # Build-free: asserts collectFunctionCfgs output is unchanged + # (fingerprint) and that wall-time, cfgSideChannel disk bytes, AND + # retained heap all stay sub-quadratic for the straight-line / + # many-functions / branchy scenarios. Catches an O(n^2) re-regression in + # the per-function CFG builder (e.g. an extendBlock concat chain) and a + # memory/disk blow-up. --expose-gc enables the retained-heap measurement. + run: node --expose-gc --import tsx bench/cfg/measure.mjs --check + working-directory: gitnexus + - name: Cross-language pipeline benchmarks (GITNEXUS_BENCH, serial) env: GITNEXUS_BENCH: '1' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 021a56930..2f925a62b 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -129,7 +129,7 @@ jobs: core.setOutput('code_review', isCodeReview ? 'true' : 'false'); - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: repository: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.repo || github.repository }} ref: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.sha || '' }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c7a95e29a..6835403db 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Don't leave GITHUB_TOKEN in .git/config for downstream steps to read. persist-credentials: false diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index d1214f849..4341e1327 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4a73329ed..39ff10494 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -101,7 +101,7 @@ jobs: # When triggered by workflow_call the caller passes the RC tag as an input; # we check out that tag so the Dockerfile and package.json match the built image. # For tag-push events github.ref is already the tag ref — no override needed. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.tag || github.ref }} @@ -138,7 +138,7 @@ jobs: # Required for multi-platform (linux/arm64) emulation. - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index f5e9af170..ee5ba8ecc 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Full history needed for the on-push full-history scan; on PRs the # action diffs against the base ref so the cost is bounded by the PR. diff --git a/.github/workflows/grammar-update-monitor.yml b/.github/workflows/grammar-update-monitor.yml new file mode 100644 index 000000000..ee14c093c --- /dev/null +++ b/.github/workflows/grammar-update-monitor.yml @@ -0,0 +1,146 @@ +name: Vendored grammar update monitor + +# Periodically checks each vendored tree-sitter grammar against its +# source-of-origin and opens a PR re-vendoring any update that is ABI-COMPATIBLE +# with the pinned tree-sitter@0.21.1 (LANGUAGE_VERSION 13–14, #1922). The version +# bump then triggers build-tree-sitter-prebuilds.yml, which cross-builds + ABI- +# validates the prebuilds — so a re-vendor that is subtly wrong can never silently +# ship: its PR's CI goes red. +# +# ABI-INCOMPATIBLE updates (the common case — upstreams move to newer tree-sitter) +# are reported as a notice + job summary, NOT applied, so the monitor never opens +# doomed PRs. tree-sitter-c is MONITORED but report-only: it is ABI-pinned at +# 0.21.4 (#1242/#858), so an available c update is surfaced (notice + summary) but +# never auto-bumped — a maintainer re-vendors it deliberately after a runtime +# upgrade. +# +# Concurrency convention: see CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention". + +on: + schedule: + - cron: '17 6 * * 1' # weekly, Monday 06:17 UTC + workflow_dispatch: + +# Least privilege; the actual writes use a short-lived App token minted below. +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + monitor: + name: Check upstreams + open update PRs + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + + # secrets aren't usable in a job/step `if:`, so compute presence here. + - name: Check release App secret + id: relapp + env: + HAS_APP: ${{ secrets.RELEASE_APP_ID != '' && secrets.RELEASE_APP_PRIVATE_KEY != '' }} + run: echo "configured=$HAS_APP" >> "$GITHUB_OUTPUT" + + - name: Mint GitHub App token + id: app-token + if: steps.relapp.outputs.configured == 'true' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - name: Detect updates, re-vendor ABI-compatible ones, open PRs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + HAS_APP: ${{ steps.relapp.outputs.configured }} + # App token writes; falls back to the read-only job token (PRs then skip). + GH_TOKEN: ${{ steps.app-token.outputs.token || github.token }} + with: + github-token: ${{ steps.app-token.outputs.token || github.token }} + script: | + const { execFileSync } = require('node:child_process'); + const SCRIPT = '.github/scripts/update-vendored-grammars.mjs'; + const run = (cmd, args, opts = {}) => + execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }); + + const report = JSON.parse(run('node', [SCRIPT])); + const { owner, repo } = context.repo; + const hasApp = process.env.HAS_APP === 'true'; + const applied = [], held = [], errors = [], skipped = []; + + run('git', ['config', 'user.name', 'gitnexus-release-bot[bot]']); + run('git', ['config', 'user.email', 'gitnexus-release-bot[bot]@users.noreply.github.com']); + const baseSha = run('git', ['rev-parse', 'HEAD']).trim(); + + for (const r of report) { + if (r.error) { errors.push(r); continue; } + if (!r.update) continue; + if (!r.applicable) { held.push(r); continue; } // ABI-incompatible / unknown + + const name = `tree-sitter-${r.grammar}`; + const branch = `chore/update-${name}-${r.upstream}`.replace(/[^a-z0-9._/-]+/gi, '-'); + + // Idempotency: don't reopen an existing PR for this exact version. + const existing = await github.rest.pulls.list({ owner, repo, head: `${owner}:${branch}`, state: 'all' }); + if (existing.data.length > 0) { skipped.push({ ...r, reason: 'PR exists' }); continue; } + + // Re-vendor in place (refuses + exits non-zero if ABI turns out wrong). + try { + run('node', [SCRIPT, '--apply', r.grammar]); + } catch (e) { + errors.push({ ...r, error: `apply failed: ${String(e.message || e).slice(0, 200)}` }); + run('git', ['checkout', '--', 'gitnexus/vendor']); + continue; + } + + if (!hasApp) { + skipped.push({ ...r, reason: 'no RELEASE_APP secret — PR not opened' }); + run('git', ['checkout', '--', 'gitnexus/vendor']); + continue; + } + + const remote = `https://x-access-token:${process.env.GH_TOKEN}@github.com/${owner}/${repo}.git`; + run('git', ['checkout', '-B', branch, baseSha]); + run('git', ['add', `gitnexus/vendor/${name}`]); + run('git', ['commit', '-m', `chore(vendor): update ${name} to ${r.upstream}`]); + run('git', ['push', '--force-with-lease', remote, `HEAD:${branch}`]); + const body = [ + `Automated re-vendor of **${name}** to \`${r.upstream}\` (from ${r.kind === 'npm' ? `npm \`${name}\`` : `\`${r.ref}\``}).`, + '', + `Verified ABI **${r.abi}** — compatible with the pinned \`tree-sitter@0.21.1\` (13–14).`, + 'Source-build inputs refreshed; the GitNexus binding.gyp / README / prebuilds are preserved.', + 'The version bump triggers `build-tree-sitter-prebuilds.yml` to rebuild + ABI-validate the', + 'prebuilds — review its result before merging.', + ].join('\n'); + const pr = await github.rest.pulls.create({ + owner, repo, head: branch, base: 'main', + title: `chore(vendor): update ${name} to ${r.upstream}`, body, + }); + applied.push({ ...r, pr: pr.data.number }); + run('git', ['checkout', '--force', baseSha]); + } + + // Summary + const s = core.summary.addHeading('Vendored grammar update monitor'); + if (applied.length) s.addRaw(`\n**Opened PRs:** ${applied.map((a) => `${a.grammar}→${a.upstream} (#${a.pr})`).join(', ')}\n`); + if (held.length) s.addRaw(`\n**Held (not auto-applied):** ${held.map((h) => `${h.grammar} ${h.upstream} (${h.hold ? 'report-only: ' + h.hold : 'ABI ' + (h.abi ?? '?') + ' — needs the tree-sitter runtime upgrade'})`).join(', ')}\n`); + if (skipped.length) s.addRaw(`\n**Skipped:** ${skipped.map((x) => `${x.grammar} (${x.reason})`).join(', ')}\n`); + if (errors.length) s.addRaw(`\n**Errors:** ${errors.map((e) => `${e.grammar}: ${e.error}`).join('; ')}\n`); + if (!applied.length && !held.length && !skipped.length && !errors.length) s.addRaw('\nAll vendored grammars are up to date. ✅\n'); + await s.write(); + + for (const h of held) core.notice(`${h.grammar}: update to ${h.upstream} available — ${h.hold ? `report-only (${h.hold})` : `ABI ${h.abi ?? 'unknown'} (need 13/14), held until the tree-sitter runtime upgrade`}.`); + if (!hasApp && (applied.length || skipped.some((x) => /secret/.test(x.reason)))) { + core.notice('RELEASE_APP_ID / RELEASE_APP_PRIVATE_KEY not configured — update PRs were not opened. Provision the App to enable auto-PRs.'); + } diff --git a/.github/workflows/pr-autofix-apply.yml b/.github/workflows/pr-autofix-apply.yml index 73bf8ceb0..9d437307b 100644 --- a/.github/workflows/pr-autofix-apply.yml +++ b/.github/workflows/pr-autofix-apply.yml @@ -336,7 +336,7 @@ jobs: # Push auth is provided inline at push time via the URL. - name: Checkout PR head if: steps.locate.outputs.found == 'true' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v5.0.4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v5.0.4 with: repository: ${{ steps.locate.outputs.head_repo }} ref: ${{ steps.locate.outputs.head_sha }} diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml index 04eb2468d..a531612f0 100644 --- a/.github/workflows/pr-autofix.yml +++ b/.github/workflows/pr-autofix.yml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # PR head commit (not the synthetic merge ref) — we need the # exact tree the contributor pushed so suggestions line up. diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 0938b5507..2341f4559 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -108,7 +108,7 @@ jobs: # Pinned to v7.2.0. Verify SHA via: # gh api repos/release-drafter/release-drafter/git/refs/tags/v7.2.0 # v7 removed `disable-releaser`; use `dry-run: true` to only autolabel. - - uses: release-drafter/release-drafter@c2e2804cc59f45f57076a99af580d0fedb697927 # v7.3.0 + - uses: release-drafter/release-drafter@693d20e7c1ce1a81d3a41962f85914253b518449 # v7.3.1 with: config-name: release-drafter.yml dry-run: true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5265d073..942bb2bd6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -162,7 +162,7 @@ jobs: should_run: ${{ steps.decide.outputs.should_run }} head_sha: ${{ steps.decide.outputs.head_sha }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 fetch-tags: true @@ -332,7 +332,7 @@ jobs: # on the RC path. - name: Checkout (RC) if: needs.route.outputs.mode == 'rc' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 fetch-tags: true @@ -349,7 +349,7 @@ jobs: - name: Checkout (stable) if: needs.route.outputs.mode == 'stable' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 # No `token:` — actions/checkout uses GITHUB_TOKEN by default. Stable # path performs no git pushes; the default scope is sufficient. with: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 1de6cd5db..229286daf 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml index 74ce72a27..1eca8861d 100644 --- a/.github/workflows/tree-sitter-upgrade-readiness.yml +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -37,7 +37,7 @@ jobs: # Needed to open/update the tracking issue on scheduled runs. issues: write steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus with: diff --git a/.github/workflows/triage-sweep.yml b/.github/workflows/triage-sweep.yml index 43d67828d..ea605eb15 100644 --- a/.github/workflows/triage-sweep.yml +++ b/.github/workflows/triage-sweep.yml @@ -59,7 +59,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: sparse-checkout: .github/scripts/triage sparse-checkout-cone-mode: false diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index a8ca6c839..fd7ade8b5 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index b74387116..678ed1a4a 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -31,7 +31,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false @@ -53,7 +53,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/AGENTS.md b/AGENTS.md index 1e31004e4..64d264126 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,18 +80,18 @@ This project is indexed by GitNexus as **GitNexus** (26675 symbols, 35395 relati ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. +- When exploring unfamiliar code, use `query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. ## Never Do -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. +- NEVER edit a function, class, or method without first running `impact` on it. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. +- NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. +- NEVER commit changes without running `detect_changes()` to check affected scope. ## Resources @@ -173,6 +173,6 @@ npx gitnexus serve # HTTP API on port 4747 (from any ind ### Gotchas -- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift, builds tree-sitter-proto). Native bindings need `python3`, `make`, `g++`. -- `tree-sitter-kotlin` and `tree-sitter-swift` are optional — install warnings expected. +- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (materializes the vendored grammars into `node_modules/`, then prefers a committed prebuild per platform-arch and only source-builds when none matches). A C/C++ toolchain (`python3`, `make`, `g++`) is needed only for that source-build fallback. +- The vendored grammars `tree-sitter-{c,dart,proto,swift,kotlin}` are handled uniformly: c is required; dart/proto/swift/kotlin are optional and skippable via `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`. Install warnings appear only when no prebuild matches the platform-arch and no toolchain is present, and are non-fatal — only that language's parsing is unavailable. - ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index f65c175f9..b3319f172 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -15,7 +15,7 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## End-to-end flow: index → graph → tools -1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 12 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. +1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 14 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. 2. **Persistence** — `repo-manager.ts` (paths, registry, KuzuDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches). @@ -77,11 +77,11 @@ Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## Pipeline Phase DAG -12 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. +14 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. ``` scan → structure → [markdown, cobol] → parse → [routes, tools, orm] - → crossFile → mro → communities → processes + → crossFile → scopeResolution → pruneLocalSymbols → mro → communities → processes ``` | Phase | File | Deps | Output | @@ -95,11 +95,13 @@ scan → structure → [markdown, cobol] → parse → [routes, tools, orm] | `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges | | `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) | | `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order | -| `mro` | `mro.ts` | `crossFile`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | -| `communities` | `communities.ts` | `mro`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | -| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `structure` | Process nodes + STEP_IN_PROCESS edges | +| `scopeResolution` | `scope-resolution/pipeline/phase.ts` | `parse`, `crossFile`, `structure` | Binding/reference + inheritance edges; disposes BindingAccumulator | +| `pruneLocalSymbols` | `prune-local-symbols.ts` | `scopeResolution` | Drops inert block-local `Const`/`Variable`/`Static` nodes (only a `File→DEFINES` edge) post-resolution | +| `mro` | `mro.ts` | `crossFile`, `scopeResolution`, `pruneLocalSymbols`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | +| `communities` | `communities.ts` | `mro`, `pruneLocalSymbols`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | +| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `pruneLocalSymbols`, `structure` | Process nodes + STEP_IN_PROCESS edges | -**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `orm-extraction.ts` (sequential ORM fallback), `types.ts`, `runner.ts`, `index.ts`. +**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `types.ts`, `runner.ts`, `index.ts`. ### DAG runner @@ -119,7 +121,8 @@ scan → structure → [markdown, cobol] → parse → [routes, tools, orm] - **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output. - **Typed phase access** — `getPhaseOutput(deps, 'name')` for type-safe upstream results. - **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership. -- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests). `skipWorkers` forces sequential parsing. +- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests); `pruneLocalSymbols` still runs (it is graph cleanup, not analysis). `skipWorkers` is no longer a sequential escape hatch — it (like `--workers 0` / `GITNEXUS_WORKER_POOL_SIZE=0`) is rejected with an actionable error, since the worker pool is the sole parse path (§ Chunked parse-and-resolve). +- **Local-symbol pruning** — `pruneLocalSymbols` removes inert block-local value symbols after scope resolution has consumed them. Opt out per-call with `PipelineOptions.keepLocalValueSymbols` or globally with the `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` env var. ### How to add a new phase @@ -199,7 +202,11 @@ Language-agnostic scope-resolution resolver. This is the resolution path for eve ``` Orchestrator: `runScopeResolution(input, provider)` in `scope-resolution/pipeline/run.ts`. -Pipeline phase: `scopeResolutionPhase` in `scope-resolution/pipeline/phase.ts` — iterates the registered `SCOPE_RESOLVERS`, reads per-file Trees from the parse phase's `scopeTreeCache`, disposes the cache at the end. +Pipeline phase: `scopeResolutionPhase` in `scope-resolution/pipeline/phase.ts` — iterates the registered `SCOPE_RESOLVERS` over the worker-serialized `ParsedFile`s. (Per-language `emitScopeCaptures` hooks may reuse a cached Tree via the orchestrator's `treeCache`, but in worker-pool runs that cache is empty — Trees can't cross MessageChannels — so they consume the pre-extracted `ParsedFile` instead; § Performance notes.) + +### Optional CFG/PDG emission (`--pdg`, #2081 M1) + +On a `--pdg` run, the parse worker builds a per-function control-flow graph from the tree-sitter AST (`LanguageProvider.cfgVisitor`; TypeScript/JavaScript in M1) and serializes it onto `ParsedFile.cfgSideChannel` as plain data. Scope-resolution then emits `BasicBlock` nodes + `CFG` edges from that side-channel **inside Phase 4 of `runScopeResolution`, while the disk-backed ParsedFile store is still live** — the only window where the worker-built CFGs are loaded (the store is cleared right after the phase returns). A standalone post-`mro` phase would read an empty store, so the CFG emit deliberately lives in-phase, mirroring the `applyCaptureSideChannel` pattern. The opt-in is off by default (graph byte-identical), folded into the parse-cache key (a pdg-off warm cache is never reused on a `--pdg` run), and bounded by a per-function edge cap that logs any dropped edges. Edge *kind* (`seq`/`cond-true`/`loop-back`/…) rides in the `CFG` relationship's `reason` (CFG is a single `CodeRelation` type, not one type per kind). See `core/ingestion/cfg/`. ### `ScopeResolver` contract @@ -248,7 +255,7 @@ CI auto-discovers the set via `tsx`. No workflow edit required. ### Performance notes -- **Cross-phase Tree cache**: parse phase writes Trees into `scopeTreeCache` (separate from the chunk-local `astCache`) ONLY for languages with `emitScopeCaptures`. Scope-resolution reads from it to skip the second parse. Cleared at end of the phase. Workers leave the cache empty — Trees can't cross MessageChannels; cache miss = fresh parse. `PROF_SCOPE_RESOLUTION=1` emits hit/miss counters and a worker-engaged warning. +- **Cross-phase Tree cache**: the orchestrator's `treeCache` (`RunScopeResolutionInput.treeCache`) lets a scope-resolution per-language hook (`emitScopeCaptures`) reuse a tree instead of re-parsing. Workers leave it empty — Trees can't cross MessageChannels — so in normal (worker-pool) runs scope-resolution does NOT rely on it: workers serialize each file's `ParsedFile` (+ capture side-channel) and stream them in, so scope-resolution consumes the pre-extracted artifact rather than re-parsing on the main thread (§ Chunked parse-and-resolve). `PROF_SCOPE_RESOLUTION=1` emits hit/miss counters and a worker-engaged warning. - **Typed relationship iteration**: heritage + MRO walk only the EXTENDS / IMPLEMENTS / HAS_METHOD edges via `iterRelationshipsByType`, not the full relationship map. - **Workspace-resolution-index**: O(1) `findOwnedMember` / `findExportedDef` / `classScopeByDefId` built once per run. - **SCC-ordered cross-file return-type propagation** (PR #1050): `propagateImportedReturnTypes` walks `indexes.sccs` in reverse-topological order (leaves first), so multi-hop alias chains like `models.User → service.user → app.user` collapse to the terminal class in a single linear pass. Within each importer, the source module's `typeBindings` is chain-followed BEFORE mirroring (so we mirror terminal types, not intermediate refs), and the importer's own `typeBindings` is chain-followed AFTER mirroring (so local `const x = importedFn()` resolves before downstream importers run). Cyclic SCCs reach a partial fixpoint within a single pass without iterating to convergence — see the `ts-circular` cross-file-binding fixture which only asserts pipeline-no-throw. PROF output (`PROF_SCOPE_RESOLUTION=1`) splits `finalize` from `propagate` so quadratic regressions in the chain-follow surface independently. @@ -311,7 +318,7 @@ Unified 3-tier algorithm (`model/resolution-context.ts`), per-language `importSe ### Chunked parse-and-resolve `parse` processes files in ~20 MB byte-budget chunks to bound memory. Per chunk: -1. Worker pool dispatches files (or sequential fallback via `skipWorkers`) +1. Worker pool dispatches files (the sole parse path — there is no sequential fallback; `skipWorkers`, `--workers 0`, and `GITNEXUS_WORKER_POOL_SIZE=0` are rejected with an actionable error) 2. Each worker: detect language → load grammar → run queries → return unified `ParseWorkerResult` 3. Synthesize wildcard bindings (`wildcard-synthesis.ts`) 4. Resolve imports @@ -321,6 +328,8 @@ Inheritance edges are emitted later, by the scope-resolution phase (`preEmitInhe Workers: `workers/worker-pool.ts`, `workers/parse-worker.ts`. +**Worker-serialized ParsedFiles (#2038).** To index very large repos (e.g. the Linux kernel) without OOM, the worker pool is the *sole* parse path and workers serialize each file's `ParsedFile` (plus its capture side-channel) in parallel, streaming them to scope-resolution through a disk-backed store. Scope-resolution consumes the pre-extracted artifact instead of re-parsing every file on the main thread — tree-sitter's native input buffers are not GC-reclaimable, so the former main-thread re-parse leaked native memory until the process died. Pool creation is lazy / cache-miss-gated, so a warm all-cache-hit run replays cached worker output without spawning a worker (hence `usedWorkerPool` can be false even when the repo has parseable files). + ### Inheritance and MRO Inheritance is captured by the `@reference.inherits` tag and emitted by the scope-resolution phase: `preEmitInheritanceEdges` resolves each base in scope, then `emitHeritageEdges` writes the `EXTENDS`/`IMPLEMENTS` edges. The phase then computes method resolution order via each `ScopeResolver`'s `buildMro` hook, feeding a `MethodDispatchIndex` used for owner-scoped lookups. Per-language strategy: diff --git a/CLAUDE.md b/CLAUDE.md index bbb991589..f2bf1e487 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -62,18 +62,18 @@ This project is indexed by GitNexus as **GitNexus** (26675 symbols, 35395 relati ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run `detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. +- When exploring unfamiliar code, use `query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `context({name: "symbolName"})`. ## Never Do -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. +- NEVER edit a function, class, or method without first running `impact` on it. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. +- NEVER rename symbols with find-and-replace — use `rename` which understands the call graph. +- NEVER commit changes without running `detect_changes()` to check affected scope. ## Resources diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ddefad384..848884be4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -157,7 +157,12 @@ routes between two modes based on the triggering event: suffix; RC tags are excluded at trigger via a negative glob). Publishes to the `latest` dist-tag with a changelog-backed GitHub release. Maintainers are expected to tag from `main` as a convention; the workflow itself does - not enforce branch reachability. No Docker build (RC-only). + not enforce branch reachability. No Docker build (RC-only). Before cutting a + stable release, keep `gitnexus/package.json`, + `gitnexus-claude-plugin/.claude-plugin/plugin.json`, + `.claude-plugin/marketplace.json`, and the matching `CHANGELOG.md` entry in + lockstep — the always-on `gitnexus` unit suite now fails if those manifest + versions drift. - **Release-candidate mode** — runs on every push to `main` (typically a merged PR) plus manual `workflow_dispatch`. Docs-only changes are skipped via `paths-ignore`. Publishes to the `rc` dist-tag with version diff --git a/Dockerfile.cli b/Dockerfile.cli index 3275c8f7e..633d23f5d 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -36,6 +36,17 @@ RUN npm ci --prefix gitnexus # Drop dev dependencies for a smaller runtime layer. RUN npm prune --omit=dev --prefix gitnexus +# `npm prune` removes anything not in package.json's dependency tree — which +# includes the VENDORED tree-sitter grammars (materialized into node_modules/ by +# postinstall, but not declared as deps) and their freshly-built native bindings. +# The `serve` image analyzes/parses uploaded repos at runtime, so those grammars +# must survive into the runtime layer. Re-run the grammar postinstall here in the +# builder (which still has python3/make/g++ and the hoisted node-addon-api / +# node-gyp-build) to re-materialize + rebuild them after the prune. This is +# load-bearing for tree-sitter-c (a core, REQUIRED grammar now vendored, #2116): +# as a former `dependency` it used to survive prune; vendored, it would not. +RUN npm run postinstall --prefix gitnexus + # -- Runtime ----------------------------------------------------------- # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime @@ -67,6 +78,44 @@ COPY --from=builder --chown=node:node /app/gitnexus/vendor ./gitnexus/vendor # unreachable from $PATH. RUN ln -s /app/gitnexus/dist/cli/index.js /usr/local/bin/gitnexus +# Bake the LadybugDB FTS extension into the image so BM25 keyword search works +# at runtime. The server runs the default `load-only` extension policy (the read +# pool pins `{ policy: 'load-only' }`), so a runtime `LOAD EXTENSION fts` never +# INSTALLs — the extension must already exist in the runtime user's HOME +# extension dir, or every keyword search silently degrades (no FTS indexes are +# written and ranking falls back to vector-only with only a `warning` field). +# Run the installer as the `node` user with the SAME HOME the server runs under, +# so `INSTALL fts` materializes the extension under `$HOME/.lbdb/extension` where +# the runtime `LOAD` resolves it offline. `ENV HOME` is pinned because Docker +# does not derive HOME from `USER`, so without it build-install and runtime-load +# would resolve different paths. Requires network egress for the one-time +# INSTALL; the build fails loudly if it cannot fetch the extension. The DB-size +# default comes from GITNEXUS_LBUG_MAX_DB_SIZE (single source of truth, matches +# the runtime) — it only sizes the throwaway scratch DB used to run INSTALL. +# The second `--verify-only` step re-LOADs the extension in a FRESH process +# under the same HOME, so a HOME/extension-dir mismatch fails the build here +# rather than silently degrading keyword search to vector-only at runtime. +ENV HOME=/home/node \ + GITNEXUS_LBUG_MAX_DB_SIZE=17179869184 +RUN su node -s /bin/sh -c "HOME=/home/node node /app/gitnexus/scripts/install-duckdb-extension.mjs fts" \ + && su node -s /bin/sh -c "HOME=/home/node node /app/gitnexus/scripts/install-duckdb-extension.mjs fts --verify-only" + +# Published runtime assets (in package.json `files`). Placed AFTER the DuckDB +# FTS-extension RUN above so editing hook/skill content does not invalidate that +# network-fetching cache layer; they have no input dependency on it. +# `hooks/`: dist/cli/resolve-invocation.js does +# `require('../../hooks/claude/resolve-analyze-cmd.cjs')` at module load — the +# single source of truth for the npm-11 npx-crash invocation decision (#1939). +# Without it, `gitnexus analyze` inside the image crashes with MODULE_NOT_FOUND +# before it does any work (#2130). `skills/`: the CLI reads the bundled SKILL.md +# templates from `/skills/` for `gitnexus analyze --skills` and `gitnexus +# setup`/`uninstall`; absent, those degrade silently (placeholder content / zero +# skills installed). (The web UI bundle `web/`, also in `files`, is deliberately +# NOT shipped: this builder never builds gitnexus-web, so the image is API-only; +# the UI is the separate Dockerfile.web image / hosted app.) +COPY --from=builder --chown=node:node /app/gitnexus/hooks ./gitnexus/hooks +COPY --from=builder --chown=node:node /app/gitnexus/skills ./gitnexus/skills + USER node # The web UI defaults to http://localhost:4747 - keep that contract. diff --git a/README.md b/README.md index 28229c98c..aea1079ee 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,9 @@ That's it. This indexes the codebase, installs agent skills, registers Claude Co To configure MCP for your editor, run `npx gitnexus setup` once — or set it up manually below. -> **Faster install (no C++ toolchain needed):** set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip vendored grammar materialize/build (`tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`). Dart/Proto/Swift files won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild. +> **Faster install (no C++ toolchain needed):** set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` — those four won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild. See the `tree-sitter-kotlin` note below. +> +> **About `tree-sitter-kotlin`:** like Dart/Proto/Swift, Kotlin is a **vendored** grammar (under `gitnexus/vendor/tree-sitter-kotlin`). Upstream `tree-sitter-kotlin` ships **source only** (no prebuilt binaries), so GitNexus builds the Kotlin platform prebuilds itself (via the `build-tree-sitter-prebuilds` GitHub Actions workflow) and vendors them — the same uniform pipeline now used for Dart, Proto, and Swift (Swift's prebuilds were originally copied from upstream; they're now GitNexus-cross-built too). `node-gyp-build` selects the right `.node` at require time, so **no C/C++ toolchain is needed**. If no prebuild matches your platform-arch, only Kotlin (`.kt`/`.kts`) parsing is unavailable; the rest of `gitnexus` is unaffected. ### MCP Setup @@ -223,6 +225,7 @@ args = ["-y", "gitnexus@latest", "mcp"] ```bash gitnexus setup # Configure MCP for your editors (one-time) +gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks (add --force to apply) gitnexus analyze [path] # Index a repository (or update stale index) gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild @@ -236,7 +239,7 @@ gitnexus analyze --embeddings [limit] # Enable embedding generation (slower, be gitnexus analyze --verbose # Log skipped files when parsers are unavailable gitnexus analyze --worker-timeout 60 # Increase worker idle timeout for slow parses gitnexus analyze --wal-checkpoint-threshold 67108864 # 64 MiB. Control LadybugDB WAL auto-checkpoint threshold (default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB) -gitnexus analyze --workers # Parse worker pool size (default: cores-1, capped at 16; 0 = sequential) +gitnexus analyze --workers # Parse worker pool size (>=1; default: cores-1, capped at 16, auto-sized to the repo). 0 is rejected — there is no sequential mode. gitnexus mcp # Start MCP server (stdio) — serves all indexed repos gitnexus serve # Start local HTTP server (multi-repo) for web UI connection gitnexus list # List all indexed repositories @@ -259,6 +262,8 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +> **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. + If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. #### Embeddings node limit @@ -314,7 +319,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | Variable | Default | Effect | Tune when… | | -------------------------------------- | ------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size. `0` disables the pool (sequential fallback). Equivalent to `--workers `. | Constrained containers (cgroup CPU limits), CI runners with explicit quotas, or debugging a worker-only crash via `0`. | +| `GITNEXUS_WORKER_POOL_SIZE` | `cores - 1`, capped at 16 | Parse worker pool size (must be ≥ 1). Equivalent to `--workers `. The worker pool is the sole parse path — there is no sequential parser, so `0` is rejected with an actionable error (the pool self-heals via quarantine + respawn). | Constrained containers (cgroup CPU limits) or CI runners with explicit quotas. To narrow down a worker crash set `1` for a single-worker pool — not `0`. | | `GITNEXUS_PARSE_CHUNK_CONCURRENCY` | `2` | Number of chunks whose file contents may be read into memory in parallel while the pool dispatches the current chunk. Worker dispatch itself stays serial. | Repos large enough to chunk (multi-MB total source) where disk I/O is a measurable fraction of analyze wall-clock. | | `GITNEXUS_VERBOSE` | unset | When `1`, enables verbose ingestion logs (skipped-file warnings, per-chunk throughput, parse-cache stats). Equivalent to `--verbose`. | Debugging an analyze that "completed" but seems to have missed files; tuning `--workers` / chunk concurrency against observable throughput. | | `GITNEXUS_PROFILE_DEFERRED` | unset | When `1`, emits `[deferred-profile]` timing/progress logs for the post-chunk deferred resolution band (imports → heritage → buildHeritageMap → legacy call resolution). Implied by `GITNEXUS_VERBOSE`. | Diagnosing analyze stalls in "Resolving calls (all chunks)" on large Java/Kotlin repos (issue #1741) without the full verbose ingestion noise. | @@ -328,7 +333,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD`| `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | | `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | | `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | -| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, and `tree-sitter-swift` at install time. | Installing on a host without a C++ toolchain or where Swift prebuilds don't match; you're willing to skip Dart/Proto/Swift parsing. | +| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | #### Publishing to understand-quickly (opt-in) @@ -342,7 +347,7 @@ It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained G | Tool | What It Does | `repo` Param | | ----------------- | ---------------------------------------------------------------- | ------------ | -| `list_repos` | Discover all indexed repositories | — | +| `list_repos` | Discover all indexed repositories (paginated — `limit`/`offset`) | — | | `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | | `context` | 360-degree symbol view — categorized refs, process participation | Optional | | `impact` | Blast radius analysis with depth grouping and confidence | Optional | @@ -698,6 +703,8 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas **Imports** — cross-file import resolution · **Named Bindings** — `import { X as Y }` / re-export tracking · **Exports** — public/exported symbol detection · **Heritage** — class inheritance, interfaces, mixins · **Type Annotations** — explicit type extraction for receiver resolution · **Constructor Inference** — infer receiver type from constructor calls (`self`/`this` resolution included for all languages) · **Config** — language toolchain config parsing (tsconfig, go.mod, etc.) · **Frameworks** — AST-based framework pattern detection · **Entry Points** — entry point scoring heuristics +**Control flow (CFG, opt-in `--pdg`)** — per-function control-flow graphs (`BasicBlock` nodes + `CFG` edges) feeding the PDG/taint substrate, currently **TypeScript & JavaScript** (#2081 M1); other languages planned. Off by default. + --- ## Tool Examples diff --git a/gitnexus-claude-plugin/.claude-plugin/plugin.json b/gitnexus-claude-plugin/.claude-plugin/plugin.json index bd4b8c426..fe3e641ac 100644 --- a/gitnexus-claude-plugin/.claude-plugin/plugin.json +++ b/gitnexus-claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.3.6", + "version": "1.6.7", "author": { "name": "GitNexus" }, diff --git a/gitnexus-claude-plugin/hooks/gitnexus-hook.js b/gitnexus-claude-plugin/hooks/gitnexus-hook.js index e3c62c769..c3ec2ecf5 100644 --- a/gitnexus-claude-plugin/hooks/gitnexus-hook.js +++ b/gitnexus-claude-plugin/hooks/gitnexus-hook.js @@ -110,10 +110,20 @@ function hasGitNexusServerOwner(gitNexusDir) { return hasGitNexusDbLockedByGitNexusServer(path.join(gitNexusDir, 'lbug'), process.pid); } +/** + * Whether opt-in diagnostics should be written to the hook's stderr. Strict + * hook runners (e.g. Codex `PreToolUse`) validate hook output, so normal, + * non-error skip paths must stay silent unless the operator explicitly asks + * for diagnostics via GITNEXUS_DEBUG. See issue #1913. + */ +function isDebugEnabled() { + return process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; +} + function extractAugmentContext(stderr) { const output = (stderr || '').trim(); const marker = output.indexOf('[GitNexus]'); - const debug = process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; + const debug = isDebugEnabled(); if (debug && output.length > 0) { // Emit the FULL discarded prefix (everything before the marker, or all of // it when no marker is present) so suppressed diagnostics — LadybugDB lock @@ -267,7 +277,12 @@ function handlePreToolUse(input) { const pattern = extractPattern(toolName, toolInput); if (!pattern || pattern.length < 3) return; if (hasGitNexusServerOwner(gitNexusDir)) { - process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + // Normal skip path: the MCP server owns the DB, so the CLI augment would + // contend on the lock. Stay silent for strict hook runners (issue #1913); + // surface the reason only when diagnostics are explicitly requested. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + } return; } @@ -366,7 +381,7 @@ function main() { const handler = handlers[input.hook_event_name || '']; if (handler) handler(input); } catch (err) { - if (process.env.GITNEXUS_DEBUG) { + if (isDebugEnabled()) { console.error('GitNexus hook error:', (err.message || '').slice(0, 200)); } } diff --git a/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md index 937b5e2a4..9834f94b7 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-debugging/SKILL.md @@ -16,10 +16,10 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ## Workflow ``` -1. gitnexus_query({query: ""}) → Find related execution flows -2. gitnexus_context({name: ""}) → See callers/callees/processes +1. query({query: ""}) → Find related execution flows +2. context({name: ""}) → See callers/callees/processes 3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow -4. gitnexus_cypher({query: "MATCH path..."}) → Custom traces if needed +4. cypher({query: "MATCH path..."}) → Custom traces if needed ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. @@ -28,11 +28,11 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ``` - [ ] Understand the symptom (error message, unexpected behavior) -- [ ] gitnexus_query for error text or related code +- [ ] query for error text or related code - [ ] Identify the suspect function from returned processes -- [ ] gitnexus_context to see callers and callees +- [ ] context to see callers and callees - [ ] Trace execution flow via process resource if applicable -- [ ] gitnexus_cypher for custom call chain traces if needed +- [ ] cypher for custom call chain traces if needed - [ ] Read source files to confirm root cause ``` @@ -40,7 +40,7 @@ description: "Use when the user is debugging a bug, tracing an error, or asking | Symptom | GitNexus Approach | | -------------------- | ---------------------------------------------------------- | -| Error message | `gitnexus_query` for error text → `context` on throw sites | +| Error message | `query` for error text → `context` on throw sites | | Wrong return value | `context` on the function → trace callees for data flow | | Intermittent failure | `context` → look for external calls, async deps | | Performance issue | `context` → find symbols with many callers (hot paths) | @@ -48,24 +48,24 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ## Tools -**gitnexus_query** — find code related to error: +**query** — find code related to error: ``` -gitnexus_query({query: "payment validation error"}) +query({query: "payment validation error"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError, PaymentException ``` -**gitnexus_context** — full context for a suspect: +**context** — full context for a suspect: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates (external API!) → Processes: CheckoutFlow (step 3/7) ``` -**gitnexus_cypher** — custom call chain traces: +**cypher** — custom call chain traces: ```cypher MATCH path = (a)-[:CodeRelation {type: 'CALLS'}*1..2]->(b:Function {name: "validatePayment"}) @@ -75,11 +75,11 @@ RETURN [n IN nodes(path) | n.name] AS chain ## Example: "Payment endpoint returns 500 intermittently" ``` -1. gitnexus_query({query: "payment error handling"}) +1. query({query: "payment error handling"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError -2. gitnexus_context({name: "validatePayment"}) +2. context({name: "validatePayment"}) → Outgoing calls: verifyCard, fetchRates (external API!) 3. READ gitnexus://repo/my-app/process/CheckoutFlow diff --git a/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md index 2dcf7b578..ccf684c28 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-exploring/SKILL.md @@ -18,8 +18,8 @@ description: "Use when the user asks how code works, wants to understand archite ``` 1. READ gitnexus://repos → Discover indexed repos 2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. gitnexus_query({query: ""}) → Find related execution flows -4. gitnexus_context({name: ""}) → Deep dive on specific symbol +3. query({query: ""}) → Find related execution flows +4. context({name: ""}) → Deep dive on specific symbol 5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow ``` @@ -29,9 +29,9 @@ description: "Use when the user asks how code works, wants to understand archite ``` - [ ] READ gitnexus://repo/{name}/context -- [ ] gitnexus_query for the concept you want to understand +- [ ] query for the concept you want to understand - [ ] Review returned processes (execution flows) -- [ ] gitnexus_context on key symbols for callers/callees +- [ ] context on key symbols for callers/callees - [ ] READ process resource for full execution traces - [ ] Read source files for implementation details ``` @@ -47,18 +47,18 @@ description: "Use when the user asks how code works, wants to understand archite ## Tools -**gitnexus_query** — find execution flows related to a concept: +**query** — find execution flows related to a concept: ``` -gitnexus_query({query: "payment processing"}) +query({query: "payment processing"}) → Processes: CheckoutFlow, RefundFlow, WebhookHandler → Symbols grouped by flow with file locations ``` -**gitnexus_context** — 360-degree view of a symbol: +**context** — 360-degree view of a symbol: ``` -gitnexus_context({name: "validateUser"}) +context({name: "validateUser"}) → Incoming calls: loginHandler, apiMiddleware → Outgoing calls: checkToken, getUserById → Processes: LoginFlow (step 2/5), TokenRefresh (step 1/3) @@ -68,10 +68,10 @@ gitnexus_context({name: "validateUser"}) ``` 1. READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. gitnexus_query({query: "payment processing"}) +2. query({query: "payment processing"}) → CheckoutFlow: processPayment → validateCard → chargeStripe → RefundFlow: initiateRefund → calculateRefund → processRefund -3. gitnexus_context({name: "processPayment"}) +3. context({name: "processPayment"}) → Incoming: checkoutHandler, webhookHandler → Outgoing: validateCard, chargeStripe, saveTransaction 4. Read src/payments/processor.ts for implementation details diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md index b81900b5e..cacc4e886 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md @@ -38,7 +38,38 @@ For any task involving code understanding, debugging, impact analysis, or refact | `detect_changes` | Git-diff impact — what do your current changes affect | | `rename` | Multi-file coordinated rename with confidence-tagged edits | | `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | +| `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | + +### Paginating `list_repos` + +`list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: + +```jsonc +{ + "repositories": [ + { "name": "...", "path": "...", "indexedAt": "...", "lastCommit": "...", "stats": { } } + ], + "pagination": { + "total": 437, + "limit": 50, + "offset": 0, + "returned": 50, + "hasMore": true, + "nextOffset": 50 + } +} +``` + +To enumerate **every** repository, keep calling with `offset` set to `pagination.nextOffset` until `hasMore` is `false`: + +```text +list_repos {} → repos 1–50, nextOffset 50, hasMore true +list_repos { offset: 50 } → repos 51–100, nextOffset 100, hasMore true +… +list_repos { offset: 400 } → repos 401–437, hasMore false (done) +``` + +Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. ## Resources Reference diff --git a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md index 7206ca506..45eb7ce87 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-impact-analysis/SKILL.md @@ -17,9 +17,9 @@ description: "Use when the user wants to know what will break if they change som ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) → What depends on this 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. gitnexus_detect_changes() → Map current git changes to affected flows +3. detect_changes() → Map current git changes to affected flows 4. Assess risk and report to user ``` @@ -28,11 +28,11 @@ description: "Use when the user wants to know what will break if they change som ## Checklist ``` -- [ ] gitnexus_impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] gitnexus_detect_changes() for pre-commit check +- [ ] detect_changes() for pre-commit check - [ ] Assess risk level and report to user ``` @@ -55,10 +55,10 @@ description: "Use when the user wants to know what will break if they change som ## Tools -**gitnexus_impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius: ``` -gitnexus_impact({ +impact({ target: "validateUser", direction: "upstream", minConfidence: 0.8, @@ -73,10 +73,10 @@ gitnexus_impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**gitnexus_detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis: ``` -gitnexus_detect_changes({scope: "staged"}) +detect_changes({scope: "staged"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -86,7 +86,7 @@ gitnexus_detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. gitnexus_impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/gitnexus-claude-plugin/skills/gitnexus-pr-review/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-pr-review/SKILL.md index 319c063f9..9f1d362e5 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-pr-review/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-pr-review/SKILL.md @@ -18,10 +18,10 @@ description: "Use when the user wants to review a pull request, understand what ``` 1. gh pr diff → Get the raw diff -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows +2. detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows 3. For each changed symbol: - gitnexus_impact({target: "", direction: "upstream"}) → Blast radius per change -4. gitnexus_context({name: ""}) → Understand callers/callees + impact({target: "", direction: "upstream"}) → Blast radius per change +4. context({name: ""}) → Understand callers/callees 5. READ gitnexus://repo/{name}/processes → Check affected execution flows 6. Summarize findings with risk assessment ``` @@ -32,10 +32,10 @@ description: "Use when the user wants to review a pull request, understand what ``` - [ ] Fetch PR diff (gh pr diff or git diff base...head) -- [ ] gitnexus_detect_changes to map changes to affected execution flows -- [ ] gitnexus_impact on each non-trivial changed symbol +- [ ] detect_changes to map changes to affected execution flows +- [ ] impact on each non-trivial changed symbol - [ ] Review d=1 items (WILL BREAK) — are callers updated? -- [ ] gitnexus_context on key changed symbols to understand full picture +- [ ] context on key changed symbols to understand full picture - [ ] Check if affected processes have test coverage - [ ] Assess overall risk level - [ ] Write review summary with findings @@ -63,20 +63,20 @@ description: "Use when the user wants to review a pull request, understand what ## Tools -**gitnexus_detect_changes** — map PR diff to affected execution flows: +**detect_changes** — map PR diff to affected execution flows: ``` -gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +detect_changes({scope: "compare", base_ref: "main"}) → Changed: 8 symbols in 4 files → Affected processes: CheckoutFlow, RefundFlow, WebhookHandler → Risk: MEDIUM ``` -**gitnexus_impact** — blast radius per changed symbol: +**impact** — blast radius per changed symbol: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream"}) +impact({target: "validatePayment", direction: "upstream"}) → d=1 (WILL BREAK): - processCheckout (src/checkout.ts:42) [CALLS, 100%] @@ -86,20 +86,20 @@ gitnexus_impact({target: "validatePayment", direction: "upstream"}) - checkoutRouter (src/routes/checkout.ts:22) [CALLS, 95%] ``` -**gitnexus_impact with tests** — check test coverage: +**impact with tests** — check test coverage: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream", includeTests: true}) +impact({target: "validatePayment", direction: "upstream", includeTests: true}) → Tests that cover this symbol: - validatePayment.test.ts [direct] - checkout.integration.test.ts [via processCheckout] ``` -**gitnexus_context** — understand a changed symbol's role: +**context** — understand a changed symbol's role: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates @@ -112,20 +112,20 @@ gitnexus_context({name: "validatePayment"}) 1. gh pr diff 42 > /tmp/pr42.diff → 4 files changed: payments.ts, checkout.ts, types.ts, utils.ts -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +2. detect_changes({scope: "compare", base_ref: "main"}) → Changed symbols: validatePayment, PaymentInput, formatAmount → Affected processes: CheckoutFlow, RefundFlow → Risk: MEDIUM -3. gitnexus_impact({target: "validatePayment", direction: "upstream"}) +3. impact({target: "validatePayment", direction: "upstream"}) → d=1: processCheckout, webhookHandler (WILL BREAK) → webhookHandler is NOT in the PR diff — potential breakage! -4. gitnexus_impact({target: "PaymentInput", direction: "upstream"}) +4. impact({target: "PaymentInput", direction: "upstream"}) → d=1: validatePayment (in PR), createPayment (NOT in PR) → createPayment uses the old PaymentInput shape — breaking change! -5. gitnexus_context({name: "formatAmount"}) +5. context({name: "formatAmount"}) → Called by 12 functions — but change is backwards-compatible (added optional param) 6. Review summary: diff --git a/gitnexus-claude-plugin/skills/gitnexus-refactoring/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-refactoring/SKILL.md index c749eb384..e13c04e14 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-refactoring/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-refactoring/SKILL.md @@ -16,9 +16,9 @@ description: "Use when the user wants to rename, extract, split, move, or restru ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → Map all dependents -2. gitnexus_query({query: "X"}) → Find execution flows involving X -3. gitnexus_context({name: "X"}) → See all incoming/outgoing refs +1. impact({target: "X", direction: "upstream"}) → Map all dependents +2. query({query: "X"}) → Find execution flows involving X +3. context({name: "X"}) → See all incoming/outgoing refs 4. Plan update order: interfaces → implementations → callers → tests ``` @@ -29,65 +29,65 @@ description: "Use when the user wants to rename, extract, split, move, or restru ### Rename Symbol ``` -- [ ] gitnexus_rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits +- [ ] rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits - [ ] Review graph edits (high confidence) and ast_search edits (review carefully) -- [ ] If satisfied: gitnexus_rename({..., dry_run: false}) — apply edits -- [ ] gitnexus_detect_changes() — verify only expected files changed +- [ ] If satisfied: rename({..., dry_run: false}) — apply edits +- [ ] detect_changes() — verify only expected files changed - [ ] Run tests for affected processes ``` ### Extract Module ``` -- [ ] gitnexus_context({name: target}) — see all incoming/outgoing refs -- [ ] gitnexus_impact({target, direction: "upstream"}) — find all external callers +- [ ] context({name: target}) — see all incoming/outgoing refs +- [ ] impact({target, direction: "upstream"}) — find all external callers - [ ] Define new module interface - [ ] Extract code, update imports -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ### Split Function/Service ``` -- [ ] gitnexus_context({name: target}) — understand all callees +- [ ] context({name: target}) — understand all callees - [ ] Group callees by responsibility -- [ ] gitnexus_impact({target, direction: "upstream"}) — map callers to update +- [ ] impact({target, direction: "upstream"}) — map callers to update - [ ] Create new functions/services - [ ] Update callers -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ## Tools -**gitnexus_rename** — automated multi-file rename: +**rename** — automated multi-file rename: ``` -gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits across 8 files → 10 graph edits (high confidence), 2 ast_search edits (review) → Changes: [{file_path, edits: [{line, old_text, new_text, confidence}]}] ``` -**gitnexus_impact** — map all dependents first: +**impact** — map all dependents first: ``` -gitnexus_impact({target: "validateUser", direction: "upstream"}) +impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware, testUtils → Affected Processes: LoginFlow, TokenRefresh ``` -**gitnexus_detect_changes** — verify your changes after refactoring: +**detect_changes** — verify your changes after refactoring: ``` -gitnexus_detect_changes({scope: "all"}) +detect_changes({scope: "all"}) → Changed: 8 files, 12 symbols → Affected processes: LoginFlow, TokenRefresh → Risk: MEDIUM ``` -**gitnexus_cypher** — custom reference queries: +**cypher** — custom reference queries: ```cypher MATCH (caller)-[:CodeRelation {type: 'CALLS'}]->(f:Function {name: "validateUser"}) @@ -98,24 +98,24 @@ RETURN caller.name, caller.filePath ORDER BY caller.filePath | Risk Factor | Mitigation | | ------------------- | ----------------------------------------- | -| Many callers (>5) | Use gitnexus_rename for automated updates | +| Many callers (>5) | Use rename for automated updates | | Cross-area refs | Use detect_changes after to verify scope | -| String/dynamic refs | gitnexus_query to find them | +| String/dynamic refs | query to find them | | External/public API | Version and deprecate properly | ## Example: Rename `validateUser` to `authenticateUser` ``` -1. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +1. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits: 10 graph (safe), 2 ast_search (review) → Files: validator.ts, login.ts, middleware.ts, config.json... 2. Review ast_search edits (config.json: dynamic reference!) -3. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) +3. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) → Applied 12 edits across 8 files -4. gitnexus_detect_changes({scope: "all"}) +4. detect_changes({scope: "all"}) → Affected: LoginFlow, TokenRefresh → Risk: MEDIUM — run tests for these flows ``` diff --git a/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md index a7e250647..a88b76430 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-debugging/SKILL.md @@ -15,10 +15,10 @@ description: Trace bugs through call chains using knowledge graph ## Workflow ``` -1. gitnexus_query({query: ""}) → Find related execution flows -2. gitnexus_context({name: ""}) → See callers/callees/processes +1. query({query: ""}) → Find related execution flows +2. context({name: ""}) → See callers/callees/processes 3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow -4. gitnexus_cypher({query: "MATCH path..."}) → Custom traces if needed +4. cypher({query: "MATCH path..."}) → Custom traces if needed ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. @@ -27,11 +27,11 @@ description: Trace bugs through call chains using knowledge graph ``` - [ ] Understand the symptom (error message, unexpected behavior) -- [ ] gitnexus_query for error text or related code +- [ ] query for error text or related code - [ ] Identify the suspect function from returned processes -- [ ] gitnexus_context to see callers and callees +- [ ] context to see callers and callees - [ ] Trace execution flow via process resource if applicable -- [ ] gitnexus_cypher for custom call chain traces if needed +- [ ] cypher for custom call chain traces if needed - [ ] Read source files to confirm root cause ``` @@ -39,7 +39,7 @@ description: Trace bugs through call chains using knowledge graph | Symptom | GitNexus Approach | |---------|-------------------| -| Error message | `gitnexus_query` for error text → `context` on throw sites | +| Error message | `query` for error text → `context` on throw sites | | Wrong return value | `context` on the function → trace callees for data flow | | Intermittent failure | `context` → look for external calls, async deps | | Performance issue | `context` → find symbols with many callers (hot paths) | @@ -47,22 +47,22 @@ description: Trace bugs through call chains using knowledge graph ## Tools -**gitnexus_query** — find code related to error: +**query** — find code related to error: ``` -gitnexus_query({query: "payment validation error"}) +query({query: "payment validation error"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError, PaymentException ``` -**gitnexus_context** — full context for a suspect: +**context** — full context for a suspect: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates (external API!) → Processes: CheckoutFlow (step 3/7) ``` -**gitnexus_cypher** — custom call chain traces: +**cypher** — custom call chain traces: ```cypher MATCH path = (a)-[:CodeRelation {type: 'CALLS'}*1..2]->(b:Function {name: "validatePayment"}) RETURN [n IN nodes(path) | n.name] AS chain @@ -71,11 +71,11 @@ RETURN [n IN nodes(path) | n.name] AS chain ## Example: "Payment endpoint returns 500 intermittently" ``` -1. gitnexus_query({query: "payment error handling"}) +1. query({query: "payment error handling"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError -2. gitnexus_context({name: "validatePayment"}) +2. context({name: "validatePayment"}) → Outgoing calls: verifyCard, fetchRates (external API!) 3. READ gitnexus://repo/my-app/process/CheckoutFlow diff --git a/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md index 4549505e5..73df1353a 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-exploring/SKILL.md @@ -17,8 +17,8 @@ description: Navigate unfamiliar code using GitNexus knowledge graph ``` 1. READ gitnexus://repos → Discover indexed repos 2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. gitnexus_query({query: ""}) → Find related execution flows -4. gitnexus_context({name: ""}) → Deep dive on specific symbol +3. query({query: ""}) → Find related execution flows +4. context({name: ""}) → Deep dive on specific symbol 5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow ``` @@ -28,9 +28,9 @@ description: Navigate unfamiliar code using GitNexus knowledge graph ``` - [ ] READ gitnexus://repo/{name}/context -- [ ] gitnexus_query for the concept you want to understand +- [ ] query for the concept you want to understand - [ ] Review returned processes (execution flows) -- [ ] gitnexus_context on key symbols for callers/callees +- [ ] context on key symbols for callers/callees - [ ] READ process resource for full execution traces - [ ] Read source files for implementation details ``` @@ -46,16 +46,16 @@ description: Navigate unfamiliar code using GitNexus knowledge graph ## Tools -**gitnexus_query** — find execution flows related to a concept: +**query** — find execution flows related to a concept: ``` -gitnexus_query({query: "payment processing"}) +query({query: "payment processing"}) → Processes: CheckoutFlow, RefundFlow, WebhookHandler → Symbols grouped by flow with file locations ``` -**gitnexus_context** — 360-degree view of a symbol: +**context** — 360-degree view of a symbol: ``` -gitnexus_context({name: "validateUser"}) +context({name: "validateUser"}) → Incoming calls: loginHandler, apiMiddleware → Outgoing calls: checkToken, getUserById → Processes: LoginFlow (step 2/5), TokenRefresh (step 1/3) @@ -65,10 +65,10 @@ gitnexus_context({name: "validateUser"}) ``` 1. READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. gitnexus_query({query: "payment processing"}) +2. query({query: "payment processing"}) → CheckoutFlow: processPayment → validateCard → chargeStripe → RefundFlow: initiateRefund → calculateRefund → processRefund -3. gitnexus_context({name: "processPayment"}) +3. context({name: "processPayment"}) → Incoming: checkoutHandler, webhookHandler → Outgoing: validateCard, chargeStripe, saveTransaction 4. Read src/payments/processor.ts for implementation details diff --git a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md index 0733b09ac..139b897e4 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-impact-analysis/SKILL.md @@ -16,9 +16,9 @@ description: Analyze blast radius before making code changes ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) → What depends on this 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. gitnexus_detect_changes() → Map current git changes to affected flows +3. detect_changes() → Map current git changes to affected flows 4. Assess risk and report to user ``` @@ -27,11 +27,11 @@ description: Analyze blast radius before making code changes ## Checklist ``` -- [ ] gitnexus_impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] gitnexus_detect_changes() for pre-commit check +- [ ] detect_changes() for pre-commit check - [ ] Assess risk level and report to user ``` @@ -54,9 +54,9 @@ description: Analyze blast radius before making code changes ## Tools -**gitnexus_impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius: ``` -gitnexus_impact({ +impact({ target: "validateUser", direction: "upstream", minConfidence: 0.8, @@ -71,9 +71,9 @@ gitnexus_impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**gitnexus_detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis: ``` -gitnexus_detect_changes({scope: "staged"}) +detect_changes({scope: "staged"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -83,7 +83,7 @@ gitnexus_detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. gitnexus_impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/gitnexus-cursor-integration/skills/gitnexus-pr-review/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-pr-review/SKILL.md index 319c063f9..9f1d362e5 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-pr-review/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-pr-review/SKILL.md @@ -18,10 +18,10 @@ description: "Use when the user wants to review a pull request, understand what ``` 1. gh pr diff → Get the raw diff -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows +2. detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows 3. For each changed symbol: - gitnexus_impact({target: "", direction: "upstream"}) → Blast radius per change -4. gitnexus_context({name: ""}) → Understand callers/callees + impact({target: "", direction: "upstream"}) → Blast radius per change +4. context({name: ""}) → Understand callers/callees 5. READ gitnexus://repo/{name}/processes → Check affected execution flows 6. Summarize findings with risk assessment ``` @@ -32,10 +32,10 @@ description: "Use when the user wants to review a pull request, understand what ``` - [ ] Fetch PR diff (gh pr diff or git diff base...head) -- [ ] gitnexus_detect_changes to map changes to affected execution flows -- [ ] gitnexus_impact on each non-trivial changed symbol +- [ ] detect_changes to map changes to affected execution flows +- [ ] impact on each non-trivial changed symbol - [ ] Review d=1 items (WILL BREAK) — are callers updated? -- [ ] gitnexus_context on key changed symbols to understand full picture +- [ ] context on key changed symbols to understand full picture - [ ] Check if affected processes have test coverage - [ ] Assess overall risk level - [ ] Write review summary with findings @@ -63,20 +63,20 @@ description: "Use when the user wants to review a pull request, understand what ## Tools -**gitnexus_detect_changes** — map PR diff to affected execution flows: +**detect_changes** — map PR diff to affected execution flows: ``` -gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +detect_changes({scope: "compare", base_ref: "main"}) → Changed: 8 symbols in 4 files → Affected processes: CheckoutFlow, RefundFlow, WebhookHandler → Risk: MEDIUM ``` -**gitnexus_impact** — blast radius per changed symbol: +**impact** — blast radius per changed symbol: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream"}) +impact({target: "validatePayment", direction: "upstream"}) → d=1 (WILL BREAK): - processCheckout (src/checkout.ts:42) [CALLS, 100%] @@ -86,20 +86,20 @@ gitnexus_impact({target: "validatePayment", direction: "upstream"}) - checkoutRouter (src/routes/checkout.ts:22) [CALLS, 95%] ``` -**gitnexus_impact with tests** — check test coverage: +**impact with tests** — check test coverage: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream", includeTests: true}) +impact({target: "validatePayment", direction: "upstream", includeTests: true}) → Tests that cover this symbol: - validatePayment.test.ts [direct] - checkout.integration.test.ts [via processCheckout] ``` -**gitnexus_context** — understand a changed symbol's role: +**context** — understand a changed symbol's role: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates @@ -112,20 +112,20 @@ gitnexus_context({name: "validatePayment"}) 1. gh pr diff 42 > /tmp/pr42.diff → 4 files changed: payments.ts, checkout.ts, types.ts, utils.ts -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +2. detect_changes({scope: "compare", base_ref: "main"}) → Changed symbols: validatePayment, PaymentInput, formatAmount → Affected processes: CheckoutFlow, RefundFlow → Risk: MEDIUM -3. gitnexus_impact({target: "validatePayment", direction: "upstream"}) +3. impact({target: "validatePayment", direction: "upstream"}) → d=1: processCheckout, webhookHandler (WILL BREAK) → webhookHandler is NOT in the PR diff — potential breakage! -4. gitnexus_impact({target: "PaymentInput", direction: "upstream"}) +4. impact({target: "PaymentInput", direction: "upstream"}) → d=1: validatePayment (in PR), createPayment (NOT in PR) → createPayment uses the old PaymentInput shape — breaking change! -5. gitnexus_context({name: "formatAmount"}) +5. context({name: "formatAmount"}) → Called by 12 functions — but change is backwards-compatible (added optional param) 6. Review summary: diff --git a/gitnexus-cursor-integration/skills/gitnexus-refactoring/SKILL.md b/gitnexus-cursor-integration/skills/gitnexus-refactoring/SKILL.md index a49b58be4..76c9d3351 100644 --- a/gitnexus-cursor-integration/skills/gitnexus-refactoring/SKILL.md +++ b/gitnexus-cursor-integration/skills/gitnexus-refactoring/SKILL.md @@ -15,9 +15,9 @@ description: Plan safe refactors using blast radius and dependency mapping ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → Map all dependents -2. gitnexus_query({query: "X"}) → Find execution flows involving X -3. gitnexus_context({name: "X"}) → See all incoming/outgoing refs +1. impact({target: "X", direction: "upstream"}) → Map all dependents +2. query({query: "X"}) → Find execution flows involving X +3. context({name: "X"}) → See all incoming/outgoing refs 4. Plan update order: interfaces → implementations → callers → tests ``` @@ -27,60 +27,60 @@ description: Plan safe refactors using blast radius and dependency mapping ### Rename Symbol ``` -- [ ] gitnexus_rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits +- [ ] rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits - [ ] Review graph edits (high confidence) and ast_search edits (review carefully) -- [ ] If satisfied: gitnexus_rename({..., dry_run: false}) — apply edits -- [ ] gitnexus_detect_changes() — verify only expected files changed +- [ ] If satisfied: rename({..., dry_run: false}) — apply edits +- [ ] detect_changes() — verify only expected files changed - [ ] Run tests for affected processes ``` ### Extract Module ``` -- [ ] gitnexus_context({name: target}) — see all incoming/outgoing refs -- [ ] gitnexus_impact({target, direction: "upstream"}) — find all external callers +- [ ] context({name: target}) — see all incoming/outgoing refs +- [ ] impact({target, direction: "upstream"}) — find all external callers - [ ] Define new module interface - [ ] Extract code, update imports -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ### Split Function/Service ``` -- [ ] gitnexus_context({name: target}) — understand all callees +- [ ] context({name: target}) — understand all callees - [ ] Group callees by responsibility -- [ ] gitnexus_impact({target, direction: "upstream"}) — map callers to update +- [ ] impact({target, direction: "upstream"}) — map callers to update - [ ] Create new functions/services - [ ] Update callers -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ## Tools -**gitnexus_rename** — automated multi-file rename: +**rename** — automated multi-file rename: ``` -gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits across 8 files → 10 graph edits (high confidence), 2 ast_search edits (review) → Changes: [{file_path, edits: [{line, old_text, new_text, confidence}]}] ``` -**gitnexus_impact** — map all dependents first: +**impact** — map all dependents first: ``` -gitnexus_impact({target: "validateUser", direction: "upstream"}) +impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware, testUtils → Affected Processes: LoginFlow, TokenRefresh ``` -**gitnexus_detect_changes** — verify your changes after refactoring: +**detect_changes** — verify your changes after refactoring: ``` -gitnexus_detect_changes({scope: "all"}) +detect_changes({scope: "all"}) → Changed: 8 files, 12 symbols → Affected processes: LoginFlow, TokenRefresh → Risk: MEDIUM ``` -**gitnexus_cypher** — custom reference queries: +**cypher** — custom reference queries: ```cypher MATCH (caller)-[:CodeRelation {type: 'CALLS'}]->(f:Function {name: "validateUser"}) RETURN caller.name, caller.filePath ORDER BY caller.filePath @@ -90,24 +90,24 @@ RETURN caller.name, caller.filePath ORDER BY caller.filePath | Risk Factor | Mitigation | |-------------|------------| -| Many callers (>5) | Use gitnexus_rename for automated updates | +| Many callers (>5) | Use rename for automated updates | | Cross-area refs | Use detect_changes after to verify scope | -| String/dynamic refs | gitnexus_query to find them | +| String/dynamic refs | query to find them | | External/public API | Version and deprecate properly | ## Example: Rename `validateUser` to `authenticateUser` ``` -1. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +1. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits: 10 graph (safe), 2 ast_search (review) → Files: validator.ts, login.ts, middleware.ts, config.json... 2. Review ast_search edits (config.json: dynamic reference!) -3. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) +3. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) → Applied 12 edits across 8 files -4. gitnexus_detect_changes({scope: "all"}) +4. detect_changes({scope: "all"}) → Affected: LoginFlow, TokenRefresh → Risk: MEDIUM — run tests for these flows ``` diff --git a/gitnexus-shared/src/graph/types.ts b/gitnexus-shared/src/graph/types.ts index ede8bc906..86abc9eba 100644 --- a/gitnexus-shared/src/graph/types.ts +++ b/gitnexus-shared/src/graph/types.ts @@ -44,7 +44,10 @@ export type NodeLabel = | 'Template' | 'Section' | 'Route' - | 'Tool'; + | 'Tool' + // Taint/PDG substrate (issue #2080). Intra-procedural control-flow node. + // Emitted by no phase yet — M1 (#2081) populates these behind an opt-in. + | 'BasicBlock'; export type NodeProperties = { name: string; @@ -89,6 +92,8 @@ export type NodeProperties = { responseKeys?: string[]; errorKeys?: string[]; middleware?: string[]; + // BasicBlock (taint/PDG substrate, issue #2080) — reuses filePath/startLine/endLine. + text?: string; // Extensible [key: string]: unknown; }; @@ -131,7 +136,28 @@ export type RelationshipType = * `reason` encodes the event name: `vue-emit: `. * Complements `BINDS_EVENT_HANDLER`; a Cypher query joining on the * component File node reveals all (emitter, handler) pairs. */ - | 'EMITS_EVENT'; + | 'EMITS_EVENT' + // ── Taint/PDG substrate (issue #2080) ──────────────────────────────────── + // Reserved edge types for the taint-first PDG substrate. No phase emits any + // of these yet; they are populated behind an opt-in by later milestones + // (CFG → M1 #2081, REACHING_DEF → M2 #2082, TAINTED/SANITIZES/TAINT_PATH → + // M3/M4 #2083/#2084). Adding them here keeps the shared schema stable so + // downstream work does not re-ripple the exhaustiveness sites. + /** Control-flow edge between two BasicBlock nodes (intra-procedural CFG). */ + | 'CFG' + /** Data-dependence edge: a definition of `variable` reaches a use of it. + * The `variable` name is stored in the relation's existing `reason` column + * (M0/S1 verdict: LadybugDB has no secondary index on relationship + * properties, so a dedicated indexed column would not speed the + * variable-filtered path query). */ + | 'REACHING_DEF' + /** A tainted value flows from source toward sink. */ + | 'TAINTED' + /** A sanitizer clears taint along a flow. */ + | 'SANITIZES' + /** Materialized source→sink taint path. Working name — final name/representation + * is confirmed when M3/M4 emits it; no persisted edge exists before then. */ + | 'TAINT_PATH'; export interface GraphNode { id: string; diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index d732d2633..1b1d9c9a9 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -183,13 +183,3 @@ export { stripGitSuffix, } from './integrations/understand-quickly.js'; export type { UqDispatchPayload } from './integrations/understand-quickly.js'; - -// Shadow-mode diff + aggregation (RFC §6.3; Ring 2 SHARED #918) -export { diffResolutions } from './scope-resolution/shadow/diff.js'; -export type { - ShadowAgreement, - ShadowCallsite, - ShadowDiff, -} from './scope-resolution/shadow/diff.js'; -export { aggregateDiffs } from './scope-resolution/shadow/aggregate.js'; -export type { LanguageParityRow, ShadowParityReport } from './scope-resolution/shadow/aggregate.js'; diff --git a/gitnexus-shared/src/lbug/schema-constants.ts b/gitnexus-shared/src/lbug/schema-constants.ts index 656ffe552..d022ba5c4 100644 --- a/gitnexus-shared/src/lbug/schema-constants.ts +++ b/gitnexus-shared/src/lbug/schema-constants.ts @@ -40,6 +40,8 @@ export const NODE_TABLES = [ 'Module', 'Route', 'Tool', + // Taint/PDG substrate (issue #2080) — inert until M1 (#2081) emits blocks. + 'BasicBlock', ] as const; export type NodeTableName = (typeof NODE_TABLES)[number]; @@ -67,6 +69,14 @@ export const REL_TYPES = [ 'ENTRY_POINT_OF', 'WRAPS', 'QUERIES', + // Taint/PDG substrate (issue #2080) — reserved edge types, emitted by no + // phase yet (CFG → M1, REACHING_DEF → M2, TAINTED/SANITIZES/TAINT_PATH → + // M3/M4). REACHING_DEF's variable name rides the relation's `reason` column. + 'CFG', + 'REACHING_DEF', + 'TAINTED', + 'SANITIZES', + 'TAINT_PATH', ] as const; export type RelType = (typeof REL_TYPES)[number]; diff --git a/gitnexus-shared/src/scope-resolution/module-scope-index.ts b/gitnexus-shared/src/scope-resolution/module-scope-index.ts index a57c02d27..bb1b4bab2 100644 --- a/gitnexus-shared/src/scope-resolution/module-scope-index.ts +++ b/gitnexus-shared/src/scope-resolution/module-scope-index.ts @@ -8,8 +8,7 @@ * * Part of RFC #909 Ring 2 SHARED — #913. * - * Consumed by: #915 (SCC finalize link pass), #923 (shadow harness when - * resolving callsite file → enclosing module). + * Consumed by: #915 (SCC finalize link pass). */ import type { ScopeId } from './types.js'; diff --git a/gitnexus-shared/src/scope-resolution/parsed-file.ts b/gitnexus-shared/src/scope-resolution/parsed-file.ts index 50eb5a795..01b70f206 100644 --- a/gitnexus-shared/src/scope-resolution/parsed-file.ts +++ b/gitnexus-shared/src/scope-resolution/parsed-file.ts @@ -74,4 +74,50 @@ export interface ParsedFile { */ readonly localDefs: readonly SymbolDefinition[]; readonly referenceSites: readonly ReferenceSite[]; + /** + * Opaque, language-private serialization of capture-time side-channel + * state that a provider's `emitScopeCaptures` populates into module-level + * maps as a SIDE EFFECT (not onto the scopes/defs of this `ParsedFile`). + * + * Such state is computed inside the parse worker (where `emitScopeCaptures` + * runs) and would otherwise be lost across the worker→main MessageChannel + * and the disk store, because scope-resolution reuses the serialized + * `ParsedFile` and SKIPS re-extraction on the main thread (#1983 — the + * whole point is to avoid a main-thread tree-sitter re-parse). Carrying the + * data here lets the main thread repopulate those maps WITHOUT re-parsing. + * + * Shared / ingestion code treats this as opaque (`unknown`) per AGENTS.md + * (no language names in shared code). The producing language fills it via + * the `LanguageProvider.collectCaptureSideChannel` hook (worker side) and + * consumes it via the `ScopeResolver.applyCaptureSideChannel` hook + * (main-thread resolution side). It MUST be plain JSON-serializable data + * (objects / arrays / primitives) so it round-trips through the disk-backed + * `parsedfile-store` (JSON.stringify + interning reviver). + * + * Optional: providers whose `emitScopeCaptures` is pure (no module-level + * side effects — the contract default) leave this undefined. + */ + readonly captureSideChannel?: unknown; + + /** + * Per-function control-flow graphs for this file (#2081 M1, PDG/taint + * substrate). A DISTINCT field from {@link captureSideChannel} — different + * producer, consumer, and lifecycle: the worker builds it from the + * tree-sitter AST via `LanguageProvider.cfgVisitor` (only on a `--pdg` run), + * and scope-resolution emits BasicBlock nodes + CFG edges from it while the + * disk-backed ParsedFile store is still live (it is NOT a capture-time + * marker the resolver restores into module maps). Kept separate so a future + * change to either channel's shape invalidates independently. + * + * Shared / ingestion code treats this as opaque (`unknown`) per AGENTS.md. + * Concretely it is a `readonly FunctionCfg[]` (see + * `core/ingestion/cfg/types.ts`) — plain JSON-serializable data (no AST + * refs, no class instances) so it round-trips through the parse cache and + * the `parsedfile-store` (whose interning reviver keys on `nodeId`, which + * these blocks/edges deliberately lack). + * + * Optional: `undefined` on non-`--pdg` runs and for languages with no + * `cfgVisitor` — the default for every run today. + */ + readonly cfgSideChannel?: unknown; } diff --git a/gitnexus-shared/src/scope-resolution/registries/evidence.ts b/gitnexus-shared/src/scope-resolution/registries/evidence.ts index cabeb6a95..855d1f15a 100644 --- a/gitnexus-shared/src/scope-resolution/registries/evidence.ts +++ b/gitnexus-shared/src/scope-resolution/registries/evidence.ts @@ -57,8 +57,7 @@ export interface RawSignals { * * Emission order mirrors the `EvidenceWeights` layout: where-found → * type-binding → corroborators → arity → degraded. Stable order makes - * the per-signal contributions easy to reason about in tests and in the - * shadow-mode parity dashboard. + * the per-signal contributions easy to reason about in tests. */ export function composeEvidence(signals: RawSignals): readonly ResolutionEvidence[] { const out: ResolutionEvidence[] = []; @@ -141,7 +140,7 @@ export function composeEvidence(signals: RawSignals): readonly ResolutionEvidenc /** * Sum evidence weights and clamp to `[0, 1]`. Separate from `composeEvidence` - * so tests and the parity dashboard can inspect the raw evidence list. + * so tests can inspect the raw evidence list. */ export function confidenceFromEvidence(evidence: readonly ResolutionEvidence[]): number { let sum = 0; diff --git a/gitnexus-shared/src/scope-resolution/shadow/aggregate.ts b/gitnexus-shared/src/scope-resolution/shadow/aggregate.ts deleted file mode 100644 index 27c24ff92..000000000 --- a/gitnexus-shared/src/scope-resolution/shadow/aggregate.ts +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Shadow-mode aggregation — per-language parity %, per-evidence-kind - * breakdown of divergences. Consumed by the parity dashboard (RING2-PKG-5). - * - * Pure functions; no I/O. The harness persists per-run JSON; the dashboard - * reads `.gitnexus/shadow-parity/latest.json` and renders. - * - * Related types — `ShadowAgreement`, `ShadowCallsite`, `ShadowDiff` — are - * defined alongside `diffResolutions` in `./diff.ts` and re-exported - * through the top-level `gitnexus-shared` barrel. Consumers import all - * three from `gitnexus-shared`, not from this module. - * - * Part of RFC #909 Ring 2 SHARED — #918. - */ - -import type { SupportedLanguages } from '../../languages.js'; -import type { ResolutionEvidence } from '../types.js'; -import type { ShadowAgreement, ShadowDiff } from './diff.js'; - -// ─── Aggregated report shape ──────────────────────────────────────────────── - -export interface LanguageParityRow { - readonly language: SupportedLanguages; - readonly totalCalls: number; - readonly bothAgree: number; - readonly onlyLegacy: number; - readonly onlyNew: number; - readonly bothDisagree: number; - readonly bothEmpty: number; - /** - * Fraction in [0, 1]. Numerator = `bothAgree`; denominator = "calls where - * at least one side resolved" = `totalCalls - bothEmpty`. - * - * When the denominator is 0 (all calls for this language were - * `both-empty`), returns 0. Callers rendering the dashboard should treat - * a 0 parity alongside `totalCalls === bothEmpty` as "no signal" rather - * than "total disagreement". - */ - readonly parity: number; - /** - * Divergence signals broken down by `ResolutionEvidence.kind`. Sourced - * from `ShadowDiff.evidenceDelta` on non-agreeing rows only — `both-agree` - * and `both-empty` do not contribute. - */ - readonly evidenceBreakdown: ReadonlyMap; -} - -export interface ShadowParityReport { - readonly generatedAt: string; // ISO 8601 - readonly perLanguage: readonly LanguageParityRow[]; - readonly overall: Omit; -} - -// ─── Public API ───────────────────────────────────────────────────────────── - -/** - * Aggregate a stream of `ShadowDiff` records into a `ShadowParityReport`, - * bucketed by language. Pure function. - * - * - `perLanguage` rows are sorted alphabetically by `SupportedLanguages` - * value for stable JSON output (the dashboard reads - * `.gitnexus/shadow-parity/latest.json` and diffing snapshots is useful). - * - `overall` is the column-wise sum across languages. - * - `generatedAt` is injected via the `now` parameter so tests stay - * deterministic; production callers let it default to `new Date()`. - */ -export function aggregateDiffs( - diffs: readonly { readonly language: SupportedLanguages; readonly diff: ShadowDiff }[], - now: Date = new Date(), -): ShadowParityReport { - const perLanguageMap = new Map(); - - for (const { language, diff } of diffs) { - let counts = perLanguageMap.get(language); - if (!counts) { - counts = makeEmptyCounts(); - perLanguageMap.set(language, counts); - } - tallyDiff(counts, diff); - } - - const perLanguage: LanguageParityRow[] = Array.from(perLanguageMap.entries()) - .map(([language, counts]) => buildRow(language, counts)) - .sort((a, b) => a.language.localeCompare(b.language)); - - const overall = buildOverallRow(perLanguage); - - return { - generatedAt: now.toISOString(), - perLanguage, - overall, - }; -} - -// ─── Internal helpers ─────────────────────────────────────────────────────── - -interface MutableCounts { - totalCalls: number; - bothAgree: number; - onlyLegacy: number; - onlyNew: number; - bothDisagree: number; - bothEmpty: number; - evidenceBreakdown: Map; -} - -function makeEmptyCounts(): MutableCounts { - return { - totalCalls: 0, - bothAgree: 0, - onlyLegacy: 0, - onlyNew: 0, - bothDisagree: 0, - bothEmpty: 0, - evidenceBreakdown: new Map(), - }; -} - -function tallyDiff(counts: MutableCounts, diff: ShadowDiff): void { - counts.totalCalls += 1; - incrementAgreement(counts, diff.agreement); - if (diff.agreement === 'both-agree' || diff.agreement === 'both-empty') return; - for (const ev of diff.evidenceDelta) { - counts.evidenceBreakdown.set(ev.kind, (counts.evidenceBreakdown.get(ev.kind) ?? 0) + 1); - } -} - -function incrementAgreement(counts: MutableCounts, agreement: ShadowAgreement): void { - switch (agreement) { - case 'both-agree': - counts.bothAgree += 1; - return; - case 'only-legacy': - counts.onlyLegacy += 1; - return; - case 'only-new': - counts.onlyNew += 1; - return; - case 'both-disagree': - counts.bothDisagree += 1; - return; - case 'both-empty': - counts.bothEmpty += 1; - return; - } -} - -function buildRow(language: SupportedLanguages, counts: MutableCounts): LanguageParityRow { - const resolved = counts.totalCalls - counts.bothEmpty; - const parity = resolved > 0 ? counts.bothAgree / resolved : 0; - return { - language, - totalCalls: counts.totalCalls, - bothAgree: counts.bothAgree, - onlyLegacy: counts.onlyLegacy, - onlyNew: counts.onlyNew, - bothDisagree: counts.bothDisagree, - bothEmpty: counts.bothEmpty, - parity, - // Freeze via `new Map` on a sorted-kind copy so downstream consumers - // can't mutate the aggregator's internal state. - evidenceBreakdown: new Map( - Array.from(counts.evidenceBreakdown.entries()).sort(([a], [b]) => a.localeCompare(b)), - ), - }; -} - -function buildOverallRow( - perLanguage: readonly LanguageParityRow[], -): Omit { - let totalCalls = 0; - let bothAgree = 0; - let onlyLegacy = 0; - let onlyNew = 0; - let bothDisagree = 0; - let bothEmpty = 0; - for (const row of perLanguage) { - totalCalls += row.totalCalls; - bothAgree += row.bothAgree; - onlyLegacy += row.onlyLegacy; - onlyNew += row.onlyNew; - bothDisagree += row.bothDisagree; - bothEmpty += row.bothEmpty; - } - const resolved = totalCalls - bothEmpty; - const parity = resolved > 0 ? bothAgree / resolved : 0; - return { totalCalls, bothAgree, onlyLegacy, onlyNew, bothDisagree, bothEmpty, parity }; -} diff --git a/gitnexus-shared/src/scope-resolution/shadow/diff.ts b/gitnexus-shared/src/scope-resolution/shadow/diff.ts deleted file mode 100644 index a1c8755c6..000000000 --- a/gitnexus-shared/src/scope-resolution/shadow/diff.ts +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Shadow-mode diff logic — RFC §6.3. - * - * Pure comparison logic for shadow mode. Takes two `Resolution[]` (legacy - * DAG result + new scope-based registry result) and produces a structured - * diff record for the parity dashboard. - * - * Consumed by the Ring 2 PKG shadow harness (#923), which dual-runs each - * call through legacy + new paths, diffs results, and persists per-run JSON - * for the parity dashboard. - * - * Part of RFC #909 Ring 2 SHARED — #918. - */ - -import type { Resolution, ResolutionEvidence } from '../types.js'; - -// ─── Diff record shape ────────────────────────────────────────────────────── - -export type ShadowAgreement = - | 'both-agree' // top match identical (same DefId) - | 'only-legacy' // legacy resolved; new did not - | 'only-new' // new resolved; legacy did not - | 'both-disagree' // both resolved, but to different targets - | 'both-empty'; // both returned empty - -export interface ShadowDiff { - readonly callsite: ShadowCallsite; - readonly legacy: Resolution | null; - readonly newResult: Resolution | null; - readonly agreement: ShadowAgreement; - /** - * Symmetric difference of the two top resolutions' `evidence` arrays, - * keyed on `ResolutionEvidence.kind`. - * - * - For `'both-agree'` and `'both-empty'` agreements, always empty. - * - For `'both-disagree'`, contains evidence kinds present on exactly one - * side (not in both). - * - For `'only-legacy'`, contains all of legacy's top evidence. - * - For `'only-new'`, contains all of new's top evidence. - */ - readonly evidenceDelta: readonly ResolutionEvidence[]; -} - -export interface ShadowCallsite { - readonly filePath: string; - readonly line: number; - readonly col: number; - readonly calledName: string; -} - -// ─── Public API ───────────────────────────────────────────────────────────── - -/** - * Compare two `Resolution[]` arrays (top matches at `[0]`) and produce a - * `ShadowDiff`. Pure function. - * - * Agreement rules: - * - both arrays empty → `'both-empty'`, `evidenceDelta: []` - * - legacy empty, new non-empty → `'only-new'`, `evidenceDelta` = new's top evidence - * - legacy non-empty, new empty → `'only-legacy'`, `evidenceDelta` = legacy's top evidence - * - both non-empty, same top `def.nodeId` → `'both-agree'`, `evidenceDelta: []` - * - both non-empty, different top `def.nodeId` → `'both-disagree'`, - * `evidenceDelta` = symmetric difference by `ResolutionEvidence.kind` - * (first occurrence of a kind-only-on-legacy then kind-only-on-new; order - * preserved from input arrays) - * - * Evidence-delta rationale: callers aggregating divergences want to know - * which signal kinds explain a disagreement. Keying on `kind` (not full - * equality over `weight`/`note`) avoids spurious deltas when the same - * signal fires with slightly different calibration weights on each side. - */ -export function diffResolutions( - callsite: ShadowCallsite, - legacy: readonly Resolution[], - newResult: readonly Resolution[], -): ShadowDiff { - const legacyTop: Resolution | null = legacy.length > 0 ? legacy[0] : null; - const newTop: Resolution | null = newResult.length > 0 ? newResult[0] : null; - - const agreement: ShadowAgreement = (() => { - if (legacyTop === null && newTop === null) return 'both-empty'; - if (legacyTop === null) return 'only-new'; - if (newTop === null) return 'only-legacy'; - return legacyTop.def.nodeId === newTop.def.nodeId ? 'both-agree' : 'both-disagree'; - })(); - - const evidenceDelta = computeEvidenceDelta(legacyTop, newTop, agreement); - - return { - callsite, - legacy: legacyTop, - newResult: newTop, - agreement, - evidenceDelta, - }; -} - -// ─── Internal helpers ─────────────────────────────────────────────────────── - -/** - * Symmetric difference of two evidence arrays, keyed on - * `ResolutionEvidence.kind`. Preserves input order: legacy-only signals - * first (in legacy's original order), then new-only signals (in new's order). - * - * For `'both-agree'` / `'both-empty'` the delta is empty by contract. For - * `'only-legacy'` / `'only-new'` one side's evidence is the delta (nothing to - * subtract against). - */ -function computeEvidenceDelta( - legacy: Resolution | null, - newResult: Resolution | null, - agreement: ShadowAgreement, -): readonly ResolutionEvidence[] { - if (agreement === 'both-agree' || agreement === 'both-empty') return []; - if (agreement === 'only-legacy') return legacy!.evidence; - if (agreement === 'only-new') return newResult!.evidence; - - // both-disagree: symmetric difference keyed on `kind` - const legacyKinds = new Set(legacy!.evidence.map((e) => e.kind)); - const newKinds = new Set(newResult!.evidence.map((e) => e.kind)); - - const onlyInLegacy = legacy!.evidence.filter((e) => !newKinds.has(e.kind)); - const onlyInNew = newResult!.evidence.filter((e) => !legacyKinds.has(e.kind)); - - return [...onlyInLegacy, ...onlyInNew]; -} diff --git a/gitnexus-shared/src/scope-resolution/symbol-definition.ts b/gitnexus-shared/src/scope-resolution/symbol-definition.ts index 06814db3c..3bcdb43eb 100644 --- a/gitnexus-shared/src/scope-resolution/symbol-definition.ts +++ b/gitnexus-shared/src/scope-resolution/symbol-definition.ts @@ -57,6 +57,10 @@ export interface SymbolDefinition { * Currently used by C++ overload ranking to exclude explicit constructors * from implicit user-defined conversion candidates. */ isExplicit?: boolean; + /** True when the callable is declared unavailable (for example C++ `= delete`). + * Unavailable callables still participate in overload selection, but a + * selected unavailable target must suppress edge emission. */ + isDeleted?: boolean; /** Links Method/Constructor/Property to owning Class/Struct/Trait nodeId */ ownerId?: string; /** #1982/#1993: bridge-held enclosing-namespace path (e.g. `NS1`, `Outer.Inner`) diff --git a/gitnexus-web/e2e/folder-upload.spec.ts b/gitnexus-web/e2e/folder-upload.spec.ts new file mode 100644 index 000000000..ba31b752c --- /dev/null +++ b/gitnexus-web/e2e/folder-upload.spec.ts @@ -0,0 +1,112 @@ +import { test, expect } from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * E2E for the browser folder-upload flow (replaces the removed server-side + * directory picker). Mocks the backend so no live gitnexus server is needed. + */ + +const BACKEND_URL = 'http://localhost:4747'; + +let fixtureDir: string; + +test.beforeAll(() => { + // A tiny "repo" folder; Playwright sets webkitRelativePath = /. + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-upload-e2e-')); + fixtureDir = path.join(root, 'myrepo'); + fs.mkdirSync(path.join(fixtureDir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(fixtureDir, 'README.md'), '# hi\n'); + fs.writeFileSync(path.join(fixtureDir, 'src', 'index.ts'), 'export const x = 1;\n'); +}); + +test.beforeEach(async ({ page }) => { + await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] })); + await page.route(`${BACKEND_URL}/api/info`, (route) => + route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }), + ); + await page.route(`${BACKEND_URL}/api/heartbeat`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: ':ok\n\n', + }), + ); +}); + +test('uploading a folder posts a multipart upload and starts analysis', async ({ page }) => { + let uploadContentType = ''; + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + uploadContentType = route.request().headers()['content-type'] ?? ''; + await route.fulfill({ json: { jobId: 'job-e2e', status: 'analyzing' } }); + }); + // SSE progress → immediately complete. + await page.route(`${BACKEND_URL}/api/analyze/job-e2e/progress`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }), + ); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); + + // Select the fixture folder via the hidden webkitdirectory input. + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + + // The upload endpoint should be hit with a multipart body, and the UI should + // leave the input phase (upload button no longer shown). + await expect.poll(() => uploadContentType).toContain('multipart/form-data'); + await expect(page.locator('[data-testid="upload-folder"]')).toBeHidden({ timeout: 10_000 }); +}); + +test('switching modes mid-upload aborts it and never shows progress', async ({ page }) => { + // Hold the upload response until the test releases it, so the mode switch + // happens while the POST is in flight (the review 4470339833 repro). + let releaseUpload!: () => void; + const uploadGate = new Promise((res) => (releaseUpload = res)); + let uploadAborted = false; + let progressOpened = false; + + // The client-side AbortController kills the POST at mode-switch time; that + // surfaces as a failed request (net::ERR_ABORTED), not as a response. + page.on('requestfailed', (req) => { + if (req.url().includes('/api/analyze/upload') && /ABORTED/.test(req.failure()?.errorText ?? '')) + uploadAborted = true; + }); + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + await uploadGate; + await route.fulfill({ json: { jobId: 'job-stale', status: 'analyzing' } }).catch(() => {}); // the request may already be gone — that's the point + }); + await page.route(`${BACKEND_URL}/api/analyze/job-stale/progress`, (route) => { + progressOpened = true; + return route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }); + }); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + await expect(page.locator('[data-testid="upload-progress"]')).toBeVisible(); + + // Switch back to GitHub while the upload POST is still pending, then let + // the (now-stale) route handler finish. + await page.getByRole('tab', { name: 'GitHub URL' }).click(); + await expect.poll(() => uploadAborted, { timeout: 10_000 }).toBe(true); + releaseUpload(); + + // The GitHub form stays clean (no error, immediately usable), and no SSE + // progress stream is ever opened by the stale upload. + await expect(page.getByPlaceholder('https://github.com/owner/repo')).toBeEditable(); + await expect(page.locator('[data-testid="upload-progress"]')).toBeHidden(); + expect(progressOpened).toBe(false); +}); diff --git a/gitnexus-web/e2e/onboarding.spec.ts b/gitnexus-web/e2e/onboarding.spec.ts index 5b70899c6..147f55632 100644 --- a/gitnexus-web/e2e/onboarding.spec.ts +++ b/gitnexus-web/e2e/onboarding.spec.ts @@ -218,8 +218,8 @@ test.describe('Flow 3: Analyze form', () => { // Switch to Local Folder tab await page.getByRole('tab', { name: 'Local Folder' }).click(); - // Browse button should be visible - await expect(page.getByText('Browse for folder')).toBeVisible(); + // Upload-a-folder button should be visible (browser folder upload) + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); await page.screenshot({ path: testInfo.outputPath('local-folder-tab.png') }); }); diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index a69275159..e4efc3aa2 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -18,7 +18,7 @@ "@tailwindcss/vite": "^4.3.0", "axios": "^1.16.1", "d3": "^7.9.0", - "dompurify": "^3.4.7", + "dompurify": "^3.4.8", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -28,7 +28,7 @@ "graphology-utils": "^2.3.0", "i18next": "^26.3.0", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.4.2", + "langchain": "^1.4.4", "lru-cache": "^11.2.4", "lucide-react": "^1.16.0", "mermaid": "^11.15.0", @@ -41,7 +41,7 @@ "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", "remark-gfm": "^4.0.1", - "sigma": "^3.0.2", + "sigma": "^3.0.3", "tailwindcss": "^4.2.4", "uuid": "^14.0.0", "zod": "^4.4.3" @@ -57,9 +57,9 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.8.8", + "@vercel/node": "^5.8.12", "@vitejs/plugin-react": "^5.1.4", - "@vitest/coverage-v8": "^4.1.5", + "@vitest/coverage-v8": "^4.1.8", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", @@ -2930,9 +2930,9 @@ } }, "node_modules/@vercel/build-utils": { - "version": "13.26.4", - "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-13.26.4.tgz", - "integrity": "sha512-0g3ZxtZUJZbt4y0Vu4pkHtu1UN58FbVF9cqGT8T6jHp0EHdLGFj5TVCiME8ALeK4tjPImSmxnKZvvB5yb2hqEw==", + "version": "13.27.1", + "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-13.27.1.tgz", + "integrity": "sha512-BD9H2U8I/IPGS1c1stSIkdPxBRu6bkCQFqtRjcT2dcdnBawyHXvzTsnnBQhgB3fJVQtgJT47gVZe1oHDmv0Ktg==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2949,9 +2949,9 @@ "license": "MIT" }, "node_modules/@vercel/error-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@vercel/error-utils/-/error-utils-2.1.0.tgz", - "integrity": "sha512-DiJcXBOB9N6QM4d7hYPM9Ck/AUjzBl58XNQPxS74o7CuvIanjzrGgygP/70VsyEASeIJMazk1LrhwcNTR/eZGQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@vercel/error-utils/-/error-utils-2.2.0.tgz", + "integrity": "sha512-WFWiRxfPzoYWYifaj4thSKvAaZZwUOqD4k5GINRIgZgCiS2E3iAJbWbIsIZmkQdTecWFHcWGA6q48CjisgpOBA==", "dev": true, "license": "Apache-2.0" }, @@ -2983,9 +2983,9 @@ } }, "node_modules/@vercel/node": { - "version": "5.8.8", - "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.8.8.tgz", - "integrity": "sha512-+uRT9evnGWUE6klrJJED4fCvlSxNShbIc/UY4FeUzt2sdcy5a5b1IoYlo94RJd7tAY9Jg2lR2cVfGfsnWH81ZA==", + "version": "5.8.12", + "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.8.12.tgz", + "integrity": "sha512-XK2ML9YVdAlZ3BmGTW4jQL0D55ZHeRWKS+CLPSWReDyOBKaC4tTnTL2tp3z76bAs0pfgbT55nplMiX2mneSbLA==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2993,8 +2993,8 @@ "@edge-runtime/primitives": "4.1.0", "@edge-runtime/vm": "3.2.0", "@types/node": "20.11.0", - "@vercel/build-utils": "13.26.4", - "@vercel/error-utils": "2.1.0", + "@vercel/build-utils": "13.27.1", + "@vercel/error-utils": "2.2.0", "@vercel/nft": "1.10.0", "@vercel/static-config": "3.4.0", "async-listen": "3.0.0", @@ -3101,14 +3101,14 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.5.tgz", - "integrity": "sha512-38C0/Ddb7HcRG0Z4/DUem8x57d2p9jYgp18mkaYswEOQBGsI1CG4f/hjm0ZCeaJfWhSZ4k7jgs29V1Zom7Ki9A==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.8.tgz", + "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.5", + "@vitest/utils": "4.1.8", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -3122,8 +3122,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.5", - "vitest": "4.1.5" + "@vitest/browser": "4.1.8", + "vitest": "4.1.8" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -3132,16 +3132,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz", - "integrity": "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", + "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -3150,13 +3150,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.5.tgz", - "integrity": "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", + "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.5", + "@vitest/spy": "4.1.8", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3187,9 +3187,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.5.tgz", - "integrity": "sha512-7I3q6l5qr03dVfMX2wCo9FxwSJbPdwKjy2uu/YPpU3wfHvIL4QHwVRp57OfGrDFeUJ8/8QdfBKIV12FTtLn00g==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", + "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", "dev": true, "license": "MIT", "dependencies": { @@ -3200,13 +3200,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.5.tgz", - "integrity": "sha512-2D+o7Pr82IEO46YPpoA/YU0neeyr6FTerQb5Ro7BUnBuv6NQtT/kmVnczngiMEBhzgqz2UZYl5gArejsyERDSQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", + "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.5", + "@vitest/utils": "4.1.8", "pathe": "^2.0.3" }, "funding": { @@ -3214,14 +3214,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.5.tgz", - "integrity": "sha512-zypXEt4KH/XgKGPUz4eC2AvErYx0My5hfL8oDb1HzGFpEk1P62bxSohdyOmvz+d9UJwanI68MKwr2EquOaOgMQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", + "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/pretty-format": "4.1.8", + "@vitest/utils": "4.1.8", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3230,9 +3230,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.5.tgz", - "integrity": "sha512-2lNOsh6+R2Idnf1TCZqSwYlKN2E/iDlD8sgU59kYVl+OMDmvldO1VDk39smRfpUNwYpNRVn3w4YfuC7KfbBnkQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", + "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", "dev": true, "license": "MIT", "funding": { @@ -3240,13 +3240,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.5.tgz", - "integrity": "sha512-76wdkrmfXfqGjueGgnb45ITPyUi1ycZ4IHgC2bhPDUfWHklY/q3MdLOAB+TF1e6xfl8NxNY0ZYaPCFNWSsw3Ug==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", + "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.5", + "@vitest/pretty-format": "4.1.8", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -4461,9 +4461,9 @@ "peer": true }, "node_modules/dompurify": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.7.tgz", - "integrity": "sha512-2jBxDJY4RR06tQNy4w5FlFH7kfxsQZlufd0sbv+chfHCxeJwrFw2baUDsSwvBISD4K4RDbd0PTfy3uNXsR6siA==", + "version": "3.4.8", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.8.tgz", + "integrity": "sha512-yb1cEmaOum7wFvOCSQxyfgVlv5D47Rc30iZWoMpbDIWTnJ6grDDQyu2KFJzB2k7u0pMuJcQ1zphH//fFnw2tjQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -5752,9 +5752,9 @@ "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" }, "node_modules/langchain": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.4.2.tgz", - "integrity": "sha512-SLGipy0r4nqQD0aiUOBYLMeGFfB/QiYnMndfZ8sGN89vXDCIXbYqcE7G/4QDDX3nZsM7/emQpoScmlxEX6sDnQ==", + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.4.4.tgz", + "integrity": "sha512-tepOCwUDaIZOYJ9Eo0O6o5dXEN/0KJheiFDnHHFL8Tx8rfkDLL4cOTSTln4Vpn9LpWzXYkjQ8lkHnnNDQWZPeg==", "license": "MIT", "dependencies": { "@langchain/langgraph": "^1.3.2", @@ -8153,9 +8153,9 @@ "license": "ISC" }, "node_modules/sigma": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/sigma/-/sigma-3.0.2.tgz", - "integrity": "sha512-/BUbeOwPGruiBOm0YQQ6ZMcLIZ6tf/W+Jcm7dxZyAX0tK3WP9/sq7/NAWBxPIxVahdGjCJoGwej0Gdrv0DxlQQ==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sigma/-/sigma-3.0.3.tgz", + "integrity": "sha512-5H0zFlx6/NTQpqBg4Rm569ZOpnBOXMaS25UQThIWMU3XyzI5AhmorK/gnl87BvJBLhQd0tW4C0LIp3enWzMoNw==", "license": "MIT", "dependencies": { "events": "^3.3.0", @@ -8822,19 +8822,19 @@ } }, "node_modules/vitest": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.5.tgz", - "integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", + "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.5", - "@vitest/mocker": "4.1.5", - "@vitest/pretty-format": "4.1.5", - "@vitest/runner": "4.1.5", - "@vitest/snapshot": "4.1.5", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/expect": "4.1.8", + "@vitest/mocker": "4.1.8", + "@vitest/pretty-format": "4.1.8", + "@vitest/runner": "4.1.8", + "@vitest/snapshot": "4.1.8", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -8862,12 +8862,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.5", - "@vitest/browser-preview": "4.1.5", - "@vitest/browser-webdriverio": "4.1.5", - "@vitest/coverage-istanbul": "4.1.5", - "@vitest/coverage-v8": "4.1.5", - "@vitest/ui": "4.1.5", + "@vitest/browser-playwright": "4.1.8", + "@vitest/browser-preview": "4.1.8", + "@vitest/browser-webdriverio": "4.1.8", + "@vitest/coverage-istanbul": "4.1.8", + "@vitest/coverage-v8": "4.1.8", + "@vitest/ui": "4.1.8", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index c09ad428f..d449863c6 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -28,7 +28,7 @@ "@tailwindcss/vite": "^4.3.0", "axios": "^1.16.1", "d3": "^7.9.0", - "dompurify": "^3.4.7", + "dompurify": "^3.4.8", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -38,7 +38,7 @@ "graphology-utils": "^2.3.0", "i18next": "^26.3.0", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.4.2", + "langchain": "^1.4.4", "lru-cache": "^11.2.4", "lucide-react": "^1.16.0", "mermaid": "^11.15.0", @@ -51,7 +51,7 @@ "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", "remark-gfm": "^4.0.1", - "sigma": "^3.0.2", + "sigma": "^3.0.3", "tailwindcss": "^4.2.4", "uuid": "^14.0.0", "zod": "^4.4.3" @@ -67,9 +67,9 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.8.8", + "@vercel/node": "^5.8.12", "@vitejs/plugin-react": "^5.1.4", - "@vitest/coverage-v8": "^4.1.5", + "@vitest/coverage-v8": "^4.1.8", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", diff --git a/gitnexus-web/src/components/RepoAnalyzer.tsx b/gitnexus-web/src/components/RepoAnalyzer.tsx index 0b7f0abbd..613284830 100644 --- a/gitnexus-web/src/components/RepoAnalyzer.tsx +++ b/gitnexus-web/src/components/RepoAnalyzer.tsx @@ -21,9 +21,11 @@ import { startAnalyze, cancelAnalyze, streamAnalyzeProgress, + uploadFolder, type JobProgress, } from '../services/backend-client'; import { AnalyzeProgress } from './AnalyzeProgress'; +import { filterRepoFiles } from '@/lib/upload-filter'; import { useTranslation } from 'react-i18next'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -165,8 +167,11 @@ export interface RepoAnalyzerProps { export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProps) => { const { t } = useTranslation(['common', 'errors', 'onboarding']); const inputId = useId(); - const folderInputRef = useRef(null); const [mode, setMode] = useState('github'); + const [uploading, setUploading] = useState(false); + const [uploadSummary, setUploadSummary] = useState<{ count: number; dropped: number } | null>( + null, + ); const [githubUrl, setGithubUrl] = useState(''); const [gitlabUrl, setGitlabUrl] = useState(''); const [localPath, setLocalPath] = useState(''); @@ -181,28 +186,73 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const jobIdRef = useRef(null); const sseControllerRef = useRef(null); + // Owns the in-flight analyze/upload request. The controller doubles as the + // staleness token: each request captures its own controller in a closure and + // bails after the await when that controller was aborted, so a resolution + // arriving after a mode switch / cancel / unmount can never drive state. + const requestControllerRef = useRef(null); const completeTimerRef = useRef | null>(null); + const folderInputRef = useRef(null); useEffect(() => { return () => { sseControllerRef.current?.abort(); + requestControllerRef.current?.abort(); if (completeTimerRef.current) clearTimeout(completeTimerRef.current); }; }, []); + // Abort any in-flight analyze/upload request so its settlement can't drive + // state. Aborting is load-bearing: once a mode switch resets `uploading`, + // the `uploading || isLoading` re-entry guard no longer covers the stale + // request — only its aborted signal does. + const invalidateRequest = (): void => { + requestControllerRef.current?.abort(); + requestControllerRef.current = null; + }; + + // Invalidate the previous request and hand the caller a fresh controller. + const renewRequestController = (): AbortController => { + invalidateRequest(); + const controller = new AbortController(); + requestControllerRef.current = controller; + return controller; + }; + + // An upload that resolved after invalidation has still created a server-side + // job; cancel it so the single analyze slot isn't held for the job's full + // duration. Upload-path only: every upload owns a fresh job (the server + // stages each upload into a unique dir, never dedup-aliasing), whereas URL + // analyzes dedup-alias by repo — the returned jobId may belong to a job + // another session (or this user's own resubmit) is actively watching, so + // cancelling on that path could kill a live analysis. A stale URL job is + // left to finish: a same-URL resubmit re-attaches to it via dedup, and the + // server's job timeout/TTL sweep bounds the slot occupancy. + const cancelStaleUploadJob = (jobId: string): void => { + void cancelAnalyze(jobId).catch(() => {}); + }; + const handleModeChange = (m: InputMode) => { + // ModeTabs fires onChange on every click, including the already-active + // tab — never abort the user's own in-flight request for a no-op click. + if (m === mode) return; + invalidateRequest(); setMode(m); setGithubUrl(''); setGitlabUrl(''); setLocalPath(''); setValidationError(null); + setUploadSummary(null); + setUploading(false); + // An aborted request no longer resolves to move `phase` off 'starting'; + // reset so the new mode's form is immediately usable (also clears a stale + // 'error' phase). Only reachable while showInput is true. + setPhase('input'); }; - // Use the browser's native directory picker (webkitdirectory doesn't give paths, - // so we use a text input + a "Browse" button that opens a standard file input - // to let users pick files from the folder — the path is typed manually since - // browsers don't expose absolute paths for security reasons). - // For local paths, the user types or pastes the absolute path. + // Local-folder mode uploads the selected folder's files (the browser never + // exposes an absolute path, so the old typed-path/browse approach couldn't + // work — see handleFolderUpload). A typed server path is also still accepted. const canSubmit = mode === 'github' @@ -228,6 +278,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp setValidationError(null); setPhase('starting'); + // Staleness guard only (no wire abort): the POST is short-lived and + // self-terminates, but its resolution must not drive state after a mode + // switch / cancel / unmount invalidated this request. + const controller = renewRequestController(); try { const request = mode === 'github' @@ -236,8 +290,9 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp ? { url: gitlabUrl.trim() } : { path: localPath.trim() }; const { jobId } = await startAnalyze(request); - jobIdRef.current = jobId; - setPhase('analyzing'); + // Stale resolution: return without cancelling — URL jobIds may be + // dedup-aliased to a job another session owns (see cancelStaleUploadJob). + if (controller.signal.aborted) return; const nameSource = mode === 'github' @@ -245,29 +300,84 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp : mode === 'gitlab' ? gitlabUrl.trim() : localPath.trim(); - const controller = streamAnalyzeProgress( - jobId, - (p) => setProgress(p), - (data) => { - const name = - data.repoName ?? - nameSource.split(/[/\\]/).filter(Boolean).at(-1) ?? - t('onboarding:repoAnalyzer.defaultRepoName'); - setCompletedRepoName(name); - setPhase('done'); - sseControllerRef.current = null; - completeTimerRef.current = setTimeout(() => { - completeTimerRef.current = null; - onComplete(name); - }, 1200); - }, - (errMsg) => { - setValidationError(errMsg || t('errors:analysisFailed')); - setPhase('error'); - }, - ); - sseControllerRef.current = controller; + trackJob(jobId, nameSource); } catch (err) { + // Unmount aborts the controller, so this also covers the unmounted case. + if (controller.signal.aborted) return; + setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); + setPhase('error'); + } + }; + + // Drive an already-created analysis job through the SSE progress stream to + // completion. Shared by the path/URL analyze flow and the folder-upload flow. + const trackJob = (jobId: string, fallbackNameSource: string | null) => { + // Callers reach here only with a live (non-aborted) request controller, so + // the component is mounted — unmount aborts the controller. + jobIdRef.current = jobId; + setPhase('analyzing'); + const controller = streamAnalyzeProgress( + jobId, + (p) => setProgress(p), + (data) => { + const name = + data.repoName ?? + (fallbackNameSource + ? fallbackNameSource.split(/[/\\]/).filter(Boolean).at(-1) + : undefined) ?? + t('onboarding:repoAnalyzer.defaultRepoName'); + setCompletedRepoName(name); + setPhase('done'); + sseControllerRef.current = null; + completeTimerRef.current = setTimeout(() => { + completeTimerRef.current = null; + onComplete(name); + }, 1200); + }, + (errMsg) => { + setValidationError(errMsg || t('errors:analysisFailed')); + setPhase('error'); + }, + ); + sseControllerRef.current = controller; + }; + + // Upload a browser-selected folder (webkitdirectory) and start analysis. The + // upload endpoint returns a jobId, which then joins the normal SSE flow. + const handleFolderUpload = async (fileList: FileList) => { + if (uploading || isLoading) return; // guard against a concurrent upload + const { files, manifest, droppedCount } = filterRepoFiles(fileList); + if (files.length === 0) { + setValidationError(t('onboarding:repoAnalyzer.upload.empty')); + return; + } + setValidationError(null); + setUploadSummary({ count: files.length, dropped: droppedCount }); + setUploading(true); + setPhase('starting'); + // The selected folder's name (manifest entries are `/`) is a + // sensible fallback if the server's complete event omits repoName. + const folderName = manifest[0]?.split('/')[0] ?? null; + const controller = renewRequestController(); + try { + const { jobId } = await uploadFolder(files, manifest, controller.signal); + if (controller.signal.aborted) { + // The abort raced the response: the server already created the job. + // (Unmount aborts the controller, so this also covers unmounted.) + cancelStaleUploadJob(jobId); + return; + } + setUploading(false); + trackJob(jobId, folderName); + } catch (err) { + // An abort surfaces in two shapes — BackendError('Request aborted') + // when it lands during fetch, raw AbortError when it lands during the + // response-body read — so branch on the closure controller's signal, + // never on the error identity. In the second shape the server may have + // already launched a job whose id we never learn; that orphan is bounded + // by the server's job timeout and terminal-job TTL sweep. + if (controller.signal.aborted) return; + setUploading(false); setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); setPhase('error'); } @@ -276,6 +386,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const handleCancel = async () => { sseControllerRef.current?.abort(); sseControllerRef.current = null; + // Defensive: no UI path can reach handleCancel while a request is in + // flight (the cancel affordance renders only at phase === 'analyzing'), + // but invalidate it anyway so the guard topology has no holes. + invalidateRequest(); if (jobIdRef.current) { try { await cancelAnalyze(jobIdRef.current); @@ -284,6 +398,8 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp } setPhase('input'); setProgress({ phase: 'queued', percent: 0, message: t('common:analyzePhases.queued') }); + setUploading(false); + setUploadSummary(null); }; const isLoading = phase === 'starting'; @@ -443,35 +559,51 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp )} - {/* Native folder picker + Browse button — below the input */} + {/* Upload a folder from your computer — no server path or mount needed. + The browser can't expose an absolute path, so we upload the files. */} { - const files = e.target.files; - if (files && files.length > 0) { - const rel = files[0].webkitRelativePath; - const folderName = rel.split('/')[0]; - if (folderName) { - setLocalPath(folderName); - setValidationError(null); - } + if (e.target.files && e.target.files.length > 0) { + handleFolderUpload(e.target.files); } e.target.value = ''; }} /> + {uploading && ( +
+
+
+
+

+ {t('onboarding:repoAnalyzer.upload.uploading')} +

+
+ )} + {uploadSummary && !uploading && phase !== 'error' && ( +

+ {t('onboarding:repoAnalyzer.upload.selected', { + fileCount: uploadSummary.count, + dropped: uploadSummary.dropped, + })} +

+ )}
)} diff --git a/gitnexus-web/src/lib/constants.ts b/gitnexus-web/src/lib/constants.ts index fe0505483..2f717cab3 100644 --- a/gitnexus-web/src/lib/constants.ts +++ b/gitnexus-web/src/lib/constants.ts @@ -38,6 +38,7 @@ export const NODE_COLORS: Record = { Template: '#a78bfa', // Violet light - like Type Route: '#f43f5e', // Rose - like Process Tool: '#a855f7', // Purple - like Project + BasicBlock: '#475569', // Slate darker - control-flow node (muted, taint/PDG substrate) }; // Node sizes by type - clear visual hierarchy with dramatic size differences @@ -79,6 +80,7 @@ export const NODE_SIZES: Record = { Template: 3, // Like Type Route: 5, // Like Enum Tool: 5, // Like Enum + BasicBlock: 2, // Tiny - control-flow node (taint/PDG substrate) }; // Community color palette for cluster-based coloring diff --git a/gitnexus-web/src/lib/upload-filter.test.ts b/gitnexus-web/src/lib/upload-filter.test.ts new file mode 100644 index 000000000..e97b9ec2c --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import { filterRepoFiles, MAX_FILE_BYTES } from './upload-filter'; + +type FileLike = { name: string; size: number; webkitRelativePath?: string }; + +function f(webkitRelativePath: string, size = 10): FileLike { + const name = webkitRelativePath.split('/').pop() ?? webkitRelativePath; + return { name, size, webkitRelativePath }; +} + +describe('filterRepoFiles', () => { + it('keeps source files and builds an order-aligned manifest', () => { + const input = [f('repo/src/index.ts', 100), f('repo/README.md', 50)]; + const r = filterRepoFiles(input); + expect(r.files).toHaveLength(2); + expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/README.md']); + expect(r.totalBytes).toBe(150); + expect(r.droppedCount).toBe(0); + }); + + it('excludes .git / node_modules / build dirs anywhere in the path', () => { + const input = [ + f('repo/.git/HEAD'), + f('repo/node_modules/x/index.js'), + f('repo/dist/bundle.js'), + f('repo/src/app.ts'), + f('repo/.gitnexus/meta.json'), + ]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/src/app.ts']); + expect(r.droppedCount).toBe(4); + }); + + it('drops files over the per-file size cap', () => { + const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/small.ts']); + expect(r.droppedCount).toBe(1); + }); + + it('falls back to name when webkitRelativePath is absent', () => { + const r = filterRepoFiles([{ name: 'lone.ts', size: 5 }]); + expect(r.manifest).toEqual(['lone.ts']); + }); +}); diff --git a/gitnexus-web/src/lib/upload-filter.ts b/gitnexus-web/src/lib/upload-filter.ts new file mode 100644 index 000000000..a24520f74 --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.ts @@ -0,0 +1,73 @@ +/** + * Client-side pre-filter for a webkitdirectory folder upload. + * + * Drops VCS metadata, dependency/build directories, and oversized files before + * upload — `.git` alone is often larger than the working tree — so payloads + * stay small and the upload matches what the analyzer actually needs. Produces + * an order-aligned `manifest` of webkitRelativePaths (the server keys on this, + * not the multipart filename, which browsers rewrite). + */ + +/** Directory names excluded anywhere in a file's path. */ +export const EXCLUDED_DIRS = new Set([ + '.git', + '.hg', + '.svn', + 'node_modules', + 'vendor', + '.venv', + '__pycache__', + 'target', + 'dist', + 'build', + 'out', + '.next', + '.nuxt', + '.cache', + 'coverage', + '.idea', + '.gitnexus', +]); + +/** Per-file size cap; matches the server's per-file limit. */ +export const MAX_FILE_BYTES = 25 * 1024 * 1024; + +export interface FilterResult { + files: File[]; + manifest: string[]; + droppedCount: number; + totalBytes: number; +} + +type FileLike = Pick & { webkitRelativePath?: string }; + +/** + * Filter a webkitdirectory `FileList` (or array) into the files to upload plus + * their relative-path manifest. + */ +export function filterRepoFiles(input: ArrayLike): FilterResult { + const files: File[] = []; + const manifest: string[] = []; + let droppedCount = 0; + let totalBytes = 0; + + for (let i = 0; i < input.length; i++) { + const f = input[i]; + const rel = + f.webkitRelativePath && f.webkitRelativePath.length > 0 ? f.webkitRelativePath : f.name; + const segments = rel.split('/'); + if (segments.some((s) => EXCLUDED_DIRS.has(s))) { + droppedCount++; + continue; + } + if (f.size > MAX_FILE_BYTES) { + droppedCount++; + continue; + } + files.push(f as File); + manifest.push(rel); + totalBytes += f.size; + } + + return { files, manifest, droppedCount, totalBytes }; +} diff --git a/gitnexus-web/src/locales/en/onboarding.json b/gitnexus-web/src/locales/en/onboarding.json index cd12c2095..8be58efce 100644 --- a/gitnexus-web/src/locales/en/onboarding.json +++ b/gitnexus-web/src/locales/en/onboarding.json @@ -61,7 +61,12 @@ "gitlabRepositoryUrl": "GitLab Repository URL", "gitlabSupported": "Supports GitLab.com and self-hosted GitLab instances.", "localFolderPath": "Local Folder Path", - "browseForFolder": "Browse for folder", - "hideBackground": "Hide (analysis continues in background)" + "hideBackground": "Hide (analysis continues in background)", + "upload": { + "button": "Upload a folder", + "uploading": "Uploading…", + "selected": "{{fileCount}} files ready ({{dropped}} skipped: .git, node_modules, build output)", + "empty": "No analyzable files found in that folder." + } } } diff --git a/gitnexus-web/src/locales/zh-CN/onboarding.json b/gitnexus-web/src/locales/zh-CN/onboarding.json index 6199511f3..f20e60b6d 100644 --- a/gitnexus-web/src/locales/zh-CN/onboarding.json +++ b/gitnexus-web/src/locales/zh-CN/onboarding.json @@ -61,7 +61,12 @@ "gitlabRepositoryUrl": "GitLab 仓库 URL", "gitlabSupported": "支持 GitLab.com 和自托管 GitLab 实例。", "localFolderPath": "本地文件夹路径", - "browseForFolder": "浏览文件夹", - "hideBackground": "隐藏（分析继续在后台进行）" + "hideBackground": "隐藏（分析继续在后台进行）", + "upload": { + "button": "上传文件夹", + "uploading": "上传中…", + "selected": "已准备 {{fileCount}} 个文件（已跳过 {{dropped}} 个：.git、node_modules、构建产物）", + "empty": "该文件夹中未找到可分析的文件。" + } } } diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index e887e3901..286a13187 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -283,12 +283,11 @@ const fetchWithTimeout = async ( ): Promise => { // Merge the external caller signal (if any) with an // `AbortSignal.timeout()` so a timer-fired abort produces a - // `DOMException` with `name === 'TimeoutError'` — which - // `resilientFetch` correctly classifies as terminal-network (no - // retry, no breaker hit). A manual `AbortController.abort()` would - // produce `name === 'AbortError'` and route through the - // retryable-network branch, which mis-penalizes the breaker for - // user-side network slowness. + // `DOMException` with `name === 'TimeoutError'`. Both shapes are + // breaker-safe: `resilientFetch` classifies TimeoutError AND a manual + // `AbortController.abort()`'s AbortError as terminal-network (no + // retry, breaker-neutral via recordNeutral), so caller-driven + // cancellation never penalizes the breaker. const timeoutSignal = AbortSignal.timeout(timeoutMs); const externalSignal = init.signal; const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal; @@ -755,6 +754,35 @@ export const fetchClusterDetail = async (repo: string, name: string): Promise`) and start analysis. + * Sends the file blobs plus a JSON `manifest` of their relative paths — the + * multipart filename can't carry the path (browsers strip separators), so the + * manifest is the source of truth. Routed through fetchWithTimeout (the shared, + * origin-validated request path) rather than a raw XHR; returns the analysis + * jobId, which the caller drives through the normal SSE flow. + */ +export const uploadFolder = async ( + files: File[], + manifest: string[], + signal?: AbortSignal, +): Promise<{ jobId: string; status: string }> => { + const form = new FormData(); + // Manifest MUST precede the file parts (the server enforces this). + form.append('manifest', JSON.stringify(manifest)); + for (const f of files) form.append('files', f); + + const response = await fetchWithTimeout( + `${_backendUrl}/api/analyze/upload`, + { method: 'POST', body: form, signal }, + 5 * 60_000, // up to 5 min for large repos + ); + await assertOk(response); + return response.json() as Promise<{ jobId: string; status: string }>; +}; + // ── Analyze API ──────────────────────────────────────────────────────────── /** Start a server-side analysis job. */ diff --git a/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx b/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx new file mode 100644 index 000000000..094641b93 --- /dev/null +++ b/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx @@ -0,0 +1,218 @@ +/** + * Stale-request guards in RepoAnalyzer (PR #1850 review 4470339833). + * + * An analyze request (folder upload or URL analyze) that is still in flight + * when the user switches modes, cancels, or unmounts must not drive state + * when it later settles: no SSE stream, no phase/error flip — and a + * stale-but-created server job gets a fire-and-forget cancel so the single + * analyze slot is freed. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { RepoAnalyzer } from '../../src/components/RepoAnalyzer'; +import { i18nReady } from '../../src/i18n'; +import { + cancelAnalyze, + startAnalyze, + streamAnalyzeProgress, + uploadFolder, +} from '../../src/services/backend-client'; + +vi.mock('../../src/services/backend-client', () => ({ + startAnalyze: vi.fn(), + cancelAnalyze: vi.fn(), + streamAnalyzeProgress: vi.fn(), + uploadFolder: vi.fn(), +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +const JOB = { jobId: 'job-1', status: 'queued' }; + +/** Gate uploadFolder on a deferred promise and expose the signal it received. */ +function mockUploadWith(d: { promise: Promise }) { + let captured: AbortSignal | undefined; + vi.mocked(uploadFolder).mockImplementation((_files, _manifest, signal) => { + captured = signal; + return d.promise; + }); + return { signal: () => captured }; +} + +/** Render, switch to Local Folder mode, and fire a folder selection. */ +function startUpload() { + const view = render(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + fireEvent.change(screen.getByTestId('folder-upload-input'), { + target: { files: [new File(['x'], 'a.ts')] }, + }); + return view; +} + +beforeEach(async () => { + await i18nReady; + vi.clearAllMocks(); + vi.mocked(cancelAnalyze).mockResolvedValue(undefined as never); + vi.mocked(streamAnalyzeProgress).mockImplementation(() => new AbortController()); +}); + +describe('folder upload', () => { + it('a mode switch mid-upload makes the resolution inert and cancels the job', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'GitHub URL' })); + + // The wire abort happened at mode-switch time, not at resolution time. + expect(upload.signal()?.aborted).toBe(true); + + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + expect(cancelAnalyze).toHaveBeenCalledWith('job-1'); + // The GitHub form is clean and submittable (phase back to 'input'). + expect(screen.getByRole('textbox')).toBeEnabled(); + expect(screen.queryByTestId('upload-progress')).not.toBeInTheDocument(); + }); + + it.each([ + ['BackendError shape', new Error('Request aborted')], + ['raw AbortError shape', new DOMException('The operation was aborted.', 'AbortError')], + ])('an aborted rejection is silent — %s', async (_label, err) => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'GitHub URL' })); + await act(async () => { + d.reject(err); + }); + + expect(screen.queryByText('Request aborted')).not.toBeInTheDocument(); + expect(screen.queryByText('The operation was aborted.')).not.toBeInTheDocument(); + expect(screen.getByRole('textbox')).toBeEnabled(); + }); + + it('a same-tab click does not abort the in-flight upload', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + + expect(upload.signal()?.aborted).toBe(false); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('an unmount mid-upload makes the resolution inert', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + const { unmount } = startUpload(); + unmount(); + + expect(upload.signal()?.aborted).toBe(true); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + }); + + it('the happy path still tracks the job', async () => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(vi.mocked(streamAnalyzeProgress).mock.calls[0][0]).toBe('job-1'); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('a genuine error still surfaces', async () => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + await act(async () => { + d.reject(new Error('upload exploded')); + }); + + expect(screen.getByText('upload exploded')).toBeInTheDocument(); + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + }); +}); + +describe('URL analyze', () => { + function startGithubAnalyze() { + render(); + fireEvent.change(screen.getByRole('textbox'), { + target: { value: 'https://github.com/owner/repo' }, + }); + fireEvent.click(screen.getByRole('button', { name: /Analyze Repository/ })); + } + + it('a mode switch mid-analyze makes the resolution inert without cancelling', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + await act(async () => { + d.resolve({ jobId: 'job-2', status: 'queued' }); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + // No cancel on the URL path: the jobId may be dedup-aliased to a job + // another session owns, so cancelling could kill a live analysis. + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('a stale rejection is silent', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + await act(async () => { + d.reject(new Error('analyze exploded')); + }); + + expect(screen.queryByText('analyze exploded')).not.toBeInTheDocument(); + expect(screen.getByTestId('upload-folder')).toBeEnabled(); + }); + + it('the happy path still tracks the job', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + await act(async () => { + d.resolve({ jobId: 'job-3', status: 'queued' }); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(vi.mocked(streamAnalyzeProgress).mock.calls[0][0]).toBe('job-3'); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/.npmignore b/gitnexus/.npmignore index cf403314a..bf2c8b7c9 100644 --- a/gitnexus/.npmignore +++ b/gitnexus/.npmignore @@ -13,6 +13,26 @@ node_modules/ vendor/**/node_modules vendor/**/build +# ── Lean publish (FUTURE optimization — NOT done here) ───────────────────────── +# Once the build-tree-sitter-prebuilds workflow has committed 6/6 prebuilds for +# EVERY vendored grammar (c, dart, proto, kotlin, swift), the ~50 MB of generated +# source (parser.c etc.) can be dropped from the tarball — node-gyp-build never +# needs the source when a prebuild matches. +# +# IMPORTANT: this CANNOT be done from this file. package.json's `files: ["vendor"]` +# allow-list OVERRIDES .npmignore for the vendor/ subtree (verified: an active +# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). To slim +# the tarball, narrow the `files` field instead — replace the blanket "vendor" +# with the non-source subpaths only (vendor/**/prebuilds/**, +# vendor/**/bindings/node/index.*, vendor/**/src/node-types.json, +# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md). +# +# Whatever the mechanism, the prepack guard +# (scripts/assert-publish-grammar-coverage.cjs, also `npm run +# assert-publish-coverage`) inspects the EFFECTIVE `npm pack` file list and FAILS +# the publish whenever a grammar with <6 prebuilds loses a source-build input — so +# the slim can never silently ship a dead grammar. Do not bypass it. + # Package lock (consumers use their own) package-lock.json diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index 39db0eb90..e860e71cd 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -4,6 +4,100 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +## [1.6.7] - 2026-06-09 + +### Added + +- **Toolchain-free tree-sitter install** — the `c`, `dart`, `proto`, `kotlin`, and `swift` grammars now ship vendored native prebuilds (six platform/arch each — linux/darwin/win32 × x64/arm64, every `.node` load-and-parse verified with committed `SHA256SUMS` and SLSA build provenance), so a fresh install no longer requires a C/C++ toolchain; `kotlin` moved off its `optionalDependency` into the vendored path, `dart`/`proto` keep a source-build fallback when no prebuild matches, and a registry-parameterized CI workflow builds, load-validates, and vendors the binaries (#2113, #2125, #2110) +- **`gitnexus uninstall`** — reverses `gitnexus setup` target-by-target, surgically removing GitNexus MCP server entries (Cursor, Claude Code, Antigravity, OpenCode, Codex), installed skill directories, and Claude Code / Antigravity hook entries with their bundled scripts; idempotent, JSONC-preserving, dry-run by default with `--force` to apply (#2062, #2060) +- **MCP `list_repos` pagination** — bounded `limit`/`offset` paging so clients can reliably enumerate every indexed repository instead of having the unpaginated array truncated by LLM token limits; the result is now a `{ repositories, pagination }` object (page until `pagination.hasMore` is false), with deterministic `(lower-cased name, path)` ordering (#2120, #2119) +- **C++ inheritance-lattice member lookup** — receiver members now resolve through the inheritance lattice with dominance hiding, ambiguous-base suppression, virtual-diamond deduplication, and overload ranking, and class-scope `using Base::member` declarations are no longer mistaken for namespace imports (#2077, #1891) +- **Taint/PDG substrate (M0)** — foundational graph schema and pipeline seams for reliable taint analysis on a PDG-expandable substrate: the `BasicBlock` node label and `CFG` / `REACHING_DEF` / `TAINTED` / `SANITIZES` / `TAINT_PATH` relationship types (round-tripped through the bulk-COPY path), a phase-registry seam (`registerPhase` / `enabledWhen`) generalising the graph-phase opt-in guard, and a per-language source/sink/sanitizer config registry. All additive and inert — no phase emits the new nodes/edges yet and a default `analyze` run is byte-identical to before (#2092, #2080) + +### Fixed + +- **Optional grammars lazy-loaded so `analyze` never crashes when one is missing** — the swift/dart/kotlin `query.ts` modules no longer statically import their tree-sitter binding at module load, so a missing optional grammar can no longer abort `gitnexus analyze` (or the MCP server, `doctor`, and `.githooks` auto-reindex) with `ERR_MODULE_NOT_FOUND` regardless of the repo's actual languages; grammars now resolve lazily at first use inside the worker, `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` is honored at runtime, the scope-resolution phase excludes unavailable-language files, and skip diagnostics/precheck globs were corrected (#2101, #2091, #2093) +- **`tree-sitter-kotlin` optional-grammar install** — install now fails soft when no C/C++ toolchain is present, emitting one clear warning and always exiting 0 (mirroring the Swift/Dart/Proto probes) instead of breaking `gitnexus` install; optional-grammar/toolchain docs corrected to include Kotlin (#2110, #2107) +- **CLI image FTS keyword search** — the full-text-search extension is now baked into the CLI Docker image so a containerized `serve` does offline keyword search instead of silently degrading to vector-only (#2108) + +### Changed + +- **Tree-sitter prebuild CI matrix greened and made re-run-safe** — dropped the broken `-t 22` flag from the `prebuildify` invocation that crashed every matrix job (`v.indexOf is not a function`; N-API prebuilds are Node-version-agnostic, so no target is needed) (#2121), cleared npm-bundled `prebuilds/` before prebuildify so the host tuple is detected (not a stray `win32-x64`) and source-built the `tree-sitter` runtime peer on `linux-arm64` where upstream ships no prebuild (#2122), and switched the vendor-prebuilds push to `git push --force` so re-running a workflow no longer fails with a stale-lease rejection (#2123) + +### Performance + +- **MCP `query` enrichment batched** — the `query` tool now batches its per-symbol enrichment lookups (3N sequential pool round-trips collapsed to 2–3 `WHERE n.id IN $nodeIds` queries), cutting N+1 round-trips with byte-identical output (#2108) + +### Chore / Dependencies + +- **`@ladybugdb/core` bumped 0.17.0 → 0.17.1 in /gitnexus** (#2098) +- **Claude plugin manifests synced to the release version** — bumped `plugin.json` and the `gitnexus` `marketplace.json` entry to match the published npm version (stale `1.3.x` manifests had blocked marketplace updates), added a Vitest guard asserting all three manifests advertise one version, and documented the sync step in `CONTRIBUTING.md` (#2090) + +## [1.6.6] - 2026-06-08 + +### Added + +- **Scope-resolution (RFC #909) migrations completed across the language matrix** — Rust (#1639), JavaScript (#1640), Ruby (#1831), Swift (#937, #1948), Vue SFC (#940, #1950), Dart (#939, #1970), COBOL (#941, #1835, #1842), and Kotlin (#1727, #1746, #1782) now run on the registry-primary path; Java reached 100% scope-resolution parity and joined `MIGRATED_LANGUAGES` (#1805); per-language progress reporting added to the scope-resolution phase (#1813) +- **HTTP route & consumer contract extraction (group mode)** — Spring interface routes attributed to controllers (#1743); named/positional Java Spring route args (#1834); Kotlin Spring HTTP route, consumer, and WebClient long-form extraction (#1849, #1855, #1884); Java HTTP consumer contracts (#1872); OpenFeign `@RequestLine` consumer contracts incl. plain interfaces without `@FeignClient` (#1904, #1917); FastAPI `include_router(prefix=...)` cross-file routes (#1877); indirect call patterns via FastAPI `Depends()` and frontend HTTP consumers (#1852); gRPC consumer FQN derivation from Java imports for client-jar consumers (#1889) +- **C++ overload & template resolution** — operator-call resolution (#1754), template partial ordering (#1885), user-defined conversion ranking (#1829), nullptr/ellipsis pointer conversion ranks (#1708), SFINAE filter (#1623), expanded `type_traits` constraint registry (#1648), structured resolver-suppression outcomes (#1785), function-type ADL entities (#1822), and a parameter-type class sidecar (#1642) +- **Go enhancements** — structural interface implementation inference (#1966) and a `builtInNames` set for the Go language provider (#1886) +- **Self-healing worker pool** — automatic worker replacement plus deferred-resolution observability and verbose progress logging (#1741, #1773, #1947) +- **`.gitnexusrc` config file and `gitnexus analyze --default-branch`** (#243, #1996) +- **CLI / MCP impact ergonomics** — `--uid/--file/--kind` disambiguation flags (#1907, #1914), `limit/offset/summaryOnly` pagination on the impact tool (#1818), and a per-symbol `processes` field on `byDepth` items (#1867) +- **`gitnexus analyze --repair-fts`** — enforces FTS verification with hardened repair safeguards (#1720) +- **Web viewer** — Tree View and Circles View (#1799), GitLab repository URLs (#1565), `GITNEXUS_BACKEND_URL` env var for Docker deployments (#1286), and web + CLI internationalization (#1748) +- **Wiki** — local Claude/Codex providers (#1769), an opencode local provider (#2039), and `gitnexus wiki --lang ` for multilanguage wiki generation (#1613) +- **`detect-changes` git-worktree support** (#1654) +- **DeepSeek V4 API support** (#1594) +- **Devcontainer for the Claude / Codex / Cursor CLIs** (#1875) and antigravity integration setup + hook adapter (#1730) +- **Object-literal methods linked to exported bindings** (#1718) +- **`eval-server --host`** for a user-configured bind IP (#1667) +- **PR reviewer swarm agents** (#1851) +- **tree-sitter node-type/field validation gate** — validates against the grammar and removes dead literal handling (#1937) + +### Fixed + +- **Parsing-layer coverage gaps closed across the language matrix** (umbrella #1919) — remaining open gaps (#2072) plus Java F35/F38/F41 (#1928, #2045), PHP F53/F54/F55 (#1931, #1989), COBOL F17–F23 (#1925, #1959), Rust F66/F68/F71/F72 (#1934, #1974), Python F57/F58/F61 (#1932, #1964), JS/TS F44/F83/F85/F86/F87 (#1929, #1968), and Ruby F62 (#1933, #1972) +- **Fully-qualified nested-type identity for C++ and Ruby** — distinct nodes for union-, anonymous-namespace-, and same-tail-nested types (#1978, #1981, #2004, #2005); cross-namespace same-tail inheritance bases resolved (#1993, #2005); Ruby same-tail nested mixin modules qualified with `IMPLEMENTS` routed by scope (#1991, #2006); shared codec for `__heritage__`/`__property__` markers (#1994, #2007); graph nodes materialized for scoped class/module/impl declarations (#1975, #1977); generic Rust inherent-impl methods owned through the mod-qualified `Impl` node (#1992, #2003) +- **C# resolution & memory** — global-namespace `typeBindings` O(files²) OOM eliminated (#1871, #1954) and namespace-siblings OOM with worker-path re-parse removed (#1905); qualified/alias constructor names, `:base`/`:this` initializers, and generic type-arg stripping (#2046); primary-base receiver type normalization (#2036); spurious `IMPORTS` edges from ungated `using` resolution stopped (#1881, #1908) +- **C++ dependent-base and member lookup** — resolution across nested/inline namespaces (#1634, #1814), base-specifier qualifier threading (#1815, #1819), call-site types threaded into qualified member lookup (#1632, #1810), variadic pack dependent lookup (#1909), uninitialized multi-declarators (#1965), and typedef-enum / anonymous-struct declarations (#1941) +- **Kotlin type resolution** — smart-cast refinement for `when/is` and `if/is` (#1758, #1774), overload target-id by parameter types (#1761, #1777), cross-file iterable return propagation (#1759, #1775), method-chain fixpoint receiver types (#1760, #1776), virtual dispatch via constructor type override (#1762, #1778), interface default-method dispatch via implements-split MRO (#1763, #1779), and default-parameter arity detection (#2034) +- **Go declarations** — multi-name declaration capture (#2032), fixed-array parameter binding normalization (#1988), and generic composite-literal constructor inference F33 (#1976) +- **Rust / PHP / Vue / Java parsing** — Rust `struct_expression` name pattern split (#2051); PHP import decomposition, namespace-less `.phtml` module scopes, and Blade-template exclusion (#1801, #1790, #1989); Vue JSDoc, dual-script merge, and lang plumbing F89/F90/F92 (#1936, #2050); Java inherited `RequestMapping` prefix deduplication (#2057) and same-module type resolution for duplicate FQNs (#1712) +- **TypeScript** — HOC pattern false positives fixed with `export default` HOC support (#1943) and suffix-index reuse in the scope resolver (#1840) +- **Inheritance on the worker path** — all languages' inheritance migrated to scope-resolution in worker mode (#1951, #1956); centralized heritage supertype matching (#1921, #1922, #1940); `File->Member` `DEFINES` edges skipped for class members (#1949); phantom `Function` defs for array-method callbacks no longer emitted (#1906) +- **MCP** — sibling-clone repo-ID collisions prevented and generated MCP tool names corrected (#2067); orphan processes avoided by handling stdin close/end and the startup race (#2049); duplicate-name repo resolution disambiguated for worktrees (#1753); Windows setup fallback when global `gitnexus` resolves to a non-spawnable shim (#1694) +- **Worker pool** — resilient zero-copy ingestion worker pool prevents analyze hangs on TS-root-scale loads (#1693); cache-hit native workers no longer abort (#1751, #1833); worker-pool docs drift corrected and worker-side stack surfaced on crash (#2068, #2070) +- **LadybugDB** — FTS loaded in the Windows read pool (#2040) and probed-then-loaded on Windows (#1690, #1692); non-ASCII KuzuDB paths resolved on Windows (#1811, #1817); WAL corruption detected in schema init with recovery surfaced (#1647, #1650); WAL checkpoint-threshold control (#1772); init lock skipped for read-only opens (#1783, #1784); `serve` kept stable when sidecars are missing (#1747) +- **Server / API** — `gitnexus serve` startup restored under Express 5 (#1749); `/api/graph`, `/api/search`, `/api/grep` opened read-only (#1686); native read-only enforcement and prepared statements for Cypher query paths (#1655); `eval-server` localhost binding left to the OS (#1722) +- **Embeddings** — local ONNX runtime guarded on macOS Intel before the transformers.js import (#1987) +- **Web agent** — Nexus AI agent system prompt aligned with registered tools (#1984) and the agent stopped cleanly on user Stop (#1820) +- **Group / contracts** — HTTP graph and source contracts unioned (#1709); `httpx` `AsyncClient` alias imports detected (#1687); Node gRPC `loadPackageDefinition` gate no longer matches every member call (#1916); manifest/workspace extraction moved before `closeLbug` (#1802, #1807) +- **Hooks / install** — `gitnexus` resolved on `PATH` via a pure-Node, all-OS scan (#1938, #1980); offline-first extension installs (#1161); actionable error and docs for the `pnpm dlx`/`pnpx` native-load crash (#307, #1967); `onnxruntime-common` declared as a runtime dependency (#2074); vendored grammars materialized to fix Windows EPERM (#1728, #1729) +- **CLI** — missing LadybugDB native binary detected at startup with actionable guidance (#835, #1837); `--no-stats` applied to the keep-marker stats line (#1706, #1765); skipped large-file paths surfaced by default (#1659, #1661); build.js skipped when running outside the monorepo (#1795, #1816); auto-heap raised to 16 GB with tightened cross-platform OOM guidance for UE5-scale repos (#1652) +- **Wiki** — hidden 60s default timeout removed with timeout/retry flag validation and surfaced timeout errors (#1651); budget-aware grouping to prevent context overflow on large repos (#627, #1832) +- **`detect-changes`** — `resolveWorktreeCwd` guarded against overriding a separately-indexed worktree (#1691) +- **Windows reliability** — `windowsHide:true` passed to every `child_process` spawn-family call (#1794) + +### Changed + +- **Legacy resolution deletion (Ring 4)** — removed the legacy call-resolution DAG + heritage processor (RING4-1, #942, #2023), the legacy resolution-context + tiered-lookup plumbing (RING4-2, #943, #2033), and the shadow-mode parity harness (RING4-3, #944, #2071) +- **CONTRIBUTING** — clarified local development setup (#2024) +- **Tests / CI** — cli-e2e made read-only and eval-server tests hardened under load (#2000, #1786, #1838, #1688); parity shards consolidated and the cross-platform matrix narrowed (#1798); devcontainer smoke build hardened against Docker Hub flakes (#1969); gitleaks stabilized (#2027) + +### Performance + +- **Linux-kernel-scale analysis overhaul** — worker-pool parse, finalize O(n²), and the scope-resolution memory wall (#1983, #2038) +- **Scope-capture linearized across all languages (O(n²)→O(n))** plus Python import-resolution linearization (#1918), the Go-specific re-walk fix (#1848, #1915), and owner-keyed lookup for Step 2 member resolution (#1657) +- **C++ ADL candidates indexed once instead of per-site rescans** (#1990) +- **Inert local value symbols pruned** during ingestion (#2065) + +### Chore / Dependencies + +- `@ladybugdb/core` bump in /gitnexus (#2056) +- Routine dependency bumps across /gitnexus, /gitnexus-web, /eval, and GitHub Actions — incl. `hono`, `vitest`, `@vitest/coverage-v8`, `tsx`, `lru-cache`, `express`/`@types/express`, `express-rate-limit`, `qs`, `node-addon-api`, `brace-expansion`, `langchain`, `i18next`, `dompurify`, `lucide-react`, `axios`, `zod`, `@langchain/langgraph`, `@vercel/node`, `langsmith`, `aiohttp`, `idna`, and the `docker/*` / `github/codeql-action` / `release-drafter` / `dependency-review-action` actions (#2056, #2044, #2043, #2042, #2016, #2015, #2013, #2012, #2011, #2010, #2009, #2008, #2018, #2019, #2017, #2020, #1986, #1911, #1864, #1863, #1861, #1860, #1866, #1844, #1845, #1826, #1825, #1824, #1791, #1789, #1768, #1767, #1739, #1740, #1738, #1736, #1735, #1734, #1731, #1713, #1698, #1697, #1696, #1689, #1604, #1552, #1464, #872) +- **Security** — `@vercel/node` upgraded in /gitnexus-web with transitive advisories remediated (#1705) + ## [1.6.5] - 2026-05-16 ### Added diff --git a/gitnexus/README.md b/gitnexus/README.md index 09c3ba464..c331e7ae2 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -126,7 +126,7 @@ Your AI agent gets these tools automatically: | Tool | What It Does | `repo` Param | | ---------------- | ---------------------------------------------------------------- | ------------ | -| `list_repos` | Discover all indexed repositories | — | +| `list_repos` | Discover all indexed repositories (paginated — `limit`/`offset`) | — | | `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | | `context` | 360-degree symbol view — categorized refs, process participation | Optional | | `impact` | Blast radius analysis with depth grouping and confidence | Optional | @@ -159,6 +159,7 @@ Your AI agent gets these tools automatically: ```bash gitnexus setup # Configure MCP for your editors (one-time) +gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks (add --force to apply) gitnexus analyze [path] # Index a repository (or update stale index) gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild @@ -196,6 +197,8 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +> **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. + ## Remote Embeddings Set these env vars to use a remote OpenAI-compatible `/v1/embeddings` endpoint instead of the local model: @@ -401,7 +404,7 @@ Values above **32768 KB (32 MB)** are clamped to the tree-sitter parser ceiling; ### Analyze reports a worker timeout -Worker parse timeouts are recoverable. GitNexus retries stalled worker jobs with backoff, splits large jobs to isolate slow files, and falls back to the sequential parser when needed. If a large repository needs more time per worker job, use either: +Worker parse timeouts are recoverable. GitNexus retries stalled worker jobs with backoff, splits large jobs to isolate slow files, and quarantines a file that repeatedly crashes its worker (respawning the slot so the pool keeps going). If a large repository needs more time per worker job, use either: ```bash # CLI flag, in seconds @@ -424,6 +427,36 @@ Three env vars expose the pool's resilience layers (respawn budget, cumulative-t | `GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS` | `5 × subBatchTimeoutMs` | Total retry wall-time budget per job before quarantining. Bounds exponentially-growing retry waits. | | `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD` | `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, dispatches require a fresh pool. | +### Graph cleanup tuning + +After scope resolution, analyze prunes inert block-local value symbols (a function-local `const`/`let`/`var` that ends up with only its structural `File→DEFINES` edge) to keep the graph focused on cross-symbol relationships. Module/file-scope symbols, class members, and any local with a real edge are always kept. + +| Variable | Default | Effect | +| ------------------------------------ | ------- | ------------------------------------------------------------------------------------------------------- | +| `GITNEXUS_KEEP_LOCAL_VALUE_SYMBOLS` | unset | Set to `1`/`true` to keep inert block-local value symbols instead of pruning them. | + +Programmatic callers can pass `keepLocalValueSymbols: true` in `PipelineOptions` instead of setting the env var. + +### Hook augmentation/notifications are silently skipped + +The Claude Code / Antigravity hooks intentionally stay **silent** on normal skip +paths so strict hook runners (e.g. Codex `PreToolUse`) never see unexpected +output. A search may not be augmented — or a stale-index reminder may not appear +on stderr — when the GitNexus MCP server owns the repo DB, when the DB-lock probe +times out and fails closed, or when the index is already current. + +To see why a hook skipped, set `GITNEXUS_DEBUG=1` and re-run the action — the hook +writes the reason (e.g. `[GitNexus] augment skipped: MCP server owns DB`) and the +stale-index hint to its stderr: + +```bash +GITNEXUS_DEBUG=1 # surfaces hook skip/diagnostic reasons on stderr +``` + +Only `GITNEXUS_DEBUG=1` and `GITNEXUS_DEBUG=true` enable diagnostics; every other +value (including `0` and `false`) is treated as off. Diagnostics go to stderr +only — the hook's structured stdout (the JSON the agent consumes) is unaffected. + ## Privacy - All processing happens locally on your machine diff --git a/gitnexus/bench/cfg/baselines.json b/gitnexus/bench/cfg/baselines.json new file mode 100644 index 000000000..f830cc680 --- /dev/null +++ b/gitnexus/bench/cfg/baselines.json @@ -0,0 +1,23 @@ +{ + "straight-line": { + "fingerprint": "f5524690b5b7d484573710938c5e9a28e08ef0882fea95111f01575c71f4a66a", + "scaling_budget": 1.5, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "_note": "#2081 M1: ONE function, N coalescing statements (extendBlock text accumulation). Runs at 2000->8000 (larger than the other scenarios — output is constant 4 blocks, so disk/heap can't see this path; the TIME ratio is the sole guard). Verified at this N: the array-join impl is ~1.0, a V8-rope-optimized `+=` is also ~1.0 (correctly NOT a real regression — ropes keep naive concat linear), but a genuine O(n²) accumulation (e.g. re-join-the-array-every-append) is ~3.8 — so budget 1.5 catches a true superlinear regression while passing linear concat. disk ~1.03, retained heap ~0.98. Re-baseline the fingerprint only on an intentional CFG-shape change." + }, + "many-functions": { + "fingerprint": "c167ccd83086254e2b71eca153ca4a833be14b2d2a3827ab76b49f643aad13d5", + "scaling_budget": 1.5, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "_note": "#2081 M1: N small branchy functions (collect walk + per-function build). Time ~1.0, disk ~1.01, retained heap ~1.0 (~1KB/function; ~2MB at 2000 fns)." + }, + "branchy": { + "fingerprint": "944ab56ffc70e195f74d8533a8aadf4930d37d13bcfa47cc4feff29e74ddca5c", + "scaling_budget": 1.8, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "_note": "#2081 M1: ONE function, N sequential ifs (block/edge growth in one CFG). Time ~1.1-1.25 (REPS=15 median; noisiest scenario), disk ~1.04, retained heap ~1.0. Time budget 1.8 absorbs noise while catching ~4.0 quadratic." + } +} diff --git a/gitnexus/bench/cfg/measure.mjs b/gitnexus/bench/cfg/measure.mjs new file mode 100644 index 000000000..115c878a8 --- /dev/null +++ b/gitnexus/bench/cfg/measure.mjs @@ -0,0 +1,288 @@ +/** + * Build-free CFG-construction measurement harness (#2081 M1). + * + * Times `collectFunctionCfgs` (the per-function CFG builder the parse worker + * runs on a `--pdg` run) on synthetic TS sources at two sizes, in three + * scenarios that each stress a distinct cost dimension: + * - `straight-line`: ONE function with N coalescing statements — stresses the + * basic-block text accumulation (the `extendBlock` path); + * - `many-functions`: N small branchy functions — stresses the collect walk + + * per-function build + the tree-sitter `namedChildren` accesses; + * - `branchy`: ONE function with N sequential `if`s — stresses block/edge + * growth within a single CFG. + * + * For each scenario it reports three scaling ratios at small→large + * (`(metric_large/metric_small)/(N_large/N_small)`: ~1.0 is linear, ~4.0 is the + * O(n²) shape the M1 perf review flagged for `extendBlock`'s concat chain): + * - TIME — wall-clock of `collectFunctionCfgs` (median of reps); + * - DISK — utf8 byte size of the serialized `cfgSideChannel` (what a `--pdg` + * run writes onto every ParsedFile shard); + * - MEMORY — retained JS heap of the `cfgSideChannel` payload, by the + * release-delta method (heap held minus heap after dropping it). Requires + * `node --expose-gc`; without it the heap metric is null and its gate skips. + * It also computes an order-independent sha256 fingerprint over the emitted + * blocks/edges of a fixed-size source — the correctness gate that a structural + * speedup must leave behavior-identical. + * + * Build-free: imports the `.ts` hotpaths through tsx + * (`node --expose-gc --import tsx bench/cfg/measure.mjs`). Parsing happens ONCE + * per size and the tree is reused across reps so the time measurement isolates + * CFG build cost, not tree-sitter parse time. `maxFunctionLines` is 0 (no cap) + * here on purpose — the bench measures the algorithm; the production default cap + * is a separate safety net (and would otherwise skip the large straight-line fn). + * + * Without args: prints one JSON object per scenario. + * With `--check`: asserts each scenario's fingerprint == its committed baseline + * (baselines.json) AND each of the time / disk / heap ratios is below its + * recorded budget; exits non-zero on any drift/regression. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; + +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import { collectFunctionCfgs } from '../../src/core/ingestion/cfg/collect.ts'; +import { createTypeScriptCfgVisitor } from '../../src/core/ingestion/cfg/visitors/typescript.ts'; +import { getTreeSitterBufferSize } from '../../src/core/ingestion/constants.ts'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const visitor = createTypeScriptCfgVisitor(); +const parser = new Parser(); +parser.setLanguage(TypeScript.typescript); +// Large synthetic sources exceed tree-sitter's default read buffer; size it +// from the content exactly as the parse worker does (getTreeSitterBufferSize). +const parse = (src) => parser.parse(src, undefined, { bufferSize: getTreeSitterBufferSize(src) }); + +// ---- synthetic generators (one cost dimension each) ---- + +const SCENARIOS = [ + { + name: 'straight-line', + // One function, N coalescing simple statements → all fold into one basic + // block whose text is accumulated statement-by-statement (extendBlock). + // Uses LARGER sizes than the other scenarios: this scenario's only cost + // dimension is text accumulation (output size is constant — 4 blocks at any + // N — so the disk/heap ratios can't see it), so the TIME ratio is the sole + // guard against an extendBlock O(n²)-concat re-regression. At small N a + // quadratic is masked by V8 cons-strings + the linear tree-walk and slips + // under the budget; these larger sizes make a real quadratic separate + // cleanly (verified: a `+=` regression here exceeds the budget, the + // array-join impl stays ~1). + small: 2000, + large: 8000, + gen: (n) => { + let s = 'function f() {\n'; + for (let i = 0; i < n; i++) s += ` let v${i} = ${i} + 1;\n`; + return s + ' return v0;\n}\n'; + }, + }, + { + name: 'many-functions', + // N independent small functions with a branch + return → stresses the + // tree walk in collectFunctionCfgs and the per-function build. + gen: (n) => { + let s = ''; + for (let i = 0; i < n; i++) { + s += `function f${i}(x: number) { if (x > ${i}) { a(); } else { b(); } return x + ${i}; }\n`; + } + return s; + }, + }, + { + name: 'branchy', + // One function, N sequential `if`s → N condition blocks + 2N+ edges in a + // single CFG; stresses block/edge growth and namedChildren on the body. + gen: (n) => { + let s = 'function f(x: number) {\n'; + for (let i = 0; i < n; i++) s += ` if (x > ${i}) { s${i}(); }\n`; + return s + '}\n'; + }, + }, +]; + +const SMALL = 500; +const LARGE = 2000; // 4× — O(n) ⇒ ratio ~1, O(n²) ⇒ ratio ~4 +const REPS = 15; // median over more reps → stabler time signal at small absolute ms +const FP_SIZE = 15; // fixed size for the behavior fingerprint +const NO_CAP = 0; // measure the algorithm, not the production safety cap + +// ---- timing ---- + +function median(xs) { + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +function measureCollect(src, file, reps) { + const root = parse(src).rootNode; // parse ONCE; reuse across reps + collectFunctionCfgs(root, visitor, `warmup-${file}`, NO_CAP); // warm JIT (uncounted) + const samples = []; + let out; + for (let i = 0; i < reps; i++) { + const start = process.hrtime.bigint(); + out = collectFunctionCfgs(root, visitor, file, NO_CAP); + samples.push(Number(process.hrtime.bigint() - start) / 1e6); + } + return { + ms: median(samples), + blockCount: out.cfgs.reduce((a, c) => a + c.blocks.length, 0), + // DISK growth: utf8 byte size of the serialized cfgSideChannel — exactly + // what a --pdg run writes onto every ParsedFile shard in the durable store + // + parse cache (the field is plain JSON, so this is the on-disk delta). + // Should scale linearly with source covered; a super-linear ratio means the + // CFG duplicates text and bloats warm-cache shards at scale. + diskBytes: Buffer.byteLength(JSON.stringify(out.cfgs), 'utf8'), + }; +} + +// ---- memory growth: retained heap of the cfgSideChannel payload ---- + +// Needs `node --expose-gc` to force collection for a clean delta; without it the +// heap metric is reported as null and its --check gate is skipped (so a local +// run without the flag still works). +const GC = typeof global.gc === 'function' ? () => (global.gc(), global.gc()) : null; + +function retainedHeapBytes(src, file) { + if (!GC) return null; + // Retained-size-by-RELEASE: measure the heap with the CFGs held, drop them, + // GC, measure again. The drop isolates exactly the JS heap the cfgSideChannel + // payload retains (the extra RAM a --pdg run carries per file until the shard + // is flushed) — robust to pre-existing garbage, which is constant across both + // measurements. The parse tree is a temporary (its native memory isn't on the + // JS heap); block text strings are fresh copies, so they count here. + let cfgs = collectFunctionCfgs(parse(src).rootNode, visitor, file, NO_CAP).cfgs; + GC(); + const withCfgs = process.memoryUsage().heapUsed; + if (cfgs.length < 0) throw new Error('unreachable'); // keep cfgs live past withCfgs + cfgs = null; + GC(); + const withoutCfgs = process.memoryUsage().heapUsed; + return Math.max(0, withCfgs - withoutCfgs); +} + +// ---- correctness fingerprint (order-independent over blocks + edges) ---- + +function canonicalizeCfg(cfg) { + const blocks = cfg.blocks + .map((b) => `B|${b.index}|${b.startLine}-${b.endLine}|${b.kind}|${b.text}`) + .sort(); + const edges = cfg.edges.map((e) => `E|${e.from}->${e.to}|${e.kind}`).sort(); + return `${cfg.functionStartLine}:${cfg.functionStartColumn}\n${blocks.join('\n')}\n${edges.join('\n')}`; +} + +function fingerprint(scenario) { + const out = collectFunctionCfgs(parse(scenario.gen(FP_SIZE)).rootNode, visitor, 'fp.ts', NO_CAP); + const canon = out.cfgs.map(canonicalizeCfg).sort().join('\n====\n'); + return { + fingerprint: crypto.createHash('sha256').update(canon).digest('hex'), + fp_cfgs: out.cfgs.length, + fp_blocks: out.cfgs.reduce((a, c) => a + c.blocks.length, 0), + fp_edges: out.cfgs.reduce((a, c) => a + c.edges.length, 0), + }; +} + +function measureScenario(scenario) { + // Per-scenario sizes (straight-line needs larger N to separate a concat + // quadratic from noise — see its comment); the rest default to the globals. + const nSmall = scenario.small ?? SMALL; + const nLarge = scenario.large ?? LARGE; + const small = measureCollect(scenario.gen(nSmall), `${scenario.name}.ts`, REPS); + const large = measureCollect(scenario.gen(nLarge), `${scenario.name}.ts`, REPS); + const sizeRatio = nLarge / nSmall; + const scalingRatio = small.ms > 0 ? large.ms / small.ms / sizeRatio : 0; + const diskRatio = small.diskBytes > 0 ? large.diskBytes / small.diskBytes / sizeRatio : 0; + + // Memory growth (only when --expose-gc gave us a forced GC). + const heapSmall = retainedHeapBytes(scenario.gen(nSmall), `${scenario.name}.ts`); + const heapLarge = retainedHeapBytes(scenario.gen(nLarge), `${scenario.name}.ts`); + const heapRatio = + heapSmall !== null && heapLarge !== null && heapSmall > 0 + ? heapLarge / heapSmall / sizeRatio + : null; + + return { + scenario: scenario.name, + elapsed_ms_small: Number(small.ms.toFixed(3)), + elapsed_ms_large: Number(large.ms.toFixed(3)), + scaling_ratio: Number(scalingRatio.toFixed(3)), + disk_bytes_small: small.diskBytes, + disk_bytes_large: large.diskBytes, + disk_bytes_ratio: Number(diskRatio.toFixed(3)), + heap_bytes_small: heapSmall, + heap_bytes_large: heapLarge, + heap_ratio: heapRatio === null ? null : Number(heapRatio.toFixed(3)), + blocks_small: small.blockCount, + blocks_large: large.blockCount, + ...fingerprint(scenario), + }; +} + +// ---- run ---- + +const CHECK = process.argv.includes('--check'); + +// The retained-heap budget is a primary regression detector, but it can only be +// measured with a forced GC. Rather than let `--check` silently PASS with the +// heap gate skipped (a green no-op if someone drops --expose-gc), fail loudly. +if (CHECK && !GC) { + process.stderr.write( + '[cfg --check] FAIL: retained-heap gate requires --expose-gc. ' + + 'Run: node --expose-gc --import tsx bench/cfg/measure.mjs --check\n', + ); + process.exit(1); +} + +const results = SCENARIOS.map(measureScenario); + +if (!CHECK) { + for (const r of results) process.stdout.write(JSON.stringify(r) + '\n'); +} else { + const baselines = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8')); + const failures = []; + for (const r of results) { + const base = baselines[r.scenario]; + if (base === undefined) { + failures.push(`${r.scenario}: no baseline recorded`); + continue; + } + if (r.fingerprint !== base.fingerprint) { + failures.push( + `${r.scenario}: CFG fingerprint drift (got ${r.fingerprint}, expected ${base.fingerprint})`, + ); + } + if (r.scaling_ratio >= base.scaling_budget) { + failures.push( + `${r.scenario}: scaling ratio ${r.scaling_ratio} >= budget ${base.scaling_budget} ` + + `(${SMALL}->${LARGE} stmts/fns, ms ${r.elapsed_ms_small}->${r.elapsed_ms_large})`, + ); + } + if (base.disk_bytes_budget !== undefined && r.disk_bytes_ratio >= base.disk_bytes_budget) { + failures.push( + `${r.scenario}: cfgSideChannel disk-bytes ratio ${r.disk_bytes_ratio} >= budget ` + + `${base.disk_bytes_budget} (bytes ${r.disk_bytes_small}->${r.disk_bytes_large})`, + ); + } + // Heap gate only when measured (--expose-gc present) AND a budget exists. + if ( + base.heap_budget !== undefined && + r.heap_ratio !== null && + r.heap_ratio >= base.heap_budget + ) { + failures.push( + `${r.scenario}: retained-heap ratio ${r.heap_ratio} >= budget ${base.heap_budget} ` + + `(heap ${r.heap_bytes_small}->${r.heap_bytes_large})`, + ); + } + process.stdout.write(JSON.stringify(r) + '\n'); + } + if (failures.length > 0) { + for (const f of failures) process.stderr.write(`[cfg --check] FAIL: ${f}\n`); + process.exit(1); + } + process.stderr.write(`[cfg --check] PASS (${results.length} scenarios)\n`); +} diff --git a/gitnexus/bench/parse-throughput.md b/gitnexus/bench/parse-throughput.md index 24a7c98b1..4d53cd2dc 100644 --- a/gitnexus/bench/parse-throughput.md +++ b/gitnexus/bench/parse-throughput.md @@ -70,10 +70,17 @@ this doc, run it under instrumentation: ```bash # From the gitnexus/ subdir: cd gitnexus -# Single-threaded baseline (sequential fallback): -npx vitest run test/integration/parse-impl-large-fixture.test.ts --reporter=verbose +# The worker pool is the sole parse path, so every run needs the dist worker +# (`npm run build`) and a pool size pinned via GITNEXUS_WORKER_POOL_SIZE. -# Worker-pool path (requires built dist/ — pre-built by `npm run build`): +# Single-worker-pool baseline (closest analog to the old single-threaded run — +# sequential parsing was removed, so a 1-worker pool is the floor): +npm run build && \ + GITNEXUS_WORKER_POOL_SIZE=1 \ + GITNEXUS_VERBOSE=1 \ + npx vitest run test/integration/parse-impl-large-fixture.test.ts --reporter=verbose + +# Multi-worker path: npm run build && \ GITNEXUS_WORKER_POOL_SIZE=4 \ GITNEXUS_PARSE_CHUNK_CONCURRENCY=2 \ @@ -97,22 +104,26 @@ node --inspect=0 \ ## Latest measurement > _No measurement data has been collected yet — this file is the -> methodology + harness scaffold. The single recorded data point is the -> U6 wall-clock smoke baseline below; the worker-pool rows are -> placeholders for future bench-pass output._ +> methodology + harness scaffold. The U6 smoke test confirms the +> worker-pool path stays well within its wall-clock budget, but every +> throughput/heap cell below is a `_TBD_` placeholder for a future +> bench-pass._ The U6 integration test (`gitnexus/test/integration/parse-impl-large-fixture.test.ts`) -was observed completing the synthetic fixture in **~6 seconds** under -the sequential path (`skipWorkers: true`) on the development machine, -well under the 30 s `Promise.race` wall-clock budget. That number is a -smoke baseline only — recorded here for reference, not as a regression -target. +runs the worker pool — the sole parse path now that sequential parsing +has been removed (disabling the pool on a repo with parseable files +raises a hard `WorkerPoolDisabledError`). It completes the synthetic +fixture well within the 30 s `Promise.race` wall-clock budget on the +development machine, but no worker-pool throughput/heap numbers have been +captured yet, so the rows below are all `_TBD_`. (An earlier ~6 s figure +recorded here was measured on the now-removed sequential path; it has +been dropped rather than relabelled as a worker-pool baseline, since the +two paths are not comparable.) -| Path | files/s | wall-clock | peak heap | chunks | quarantined | -| ------------------------------------------ | ------- | -------------------- | --------- | ------ | ----------- | -| Sequential (`skipWorkers: true`, U6 smoke) | _TBD_ | ~6 s _(observation)_ | _TBD_ | 17 | 0 | -| Worker pool, `--workers 4`, concurrency 2 | _TBD_ | _TBD_ | _TBD_ | _TBD_ | 0 | -| Worker pool, `--workers 1`, concurrency 1 | _TBD_ | _TBD_ | _TBD_ | _TBD_ | 0 | +| Path | files/s | wall-clock | peak heap | chunks | quarantined | +| ------------------------------------------------------------------------- | ------- | ---------- | --------- | ------ | ----------- | +| Worker pool, `--workers 1` (`GITNEXUS_WORKER_POOL_SIZE=1`), concurrency 1 | _TBD_ | _TBD_ | _TBD_ | _TBD_ | 0 | +| Worker pool, `--workers 4`, concurrency 2 | _TBD_ | _TBD_ | _TBD_ | _TBD_ | 0 | **Hardware:** _TBD — record OS, CPU, RAM, Node version, gitnexus SHA at the time of the bench-pass that populates the table above._ diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index 85b2efd58..f6f68252a 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -11,27 +11,30 @@ "_note": "Updated for F17-F23 fixes (P2: TIMES guard, ADD GIVING, SQL AS alias). See PR #1959." }, "c": { - "fingerprint": "0de009bdbfe095f530fa87eb32bce6ab83092c904f26b3c8fe8d8ab587cf6dc9", + "fingerprint": "12a196b2d6249c8d86a931b12ecebc2a0cdf8d6f47683acdd0d8e9d8bc7657f5", "scaling_budget": 1.5, - "_added": "#1956: c added to the scope-capture bench (was UNBENCHED). C has no inheritance \u2014 flat scale source. Adding it exposed + fixed a pre-existing O(n^2) findNodeAtRange root-walk in c/captures.ts (threaded c.node, byte-identical over c-* fixtures); scaling 3.475 -> 0.96." + "_added": "#1956: c added to the scope-capture bench (was UNBENCHED). C has no inheritance — flat scale source. Adding it exposed + fixed a pre-existing O(n^2) findNodeAtRange root-walk in c/captures.ts (threaded c.node, byte-identical over c-* fixtures); scaling 3.475 -> 0.96.", + "_note": "#1983: + c-static-linkage-worker fixture (caller.c/lib.c/lib.h/local.c — worker-path static-linkage side-channel test). Pure fixture-corpus drift: no c/captures.ts or query change branch-vs-main, existing fixtures' captures byte-identical (c-captures.test.ts 45/45), scaling stays linear (~0.97). The baseline was missed when the fixture landed; regenerated here. fingerprint 0de009b->39f3a83.", + "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression)." }, "cpp": { - "fingerprint": "6d6207ae1df3943c5fae28983e0c294e55225456e7cf39af1d46fda21b6787c4", + "fingerprint": "9b5b4393d158d76dcf1ef9807e0326462c45a5266310f0ae7894d017f3858219", "scaling_budget": 1.5, "_added": "#1956: cpp added to the scope-capture bench (was UNBENCHED). Heritage-bearing scale source (: public Base, public Mixin) drives emitCppInheritanceCaptures at scale. Adding it exposed + fixed a pre-existing O(n^2) findNodeAtRange root-walk in cpp/captures.ts (~12 sites, threaded c.node, byte-identical over 263 cpp-* fixtures); scaling 2.30 -> 1.12.", - "_rebaselined": "#1965 / #1923 F4: uninitialized non-leading multi-declarators now emit @declaration.variable captures; cpp-adl-inner-callable-outer-noncallable data::Pair a, b adds the legitimate fixture drift. Linear (~1.06).", - "_note": "#1975: + cpp-out-of-line-class fixture, fixture_count 263->265. #1990: + cpp-adl-ns-plus-hidden-friend-same-name fixture (ADL hidden-friend + namespace-callable merge parity test). Pure fixture-corpus drift \u2014 no scope-extractor change; existing fixtures' captures byte-identical. fixture_count 265->267. #1995: + cpp-union-nested-tail-collision and cpp-anon-ns-tail-collision fixtures \u2014 pure fixture-corpus drift; fixture_count 270->272, fingerprint 538e8be->d63ded6. #1993: + cpp-cross-namespace-same-tail fixture \u2014 pure fixture-corpus drift; fixture_count 272->273, fingerprint d63ded6->6d6207ae." + "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression). #2094: deleted C++ declarations retain @declaration.is-deleted metadata; deleted operator and pointer-return shapes plus the expanded deleted-overload fixture are included. Intended capture drift; scaling remains linear (1.139 < 1.5).", + "_note": "#1975: + cpp-out-of-line-class fixture, fixture_count 263->265. #1990: + cpp-adl-ns-plus-hidden-friend-same-name fixture (ADL hidden-friend + namespace-callable merge parity test). Pure fixture-corpus drift — no scope-extractor change; existing fixtures' captures byte-identical. fixture_count 265->267. #1995: + cpp-union-nested-tail-collision and cpp-anon-ns-tail-collision fixtures — pure fixture-corpus drift; fixture_count 270->272, fingerprint 538e8be->d63ded6. #1993: + cpp-cross-namespace-same-tail fixture — pure fixture-corpus drift; fixture_count 272->273, fingerprint d63ded6->6d6207ae. #2077 review follow-up: cpp-member-lattice adds cross-file, qualified-base, nested-template, inherited-using, this-receiver, and non-virtual-override regressions; fixture_count 274->275. Capture scaling remains linear (1.134 < 1.5)." }, "csharp": { "_rebaselined": "#1956 synth-widening: + csharp-qualified-base fixture; the synth now walks record_declaration + struct_declaration base_lists and handles alias_qualified_name (matching the #1940 legacy leg), so record/struct heritage now emits. csharp-record-base gains a record inherits capture. (record->record SAME-namespace EXTENDS is a separate registry resolution gap, tracked as follow-up.) Linear (~1.00). (Earlier #1956: heritage-bearing scale source.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged. | #1924 F16: record primary-constructor base bindings now exclude constructor arguments; capture fingerprint changes, scaling remains linear. | #2036 review follow-up: csharp-record-base now exercises primary-constructor base dispatch end to end; +2 capture groups, scaling remains linear.", - "fingerprint": "701b4274643a9a5ee03a71e9fc0dede28a89524eedb976affc48df72bdd4adcc", - "scaling_budget": 1.5 + "fingerprint": "2bb5bc8c19cb8eb08c9590545ad8a1968a7152951f7e12746e2d7901d542fed9", + "scaling_budget": 1.5, + "_note": "#2046: F35 qualified-constructor captures now emit @reference.qualified-name + a simple-name @reference.name on `new Ns.Foo()`/`new A.B.Foo()`; namespace_declaration/file_scoped_namespace_declaration now emit @declaration.namespace name captures (feeding the non-destructive namespacePrefix sidecar for `new B.Foo()` same-tail disambiguation). + csharp-interface-only-base and csharp-namespace-qualified-ctor fixtures. Pure capture-additive + fixture-corpus drift; scaling stays linear (~1.11)." }, "rust": { "fingerprint": "ac610bbe97666bf285923479dd7b43a2fe4c5354aae8df1bcbafdc04fb220f82", "scaling_budget": 1.5, - "_rebaselined": "#1956 tri-review U1: rust-qualified-trait fixture (scoped + generic-of-scoped impl trait paths); bareTypeIdentifier now resolves scoped_type_identifier bases by their name: tail (additive, no existing-fixture drift); linear (~1.04). #1975: + rust-scoped-impl fixture (impl a::Inner / b::Inner inherent scoped impls) \u2014 legacy @definition.impl scoped arm + findEnclosingClassInfo inherent-impl scoped target; rust scope-extractor captures byte-identical. | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged.", - "_note": "PR #1934: F66/F68 let-binding pattern narrowing; F71 union (Struct-labeled, now materialized via legacy @definition.struct + resolvable); F72 macro FULLY WIRED \u2014 @declaration.macro/@reference.macro + MacroRegistry \u2192 USES edges to Macro nodes (never a same-named fn). + rust-macro / rust-union fixtures and merged with origin/main #1975 rust-scoped-impl; fingerprint re-baselined (scaling ~0.99, fixture_count 126). #1992: + rust-nested-tail-collision-generic and rust-generic-impl-same-method-name (F3) fixtures \u2014 pure fixture-corpus drift, no scope-extractor change; fixture_count 127->129, fingerprint 56ffc1c0->b00aea0f." + "_rebaselined": "#1956 tri-review U1: rust-qualified-trait fixture (scoped + generic-of-scoped impl trait paths); bareTypeIdentifier now resolves scoped_type_identifier bases by their name: tail (additive, no existing-fixture drift); linear (~1.04). #1975: + rust-scoped-impl fixture (impl a::Inner / b::Inner inherent scoped impls) — legacy @definition.impl scoped arm + findEnclosingClassInfo inherent-impl scoped target; rust scope-extractor captures byte-identical. | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged.", + "_note": "PR #1934: F66/F68 let-binding pattern narrowing; F71 union (Struct-labeled, now materialized via legacy @definition.struct + resolvable); F72 macro FULLY WIRED — @declaration.macro/@reference.macro + MacroRegistry → USES edges to Macro nodes (never a same-named fn). + rust-macro / rust-union fixtures and merged with origin/main #1975 rust-scoped-impl; fingerprint re-baselined (scaling ~0.99, fixture_count 126). #1992: + rust-nested-tail-collision-generic and rust-generic-impl-same-method-name (F3) fixtures — pure fixture-corpus drift, no scope-extractor change; fixture_count 127->129, fingerprint 56ffc1c0->b00aea0f." }, "php": { "fingerprint": "bc2c27c5ba26d5aea61142a2a99fb772222f5b969205260eb7a71b4c0bd73cdb", @@ -43,29 +46,30 @@ "fingerprint": "b5ea93bb3d0469c3821a8c70f5d5991c6f326e41097c119ad691154301dcc753", "scaling_budget": 1.5, "_rebaselined": "#1956 synth-widening: + ruby-qualified-base fixture; synth now reduces a scope_resolution superclass (class C < Mod::Super) to its trailing constant (matching the #1940 legacy leg), at parity. Linear (~1.03). (Earlier #1956: heritage-bearing scale source.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged.", - "_note": "F62: + scope_resolution class/module declaration captures \u2014 fixture count 78\u219281, fingerprint drift expected. #1975: + ruby-tail-collision fixture (Foo::Bar vs Baz::Bar stay distinct nodes) \u2014 pure fixture-corpus drift, scope-extractor captures unchanged; 81\u219282. #1991: + ruby-nested-mixin-tail-collision fixture (85\u219286). Recomputed on the #942 merge (fixture-comment rewording shifts capture byte-positions, capture LOGIC unchanged): bf6b13a -> b5ea93bb." + "_note": "F62: + scope_resolution class/module declaration captures — fixture count 78→81, fingerprint drift expected. #1975: + ruby-tail-collision fixture (Foo::Bar vs Baz::Bar stay distinct nodes) — pure fixture-corpus drift, scope-extractor captures unchanged; 81→82. #1991: + ruby-nested-mixin-tail-collision fixture (85→86). Recomputed on the #942 merge (fixture-comment rewording shifts capture byte-positions, capture LOGIC unchanged): bf6b13a -> b5ea93bb." }, "swift": { - "fingerprint": "53325c6345161c5a495f997297af5a24fb718fd3e6647040160f8ab2a2c8e4c0", + "fingerprint": "180ac68e780bdf6f9089d53f51cbb9a66aed3e7774631cc3fcbaae5020213998", "scaling_budget": 1.5, - "_rebaselined": "#1956: swift-qualified-base fixture + heritage-bearing scale source (class: Base, Serviceable \u2014 extends + protocol conformance); linear (~1.03)." + "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression)." }, "dart": { - "fingerprint": "a9e882b537765e8fd0ddfcd33b38b253dd86fc5ddffa6e4bf5a85ed8ee615eaa", + "fingerprint": "94bf2c26e1ba96f4211634aa572c0a989b503e717e75dfc5df04f66c417de80f", "scaling_budget": 1.5, "_added": "#939: dart added to the scope-capture bench with the registry-primary migration. Heritage-bearing scale source (Entity extends Base implements Marker) gates the @reference.inherits synth + the postfix-chain reference walk at scale. emitDartScopeCaptures threads tree-sitter captured nodes (no findNodeAtRange root-walk), so it is linear (~1.0).", - "_rebaselined": "#1970 review + tri-review follow-ups: constructor-call retag, cascade calls, built-in suppression, enum scope, #1926 F24/F25, named-ctor dedup (crash fix), container-name binding suppression; heritage file-affinity resolution. Fixtures: member-call-contexts, constructor-body, named-constructor-body, heritage-name-collision, construct-cascade." + "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0." }, "java": { - "fingerprint": "d5cf68e9faf92fffd928c1ee6e584c72cc65918d1f1b5078abb3bfe09ac699bf", + "fingerprint": "9b29cafe32873b4902bda311bd089ffc04efe08f13557b966d29544be514080a", "scaling_budget": 1.5, - "_rebaselined": "#1956 synth-widening: + java-iface-extends fixture; synthesizeJavaInheritanceReferences now ALSO walks interface_declaration extends_interfaces (interface IA extends IB, IC), matching the #1940 legacy leg. (Earlier U2+review: java-qualified-base fixture covers 2- AND 3-segment qualified bases guarding the legacy end-anchor; synth tail-resolves scoped bases.) Linear (~1.03). (Earliest: java added to bench, exposed+fixed the O(n^2) findNodeAtRange root-walk; 3.09 -> ~0.99.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged." + "_rebaselined": "#1956 synth-widening: + java-iface-extends fixture; synthesizeJavaInheritanceReferences now ALSO walks interface_declaration extends_interfaces (interface IA extends IB, IC), matching the #1940 legacy leg. (Earlier U2+review: java-qualified-base fixture covers 2- AND 3-segment qualified bases guarding the legacy end-anchor; synth tail-resolves scoped bases.) Linear (~1.03). (Earliest: java added to bench, exposed+fixed the O(n^2) findNodeAtRange root-walk; 3.09 -> ~0.99.) | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged.", + "_note": "#1928 / #2045: F35 adds qualified + qualified-generic constructor query captures (`new pkg.Foo()`, `new a.b.Foo()`, `new pkg.Box()`); F38 synthesizes `@reference.call.constructor` on `super(...)`/`this(...)` explicit_constructor_invocation nodes; F41 generic-aware stripQualifier in interpret (type-binding normalization). + java-qualified-constructor and java-explicit-constructor fixtures. Pure capture-additive + fixture-corpus drift; scaling stays linear (~1.06)." }, "typescript": { "fingerprint": "3f44a4a6892698df2d145c8ff2812c3b318807648983c88aca28fbd694f172f9", "scaling_budget": 1.5, - "_rebaselined": "#1962: F44 (class scope@), F85 (enum member declarations), F87 (optional_parameter type annotations) add new captures \u2014 fingerprint drift expected.", - "_note": "#1968: F44, F85, F87 \u2014 fingerprint drift expected." + "_rebaselined": "#1962: F44 (class scope@), F85 (enum member declarations), F87 (optional_parameter type annotations) add new captures — fingerprint drift expected.", + "_note": "#1968: F44, F85, F87 — fingerprint drift expected." }, "javascript": { "fingerprint": "d72f03c6c502235d2d4b74d66baa5c7d361f040d7a1b72e84acad61210d05ae8", @@ -74,9 +78,9 @@ "_rebaselined": "#1956 synth-widening: + javascript-qualified-base fixture; synthesizeJsInheritanceReferences now handles a member_expression base (class S extends ns.Base -> Base), matching the #1940 legacy leg + the TS terminalTsTypeNameNode property_identifier case, at parity. Linear (~1.05). | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged." }, "kotlin": { - "fingerprint": "a16400622892183581b8f5f8fa01f07842d19b8cf49ed021df52bd17009d749f", + "fingerprint": "90aa832978d9744e50058e77a04748390a7e34e36b309f6c1d178eb07280b7ea", "scaling_budget": 1.5, "_added": "#1951: bench coverage added (was ungated); scale source heritage-bearing (: Base()); js/kotlin O(n^2) findNodeAtRange-per-match fixed to threaded captured node, now linear.", - "_rebaselined": "#1956 synth-widening: + kotlin-qualified-base fixture; synthesizeKotlinInheritanceReferences now handles the explicit_delegation form (class F : Iface by d -> Iface), matching the #1940 legacy leg, at parity. Linear (~0.87). | #942: scope-resolution-only cleanup reworded fixture comments; capture byte-positions shift, capture LOGIC unchanged. | #1930 F45: default parameters now emit optional-arity metadata; capture fingerprint changes, scaling remains linear." + "_rebaselined": "#1919 review CF3 fix: extended kotlin-local-property-owner (init/accessor destructuring) + new dart-accessor-owner fixture (getter/setter ownership). Fingerprint-only corpus drift; scaling ~1.0." } } diff --git a/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs b/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs index bbfccb92e..0d837fb2c 100755 --- a/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs +++ b/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs @@ -91,10 +91,20 @@ function hasGitNexusServerOwner(gitNexusDir) { return hasGitNexusDbLockedByGitNexusServer(path.join(gitNexusDir, 'lbug'), process.pid); } +/** + * Whether opt-in diagnostics should be written to the hook's stderr. Strict + * hook runners validate hook output, so normal, non-error skip paths must stay + * silent unless the operator explicitly asks for diagnostics via GITNEXUS_DEBUG. + * See issue #1913. + */ +function isDebugEnabled() { + return process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; +} + function extractAugmentContext(stderr) { const output = (stderr || '').trim(); const marker = output.indexOf('[GitNexus]'); - const debug = process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; + const debug = isDebugEnabled(); if (debug && output.length > 0) { // Emit the FULL discarded prefix (everything before the marker, or all of // it when no marker is present) so suppressed diagnostics — LadybugDB lock @@ -258,8 +268,14 @@ function buildAfterToolContext(input) { if (/\bgit\s+(commit|merge|rebase|cherry-pick|pull)(\s|$)/.test(command)) { const hint = buildStaleIndexHint(gitNexusDir, cwd); if (hint) { - process.stderr.write(`${hint}\n`); + // The hint always reaches the agent via additionalContext (parts). Mirror + // it to stderr (for terminal users) only under GITNEXUS_DEBUG, so strict + // hook runners see no unexpected output on this normal path (#1913). The + // claude hook never mirrored this to stderr — this aligns the two adapters. parts.push(hint); + if (isDebugEnabled()) { + process.stderr.write(`${hint}\n`); + } } } } @@ -269,7 +285,11 @@ function buildAfterToolContext(input) { function runAugment(gitNexusDir, cwd, pattern) { if (hasGitNexusServerOwner(gitNexusDir)) { - process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + // Normal skip path: the MCP server owns the DB. Stay silent for strict + // hook runners (issue #1913); surface the reason only under GITNEXUS_DEBUG. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + } return ''; } const release = acquireHookSlot(gitNexusDir); @@ -338,7 +358,7 @@ function main() { const handler = handlers[input.hook_event_name || '']; if (handler) handler(input); } catch (err) { - if (process.env.GITNEXUS_DEBUG) { + if (isDebugEnabled()) { console.error('GitNexus antigravity hook error:', (err.message || '').slice(0, 200)); } } diff --git a/gitnexus/hooks/claude/gitnexus-hook.cjs b/gitnexus/hooks/claude/gitnexus-hook.cjs index 8bfa49381..40d0b08df 100755 --- a/gitnexus/hooks/claude/gitnexus-hook.cjs +++ b/gitnexus/hooks/claude/gitnexus-hook.cjs @@ -110,10 +110,20 @@ function hasGitNexusServerOwner(gitNexusDir) { return hasGitNexusDbLockedByGitNexusServer(path.join(gitNexusDir, 'lbug'), process.pid); } +/** + * Whether opt-in diagnostics should be written to the hook's stderr. Strict + * hook runners (e.g. Codex `PreToolUse`) validate hook output, so normal, + * non-error skip paths must stay silent unless the operator explicitly asks + * for diagnostics via GITNEXUS_DEBUG. See issue #1913. + */ +function isDebugEnabled() { + return process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; +} + function extractAugmentContext(stderr) { const output = (stderr || '').trim(); const marker = output.indexOf('[GitNexus]'); - const debug = process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; + const debug = isDebugEnabled(); if (debug && output.length > 0) { // Emit the FULL discarded prefix (everything before the marker, or all of // it when no marker is present) so suppressed diagnostics — KuzuDB lock @@ -250,7 +260,12 @@ function handlePreToolUse(input) { const pattern = extractPattern(toolName, toolInput); if (!pattern || pattern.length < 3) return; if (hasGitNexusServerOwner(gitNexusDir)) { - process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + // Normal skip path: the MCP server owns the DB, so the CLI augment would + // contend on the lock. Stay silent for strict hook runners (issue #1913); + // surface the reason only when diagnostics are explicitly requested. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + } return; } @@ -361,7 +376,7 @@ function main() { const handler = handlers[input.hook_event_name || '']; if (handler) handler(input); } catch (err) { - if (process.env.GITNEXUS_DEBUG) { + if (isDebugEnabled()) { console.error('GitNexus hook error:', (err.message || '').slice(0, 200)); } } diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 810923e40..9fad8f440 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,19 +1,20 @@ { "name": "gitnexus", - "version": "1.6.5", + "version": "1.6.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.6.5", + "version": "1.6.7", "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { "@huggingface/transformers": "^4.1.0", - "@ladybugdb/core": "^0.16.1", + "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -26,14 +27,15 @@ "ignore": "^7.0.5", "js-yaml": "^4.1.1", "jsonc-parser": "^3.3.1", - "lru-cache": "^11.0.0", "mnemonist": "^0.40.3", + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0", + "onnxruntime-common": "^1.26.0", "onnxruntime-node": "^1.24.0", "pandemonium": "^2.4.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tree-sitter": "0.21.1", - "tree-sitter-c": "0.21.4", "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "0.23.2", "tree-sitter-go": "^0.23.0", @@ -50,6 +52,7 @@ "gitnexus": "dist/cli/index.js" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", @@ -64,11 +67,6 @@ }, "engines": { "node": ">=22.0.0" - }, - "optionalDependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.0", - "tree-sitter-kotlin": "^0.3.8" } }, "../gitnexus-shared": { @@ -1159,27 +1157,28 @@ } }, "node_modules/@ladybugdb/core": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/@ladybugdb/core/-/core-0.16.1.tgz", - "integrity": "sha512-qwuEcR8CVMKb6tNDaHtq7Ux8hT/XbPC0db+vwutX6JxNAejyx7YomHKPSy9XAKURhYK8mezZe3UN8rf+xpHOjQ==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core/-/core-0.17.1.tgz", + "integrity": "sha512-K1bHnQrRy3bxkyrFHlxGqKUyIUS1LsRXKOSt14XGY/msBZHaDat/uBrlHiWpM4/24OtfOq/qwTqcTCXannnEjw==", "hasInstallScript": true, "license": "MIT", "dependencies": { + "apache-arrow": "^21.1.0", "cmake-js": "^8.0.0", "node-addon-api": "^6.0.0" }, "optionalDependencies": { - "@ladybugdb/core-darwin-arm64": "0.16.1", - "@ladybugdb/core-darwin-x64": "0.16.1", - "@ladybugdb/core-linux-arm64": "0.16.1", - "@ladybugdb/core-linux-x64": "0.16.1", - "@ladybugdb/core-win32-x64": "0.16.1" + "@ladybugdb/core-darwin-arm64": "0.17.1", + "@ladybugdb/core-darwin-x64": "0.17.1", + "@ladybugdb/core-linux-arm64": "0.17.1", + "@ladybugdb/core-linux-x64": "0.17.1", + "@ladybugdb/core-win32-x64": "0.17.1" } }, "node_modules/@ladybugdb/core-darwin-arm64": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-arm64/-/core-darwin-arm64-0.16.1.tgz", - "integrity": "sha512-Nl+Cf70rD+HaC9IBHv+oeUwqX9plghXD7PN9tyMzMohRVPvcGEbqWPB6YcdJa8rR7qRqCCbmaNMDen5wg4rY2w==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-arm64/-/core-darwin-arm64-0.17.1.tgz", + "integrity": "sha512-JG/uzmolEh3wXJ/ME1EaTH5LTDQ9Cs+Q3Czul8pW2eWbWQZghQU3jjM++7ST7Bla5BX/WITqwPqPoC+sL+slfA==", "cpu": [ "arm64" ], @@ -1190,9 +1189,9 @@ ] }, "node_modules/@ladybugdb/core-darwin-x64": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-x64/-/core-darwin-x64-0.16.1.tgz", - "integrity": "sha512-4eAjfimAAQRSmDfUUkGrl9OhefxcW1ziA9tl0eljBlGoUseE7dL02+RSqjGohYMcQ+lzuHAq1QWb0XRlMA8YTQ==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-darwin-x64/-/core-darwin-x64-0.17.1.tgz", + "integrity": "sha512-Enjm+/V9/jpKmtzF2PB0muVkgpFUGHEvA7r16eJWxVRA/BeO8VPmngTKy9rf/4Yc6TWexjoHRug04BbTXEmerg==", "cpu": [ "x64" ], @@ -1203,9 +1202,9 @@ ] }, "node_modules/@ladybugdb/core-linux-arm64": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-arm64/-/core-linux-arm64-0.16.1.tgz", - "integrity": "sha512-zkctksev+hsPFrNxHHdq4lYK5OWdLhWfRdQzjzkgDyaHayHU6yCL2fgD6uPGQ8TRQ6/2DxMErb4p3FzGW85Ubw==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-arm64/-/core-linux-arm64-0.17.1.tgz", + "integrity": "sha512-P+xM9o4I3JAQtXpX19ZuLj9EeO2gppa+IdmAqhpI8tuhyA3/a85Eaxby1fXOjsbrnOAEyFJczUdyoDkhCPSyiw==", "cpu": [ "arm64" ], @@ -1216,9 +1215,9 @@ ] }, "node_modules/@ladybugdb/core-linux-x64": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-x64/-/core-linux-x64-0.16.1.tgz", - "integrity": "sha512-5rAb9T5vif8WKhHwhobosu2/aiOwJkWb/ViybvUc5GFKunKl8VI6RmZQVeufT9zUzRktUwrxBrxblCxsnamXJw==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-linux-x64/-/core-linux-x64-0.17.1.tgz", + "integrity": "sha512-N2ujE0CrsToBpVBpou1iWwEkK7CgVxucnUNxteySrnDccZwICXFP5BlcFpKE0qq3Eqmqszh4ptR4GuSi6rKPGw==", "cpu": [ "x64" ], @@ -1229,9 +1228,9 @@ ] }, "node_modules/@ladybugdb/core-win32-x64": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/@ladybugdb/core-win32-x64/-/core-win32-x64-0.16.1.tgz", - "integrity": "sha512-ShOUTrIuZKQ63J95tcRJxKf1cvg8yi2FSYx9kMTSercc1FdQZPV+zxUN0myMq3MTWOl7xDxsVMmdp/t80O29UQ==", + "version": "0.17.1", + "resolved": "https://registry.npmjs.org/@ladybugdb/core-win32-x64/-/core-win32-x64-0.17.1.tgz", + "integrity": "sha512-9i3xNfFAMqFRuQG3F1hOCWYGna6eTg8HJ/XYhWVDGkeFJNUV3IdneEiYttF5B2qAtQYUd4sAikScsImrMRw+6g==", "cpu": [ "x64" ], @@ -1307,9 +1306,9 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.132.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.132.0.tgz", - "integrity": "sha512-FESMOxil5Se014ui/Eq8fT5uHJo6nIRwH0PfJrZJXs6Gek3ZVFOrpUv3YIZT20m+extU98Hg1Ym72U58rlsxUQ==", + "version": "0.133.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", + "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", "dev": true, "license": "MIT", "funding": { @@ -1387,9 +1386,9 @@ "license": "BSD-3-Clause" }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.2.tgz", - "integrity": "sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", + "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", "cpu": [ "arm64" ], @@ -1404,9 +1403,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.2.tgz", - "integrity": "sha512-vdFA9+C/rekyGce7WqHs/xoT0ioZEWaOFyZLIV1mEeNFaFDUQrPIo8Vs2GvJ6eetb3rzDUtUBgzto3ExpXJB3w==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", + "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", "cpu": [ "arm64" ], @@ -1421,9 +1420,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.2.tgz", - "integrity": "sha512-BewSOwTHazv77DTYiAZXSqqKZ4KP/KonFisDMVU7PImxoWfB2aepnPhd2E4SWz3zDzYgDNbs6jBmTdgNnF02GA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", + "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", "cpu": [ "x64" ], @@ -1438,9 +1437,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.2.tgz", - "integrity": "sha512-m41o7M0YWtUdqk61Tb+jnKb2rN++iRdIASlExkUoKfIAH30DOHCB8fVLzSUpbWHHU8esmEioY62PxzexE8MBuA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", + "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", "cpu": [ "x64" ], @@ -1455,9 +1454,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.2.tgz", - "integrity": "sha512-jcojB9H7W/jS29pMKWAK1N+fU99vXodHDTatS3b3y/XSOCiHo0kkA74pL3jJmkoQtYpOCxDvaKs1fo2Ij/1X5w==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", + "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", "cpu": [ "arm" ], @@ -1472,9 +1471,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.2.tgz", - "integrity": "sha512-1jn6qDU5iiOgFgygDzKUuKP0maTi0/f1+sBLgvij/76C77Nm3ts6ufz9Bjg5q5dduxiUIxtq86JIoBvo1xQ4Ig==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", + "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", "cpu": [ "arm64" ], @@ -1489,9 +1488,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.2.tgz", - "integrity": "sha512-QVLO/czFMdoMFSqlX3bcswcJNm/23r+qoa/jgtmFc/qEp6/jXmIkDjF/XIo8dPfGaiwy1xfQn8o77L79GeXFgw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", + "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", "cpu": [ "arm64" ], @@ -1506,9 +1505,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.2.tgz", - "integrity": "sha512-hgO5Abm0w5UL6FEa2iFnZqo2KlK7TQ5QhV5x09hujBf7t5KzHQ1VmfPuTpqRy/rNlSxua3eWH374xxiVrP+lcA==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", + "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", "cpu": [ "ppc64" ], @@ -1523,9 +1522,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.2.tgz", - "integrity": "sha512-fy8rXxuYEu602abC8MUNaPjYLIFzReOaEIEMKMUa0rFEUxNpVXhs15KSSQ4qlqSaM7B6rcj9rDZgADh/IGDzLQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", + "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", "cpu": [ "s390x" ], @@ -1540,9 +1539,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.2.tgz", - "integrity": "sha512-0+bOkiQ779+r1WpoHOWHqncvyySci0vKph+myNDYb+im6meJAzHQXay6oEgnkHuUGouM1LKTZwqKpBow6Kj7CQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", + "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", "cpu": [ "x64" ], @@ -1557,9 +1556,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.2.tgz", - "integrity": "sha512-mjSkrzZK5Qsl0a9d1JgILOiuZOSDTVdKENcSXBoqbzSrspLR/4/IRVDo5wd2GgZjNss/viBFJdeq+j7qH2nypw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", + "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", "cpu": [ "x64" ], @@ -1574,9 +1573,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.2.tgz", - "integrity": "sha512-1v5vHasdfQAZoEHakBV72LIFAC9JjnymsiKxp+GEr/ma3+NJCPSaYK+qavInOovJkgwFrs7GccX2d6IgDA3Z5w==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", + "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", "cpu": [ "arm64" ], @@ -1591,9 +1590,9 @@ } }, "node_modules/@rolldown/binding-wasm32-wasi": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.2.tgz", - "integrity": "sha512-mb1VobWn6NheziTk5/WEaR6AKVbrwT5sOi6C7zk3gy/pD1qtJfU1j4PgTo2NJnOtbL9Dl3Aeei8w9jJ7qC2jZQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", + "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", "cpu": [ "wasm32" ], @@ -1621,9 +1620,9 @@ } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.2.tgz", - "integrity": "sha512-SqKonF56vA/L2yHwHYcEp2P34URpOZ7d1fS635cTkpDnUtEGdUbhI6NzsPdqeSWvAAeGDrxjWjNmibDIdFf9/A==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", + "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", "cpu": [ "arm64" ], @@ -1638,9 +1637,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.2.tgz", - "integrity": "sha512-v7qRI7gXLRINcOGXt+7YmAZ6iFuyZVMIoXAxhd8oP+DR9dLfL9GfNIx7PLMxmhZdvq8waUJBQiWN9EKNy+TRBQ==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", + "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", "cpu": [ "x64" ], @@ -1675,6 +1674,15 @@ "dev": true, "license": "MIT" }, + "node_modules/@swc/helpers": { + "version": "0.5.23", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", + "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, "node_modules/@tybys/wasm-util": { "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", @@ -1697,6 +1705,16 @@ "@types/node": "*" } }, + "node_modules/@types/busboy": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz", + "integrity": "sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -1718,6 +1736,18 @@ "@types/node": "*" } }, + "node_modules/@types/command-line-args": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", + "license": "MIT" + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", + "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", + "license": "MIT" + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -1792,9 +1822,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.9.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz", - "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", + "version": "25.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.2.tgz", + "integrity": "sha512-G05zqtJhcDLb8uslf5EjCxXg9G1KQxiV8OS0R26IC//Eoyitzqe8z37I7cqvnZlrlSfgocQRfSn/AHBZJJFyGw==", "license": "MIT", "dependencies": { "undici-types": ">=7.24.0 <7.24.7" @@ -1847,14 +1877,14 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.7.tgz", - "integrity": "sha512-qsYPeXc5Q9dFLd1i8Ap+Bx8sQgcp+rFVQo4R0dDsWNBzl26ldVF1qOO+RL24K7FDrR6pA+50XedRLSoSG24bVQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.8.tgz", + "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.7", + "@vitest/utils": "4.1.8", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -1868,8 +1898,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.7", - "vitest": "4.1.7" + "@vitest/browser": "4.1.8", + "vitest": "4.1.8" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -1878,16 +1908,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.7.tgz", - "integrity": "sha512-1R+tw0ortHEbZDGMymm+pN7/AFQ/RkFFdtd7EN+VBpynKmLbP8A3rpEXdshBJ7+8hQ9zBJh/i1s0yKNtxAnU7w==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", + "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.7", - "@vitest/utils": "4.1.7", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -1896,13 +1926,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.7.tgz", - "integrity": "sha512-vY7nuamKgfvpA1Koa3oYIw/k7D6kZnpGyNMZW8loow2bsBYla1TFdqTaXncWdRn4pgwNs+90RhnXhJScDwQeJA==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", + "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.7", + "@vitest/spy": "4.1.8", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -1923,9 +1953,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.7.tgz", - "integrity": "sha512-umgCarTOYQWIaDMvGDRZij+6b9oVeLIyJzfN+AS88e0ZOU3QTgNNSTtjQOpcvWr3np1N0j4WgZj+sb3oYBDscw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", + "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", "dev": true, "license": "MIT", "dependencies": { @@ -1936,13 +1966,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.7.tgz", - "integrity": "sha512-BapjmAQ2aI78WdMEfeUWivnfVzB+VPGwWRQcJE0OUq7qEeEcBsCSf+0T5iREBNE5nBb4wA5Ya0W6IA+sghdEFw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", + "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.7", + "@vitest/utils": "4.1.8", "pathe": "^2.0.3" }, "funding": { @@ -1950,14 +1980,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.7.tgz", - "integrity": "sha512-ZacLzja+TmJeZ1h14xW2FB/WpeimUD3haBXQPyJqxvo8jQTmfeA8zv58mtjN2C7EHXZDYVcVYdYmAxjkWVvKCw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", + "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.7", - "@vitest/utils": "4.1.7", + "@vitest/pretty-format": "4.1.8", + "@vitest/utils": "4.1.8", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -1966,9 +1996,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.7.tgz", - "integrity": "sha512-kbkI5LMWakyuTIvs6fUJ5qdIVb1XVKsYJAT4OJ938cHMROYMSfmoQdZy0aaAnjbbc8F61vkoTqz/Az+/HiIu5Q==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", + "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", "dev": true, "license": "MIT", "funding": { @@ -1976,13 +2006,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.7.tgz", - "integrity": "sha512-T532WBu791cBxJlCl6SO+J14l81DQx6uQHm1bQbmCDY7nqlEIgkza/UFnSBNaUtSf41unldDFjdOBYEQC4b5Hw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", + "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.7", + "@vitest/pretty-format": "4.1.8", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -2069,12 +2099,56 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/apache-arrow": { + "version": "21.1.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.1.0.tgz", + "integrity": "sha512-kQrYLxhC+NTVVZ4CCzGF6L/uPVOzJmD1T3XgbiUnP7oTeVFOFgEUu6IKNwCDkpFoBVqDKQivlX4RUFqqnWFlEA==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.11", + "@types/command-line-args": "^5.2.3", + "@types/command-line-usage": "^5.0.4", + "@types/node": "^24.0.3", + "command-line-args": "^6.0.1", + "command-line-usage": "^7.0.1", + "flatbuffers": "^25.1.24", + "json-bignum": "^0.0.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "24.13.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.13.0.tgz", + "integrity": "sha512-5vtOqGQr4NJKeEzV441FcOi2MeG9UTWq9LqVLGneDdu4vlX17H8kQ2PA2UmNwCUGPVDj4oBjNhS7ReVEIWJJrg==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.18.0" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "7.18.2", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.18.2.tgz", + "integrity": "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==", + "license": "MIT" + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "license": "Python-2.0" }, + "node_modules/array-back": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", + "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/assertion-error": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", @@ -2151,6 +2225,17 @@ "node": "18 || 20 || >=22" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -2199,6 +2284,37 @@ "node": ">=18" } }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, "node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -2281,6 +2397,44 @@ "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", "license": "MIT" }, + "node_modules/command-line-args": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-6.0.2.tgz", + "integrity": "sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.3", + "find-replace": "^5.0.2", + "lodash.camelcase": "^4.3.0", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20" + }, + "peerDependencies": { + "@75lb/nature": "latest" + }, + "peerDependenciesMeta": { + "@75lb/nature": { + "optional": true + } + } + }, + "node_modules/command-line-usage": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.4.tgz", + "integrity": "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^4.1.1", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/commander": { "version": "14.0.3", "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.3.tgz", @@ -2819,6 +2973,23 @@ "url": "https://opencollective.com/express" } }, + "node_modules/find-replace": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-5.0.2.tgz", + "integrity": "sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@75lb/nature": "latest" + }, + "peerDependenciesMeta": { + "@75lb/nature": { + "optional": true + } + } + }, "node_modules/flatbuffers": { "version": "25.9.23", "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", @@ -3057,7 +3228,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -3106,9 +3276,9 @@ "license": "MIT" }, "node_modules/hono": { - "version": "4.12.18", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.18.tgz", - "integrity": "sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==", + "version": "4.12.23", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.23.tgz", + "integrity": "sha512-eIaZ9qDgu7XV0pxOCrg7/WhnQ6Ivm22UcxhXx/A3dcbqbbYgBEkc6e/J/s7j2tS96zoB0S9VBdLwQNCWwUo4LA==", "license": "MIT", "engines": { "node": ">=16.9.0" @@ -3285,9 +3455,19 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -3296,6 +3476,14 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/json-schema-traverse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", @@ -3587,6 +3775,12 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -4291,13 +4485,13 @@ } }, "node_modules/rolldown": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.2.tgz", - "integrity": "sha512-oZx5zVDtVB44AW3eaifgDml1gWRDZGvjcfdxonE4swNPG98PrrXjaO/KrnUjzlMnztCCRVlUueA1kCXhARGk6g==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", + "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.132.0", + "@oxc-project/types": "=0.133.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -4307,21 +4501,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm64": "1.0.2", - "@rolldown/binding-darwin-arm64": "1.0.2", - "@rolldown/binding-darwin-x64": "1.0.2", - "@rolldown/binding-freebsd-x64": "1.0.2", - "@rolldown/binding-linux-arm-gnueabihf": "1.0.2", - "@rolldown/binding-linux-arm64-gnu": "1.0.2", - "@rolldown/binding-linux-arm64-musl": "1.0.2", - "@rolldown/binding-linux-ppc64-gnu": "1.0.2", - "@rolldown/binding-linux-s390x-gnu": "1.0.2", - "@rolldown/binding-linux-x64-gnu": "1.0.2", - "@rolldown/binding-linux-x64-musl": "1.0.2", - "@rolldown/binding-openharmony-arm64": "1.0.2", - "@rolldown/binding-wasm32-wasi": "1.0.2", - "@rolldown/binding-win32-arm64-msvc": "1.0.2", - "@rolldown/binding-win32-x64-msvc": "1.0.2" + "@rolldown/binding-android-arm64": "1.0.3", + "@rolldown/binding-darwin-arm64": "1.0.3", + "@rolldown/binding-darwin-x64": "1.0.3", + "@rolldown/binding-freebsd-x64": "1.0.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", + "@rolldown/binding-linux-arm64-gnu": "1.0.3", + "@rolldown/binding-linux-arm64-musl": "1.0.3", + "@rolldown/binding-linux-ppc64-gnu": "1.0.3", + "@rolldown/binding-linux-s390x-gnu": "1.0.3", + "@rolldown/binding-linux-x64-gnu": "1.0.3", + "@rolldown/binding-linux-x64-musl": "1.0.3", + "@rolldown/binding-openharmony-arm64": "1.0.3", + "@rolldown/binding-wasm32-wasi": "1.0.3", + "@rolldown/binding-win32-arm64-msvc": "1.0.3", + "@rolldown/binding-win32-x64-msvc": "1.0.3" } }, "node_modules/router": { @@ -4654,6 +4848,14 @@ "dev": true, "license": "MIT" }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -4693,7 +4895,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -4702,6 +4903,19 @@ "node": ">=8" } }, + "node_modules/table-layout": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", + "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "wordwrapjs": "^5.1.0" + }, + "engines": { + "node": ">=12.17" + } + }, "node_modules/tar": { "version": "7.5.13", "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", @@ -4748,9 +4962,9 @@ } }, "node_modules/tinyglobby": { - "version": "0.2.16", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", - "integrity": "sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==", + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", "dependencies": { @@ -4794,25 +5008,6 @@ "node-gyp-build": "^4.8.0" } }, - "node_modules/tree-sitter-c": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/tree-sitter-c/-/tree-sitter-c-0.21.4.tgz", - "integrity": "sha512-IahxFIhXiY15SUlrt2upBiKSBGdOaE1fjKLK1Ik5zxqGHf6T1rvr3IJrovbsE5sXhypx7Hnmf50gshsppaIihA==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.1" - }, - "peerDependencies": { - "tree-sitter": "^0.21.0" - }, - "peerDependenciesMeta": { - "tree_sitter": { - "optional": true - } - } - }, "node_modules/tree-sitter-c-sharp": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/tree-sitter-c-sharp/-/tree-sitter-c-sharp-0.23.1.tgz", @@ -4908,33 +5103,6 @@ } } }, - "node_modules/tree-sitter-kotlin": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/tree-sitter-kotlin/-/tree-sitter-kotlin-0.3.8.tgz", - "integrity": "sha512-A4obq6bjzmYrA+F0JLLoheFPcofFkctNaZSpnDd+GPn1SfVZLY4/GG4C0cYVBTOShuPBGGAOPLM1JWLZQV4m1g==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "node-addon-api": "^7.1.0", - "node-gyp-build": "^4.8.0" - }, - "peerDependencies": { - "tree-sitter": "^0.21.0" - }, - "peerDependenciesMeta": { - "tree_sitter": { - "optional": true - } - } - }, - "node_modules/tree-sitter-kotlin/node_modules/node-addon-api": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", - "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", - "license": "MIT", - "optional": true - }, "node_modules/tree-sitter-php": { "version": "0.23.12", "resolved": "https://registry.npmjs.org/tree-sitter-php/-/tree-sitter-php-0.23.12.tgz", @@ -5035,8 +5203,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true + "license": "0BSD" }, "node_modules/tsx": { "version": "4.22.4", @@ -5114,6 +5281,15 @@ "node": ">=14.17" } }, + "node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/undici-types": { "version": "7.24.6", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", @@ -5167,17 +5343,17 @@ } }, "node_modules/vite": { - "version": "8.0.14", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.14.tgz", - "integrity": "sha512-s4BJJ+5y1pYL6Otw51FHhVJQhPnuRinKig64g/1+EUNaJsd3gCKdD31IPFvswUgW9/60QT9oFHbZHbQK5imcxw==", + "version": "8.0.16", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", + "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.32.0", "picomatch": "^4.0.4", "postcss": "^8.5.15", - "rolldown": "1.0.2", - "tinyglobby": "^0.2.16" + "rolldown": "1.0.3", + "tinyglobby": "^0.2.17" }, "bin": { "vite": "bin/vite.js" @@ -5245,19 +5421,19 @@ } }, "node_modules/vitest": { - "version": "4.1.7", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.7.tgz", - "integrity": "sha512-flYyaFd2CgoCoU+0UKt3pxksgC+S02iTDN0n3LtqaMeXsI9SBcdNujc2k0DeFLzUn/0k538yNjOSdwgCqcrwJA==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", + "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.7", - "@vitest/mocker": "4.1.7", - "@vitest/pretty-format": "4.1.7", - "@vitest/runner": "4.1.7", - "@vitest/snapshot": "4.1.7", - "@vitest/spy": "4.1.7", - "@vitest/utils": "4.1.7", + "@vitest/expect": "4.1.8", + "@vitest/mocker": "4.1.8", + "@vitest/pretty-format": "4.1.8", + "@vitest/runner": "4.1.8", + "@vitest/snapshot": "4.1.8", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -5285,12 +5461,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.7", - "@vitest/browser-preview": "4.1.7", - "@vitest/browser-webdriverio": "4.1.7", - "@vitest/coverage-istanbul": "4.1.7", - "@vitest/coverage-v8": "4.1.7", - "@vitest/ui": "4.1.7", + "@vitest/browser-playwright": "4.1.8", + "@vitest/browser-preview": "4.1.8", + "@vitest/browser-webdriverio": "4.1.8", + "@vitest/coverage-istanbul": "4.1.8", + "@vitest/coverage-v8": "4.1.8", + "@vitest/ui": "4.1.8", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -5366,6 +5542,15 @@ "node": ">=8" } }, + "node_modules/wordwrapjs": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", + "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 53eb7dedd..6fa0c9007 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.6.5", + "version": "1.6.7", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", @@ -49,15 +49,17 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:cross-platform": "tsx scripts/run-cross-platform.ts", - "postinstall": "node scripts/materialize-vendor-grammars.cjs && node scripts/build-tree-sitter-dart.cjs && node scripts/build-tree-sitter-proto.cjs && node scripts/build-tree-sitter-swift.cjs", + "postinstall": "node scripts/build-tree-sitter-grammars.cjs", + "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", - "prepack": "node scripts/build.js" + "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js" }, "dependencies": { "@huggingface/transformers": "^4.1.0", - "@ladybugdb/core": "^0.16.1", + "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -70,14 +72,15 @@ "ignore": "^7.0.5", "js-yaml": "^4.1.1", "jsonc-parser": "^3.3.1", - "lru-cache": "^11.0.0", "mnemonist": "^0.40.3", + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0", + "onnxruntime-common": "^1.26.0", "onnxruntime-node": "^1.24.0", "pandemonium": "^2.4.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tree-sitter": "0.21.1", - "tree-sitter-c": "0.21.4", "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "0.23.2", "tree-sitter-go": "^0.23.0", @@ -90,12 +93,8 @@ "tree-sitter-typescript": "^0.23.2", "uuid": "^14.0.0" }, - "optionalDependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.0", - "tree-sitter-kotlin": "^0.3.8" - }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", diff --git a/gitnexus/scripts/assert-publish-grammar-coverage.cjs b/gitnexus/scripts/assert-publish-grammar-coverage.cjs new file mode 100644 index 000000000..b523f68e8 --- /dev/null +++ b/gitnexus/scripts/assert-publish-grammar-coverage.cjs @@ -0,0 +1,204 @@ +#!/usr/bin/env node +/** + * Publish guard: every vendored tree-sitter grammar must ship a loadable binding. + * + * The npm tarball includes gitnexus/vendor/ (package.json `files`). A grammar is + * "covered" on a platform-arch tuple if EITHER a prebuild ships for it OR the + * grammar's full source-build set ships (so the install can source-build it, + * toolchain permitting). A future lean publish — dropping the ~50 MB of generated + * source to ship prebuilds only — is safe ONLY once every grammar has all six + * prebuilds; doing it while any grammar still lacks a prebuild would ship a + * grammar with NO loadable binding (neither prebuild nor buildable source) → that + * language is silently dead for users. + * + * HOW SOURCE INCLUSION IS DECIDED. The `files` allow-list OVERRIDES `.npmignore` + * for the vendored subtree (verified: an active "vendor/(star-star)/src/parser.c" + * in .npmignore does NOT drop it from `npm pack`). So `.npmignore` can never + * exclude vendored source — the ONLY lever is the `files` field. A broad `vendor` + * ships the whole subtree (source + prebuilds); a lean publish narrows `files` to + * non-source subpaths. This guard therefore reads `files` directly rather than + * shelling out to `npm pack` (which, in prepack, would re-enter this guard and, + * on npm versions that don't honor --ignore-scripts for prepare/prepack, run the + * full build — slow enough to time out and fragile). + * + * Wired via `prepack`, so it fails `npm pack` / `npm publish` if the invariant is + * violated. + */ +const fs = require('fs'); +const path = require('path'); + +const TUPLES = [ + 'linux-x64', + 'linux-arm64', + 'darwin-x64', + 'darwin-arm64', + 'win32-x64', + 'win32-arm64', +]; + +// Source-build inputs (relative to vendor//) whose presence makes a grammar +// source-buildable. Per-grammar we only require the ones that exist on disk (e.g. +// tree-sitter-c has no external scanner.c). +const SOURCE_BUILD_REL = [ + 'binding.gyp', + 'bindings/node/binding.cc', + 'src/parser.c', + 'src/scanner.c', + 'src/tree_sitter/parser.h', +]; + +/** + * Does the package.json `files` allow-list ship the WHOLE vendor subtree (and + * therefore the vendored grammar source)? A bare `vendor` (optionally with a + * trailing slash or `/**`/`/*`) includes everything under vendor/. A lean publish + * replaces that with non-source subpaths, so this returns false and grammars must + * then rely on prebuilds. + */ +function filesShipsVendorSource(filesField) { + return (filesField || []).some((f) => { + const n = String(f) + .replace(/\\/g, '/') + .replace(/\/+$/, '') + .replace(/\/\*\*?$/, ''); + return n === 'vendor'; + }); +} + +/** The on-disk source-build inputs for a grammar (relative paths). */ +function sourceBuildSet(grammarDir) { + return SOURCE_BUILD_REL.filter((rel) => fs.existsSync(path.join(grammarDir, rel))); +} + +/** True when a grammar can be source-built from its vendored files (has gyp + parser). */ +function isBuildableFromSource(grammarDir) { + const set = sourceBuildSet(grammarDir); + return set.includes('binding.gyp') && set.includes('src/parser.c'); +} + +/** Count platform-arch tuples with a committed prebuilt .node on disk. */ +function countPrebuiltTuples(grammarDir) { + const pdir = path.join(grammarDir, 'prebuilds'); + let n = 0; + for (const t of TUPLES) { + const td = path.join(pdir, t); + try { + if (fs.statSync(td).isDirectory() && fs.readdirSync(td).some((f) => f.endsWith('.node'))) { + n++; + } + } catch { + /* tuple dir absent — not covered */ + } + } + return n; +} + +/** + * Pure core (exported for tests). `grammars` is a list of + * `{ name, prebuilt: 0..6, shipsSource: boolean }`. Returns human-readable + * problem strings; an empty array means the pack is publish-safe. + */ +function findCoverageProblems({ grammars }) { + const problems = []; + for (const g of grammars) { + if (g.prebuilt < 6 && !g.shipsSource) { + const missing = 6 - g.prebuilt; + problems.push( + `${g.name}: ${g.prebuilt}/6 prebuilds and its vendored source is not shipped ` + + `(the package.json \`files\` field excludes it, or it is not buildable) — would ship ` + + `with no loadable binding on ${missing} platform-arch tuple(s).`, + ); + } + } + return problems; +} + +/** + * Stray local source-build outputs under `vendor//build/`. These would + * ship in the tarball (`files: ["vendor"]` overrides .gitignore/.npmignore) AND + * shadow the committed prebuilds — `node-gyp-build` resolves `build/Release` + * BEFORE `prebuilds/`, so a consumer on the publisher's platform would load the + * stray (possibly stale/wrong) binding instead of the curated prebuild. The + * build dir is gitignored and only appears if a maintainer source-built locally + * (e.g. on a no-prebuild platform); refuse to publish it. (#2144 review.) + */ +function findStrayBuildArtifacts(vendorDir) { + if (!fs.existsSync(vendorDir)) return []; + return fs + .readdirSync(vendorDir) + .filter((d) => /^tree-sitter-/.test(d)) + .filter((d) => fs.existsSync(path.join(vendorDir, d, 'build'))) + .map((d) => `vendor/${d}/build`); +} + +function collectGrammars(vendorDir, shipsVendorSource) { + if (!fs.existsSync(vendorDir)) return []; + return fs + .readdirSync(vendorDir) + .filter((d) => /^tree-sitter-/.test(d)) + .map((name) => { + const dir = path.join(vendorDir, name); + return { + name, + prebuilt: countPrebuiltTuples(dir), + // Source ships when `files` includes the vendor subtree AND the grammar + // actually carries a buildable source set on disk. + shipsSource: shipsVendorSource && isBuildableFromSource(dir), + }; + }); +} + +function main() { + const gitnexusRoot = path.join(__dirname, '..'); + const vendorDir = path.join(gitnexusRoot, 'vendor'); + const pkg = JSON.parse(fs.readFileSync(path.join(gitnexusRoot, 'package.json'), 'utf8')); + const shipsVendorSource = filesShipsVendorSource(pkg.files); + + const grammars = collectGrammars(vendorDir, shipsVendorSource); + if (grammars.length === 0) { + console.error(`[publish-guard] No vendored tree-sitter grammars found under ${vendorDir}.`); + process.exit(1); + } + + const stray = findStrayBuildArtifacts(vendorDir); + if (stray.length > 0) { + console.error( + '[publish-guard] Refusing to publish — stray source-build output under vendor/ would\n' + + 'ship and shadow the committed prebuilds (node-gyp-build loads build/Release before\n' + + 'prebuilds/):', + ); + for (const s of stray) console.error(` - ${s}`); + console.error('\nFix: remove it before packing, e.g. `rm -rf gitnexus/vendor/*/build`.'); + process.exit(1); + } + + const problems = findCoverageProblems({ grammars }); + if (problems.length > 0) { + console.error('[publish-guard] Refusing to publish — a vendored grammar would ship unusable:'); + for (const p of problems) console.error(` - ${p}`); + console.error( + '\nFix: either commit the missing prebuilds (run the build-tree-sitter-prebuilds\n' + + 'workflow) or keep the vendored source in the package.json `files` field.', + ); + process.exit(1); + } + + const sourceShippers = grammars.filter((g) => g.shipsSource).length; + console.log( + `[publish-guard] OK — ${grammars.length} vendored grammar(s) covered ` + + `(${sourceShippers} shipping source, ${grammars.length - sourceShippers} prebuilds-only).`, + ); +} + +if (require.main === module) main(); + +module.exports = { + findCoverageProblems, + findStrayBuildArtifacts, + filesShipsVendorSource, + isBuildableFromSource, + sourceBuildSet, + countPrebuiltTuples, + collectGrammars, + TUPLES, + SOURCE_BUILD_REL, +}; diff --git a/gitnexus/scripts/bench/fts-evict-reload-rss.mjs b/gitnexus/scripts/bench/fts-evict-reload-rss.mjs new file mode 100644 index 000000000..30d3f359a --- /dev/null +++ b/gitnexus/scripts/bench/fts-evict-reload-rss.mjs @@ -0,0 +1,374 @@ +#!/usr/bin/env node +// FTS evict→reload RSS repro (gitnexus-enterprise PR #222 / local U3). +// +// Settles ONE empirical question that no static read can answer: when a +// LadybugDB database that has `LOAD EXTENSION fts` applied is closed and a +// fresh one is opened + re-LOADed (the pool's evict→reload cycle), does the +// native FTS arena get reclaimed by `db.close()` — or is it stranded, so RSS +// climbs without bound over a long-lived MCP `serve` session? +// +// • PLATEAU across cycles → db.close() reclaims the FTS arena; the OSS pool's +// footprint is bounded by MAX_POOL_SIZE (~5 live arenas). No unbounded leak; +// the #222 worker-isolation rewrite (plan U4) is NOT justified for OSS. +// • MONOTONIC CLIMB → the FTS arena is stranded per reopen; the user's +// hypothesis holds and U4 (route FTS reads through a reclaimable worker) is +// justified. +// +// SCOPE OF THE VERDICT (read before citing it). A per-reload FTS-arena leak +// would be PROPORTIONAL to the index size. A small fixture therefore produces a +// small per-cycle increment that an absolute threshold can read as PLATEAU even +// when a production-scale graph would leak visibly. So: +// - `--rows` controls fixture size; run it LARGE (tens of thousands) before +// concluding "no leak". The default is deliberately not tiny. +// - The verdict (in fts-rss-verdict.mjs) keys on slope DECELERATION, not total +// delta, with a noise floor that scales with the working-set growth +// (peak−baseline) so sensitivity tracks fixture/arena size — NOT the pre-DB +// baseline RSS. A sustained sub-floor positive slope is INCONCLUSIVE (a slow +// creep RSS can't distinguish from noise), never a clean PLATEAU. +// - The PLATEAU verdict is only valid for the corpus size it was run at; the +// output states that size. The production-faithful confirmation is a +// `--via-pool` run against a real large analyzed repo over a long session. +// +// Two modes: +// (default) NATIVE — reproduces the native sequence doInitLbug()+closeOne() +// perform (open Database → new Connection → LOAD EXTENSION fts → +// QUERY_FTS_INDEX → close), against K self-built FTS fixtures, with no +// gitnexus build required. `--no-await-close` mirrors the pool's +// fire-and-forget close instead of awaiting (the production close shape). +// --via-pool — drives the REAL gitnexus pool from compiled dist +// (initLbug → executeParameterized → closeLbug) against an existing analyzed +// repo, exercising the production path + the GITNEXUS_POOL_RSS_TRACE +// instrumentation. Probes ALL FTS indexes the repo has. Forces an explicit +// close+reinit each cycle. Run `node scripts/build.js` first so the dist +// reflects the current pool-adapter (incl. the RSS trace). +// +// Run with --expose-gc so RSS excludes V8-heap noise: +// node --expose-gc gitnexus/scripts/bench/fts-evict-reload-rss.mjs +// node --expose-gc gitnexus/scripts/bench/fts-evict-reload-rss.mjs --rows 40000 --cycles 30 +// GITNEXUS_POOL_RSS_TRACE=1 node --expose-gc \ +// gitnexus/scripts/bench/fts-evict-reload-rss.mjs --via-pool /path/to/repo/.gitnexus/lbug +// +// Flags by mode: --rows/--repos/--read-write/--no-await-close apply to NATIVE +// only; --cycles applies to both. VIA-POOL warns when a NATIVE-only flag is set. +// +// Memory benches are noisy. Default is 24 cycles; trust the TREND (slope / +// first-third vs last-third), never a single delta. A flat trend at a LARGE +// fixture is a real NEGATIVE result (no unbounded leak), not a failed run. + +import { createRequire } from 'node:module'; +import os from 'node:os'; +import path from 'node:path'; +import fs from 'node:fs'; +// Pure verdict classifier (median, slopeMbPerCycle, classifyVerdict) lives in a +// side-effect-free sibling module so it is unit-testable without loading the +// native addon or running this bench. See fts-rss-verdict.mjs. +import { classifyVerdict, median, slopeMbPerCycle } from './fts-rss-verdict.mjs'; + +const require = createRequire(import.meta.url); +const lbugModule = require('@ladybugdb/core'); +const lbug = lbugModule.default ?? lbugModule; + +const LBUG_MAX_DB_SIZE = 16 * 1024 * 1024 * 1024; + +// ── args ────────────────────────────────────────────────────────────────── +function argVal(flag, dflt) { + const i = process.argv.indexOf(flag); + return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : dflt; +} +const CYCLES = Math.max(6, parseInt(argVal('--cycles', '24'), 10) || 24); +const REPOS = Math.max(1, parseInt(argVal('--repos', '6'), 10) || 6); // >5 mirrors LRU thrash +// Fixture size. Default is large enough that a size-proportional leak would be +// visible across cycles; raise it further before trusting a PLATEAU verdict. +const ROWS = Math.max(100, parseInt(argVal('--rows', '8000'), 10) || 8000); +const VIA_POOL = argVal('--via-pool', null); +const READONLY = !process.argv.includes('--read-write'); +const AWAIT_CLOSE = !process.argv.includes('--no-await-close'); + +if (VIA_POOL) { + // These flags are consumed only by NATIVE mode; warn rather than ignore + // silently so a VIA-POOL run is not misread as honoring them. + const ignored = ['--rows', '--repos', '--read-write', '--no-await-close'].filter((f) => + process.argv.includes(f), + ); + if (ignored.length) { + console.error( + `[fts-rss] NOTE: ${ignored.join(', ')} apply to NATIVE mode only; ignored in --via-pool.`, + ); + } +} + +if (typeof global.gc !== 'function') { + console.error( + '[fts-rss] WARNING: run with --expose-gc for clean RSS samples ' + + '(`node --expose-gc `). Continuing without forced GC — results are noisier.', + ); +} + +const gc = () => { + if (typeof global.gc === 'function') { + global.gc(); + global.gc(); + } +}; +const rssMb = () => Math.round(process.memoryUsage().rss / (1024 * 1024)); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// ── fixture: a minimal FTS-bearing .lbug ──────────────────────────────────── +const WORDS = [ + 'login auth session token user password validate verify credential', + 'parse tree syntax node grammar lexer token ast traversal visitor', + 'graph query cypher match relation node edge pattern aggregate index', + 'memory pool buffer arena allocate reclaim evict cache resident heap', + 'search rank score bm25 fts index stem porter keyword document corpus', + 'worker fork process spawn kill reclaim isolate native binding addon', +]; + +function buildFixture(dir) { + fs.mkdirSync(dir, { recursive: true }); + const dbPath = path.join(dir, 'fixture.lbug'); + const db = new lbug.Database(dbPath, 0, false, false, LBUG_MAX_DB_SIZE); + const conn = new lbug.Connection(db); + return (async () => { + await conn.query('LOAD EXTENSION fts'); + await conn.query( + 'CREATE NODE TABLE Doc(id STRING, name STRING, content STRING, PRIMARY KEY(id))', + ); + // Batch-insert via UNWIND so large fixtures (`--rows`) build in seconds + // instead of one round-trip per row. The fixture size drives the per-arena + // FTS allocation, which is what makes a size-proportional leak observable. + const rows = []; + for (let i = 0; i < ROWS; i++) { + const w = WORDS[i % WORDS.length]; + const name = `sym_${i}`; + const content = `${w} ${name} block number ${i} ${WORDS[(i + 3) % WORDS.length]}`; + rows.push({ id: `doc:${i}`, name, content }); + } + const INSERT_CHUNK = 2000; + for (let i = 0; i < rows.length; i += INSERT_CHUNK) { + const chunk = rows.slice(i, i + INSERT_CHUNK); + const stmt = await conn.prepare( + 'UNWIND $rows AS r CREATE (:Doc {id: r.id, name: r.name, content: r.content})', + ); + await conn.execute(stmt, { rows: chunk }); + } + await conn.query( + "CALL CREATE_FTS_INDEX('Doc', 'doc_fts', ['name', 'content'], stemmer := 'porter')", + ); + await conn.close(); + await db.close(); + return dbPath; + })(); +} + +const QUERIES = ['login token', 'parse node', 'memory arena', 'search index', 'worker reclaim']; + +// ── NATIVE mode ───────────────────────────────────────────────────────────── +async function runNative() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'fts-rss-')); + console.error( + `[fts-rss] NATIVE: ${REPOS} fixtures × ${ROWS} rows × ${CYCLES} cycles ` + + `(readOnly=${READONLY}, awaitClose=${AWAIT_CLOSE})`, + ); + console.error(`[fts-rss] building ${REPOS} FTS fixture(s) under ${root} …`); + + const srcDb = await buildFixture(path.join(root, 'src')); + const repoPaths = []; + for (let k = 0; k < REPOS; k++) { + const dst = path.join(root, `repo-${k}`); + fs.cpSync(path.dirname(srcDb), dst, { recursive: true }); + repoPaths.push(path.join(dst, 'fixture.lbug')); + } + + // Mirror the pool's evict→reload: each visit opens a FRESH Database, makes a + // Connection, LOADs fts, runs an FTS query, then closes — no caching, so every + // visit is a reload. K>5 amplifies the LRU-thrash signal the pool would see. + const series = []; + gc(); + await sleep(50); + const baseline = rssMb(); + console.error(`[fts-rss] baseline RSS=${baseline}MB`); + + for (let cycle = 0; cycle < CYCLES; cycle++) { + for (let k = 0; k < REPOS; k++) { + const db = new lbug.Database(repoPaths[k], 0, false, READONLY, LBUG_MAX_DB_SIZE); + const conn = new lbug.Connection(db); + try { + await conn.query('LOAD EXTENSION fts'); // the per-reload re-LOAD under test + const q = QUERIES[(cycle + k) % QUERIES.length]; + const res = await conn.query( + `CALL QUERY_FTS_INDEX('Doc', 'doc_fts', '${q}') RETURN node.id AS id, score ORDER BY score DESC LIMIT 20`, + ); + // Drain so the query actually materializes results. + if (res && typeof res.getAll === 'function') await res.getAll(); + } catch (e) { + console.error(`[fts-rss] query error (cycle ${cycle}, repo ${k}): ${e?.message || e}`); + } finally { + // AWAIT_CLOSE (default) is the best case for reclamation. --no-await-close + // mirrors the pool's fire-and-forget close (closeOne: db.close().catch()) + // so a leak that only manifests without awaiting is not hidden. + if (AWAIT_CLOSE) { + try { + await conn.close(); + await db.close(); + } catch { + /* ignore */ + } + } else { + conn.close().catch(() => {}); + db.close().catch(() => {}); + } + } + } + gc(); + // Longer settle when not awaiting close, so fire-and-forget native teardown + // has a chance to complete before the RSS sample (avoids a false PLATEAU). + await sleep(AWAIT_CLOSE ? 20 : 200); + const rss = rssMb(); + series.push(rss); + console.error(`[fts-rss] cycle ${String(cycle + 1).padStart(3)}/${CYCLES} rssMB=${rss}`); + } + + fs.rmSync(root, { recursive: true, force: true }); + return { baseline, series, corpus: `${REPOS}×${ROWS} rows, native, awaitClose=${AWAIT_CLOSE}` }; +} + +// ── VIA-POOL mode (real gitnexus pool from compiled dist) ─────────────────── +async function runViaPool(lbugPath) { + if (!fs.existsSync(lbugPath)) { + console.error(`[fts-rss] --via-pool path not found: ${lbugPath}`); + process.exit(2); + } + // Compiled dist is required (the pool pulls the native addon + many modules). + const distUrl = new URL('../../dist/core/lbug/pool-adapter.js', import.meta.url); + let pool; + try { + pool = await import(distUrl.href); + } catch (e) { + console.error( + `[fts-rss] could not import compiled pool-adapter (${e?.message}). ` + + `Run \`node scripts/build.js\` first, or use NATIVE mode.`, + ); + process.exit(2); + } + const { initLbug, executeParameterized, closeLbug } = pool; + console.error( + `[fts-rss] VIA-POOL on ${lbugPath} × ${CYCLES} cycles ` + + `(explicit closeLbug+initLbug per cycle = forced evict→reload)`, + ); + + // Probe ALL FTS indexes the analyzed graph carries (mirrors fts-schema.ts + // FTS_INDEXES) so the per-cycle FTS arena load matches production, not a + // 2-of-5 subset that would understate it. + const FTS_INDEXES = [ + { table: 'File', indexName: 'file_fts' }, + { table: 'Function', indexName: 'function_fts' }, + { table: 'Class', indexName: 'class_fts' }, + { table: 'Method', indexName: 'method_fts' }, + { table: 'Interface', indexName: 'interface_fts' }, + ]; + + const series = []; + gc(); + const baseline = rssMb(); + console.error(`[fts-rss] baseline RSS=${baseline}MB`); + + for (let cycle = 0; cycle < CYCLES; cycle++) { + try { + await initLbug(lbugPath, lbugPath); + const q = QUERIES[cycle % QUERIES.length]; + for (const { table, indexName } of FTS_INDEXES) { + await executeParameterized( + lbugPath, + `CALL QUERY_FTS_INDEX('${table}', '${indexName}', $q) RETURN node.id AS id, score ORDER BY score DESC LIMIT 20`, + { q }, + ).catch(() => []); // index may not exist for this graph — that's fine + } + await closeLbug(lbugPath); // force eviction → next cycle reopens + re-LOADs fts + } catch (e) { + console.error(`[fts-rss] pool cycle ${cycle} error: ${e?.message || e}`); + } + gc(); + // closeLbug fires a fire-and-forget native close (pool closeOne: + // db.close().catch()), so settle longer than NATIVE's awaited close to let + // native teardown finish before sampling — else a real leak reads PLATEAU. + await sleep(200); + const rss = rssMb(); + series.push(rss); + console.error(`[fts-rss] cycle ${String(cycle + 1).padStart(3)}/${CYCLES} rssMB=${rss}`); + } + await closeLbug().catch(() => {}); + return { baseline, series, corpus: `via-pool ${path.basename(path.dirname(lbugPath))}` }; +} + +// ── verdict ───────────────────────────────────────────────────────────────── +function verdict({ baseline, series, corpus }) { + const third = Math.max(1, Math.floor(series.length / 3)); + const firstMed = median(series.slice(0, third)); + const lastMed = median(series.slice(-third)); + const delta = lastMed - firstMed; + const slope = slopeMbPerCycle(series); + + // All label logic lives in the pure, unit-tested classifier (fts-rss-verdict.mjs): + // epsilon-first flat→PLATEAU, decelerated→PLATEAU, sustained-sub-floor→INCONCLUSIVE, + // ≥floor sustained→CLIMB, step→INCONCLUSIVE; floor scales with the working-set + // growth (peak−baseline), not the pre-DB baseline RSS. + const { + verdict: label, + firstHalfSlope, + secondHalfSlope, + decelRatio, + floor, + stepDiscontinuity, + maxJump, + peak, + } = classifyVerdict(series, baseline); + + console.log('\n==================== FTS evict→reload RSS verdict ===================='); + console.log(`corpus: ${corpus}`); + console.log(`samples (MB): ${series.join(' ')}`); + console.log( + `baseline=${baseline} firstThirdMed=${firstMed} lastThirdMed=${lastMed} delta=${delta}MB ` + + `peak=${peak} overallSlope=${slope.toFixed(2)} firstHalfSlope=${firstHalfSlope.toFixed(2)} ` + + `secondHalfSlope=${secondHalfSlope.toFixed(2)}MB/cycle floor=${floor.toFixed(2)} decelRatio=${decelRatio.toFixed(2)} ` + + `maxJump=${maxJump}MB step=${stepDiscontinuity} cycles=${series.length}`, + ); + if (label === 'CLIMB') { + console.log( + 'VERDICT: CLIMB — the per-cycle increment is SUSTAINED (second-half slope ≈ first-half),\n' + + ' i.e. RSS rises ~linearly with no decay. The native FTS arena is NOT reclaimed\n' + + ' by db.close(); the leak is real over a long-lived session.\n' + + ' → plan U4 (worker/process isolation of the FTS read path) is JUSTIFIED.', + ); + } else if (label === 'PLATEAU') { + console.log( + `VERDICT: PLATEAU at this corpus (${corpus}) — the per-cycle increment DECAYS to flat\n` + + ' (second-half slope below the noise floor). db.close() reclaims the FTS arena;\n' + + ' footprint is bounded (and the pool further caps it at MAX_POOL_SIZE). No\n' + + ' unbounded leak. Caveat: synthetic fixture — confirm with a --via-pool run\n' + + ' against a real large analyzed repo before fully closing plan U4.', + ); + } else { + console.log( + `VERDICT: INCONCLUSIVE at this corpus (${corpus}) — the run is noisy (step discontinuity)\n` + + ' or still decelerating without reaching flat, so neither a clean PLATEAU nor a\n' + + ' sustained linear CLIMB can be asserted. NATIVE synthetic runs do not resolve\n' + + ' this reliably at scale. The definitive test is a --via-pool run against a real\n' + + ' large analyzed repo over many cycles (with GITNEXUS_POOL_RSS_TRACE=1). Plan U4\n' + + ' stays GATED — neither closed nor built on this evidence.', + ); + } + console.log( + `MACHINE: ${JSON.stringify({ mode: VIA_POOL ? 'via-pool' : 'native', corpus, baseline, firstMed, lastMed, delta, overallSlope: Number(slope.toFixed(3)), firstHalfSlope: Number(firstHalfSlope.toFixed(3)), secondHalfSlope: Number(secondHalfSlope.toFixed(3)), floor: Number(floor.toFixed(3)), decelRatio: Number(decelRatio.toFixed(3)), maxJump, stepDiscontinuity, peak, cycles: series.length, verdict: label })}`, + ); + console.log('=====================================================================\n'); +} + +// ── main ──────────────────────────────────────────────────────────────────── +(async () => { + const result = VIA_POOL ? await runViaPool(VIA_POOL) : await runNative(); + verdict(result); + process.exit(0); +})().catch((e) => { + console.error('[fts-rss] fatal:', e?.stack || e); + process.exit(1); +}); diff --git a/gitnexus/scripts/bench/fts-rss-verdict.mjs b/gitnexus/scripts/bench/fts-rss-verdict.mjs new file mode 100644 index 000000000..b198234ee --- /dev/null +++ b/gitnexus/scripts/bench/fts-rss-verdict.mjs @@ -0,0 +1,105 @@ +// Pure, side-effect-free verdict classifier for the FTS evict→reload RSS bench +// (fts-evict-reload-rss.mjs). Extracted so it can be unit-tested WITHOUT importing +// the native LadybugDB addon or running the bench — this module has zero imports +// and zero module-scope side effects. Do not add imports or top-level statements. +// +// The discriminant between a real leak and allocator warmup is SLOPE DECELERATION, +// not total delta. A true per-reload leak (stranded FTS arena) rises ~linearly: +// the second-half slope stays ≈ the first-half slope. Allocator working-set warmup +// rises then flattens: the second-half slope decays to a fraction of the first. +// +// Thresholds: +// EPSILON (~0.1 MB/cycle) — below this the tail is effectively flat (no leak). +// SUSTAIN_FLOOR (0.5 MB/cycle) — the base noise floor. +// The floor SCALES with the working-set growth (peak − baseline), NOT the pre-DB +// `baseline` RSS: baseline is interpreter/addon overhead (and is LARGER in +// --via-pool mode), so a baseline-keyed floor would inflate and HIDE leaks. A +// bigger fixture has a bigger arena and bigger per-cycle noise, so the floor +// rises with the working set: floor = SUSTAIN_FLOOR · max(1, (peak−baseline)/REF). + +export const EPSILON_MB_PER_CYCLE = 0.1; +export const SUSTAIN_FLOOR = 0.5; +// Reference working-set (MB) at which the floor equals SUSTAIN_FLOOR; the floor +// scales up linearly for larger arenas. ~200 MB ≈ a small FTS fixture's footprint. +export const FLOOR_REF_WORKINGSET_MB = 200; + +export function median(xs) { + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : Math.round((s[m - 1] + s[m]) / 2); +} + +export function slopeMbPerCycle(series) { + // Least-squares slope of rss vs cycle index. + const n = series.length; + if (n < 2) return 0; + const xs = series.map((_, i) => i); + const xMean = xs.reduce((a, b) => a + b, 0) / n; + const yMean = series.reduce((a, b) => a + b, 0) / n; + let num = 0; + let den = 0; + for (let i = 0; i < n; i++) { + num += (xs[i] - xMean) * (series[i] - yMean); + den += (xs[i] - xMean) ** 2; + } + return den === 0 ? 0 : num / den; +} + +/** + * Classify an RSS-per-cycle series into PLATEAU / CLIMB / INCONCLUSIVE. + * Pure: no I/O, no globals. `baseline` is the pre-DB RSS; `peak` defaults to the + * series max. Returns the label plus the diagnostics the bench prints. + */ +export function classifyVerdict(series, baseline, peak = Math.max(...series)) { + const cycles = series.length; + const half = Math.max(1, Math.floor(cycles / 2)); + const firstHalfSlope = slopeMbPerCycle(series.slice(0, half)); + const secondHalfSlope = slopeMbPerCycle(series.slice(-half)); + const decelRatio = secondHalfSlope / Math.max(firstHalfSlope, 1e-9); + + // Step discontinuity: a single cycle-to-cycle jump far larger than the typical + // per-cycle delta — a one-time allocator/arena reservation (then flat), not a + // per-reload leak, but a noisy run we won't claim a clean result on. + const deltas = series.slice(1).map((v, i) => v - series[i]); + const absDeltas = deltas.map(Math.abs).sort((a, b) => a - b); + const medAbsDelta = absDeltas.length ? absDeltas[Math.floor(absDeltas.length / 2)] : 0; + const maxJump = deltas.length ? Math.max(...deltas) : 0; + const stepDiscontinuity = maxJump > Math.max(30, 5 * Math.max(medAbsDelta, 1)); + + // Working-set-scaled floor (see header). Guard against a negative working set. + const workingSet = Math.max(0, peak - baseline); + const floor = SUSTAIN_FLOOR * Math.max(1, workingSet / FLOOR_REF_WORKINGSET_MB); + + const SUSTAINED = 0.6; // decelRatio at/above which the tail is "not decaying" + let verdict; + if (stepDiscontinuity) { + verdict = 'INCONCLUSIVE'; + } else if (secondHalfSlope < EPSILON_MB_PER_CYCLE) { + // Effectively flat — no leak, regardless of decelRatio (a flat-from-start run + // has decelRatio ≈ 1 but is still PLATEAU). This gate is what keeps a true + // negative from being over-corrected into INCONCLUSIVE. + verdict = 'PLATEAU'; + } else if (secondHalfSlope >= floor) { + // Tail is still substantial: sustained → real leak; decelerating → unresolved. + verdict = decelRatio >= SUSTAINED ? 'CLIMB' : 'INCONCLUSIVE'; + } else if (decelRatio < SUSTAINED) { + // Below the floor AND decelerating — warmup converged toward flat → PLATEAU. + verdict = 'PLATEAU'; + } else { + // Below the floor but SUSTAINED — a slow steady creep RSS can't distinguish + // from noise at this scale. The honest label is "not resolved", NEVER a clean + // PLATEAU ("no leak"). This is the headline tri-review fix. + verdict = 'INCONCLUSIVE'; + } + + return { + verdict, + firstHalfSlope, + secondHalfSlope, + decelRatio, + floor, + stepDiscontinuity, + maxJump, + peak, + }; +} diff --git a/gitnexus/scripts/build-tree-sitter-dart.cjs b/gitnexus/scripts/build-tree-sitter-dart.cjs deleted file mode 100644 index d7542e253..000000000 --- a/gitnexus/scripts/build-tree-sitter-dart.cjs +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env node -/** - * Build tree-sitter-dart native binding in node_modules/ after materialize-vendor-grammars.cjs. - * Vendored source lives in vendor/ only; see #836 and #1728. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: skip the native rebuild entirely. Dart parsing becomes -// unavailable but `npm install gitnexus` finishes much faster on machines -// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0' -// (read as a string), and any other value all fall through to the rebuild. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-dart] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const dartDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-dart'); -const bindingGyp = path.join(dartDir, 'binding.gyp'); -const bindingNode = path.join(dartDir, 'build', 'Release', 'tree_sitter_dart_binding.node'); - -try { - if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { - process.exit(0); - } - - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-dart] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-dart] Dart parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', - ); - process.exit(0); - } - - console.log('[tree-sitter-dart] Building native binding...'); - execSync('npx node-gyp rebuild', { - cwd: dartDir, - stdio: 'pipe', - timeout: 180000, - }); - console.log('[tree-sitter-dart] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-dart] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-dart] Dart parsing will be unavailable. Non-Dart functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-grammars.cjs b/gitnexus/scripts/build-tree-sitter-grammars.cjs new file mode 100644 index 000000000..842bd0d99 --- /dev/null +++ b/gitnexus/scripts/build-tree-sitter-grammars.cjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +/** + * Activate the vendored tree-sitter native bindings IN PLACE under `vendor/`. + * One registry-driven script replaces the former per-grammar + * build-tree-sitter-.cjs files (they were ~95% identical). + * + * The grammars (tree-sitter-c/dart/proto/swift/kotlin) are loaded from + * `vendor//` by absolute path at runtime (see + * src/core/tree-sitter/vendored-grammars.ts) and are NEVER copied into + * node_modules — an undeclared package under node_modules is "extraneous" to + * every subsequent npm/npx reify, which prunes/relocates it (Windows + * `EPERM: …, symlink` + a silent grammar deletion on the 2nd run; #2111/#1728). + * + * For each grammar the resolution order is identical: + * 1. If the vendored source is absent (no binding.gyp) or the binding is + * already built, do nothing. + * 2. Prefer a committed prebuild for this platform-arch (toolchain-free) via + * node-gyp-build — `vendor//prebuilds/` ships all six tuples, so on a + * supported platform this returns immediately and writes nothing. + * 3. Otherwise source-build from the vendored grammar source (binding.gyp + + * src/) into `vendor//build/` (gitignored) so parsing still works on + * a toolchain host that lacks a matching prebuild. + * + * HARD INVARIANT: this runs in `gitnexus`'s postinstall, so it MUST NEVER throw + * or exit non-zero — a failure for any single grammar must not break the install. + * + * Opt-out: GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (strict '1') skips the OPTIONAL + * grammars only. tree-sitter-c is REQUIRED (it backstops upstream's 4/6 ARM + * prebuild gap, #2116) and is always built. + * + * Usage: + * node build-tree-sitter-grammars.cjs # all grammars (postinstall) + * node build-tree-sitter-grammars.cjs swift c # only the named grammars + */ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +// Registry. `display`/`ext` drive the human-readable warnings; `required` +// grammars ignore the opt-out gate. Insertion order == build order (c first). +const GRAMMARS = { + c: { required: true, display: 'C', ext: '.c' }, + dart: { required: false, display: 'Dart', ext: '.dart' }, + proto: { required: false, display: 'Proto', ext: '.proto' }, + swift: { required: false, display: 'Swift', ext: '.swift' }, + kotlin: { required: false, display: 'Kotlin', ext: '.kt/.kts' }, +}; + +const skipOptional = process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1'; + +function buildGrammar(short) { + const cfg = GRAMMARS[short]; + const tag = `[tree-sitter-${short}]`; + + if (!cfg.required && skipOptional) { + console.warn( + `${tag} Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). ${cfg.display} parsing will be unavailable until reinstalled without the env var.`, + ); + return; + } + + const dir = path.join(__dirname, '..', 'vendor', `tree-sitter-${short}`); + const bindingGyp = path.join(dir, 'binding.gyp'); + const bindingNode = path.join(dir, 'build', 'Release', `tree_sitter_${short}_binding.node`); + + try { + // Not materialized (no source), or already built — nothing to do. + if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { + return; + } + + // Prefer a committed prebuild for this platform-arch (no toolchain needed). + try { + require('node-gyp-build').path(dir); + return; + } catch { + // No matching prebuild — fall through to the source build below. + } + + // The hoisted build deps must be resolvable to source-build. + try { + require.resolve('node-addon-api'); + require.resolve('node-gyp-build'); + } catch (resolveErr) { + console.warn( + `${tag} Skipping build: hoisted build deps not resolvable (${resolveErr.message}).`, + ); + console.warn( + `${tag} ${cfg.display} parsing will be unavailable until a prebuild or toolchain is present.`, + ); + return; + } + + console.log(`${tag} No prebuild for this platform — building native binding from source...`); + execSync('npx node-gyp rebuild', { cwd: dir, stdio: 'pipe', timeout: 180000 }); + console.log(`${tag} Native binding built successfully`); + } catch (err) { + console.warn(`${tag} Could not build native binding:`, err.message); + console.warn( + `${tag} ${cfg.display} (${cfg.ext}) parsing will be unavailable. Non-${cfg.display} functionality is unaffected.`, + ); + } +} + +function main() { + const args = process.argv.slice(2).filter(Boolean); + const targets = args.length > 0 ? args : Object.keys(GRAMMARS); + for (const short of targets) { + if (!GRAMMARS[short]) { + console.warn(`[tree-sitter] Unknown grammar '${short}' — skipping.`); + continue; + } + // Defensive: never let an unexpected throw escape and fail the install. + try { + buildGrammar(short); + } catch (err) { + console.warn(`[tree-sitter-${short}] Unexpected build error (ignored): ${err.message}`); + } + } + // Hard guarantee: postinstall must never exit non-zero. + process.exit(0); +} + +if (require.main === module) main(); + +module.exports = { GRAMMARS, buildGrammar }; diff --git a/gitnexus/scripts/build-tree-sitter-proto.cjs b/gitnexus/scripts/build-tree-sitter-proto.cjs deleted file mode 100644 index eaefd14e6..000000000 --- a/gitnexus/scripts/build-tree-sitter-proto.cjs +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env node -/** - * Build tree-sitter-proto native binding. - * - * Why this script exists: - * tree-sitter-proto is vendored under gitnexus/vendor/tree-sitter-proto/ - * and copied into node_modules/ by materialize-vendor-grammars.cjs. Previously, the vendored - * package had its own `dependencies` and `install` script, which caused - * npm to create `vendor/tree-sitter-proto/node_modules/` and - * `vendor/tree-sitter-proto/build/` during install. Those directories - * blocked `rmdir` on global-install upgrade, producing: - * - * ENOTEMPTY: directory not empty, rmdir - * '.../gitnexus/vendor/tree-sitter-proto/node_modules/node-addon-api' - * - * (See https://github.com/abhigyanpatwari/GitNexus/issues/836.) - * - * We stripped `dependencies` and the `install` script from the vendored - * package.json, hoisted `node-addon-api` and `node-gyp-build` into - * gitnexus's own optionalDependencies, and moved native compilation here. - * - * What this does: - * Runs `npx node-gyp rebuild` inside `node_modules/tree-sitter-proto/`. - * Build output lands in - * `node_modules/tree-sitter-proto/build/Release/tree_sitter_proto_binding.node` - * — under npm-managed territory, safe on upgrade. - * - * Mirrors the tree-sitter-dart build helper. Best-effort: if any - * precondition fails (optional dep absent, no toolchain, --ignore-scripts), - * warn and exit 0 so gitnexus install still succeeds. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: skip the native rebuild entirely. Proto parsing becomes -// unavailable but `npm install gitnexus` finishes much faster on machines -// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0' -// (read as a string), and any other value all fall through to the rebuild. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-proto] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Proto parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto'); -const bindingGyp = path.join(protoDir, 'binding.gyp'); -const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node'); - -try { - if (!fs.existsSync(bindingGyp)) { - // tree-sitter-proto is an optionalDependency; absent when install - // skipped optional deps or the file: dep was not resolved. - process.exit(0); - } - - // Skip if the native binding already exists (idempotent re-run). - if (fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Pre-flight: the hoisted build deps must be resolvable. - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-proto] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-proto] Proto parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', - ); - process.exit(0); - } - - console.log('[tree-sitter-proto] Building native binding...'); - execSync('npx node-gyp rebuild', { - cwd: protoDir, - stdio: 'pipe', - timeout: 180000, - }); - console.log('[tree-sitter-proto] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-proto] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-proto] Proto (.proto) parsing will be unavailable. Non-proto gitnexus functionality is unaffected.', - ); - // Exit 0: optionalDependency failures must not fail the gitnexus install. - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-swift.cjs b/gitnexus/scripts/build-tree-sitter-swift.cjs deleted file mode 100644 index cbdd6eb54..000000000 --- a/gitnexus/scripts/build-tree-sitter-swift.cjs +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env node -/** - * Probe tree-sitter-swift prebuild availability at install time. - * - * The vendored package ships platform prebuilds; node-gyp-build selects the - * correct binary at require time. This script calls node-gyp-build once - * against the materialized package so a missing-prebuild failure surfaces - * as an install-time warning (with the rest of the gitnexus install - * succeeding) rather than as a runtime error the first time Swift parsing - * is requested. The result is discarded — it does not copy, register, or - * mutate anything; the runtime require() path in parser-loader does the - * actual load. Running this probe here instead of an npm `install` script - * on the vendored package preserves the #836 hygiene (no scripts.install - * inside vendor/). - */ -const fs = require('fs'); -const path = require('path'); - -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn('[tree-sitter-swift] Skipping prebuild probe (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1).'); - process.exit(0); -} - -const swiftDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-swift'); - -try { - if (!fs.existsSync(path.join(swiftDir, 'bindings', 'node', 'index.js'))) { - process.exit(0); - } - - const nodeGypBuild = require('node-gyp-build'); - nodeGypBuild(swiftDir); -} catch (err) { - console.warn('[tree-sitter-swift] Prebuild probe failed:', err.message); - console.warn( - '[tree-sitter-swift] Swift parsing will be unavailable. Non-Swift functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/scripts/cross-platform-tests.ts b/gitnexus/scripts/cross-platform-tests.ts index 364eac683..d332ba1d6 100644 --- a/gitnexus/scripts/cross-platform-tests.ts +++ b/gitnexus/scripts/cross-platform-tests.ts @@ -33,7 +33,7 @@ const PLATFORM_LOGIC = [ 'test/unit/resolve-invocation.test.ts', 'test/unit/platform-capabilities.test.ts', 'test/unit/worker-pool-windows-quarantine.test.ts', - 'test/unit/lbug-pool-win-fts-probe.test.ts', + 'test/unit/lbug-pool-fts-load.test.ts', 'test/unit/repo-manager.test.ts', 'test/unit/repo-manager-finalize-invariant.test.ts', 'test/unit/hooks.test.ts', diff --git a/gitnexus/scripts/install-duckdb-extension.mjs b/gitnexus/scripts/install-duckdb-extension.mjs index 2bc65a05e..7492e084f 100644 --- a/gitnexus/scripts/install-duckdb-extension.mjs +++ b/gitnexus/scripts/install-duckdb-extension.mjs @@ -14,7 +14,7 @@ function parseLbugMaxDbSize(raw) { return Math.floor(parsed); } -async function installDuckDbExtension(extensionName) { +async function installDuckDbExtension(extensionName, verifyOnly = false) { if (!extensionName || !EXTENSION_NAME_PATTERN.test(extensionName)) { throw new Error(`Invalid DuckDB extension name: ${extensionName ?? ''}`); } @@ -22,9 +22,11 @@ async function installDuckDbExtension(extensionName) { const require = createRequire(import.meta.url); const lbugModule = require('@ladybugdb/core'); const lbug = lbugModule.default ?? lbugModule; - const lbugMaxDbSize = parseLbugMaxDbSize( - process.argv[3] ?? process.env.GITNEXUS_LBUG_MAX_DB_SIZE, - ); + // argv[3] is the optional positional size; ignore it when it is actually a + // flag token (e.g. `--verify-only`) and fall back to the env default. + const sizeArg = + process.argv[3] && !process.argv[3].startsWith('--') ? process.argv[3] : undefined; + const lbugMaxDbSize = parseLbugMaxDbSize(sizeArg ?? process.env.GITNEXUS_LBUG_MAX_DB_SIZE); const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-ext-install-')); const dbPath = path.join(tmpDir, 'install.lbug'); @@ -34,7 +36,18 @@ async function installDuckDbExtension(extensionName) { try { db = new lbug.Database(dbPath, 0, false, false, lbugMaxDbSize); conn = new lbug.Connection(db); - await conn.query(`INSTALL ${extensionName}`); + if (verifyOnly) { + // Prove a previously-baked extension is resolvable by a FRESH process + // under the current HOME (the runtime `LOAD EXTENSION` path) — no INSTALL, + // no network. Used as a Docker build-time gate so a HOME/extension-dir + // mismatch fails the build instead of silently degrading search at runtime. + await conn.query(`LOAD EXTENSION ${extensionName}`); + console.log( + `[install-ext] LOAD-only verify OK for '${extensionName}' (HOME=${process.env.HOME})`, + ); + } else { + await conn.query(`INSTALL ${extensionName}`); + } } finally { if (conn) await conn.close().catch(() => {}); if (db) await db.close().catch(() => {}); @@ -42,7 +55,10 @@ async function installDuckDbExtension(extensionName) { } } -installDuckDbExtension(process.argv[2] ?? process.env.GITNEXUS_LBUG_EXTENSION_NAME).catch((err) => { +installDuckDbExtension( + process.argv[2] ?? process.env.GITNEXUS_LBUG_EXTENSION_NAME, + process.argv.includes('--verify-only'), +).catch((err) => { console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)); process.exitCode = 1; }); diff --git a/gitnexus/scripts/materialize-vendor-grammars.cjs b/gitnexus/scripts/materialize-vendor-grammars.cjs deleted file mode 100644 index 399696f9b..000000000 --- a/gitnexus/scripts/materialize-vendor-grammars.cjs +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env node -/** - * Copy vendored tree-sitter grammars into node_modules/ using real files (fs.cpSync). - * - * Published gitnexus used to declare these as optionalDependencies with - * `file:./vendor/...`, which makes npm symlink/junction vendor → node_modules on - * install. Windows without Developer Mode often fails with EPERM (#1728). - * - * Vendor trees stay read-only in gitnexus/vendor/; build artifacts must only - * land under node_modules/ (see #836). - */ -const fs = require('fs'); -const path = require('path'); - -const ROOT = path.join(__dirname, '..'); -const VENDORED_GRAMMARS = ['tree-sitter-dart', 'tree-sitter-proto', 'tree-sitter-swift']; - -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[gitnexus] Skipping vendored grammar materialize (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart/Proto/Swift parsing will be unavailable.', - ); - process.exit(0); -} - -for (const name of VENDORED_GRAMMARS) { - const src = path.join(ROOT, 'vendor', name); - const dest = path.join(ROOT, 'node_modules', name); - - if (!fs.existsSync(src)) { - console.warn(`[gitnexus] vendor/${name} missing; skipping materialize.`); - continue; - } - - // Sequence: copy src → partial; rename dest → backup; rename partial → dest; - // remove backup. If any step fails, restore from backup so a previously- - // materialized grammar is never lost. Targets the #1728 EPERM scenario plus - // narrower failure modes (Windows AV scanner racing on rename, EBUSY mid-swap). - const partial = `${dest}.materialize-tmp`; - const backup = `${dest}.materialize-bak`; - try { - fs.mkdirSync(path.join(ROOT, 'node_modules'), { recursive: true }); - fs.rmSync(partial, { recursive: true, force: true }); - fs.rmSync(backup, { recursive: true, force: true }); - fs.cpSync(src, partial, { recursive: true, verbatim: true }); - if (fs.existsSync(dest)) { - fs.renameSync(dest, backup); - } - try { - fs.renameSync(partial, dest); - } catch (renameErr) { - // Best-effort rollback: restore the previous dest from backup. - if (fs.existsSync(backup)) { - try { - fs.renameSync(backup, dest); - } catch { - // If rollback also fails, the prior backup directory still exists on - // disk — the catch block below surfaces both errors via the warning. - } - } - throw renameErr; - } - fs.rmSync(backup, { recursive: true, force: true }); - } catch (err) { - // Fail-soft: a single locked/inaccessible file (common on Windows) must not - // abort the whole gitnexus install. Matches build-tree-sitter-*.cjs pattern. - fs.rmSync(partial, { recursive: true, force: true }); - console.warn(`[gitnexus] Could not materialize vendor/${name}: ${err.message}`); - console.warn( - `[gitnexus] ${name} parsing will be unavailable. Other functionality is unaffected.`, - ); - } -} diff --git a/gitnexus/scripts/spikes/s1-reaching-def-index-bench.ts b/gitnexus/scripts/spikes/s1-reaching-def-index-bench.ts new file mode 100644 index 000000000..98eb40bfc --- /dev/null +++ b/gitnexus/scripts/spikes/s1-reaching-def-index-bench.ts @@ -0,0 +1,151 @@ +/** + * Spike S1 (issue #2080, M0) — THROWAWAY benchmark. Not part of the build + * (scripts/ is excluded from tsconfig) or the test suite. + * + * Question: can LadybugDB serve the headline REACHING_DEF query + * [:REACHING_DEF*1..5 {variable}] + * fast enough, and what is the right storage shape for the `variable`? + * + * What it does: + * 1. Builds a synthetic ~100K-edge graph of BasicBlock nodes + REACHING_DEF + * edges (variable carried in the CodeRelation `reason` column) with a + * realistic per-variable fan-out distribution, and loads it through the + * real bulk-COPY path (loadGraphToLbug). + * 2. Probes whether LadybugDB supports a secondary index on a relationship + * property (the crux of the "edge property vs side table" decision). + * 3. Times the variable-filtered bounded var-length path query. + * + * Run: npx tsx scripts/spikes/s1-reaching-def-index-bench.ts [edgeCount] + */ +import fs from 'fs/promises'; +import path from 'path'; +import os from 'os'; +import { performance } from 'node:perf_hooks'; +import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; +import type { KnowledgeGraph } from '../../src/core/graph/types.js'; + +const EDGE_COUNT = Number(process.argv[2] ?? 30_000); +// Realistic-ish def-use shape: many short chains, variables reused across them. +const CHAIN_LEN = 6; // blocks per function-ish chain +const DISTINCT_VARS = Math.max(1, Math.floor(EDGE_COUNT / 20)); // ~20 edges/variable fan-out + +const log = (m: string) => process.stdout.write(m + '\n'); + +function buildSynthGraph(edgeCount: number): KnowledgeGraph { + const g = createKnowledgeGraph(); + let edges = 0; + let chain = 0; + while (edges < edgeCount) { + const base = `BasicBlock:synth/f${chain}.ts`; + for (let i = 0; i <= CHAIN_LEN; i++) { + g.addNode({ + id: `${base}:${i}`, + label: 'BasicBlock', + properties: { + name: '', + filePath: `synth/f${chain}.ts`, + startLine: i, + endLine: i, + text: '', + }, + }); + } + for (let i = 0; i < CHAIN_LEN && edges < edgeCount; i++) { + const variable = `v${edges % DISTINCT_VARS}`; + g.addRelationship({ + id: `${base}:${i}->${i + 1}:${variable}`, + sourceId: `${base}:${i}`, + targetId: `${base}:${i + 1}`, + type: 'REACHING_DEF', + confidence: 1.0, + reason: variable, // M0 storage: variable rides `reason` + }); + edges++; + } + chain++; + } + return g; +} + +async function main() { + const tmp = path.join(os.tmpdir(), `s1-spike-${Date.now()}`); + const storagePath = path.join(tmp, '.gitnexus'); + const dbPath = path.join(storagePath, 'lbug'); + await fs.mkdir(dbPath, { recursive: true }); + + const adapter = await import('../../src/core/lbug/lbug-adapter.js'); + await adapter.initLbug(dbPath); + + log( + `[S1] building synthetic graph: ~${EDGE_COUNT} REACHING_DEF edges, ` + + `${DISTINCT_VARS} distinct variables (~20 edges/var fan-out), chains of ${CHAIN_LEN}`, + ); + const g = buildSynthGraph(EDGE_COUNT); + + let t = performance.now(); + await adapter.loadGraphToLbug(g, tmp, storagePath); + const loadMs = performance.now() - t; + const stats = await adapter.getLbugStats(); + log(`[S1] bulk-COPY load: ${loadMs.toFixed(0)}ms (nodes=${stats.nodes}, edges=${stats.edges})`); + + // (2) Probe: does LadybugDB support a secondary index on a REL property? + let relIndexSupported = false; + let relIndexErr = ''; + for (const stmt of [ + "CALL CREATE_REL_INDEX('CodeRelation', 'cr_reason_idx', 'reason')", + 'CREATE INDEX cr_reason_idx ON CodeRelation(reason)', + ]) { + try { + await adapter.executeQuery(stmt); + relIndexSupported = true; + break; + } catch (e: any) { + relIndexErr = String(e?.message ?? e).split('\n')[0]; + } + } + log( + `[S1] rel-property secondary index supported? ${relIndexSupported} ` + + `(last error: ${relIndexErr})`, + ); + + // (3a) Single-hop variable filter — the common case M3 runs most. + const probeVar = 'v0'; + t = performance.now(); + const single = await adapter.executeQuery( + `MATCH (a:BasicBlock)-[r:CodeRelation {type: 'REACHING_DEF', reason: '${probeVar}'}]->(b:BasicBlock) + RETURN count(r) AS c`, + ); + const singleMs = performance.now() - t; + log(`[S1] single-hop variable filter → ${single[0]?.c} edges in ${singleMs.toFixed(0)}ms`); + + // (3b) SOURCE-ANCHORED bounded var-length path — the realistic taint query + // (anchor the source block, then walk REACHING_DEF up to 5 hops). The + // UNANCHORED global form ([:REACHING_DEF*1..5] from every block) is + // impractical at scale (path explosion) — that is itself an S1 finding: + // taint queries MUST be scoped to a source block, not run graph-wide. + const srcId = 'BasicBlock:synth/f0.ts:0'; + t = performance.now(); + const anchored = await adapter.executeQuery( + `MATCH p = (a:BasicBlock)-[:CodeRelation*1..5 {type: 'REACHING_DEF'}]->(b:BasicBlock) + WHERE a.id = '${srcId}' AND all(rel IN relationships(p) WHERE rel.reason = '${probeVar}') + RETURN count(p) AS paths`, + ); + const pathMs = performance.now() - t; + log( + `[S1] source-anchored [:REACHING_DEF*1..5 {reason='${probeVar}'}] from one block → ` + + `${anchored[0]?.paths} paths in ${pathMs.toFixed(0)}ms`, + ); + + await adapter.closeLbug(); + await fs.rm(tmp, { recursive: true, force: true }); + + log('\n[S1] VERDICT INPUTS:'); + log( + ` load_ms=${loadMs.toFixed(0)} single_hop_ms=${singleMs.toFixed(0)} anchored_path_ms=${pathMs.toFixed(0)} rel_index=${relIndexSupported}`, + ); +} + +main().catch((e) => { + console.error('[S1] FAILED:', e); + process.exit(1); +}); diff --git a/gitnexus/scripts/spikes/s2-postdom-prototype.ts b/gitnexus/scripts/spikes/s2-postdom-prototype.ts new file mode 100644 index 000000000..da06d8e7e --- /dev/null +++ b/gitnexus/scripts/spikes/s2-postdom-prototype.ts @@ -0,0 +1,162 @@ +/** + * Spike S2 (issue #2080, M0) — THROWAWAY post-dominator feasibility prototype. + * Not part of the build (scripts/ excluded from tsconfig) or the test suite. + * + * Question (per maintainer review): does the post-dominator algorithm Epic B + * (#2085, CDG) depends on hold up on real TS/JS control-flow shapes — the + * classic CFG hazards — before Epic B commits to it? + * + * Scope boundary: post-dominators operate on a CFG, not on the AST directly. + * This prototype validates the ALGORITHM (iterative dataflow on the reverse + * CFG, EXIT-rooted, → immediate-post-dominator tree) against CFGs that model + * each hazard's real TS control flow (the TS source each CFG represents is + * shown inline). Building the CFG from a tree-sitter AST is M1's job (#2081); + * this spike deliberately does not reimplement it. + * + * Run: npx tsx scripts/spikes/s2-postdom-prototype.ts + */ + +type CFG = { + name: string; + tsSource: string; + entry: string; + exit: string; + // adjacency: block -> successors + succ: Record; + hazard: string; +}; + +// Iterative post-dominator dataflow on the reverse CFG. +// PostDom(EXIT) = {EXIT}; PostDom(n) = {n} ∪ (⋂ PostDom(s) for s ∈ succ(n)). +// Monotone over a finite lattice (powerset of blocks) ⇒ guaranteed to converge. +function postDominators(cfg: CFG): { pdom: Record>; iterations: number } { + const blocks = Object.keys(cfg.succ); + const all = new Set(blocks); + const pdom: Record> = {}; + for (const b of blocks) pdom[b] = b === cfg.exit ? new Set([cfg.exit]) : new Set(all); + + let changed = true; + let iterations = 0; + while (changed) { + changed = false; + iterations++; + for (const b of blocks) { + if (b === cfg.exit) continue; + const succs = cfg.succ[b] ?? []; + let inter: Set | null = null; + for (const s of succs) { + if (inter === null) inter = new Set(pdom[s]); + else inter = new Set([...inter].filter((x) => pdom[s].has(x))); + } + const next = new Set(inter ?? []); + next.add(b); + if (next.size !== pdom[b].size || [...next].some((x) => !pdom[b].has(x))) { + pdom[b] = next; + changed = true; + } + } + if (iterations > blocks.length + 5) + throw new Error('post-dom did not converge (suspected bug)'); + } + return { pdom, iterations }; +} + +// Immediate post-dominator: the closest strict post-dominator. +function ipdom(cfg: CFG, pdom: Record>): Record { + const res: Record = {}; + for (const b of Object.keys(cfg.succ)) { + if (b === cfg.exit) { + res[b] = null; + continue; + } + const strict = [...pdom[b]].filter((x) => x !== b); + // ipdom = the strict post-dom that does not post-dominate any other strict post-dom. + res[b] = + strict.find((cand) => strict.every((other) => other === cand || !pdom[other].has(cand))) ?? + null; + } + return res; +} + +const CFGS: CFG[] = [ + { + name: 'early-return', + hazard: 'early return / multiple paths to EXIT', + tsSource: `function f(x){ if (x) { return 1; } g(); return 2; }`, + entry: 'ENTRY', + exit: 'EXIT', + succ: { ENTRY: ['ret1', 'g'], ret1: ['EXIT'], g: ['ret2'], ret2: ['EXIT'], EXIT: [] }, + }, + { + name: 'try-throw-finally', + hazard: 'try/throw/finally with multiple exits through finally', + tsSource: `function f(){ try { risky(); } catch(e){ handle(e); } finally { cleanup(); } done(); }`, + entry: 'ENTRY', + exit: 'EXIT', + // try → (normal | throw→catch) → finally → done → EXIT; finally also reached on rethrow + succ: { + ENTRY: ['try'], + try: ['finally', 'catch'], + catch: ['finally'], + finally: ['done', 'EXIT'], + done: ['EXIT'], + EXIT: [], + }, + }, + { + name: 'labeled-break', + hazard: 'labeled break/continue across nested loops', + tsSource: `outer: for(;;){ for(;;){ if (a) break outer; if (b) continue outer; work(); } }`, + entry: 'ENTRY', + exit: 'EXIT', + succ: { + ENTRY: ['outerHead'], + outerHead: ['innerHead', 'EXIT'], + innerHead: ['breakOuter', 'afterIf1'], + breakOuter: ['EXIT'], + afterIf1: ['contOuter', 'work'], + contOuter: ['outerHead'], + work: ['innerHead'], + EXIT: [], + }, + }, + { + name: 'if-else-diamond', + hazard: 'baseline reducible diamond (sanity)', + tsSource: `function f(x){ if (x) { a(); } else { b(); } c(); }`, + entry: 'ENTRY', + exit: 'EXIT', + succ: { ENTRY: ['a', 'b'], a: ['c'], b: ['c'], c: ['EXIT'], EXIT: [] }, + }, +]; + +function main() { + let allOk = true; + for (const cfg of CFGS) { + try { + const { pdom, iterations } = postDominators(cfg); + const idom = ipdom(cfg, pdom); + // Sanity invariants: EXIT post-dominates every block; ipdom tree reaches EXIT. + const exitPostDomsAll = Object.keys(cfg.succ).every((b) => pdom[b].has(cfg.exit)); + console.log(`\n[S2] ${cfg.name} — ${cfg.hazard}`); + console.log(` TS: ${cfg.tsSource}`); + console.log( + ` converged in ${iterations} iters; EXIT post-dominates all blocks: ${exitPostDomsAll}`, + ); + console.log( + ` ipdom tree: ${Object.entries(idom) + .map(([b, p]) => `${b}->${p ?? '∅'}`) + .join(' ')}`, + ); + if (!exitPostDomsAll) allOk = false; + } catch (e) { + allOk = false; + console.log(`\n[S2] ${cfg.name} FAILED: ${(e as Error).message}`); + } + } + console.log( + `\n[S2] VERDICT INPUT: all hazard CFGs converged + EXIT post-dominates all = ${allOk}`, + ); +} + +main(); diff --git a/gitnexus/shadow-parity-dashboard/index.html b/gitnexus/shadow-parity-dashboard/index.html deleted file mode 100644 index 104d7b026..000000000 --- a/gitnexus/shadow-parity-dashboard/index.html +++ /dev/null @@ -1,291 +0,0 @@ - - - - - - GitNexus — Shadow Parity Dashboard - - - - -
-

Shadow Parity — RFC #909

-
loading latest.json…
-
- - - - - - - - - - - - - - -
LanguageTotalAgreeOnly legacyOnly newDisagreeBoth emptyParity
- -
- - - diff --git a/gitnexus/skills/gitnexus-debugging.md b/gitnexus/skills/gitnexus-debugging.md index 937b5e2a4..9834f94b7 100644 --- a/gitnexus/skills/gitnexus-debugging.md +++ b/gitnexus/skills/gitnexus-debugging.md @@ -16,10 +16,10 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ## Workflow ``` -1. gitnexus_query({query: ""}) → Find related execution flows -2. gitnexus_context({name: ""}) → See callers/callees/processes +1. query({query: ""}) → Find related execution flows +2. context({name: ""}) → See callers/callees/processes 3. READ gitnexus://repo/{name}/process/{name} → Trace execution flow -4. gitnexus_cypher({query: "MATCH path..."}) → Custom traces if needed +4. cypher({query: "MATCH path..."}) → Custom traces if needed ``` > If "Index is stale" → run `node .gitnexus/run.cjs analyze` in terminal. @@ -28,11 +28,11 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ``` - [ ] Understand the symptom (error message, unexpected behavior) -- [ ] gitnexus_query for error text or related code +- [ ] query for error text or related code - [ ] Identify the suspect function from returned processes -- [ ] gitnexus_context to see callers and callees +- [ ] context to see callers and callees - [ ] Trace execution flow via process resource if applicable -- [ ] gitnexus_cypher for custom call chain traces if needed +- [ ] cypher for custom call chain traces if needed - [ ] Read source files to confirm root cause ``` @@ -40,7 +40,7 @@ description: "Use when the user is debugging a bug, tracing an error, or asking | Symptom | GitNexus Approach | | -------------------- | ---------------------------------------------------------- | -| Error message | `gitnexus_query` for error text → `context` on throw sites | +| Error message | `query` for error text → `context` on throw sites | | Wrong return value | `context` on the function → trace callees for data flow | | Intermittent failure | `context` → look for external calls, async deps | | Performance issue | `context` → find symbols with many callers (hot paths) | @@ -48,24 +48,24 @@ description: "Use when the user is debugging a bug, tracing an error, or asking ## Tools -**gitnexus_query** — find code related to error: +**query** — find code related to error: ``` -gitnexus_query({query: "payment validation error"}) +query({query: "payment validation error"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError, PaymentException ``` -**gitnexus_context** — full context for a suspect: +**context** — full context for a suspect: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates (external API!) → Processes: CheckoutFlow (step 3/7) ``` -**gitnexus_cypher** — custom call chain traces: +**cypher** — custom call chain traces: ```cypher MATCH path = (a)-[:CodeRelation {type: 'CALLS'}*1..2]->(b:Function {name: "validatePayment"}) @@ -75,11 +75,11 @@ RETURN [n IN nodes(path) | n.name] AS chain ## Example: "Payment endpoint returns 500 intermittently" ``` -1. gitnexus_query({query: "payment error handling"}) +1. query({query: "payment error handling"}) → Processes: CheckoutFlow, ErrorHandling → Symbols: validatePayment, handlePaymentError -2. gitnexus_context({name: "validatePayment"}) +2. context({name: "validatePayment"}) → Outgoing calls: verifyCard, fetchRates (external API!) 3. READ gitnexus://repo/my-app/process/CheckoutFlow diff --git a/gitnexus/skills/gitnexus-exploring.md b/gitnexus/skills/gitnexus-exploring.md index 2dcf7b578..ccf684c28 100644 --- a/gitnexus/skills/gitnexus-exploring.md +++ b/gitnexus/skills/gitnexus-exploring.md @@ -18,8 +18,8 @@ description: "Use when the user asks how code works, wants to understand archite ``` 1. READ gitnexus://repos → Discover indexed repos 2. READ gitnexus://repo/{name}/context → Codebase overview, check staleness -3. gitnexus_query({query: ""}) → Find related execution flows -4. gitnexus_context({name: ""}) → Deep dive on specific symbol +3. query({query: ""}) → Find related execution flows +4. context({name: ""}) → Deep dive on specific symbol 5. READ gitnexus://repo/{name}/process/{name} → Trace full execution flow ``` @@ -29,9 +29,9 @@ description: "Use when the user asks how code works, wants to understand archite ``` - [ ] READ gitnexus://repo/{name}/context -- [ ] gitnexus_query for the concept you want to understand +- [ ] query for the concept you want to understand - [ ] Review returned processes (execution flows) -- [ ] gitnexus_context on key symbols for callers/callees +- [ ] context on key symbols for callers/callees - [ ] READ process resource for full execution traces - [ ] Read source files for implementation details ``` @@ -47,18 +47,18 @@ description: "Use when the user asks how code works, wants to understand archite ## Tools -**gitnexus_query** — find execution flows related to a concept: +**query** — find execution flows related to a concept: ``` -gitnexus_query({query: "payment processing"}) +query({query: "payment processing"}) → Processes: CheckoutFlow, RefundFlow, WebhookHandler → Symbols grouped by flow with file locations ``` -**gitnexus_context** — 360-degree view of a symbol: +**context** — 360-degree view of a symbol: ``` -gitnexus_context({name: "validateUser"}) +context({name: "validateUser"}) → Incoming calls: loginHandler, apiMiddleware → Outgoing calls: checkToken, getUserById → Processes: LoginFlow (step 2/5), TokenRefresh (step 1/3) @@ -68,10 +68,10 @@ gitnexus_context({name: "validateUser"}) ``` 1. READ gitnexus://repo/my-app/context → 918 symbols, 45 processes -2. gitnexus_query({query: "payment processing"}) +2. query({query: "payment processing"}) → CheckoutFlow: processPayment → validateCard → chargeStripe → RefundFlow: initiateRefund → calculateRefund → processRefund -3. gitnexus_context({name: "processPayment"}) +3. context({name: "processPayment"}) → Incoming: checkoutHandler, webhookHandler → Outgoing: validateCard, chargeStripe, saveTransaction 4. Read src/payments/processor.ts for implementation details diff --git a/gitnexus/skills/gitnexus-guide.md b/gitnexus/skills/gitnexus-guide.md index b81900b5e..a54337879 100644 --- a/gitnexus/skills/gitnexus-guide.md +++ b/gitnexus/skills/gitnexus-guide.md @@ -38,7 +38,38 @@ For any task involving code understanding, debugging, impact analysis, or refact | `detect_changes` | Git-diff impact — what do your current changes affect | | `rename` | Multi-file coordinated rename with confidence-tagged edits | | `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | +| `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | + +### Paginating `list_repos` + +`list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: + +```jsonc +{ + "repositories": [ + { "name": "...", "path": "...", "indexedAt": "...", "lastCommit": "...", "stats": { } } + ], + "pagination": { + "total": 437, + "limit": 50, + "offset": 0, + "returned": 50, + "hasMore": true, + "nextOffset": 50 + } +} +``` + +To enumerate **every** repository, keep calling with `offset` set to `pagination.nextOffset` until `hasMore` is `false`: + +```text +list_repos {} → repos 1–50, nextOffset 50, hasMore true +list_repos { offset: 50 } → repos 51–100, nextOffset 100, hasMore true +… +list_repos { offset: 400 } → repos 401–437, hasMore false (done) +``` + +Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. ## Resources Reference diff --git a/gitnexus/skills/gitnexus-impact-analysis.md b/gitnexus/skills/gitnexus-impact-analysis.md index 7206ca506..45eb7ce87 100644 --- a/gitnexus/skills/gitnexus-impact-analysis.md +++ b/gitnexus/skills/gitnexus-impact-analysis.md @@ -17,9 +17,9 @@ description: "Use when the user wants to know what will break if they change som ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → What depends on this +1. impact({target: "X", direction: "upstream"}) → What depends on this 2. READ gitnexus://repo/{name}/processes → Check affected execution flows -3. gitnexus_detect_changes() → Map current git changes to affected flows +3. detect_changes() → Map current git changes to affected flows 4. Assess risk and report to user ``` @@ -28,11 +28,11 @@ description: "Use when the user wants to know what will break if they change som ## Checklist ``` -- [ ] gitnexus_impact({target, direction: "upstream"}) to find dependents +- [ ] impact({target, direction: "upstream"}) to find dependents - [ ] Review d=1 items first (these WILL BREAK) - [ ] Check high-confidence (>0.8) dependencies - [ ] READ processes to check affected execution flows -- [ ] gitnexus_detect_changes() for pre-commit check +- [ ] detect_changes() for pre-commit check - [ ] Assess risk level and report to user ``` @@ -55,10 +55,10 @@ description: "Use when the user wants to know what will break if they change som ## Tools -**gitnexus_impact** — the primary tool for symbol blast radius: +**impact** — the primary tool for symbol blast radius: ``` -gitnexus_impact({ +impact({ target: "validateUser", direction: "upstream", minConfidence: 0.8, @@ -73,10 +73,10 @@ gitnexus_impact({ - authRouter (src/routes/auth.ts:22) [CALLS, 95%] ``` -**gitnexus_detect_changes** — git-diff based impact analysis: +**detect_changes** — git-diff based impact analysis: ``` -gitnexus_detect_changes({scope: "staged"}) +detect_changes({scope: "staged"}) → Changed: 5 symbols in 3 files → Affected: LoginFlow, TokenRefresh, APIMiddlewarePipeline @@ -86,7 +86,7 @@ gitnexus_detect_changes({scope: "staged"}) ## Example: "What breaks if I change validateUser?" ``` -1. gitnexus_impact({target: "validateUser", direction: "upstream"}) +1. impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware (WILL BREAK) → d=2: authRouter, sessionManager (LIKELY AFFECTED) diff --git a/gitnexus/skills/gitnexus-pr-review.md b/gitnexus/skills/gitnexus-pr-review.md index 319c063f9..9f1d362e5 100644 --- a/gitnexus/skills/gitnexus-pr-review.md +++ b/gitnexus/skills/gitnexus-pr-review.md @@ -18,10 +18,10 @@ description: "Use when the user wants to review a pull request, understand what ``` 1. gh pr diff → Get the raw diff -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows +2. detect_changes({scope: "compare", base_ref: "main"}) → Map diff to affected flows 3. For each changed symbol: - gitnexus_impact({target: "", direction: "upstream"}) → Blast radius per change -4. gitnexus_context({name: ""}) → Understand callers/callees + impact({target: "", direction: "upstream"}) → Blast radius per change +4. context({name: ""}) → Understand callers/callees 5. READ gitnexus://repo/{name}/processes → Check affected execution flows 6. Summarize findings with risk assessment ``` @@ -32,10 +32,10 @@ description: "Use when the user wants to review a pull request, understand what ``` - [ ] Fetch PR diff (gh pr diff or git diff base...head) -- [ ] gitnexus_detect_changes to map changes to affected execution flows -- [ ] gitnexus_impact on each non-trivial changed symbol +- [ ] detect_changes to map changes to affected execution flows +- [ ] impact on each non-trivial changed symbol - [ ] Review d=1 items (WILL BREAK) — are callers updated? -- [ ] gitnexus_context on key changed symbols to understand full picture +- [ ] context on key changed symbols to understand full picture - [ ] Check if affected processes have test coverage - [ ] Assess overall risk level - [ ] Write review summary with findings @@ -63,20 +63,20 @@ description: "Use when the user wants to review a pull request, understand what ## Tools -**gitnexus_detect_changes** — map PR diff to affected execution flows: +**detect_changes** — map PR diff to affected execution flows: ``` -gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +detect_changes({scope: "compare", base_ref: "main"}) → Changed: 8 symbols in 4 files → Affected processes: CheckoutFlow, RefundFlow, WebhookHandler → Risk: MEDIUM ``` -**gitnexus_impact** — blast radius per changed symbol: +**impact** — blast radius per changed symbol: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream"}) +impact({target: "validatePayment", direction: "upstream"}) → d=1 (WILL BREAK): - processCheckout (src/checkout.ts:42) [CALLS, 100%] @@ -86,20 +86,20 @@ gitnexus_impact({target: "validatePayment", direction: "upstream"}) - checkoutRouter (src/routes/checkout.ts:22) [CALLS, 95%] ``` -**gitnexus_impact with tests** — check test coverage: +**impact with tests** — check test coverage: ``` -gitnexus_impact({target: "validatePayment", direction: "upstream", includeTests: true}) +impact({target: "validatePayment", direction: "upstream", includeTests: true}) → Tests that cover this symbol: - validatePayment.test.ts [direct] - checkout.integration.test.ts [via processCheckout] ``` -**gitnexus_context** — understand a changed symbol's role: +**context** — understand a changed symbol's role: ``` -gitnexus_context({name: "validatePayment"}) +context({name: "validatePayment"}) → Incoming calls: processCheckout, webhookHandler → Outgoing calls: verifyCard, fetchRates @@ -112,20 +112,20 @@ gitnexus_context({name: "validatePayment"}) 1. gh pr diff 42 > /tmp/pr42.diff → 4 files changed: payments.ts, checkout.ts, types.ts, utils.ts -2. gitnexus_detect_changes({scope: "compare", base_ref: "main"}) +2. detect_changes({scope: "compare", base_ref: "main"}) → Changed symbols: validatePayment, PaymentInput, formatAmount → Affected processes: CheckoutFlow, RefundFlow → Risk: MEDIUM -3. gitnexus_impact({target: "validatePayment", direction: "upstream"}) +3. impact({target: "validatePayment", direction: "upstream"}) → d=1: processCheckout, webhookHandler (WILL BREAK) → webhookHandler is NOT in the PR diff — potential breakage! -4. gitnexus_impact({target: "PaymentInput", direction: "upstream"}) +4. impact({target: "PaymentInput", direction: "upstream"}) → d=1: validatePayment (in PR), createPayment (NOT in PR) → createPayment uses the old PaymentInput shape — breaking change! -5. gitnexus_context({name: "formatAmount"}) +5. context({name: "formatAmount"}) → Called by 12 functions — but change is backwards-compatible (added optional param) 6. Review summary: diff --git a/gitnexus/skills/gitnexus-refactoring.md b/gitnexus/skills/gitnexus-refactoring.md index c749eb384..e13c04e14 100644 --- a/gitnexus/skills/gitnexus-refactoring.md +++ b/gitnexus/skills/gitnexus-refactoring.md @@ -16,9 +16,9 @@ description: "Use when the user wants to rename, extract, split, move, or restru ## Workflow ``` -1. gitnexus_impact({target: "X", direction: "upstream"}) → Map all dependents -2. gitnexus_query({query: "X"}) → Find execution flows involving X -3. gitnexus_context({name: "X"}) → See all incoming/outgoing refs +1. impact({target: "X", direction: "upstream"}) → Map all dependents +2. query({query: "X"}) → Find execution flows involving X +3. context({name: "X"}) → See all incoming/outgoing refs 4. Plan update order: interfaces → implementations → callers → tests ``` @@ -29,65 +29,65 @@ description: "Use when the user wants to rename, extract, split, move, or restru ### Rename Symbol ``` -- [ ] gitnexus_rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits +- [ ] rename({symbol_name: "oldName", new_name: "newName", dry_run: true}) — preview all edits - [ ] Review graph edits (high confidence) and ast_search edits (review carefully) -- [ ] If satisfied: gitnexus_rename({..., dry_run: false}) — apply edits -- [ ] gitnexus_detect_changes() — verify only expected files changed +- [ ] If satisfied: rename({..., dry_run: false}) — apply edits +- [ ] detect_changes() — verify only expected files changed - [ ] Run tests for affected processes ``` ### Extract Module ``` -- [ ] gitnexus_context({name: target}) — see all incoming/outgoing refs -- [ ] gitnexus_impact({target, direction: "upstream"}) — find all external callers +- [ ] context({name: target}) — see all incoming/outgoing refs +- [ ] impact({target, direction: "upstream"}) — find all external callers - [ ] Define new module interface - [ ] Extract code, update imports -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ### Split Function/Service ``` -- [ ] gitnexus_context({name: target}) — understand all callees +- [ ] context({name: target}) — understand all callees - [ ] Group callees by responsibility -- [ ] gitnexus_impact({target, direction: "upstream"}) — map callers to update +- [ ] impact({target, direction: "upstream"}) — map callers to update - [ ] Create new functions/services - [ ] Update callers -- [ ] gitnexus_detect_changes() — verify affected scope +- [ ] detect_changes() — verify affected scope - [ ] Run tests for affected processes ``` ## Tools -**gitnexus_rename** — automated multi-file rename: +**rename** — automated multi-file rename: ``` -gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits across 8 files → 10 graph edits (high confidence), 2 ast_search edits (review) → Changes: [{file_path, edits: [{line, old_text, new_text, confidence}]}] ``` -**gitnexus_impact** — map all dependents first: +**impact** — map all dependents first: ``` -gitnexus_impact({target: "validateUser", direction: "upstream"}) +impact({target: "validateUser", direction: "upstream"}) → d=1: loginHandler, apiMiddleware, testUtils → Affected Processes: LoginFlow, TokenRefresh ``` -**gitnexus_detect_changes** — verify your changes after refactoring: +**detect_changes** — verify your changes after refactoring: ``` -gitnexus_detect_changes({scope: "all"}) +detect_changes({scope: "all"}) → Changed: 8 files, 12 symbols → Affected processes: LoginFlow, TokenRefresh → Risk: MEDIUM ``` -**gitnexus_cypher** — custom reference queries: +**cypher** — custom reference queries: ```cypher MATCH (caller)-[:CodeRelation {type: 'CALLS'}]->(f:Function {name: "validateUser"}) @@ -98,24 +98,24 @@ RETURN caller.name, caller.filePath ORDER BY caller.filePath | Risk Factor | Mitigation | | ------------------- | ----------------------------------------- | -| Many callers (>5) | Use gitnexus_rename for automated updates | +| Many callers (>5) | Use rename for automated updates | | Cross-area refs | Use detect_changes after to verify scope | -| String/dynamic refs | gitnexus_query to find them | +| String/dynamic refs | query to find them | | External/public API | Version and deprecate properly | ## Example: Rename `validateUser` to `authenticateUser` ``` -1. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) +1. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: true}) → 12 edits: 10 graph (safe), 2 ast_search (review) → Files: validator.ts, login.ts, middleware.ts, config.json... 2. Review ast_search edits (config.json: dynamic reference!) -3. gitnexus_rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) +3. rename({symbol_name: "validateUser", new_name: "authenticateUser", dry_run: false}) → Applied 12 edits across 8 files -4. gitnexus_detect_changes({scope: "all"}) +4. detect_changes({scope: "all"}) → Affected: LoginFlow, TokenRefresh → Risk: MEDIUM — run tests for these flows ``` diff --git a/gitnexus/src/cli/ai-context.ts b/gitnexus/src/cli/ai-context.ts index 6470c6430..641e7ee94 100644 --- a/gitnexus/src/cli/ai-context.ts +++ b/gitnexus/src/cli/ai-context.ts @@ -174,18 +174,18 @@ This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${s ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run \`gitnexus_impact({target: "symbolName", direction: "upstream"})\` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run \`gitnexus_detect_changes()\` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: \`gitnexus_detect_changes({scope: "compare", base_ref: ${JSON.stringify(markdownSafeBranch(defaultBranch))}})\`. +- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run \`impact({target: "symbolName", direction: "upstream"})\` and report the blast radius (direct callers, affected processes, risk level) to the user. +- **MUST run \`detect_changes()\` before committing** to verify your changes only affect expected symbols and execution flows. For regression review, compare against the default branch: \`detect_changes({scope: "compare", base_ref: ${JSON.stringify(markdownSafeBranch(defaultBranch))}})\`. - **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use \`gitnexus_query({query: "concept"})\` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use \`gitnexus_context({name: "symbolName"})\`. +- When exploring unfamiliar code, use \`query({query: "concept"})\` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. +- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use \`context({name: "symbolName"})\`. ## Never Do -- NEVER edit a function, class, or method without first running \`gitnexus_impact\` on it. +- NEVER edit a function, class, or method without first running \`impact\` on it. - NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use \`gitnexus_rename\` which understands the call graph. -- NEVER commit changes without running \`gitnexus_detect_changes()\` to check affected scope. +- NEVER rename symbols with find-and-replace — use \`rename\` which understands the call graph. +- NEVER commit changes without running \`detect_changes()\` to check affected scope. ## Resources diff --git a/gitnexus/src/cli/analyze-config.ts b/gitnexus/src/cli/analyze-config.ts index 8fbe40feb..63a328845 100644 --- a/gitnexus/src/cli/analyze-config.ts +++ b/gitnexus/src/cli/analyze-config.ts @@ -56,6 +56,7 @@ type ValueKind = | 'boolean' | 'boolean-negate' | 'string' + | 'string-array' | 'numeric-string' | 'embeddings' | 'branch'; @@ -84,6 +85,7 @@ const KEY_SPECS: Record = { skipContextFiles: { target: 'skipAgentsMd', kind: 'boolean' }, skipAiContext: { target: 'skipAgentsMd', kind: 'boolean' }, skipSkills: { target: 'skipSkills', kind: 'boolean' }, + pdg: { target: 'pdg', kind: 'boolean' }, indexOnly: { target: 'indexOnly', kind: 'boolean' }, stats: { target: 'stats', kind: 'boolean' }, noStats: { target: 'stats', kind: 'boolean-negate' }, @@ -99,6 +101,12 @@ const KEY_SPECS: Record = { embeddingBatchSize: { target: 'embeddingBatchSize', kind: 'numeric-string' }, embeddingSubBatchSize: { target: 'embeddingSubBatchSize', kind: 'numeric-string' }, embeddingDevice: { target: 'embeddingDevice', kind: 'string' }, + // #1589/#1852 residual — extra fetch-wrapper function names to treat as HTTP + // consumers. The auto-detector only flags functions that call the bare global + // `fetch()`; a wrapper built on axios / a custom client, or named outside the + // built-in convention set, is otherwise invisible to route_map consumers. + // Listing it here adds it to the cross-file consumer scan. + fetchWrappers: { target: 'fetchWrappers', kind: 'string-array' }, }; /** Top-level container key for the nested form; not itself an `AnalyzeOptions` field. */ @@ -230,6 +238,41 @@ const normalizeValue = (kind: ValueKind, value: unknown, key: string): unknown = } return trimmed; } + case 'string-array': { + // Generic shared validator — `source` already names the config key, so + // messages here stay key-agnostic (no fetch-wrapper coupling in the + // shared normalizer; #1589/#1852 review F7). + if (!Array.isArray(value)) { + throw new GitNexusRcError(`${source} must be an array of strings.`); + } + const names: string[] = []; + for (const item of value) { + if (typeof item !== 'string') { + throw new GitNexusRcError(`${source} entries must all be strings.`); + } + const trimmed = item.trim(); + if (!trimmed) { + throw new GitNexusRcError(`${source} entries must not be empty.`); + } + assertNoHiddenChars(trimmed, source); + // Values may be interpolated into a RegExp downstream. Restrict to + // identifier / member-access shapes so a config value can never smuggle + // regex metacharacters into a consumer. + if (!/^[A-Za-z_$][A-Za-z0-9_$.]*$/.test(trimmed)) { + throw new GitNexusRcError( + `${source} entry "${trimmed}" must be an identifier or member name ` + + `(letters, digits, _, $, . — e.g. "client.get").`, + ); + } + names.push(trimmed); + } + if (names.length === 0) { + throw new GitNexusRcError(`${source} must list at least one string.`); + } + // De-duplicate and cap to a sane bound so a pathological config cannot + // blow up the consumer scan's alternation. + return Array.from(new Set(names)).slice(0, 100); + } case 'numeric-string': { // Mirror Commander's contract: these options reach the existing CLI // validation as strings. Accept a JSON number or a string; normalize to a diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 6801f3f77..e54601bab 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -9,6 +9,7 @@ */ import path from 'path'; +import os from 'os'; import { spawn } from 'child_process'; import v8 from 'v8'; import cliProgress from 'cli-progress'; @@ -36,7 +37,7 @@ import { } from './analyze-config.js'; import { runFullAnalysis } from '../core/run-analyze.js'; import { getMaxFileSizeBannerMessage } from '../core/ingestion/utils/max-file-size.js'; -import { warnMissingOptionalGrammars } from './optional-grammars.js'; +import { warnMissingOptionalGrammars, getOptionalGrammarExtensions } from './optional-grammars.js'; import { glob } from 'glob'; import fs from 'fs/promises'; import { cliError } from './cli-message.js'; @@ -59,6 +60,22 @@ const writeFatalToStderr = (label: string, err: unknown): void => { const message = isErr ? err.message : String(err); realStderrWrite(`\n ${label}: ${message}\n`); if (isErr && err.stack) realStderrWrite(`${err.stack}\n`); + // Walk and print the `cause` chain. The phase runner wraps the underlying + // failure as `new Error("Phase 'X' failed: …", { cause })`, so the original + // error (e.g. a WorkerPoolDispatchError carrying the worker-side stack from + // #2068) is only reachable via `.cause`. Without this the user sees the + // wrapper's main-thread stack and never the real frame. `cause.stack` already + // begins with the cause's message, so we print the stack alone (not message + + // stack) to avoid repeating it. Depth-bounded so a cyclic `cause` can't loop + // (the phase runner wraps one level; the bound leaves headroom for future + // nesting); uses realStderrWrite so the redirected console.error's ANSI + // clear-line wrapping can't erase it (#1169). + const MAX_CAUSE_DEPTH = 5; + let cause: unknown = isErr ? (err as { cause?: unknown }).cause : undefined; + for (let depth = 0; depth < MAX_CAUSE_DEPTH && cause instanceof Error; depth++) { + realStderrWrite(`\n Caused by: ${cause.stack ?? cause.message}\n`); + cause = (cause as { cause?: unknown }).cause; + } }; let fatalHandlersInstalled = false; @@ -84,13 +101,45 @@ const installFatalHandlers = (): void => { }); }; -const HEAP_MB = 16384; +/** Historical floor for the re-exec heap cap — the auto-sizer never goes below + * this, so small boxes / CI never regress. */ +const DEFAULT_HEAP_MB = 16384; + +/** + * RAM-aware re-exec heap cap (MB): `0.75 × effective RAM`, clamped to + * `>= DEFAULT_HEAP_MB`. Kept BELOW physical RAM on purpose — a cap `>=` RAM makes + * V8 collect lazily and inflate the heap into swap-thrash (observed analyzing the + * Linux kernel at a 30GB cap on a 31GB box). `constrainedBytes` is the cgroup + * limit or `null`; it is honored only as a real, smaller-than-physical cap, because + * `process.constrainedMemory()` returns a huge sentinel when UNCONSTRAINED. + */ +export function computeHeapCapMb(totalBytes: number, constrainedBytes: number | null): number { + const effectiveBytes = + constrainedBytes !== null && constrainedBytes > 0 && constrainedBytes < totalBytes + ? constrainedBytes + : totalBytes; + const effectiveMb = Math.floor(effectiveBytes / (1024 * 1024)); + return Math.max(DEFAULT_HEAP_MB, Math.floor(0.75 * effectiveMb)); +} + +function readConstrainedBytes(): number | null { + if (typeof process.constrainedMemory !== 'function') return null; + const c = process.constrainedMemory(); + return typeof c === 'number' && c > 0 ? c : null; +} + +const HEAP_MB = computeHeapCapMb(os.totalmem(), readConstrainedBytes()); const TEST_RESPAWN_HEAP_MB = Number(process.env.GITNEXUS_TEST_RESPAWN_HEAP_MB); const RESPAWN_HEAP_MB = Number.isFinite(TEST_RESPAWN_HEAP_MB) && TEST_RESPAWN_HEAP_MB > 0 ? Math.floor(TEST_RESPAWN_HEAP_MB) : HEAP_MB; const HEAP_FLAG = `--max-old-space-size=${RESPAWN_HEAP_MB}`; +/** Larger semi-space (young-gen) cuts minor-GC frequency + promotion churn during + * the multi-million-node graph build/emit. Allowed in NODE_OPTIONS (unlike + * --stack-size), so it propagates to the re-exec env cleanly. */ +const SEMI_SPACE_MB = 128; +const SEMI_FLAG = `--max-semi-space-size=${SEMI_SPACE_MB}`; /** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */ const STACK_KB = 4096; const STACK_FLAG = `--stack-size=${STACK_KB}`; @@ -440,7 +489,8 @@ const forceHeapOOMForTestIfEnabled = (): void => { // `gitnexus/src/core/lbug/lbug-config.ts` in sync with this value. const RECOMMENDED_WAL_CHECKPOINT_THRESHOLD = 64 * 1024 * 1024; -/** Re-exec the process with a 16GB heap and larger stack if we're currently below that. */ +/** Re-exec the process with the RAM-aware auto heap cap + larger semi-space/stack + * if we're currently below that. A user-supplied NODE_OPTIONS heap wins (no re-exec). */ async function ensureHeap(): Promise { const nodeOpts = process.env.NODE_OPTIONS || ''; if (nodeOpts.includes('--max-old-space-size')) return false; @@ -448,25 +498,26 @@ async function ensureHeap(): Promise { const v8Heap = v8.getHeapStatistics().heap_size_limit; if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false; - // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, - // so pass it only as a direct CLI argument, not via the environment. - const cliFlags = [HEAP_FLAG]; + // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, so pass it + // only as a direct CLI argument. --max-semi-space-size IS allowed in NODE_OPTIONS. + const cliFlags = [HEAP_FLAG, SEMI_FLAG]; if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG); const childArgs = [...cliFlags, ...process.argv.slice(1)]; const childEnv = { ...process.env, - NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim(), + NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG} ${SEMI_FLAG}`.trim(), }; if (shouldBridgeRespawnProgressTty()) childEnv[RESPAWN_PROGRESS_ENV] = '1'; const childExit = await runRespawnedAnalyze(childArgs, childEnv); if (childExit.status !== 0 || childExit.signal) { if (childProcessLikelyOom(childExit)) { cliError( - ` Analysis likely ran out of memory.\n` + - ` Retry with a larger heap if your machine allows it:\n` + - ` NODE_OPTIONS="--max-old-space-size=24576" gitnexus analyze [your-args]\n` + - ` (Windows: set NODE_OPTIONS=--max-old-space-size=24576 && gitnexus analyze [your-args])\n` + + ` Analysis likely ran out of memory (heap cap auto-sized to ${RESPAWN_HEAP_MB}MB ≈ 0.75x RAM).\n` + + ` This repository's working set exceeds available RAM. Use a machine with more RAM,\n` + + ` or override the cap (a cap above physical RAM causes swap-thrash — use with care):\n` + + ` NODE_OPTIONS="--max-old-space-size=" gitnexus analyze [your-args]\n` + + ` (Windows: set NODE_OPTIONS=--max-old-space-size= && gitnexus analyze [your-args])\n` + ` If this persists, it may be a native crash unrelated to heap size.\n`, { recoveryHint: 'heap-oom-respawn' }, ); @@ -474,8 +525,7 @@ async function ensureHeap(): Promise { cliError( ` Analysis aborted in a native worker or native binding path.\n` + ` Try one of these recovery paths:\n` + - ` gitnexus analyze --workers 0\n` + - ` npm uninstall -g gitnexus && npm install -g gitnexus@latest\n` + + ` npm uninstall -g gitnexus && npm install -g gitnexus@latest (rebuilds native bindings)\n` + ` Use Node 22 LTS if you are on a newer non-LTS runtime.\n`, { recoveryHint: 'native-worker-abort' }, ); @@ -500,6 +550,7 @@ const ANALYZE_CLI_ENV_KEYS = [ 'GITNEXUS_VERBOSE', 'GITNEXUS_PROFILE_DEFERRED', 'GITNEXUS_PROFILE_DEFERRED_SLOW_MS', + 'GITNEXUS_DEBUG_HEAP', 'GITNEXUS_MAX_FILE_SIZE', 'GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS', 'GITNEXUS_WAL_CHECKPOINT_THRESHOLD', @@ -548,6 +599,12 @@ export interface AnalyzeOptions { verbose?: boolean; /** Skip AGENTS.md and CLAUDE.md gitnexus block updates. */ skipAgentsMd?: boolean; + /** + * Build the control-flow-graph / PDG substrate (#2081 M1). Opt-in; off by + * default. Threaded to both the worker (CFG build) and scope-resolution + * (BasicBlock/CFG emit). + */ + pdg?: boolean; /** * Stats inclusion in AGENTS.md and CLAUDE.md. * @@ -569,6 +626,14 @@ export interface AnalyzeOptions { * before being threaded into the generated AGENTS.md / CLAUDE.md content. */ defaultBranch?: string; + /** + * Index-branch selector (#2106). From `--branch`. Distinct from + * `defaultBranch` (cosmetic base_ref): this routes the index to a per-branch + * slot. NOT sourced from `.gitnexusrc` — the `.gitnexusrc` `branch` key is an + * alias for `defaultBranch` and must not change index placement. Defaults to + * the checked-out branch inside `runFullAnalysis` when omitted. + */ + branch?: string; /** Pure index mode: skip all file injection (AGENTS.md, CLAUDE.md, skills). */ indexOnly?: boolean; /** Index the folder even when no .git directory is present. */ @@ -598,12 +663,20 @@ export interface AnalyzeOptions { workerTimeout?: string; /** Control LadybugDB WAL auto-checkpoint threshold during analyze. */ walCheckpointThreshold?: string; - /** Parse worker pool size; 0 disables workers (sequential fallback). */ + /** Parse worker pool size (>=1); 0 is rejected (no sequential mode). */ workers?: string; embeddingThreads?: string; embeddingBatchSize?: string; embeddingSubBatchSize?: string; embeddingDevice?: string; + /** + * Extra fetch-wrapper function names to treat as HTTP consumers (#1589/#1852 + * residual). Supplied via `.gitnexusrc` `fetchWrappers: [...]`. Threaded into + * the routes phase, where the cross-file consumer scan unions them with the + * auto-detected `fetch()` wrappers so a custom/axios-based wrapper named + * outside the built-in convention still produces `route_map` consumers. + */ + fetchWrappers?: string[]; } /** @@ -711,6 +784,21 @@ const analyzeCommandImpl = async ( } } + // Validate the index-branch selector (#2106) the same way, so a malformed + // `--branch` exits before any expensive analysis starts. Capture the TRIMMED + // return so a whitespace-padded value (e.g. " feature" from shell completion) + // normalizes before the checked-out-branch mismatch guard and slug — otherwise + // it would false-reject on-branch or create a ghost index when detached. + if (cliOptions?.branch !== undefined) { + try { + cliOptions.branch = validateBranchName(cliOptions.branch, '--branch'); + } catch (err) { + cliError(` ${err instanceof Error ? err.message : String(err)}\n`); + process.exitCode = 1; + return; + } + } + // ── Load .gitnexusrc and merge: CLI flags override config (#243) ─── // Parse/validate before the progress bar so a malformed config produces an // actionable error and exits before any expensive analysis starts. @@ -793,10 +881,11 @@ const analyzeCommandImpl = async ( let workerPoolSize: number | undefined; if (options.workers !== undefined) { const parsedWorkers = Number(options.workers); - if (!Number.isInteger(parsedWorkers) || parsedWorkers < 0) { + if (!Number.isInteger(parsedWorkers) || parsedWorkers < 1) { cliError( - ' --workers must be a non-negative integer. ' + - 'Pass 0 to disable the worker pool (sequential fallback).\n', + ' --workers must be a positive integer (>= 1). ' + + 'GitNexus parses through a worker pool only — there is no sequential ' + + 'mode, so 0 is not allowed. Omit --workers for an auto-sized pool.\n', ); process.exitCode = 1; return; @@ -891,11 +980,13 @@ const analyzeCommandImpl = async ( } // If the target repo contains files an optional grammar would parse but - // that grammar's native binding is absent, warn before analysis so users - // learn why those files end up unparsed instead of silently getting a - // degraded index. + // that grammar's native binding is absent (or disabled via + // GITNEXUS_SKIP_OPTIONAL_GRAMMARS), warn before analysis so users learn why + // those files end up unparsed instead of silently getting a degraded index. + // The extension set is derived from OPTIONAL_GRAMMARS so it can't drift. try { - const matches = await glob(['**/*.dart', '**/*.proto'], { + const optionalGlobs = getOptionalGrammarExtensions().map((e) => `**/*${e}`); + const matches = await glob(optionalGlobs, { cwd: repoPath, ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'], dot: false, @@ -1037,9 +1128,15 @@ const analyzeCommandImpl = async ( skipGit: options.skipGit, skipAgentsMd, skipSkills, + // CFG/PDG substrate opt-in (#2081 M1) — threaded to both sinks downstream. + pdg: options.pdg === true, // Resolved default branch (CLI > .gitnexusrc > auto-detect > "main") // threaded into the generated regression-compare example (#243). defaultBranch: resolvedDefaultBranch, + // Index-branch selector (#2106). Read straight from the CLI flag (not + // the .gitnexusrc-merged options) so the cosmetic defaultBranch config + // can never change index placement. Undefined → auto-detect in pipeline. + branch: cliOptions?.branch, // commander.js `.option('--no-stats', …)` registers the flag as // `options.stats` (boolean, default true; `false` when the user // passed --no-stats). Reading `options.noStats` here returns @@ -1056,6 +1153,9 @@ const analyzeCommandImpl = async ( // GITNEXUS_WORKER_POOL_SIZE env mutation. `undefined` defers to the // env / auto-formula fallback inside the pipeline. workerPoolSize, + // Extra fetch-wrapper names from `.gitnexusrc` (#1589/#1852 residual); + // forwarded to the routes phase consumer scan. + fetchWrappers: options.fetchWrappers, }, { onProgress: (_phase, percent, message) => { @@ -1077,14 +1177,20 @@ const analyzeCommandImpl = async ( // preserving the rest of the block (incl. --skills community rows). No-op // when the value already matches, so a routine up-to-date run is silent // (#1996 tri-review P2). + // Only refresh the repo-root AGENTS.md/CLAUDE.md base_ref for the + // PRIMARY/flat index (#2106 R2). A non-primary branch's up-to-date + // analyze must not churn the committed AGENTS.md — this mirrors the + // in-pipeline `if (!placement.branch)` gate around generateAIContextFiles. let baseRefRefreshed: string[] = []; - try { - const { refreshBaseRefLine } = await import('./ai-context.js'); - baseRefRefreshed = ( - await refreshBaseRefLine(repoPath, resolvedDefaultBranch, { skipAgentsMd }) - ).files; - } catch { - /* best-effort — never fail the fast path over a context refresh */ + if (result.isPrimaryBranch !== false) { + try { + const { refreshBaseRefLine } = await import('./ai-context.js'); + baseRefRefreshed = ( + await refreshBaseRefLine(repoPath, resolvedDefaultBranch, { skipAgentsMd }) + ).files; + } catch { + /* best-effort — never fail the fast path over a context refresh */ + } } clearInterval(elapsedTimer); process.removeListener('SIGINT', sigintHandler); diff --git a/gitnexus/src/cli/clean.ts b/gitnexus/src/cli/clean.ts index ef5ab8bb8..98d881471 100644 --- a/gitnexus/src/cli/clean.ts +++ b/gitnexus/src/cli/clean.ts @@ -13,6 +13,8 @@ import { unregisterRepo, listRegisteredRepos, assertSafeStoragePath, + getStoragePaths, + removeBranchIndex, UnsafeStoragePathError, } from '../storage/repo-manager.js'; import { @@ -26,7 +28,50 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean; lbugSidecars?: boolean; + branch?: string; }) => { + // --branch : remove a single non-primary branch's index (#2106 R7). + // Resolve against the RECORDED branches[] summary (never by slugging the + // user's raw input, which can disagree with the index-time-sanitized label). + if (options?.branch) { + const cwd = process.cwd(); + const repo = await findRepo(cwd); + if (!repo) { + console.log(t('clean.notFoundHere')); + return; + } + const entries = await listRegisteredRepos(); + const entry = entries.find((e) => path.resolve(e.path) === path.resolve(repo.repoPath)); + const summary = entry?.branches?.find((b) => b.branch === options.branch); + if (!summary) { + console.log(t('clean.branchNotIndexed', { branch: options.branch })); + return; + } + const { storagePath, lbugPath } = getStoragePaths(repo.repoPath, summary.branch); + const branchDir = path.dirname(lbugPath); + // Safety guard: the target MUST live under /.gitnexus/branches/. + // assertSafeStoragePath only validates the flat `/.gitnexus`, so this + // is a dedicated branches-sub-dir check before any destructive fs.rm. + const branchesRoot = path.join(storagePath, 'branches') + path.sep; + if (!branchDir.startsWith(branchesRoot)) { + logger.error(`Refusing to clean branch index outside .gitnexus/branches: ${branchDir}`); + return; + } + if (!options.force) { + console.log(t('clean.deleteBranch', { branch: summary.branch, path: branchDir })); + console.log(`\n${t('common.runForceConfirm')}`); + return; + } + try { + await fs.rm(branchDir, { recursive: true, force: true }); + await removeBranchIndex(repo.repoPath, summary.branch); + console.log(t('clean.deletedBranch', { branch: summary.branch })); + } catch (err) { + logger.error({ err }, 'Failed to delete branch index:'); + } + return; + } + if (options?.lbugSidecars) { const cwd = process.cwd(); const repo = await findRepo(cwd); diff --git a/gitnexus/src/cli/editor-targets.ts b/gitnexus/src/cli/editor-targets.ts new file mode 100644 index 000000000..e00cf9778 --- /dev/null +++ b/gitnexus/src/cli/editor-targets.ts @@ -0,0 +1,187 @@ +/** + * Editor targets — the single source of truth for *where* GitNexus writes its + * per-editor configuration and *how* its entries are identified. + * + * `setup` (writes these) and `uninstall` (removes them) both consume this + * module so the two stay structurally in lock-step: add or change a target + * here and both sides follow. This is declarative metadata only — file + * locations, JSON key paths, hook event names, command needles, and script + * directories, plus the shared `detectIndentation` formatting helper. The + * format-specific read/write logic (JSONC merge, TOML upsert, OpenCode's flat + * command array, Gemini's hook schema) deliberately stays in setup.ts / + * uninstall.ts. + * + * The `setup → uninstall` round-trip integration test verifies the two + * implementations remain behaviourally symmetrical on top of this shared + * structure. + */ + +import os from 'os'; +import path from 'path'; + +export type EditorId = 'cursor' | 'claude' | 'antigravity' | 'opencode' | 'codex'; + +/** An editor whose MCP config is a JSONC document (server keyed by name). */ +export interface McpJsoncTarget { + id: EditorId; + label: string; + /** Absolute path to the editor's MCP config file. */ + file: string; + /** + * JSON path of the gitnexus server entry within that file. Typed as + * `string[]` (all our keys are object keys) so it satisfies both setup's + * `mergeJsoncFile(string[])` and uninstall's `removeJsoncKey(JSONPath)` + * without either side needing a cast. + */ + keyPath: string[]; +} + +/** Codex stores MCP config as a TOML table, not JSONC. */ +export interface CodexMcpTarget { + id: 'codex'; + label: string; + /** Absolute path to ~/.codex/config.toml. */ + configFile: string; + /** The TOML table header (without brackets) setup writes / uninstall strips. */ + tomlSection: string; +} + +export interface SkillTarget { + id: EditorId; + label: string; + /** Absolute path to the editor's skills directory. */ + dir: string; +} + +export interface HookTarget { + id: EditorId; + label: string; + /** Absolute path to the editor's settings file (JSONC). */ + settingsFile: string; + /** Hook event arrays that may hold a gitnexus entry. */ + events: string[]; + /** Substring identifying the gitnexus command within a hook entry. */ + needle: string; + /** Absolute path to the bundled hook-script directory setup writes. */ + scriptDir: string; +} + +export interface EditorTargets { + /** JSONC-format MCP entries: Cursor, Claude Code, Antigravity, OpenCode. */ + mcpJsonc: McpJsoncTarget[]; + /** Codex MCP (TOML). */ + codex: CodexMcpTarget; + /** Skill install directories, one per editor that supports skills. */ + skills: SkillTarget[]; + /** Hook registrations + their bundled script directories. */ + hooks: HookTarget[]; +} + +/** + * Resolve all editor targets for the given home directory. Defaults to + * `os.homedir()`; call sites pass it through so tests can point HOME at a temp + * dir. Paths are computed at call time (not module load) so a test setting + * `process.env.HOME` before invoking sees the right locations. + */ +export function getEditorTargets(home: string = os.homedir()): EditorTargets { + const mcpJsonc: McpJsoncTarget[] = [ + { + id: 'cursor', + label: 'Cursor', + file: path.join(home, '.cursor', 'mcp.json'), + keyPath: ['mcpServers', 'gitnexus'], + }, + { + id: 'claude', + label: 'Claude Code', + file: path.join(home, '.claude.json'), + keyPath: ['mcpServers', 'gitnexus'], + }, + { + id: 'antigravity', + label: 'Antigravity', + file: path.join(home, '.gemini', 'antigravity', 'mcp_config.json'), + keyPath: ['mcpServers', 'gitnexus'], + }, + { + id: 'opencode', + label: 'OpenCode', + file: path.join(home, '.config', 'opencode', 'opencode.json'), + // OpenCode nests servers under `mcp`, not `mcpServers`. + keyPath: ['mcp', 'gitnexus'], + }, + ]; + + const codex: CodexMcpTarget = { + id: 'codex', + label: 'Codex', + configFile: path.join(home, '.codex', 'config.toml'), + tomlSection: 'mcp_servers.gitnexus', + }; + + const skills: SkillTarget[] = [ + { id: 'claude', label: 'Claude Code', dir: path.join(home, '.claude', 'skills') }, + { + id: 'antigravity', + label: 'Antigravity', + dir: path.join(home, '.gemini', 'antigravity', 'skills'), + }, + { id: 'cursor', label: 'Cursor', dir: path.join(home, '.cursor', 'skills') }, + { id: 'opencode', label: 'OpenCode', dir: path.join(home, '.config', 'opencode', 'skills') }, + // Codex reads skills from ~/.agents/skills (not ~/.codex). + { id: 'codex', label: 'Codex', dir: path.join(home, '.agents', 'skills') }, + ]; + + const hooks: HookTarget[] = [ + { + id: 'claude', + label: 'Claude Code', + settingsFile: path.join(home, '.claude', 'settings.json'), + events: ['PreToolUse', 'PostToolUse'], + needle: 'gitnexus-hook', + scriptDir: path.join(home, '.claude', 'hooks', 'gitnexus'), + }, + { + id: 'antigravity', + label: 'Antigravity', + settingsFile: path.join(home, '.gemini', 'settings.json'), + events: ['AfterTool'], + needle: 'gitnexus-antigravity-hook', + scriptDir: path.join(home, '.gemini', 'config', 'hooks', 'gitnexus'), + }, + ]; + + return { mcpJsonc, codex, skills, hooks }; +} + +/** Look up a single JSONC MCP target by editor id (throws if unknown). */ +export function mcpTarget(id: EditorId, home?: string): McpJsoncTarget { + const t = getEditorTargets(home).mcpJsonc.find((m) => m.id === id); + if (!t) throw new Error(`No JSONC MCP target for editor "${id}"`); + return t; +} + +/** Look up a single skill target by editor id (throws if unknown). */ +export function skillTarget(id: EditorId, home?: string): SkillTarget { + const t = getEditorTargets(home).skills.find((s) => s.id === id); + if (!t) throw new Error(`No skill target for editor "${id}"`); + return t; +} + +/** Look up a single hook target by editor id (throws if unknown). */ +export function hookTarget(id: EditorId, home?: string): HookTarget { + const t = getEditorTargets(home).hooks.find((h) => h.id === id); + if (!t) throw new Error(`No hook target for editor "${id}"`); + return t; +} + +/** + * Detect indentation style from file content so JSONC edits preserve the file's + * existing formatting. Shared by setup (writes) and uninstall (removes). + */ +export function detectIndentation(raw: string): { tabSize: number; insertSpaces: boolean } { + const firstIndented = raw.match(/^( +|\t)/m); + if (!firstIndented) return { tabSize: 2, insertSpaces: true }; + if (firstIndented[1] === '\t') return { tabSize: 1, insertSpaces: false }; + return { tabSize: firstIndented[1].length, insertSpaces: true }; +} diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index f28225e0c..a3a058264 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -32,7 +32,11 @@ import http from 'http'; import { isIPv4, isIPv6 } from 'node:net'; import { writeSync } from 'node:fs'; -import { LocalBackend } from '../mcp/local/local-backend.js'; +import { + LocalBackend, + type RepoListing, + type ListReposPagination, +} from '../mcp/local/local-backend.js'; import { logger } from '../core/logger.js'; import { cliInfo, cliWarn, cliError } from './cli-message.js'; import { formatDetectChangesResult } from './detect-changes-format.js'; @@ -185,7 +189,47 @@ export function formatImpactResult(result: any): string { const byDepth = result.byDepth || {}; const total = result.impactedCount || 0; + // #2129 — an ambiguous bare name must not print the "isolated / safe to + // refactor" headline. Surface the per-candidate blast radius + the maximum, + // mirroring formatContextResult, so the real impact under whichever symbol the + // caller meant is visible on the text surface, not just in the JSON. + if (result.status === 'ambiguous') { + // #2129 review F11 — report the FULL match count (`totalCandidates`), not the + // truncated `candidates[]` length; note when the candidate list is capped. + const shown = result.candidates?.length ?? 0; + const total = result.totalCandidates ?? shown; + const countPhrase = total > shown ? `${total} symbols (showing ${shown})` : `${total} symbols`; + const lines = [ + `${target?.name || '?'}: AMBIGUOUS — ${countPhrase} share this name. ` + + `Max blast radius ${result.maxImpactedCount ?? 0} (${result.maxRisk ?? 'UNKNOWN'} risk). ` + + `Disambiguate with --uid for one authoritative result:`, + ]; + for (const c of result.candidates || []) { + lines.push( + ` ${c.kind} ${c.name} → ${c.filePath}:${c.line || '?'} ` + + `[${c.impactedCount ?? 0} ${direction}, risk ${c.risk ?? 'UNKNOWN'}] (uid: ${c.uid})`, + ); + } + // #2129 review F1 — a failed per-candidate probe makes the max a lower bound. + if (result.partialProbe) { + lines.push( + ' ⚠️ One or more candidate probes failed — max blast radius / risk are lower bounds.', + ); + } + return lines.join('\n'); + } + if (total === 0) { + // #1858 — "isolated" is a confident claim. If an interface / indirection + // boundary is on the path, the true count is a lower bound, not zero; + // callers binding via DI / dynamic dispatch were not traced. Say so instead. + if (result.epistemic === 'lower-bound') { + const lines = [ + `${target?.name || '?'}: no direct ${direction} dependencies traced, but this is a LOWER BOUND — unresolved indirection on the path (actual impact may be higher):`, + ]; + for (const b of result.boundaries || []) lines.push(` • ${b}`); + return lines.join('\n'); + } return `${target?.name || '?'}: No ${direction} dependencies found. This symbol appears isolated.`; } @@ -198,6 +242,14 @@ export function formatImpactResult(result: any): string { if (result.partial) { lines.push('⚠️ Partial results — graph traversal was interrupted. Deeper impacts may exist.'); } + // #1858 — an interface / indirection boundary on the path makes this a lower + // bound; surface it so the count is not read as exhaustive. + if (result.epistemic === 'lower-bound') { + lines.push( + '⚠️ Lower bound — unresolved indirection on the path (callers binding via DI / dynamic dispatch are not traced; actual impact may be higher):', + ); + for (const b of result.boundaries || []) lines.push(` • ${b}`); + } lines.push(''); const depthLabels: Record = { @@ -265,13 +317,22 @@ export function formatCypherResult(result: any): string { return typeof result === 'string' ? result : JSON.stringify(result, null, 2); } -export function formatListReposResult(result: any): string { - if (!Array.isArray(result) || result.length === 0) { - return 'No indexed repositories.'; +export function formatListReposResult(result: { + repositories: RepoListing[]; + pagination?: ListReposPagination; +}): string { + // `list_repos` always returns the paginated { repositories, pagination } object (#2119). + const repos = result.repositories; + const pg = result.pagination; + + if (repos.length === 0) { + return pg && pg.total > 0 + ? `No repositories on this page (offset ${pg.offset} of ${pg.total} total).` + : 'No indexed repositories.'; } const lines = ['Indexed repositories:\n']; - for (const r of result) { + for (const r of repos) { const stats = r.stats || {}; lines.push( ` ${r.name} — ${stats.nodes || '?'} symbols, ${stats.edges || '?'} relationships, ${stats.processes || '?'} flows`, @@ -279,6 +340,13 @@ export function formatListReposResult(result: any): string { lines.push(` Path: ${r.path}`); lines.push(` Indexed: ${r.indexedAt}`); } + if (pg) { + lines.push(''); + lines.push( + ` Showing ${repos.length} of ${pg.total} (offset ${pg.offset}).` + + (pg.hasMore ? ` More available — re-run with offset ${pg.nextOffset}.` : ''), + ); + } return lines.join('\n'); } @@ -325,6 +393,9 @@ function getNextStepHint(toolName: string): string { case 'detect_changes': return '\n---\nNext: Run gitnexus-context "" on high-risk changed symbols to check their callers.'; + case 'list_repos': + return '\n---\nNext: READ gitnexus://repo/{name}/context for a repo above. If pagination.hasMore is true, re-run list_repos with offset set to pagination.nextOffset to page through the rest.'; + default: return ''; } diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index 5fb42dbe1..9b4c9a823 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -12,6 +12,7 @@ const TITLE_KEYS = { const COMMAND_DESCRIPTION_KEYS = { '': 'help.description.root', setup: 'help.command.setup.description', + uninstall: 'help.command.uninstall.description', analyze: 'help.command.analyze.description', index: 'help.command.index.description', serve: 'help.command.serve.description', @@ -69,8 +70,10 @@ const OPTION_DESCRIPTION_KEYS = { 'index|--allow-non-git': 'help.option.index.allowNonGit', 'serve|-p, --port ': 'help.option.port', 'serve|--host ': 'help.option.serve.host', + 'uninstall|-f, --force': 'help.option.uninstall.force', 'clean|-f, --force': 'help.option.force.confirmation', 'clean|--all': 'help.option.clean.all', + 'clean|--branch ': 'help.option.clean.branch', 'clean|--lbug-sidecars': 'help.option.clean.lbugSidecars', 'remove|-f, --force': 'help.option.force.confirmation', 'wiki|-f, --force': 'help.option.wiki.force', @@ -91,16 +94,19 @@ const OPTION_DESCRIPTION_KEYS = { 'publish|--id ': 'help.option.publish.id', 'publish|--skip-git': 'help.option.skipGit', 'query|-r, --repo ': 'help.option.repo.targetOmitOne', + 'query|--branch ': 'help.option.branch', 'query|-c, --context ': 'help.option.query.context', 'query|-g, --goal ': 'help.option.query.goal', 'query|-l, --limit ': 'help.option.query.limit', 'query|--content': 'help.option.content', 'context|-r, --repo ': 'help.option.repo.target', + 'context|--branch ': 'help.option.branch', 'context|-u, --uid ': 'help.option.context.uid', 'context|-f, --file ': 'help.option.context.file', 'context|--content': 'help.option.content', 'impact|-d, --direction ': 'help.option.impact.direction', 'impact|-r, --repo ': 'help.option.repo.target', + 'impact|--branch ': 'help.option.branch', 'impact|-u, --uid ': 'help.option.context.uid', 'impact|-f, --file ': 'help.option.context.file', 'impact|--kind ': 'help.option.impact.kind', @@ -110,9 +116,11 @@ const OPTION_DESCRIPTION_KEYS = { 'impact|--offset ': 'help.option.impact.offset', 'impact|--summary-only': 'help.option.impact.summaryOnly', 'cypher|-r, --repo ': 'help.option.repo.target', + 'cypher|--branch ': 'help.option.branch', 'detect-changes|-s, --scope ': 'help.option.detectChanges.scope', 'detect-changes|-b, --base-ref ': 'help.option.detectChanges.baseRef', 'detect-changes|-r, --repo ': 'help.option.repo.target', + 'detect-changes|--branch ': 'help.option.branch', 'eval-server|-p, --port ': 'help.option.port', 'eval-server|--host ': 'help.option.evalServer.host', 'eval-server|--idle-timeout ': 'help.option.evalServer.idleTimeout', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index e4778fb58..0be54ebf6 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -10,6 +10,9 @@ export const en = { 'list.title': 'Indexed Repositories ({{count}})', 'list.indexed': 'Indexed', 'list.commit': 'Commit', + 'list.branch': 'Branch', + 'list.branchIndexes': 'Branch indexes', + 'list.branchLine': '{{branch}} ({{commit}}, {{indexed}})', 'list.stats': 'Stats', 'list.statsValue': '{{files}} files, {{symbols}} symbols, {{edges}} edges', 'list.clusters': 'Clusters', @@ -23,6 +26,10 @@ export const en = { 'status.indexed': 'Indexed', 'status.indexedCommit': 'Indexed commit', 'status.currentCommit': 'Current commit', + 'status.branch': 'Branch', + 'status.detached': '(detached HEAD)', + 'status.branchNotIndexed': + "⚠️ current branch not indexed (primary index is for '{{primary}}'; run gitnexus analyze)", 'status.status': 'Status', 'status.upToDate': '✅ up-to-date', 'status.stale': '⚠️ stale (re-run gitnexus analyze)', @@ -30,6 +37,9 @@ export const en = { 'clean.deletedRepo': 'Deleted: {{name}} ({{storagePath}})', 'clean.notFoundHere': 'No indexed repository found in this directory.', 'clean.deleteCurrent': 'This will delete the GitNexus index for: {{repoName}}', + 'clean.branchNotIndexed': 'No indexed branch named "{{branch}}" for this repository.', + 'clean.deleteBranch': 'This will delete the branch index "{{branch}}" at: {{path}}', + 'clean.deletedBranch': 'Deleted branch index: {{branch}}', 'clean.lbugSidecars.state': 'LadybugDB sidecar state: {{state}}', 'clean.lbugSidecars.none': 'No quarantined LadybugDB missing-shadow WAL sidecars found.', 'clean.lbugSidecars.preview': @@ -106,6 +116,8 @@ export const en = { 'help.option.version': 'output the version number', 'help.command.setup.description': 'One-time setup: configure MCP for Cursor, Claude Code, OpenCode, Codex', + 'help.command.uninstall.description': + 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', 'help.command.analyze.description': 'Index a repository (full analysis)', 'help.command.index.description': 'Register an existing .gitnexus/ folder into the global registry (no re-analysis needed)', @@ -175,7 +187,7 @@ export const en = { 'help.option.analyze.walCheckpointThreshold': 'LadybugDB WAL auto-checkpoint threshold in bytes during analyze (integer >= -1; default: 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).', 'help.option.analyze.workers': - 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).', + 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', 'help.option.analyze.embeddingThreads': 'Limit local ONNX embedding CPU threads', 'help.option.analyze.embeddingBatchSize': 'Number of nodes per embedding batch', 'help.option.analyze.embeddingSubBatchSize': 'Number of chunks per embedding model call', @@ -185,11 +197,13 @@ export const en = { 'help.option.port': 'Port number', 'help.option.serve.host': 'Bind address (default: 127.0.0.1, use 0.0.0.0 for remote access)', 'help.option.force.confirmation': 'Skip confirmation prompt', + 'help.option.uninstall.force': 'Apply the changes (default is a dry-run preview)', 'help.option.clean.all': 'Clean all indexed repos', + 'help.option.clean.branch': 'Delete only the named branch index (not the primary)', 'help.option.clean.lbugSidecars': 'Clean quarantined LadybugDB missing-shadow WAL sidecars', 'help.option.wiki.force': 'Force full regeneration even if up to date', 'help.option.wiki.provider': - 'LLM provider: openai, openrouter, azure, custom, cursor, claude, or codex (default: openai)', + 'LLM provider: openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: openai)', 'help.option.wiki.model': 'LLM model or Azure deployment name (default: minimax/minimax-m2.5)', 'help.option.wiki.baseUrl': 'LLM API base URL. Azure v1: https://{resource}.openai.azure.com/openai/v1', @@ -214,6 +228,7 @@ export const en = { 'help.option.query.limit': 'Max processes to return (default: 5)', 'help.option.content': 'Include full symbol source code', 'help.option.repo.target': 'Target repository', + 'help.option.branch': 'Scope to a specific branch index (multi-branch repos)', 'help.option.context.uid': 'Direct symbol UID (zero-ambiguity lookup)', 'help.option.context.file': 'File path to disambiguate common names', 'help.option.impact.kind': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index cd488de0a..8b87ee567 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -14,6 +14,9 @@ export const zhCN = { 'list.title': '已索引仓库（{{count}}）', 'list.indexed': '索引时间', 'list.commit': '提交', + 'list.branch': '分支', + 'list.branchIndexes': '分支索引', + 'list.branchLine': '{{branch}}（{{commit}}，{{indexed}}）', 'list.stats': '统计', 'list.statsValue': '{{files}} 个文件，{{symbols}} 个符号，{{edges}} 条边', 'list.clusters': '聚类', @@ -27,6 +30,10 @@ export const zhCN = { 'status.indexed': '索引时间', 'status.indexedCommit': '索引提交', 'status.currentCommit': '当前提交', + 'status.branch': '分支', + 'status.detached': '（分离 HEAD）', + 'status.branchNotIndexed': + "⚠️ 当前分支未索引（主索引对应 '{{primary}}'；请运行 gitnexus analyze）", 'status.status': '状态', 'status.upToDate': '✅ 已是最新', 'status.stale': '⚠️ 已过期（重新运行 gitnexus analyze）', @@ -34,6 +41,9 @@ export const zhCN = { 'clean.deletedRepo': '已删除：{{name}}（{{storagePath}}）', 'clean.notFoundHere': '当前目录未找到已索引仓库。', 'clean.deleteCurrent': '将删除该仓库的 GitNexus 索引：{{repoName}}', + 'clean.branchNotIndexed': '该仓库没有名为 “{{branch}}” 的已索引分支。', + 'clean.deleteBranch': '将删除分支索引 “{{branch}}”，路径：{{path}}', + 'clean.deletedBranch': '已删除分支索引：{{branch}}', 'clean.lbugSidecars.state': 'LadybugDB sidecar 状态：{{state}}', 'clean.lbugSidecars.none': '未找到已隔离的 LadybugDB missing-shadow WAL sidecar。', 'clean.lbugSidecars.preview': @@ -108,6 +118,8 @@ export const zhCN = { 'help.option.help': '显示命令帮助', 'help.option.version': '输出版本号', 'help.command.setup.description': '一次性设置：为 Cursor、Claude Code、OpenCode、Codex 配置 MCP', + 'help.command.uninstall.description': + '撤销 `setup`：从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', 'help.command.analyze.description': '索引仓库（完整分析）', 'help.command.index.description': '将现有 .gitnexus/ 文件夹注册到全局注册表（无需重新分析）', 'help.command.serve.description': '启动供 Web UI 连接的本地 HTTP 服务器', @@ -164,7 +176,7 @@ export const zhCN = { 'help.option.analyze.walCheckpointThreshold': 'analyze 期间 LadybugDB WAL 自动 checkpoint 阈值（字节，整数 >= -1；默认：67108864 = 64 MiB；-1 保持 Ladybug 默认约 16 MiB）。', 'help.option.analyze.workers': - '解析 worker 池大小。默认：cores-1，最多 16。传 0 禁用 worker（顺序执行）。', + '解析 worker 池大小（>=1）。默认：cores-1，最多 16，按仓库规模自适应。', 'help.option.analyze.embeddingThreads': '限制本地 ONNX 嵌入 CPU 线程数', 'help.option.analyze.embeddingBatchSize': '每个嵌入批次的节点数', 'help.option.analyze.embeddingSubBatchSize': '每次嵌入模型调用的分块数', @@ -174,11 +186,13 @@ export const zhCN = { 'help.option.port': '端口号', 'help.option.serve.host': '绑定地址（默认：127.0.0.1；远程访问可用 0.0.0.0）', 'help.option.force.confirmation': '跳过确认提示', + 'help.option.uninstall.force': '应用更改（默认仅为预演预览）', 'help.option.clean.all': '清理所有已索引仓库', + 'help.option.clean.branch': '仅删除指定分支的索引（不影响主索引）', 'help.option.clean.lbugSidecars': '清理已隔离的 LadybugDB missing-shadow WAL sidecar', 'help.option.wiki.force': '即使已是最新也强制完整重新生成', 'help.option.wiki.provider': - 'LLM 提供商：openai、openrouter、azure、custom、cursor、claude 或 codex（默认：openai）', + 'LLM 提供商：openai、openrouter、azure、custom、cursor、claude、codex 或 opencode（默认：openai）', 'help.option.wiki.model': 'LLM 模型或 Azure deployment 名称（默认：minimax/minimax-m2.5）', 'help.option.wiki.baseUrl': 'LLM API base URL。Azure v1：https://{resource}.openai.azure.com/openai/v1', @@ -200,6 +214,7 @@ export const zhCN = { 'help.option.query.limit': '最多返回的流程数（默认：5）', 'help.option.content': '包含完整符号源码', 'help.option.repo.target': '目标仓库', + 'help.option.branch': '将查询限定到指定分支的索引（多分支仓库）', 'help.option.context.uid': '直接符号 UID（零歧义查找）', 'help.option.context.file': '用于消除常见名称歧义的文件路径', 'help.option.impact.kind': '用于消除常见名称歧义的类型过滤（如 Function、Class、Method）', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index f64b3a24d..ffe6a6483 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -23,6 +23,14 @@ program ) .action(createLazyAction(() => import('./setup.js'), 'setupCommand')); +program + .command('uninstall') + .description( + 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', + ) + .option('-f, --force', 'Apply the changes (default is a dry-run preview)') + .action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand')); + program .command('analyze [path]') .description('Index a repository (full analysis)') @@ -44,11 +52,22 @@ program '(no-op when --index-only is also set).', ) .option('--skip-agents-md', 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md') + .option( + '--pdg', + 'Build the control-flow-graph / PDG substrate (BasicBlock nodes + CFG edges) ' + + 'for supported languages. Opt-in; off by default. (#2081 M1)', + ) .option( '--default-branch ', 'Default branch used in the generated regression-compare example (base_ref). ' + 'Falls back to .gitnexusrc, then auto-detected origin/HEAD, then "main".', ) + .option( + '--branch ', + 'Index the working tree under a specific branch slot (multi-branch indexing). ' + + 'Defaults to the checked-out branch; the primary/first-indexed branch keeps the ' + + 'flat index and others get their own. Distinct from --default-branch (cosmetic base_ref).', + ) .option('--no-stats', 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md') .option( '--skip-skills', @@ -87,7 +106,7 @@ program ) .option( '--workers ', - 'Parse worker pool size. Default: cores-1 capped at 16. Pass 0 to disable workers (sequential).', + 'Parse worker pool size (>=1). Default: cores-1 capped at 16, auto-sized to the repo.', ) .option('--embedding-threads ', 'Limit local ONNX embedding CPU threads') .option('--embedding-batch-size ', 'Number of nodes per embedding batch') @@ -137,6 +156,7 @@ program .description('Delete GitNexus index for current repo') .option('-f, --force', 'Skip confirmation prompt') .option('--all', 'Clean all indexed repos') + .option('--branch ', 'Delete only the named branch index (not the primary)') .option('--lbug-sidecars', 'Clean quarantined LadybugDB missing-shadow WAL sidecars') .action(createLazyAction(() => import('./clean.js'), 'cleanCommand')); @@ -155,7 +175,7 @@ program .option('-f, --force', 'Force full regeneration even if up to date') .option( '--provider ', - 'LLM provider: openai, openrouter, azure, custom, cursor, claude, or codex (default: openai)', + 'LLM provider: openai, openrouter, azure, custom, cursor, claude, codex, or opencode (default: openai)', ) .option('--model ', 'LLM model or Azure deployment name (default: minimax/minimax-m2.5)') .option( @@ -208,6 +228,7 @@ program .command('query ') .description('Search the knowledge graph for execution flows related to a concept') .option('-r, --repo ', 'Target repository (omit if only one indexed)') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .option('-c, --context ', 'Task context to improve ranking') .option('-g, --goal ', 'What you want to find') .option('-l, --limit ', 'Max processes to return (default: 5)') @@ -218,6 +239,7 @@ program .command('context [name]') .description('360-degree view of a code symbol: callers, callees, processes') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .option('-u, --uid ', 'Direct symbol UID (zero-ambiguity lookup)') .option('-f, --file ', 'File path to disambiguate common names') .option('--content', 'Include full symbol source code') @@ -228,6 +250,7 @@ program .description('Blast radius analysis: what breaks if you change a symbol') .option('-d, --direction ', 'upstream (dependants) or downstream (dependencies)', 'upstream') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .option('-u, --uid ', 'Direct symbol UID (zero-ambiguity lookup)') .option('-f, --file ', 'File path to disambiguate common names') .option( @@ -245,6 +268,7 @@ program .command('cypher ') .description('Execute raw Cypher query against the knowledge graph') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .action(createLbugLazyAction(() => import('./tool.js'), 'cypherCommand')); program @@ -254,6 +278,7 @@ program .option('-s, --scope ', 'What to analyze: unstaged, staged, all, or compare', 'unstaged') .option('-b, --base-ref ', 'Branch/commit for compare scope (e.g. main)') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .action(createLbugLazyAction(() => import('./tool.js'), 'detectChangesCommand')); // ─── Eval Server (persistent daemon for SWE-bench) ───────────────── diff --git a/gitnexus/src/cli/list.ts b/gitnexus/src/cli/list.ts index 20214e8a8..223b662f0 100644 --- a/gitnexus/src/cli/list.ts +++ b/gitnexus/src/cli/list.ts @@ -38,6 +38,7 @@ export const listCommand = async () => { console.log(` ${t('common.path')}: ${entry.path}`); console.log(` ${t('list.indexed')}: ${indexedDate}`); console.log(` ${t('list.commit')}: ${commitShort}`); + if (entry.branch) console.log(` ${t('list.branch')}: ${entry.branch}`); console.log( ` ${t('list.stats')}: ${t('list.statsValue', { files: stats.files ?? 0, @@ -47,6 +48,18 @@ export const listCommand = async () => { ); if (stats.communities) console.log(` ${t('list.clusters')}: ${stats.communities}`); if (stats.processes) console.log(` ${t('list.processes')}: ${stats.processes}`); + // Per-branch indexes (#2106). Only rendered when extra branches were + // indexed for this path, so single-branch output is unchanged. + if (entry.branches && entry.branches.length > 0) { + console.log(` ${t('list.branchIndexes')}:`); + for (const b of entry.branches) { + const bCommit = b.lastCommit?.slice(0, 7) || t('list.unknown'); + const bIndexed = new Date(b.indexedAt).toLocaleString(); + console.log( + ` ${t('list.branchLine', { branch: b.branch, commit: bCommit, indexed: bIndexed })}`, + ); + } + } console.log(''); } }; diff --git a/gitnexus/src/cli/optional-grammars.ts b/gitnexus/src/cli/optional-grammars.ts index e12b471e0..736ee6628 100644 --- a/gitnexus/src/cli/optional-grammars.ts +++ b/gitnexus/src/cli/optional-grammars.ts @@ -1,43 +1,81 @@ /** * Optional grammar availability check. * - * tree-sitter-dart, tree-sitter-proto, and tree-sitter-swift are vendored - * under vendor/ and materialized into node_modules/ at postinstall. Dart - * and Proto are built from source with node-gyp; Swift ships platform - * prebuilds activated via node-gyp-build. All three can be skipped via + * tree-sitter-dart, -proto, -swift, and -kotlin are vendored under vendor/ and + * loaded from there by absolute path (NEVER copied into node_modules — see + * core/tree-sitter/vendored-grammars.ts / #2111). Each ships committed platform + * prebuilds activated via node-gyp-build. All can be skipped via * GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (postinstall scripts), or can silently - * soft-fail when the toolchain is missing (Dart/Proto) or no prebuild - * matches the host platform (Swift). + * soft-fail when no prebuild matches the host platform (and a source build was + * unavailable / not attempted). * * Either path produces the same observable: the .node binding is absent * at runtime. This helper detects that condition and surfaces a single - * stderr line per missing grammar so users learn why .dart/.proto/.swift + * stderr line per missing grammar so users learn why .dart/.proto/.swift/.kt * support is unavailable instead of silently getting a degraded index. */ -import { createRequire } from 'module'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { isGrammarRuntimeSkipped } from '../core/tree-sitter/parser-loader.js'; +import { requireVendoredGrammar } from '../core/tree-sitter/vendored-grammars.js'; import { cliWarn } from './cli-message.js'; -const _require = createRequire(import.meta.url); - interface OptionalGrammar { /** Display name in warnings */ name: string; - /** Module name to require.resolve */ + /** Vendored grammar package name (directory under vendor/) */ pkg: string; /** File extensions this grammar parses */ extensions: string[]; + /** + * SupportedLanguages id, when this grammar backs an ingestion language. + * Used to ask `isGrammarRuntimeSkipped` whether the grammar was disabled via + * `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` (vs. genuinely missing). Omitted for + * `.proto`, which is a gRPC-extractor concern, not a SupportedLanguages. + */ + language?: SupportedLanguages; } const OPTIONAL_GRAMMARS: OptionalGrammar[] = [ - { name: 'tree-sitter-dart', pkg: 'tree-sitter-dart', extensions: ['.dart'] }, + { + name: 'tree-sitter-dart', + pkg: 'tree-sitter-dart', + extensions: ['.dart'], + language: SupportedLanguages.Dart, + }, { name: 'tree-sitter-proto', pkg: 'tree-sitter-proto', extensions: ['.proto'] }, - { name: 'tree-sitter-swift', pkg: 'tree-sitter-swift', extensions: ['.swift'] }, + { + name: 'tree-sitter-swift', + pkg: 'tree-sitter-swift', + extensions: ['.swift'], + language: SupportedLanguages.Swift, + }, + { + name: 'tree-sitter-kotlin', + pkg: 'tree-sitter-kotlin', + extensions: ['.kt', '.kts'], + language: SupportedLanguages.Kotlin, + }, ]; +/** + * The file extensions backed by an optional grammar — the single source for + * the `analyze` preflight glob (so the glob can't drift from this list). + */ +export function getOptionalGrammarExtensions(): string[] { + return [...new Set(OPTIONAL_GRAMMARS.flatMap((g) => g.extensions))]; +} + export interface MissingGrammar { name: string; extensions: string[]; + /** + * `missing` — the native binding could not be loaded (not installed / build + * soft-failed / no prebuild). `skipped` — the binding is fine but the user + * disabled it via `GITNEXUS_SKIP_OPTIONAL_GRAMMARS`. Drives the warning text + * so a deliberate opt-out is not told to reinstall. + */ + reason: 'missing' | 'skipped'; } /** @@ -59,8 +97,15 @@ export interface MissingGrammar { export function detectMissingOptionalGrammars(): MissingGrammar[] { const missing: MissingGrammar[] = []; for (const g of OPTIONAL_GRAMMARS) { + // Deliberate runtime opt-out comes first: even an installed binding is + // treated as unavailable, with a `skipped` reason so the warning says so + // instead of suggesting a reinstall (#2101 review). + if (g.language !== undefined && isGrammarRuntimeSkipped(g.language)) { + missing.push({ name: g.name, extensions: g.extensions, reason: 'skipped' }); + continue; + } try { - _require(g.pkg); + requireVendoredGrammar(g.pkg); } catch (err) { const code = (err as NodeJS.ErrnoException | undefined)?.code; const msg = err instanceof Error ? err.message : String(err); @@ -80,7 +125,7 @@ export function detectMissingOptionalGrammars(): MissingGrammar[] { { grammar: g.name, extensions: g.extensions, error: msg }, ); } - missing.push({ name: g.name, extensions: g.extensions }); + missing.push({ name: g.name, extensions: g.extensions, reason: 'missing' }); } } return missing; @@ -110,9 +155,16 @@ export function warnMissingOptionalGrammars(opts?: { if (relevantExtensions && !g.extensions.some((e) => relevantExtensions.has(e))) { continue; } - cliWarn( - `GitNexus${ctx}: optional grammar "${g.name}" is unavailable — ${g.extensions.join('/')} files will not be parsed. Reinstall without GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (and ensure python3, make, g++) to enable.`, - { grammar: g.name, extensions: g.extensions, context: opts?.context }, - ); + const exts = g.extensions.join('/'); + const message = + g.reason === 'skipped' + ? `GitNexus${ctx}: optional grammar "${g.name}" is disabled via GITNEXUS_SKIP_OPTIONAL_GRAMMARS — ${exts} files will not be parsed. Unset the variable to re-enable.` + : `GitNexus${ctx}: optional grammar "${g.name}" is unavailable — ${exts} files will not be parsed. Reinstall without GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (and ensure python3, make, g++) to enable.`; + cliWarn(message, { + grammar: g.name, + extensions: g.extensions, + reason: g.reason, + context: opts?.context, + }); } } diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index 0aa112e6a..907b7f885 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -15,6 +15,13 @@ import { promisify } from 'util'; import { fileURLToPath } from 'url'; import { parseTree, modify, applyEdits, ParseError, parse as parseJsonc } from 'jsonc-parser'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { + getEditorTargets, + mcpTarget, + skillTarget, + hookTarget, + detectIndentation, +} from './editor-targets.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -162,17 +169,6 @@ function getOpenCodeMcpEntry() { return { type: 'local', command: ['npx', '-y', MCP_PINNED_REF, 'mcp'] }; } -/** - * Detect indentation style from file content. - * Returns formatting options matching the file's existing style. - */ -function detectIndentation(raw: string): { tabSize: number; insertSpaces: boolean } { - const firstIndented = raw.match(/^( +|\t)/m); - if (!firstIndented) return { tabSize: 2, insertSpaces: true }; - if (firstIndented[1] === '\t') return { tabSize: 1, insertSpaces: false }; - return { tabSize: firstIndented[1].length, insertSpaces: true }; -} - /** * Merge a key/value pair into a JSONC config file, preserving comments and formatting. * If the file is genuinely corrupt (not valid JSONC), leaves it untouched. @@ -233,9 +229,9 @@ async function setupCursor(result: SetupResult): Promise { return; } - const mcpPath = path.join(cursorDir, 'mcp.json'); + const { file: mcpPath, keyPath } = mcpTarget('cursor'); try { - const ok = await mergeJsoncFile(mcpPath, ['mcpServers', 'gitnexus'], getMcpEntry()); + const ok = await mergeJsoncFile(mcpPath, keyPath, getMcpEntry()); if (ok) { result.configured.push('Cursor'); } else { @@ -254,9 +250,9 @@ async function setupClaudeCode(result: SetupResult): Promise { } // Claude Code stores MCP config in ~/.claude.json - const mcpPath = path.join(os.homedir(), '.claude.json'); + const { file: mcpPath, keyPath } = mcpTarget('claude'); try { - const ok = await mergeJsoncFile(mcpPath, ['mcpServers', 'gitnexus'], getMcpEntry()); + const ok = await mergeJsoncFile(mcpPath, keyPath, getMcpEntry()); if (ok) { result.configured.push('Claude Code'); } else { @@ -276,7 +272,7 @@ async function installClaudeCodeSkills(result: SetupResult): Promise { const claudeDir = path.join(os.homedir(), '.claude'); if (!(await dirExists(claudeDir))) return; - const skillsDir = path.join(claudeDir, 'skills'); + const skillsDir = skillTarget('claude').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -422,13 +418,14 @@ async function installClaudeCodeHooks(result: SetupResult): Promise { const claudeDir = path.join(os.homedir(), '.claude'); if (!(await dirExists(claudeDir))) return; - const settingsPath = path.join(claudeDir, 'settings.json'); + const claudeHook = hookTarget('claude'); + const settingsPath = claudeHook.settingsFile; // Source hooks bundled within the gitnexus package (hooks/claude/) const pluginHooksPath = path.join(__dirname, '..', '..', 'hooks', 'claude'); // Copy unified hook script to ~/.claude/hooks/gitnexus/ - const destHooksDir = path.join(claudeDir, 'hooks', 'gitnexus'); + const destHooksDir = claudeHook.scriptDir; try { await fs.mkdir(destHooksDir, { recursive: true }); @@ -494,7 +491,7 @@ async function installClaudeCodeHooks(result: SetupResult): Promise { // NOTE: SessionStart hooks are broken on Windows (Claude Code bug #23576). // Session context is delivered via CLAUDE.md / skills instead. - if (!hasGitnexusHook(parsed?.hooks, 'PreToolUse')) { + if (!hasGitnexusHook(parsed?.hooks, 'PreToolUse', claudeHook.needle)) { hookEntries.push({ eventName: 'PreToolUse', value: { @@ -510,7 +507,7 @@ async function installClaudeCodeHooks(result: SetupResult): Promise { }, }); } - if (!hasGitnexusHook(parsed?.hooks, 'PostToolUse')) { + if (!hasGitnexusHook(parsed?.hooks, 'PostToolUse', claudeHook.needle)) { hookEntries.push({ eventName: 'PostToolUse', value: { @@ -566,9 +563,9 @@ async function setupAntigravity(result: SetupResult): Promise { return; } - const mcpPath = path.join(antigravityDir, 'mcp_config.json'); + const { file: mcpPath, keyPath } = mcpTarget('antigravity'); try { - const ok = await mergeJsoncFile(mcpPath, ['mcpServers', 'gitnexus'], getMcpEntry()); + const ok = await mergeJsoncFile(mcpPath, keyPath, getMcpEntry()); if (ok) { result.configured.push('Antigravity'); } else { @@ -590,7 +587,7 @@ async function installAntigravitySkills(result: SetupResult): Promise { const antigravityDir = path.join(os.homedir(), '.gemini', 'antigravity'); if (!(await dirExists(antigravityDir))) return; - const skillsDir = path.join(antigravityDir, 'skills'); + const skillsDir = skillTarget('antigravity').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -618,9 +615,9 @@ async function installAntigravityHooks(result: SetupResult): Promise { const antigravityDir = path.join(os.homedir(), '.gemini', 'antigravity'); if (!(await dirExists(antigravityDir))) return; - const geminiDir = path.join(os.homedir(), '.gemini'); - const settingsPath = path.join(geminiDir, 'settings.json'); - const destHooksDir = path.join(geminiDir, 'config', 'hooks', 'gitnexus'); + const antigravityHook = hookTarget('antigravity'); + const settingsPath = antigravityHook.settingsFile; + const destHooksDir = antigravityHook.scriptDir; // The antigravity adapter shares its lock/probe helpers with the claude // adapter — same DB, same concurrency rules — so we reuse those CJS files @@ -694,7 +691,7 @@ async function installAntigravityHooks(result: SetupResult): Promise { const hookEntries: Array<{ eventName: string; value: unknown }> = []; - if (!hasGitnexusHook(parsed?.hooks, 'AfterTool', 'gitnexus-antigravity-hook')) { + if (!hasGitnexusHook(parsed?.hooks, 'AfterTool', antigravityHook.needle)) { // Matcher follows the Gemini CLI built-in tool naming (snake_case). // search_file_content / glob cover content + filename search; run_shell_command // catches rg/grep invocations and the git commit family for stale-index hints. @@ -742,9 +739,9 @@ async function setupOpenCode(result: SetupResult): Promise { return; } - const configPath = path.join(opencodeDir, 'opencode.json'); + const { file: configPath, keyPath } = mcpTarget('opencode'); try { - const ok = await mergeJsoncFile(configPath, ['mcp', 'gitnexus'], getOpenCodeMcpEntry()); + const ok = await mergeJsoncFile(configPath, keyPath, getOpenCodeMcpEntry()); if (ok) { result.configured.push('OpenCode'); } else { @@ -764,7 +761,7 @@ function getCodexMcpTomlSection(): string { const entry = getMcpEntry(); const command = JSON.stringify(entry.command); const args = `[${entry.args.map((arg) => JSON.stringify(arg)).join(', ')}]`; - return `[mcp_servers.gitnexus]\ncommand = ${command}\nargs = ${args}\n`; + return `[${getEditorTargets().codex.tomlSection}]\ncommand = ${command}\nargs = ${args}\n`; } /** @@ -778,7 +775,7 @@ async function upsertCodexConfigToml(configPath: string): Promise { existing = ''; } - if (existing.includes('[mcp_servers.gitnexus]')) { + if (existing.includes(`[${getEditorTargets().codex.tomlSection}]`)) { return; } @@ -809,7 +806,7 @@ async function setupCodex(result: SetupResult): Promise { } try { - const configPath = path.join(codexDir, 'config.toml'); + const configPath = getEditorTargets().codex.configFile; await upsertCodexConfigToml(configPath); result.configured.push('Codex (MCP added to ~/.codex/config.toml)'); } catch (err: any) { @@ -920,7 +917,7 @@ async function installCursorSkills(result: SetupResult): Promise { const cursorDir = path.join(os.homedir(), '.cursor'); if (!(await dirExists(cursorDir))) return; - const skillsDir = path.join(cursorDir, 'skills'); + const skillsDir = skillTarget('cursor').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -938,7 +935,7 @@ async function installOpenCodeSkills(result: SetupResult): Promise { const opencodeDir = path.join(os.homedir(), '.config', 'opencode'); if (!(await dirExists(opencodeDir))) return; - const skillsDir = path.join(opencodeDir, 'skills'); + const skillsDir = skillTarget('opencode').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -958,7 +955,7 @@ async function installCodexSkills(result: SetupResult): Promise { const codexDir = path.join(os.homedir(), '.codex'); if (!(await dirExists(codexDir))) return; - const skillsDir = path.join(os.homedir(), '.agents', 'skills'); + const skillsDir = skillTarget('codex').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { diff --git a/gitnexus/src/cli/skill-gen.ts b/gitnexus/src/cli/skill-gen.ts index e4e66e85b..f2fb42838 100644 --- a/gitnexus/src/cli/skill-gen.ts +++ b/gitnexus/src/cli/skill-gen.ts @@ -649,9 +649,9 @@ const renderSkillMarkdown = ( : community.label; lines.push('## How to Explore'); lines.push(''); - lines.push(`1. \`gitnexus_context({name: "${firstEntry}"})\` \u2014 see callers and callees`); + lines.push(`1. \`context({name: "${firstEntry}"})\` \u2014 see callers and callees`); lines.push( - `2. \`gitnexus_query({query: "${community.label.toLowerCase()}"})\` \u2014 find related execution flows`, + `2. \`query({query: "${community.label.toLowerCase()}"})\` \u2014 find related execution flows`, ); lines.push('3. Read key files listed above for implementation details'); lines.push(''); diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index c56b027d1..89ff9697d 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -4,8 +4,9 @@ * Shows the indexing status of the current repository. */ -import { findRepo, getStoragePaths, hasKuzuIndex } from '../storage/repo-manager.js'; -import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js'; +import path from 'path'; +import { findRepo, getStoragePaths, loadMeta, hasKuzuIndex } from '../storage/repo-manager.js'; +import { getCurrentCommit, getCurrentBranch, isGitRepo, getGitRoot } from '../storage/git.js'; import { t } from './i18n/index.js'; export const statusCommand = async () => { @@ -32,11 +33,34 @@ export const statusCommand = async () => { } const currentCommit = getCurrentCommit(repo.repoPath); - const isUpToDate = currentCommit === repo.meta.lastCommit; + const currentBranch = getCurrentBranch(repo.repoPath); + + // Pick the index matching the checked-out branch (#2106). The flat index + // belongs to the primary branch (repo.meta.branch); when the current branch + // differs and has its own index, report that one. Legacy/no-branch metas and + // detached HEAD fall through to the flat index (unchanged behavior). + let activeMeta = repo.meta; + let currentBranchIndexed = true; + if (currentBranch && repo.meta.branch && currentBranch !== repo.meta.branch) { + const { metaPath } = getStoragePaths(repo.repoPath, currentBranch); + const branchMeta = await loadMeta(path.dirname(metaPath)); + if (branchMeta) activeMeta = branchMeta; + else currentBranchIndexed = false; + } console.log(`${t('status.repository')}: ${repo.repoPath}`); - console.log(`${t('status.indexed')}: ${new Date(repo.meta.indexedAt).toLocaleString()}`); - console.log(`${t('status.indexedCommit')}: ${repo.meta.lastCommit?.slice(0, 7)}`); + console.log(`${t('status.branch')}: ${currentBranch ?? t('status.detached')}`); + + if (!currentBranchIndexed) { + console.log( + `${t('status.status')}: ${t('status.branchNotIndexed', { primary: repo.meta.branch ?? '' })}`, + ); + return; + } + + const isUpToDate = currentCommit === activeMeta.lastCommit; + console.log(`${t('status.indexed')}: ${new Date(activeMeta.indexedAt).toLocaleString()}`); + console.log(`${t('status.indexedCommit')}: ${activeMeta.lastCommit?.slice(0, 7)}`); console.log(`${t('status.currentCommit')}: ${currentCommit?.slice(0, 7)}`); console.log(`${t('status.status')}: ${isUpToDate ? t('status.upToDate') : t('status.stale')}`); }; diff --git a/gitnexus/src/cli/tool.ts b/gitnexus/src/cli/tool.ts index 5801677b9..267482382 100644 --- a/gitnexus/src/cli/tool.ts +++ b/gitnexus/src/cli/tool.ts @@ -62,6 +62,7 @@ export async function queryCommand( queryText: string, options?: { repo?: string; + branch?: string; context?: string; goal?: string; limit?: string; @@ -81,6 +82,7 @@ export async function queryCommand( limit: options?.limit ? parseInt(options.limit) : undefined, include_content: options?.content ?? false, repo: options?.repo, + branch: options?.branch, }); output(result); } @@ -89,6 +91,7 @@ export async function contextCommand( name: string, options?: { repo?: string; + branch?: string; file?: string; uid?: string; content?: boolean; @@ -111,6 +114,7 @@ export async function contextCommand( file_path: options?.file, include_content: options?.content ?? false, repo: options?.repo, + branch: options?.branch, }); output(result); } @@ -120,6 +124,7 @@ export async function impactCommand( options?: { direction?: string; repo?: string; + branch?: string; uid?: string; file?: string; kind?: string; @@ -165,6 +170,7 @@ export async function impactCommand( maxDepth: options?.depth ? parseInt(options.depth, 10) : undefined, includeTests: options?.includeTests ?? false, repo: options?.repo, + branch: options?.branch, limit: parsedLimit, offset: parsedOffset, summaryOnly: options?.summaryOnly ?? undefined, @@ -188,6 +194,7 @@ export async function cypherCommand( query: string, options?: { repo?: string; + branch?: string; }, ): Promise { if (!query?.trim()) { @@ -199,6 +206,7 @@ export async function cypherCommand( const result = await backend.callTool('cypher', { query, repo: options?.repo, + branch: options?.branch, }); output(result); } @@ -207,12 +215,14 @@ export async function detectChangesCommand(options?: { scope?: string; baseRef?: string; repo?: string; + branch?: string; }): Promise { const backend = await getBackend(); const result = await backend.callTool('detect_changes', { scope: options?.scope || 'unstaged', base_ref: options?.baseRef, repo: options?.repo, + branch: options?.branch, }); output(formatDetectChangesResult(result)); } diff --git a/gitnexus/src/cli/uninstall.ts b/gitnexus/src/cli/uninstall.ts new file mode 100644 index 000000000..b67e9fcdc --- /dev/null +++ b/gitnexus/src/cli/uninstall.ts @@ -0,0 +1,518 @@ +/** + * Uninstall Command + * + * Reverses `gitnexus setup`: removes the GitNexus MCP server entries, + * skills, and hooks that setup writes into each detected AI editor's + * global configuration. The set of targets (paths, key paths, hook events, + * needles, script dirs) is shared with setup.ts via editor-targets.ts, so the + * two stay in lock-step. + * + * Surgical and idempotent: only gitnexus-owned keys/entries/dirs are + * removed. Unrelated user config (other MCP servers, other hooks, JSONC + * comments, indentation) is preserved. Files that are absent or that + * never contained a gitnexus entry are left untouched. + * + * Ownership is by name: skill directories are matched by the bundled gitnexus + * skill names, MCP entries by the `gitnexus` key, hooks by the gitnexus command + * needle. There is no per-install provenance marker yet (a user dir that + * happens to share a bundled skill name, or files a user added inside an + * installed skill dir, are matched purely by name) — which is why uninstall is + * a dry-run preview by default and prints the exact paths it will remove. + * Richer provenance tracking is a tracked follow-up. + * + * Intentionally NOT done here (printed as hints instead, since both are + * destructive in ways setup never caused): + * - per-repo indexes → `gitnexus clean --all` + * - the global npm package → `npm uninstall -g gitnexus` + * + * Default is a dry-run preview; pass --force to apply. + */ + +import fs from 'fs/promises'; +import path from 'path'; +import { execFile } from 'child_process'; +import { promisify } from 'util'; +import { fileURLToPath } from 'url'; +import { + parseTree, + modify, + applyEdits, + findNodeAtLocation, + parse as parseJsonc, + type ParseError, + type JSONPath, +} from 'jsonc-parser'; +import { getEditorTargets, detectIndentation } from './editor-targets.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const execFileAsync = promisify(execFile); + +interface UninstallResult { + removed: string[]; + skipped: string[]; + errors: string[]; +} + +type RemovalStatus = 'removed' | 'absent' | 'corrupt' | 'missing'; + +/** + * Remove a single key (by JSON path) from a JSONC file, preserving the + * surrounding comments and formatting. Returns: + * - 'missing': file does not exist + * - 'absent': file exists but the key isn't there (nothing to do) + * - 'corrupt': file isn't valid JSONC — left untouched on purpose + * - 'removed': the key was present (and removed unless dryRun) + */ +async function removeJsoncKey( + filePath: string, + keyPath: JSONPath, + dryRun: boolean, +): Promise { + let raw: string; + try { + raw = await fs.readFile(filePath, 'utf-8'); + } catch { + return 'missing'; + } + + if (raw.trim().length === 0) return 'absent'; + + const parseErrors: ParseError[] = []; + const tree = parseTree(raw, parseErrors); + if (!tree || tree.type !== 'object' || parseErrors.length > 0) return 'corrupt'; + + if (!findNodeAtLocation(tree, keyPath)) return 'absent'; + + if (!dryRun) { + const formattingOptions = detectIndentation(raw); + const edits = modify(raw, keyPath, undefined, { formattingOptions }); + await fs.writeFile(filePath, applyEdits(raw, edits), 'utf-8'); + } + return 'removed'; +} + +/** + * Remove the gitnexus hook command(s) — those whose command string contains + * `commandNeedle` — from the given `eventNames` arrays in a JSONC settings + * file. Mirrors the idempotency probes in setup.ts (hasGitnexusHook / + * geminiHasGitnexusHook). Returns how many event entries contained a gitnexus + * command. + * + * Removal is element-granular to honor the "other hooks are preserved" + * contract: only the matching command object inside an entry's `hooks[]` is + * deleted. The surrounding matcher entry is removed only when it becomes + * empty (i.e. it held nothing but gitnexus commands — which is exactly what + * setup creates). A user who hand-added their own command alongside ours + * keeps it. Edits are applied highest-index-first so earlier indices stay + * valid across edits. + */ +async function removeHookEntries( + filePath: string, + eventNames: string[], + commandNeedle: string, + dryRun: boolean, +): Promise<{ status: RemovalStatus; count: number }> { + let raw: string; + try { + raw = await fs.readFile(filePath, 'utf-8'); + } catch { + return { status: 'missing', count: 0 }; + } + + if (raw.trim().length === 0) return { status: 'absent', count: 0 }; + + const parseErrors: ParseError[] = []; + const tree = parseTree(raw, parseErrors); + if (!tree || tree.type !== 'object' || parseErrors.length > 0) { + return { status: 'corrupt', count: 0 }; + } + + const parsed = parseJsonc(raw); + const formattingOptions = detectIndentation(raw); + let current = raw; + let total = 0; + + const isGitnexusHook = (hh: any): boolean => + typeof hh?.command === 'string' && hh.command.includes(commandNeedle); + + for (const eventName of eventNames) { + const entries = parsed?.hooks?.[eventName]; + if (!Array.isArray(entries)) continue; + + // Walk entries high → low so removing a later one never shifts the + // index of an earlier one. + for (let entryIdx = entries.length - 1; entryIdx >= 0; entryIdx--) { + const entry = entries[entryIdx]; + if (!Array.isArray(entry?.hooks)) continue; + + const hookIdxs: number[] = []; + entry.hooks.forEach((hh: any, hi: number) => { + if (isGitnexusHook(hh)) hookIdxs.push(hi); + }); + if (hookIdxs.length === 0) continue; + + total += 1; + if (dryRun) continue; + + if (hookIdxs.length === entry.hooks.length) { + // The entry held only gitnexus command(s) — drop the whole entry. + const edits = modify(current, ['hooks', eventName, entryIdx], undefined, { + formattingOptions, + }); + current = applyEdits(current, edits); + } else { + // The entry also holds user command(s) — delete only ours, keep + // the rest. Highest hook index first to keep lower indices valid. + for (const hi of hookIdxs.reverse()) { + const edits = modify(current, ['hooks', eventName, entryIdx, 'hooks', hi], undefined, { + formattingOptions, + }); + current = applyEdits(current, edits); + } + } + } + } + + if (total === 0) return { status: 'absent', count: 0 }; + if (!dryRun) await fs.writeFile(filePath, current, 'utf-8'); + return { status: 'removed', count: total }; +} + +/** + * Remove a directory tree if it exists. Returns true when something was + * (or would be) removed. + */ +async function removeDir(dirPath: string, dryRun: boolean): Promise { + try { + await fs.access(dirPath); + } catch { + return false; + } + if (!dryRun) await fs.rm(dirPath, { recursive: true, force: true }); + return true; +} + +/** + * The exact set of skill directory names setup installs, derived from the + * bundled `skills/` source the same way installSkillsTo does (flat + * `{name}.md` and `{name}/SKILL.md` layouts). Deriving the set — rather + * than globbing `gitnexus-*` — ensures we never delete a user's own + * similarly-named skill folder. + */ +async function listGitnexusSkillNames(): Promise { + const skillsRoot = + process.env.GITNEXUS_TEST_SKILLS_ROOT ?? path.join(__dirname, '..', '..', 'skills'); + + const names = new Set(); + try { + const entries = await fs.readdir(skillsRoot, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith('.md')) { + // Guard against a bare `.md` file: basename('.md', '.md') === '', + // which would later resolve to the skills dir itself and wipe it. + const base = path.basename(entry.name, '.md'); + if (base) names.add(base); + } else if (entry.isDirectory()) { + try { + await fs.access(path.join(skillsRoot, entry.name, 'SKILL.md')); + names.add(entry.name); + } catch { + // Not a skill directory — skip. + } + } + } + } catch { + return []; + } + return [...names]; +} + +/** + * Remove the gitnexus skill directories from a target skills folder. Returns + * the absolute paths that were removed (or would be removed in dryRun) so the + * caller can show the user exactly what is affected. + */ +async function removeSkillsFrom( + targetDir: string, + skillNames: string[], + dryRun: boolean, +): Promise { + const removed: string[] = []; + for (const name of skillNames) { + // Defense in depth: an empty/relative/absolute name would resolve back to + // targetDir (or escape it) and wipe unrelated content. Only act on a + // plain child directory name. + if ( + !name || + name.includes('/') || + name.includes('\\') || + name === '.' || + name === '..' || + path.isAbsolute(name) + ) { + continue; + } + const dir = path.join(targetDir, name); + if (await removeDir(dir, dryRun)) removed.push(dir); + } + return removed; +} + +/** + * Remove the `[mcp_servers.gitnexus]` table — and any of its descendant + * sub-tables (`[mcp_servers.gitnexus.env]`, `[[mcp_servers.gitnexus.x]]`) — + * from Codex's config.toml. Used only as a fallback when the `codex` binary + * isn't on PATH; the CLI's `codex mcp remove` is preferred. + * + * Hand-rolled (no TOML dependency), but careful about the cases a naive + * line-scan gets wrong: + * - descendant sub-tables of the section are also removed (else they'd be + * left dangling, referencing a server that no longer exists); + * - `[...]`-shaped lines inside a multiline string (`"""`/`'''`) are NOT + * treated as table headers; + * - unrelated whitespace/formatting elsewhere in the file is left intact + * (no global blank-line reflow). Only a single blank separator line + * directly above the removed section is dropped. + */ +function stripTomlSection(raw: string, sectionName: string): string { + const header = `[${sectionName}]`; + const childTable = `[${sectionName}.`; + const childArray = `[[${sectionName}.`; + // Capture group 1 is the bracket token only, so a trailing inline comment + // (`[mcp_servers.gitnexus] # note`) is stripped before classification — + // otherwise an exact `=== header` check fails and the section is left behind. + const headerRe = /^(\[\[?[^[\]]+\]\]?)\s*(#.*)?$/; + + const isSectionHeader = (token: string): boolean => + token === header || token.startsWith(childTable) || token.startsWith(childArray); + + // Return the multiline-string delimiter still OPEN at the end of `line`, + // given the state at its start (null = outside any multiline string). Scans + // left→right so the delimiter that actually opens first wins — a line with an + // odd count of BOTH `"""` and `'''` (e.g. `x = '''has """ inside`) no longer + // mis-picks the wrong delimiter and desyncs the scanner. + const multilineStateAfter = (line: string, startState: string | null): string | null => { + let state = startState; + let i = 0; + while (i < line.length) { + if (state) { + const close = line.indexOf(state, i); + if (close === -1) return state; // still open at end of line + i = close + state.length; + state = null; + } else { + const a = line.indexOf('"""', i); + const b = line.indexOf("'''", i); + if (a === -1 && b === -1) return null; + const useA = b === -1 || (a !== -1 && a < b); + state = useA ? '"""' : "'''"; + i = (useA ? a : b) + 3; + } + } + return state; + }; + + const lines = raw.split(/\r?\n/); + const out: string[] = []; + let skipping = false; + let mlDelim: string | null = null; + + for (const line of lines) { + if (mlDelim) { + // Inside a multiline string: brackets here are data, not headers. + mlDelim = multilineStateAfter(line, mlDelim); + if (!skipping) out.push(line); + continue; + } + + const trimmed = line.trim(); + const headerMatch = trimmed.match(headerRe); + if (headerMatch) { + if (isSectionHeader(headerMatch[1])) { + // Drop a single blank separator line immediately above the section. + if (!skipping && out.length > 0 && out[out.length - 1].trim() === '') out.pop(); + skipping = true; + continue; + } + // A non-descendant header ends the section. + skipping = false; + out.push(line); + continue; + } + + // Track whether this (non-header) line opens a multiline string so a + // bracketed line inside it isn't mistaken for a header. + mlDelim = multilineStateAfter(line, null); + + if (!skipping) out.push(line); + } + + // Preserve the file's line endings: a CRLF (Windows) config.toml should not + // be silently rewritten to LF. Rejoin with the dominant EOL of the input. + const eol = raw.includes('\r\n') ? '\r\n' : '\n'; + let result = out.join(eol); + if (!result.endsWith(eol)) result += eol; + return result; +} + +async function uninstallCodex( + result: UninstallResult, + dryRun: boolean, + configPath: string, + tomlSection: string, +): Promise { + let raw: string; + try { + raw = await fs.readFile(configPath, 'utf-8'); + } catch { + result.skipped.push('Codex MCP (not configured)'); + return; + } + + if (!raw.includes(`[${tomlSection}]`)) { + result.skipped.push('Codex MCP (not configured)'); + return; + } + + if (dryRun) { + result.removed.push(`Codex MCP server — [${tomlSection}] in ${configPath}`); + return; + } + + // Prefer the official CLI (mirrors setup's `codex mcp add`); fall back + // to editing config.toml directly when the binary isn't on PATH. + try { + await execFileAsync('codex', ['mcp', 'remove', 'gitnexus'], { + shell: process.platform === 'win32', + windowsHide: true, + timeout: 10000, + }); + result.removed.push("Codex MCP server — via 'codex mcp remove gitnexus'"); + return; + } catch { + // Fall through to manual edit. + } + + try { + await fs.writeFile(configPath, stripTomlSection(raw, tomlSection), 'utf-8'); + result.removed.push(`Codex MCP server — [${tomlSection}] in ${configPath}`); + } catch (err: any) { + result.errors.push(`Codex: ${err.message}`); + } +} + +// ─── Main command ────────────────────────────────────────────────── + +export const uninstallCommand = async (options?: { force?: boolean }) => { + const dryRun = !options?.force; + const targets = getEditorTargets(); + + console.log(''); + console.log(' GitNexus Uninstall'); + console.log(' =================='); + console.log(''); + if (dryRun) { + console.log(' Dry run — nothing will be changed. Re-run with --force to apply.'); + console.log(''); + } + + const result: UninstallResult = { removed: [], skipped: [], errors: [] }; + + // ─── MCP server entries (JSONC editors) ────────────────────────── + for (const target of targets.mcpJsonc) { + try { + const status = await removeJsoncKey(target.file, target.keyPath, dryRun); + if (status === 'removed') + result.removed.push( + `${target.label} MCP server — ${target.keyPath.join('.')} in ${target.file}`, + ); + else if (status === 'corrupt') + result.errors.push( + `${target.label}: ${path.basename(target.file)} is corrupt — left untouched`, + ); + else result.skipped.push(`${target.label} MCP (not configured)`); + } catch (err: any) { + result.errors.push(`${target.label}: ${err.message}`); + } + } + + await uninstallCodex(result, dryRun, targets.codex.configFile, targets.codex.tomlSection); + + // ─── Hooks ─────────────────────────────────────────────────────── + for (const hook of targets.hooks) { + try { + const { status, count } = await removeHookEntries( + hook.settingsFile, + hook.events, + hook.needle, + dryRun, + ); + if (status === 'removed') + result.removed.push(`${hook.label} hooks (${count}) — ${hook.settingsFile}`); + else if (status === 'corrupt') + result.errors.push( + `${hook.label} hooks: ${path.basename(hook.settingsFile)} is corrupt — left untouched`, + ); + // Don't delete the hook script while a registered entry may still point + // at it (corrupt = we couldn't parse/remove the entry) — that would + // leave the editor invoking a missing script on every matched tool call. + if (status !== 'corrupt' && (await removeDir(hook.scriptDir, dryRun))) + result.removed.push(`${hook.label} hook scripts — ${hook.scriptDir}`); + } catch (err: any) { + result.errors.push(`${hook.label} hooks: ${err.message}`); + } + } + + // ─── Skills ────────────────────────────────────────────────────── + // Skill directories are identified by the bundled gitnexus skill names; the + // exact paths are listed below so the user can see what will be removed. + const skillNames = await listGitnexusSkillNames(); + for (const target of targets.skills) { + try { + const removedDirs = await removeSkillsFrom(target.dir, skillNames, dryRun); + for (const dir of removedDirs) result.removed.push(`${target.label} skill — ${dir}`); + } catch (err: any) { + result.errors.push(`${target.label} skills: ${err.message}`); + } + } + + // ─── Report ────────────────────────────────────────────────────── + const verb = dryRun ? 'Would remove' : 'Removed'; + if (result.removed.length > 0) { + console.log(` ${verb}:`); + for (const name of result.removed) console.log(` - ${name}`); + } else { + console.log(' Nothing to remove — GitNexus is not configured in any detected editor.'); + } + + if (result.skipped.length > 0) { + console.log(''); + console.log(' Skipped:'); + for (const name of result.skipped) console.log(` - ${name}`); + } + + if (result.errors.length > 0) { + console.log(''); + console.log(' Errors:'); + for (const err of result.errors) console.log(` ! ${err}`); + // Signal partial failure to callers/CI without aborting the remaining + // cleanup (which has already run by this point). + process.exitCode = 1; + } + + console.log(''); + console.log(' Note: skill directories are matched by bundled gitnexus skill name. If you'); + console.log(' customized files inside an installed skill dir, back them up before --force.'); + + console.log(''); + console.log(' Not removed automatically:'); + console.log(' - Per-repo indexes — run: gitnexus clean --all'); + console.log(' - The global npm package — run: npm uninstall -g gitnexus'); + + if (dryRun && result.removed.length > 0) { + console.log(''); + console.log(' Re-run with --force to apply the changes above.'); + } + console.log(''); +}; diff --git a/gitnexus/src/cli/wiki.ts b/gitnexus/src/cli/wiki.ts index d10d10875..446b83d30 100644 --- a/gitnexus/src/cli/wiki.ts +++ b/gitnexus/src/cli/wiki.ts @@ -58,14 +58,21 @@ function parsePositiveIntegerOption( function isLocalProvider( provider: LLMProvider | undefined, -): provider is 'cursor' | 'claude' | 'codex' { - return provider === 'cursor' || provider === 'claude' || provider === 'codex'; +): provider is 'cursor' | 'claude' | 'codex' | 'opencode' { + return ( + provider === 'cursor' || + provider === 'claude' || + provider === 'codex' || + provider === 'opencode' + ); } -function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex') { +function localModelConfigKey(provider: 'cursor' | 'claude' | 'codex' | 'opencode') { if (provider === 'cursor') return 'cursorModel'; if (provider === 'claude') return 'claudeModel'; - return 'codexModel'; + if (provider === 'codex') return 'codexModel'; + if (provider === 'opencode') return 'opencodeModel'; + throw new Error(`Unsupported local provider: ${provider satisfies never}`); } /** @@ -248,7 +255,7 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) if (!llmConfig.apiKey && !isLocalProvider(llmConfig.provider)) { console.log(' Error: No LLM API key found.'); console.log(' Set OPENAI_API_KEY or GITNEXUS_API_KEY environment variable,'); - console.log(' or pass --api-key , or use --provider cursor|claude|codex.\n'); + console.log(' or pass --api-key , or use --provider cursor|claude|codex|opencode.\n'); process.exitCode = 1; return; } @@ -256,16 +263,17 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) } else { console.log(" No LLM configured. Let's set it up.\n"); console.log( - ' Supports OpenAI, OpenRouter, Azure, any OpenAI-compatible API, Cursor CLI, Claude CLI, or Codex CLI.\n', + ' Supports OpenAI, OpenRouter, Azure, any OpenAI-compatible API, Cursor CLI, Claude CLI, Codex CLI, or OpenCode CLI.\n', ); // Check if local agent CLIs are available. const hasCursor = detectCursorCLI(); const hasClaude = detectLocalCLI('claude'); const hasCodex = detectLocalCLI('codex'); + const hasOpenCode = detectLocalCLI('opencode'); const localChoices: Array<{ choice: string; - provider: 'cursor' | 'claude' | 'codex'; + provider: 'cursor' | 'claude' | 'codex' | 'opencode'; }> = []; // Provider selection @@ -298,6 +306,14 @@ const wikiCommandImpl = async (inputPath?: string, options?: WikiCommandOptions) }); console.log(` [${choice}] Codex CLI (local, uses your Codex login)`); } + if (hasOpenCode) { + const choice = String(nextChoice++); + localChoices.push({ + choice, + provider: 'opencode', + }); + console.log(` [${choice}] OpenCode CLI (local, uses your OpenCode login/config)`); + } console.log(''); const maxChoice = String(nextChoice - 1); diff --git a/gitnexus/src/core/embeddings/embedder.ts b/gitnexus/src/core/embeddings/embedder.ts index d873f3a0a..3ec5f08e1 100644 --- a/gitnexus/src/core/embeddings/embedder.ts +++ b/gitnexus/src/core/embeddings/embedder.ts @@ -28,6 +28,7 @@ import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js'; import { resolveEmbeddingConfig } from './config.js'; import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js'; import { getLocalEmbeddingRuntimeBlocker } from './runtime-support.js'; +import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js'; import { logger } from '../logger.js'; /** @@ -179,6 +180,9 @@ export const initEmbedder = async ( try { // Lazy-load transformers.js only after the runtime guard has passed, so // unsupported platforms never reach the native ONNX import (#1515). + // Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common` + // import is unresolvable; register the fallback resolver first (#307). + ensureOnnxRuntimeCommonResolvable(); const { pipeline, env } = await import('@huggingface/transformers'); // Configure transformers.js environment diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index e394659f4..0405cd47c 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -36,13 +36,8 @@ import { } from './types.js'; import { resolveEmbeddingConfig } from './config.js'; import { rankExactEmbeddingRows, type ExactEmbeddingRow } from './exact-search.js'; -import { - EMBEDDING_TABLE_NAME, - EMBEDDING_INDEX_NAME, - CREATE_VECTOR_INDEX_QUERY, - STALE_HASH_SENTINEL, -} from '../lbug/schema.js'; -import { loadVectorExtension } from '../lbug/lbug-adapter.js'; +import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, STALE_HASH_SENTINEL } from '../lbug/schema.js'; +import { loadVectorExtension, createVectorIndex } from '../lbug/lbug-adapter.js'; import type { ExtensionInstallPolicy } from '../lbug/extension-loader.js'; import { getExactScanLimit } from '../platform/capabilities.js'; import { logger } from '../logger.js'; @@ -215,24 +210,36 @@ export const batchInsertEmbeddings = async ( }; /** - * Create the vector index for semantic search - - * Now indexes the separate CodeEmbedding table. - * Delegates extension loading to lbug-adapter's loadVectorExtension(), - * which owns the VECTOR extension lifecycle and state tracking. - + * Create the vector index for semantic search (indexes the CodeEmbedding table). + * + * Keeps the embedding-specific extension-install policy gate here + * (ensureVectorExtensionAvailable → resolveEmbeddingInstallPolicy, default + * `auto` for the analyze write path), then delegates the actual + * `CALL CREATE_VECTOR_INDEX(...)` to the adapter, which runs it through the + * unprepared `conn.query()` path. It must NOT go through the injected + * `executeQuery` (prepared `conn.prepare()`): LadybugDB cannot prepare that + * procedure and fails with "We do not support prepare multiple statements" — + * the silent degrade in #2114. */ -const createVectorIndex = async ( - executeQuery: (cypher: string) => Promise, -): Promise => { +const buildVectorIndex = async (): Promise => { + // This pre-check applies the embedding-specific install policy + // (resolveEmbeddingInstallPolicy, default `auto` for analyze) before reaching + // the adapter. The adapter's createVectorIndex() calls loadVectorExtension() + // again, but that's a no-op here: once this gate loads VECTOR the module-level + // `vectorExtensionLoaded` flag is set, so the adapter's second call + // short-circuits without re-resolving the policy — no double install. if (!(await ensureVectorExtensionAvailable())) return false; try { - await executeQuery(CREATE_VECTOR_INDEX_QUERY); - return true; + return await createVectorIndex(); } catch (error) { - if (isDev) { - logger.warn({ error }, 'Vector index creation warning:'); - } + // Surface this even outside dev: it silently downgrades a user-requested + // feature (semantic search) to exact scan. Log under `err` so pino's + // standard serializer captures the message/stack — logging under `error` + // serialized an Error to `{}` (the empty `{"error":{}}` reported in #2114). + logger.warn( + { err: error }, + 'Vector index creation failed; semantic search will use exact-scan fallback', + ); return false; } }; @@ -383,7 +390,7 @@ export const runEmbeddingPipeline = async ( // Ensure the vector index exists even when no new nodes need embedding. // A prior crash or first-time incremental run may have left CodeEmbedding // rows without ever reaching index creation. - const vectorIndexReady = await createVectorIndex(executeQuery); + const vectorIndexReady = await buildVectorIndex(); onProgress({ phase: 'ready', @@ -544,7 +551,7 @@ export const runEmbeddingPipeline = async ( logger.info('📇 Creating vector index...'); } - const vectorIndexReady = await createVectorIndex(executeQuery); + const vectorIndexReady = await buildVectorIndex(); onProgress({ phase: 'ready', diff --git a/gitnexus/src/core/embeddings/onnxruntime-common-resolver.ts b/gitnexus/src/core/embeddings/onnxruntime-common-resolver.ts new file mode 100644 index 000000000..fbb4f4082 --- /dev/null +++ b/gitnexus/src/core/embeddings/onnxruntime-common-resolver.ts @@ -0,0 +1,133 @@ +/** + * Make `@huggingface/transformers`' phantom `onnxruntime-common` import + * resolvable under strict package-manager layouts (#307, #2069). + * + * ## Why + * transformers' shipped `dist/transformers.node.mjs` does a bare + * `import 'onnxruntime-common'`, but transformers' `package.json` never declares + * onnxruntime-common (it lists onnxruntime-node / onnxruntime-web / sharp). With + * npm's flat `node_modules` — or pnpm with hoisting — the package is hoisted to + * a directory on transformers' resolution path and the import resolves by + * accident. Under pnpm's isolated store (and therefore `pnpm dlx` / `pnpx`), a + * package only sees its *declared* deps, so the import dies with + * `ERR_MODULE_NOT_FOUND` before `analyze --embeddings` can run. + * + * Declaring onnxruntime-common in gitnexus' own dependencies (#2074) does NOT + * fix this under pnpm: Node resolves the bare specifier from *transformers'* + * module scope, not ours, and overrides/resolutions can only re-version an + * existing edge, never add the missing one. + * + * ## What this does + * Install a synchronous, in-thread ESM resolution hook (`module.registerHooks`, + * Node >= 22.15) that redirects `onnxruntime-common` to a copy gitnexus can + * resolve — but only when the default resolver fails. The redirect target is + * preferentially the `onnxruntime-common` that `onnxruntime-node` (the native + * binding transformers actually loads) itself depends on, so the redirected copy + * is version-matched to that binding even under `pnpm dlx` — where gitnexus' + * npm-style `overrides` block does NOT apply, because it is honoured only from a + * root manifest and gitnexus is a transitive dependency there. It falls back to + * gitnexus' own direct `onnxruntime-common` dependency when that chain can't be + * walked. onnxruntime-common is a stable, pure-JS package whose `Tensor` surface + * is unchanged across 1.24–1.26, so either target is API-compatible. On working + * layouts the default resolver succeeds first and the hook never fires, so + * behaviour is unchanged. + * + * `registerHooks` (synchronous, in-thread) is preferred over the older + * `module.register` (async, off-thread, now deprecated — DEP0205, removed in + * Node 26): the redirect is a one-line conditional that needs no worker thread, + * no separate hook module, and no `data` marshalling. + * + * ## Safety + * Best-effort and idempotent. The hook is installed lazily, only on the + * local-embedding code path (after parsing), so it is never registered during + * analysis, in the parse workers, or in HTTP embedding mode. Once installed it + * is process-global: its resolve closure runs for every subsequent module + * resolution, but it passes all of them through untouched and only substitutes a + * result for the exact `onnxruntime-common` specifier when that specifier is + * genuinely absent — so it cannot mask an unrelated resolution error, and the + * per-resolution cost is a single string comparison. + * + * `module.registerHooks` is marked `@experimental` and requires Node >= 22.15 + * (the gitnexus engines floor is >= 22.0.0). On older runtimes it is absent and + * this is a graceful no-op: embeddings then resolve onnxruntime-common exactly + * as before — fine on hoisted layouts. Any failure during installation is + * swallowed. + */ +import { registerHooks, createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import { logger } from '../logger.js'; + +let attempted = false; + +/** + * Compute the file: URL the hook redirects `onnxruntime-common` to. + * + * Prefer the copy `onnxruntime-node` (the native binding transformers loads) + * depends on, so the redirected module is version-matched to the binding even + * under `pnpm dlx`, where transformers keeps its own pinned onnxruntime-node. + * The walk resolves transformers' MAIN entry — NOT `@huggingface/transformers/ + * package.json`, which transformers' `exports` map blocks + * (`ERR_PACKAGE_PATH_NOT_EXPORTED`) — then onnxruntime-node, then its + * onnxruntime-common. Falls back to gitnexus' own direct dependency (always + * resolvable from our scope) when any step fails. + */ +const resolveOnnxRuntimeCommonUrl = (): string => { + const require = createRequire(import.meta.url); + try { + const transformersMain = require.resolve('@huggingface/transformers'); + const ortNodePkg = createRequire(transformersMain).resolve('onnxruntime-node/package.json'); + const common = createRequire(ortNodePkg).resolve('onnxruntime-common'); + return pathToFileURL(common).href; + } catch { + return pathToFileURL(require.resolve('onnxruntime-common')).href; + } +}; + +/** + * Idempotently install the onnxruntime-common resolution fallback. Call once + * immediately before the dynamic `import('@huggingface/transformers')` on the + * local-embedding path. + */ +export const ensureOnnxRuntimeCommonResolvable = (): void => { + if (attempted) return; + // Mark attempted up-front: a failed attempt must not retry on every + // initEmbedder() call, and the hook is process-global — once is enough. + attempted = true; + + try { + // Node < 22.15 (the gitnexus engines floor is >= 22.0.0): no synchronous + // hooks API. Degrade gracefully — the import still works on hoisted layouts. + if (typeof registerHooks !== 'function') return; + + const redirectUrl = resolveOnnxRuntimeCommonUrl(); + + registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier !== 'onnxruntime-common') return nextResolve(specifier, context); + // Honour a real, package-manager-provided copy when one is on the path + // (npm / hoisted pnpm); only substitute ours when the specifier is + // genuinely absent. + try { + return nextResolve(specifier, context); + } catch (err) { + // The phantom import surfaces as ERR_MODULE_NOT_FOUND (or, for a + // present-but-exports-broken copy, ERR_PACKAGE_PATH_NOT_EXPORTED). + // Rethrow anything else so a genuinely broken install is not masked. + const code = (err as { code?: string } | null | undefined)?.code; + if (code === 'ERR_MODULE_NOT_FOUND' || code === 'ERR_PACKAGE_PATH_NOT_EXPORTED') { + return { url: redirectUrl, shortCircuit: true }; + } + throw err; + } + }, + }); + logger.debug({ redirectUrl }, 'Installed onnxruntime-common resolution fallback (#307)'); + } catch (err) { + // Never block embeddings on the fallback. On layouts where the package + // manager already resolves onnxruntime-common this is unnecessary anyway. + logger.debug( + { err: err instanceof Error ? err.message : String(err) }, + 'onnxruntime-common resolution fallback not installed', + ); + } +}; diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts b/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts index 3435b15d1..6309c1acc 100644 --- a/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts +++ b/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts @@ -1,4 +1,5 @@ import { createRequire } from 'node:module'; +import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.js'; import { compilePatterns, runCompiledPatterns, @@ -10,11 +11,11 @@ import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; /** * Protobuf (.proto) tree-sitter plugin for gRPC contract extraction. * - * Uses `tree-sitter-proto` (coder3101/tree-sitter-proto) as an - * optionalDependency — if the grammar is not installed (e.g. native - * compilation failed on an unusual platform), the plugin exports - * `null` and the orchestrator falls back to the existing manual - * string-sanitizing parser. + * Uses `tree-sitter-proto` (coder3101/tree-sitter-proto), loaded from + * `vendor/` by absolute path (NEVER copied into node_modules — see + * vendored-grammars.ts / #2111). If the grammar's binding cannot be loaded + * (e.g. no prebuild for an unusual platform), the plugin exports `null` and the + * orchestrator falls back to the existing manual string-sanitizing parser. * * The grammar is vendored in `vendor/tree-sitter-proto/` with * parser.c regenerated against tree-sitter-cli 0.24 (ABI version 14) @@ -22,10 +23,13 @@ import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; * (which loads ABI 13–14). */ +// Only for `tree-sitter` (a real npm dependency) in the smoke-test below; +// the vendored grammar goes through requireVendoredGrammar (never a bare +// `_require('tree-sitter-proto')`, which would force a node_modules copy — #2111). const _require = createRequire(import.meta.url); let ProtoGrammar: unknown = null; try { - ProtoGrammar = _require('tree-sitter-proto'); + ProtoGrammar = requireVendoredGrammar('tree-sitter-proto'); } catch { // Grammar not installed — PROTO_GRPC_PLUGIN will be null. } diff --git a/gitnexus/src/core/group/extractors/http-patterns/java.ts b/gitnexus/src/core/group/extractors/http-patterns/java.ts index 920d499aa..3ad66543e 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/java.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/java.ts @@ -6,6 +6,11 @@ import { unquoteLiteral, type LanguagePatterns, } from '../tree-sitter-scanner.js'; +import { + METHOD_ANNOTATION_TO_HTTP, + isRouteMemberKey, + findEnclosingClass, +} from '../../../ingestion/route-extractors/spring-shared.js'; import type { HttpDetection, HttpFileDetections, @@ -33,14 +38,6 @@ import type { * OkHttp, Java/Apache HttpClient) keep their own focused queries. */ -const METHOD_ANNOTATION_TO_HTTP: Record = { - GetMapping: 'GET', - PostMapping: 'POST', - PutMapping: 'PUT', - DeleteMapping: 'DELETE', - PatchMapping: 'PATCH', -}; - // Each route-defining annotation has two AST shapes — a positional argument // and a named one — that must both be matched: // @RequestMapping("/api") → (annotation_argument_list (string_literal)) @@ -55,6 +52,7 @@ const METHOD_ANNOTATION_TO_HTTP: Record = { interface SpringRouteBinding { method: string; path: string; + ownerPrefix?: string; } interface SpringMethodInfo { @@ -360,19 +358,9 @@ const APACHE_HTTP_CLIENT_PATTERNS = compilePatterns({ } satisfies LanguagePatterns>); /** - * Find the nearest enclosing class/interface declaration ancestor for - * a node, or null if the node is top-level. Tree-sitter's - * SyntaxNode.parent walks one level at a time. + * Find the nearest enclosing interface declaration ancestor for a node, or + * null if the node is top-level. */ -function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let cur: Parser.SyntaxNode | null = node.parent; - while (cur) { - if (cur.type === 'class_declaration') return cur; - cur = cur.parent; - } - return null; -} - function findEnclosingInterface(node: Parser.SyntaxNode): Parser.SyntaxNode | null { let cur: Parser.SyntaxNode | null = node.parent; while (cur) { @@ -395,6 +383,25 @@ function joinPath(prefix: string, methodPath: string): string { return `/${cleanPrefix}/${cleanSub}`; } +function joinInheritedSpringPath( + controllerPrefix: string, + inheritedPath: string, + inheritedOwnerPrefix = '', +): string { + const joined = joinPath(controllerPrefix, inheritedPath); + const cleanPrefix = controllerPrefix.replace(/^\/+/, '').replace(/\/+$/, ''); + const cleanOwnerPrefix = inheritedOwnerPrefix.replace(/^\/+/, '').replace(/\/+$/, ''); + const cleanInherited = inheritedPath.replace(/^\/+/, ''); + if (!cleanPrefix) return joined; + if ( + cleanPrefix === cleanOwnerPrefix && + (cleanInherited === cleanPrefix || cleanInherited.startsWith(`${cleanPrefix}/`)) + ) { + return `/${cleanInherited}`; + } + return joined; +} + function getNodeName(node: Parser.SyntaxNode): string | null { return node.childForFieldName('name')?.text ?? null; } @@ -419,18 +426,6 @@ function hasAnnotation(node: Parser.SyntaxNode, names: string | readonly string[ return false; } -/** - * A named annotation argument contributes a route only when its member key is - * `path` or `value`; a positional argument (no key node) always qualifies. - * This is the JS-side replacement for the in-query `^(path|value)$` filter and - * drops Spring's non-route string attributes (`produces`, `consumes`, - * `headers`, `name`, `params`) that would otherwise be mis-read as routes. - */ -function isRouteMemberKey(keyNode: Parser.SyntaxNode | undefined): boolean { - if (!keyNode) return true; - return keyNode.text === 'path' || keyNode.text === 'value'; -} - interface MethodRouteAnnotation { methodNode: Parser.SyntaxNode; methodName: string | null; @@ -634,6 +629,7 @@ function scanSpringProject(files: readonly HttpScanInput[]): HttpFileDetections[ const routes = method.routes.map((route) => ({ method: route.method, path: type.classPrefix ? joinPath(type.classPrefix, route.path) : route.path, + ownerPrefix: type.classPrefix, })); if (routes.length > 0) methodMap.set(method.name, routes); } @@ -651,7 +647,7 @@ function scanSpringProject(files: readonly HttpScanInput[]): HttpFileDetections[ const routes = routeMap.get(method.name) ?? []; return routes.map((route) => ({ method: route.method, - path: joinPath(type.classPrefix, route.path), + path: joinInheritedSpringPath(type.classPrefix, route.path, route.ownerPrefix), })); }); diff --git a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts index 0e56b554b..14dce0ae1 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts @@ -1,5 +1,5 @@ import Parser from 'tree-sitter'; -import { createRequire } from 'node:module'; +import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.js'; import { compilePatterns, runCompiledPatterns, @@ -60,17 +60,16 @@ import type { HttpDetection, HttpLanguagePlugin } from './types.js'; * value_argument * string_literal ← the path * - * tree-sitter-kotlin is an optional npm dependency — when its native - * binding is unavailable the plugin gracefully exports `null` and - * `http-patterns/index.ts` skips registration for `.kt`/`.kts` files. + * tree-sitter-kotlin is a vendored grammar loaded from `vendor/` by absolute + * path (NEVER copied into node_modules — see vendored-grammars.ts / #2111) — + * when its native binding is unavailable the plugin gracefully exports `null` + * and `http-patterns/index.ts` skips registration for `.kt`/`.kts` files. */ -const _require = createRequire(import.meta.url); - -/** Loaded lazily; null when the grammar binding isn't installed. */ +/** Loaded lazily; null when the grammar binding isn't available. */ let Kotlin: unknown | null = null; try { - Kotlin = _require('tree-sitter-kotlin'); + Kotlin = requireVendoredGrammar('tree-sitter-kotlin'); } catch { Kotlin = null; } diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts index 98cbd371f..c8b4ee662 100644 --- a/gitnexus/src/core/group/extractors/include-extractor.ts +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -2,8 +2,22 @@ import * as path from 'node:path'; import * as fs from 'node:fs/promises'; import { glob } from 'glob'; import Parser from 'tree-sitter'; -import C from 'tree-sitter-c'; import Cpp from 'tree-sitter-cpp'; +import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js'; + +// `tree-sitter-c` is vendored (#2116), loaded from `vendor/` by absolute path +// (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Load it +// via a guarded call rather than a top-level `import C from 'tree-sitter-c'`, +// which would throw ERR_MODULE_NOT_FOUND at module-load and crash analyze +// (#2091/#2093). It may be absent on a platform without a prebuild; when the +// binding is absent, `getLanguageForFile` returns null for `.c`/`.h` so C +// include-extraction is skipped (C++ is unaffected — its binding always ships). +let C: unknown = null; +try { + C = requireVendoredGrammar('tree-sitter-c'); +} catch { + /* C grammar unavailable — C include extraction degrades to a no-op. */ +} import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; import type { ExtractedContract, RepoHandle } from '../types.js'; import { readSafe } from './fs-utils.js'; diff --git a/gitnexus/src/core/ingestion/ast-cache.ts b/gitnexus/src/core/ingestion/ast-cache.ts deleted file mode 100644 index 454c60df2..000000000 --- a/gitnexus/src/core/ingestion/ast-cache.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { LRUCache } from 'lru-cache'; -import Parser from 'tree-sitter'; - -import { logger } from '../logger.js'; -/** - * Minimal structural shape consumers need when reading Trees back - * through a phase-dependency boundary. Declared here so phases that - * receive ASTCache via `getPhaseOutput<...>` don't hand-roll their - * own inline structural types that silently drift when ASTCache's - * contract changes. - * - * Typed as `unknown` at the Tree boundary because consumers on the - * other side of the phase-output map don't share tree-sitter's type - * graph (e.g. COBOL's standalone processor). - */ -export interface ASTCacheReader { - get(filePath: string): unknown; - clear(): void; -} - -// Define the interface for the Cache -export interface ASTCache extends ASTCacheReader { - get: (filePath: string) => Parser.Tree | undefined; - set: (filePath: string, tree: Parser.Tree) => void; - clear: () => void; - stats: () => { size: number; maxSize: number }; -} - -export const createASTCache = (maxSize: number = 50): ASTCache => { - const effectiveMax = Math.max(maxSize, 1); - // Initialize the cache with a 'dispose' handler - // This is the magic: When an item is evicted (dropped), this runs automatically. - const cache = new LRUCache({ - max: effectiveMax, - dispose: (tree) => { - try { - // NOTE: web-tree-sitter has tree.delete(); native tree-sitter - // trees are GC-managed and .delete is absent (no-op here). - // - // Single-owner invariant (load-bearing under WASM): a given - // Parser.Tree reference must live in AT MOST ONE ASTCache - // that disposes. The parse-phase chunk-local cache clears - // between chunks; the cross-phase `scopeTreeCache` (also an - // ASTCache today) holds the same Tree by reference. Under - // native tree-sitter this is benign (dispose is a no-op). - // If/when GitNexus adopts web-tree-sitter for sequential - // parsing, the cross-phase cache must either (a) skip - // writing Trees that are already owned by a disposing cache, - // or (b) use tree.copy() per entry. Failing to pick one - // will hand freed memory to scope-resolution. - (tree as unknown as { delete?: () => void }).delete?.(); - } catch (e) { - logger.warn({ e }, 'Failed to delete tree from WASM memory'); - } - }, - }); - - return { - get: (filePath: string) => { - const tree = cache.get(filePath); - return tree; // Returns undefined if not found - }, - - set: (filePath: string, tree: Parser.Tree) => { - cache.set(filePath, tree); - }, - - clear: () => { - cache.clear(); - }, - - stats: () => ({ - size: cache.size, - maxSize: effectiveMax, - }), - }; -}; diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 4a9fba9af..b5b258dcd 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -9,25 +9,17 @@ * * - `processRoutesFromExtracted` — CALLS edges from framework routes * (e.g. Laravel) to their controller methods. - * - `processNextjsFetchRoutes` / `extractFetchCallsFromFiles` / - * `extractConsumerAccessedKeys` — FETCHES edges from `fetch()` calls to - * Next.js Route nodes. + * - `processNextjsFetchRoutes` / `extractConsumerAccessedKeys` — FETCHES edges + * from `fetch()` calls to Next.js Route nodes. * - `buildExportedTypeMapFromGraph` — exported symbol → return/declared type * map, consumed by the cross-file enrichment pass. */ -import Parser from 'tree-sitter'; import { KnowledgeGraph } from '../graph/types.js'; -import { ASTCache } from './ast-cache.js'; import type { SemanticModel, SymbolTableReader } from './model/index.js'; -import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; -import { getProvider } from './languages/index.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename } from 'gitnexus-shared'; import type { SymbolDefinition } from 'gitnexus-shared'; import { yieldToEventLoop } from './utils/event-loop.js'; -import { parseSourceSafe } from '../tree-sitter/safe-parse.js'; -import { getTreeSitterBufferSize } from './constants.js'; import type { ExtractedRoute, ExtractedFetchCall } from './workers/parse-worker.js'; import { normalizeFetchURL, routeMatches } from './route-extractors/nextjs.js'; import { extractReturnTypeName } from './type-extractors/shared.js'; @@ -39,6 +31,34 @@ const MAX_TYPE_NAME_LENGTH = 256; * Consumed by the cross-file re-resolution / enrichment pass. */ export type ExportedTypeMap = Map>; +/** Record one exported graph node into the incremental ExportedTypeMap. */ +export const accumulateExportedTypesFromParsedNode = ( + result: ExportedTypeMap, + node: { id: string; properties?: Record }, + symbolTable: SymbolTableReader, +): void => { + if (!node.properties?.isExported) return; + if (!node.properties?.filePath || !node.properties?.name) return; + const filePath = node.properties.filePath as string; + const name = node.properties.name as string; + if (!name || name.length > MAX_TYPE_NAME_LENGTH) return; + const defs = symbolTable.lookupExactAll(filePath, name); + const def = defs.find((d) => d.nodeId === node.id) ?? defs[0]; + if (!def) return; + const typeName = def.returnType ?? def.declaredType; + if (!typeName || typeName.length > MAX_TYPE_NAME_LENGTH) return; + const simpleType = extractReturnTypeName(typeName) ?? typeName; + if (!simpleType) return; + let fileExports = result.get(filePath); + if (!fileExports) { + fileExports = new Map(); + result.set(filePath, fileExports); + } + if (fileExports.size < MAX_EXPORTS_PER_FILE) { + fileExports.set(name, simpleType); + } +}; + /** Build ExportedTypeMap from graph nodes — used for the worker path where the * sequential TypeEnv is not available in the main thread. Collects * returnType/declaredType from exported symbols with known types. */ @@ -48,29 +68,7 @@ export function buildExportedTypeMapFromGraph( ): ExportedTypeMap { const result: ExportedTypeMap = new Map(); graph.forEachNode((node) => { - if (!node.properties?.isExported) return; - if (!node.properties?.filePath || !node.properties?.name) return; - const filePath = node.properties.filePath as string; - const name = node.properties.name as string; - if (!name || name.length > MAX_TYPE_NAME_LENGTH) return; - // For callable symbols, use returnType; for properties/variables, use declaredType. - // Use lookupExactAll + nodeId match to handle same-name methods in different classes. - const defs = symbolTable.lookupExactAll(filePath, name); - const def = defs.find((d) => d.nodeId === node.id) ?? defs[0]; - if (!def) return; - const typeName = def.returnType ?? def.declaredType; - if (!typeName || typeName.length > MAX_TYPE_NAME_LENGTH) return; - // Extract simple type name (strip Promise<>, etc.) — reuse shared utility - const simpleType = extractReturnTypeName(typeName) ?? typeName; - if (!simpleType) return; - let fileExports = result.get(filePath); - if (!fileExports) { - fileExports = new Map(); - result.set(filePath, fileExports); - } - if (fileExports.size < MAX_EXPORTS_PER_FILE) { - fileExports.set(name, simpleType); - } + accumulateExportedTypesFromParsedNode(result, node, symbolTable); }); return result; } @@ -448,79 +446,3 @@ export const processNextjsFetchRoutes = ( } } }; - -/** - * Extract fetch() calls from source files (sequential path). - * Workers handle this via tree-sitter captures in parse-worker; this function - * provides the same extraction for the sequential fallback path. - */ -export const extractFetchCallsFromFiles = async ( - files: { path: string; content: string }[], - astCache: ASTCache, -): Promise => { - const parser = await loadParser(); - const result: ExtractedFetchCall[] = []; - - for (const file of files) { - const language = getLanguageFromFilename(file.path); - if (!language) continue; - if (!isLanguageAvailable(language)) continue; - - const provider = getProvider(language); - const queryStr = provider.treeSitterQueries; - if (!queryStr) continue; - - await loadLanguage(language, file.path); - - let tree = astCache.get(file.path); - if (!tree) { - const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; - try { - tree = parseSourceSafe(parser, parseContent, undefined, { - bufferSize: getTreeSitterBufferSize(parseContent), - }); - } catch { - continue; - } - astCache.set(file.path, tree); - } - - let matches; - try { - const lang = parser.getLanguage(); - const query = new Parser.Query(lang, queryStr); - matches = query.matches(tree.rootNode); - } catch { - continue; - } - - for (const match of matches) { - const captureMap: Record = {}; - match.captures.forEach((c) => (captureMap[c.name] = c.node)); - - if (captureMap['route.fetch']) { - const urlNode = captureMap['route.url'] ?? captureMap['route.template_url']; - if (urlNode) { - result.push({ - filePath: file.path, - fetchURL: urlNode.text, - lineNumber: captureMap['route.fetch'].startPosition.row, - }); - } - } else if (captureMap['http_client'] && captureMap['http_client.url']) { - const method = captureMap['http_client.method']?.text; - const url = captureMap['http_client.url'].text; - const HTTP_CLIENT_ONLY = new Set(['head', 'options', 'request', 'ajax']); - if (method && HTTP_CLIENT_ONLY.has(method) && url.startsWith('/')) { - result.push({ - filePath: file.path, - fetchURL: url, - lineNumber: captureMap['http_client'].startPosition.row, - }); - } - } - } - } - - return result; -}; diff --git a/gitnexus/src/core/ingestion/cfg/cfg-builder.ts b/gitnexus/src/core/ingestion/cfg/cfg-builder.ts new file mode 100644 index 000000000..976b46824 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/cfg-builder.ts @@ -0,0 +1,132 @@ +/** + * CfgBuilder (issue #2081, M1) — the language-agnostic accumulator. + * + * A per-language `CfgVisitor` drives this: it creates blocks as it walks + * statements, wires edges (including back-edges and break/continue/return/throw + * targets resolved via {@link ControlFlowContext}), and calls {@link finish} to + * produce the serializable {@link FunctionCfg}. The builder owns the synthetic + * ENTRY (index 0) and EXIT blocks and de-duplicates identical edges so repeated + * `connect` calls (common when wiring a set of dangling exits) stay idempotent. + * + * It has no knowledge of any AST — it is exercised directly in unit tests with + * hand-built block sequences, which is how the classic CFG hazards are pinned + * before the tree-sitter visitor (U2) drives it. + */ +import type { BasicBlockData, CfgEdgeData, CfgEdgeKind, FunctionCfg } from './types.js'; + +interface MutableBlock { + startLine: number; + endLine: number; + /** + * Block source accumulated as fragments, joined once in {@link finish}. A + * coalescing straight-line run appends one fragment per statement; storing + * them as an array and joining at the end keeps that O(n) instead of the + * O(n²) of repeatedly concatenating onto a growing string (a long generated + * init function is the worst case — see bench/cfg). + */ + textParts: string[]; + kind: BasicBlockData['kind']; +} + +export class CfgBuilder { + private readonly blocks: MutableBlock[] = []; + private readonly edges: CfgEdgeData[] = []; + private readonly edgeKeys = new Set(); + readonly entryIndex: number; + readonly exitIndex: number; + + constructor( + private readonly filePath: string, + private readonly functionStartLine: number, + private readonly functionEndLine: number, + /** Start column of the owning function — disambiguates same-line functions + * in the BasicBlock ids (see {@link FunctionCfg.functionStartColumn}). + * Defaults to 0 for hand-built test CFGs that don't model columns. */ + private readonly functionStartColumn: number = 0, + ) { + this.entryIndex = this.newBlock(functionStartLine, functionStartLine, '', 'entry'); + this.exitIndex = this.newBlock(functionEndLine, functionEndLine, '', 'exit'); + } + + /** Create a block and return its index. */ + newBlock( + startLine: number, + endLine: number, + text: string, + kind: BasicBlockData['kind'] = 'normal', + ): number { + this.blocks.push({ startLine, endLine, textParts: text ? [text] : [], kind }); + return this.blocks.length - 1; + } + + /** Add a single edge (idempotent on from+to+kind). */ + edge(from: number, to: number, kind: CfgEdgeKind): void { + const key = `${from}->${to}:${kind}`; + if (this.edgeKeys.has(key)) return; + this.edgeKeys.add(key); + this.edges.push({ from, to, kind }); + } + + /** Wire a set of dangling exits to a single target block with one kind. */ + connect(exits: readonly number[], to: number, kind: CfgEdgeKind = 'seq'): void { + for (const from of exits) this.edge(from, to, kind); + } + + /** Extend a block's end line as more statements accrue to it. */ + extendBlock(index: number, endLine: number, appendText?: string): void { + const b = this.blocks[index]; + if (!b) return; + if (endLine > b.endLine) b.endLine = endLine; + if (appendText) b.textParts.push(appendText); + } + + get blockCount(): number { + return this.blocks.length; + } + + /** Produce the serializable CFG. Caller is responsible for having wired the + * function's dangling exits to {@link exitIndex} before calling. */ + finish(): FunctionCfg { + return { + filePath: this.filePath, + functionStartLine: this.functionStartLine, + functionEndLine: this.functionEndLine, + functionStartColumn: this.functionStartColumn, + entryIndex: this.entryIndex, + exitIndex: this.exitIndex, + blocks: this.blocks.map((b, index) => ({ + index, + startLine: b.startLine, + endLine: b.endLine, + text: b.textParts.join('\n'), + kind: b.kind, + })), + edges: [...this.edges], + }; + } +} + +/** + * Block indices reachable from `entryIndex` by following edges. Backs the + * reachability property tests (R9) over hand-built and visitor-produced CFGs. + */ +export const reachableBlocks = (cfg: FunctionCfg): Set => { + const adj = new Map(); + for (const e of cfg.edges) { + const list = adj.get(e.from); + if (list) list.push(e.to); + else adj.set(e.from, [e.to]); + } + const seen = new Set([cfg.entryIndex]); + const stack = [cfg.entryIndex]; + while (stack.length) { + const n = stack.pop() as number; + for (const next of adj.get(n) ?? []) { + if (!seen.has(next)) { + seen.add(next); + stack.push(next); + } + } + } + return seen; +}; diff --git a/gitnexus/src/core/ingestion/cfg/collect.ts b/gitnexus/src/core/ingestion/cfg/collect.ts new file mode 100644 index 000000000..890987f7d --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/collect.ts @@ -0,0 +1,63 @@ +/** + * collectFunctionCfgs (issue #2081, M1). + * + * Walks a parsed file's tree-sitter tree and builds one {@link FunctionCfg} per + * CFG-bearing function via the language's {@link CfgVisitor}. Runs IN THE PARSE + * WORKER (where the AST lives — KTD1/KTD7); the result rides on + * `ParsedFile.cfgSideChannel` across the worker→main boundary. + * + * Nested functions are enumerated independently — each gets its own CFG, and + * appears as an opaque straight-line block in its enclosing function's CFG (the + * visitor does not descend into nested function bodies). `maxFunctionLines` + * bounds per-function cost: a function whose source span exceeds the cap is + * skipped (and counted) rather than walked, so a pathological mega-function + * cannot blow up worker time/memory. A cap of `0` means no limit. + */ +import type { SyntaxNode } from '../utils/ast-helpers.js'; +import type { CfgVisitor, FunctionCfg } from './types.js'; + +/** + * Default per-function source-line cap used by the worker when the `--pdg` run + * does not specify `pdgMaxFunctionLines`. A function longer than this (almost + * always minified/generated code) is skipped rather than walked — its CFG is + * both expensive and low-value. Overridable via `PipelineOptions.pdgMaxFunctionLines`. + */ +export const DEFAULT_PDG_MAX_FUNCTION_LINES = 2000; + +export interface CollectedCfgs { + readonly cfgs: readonly FunctionCfg[]; + /** Functions skipped for exceeding `maxFunctionLines` (0 ⇒ none skipped). */ + readonly skipped: number; +} + +export function collectFunctionCfgs( + root: SyntaxNode, + visitor: CfgVisitor, + filePath: string, + maxFunctionLines = 0, +): CollectedCfgs { + const cfgs: FunctionCfg[] = []; + let skipped = 0; + const stack: SyntaxNode[] = [root]; + + while (stack.length) { + const node = stack.pop() as SyntaxNode; + if (visitor.isFunction(node)) { + const lines = node.endPosition.row - node.startPosition.row + 1; + if (maxFunctionLines > 0 && lines > maxFunctionLines) { + skipped++; + } else { + const cfg = visitor.buildFunctionCfg(node, filePath); + if (cfg) cfgs.push(cfg); + } + } + // Descend regardless (a skipped mega-function may still contain small + // nested functions that are worth a CFG of their own). + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child) stack.push(child); + } + } + + return { cfgs, skipped }; +} diff --git a/gitnexus/src/core/ingestion/cfg/control-flow-context.ts b/gitnexus/src/core/ingestion/cfg/control-flow-context.ts new file mode 100644 index 000000000..38c7bcbb8 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/control-flow-context.ts @@ -0,0 +1,70 @@ +/** + * ControlFlowContext (issue #2081, M1). + * + * Resolves the targets of `break`/`continue` (plain and labeled) as the visitor + * descends through loops and switches. Loops and switches push a target frame + * on entry and pop it on exit; a labeled statement attaches its label to the + * frame of the construct it labels, so `break outer` / `continue outer` resolve + * against the right enclosing loop/switch rather than the nearest one. + */ + +interface LoopFrame { + readonly kind: 'loop'; + /** Block a `continue` jumps to (the loop header / update). */ + readonly continueTo: number; + /** Block a `break` jumps to (the loop exit / join). */ + readonly breakTo: number; + readonly label?: string; +} + +interface SwitchFrame { + readonly kind: 'switch'; + /** Block a `break` jumps to (after the switch). `continue` is invalid here. */ + readonly breakTo: number; + readonly label?: string; +} + +type Frame = LoopFrame | SwitchFrame; + +export class ControlFlowContext { + private readonly stack: Frame[] = []; + + pushLoop(continueTo: number, breakTo: number, label?: string): void { + this.stack.push({ kind: 'loop', continueTo, breakTo, label }); + } + + pushSwitch(breakTo: number, label?: string): void { + this.stack.push({ kind: 'switch', breakTo, label }); + } + + pop(): void { + this.stack.pop(); + } + + /** + * Target block for a `break`. With a label, the nearest enclosing frame + * carrying that label (loop or switch); without, the nearest frame of any + * kind. Returns `undefined` if there is no valid target (malformed input). + */ + breakTarget(label?: string): number | undefined { + for (let i = this.stack.length - 1; i >= 0; i--) { + const f = this.stack[i]; + if (label === undefined || f.label === label) return f.breakTo; + } + return undefined; + } + + /** + * Target block for a `continue`. With a label, the nearest enclosing **loop** + * carrying that label; without, the nearest loop (switches are skipped — you + * cannot `continue` a switch). Returns `undefined` if there is no valid loop. + */ + continueTarget(label?: string): number | undefined { + for (let i = this.stack.length - 1; i >= 0; i--) { + const f = this.stack[i]; + if (f.kind !== 'loop') continue; + if (label === undefined || f.label === label) return f.continueTo; + } + return undefined; + } +} diff --git a/gitnexus/src/core/ingestion/cfg/emit.ts b/gitnexus/src/core/ingestion/cfg/emit.ts new file mode 100644 index 000000000..6531b723e --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/emit.ts @@ -0,0 +1,147 @@ +/** + * cfg/emit.ts (issue #2081, M1) — serialized side-channel → graph. + * + * Pure helper: given a file's per-function CFGs (off `ParsedFile.cfgSideChannel`, + * produced by the worker in U3), emit one persisted `BasicBlock` node per block + * and one `CFG` edge per edge into the {@link KnowledgeGraph}. Invoked from + * scope-resolution (run.ts Phase 4) while the disk-backed ParsedFile store is + * still live — the only window where the worker-built CFGs are loaded (KTD1/ + * KTD5). Default (`--pdg` off) runs never call this, so the emitted graph stays + * byte-identical to a pre-#2081 run. + * + * BasicBlock id: `BasicBlock::::` + * (KTD3). The function start line+column segments disambiguate blocks across + * multiple functions in one file — including same-line functions — since each + * function's block indices restart at 0; blocks carry no `name` (the + * BasicBlock table has no such column). The edge KIND + * (`seq`/`cond-true`/…) rides in the relationship `reason` — CFG edges are + * values of the single `CodeRelation` table's `type` column (`'CFG'`), so the + * kind cannot be its own edge type and is queried via `reason`. + */ +import type { KnowledgeGraph } from '../../graph/types.js'; +import { generateId } from '../../../lib/utils.js'; +import type { FunctionCfg } from './types.js'; + +/** + * Default per-function CFG edge cap. A pathological generated function could + * otherwise emit an unbounded edge set; the cap bounds graph growth and is + * overridable via `--pdg` options. `0` (in options) means no cap (unlimited + * — see the `cap` mapping in {@link emitFileCfgs}); `undefined` means this + * default. + */ +export const DEFAULT_MAX_CFG_EDGES_PER_FUNCTION = 5000; + +export interface CfgEmitResult { + blocks: number; + edges: number; + /** Edges dropped because a function's edge count exceeded the cap. */ + droppedEdges: number; + /** Number of functions that hit the cap. */ + cappedFunctions: number; +} + +const basicBlockId = ( + filePath: string, + functionStartLine: number, + functionStartColumn: number, + blockIndex: number, +): string => `BasicBlock:${filePath}:${functionStartLine}:${functionStartColumn}:${blockIndex}`; + +/** + * Whether an untrusted `cfgSideChannel` element is safe to feed to + * {@link emitFileCfgs}. Deliberately NOT full FunctionCfg validation — it + * checks exactly the fields whose corruption is SILENT given emit's + * mechanics: {@link basicBlockId} string-templates every id-anchor value + * (filePath, function start line/column, block index, edge endpoints) and + * the graph's addNode/addRelationship are no-throw Map inserts. Unchecked, + * a missing anchor field cross-wires same-`undefined`-id blocks across + * functions (addNode is first-writer-wins), and an edge endpoint that + * matches no block index becomes a dangling `BasicBlock:…:` edge that + * detonates much later at DB bulk-load instead of throwing here — so + * endpoints are checked for MEMBERSHIP in the block-index set, not just + * integer-ness. Lives in this module so the guard evolves with the id + * templating it defends (#2099 F4; M2 fields that join the id path must + * join this check). + */ +export const isEmitSafeCfg = (cfg: FunctionCfg | undefined | null): cfg is FunctionCfg => { + if ( + typeof cfg?.filePath !== 'string' || + !Number.isInteger(cfg.functionStartLine) || + !Number.isInteger(cfg.functionStartColumn) || + !Array.isArray(cfg.blocks) || + !Array.isArray(cfg.edges) + ) { + return false; + } + const blockIndices = new Set(); + for (const b of cfg.blocks) { + if (!Number.isInteger(b?.index)) return false; + blockIndices.add(b.index); + } + return cfg.edges.every((e) => blockIndices.has(e?.from) && blockIndices.has(e?.to)); +}; + +/** + * Emit BasicBlock nodes + CFG edges for every function CFG in `cfgs`. + * + * `maxEdgesPerFunction` caps edges per function. On overflow we stop emitting + * that function's remaining edges and call `onWarn` naming the dropped count — + * no silent truncation (KTD6/R6). Block nodes are always fully emitted (their + * count is bounded by the function's statement count); only edges are capped. + */ +export function emitFileCfgs( + graph: KnowledgeGraph, + cfgs: readonly FunctionCfg[], + maxEdgesPerFunction: number = DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, + onWarn?: (message: string) => void, +): CfgEmitResult { + const result: CfgEmitResult = { blocks: 0, edges: 0, droppedEdges: 0, cappedFunctions: 0 }; + const cap = maxEdgesPerFunction > 0 ? maxEdgesPerFunction : Infinity; + + for (const cfg of cfgs) { + const { filePath, functionStartLine, functionStartColumn } = cfg; + + for (const b of cfg.blocks) { + graph.addNode({ + id: basicBlockId(filePath, functionStartLine, functionStartColumn, b.index), + label: 'BasicBlock', + properties: { + name: '', // BasicBlock has no name column; identified by id + span + filePath, + startLine: b.startLine, + endLine: b.endLine, + text: b.text, + }, + }); + result.blocks++; + } + + let emittedForFn = 0; + for (const e of cfg.edges) { + if (emittedForFn >= cap) { + const dropped = cfg.edges.length - emittedForFn; + result.droppedEdges += dropped; + result.cappedFunctions++; + onWarn?.( + `[cfg] ${filePath}:${functionStartLine}: per-function CFG edge cap ` + + `(${maxEdgesPerFunction}) reached — dropped ${dropped} of ${cfg.edges.length} edges`, + ); + break; + } + const sourceId = basicBlockId(filePath, functionStartLine, functionStartColumn, e.from); + const targetId = basicBlockId(filePath, functionStartLine, functionStartColumn, e.to); + graph.addRelationship({ + id: generateId('CFG', `${sourceId}->${targetId}:${e.kind}`), + type: 'CFG', + sourceId, + targetId, + confidence: 1.0, + reason: e.kind, // CfgEdgeKind (seq/cond-true/loop-back/…) — queryable + }); + result.edges++; + emittedForFn++; + } + } + + return result; +} diff --git a/gitnexus/src/core/ingestion/cfg/traversal-result.ts b/gitnexus/src/core/ingestion/cfg/traversal-result.ts new file mode 100644 index 000000000..d26500fa5 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/traversal-result.ts @@ -0,0 +1,21 @@ +/** + * TraversalResult (issue #2081, M1). + * + * Visiting a statement (or a statement sequence) returns the block its control + * flow ENTERS through, plus the set of blocks whose **normal** control flows + * out the bottom (the "dangling exits") — to be wired to the entry of whatever + * comes next. Abnormal exits (return/break/continue/throw) are wired directly + * to their targets during the walk and are NOT part of `exits`. + * + * A statement that cannot fall through (e.g. ends in `return`/`throw`, or both + * branches of an `if` return) yields an empty `exits` array. + */ +export interface TraversalResult { + /** Block index control enters this statement/sequence through. */ + readonly entry: number; + /** Block indices whose normal control falls out the bottom (may be empty). */ + readonly exits: readonly number[]; +} + +/** A sequence of statements that produced no blocks (e.g. an empty body). */ +export const emptyTraversal = (entry: number): TraversalResult => ({ entry, exits: [entry] }); diff --git a/gitnexus/src/core/ingestion/cfg/types.ts b/gitnexus/src/core/ingestion/cfg/types.ts new file mode 100644 index 000000000..0b28c6089 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/types.ts @@ -0,0 +1,82 @@ +/** + * CFG data model — plain, JSON-serializable types (issue #2081, M1). + * + * These cross the worker→main boundary and the disk-backed/durable ParsedFile + * store, so they must contain NO tree-sitter AST references, class instances, + * or anything that does not survive `JSON.stringify` → `JSON.parse`. Block and + * edge endpoints are referenced by integer index within a function's CFG. + * + * The per-language `CfgVisitor` (built in the parse worker, where the AST + * lives — see the M1 plan KTD1/KTD7) produces a `FunctionCfg` per function; the + * array of them is what rides on `ParsedFile.cfgSideChannel`. + */ + +/** A basic block: a maximal straight-line run of statements between leaders. */ +export interface BasicBlockData { + /** Block index within its function. The synthetic ENTRY is always 0. */ + readonly index: number; + readonly startLine: number; + readonly endLine: number; + /** Source snippet for the block (empty for synthetic ENTRY/EXIT). */ + readonly text: string; + readonly kind: 'entry' | 'exit' | 'normal'; +} + +/** Why one block flows to another — drives the `reason` on the emitted CFG edge. */ +export type CfgEdgeKind = + | 'seq' // straight-line fallthrough + | 'cond-true' // branch taken (if/while/for condition true) + | 'cond-false' // branch not taken / loop exit + | 'loop-back' // back-edge to a loop header + | 'break' // break → loop/switch exit + | 'continue' // continue → loop header + | 'return' // return → function EXIT + | 'throw' // throw → nearest handler / finally / EXIT + | 'switch-case' // dispatch to a case + | 'fallthrough'; // switch case → next case (no break) + +export interface CfgEdgeData { + readonly from: number; + readonly to: number; + readonly kind: CfgEdgeKind; +} + +/** One function's control-flow graph. `cfgSideChannel` is `readonly FunctionCfg[]`. */ +export interface FunctionCfg { + readonly filePath: string; + /** Source span of the owning function — anchors the BasicBlock node ids. */ + readonly functionStartLine: number; + readonly functionEndLine: number; + /** + * Start COLUMN of the owning function. Combined with `functionStartLine` it + * disambiguates the BasicBlock node ids when two functions share a start line + * — e.g. `{ a: () => x(), b: () => y() }`, where both arrows begin on the same + * line and each restarts its block indices at 0. Without the column the ids + * collide and the graph's first-writer-wins `addNode` silently drops the + * second function's blocks and cross-wires its edges. + */ + readonly functionStartColumn: number; + readonly entryIndex: number; + readonly exitIndex: number; + readonly blocks: readonly BasicBlockData[]; + readonly edges: readonly CfgEdgeData[]; +} + +/** + * Per-language CFG strategy. Invoked **in the parse worker** for each function + * node. `TNode` is the language's AST node type (tree-sitter `SyntaxNode` for + * TS/JS) — kept generic so this module stays AST-library-agnostic. Returns + * `undefined` when the node is not a CFG-bearing function (the caller skips it). + */ +export interface CfgVisitor { + buildFunctionCfg(fnNode: TNode, filePath: string): FunctionCfg | undefined; + + /** + * Whether `node` is a CFG-bearing function this visitor handles. Lets the + * worker enumerate functions (and apply the per-function line budget) by a + * cheap node-type test, instead of attempting to build a CFG for every AST + * node. `buildFunctionCfg` still re-checks, so this is purely an optimization + * + the seam the line-budget hooks into. + */ + isFunction(node: TNode): boolean; +} diff --git a/gitnexus/src/core/ingestion/cfg/visitors/typescript.ts b/gitnexus/src/core/ingestion/cfg/visitors/typescript.ts new file mode 100644 index 000000000..79643b8d4 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/visitors/typescript.ts @@ -0,0 +1,581 @@ +/** + * TS/JS CfgVisitor (issue #2081, M1). + * + * Walks a TypeScript/JavaScript function's tree-sitter AST and drives the + * language-agnostic {@link CfgBuilder} to produce a serializable + * {@link FunctionCfg}. TS and JS share a grammar family (tree-sitter-typescript + * reuses tree-sitter-javascript's statement nodes), so one visitor covers both. + * + * Design — a `visit_` dispatch over the statement taxonomy. The + * classic CFG hazards (R10) are handled explicitly: + * - loops allocate a dedicated **loop-exit** block so `break` has a concrete + * target before the loop's successor is known; `continue` targets the + * header/increment; the back-edge closes the loop. + * - `switch` cases fall through naturally: a case body that does not `break` + * yields non-empty `exits`, which we wire to the next case as `fallthrough`; + * a case that `break`s wires to the switch exit (via {@link ControlFlowContext}) + * and yields no fall-out. + * - `try/catch/finally` routes both normal completion AND a `throw` in the try + * through `finally` (the finally block post-dominates the try/catch); a + * `throw` with no catch propagates through finally to the enclosing handler. + * - labeled `break`/`continue` resolve against the labeled loop's frame. + * + * Known M1 limitations: + * - SOUNDNESS GAP (M2 blocker, not mere precision): a non-local jump + * (`break`/`continue`/`return`) out of a `try` that has a `finally` edges + * directly to its target rather than routing THROUGH the `finally` block + * first. A future taint/PDG pass will therefore MISS flow mediated by a + * `finally` on the early-exit path (e.g. a value the `finally` taints or + * sanitizes before the `return` reaches its target) — a false negative. The + * general fix duplicates `finally` per exit path; deferred past M1 and + * tracked for M2. Normal completion and `throw` DO route through `finally`. + * - A `break`/`continue` to a label on a non-loop/non-switch block, and the + * OUTER label of a doubly-labeled construct (`outer: inner: for (...)`), are + * not modeled. The jump is conservatively routed to the function EXIT (a + * sound over-approximation that keeps the graph single-exit — see visitBreak) + * rather than left as a dangling sink; only the precise labeled target is + * unmodeled. Single-labeled loops/switches resolve correctly. + * + * Block/edge accounting and reachability are pinned in + * `test/unit/cfg/cfg-builder.test.ts` (core) and + * `test/unit/cfg/typescript-visitor.test.ts` (this visitor, per hazard). + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { CfgBuilder } from '../cfg-builder.js'; +import { ControlFlowContext } from '../control-flow-context.js'; +import type { TraversalResult } from '../traversal-result.js'; +import type { CfgVisitor, FunctionCfg } from '../types.js'; + +/** TS/JS node types that own a CFG-bearing function body. */ +const TS_FUNCTION_TYPES = new Set([ + 'function_declaration', + 'function_expression', + 'arrow_function', + 'method_definition', + 'generator_function_declaration', + 'generator_function', + 'async_function_declaration', + 'async_arrow_function', +]); + +/** Statement node types that break a basic block (everything else coalesces). */ +const CONTROL_FLOW_TYPES = new Set([ + 'if_statement', + 'while_statement', + 'do_statement', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'switch_statement', + 'try_statement', + 'return_statement', + 'break_statement', + 'continue_statement', + 'throw_statement', + 'labeled_statement', + 'statement_block', +]); + +const LOOP_OR_SWITCH_TYPES = new Set([ + 'while_statement', + 'do_statement', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'switch_statement', +]); + +const startLineOf = (n: SyntaxNode): number => n.startPosition.row + 1; +const endLineOf = (n: SyntaxNode): number => n.endPosition.row + 1; + +/** A statement sequence that produced no blocks (empty body) is "transparent". */ +type SeqResult = TraversalResult | null; + +/** + * Per-function walk state. One instance is created per function so the + * {@link ControlFlowContext}, exception-handler stack, and pending label are + * scoped to that function and never leak across functions. + */ +class TsCfgWalk { + private readonly cfc = new ControlFlowContext(); + /** Stack of exception-handler entry blocks (catch/finally) a `throw` jumps to. */ + private readonly handlers: number[] = []; + /** Label awaiting the loop/switch it immediately precedes (labeled_statement). */ + private pendingLabel: string | undefined; + + constructor(private readonly builder: CfgBuilder) {} + + /** Statements of a block node, ignoring comments. */ + private statementsOf(block: SyntaxNode): SyntaxNode[] { + return block.namedChildren.filter((c) => c.type !== 'comment'); + } + + /** The `body` block of a node (field, or the first statement_block child). */ + private bodyBlockOf(node: SyntaxNode): SyntaxNode | undefined { + return ( + node.childForFieldName('body') ?? node.namedChildren.find((c) => c.type === 'statement_block') + ); + } + + /** Visit a body that may be a `statement_block` or a single statement. */ + private visitBody(node: SyntaxNode | undefined | null): SeqResult { + if (!node) return null; + if (node.type === 'statement_block') return this.visitSeq(this.statementsOf(node)); + return this.visitStmt(node); + } + + /** Wire a sequence of statements, coalescing straight-line runs into blocks. */ + visitSeq(stmts: SyntaxNode[]): SeqResult { + let entry: number | undefined; + let dangling: number[] = []; + let openSimple: number | undefined; + + for (const stmt of stmts) { + if (CONTROL_FLOW_TYPES.has(stmt.type)) { + openSimple = undefined; // close any open straight-line block + const res = this.visitStmt(stmt); + if (res === null) continue; // transparent (empty nested block) + if (entry === undefined) entry = res.entry; + else this.builder.connect(dangling, res.entry, 'seq'); + dangling = [...res.exits]; + } else { + // Simple statement — coalesce into the current straight-line block. + if (openSimple === undefined) { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + if (entry === undefined) entry = idx; + else this.builder.connect(dangling, idx, 'seq'); + openSimple = idx; + dangling = [idx]; + } else { + this.builder.extendBlock(openSimple, endLineOf(stmt), stmt.text); + } + } + } + + if (entry === undefined) return null; + return { entry, exits: dangling }; + } + + /** Dispatch one statement to its handler. Non-null except for empty blocks. */ + visitStmt(stmt: SyntaxNode): SeqResult { + switch (stmt.type) { + case 'if_statement': + return this.visitIf(stmt); + case 'while_statement': + return this.visitWhile(stmt); + case 'do_statement': + return this.visitDoWhile(stmt); + case 'for_statement': + return this.visitFor(stmt); + case 'for_in_statement': + case 'for_of_statement': + return this.visitForIn(stmt); + case 'switch_statement': + return this.visitSwitch(stmt); + case 'try_statement': + return this.visitTry(stmt); + case 'return_statement': + return this.visitReturn(stmt); + case 'throw_statement': + return this.visitThrow(stmt); + case 'break_statement': + return this.visitBreak(stmt); + case 'continue_statement': + return this.visitContinue(stmt); + case 'labeled_statement': + return this.visitLabeled(stmt); + case 'statement_block': + return this.visitSeq(this.statementsOf(stmt)); + default: + return this.visitSimple(stmt); + } + } + + private visitSimple(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + return { entry: idx, exits: [idx] }; + } + + private visitReturn(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + this.builder.edge(idx, this.builder.exitIndex, 'return'); + return { entry: idx, exits: [] }; + } + + private visitThrow(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + this.builder.edge(idx, this.currentHandler(), 'throw'); + return { entry: idx, exits: [] }; + } + + private visitBreak(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const target = this.cfc.breakTarget(this.labelOf(stmt)); + // An unresolved target — a label this M1 visitor doesn't model (a stacked + // outer label like `outer: inner: for`, or a labeled non-loop block) — + // would otherwise leave this block with NO out-edge, stranding it and + // breaking the single-exit invariant a downstream post-dominator / PDG pass + // relies on. Conservatively route an unresolved jump to the function EXIT + // ("escapes the function"): sound over-approximation, keeps single-exit. + this.builder.edge(idx, target ?? this.builder.exitIndex, 'break'); + return { entry: idx, exits: [] }; + } + + private visitContinue(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const target = this.cfc.continueTarget(this.labelOf(stmt)); + // See visitBreak: an unresolved label routes to EXIT to preserve single-exit. + this.builder.edge(idx, target ?? this.builder.exitIndex, 'continue'); + return { entry: idx, exits: [] }; + } + + private visitLabeled(stmt: SyntaxNode): SeqResult { + const body = + stmt.childForFieldName('body') ?? stmt.namedChildren[stmt.namedChildren.length - 1]; + if (body && LOOP_OR_SWITCH_TYPES.has(body.type)) { + this.pendingLabel = this.labelOf(stmt); + const res = this.visitStmt(body); + this.pendingLabel = undefined; // clear even if the construct didn't consume it + return res; + } + // Labeled non-loop blocks (break-to-block-label) are not modeled in M1. + return this.visitBody(body); + } + + private visitIf(stmt: SyntaxNode): TraversalResult { + const cond = stmt.childForFieldName('condition') ?? stmt; + const condBlock = this.builder.newBlock(startLineOf(stmt), endLineOf(cond), cond.text); + + const exits: number[] = []; + + const thenRes = this.visitBody(stmt.childForFieldName('consequence')); + if (thenRes) { + this.builder.edge(condBlock, thenRes.entry, 'cond-true'); + exits.push(...thenRes.exits); + } else { + exits.push(condBlock); // empty then — true path falls through + } + + const elseNode = this.elseBodyOf(stmt); + if (elseNode) { + const elseRes = this.visitBody(elseNode); + if (elseRes) { + this.builder.edge(condBlock, elseRes.entry, 'cond-false'); + exits.push(...elseRes.exits); + } else { + exits.push(condBlock); // empty else block + } + } else { + exits.push(condBlock); // no else — false path falls through to the join + } + + return { entry: condBlock, exits: [...new Set(exits)] }; + } + + /** The else body node (unwraps an `else_clause` wrapper if present). */ + private elseBodyOf(ifStmt: SyntaxNode): SyntaxNode | undefined { + const alt = ifStmt.childForFieldName('alternative'); + if (!alt) return undefined; + if (alt.type === 'else_clause') { + return alt.childForFieldName('body') ?? alt.namedChildren[0]; + } + return alt; + } + + private visitWhile(stmt: SyntaxNode): TraversalResult { + const label = this.takeLabel(); + const cond = stmt.childForFieldName('condition') ?? stmt; + const header = this.builder.newBlock(startLineOf(stmt), endLineOf(cond), cond.text); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, label); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); // empty body re-tests + } + this.builder.edge(header, loopExit, 'cond-false'); + return { entry: header, exits: [loopExit] }; + } + + private visitDoWhile(stmt: SyntaxNode): TraversalResult { + const label = this.takeLabel(); + const cond = stmt.childForFieldName('condition') ?? stmt; + const condBlock = this.builder.newBlock(startLineOf(cond), endLineOf(cond), cond.text); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(condBlock, loopExit, label); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + const backTarget = body ? body.entry : condBlock; + if (body) this.builder.connect(body.exits, condBlock, 'seq'); + this.builder.edge(condBlock, backTarget, 'loop-back'); // cond true → run body again + this.builder.edge(condBlock, loopExit, 'cond-false'); + return { entry: backTarget, exits: [loopExit] }; + } + + private visitFor(stmt: SyntaxNode): TraversalResult { + const label = this.takeLabel(); + const init = stmt.childForFieldName('initializer'); + const cond = stmt.childForFieldName('condition'); + const incr = stmt.childForFieldName('increment'); + + const header = this.builder.newBlock( + startLineOf(stmt), + cond ? endLineOf(cond) : startLineOf(stmt), + cond ? cond.text : 'for(;;)', + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + let incrBlock = header; + if (incr) { + incrBlock = this.builder.newBlock(startLineOf(incr), endLineOf(incr), incr.text); + this.builder.edge(incrBlock, header, 'loop-back'); + } + + this.cfc.pushLoop(incrBlock, loopExit, label); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + // With no increment clause the body's exits ARE the back-edge — carry + // the loop-back kind on them (mirroring visitWhile/visitForIn) instead + // of a phantom header→header self-loop that models a path which never + // executes the body. With an increment, the body falls through to the + // increment (`seq`) and the increment carries the loop-back (:338). + this.builder.connect(body.exits, incrBlock, incr ? 'seq' : 'loop-back'); + } else { + this.builder.edge(header, incrBlock, 'cond-true'); + // Empty body with no increment: the header genuinely re-tests itself. + if (!incr) this.builder.edge(header, header, 'loop-back'); + } + this.builder.edge(header, loopExit, 'cond-false'); + + let entry = header; + if (init) { + const initBlock = this.builder.newBlock(startLineOf(init), endLineOf(init), init.text); + this.builder.edge(initBlock, header, 'seq'); + entry = initBlock; + } + return { entry, exits: [loopExit] }; + } + + private visitForIn(stmt: SyntaxNode): TraversalResult { + const label = this.takeLabel(); + const header = this.builder.newBlock( + startLineOf(stmt), + startLineOf(stmt), + this.forInHeaderText(stmt), + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, label); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); + } + this.builder.edge(header, loopExit, 'cond-false'); + return { entry: header, exits: [loopExit] }; + } + + private forInHeaderText(stmt: SyntaxNode): string { + const left = stmt.childForFieldName('left')?.text ?? ''; + const right = stmt.childForFieldName('right')?.text ?? ''; + return left || right ? `for(${left} … ${right})` : 'for(… in/of …)'; + } + + private visitSwitch(stmt: SyntaxNode): TraversalResult { + const label = this.takeLabel(); + const value = stmt.childForFieldName('value') ?? stmt; + const dispatch = this.builder.newBlock(startLineOf(stmt), endLineOf(value), value.text); + const switchExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushSwitch(switchExit, label); + const body = stmt.childForFieldName('body'); + const cases = body + ? body.namedChildren.filter((c) => c.type === 'switch_case' || c.type === 'switch_default') + : []; + + const caseResults = cases.map((c) => this.visitSeq(this.caseStatements(c))); + const hasDefault = cases.some((c) => c.type === 'switch_default'); + + // entryOf[i] = block a dispatch/fallthrough INTO case i lands on (empty + // cases are transparent — they resolve to the next case, or the exit). + const entryOf: number[] = new Array(cases.length); + let after = switchExit; + for (let i = cases.length - 1; i >= 0; i--) { + entryOf[i] = caseResults[i]?.entry ?? after; + after = entryOf[i]; + } + + for (let i = 0; i < cases.length; i++) { + this.builder.edge(dispatch, entryOf[i], 'switch-case'); + } + if (!hasDefault) this.builder.edge(dispatch, switchExit, 'switch-case'); // no-match path + + for (let i = 0; i < cases.length; i++) { + const res = caseResults[i]; + if (!res) continue; + const fallTarget = i + 1 < cases.length ? entryOf[i + 1] : switchExit; + this.builder.connect(res.exits, fallTarget, 'fallthrough'); + } + + this.cfc.pop(); + return { entry: dispatch, exits: [switchExit] }; + } + + private caseStatements(caseNode: SyntaxNode): SyntaxNode[] { + const value = caseNode.childForFieldName('value'); + return caseNode.namedChildren.filter((c) => c.id !== value?.id && c.type !== 'comment'); + } + + private visitTry(stmt: SyntaxNode): SeqResult { + const bodyNode = stmt.childForFieldName('body'); + // Single pass over named children — tree-sitter's `namedChildren` getter + // allocates a fresh array on every access, so avoid the double `.find`. + let catchClause: SyntaxNode | undefined; + let finallyClause: SyntaxNode | undefined; + for (let i = 0; i < stmt.namedChildCount; i++) { + const c = stmt.namedChild(i); + if (c?.type === 'catch_clause') catchClause = c; + else if (c?.type === 'finally_clause') finallyClause = c; + } + + // Build finally first so its entry is known as both a normal join and a + // handler target. The finally body runs in the OUTER handler context. + const finallyRes = finallyClause + ? this.visitSeq(this.statementsOf(this.bodyBlockOf(finallyClause) as SyntaxNode)) + : null; + + // A throw inside catch propagates to finally (if any), else the outer handler. + let catchRes: SeqResult = null; + if (catchClause) { + if (finallyRes) this.handlers.push(finallyRes.entry); + catchRes = this.visitSeq(this.statementsOf(this.bodyBlockOf(catchClause) as SyntaxNode)); + if (finallyRes) this.handlers.pop(); + if (catchRes === null) { + // Empty (or comment-only) catch body — `catch {}`. The clause still + // CATCHES: handler semantics key off the syntactic clause, not the + // traversal result. Treating it as "no catch" sent the swallowed + // exception to the outer handler/EXIT and left post-try code + // unreachable when the body always throws — a hard false-negative + // for downstream taint. Synthesize one empty block spanning the + // clause (entry == sole exit) so exception flow lands in it and + // rejoins the normal continuation. Created BEFORE the protected + // region is walked, so it never receives a spurious throw edge. + const idx = this.builder.newBlock(startLineOf(catchClause), endLineOf(catchClause), ''); + catchRes = { entry: idx, exits: [idx] }; + } + } + + // Handler for the try body: catch if present, else finally, else outer. + const tryHandler = catchRes?.entry ?? finallyRes?.entry ?? this.currentHandler(); + const protectedStart = this.builder.blockCount; + this.handlers.push(tryHandler); + const bodyRes = bodyNode ? this.visitSeq(this.statementsOf(bodyNode)) : null; + this.handlers.pop(); + + // Conservative exceptional edges: ANY block in the protected region may raise + // to the handler — not just an explicit `throw`, and not just the body ENTRY. + // Edging every block created during the try-body walk keeps exception flow + // sound when the body BRANCHES: an `if` / nested-try / post-branch block whose + // interior blocks would otherwise have no path to the handler — i.e. a taint + // false-negative into `catch` for the downstream PDG analysis. The + // per-function edge cap bounds the count; explicit `throw`s add their own + // (idempotent) edge to the same handler. + if (catchClause || finallyClause) { + for (let b = protectedStart; b < this.builder.blockCount; b++) { + this.builder.edge(b, tryHandler, 'throw'); + } + } + + const exits: number[] = []; + if (finallyRes) { + // Normal completion of try AND catch both flow through finally. + if (bodyRes) this.builder.connect(bodyRes.exits, finallyRes.entry, 'seq'); + if (catchRes) this.builder.connect(catchRes.exits, finallyRes.entry, 'seq'); + exits.push(...finallyRes.exits); + // No catch → an exception re-propagates out after finally runs. + if (!catchRes) this.builder.connect(finallyRes.exits, this.currentHandler(), 'throw'); + } else { + if (bodyRes) exits.push(...bodyRes.exits); + if (catchRes) exits.push(...catchRes.exits); + } + + const entry = bodyRes?.entry ?? finallyRes?.entry ?? catchRes?.entry; + if (entry === undefined) return null; + return { entry, exits: [...new Set(exits)] }; + } + + /** Nearest enclosing exception handler, or the function EXIT. */ + private currentHandler(): number { + return this.handlers.length ? this.handlers[this.handlers.length - 1] : this.builder.exitIndex; + } + + /** Consume the label awaiting the loop/switch this call is building. */ + private takeLabel(): string | undefined { + const label = this.pendingLabel; + this.pendingLabel = undefined; + return label; + } + + private labelOf(stmt: SyntaxNode): string | undefined { + const id = + stmt.childForFieldName('label') ?? + stmt.namedChildren.find((c) => c.type === 'statement_identifier'); + return id?.text; + } +} + +/** Build the CFG for one TS/JS function node (or `undefined` if not a function). */ +function buildFunctionCfg(fnNode: SyntaxNode, filePath: string): FunctionCfg | undefined { + if (!TS_FUNCTION_TYPES.has(fnNode.type)) return undefined; + const startLine = startLineOf(fnNode); + const endLine = endLineOf(fnNode); + const startColumn = fnNode.startPosition.column; + const builder = new CfgBuilder(filePath, startLine, endLine, startColumn); + + const body = fnNode.childForFieldName('body'); + if (!body) return undefined; // overload signature / abstract method — no body + + if (body.type !== 'statement_block') { + // Expression-bodied arrow: `() => expr` — one block whose value is returned. + const blk = builder.newBlock(startLineOf(body), endLineOf(body), body.text); + builder.edge(builder.entryIndex, blk, 'seq'); + builder.edge(blk, builder.exitIndex, 'return'); + return builder.finish(); + } + + const walk = new TsCfgWalk(builder); + const res = walk.visitSeq(body.namedChildren.filter((c) => c.type !== 'comment')); + if (!res) { + builder.edge(builder.entryIndex, builder.exitIndex, 'seq'); // empty body + return builder.finish(); + } + builder.edge(builder.entryIndex, res.entry, 'seq'); + builder.connect(res.exits, builder.exitIndex, 'seq'); // normal fall-off → EXIT + return builder.finish(); +} + +/** Whether a node is a TS/JS function this visitor builds a CFG for. */ +function isFunction(node: SyntaxNode): boolean { + return TS_FUNCTION_TYPES.has(node.type); +} + +/** The TS/JS CFG visitor (shared by TypeScript and JavaScript). */ +export function createTypeScriptCfgVisitor(): CfgVisitor { + return { buildFunctionCfg, isFunction }; +} + +export { TS_FUNCTION_TYPES }; diff --git a/gitnexus/src/core/ingestion/export-detection.ts b/gitnexus/src/core/ingestion/export-detection.ts index 31d0722f4..17494e7bb 100644 --- a/gitnexus/src/core/ingestion/export-detection.ts +++ b/gitnexus/src/core/ingestion/export-detection.ts @@ -4,7 +4,8 @@ * Determines whether a symbol (function, class, etc.) is exported/public * in its language. This is a pure function — safe for use in worker threads. * - * Shared between parse-worker.ts (worker pool) and parsing-processor.ts (sequential fallback). + * Used by the language providers during worker parsing (parse-worker.ts) — the + * sole parse path. (Sequential parsing was removed.) */ import { findSiblingChild, type SyntaxNode } from './utils/ast-helpers.js'; diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/field-extractors/configs/dart.ts index 52f4c0ca7..00c89b607 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/dart.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/dart.ts @@ -2,15 +2,59 @@ import { SupportedLanguages } from 'gitnexus-shared'; import type { FieldExtractionConfig } from '../generic.js'; +import type { FieldVisibility } from '../../field-types.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; import { hasKeyword } from './helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; /** * Dart field extraction config. * - * Dart class fields appear as declaration nodes inside class_body. + * Dart class fields appear as `declaration` nodes inside `class_body`. + * Two shapes carry the field name(s): + * - instance / plain fields → `initialized_identifier_list` + * (`int z = 0;`, `int a = 1, b = 2;`) + * - `static const` / `static final` / `const` fields → `static_final_declaration_list` + * (`static const a = 1;`, `static final String b = 'x', c = 'y';`) + * Both shapes may declare SEVERAL fields in one declaration, so name extraction + * is multi-name (`extractNames`). The structure query (`DART_QUERIES`) emits one + * `@definition.property` per name for both shapes; this config enriches each. + * * Visibility is convention-based: underscore prefix = private. */ + +/** All field names declared by a `declaration` node, across both Dart shapes. */ +function extractDartFieldNames(node: SyntaxNode): string[] { + const names: string[] = []; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child) continue; + + // instance / plain fields: initialized_identifier_list > initialized_identifier > identifier + if (child.type === 'initialized_identifier_list') { + for (let j = 0; j < child.namedChildCount; j++) { + const init = child.namedChild(j); + if (init?.type === 'initialized_identifier') { + const ident = init.firstNamedChild; + if (ident?.type === 'identifier') names.push(ident.text); + } + } + } + + // static const / final fields: static_final_declaration_list > static_final_declaration > identifier + if (child.type === 'static_final_declaration_list') { + for (let j = 0; j < child.namedChildCount; j++) { + const decl = child.namedChild(j); + if (decl?.type === 'static_final_declaration') { + const ident = decl.firstNamedChild; + if (ident?.type === 'identifier') names.push(ident.text); + } + } + } + } + return names; +} + export const dartConfig: FieldExtractionConfig = { language: SupportedLanguages.Dart, typeDeclarationNodes: ['class_definition'], @@ -18,31 +62,20 @@ export const dartConfig: FieldExtractionConfig = { bodyNodeTypes: ['class_body'], defaultVisibility: 'public', + // One AST `declaration` node may declare several fields (`int a, b;`, + // `static final String b = 'x', c = 'y';`), so use the multi-name path. extractName(node) { - // declaration > initialized_identifier_list > initialized_identifier > identifier - for (let i = 0; i < node.namedChildCount; i++) { - const child = node.namedChild(i); - if (child?.type === 'initialized_identifier_list') { - for (let j = 0; j < child.namedChildCount; j++) { - const init = child.namedChild(j); - if (init?.type === 'initialized_identifier') { - const ident = init.firstNamedChild; - if (ident?.type === 'identifier') return ident.text; - } - } - } - if (child?.type === 'initialized_identifier') { - const ident = child.firstNamedChild; - if (ident?.type === 'identifier') return ident.text; - } - } - // fallback: look for direct identifier - const name = node.childForFieldName('name'); - return name?.text; + return extractDartFieldNames(node)[0]; + }, + + extractNames(node) { + return extractDartFieldNames(node); }, extractType(node) { - // declaration > type_identifier (first named child usually) + // declaration > type_identifier (the type annotation, present for both the + // instance-field shape and `static final String b = …`). `static const a = 1;` + // has no annotation → undefined (untyped). for (let i = 0; i < node.namedChildCount; i++) { const child = node.namedChild(i); if (child && (child.type === 'type_identifier' || child.type === 'function_type')) { @@ -52,22 +85,16 @@ export const dartConfig: FieldExtractionConfig = { return undefined; }, - extractVisibility(node) { - // Dart uses _ prefix for private - // Walk to find the identifier name - for (let i = 0; i < node.namedChildCount; i++) { - const child = node.namedChild(i); - if (child?.type === 'initialized_identifier_list') { - for (let j = 0; j < child.namedChildCount; j++) { - const init = child.namedChild(j); - if (init?.type === 'initialized_identifier') { - const ident = init.firstNamedChild; - if (ident?.text?.startsWith('_')) return 'private'; - } - } - } - } - return 'public'; + // Per-name: Dart convention is underscore-prefixed = private. A single + // declaration can mix visibilities (`static const _p = 1, q = 2;`), so the + // decision is keyed on the individual field name. + extractVisibilityForName(_node, name): FieldVisibility { + return name.startsWith('_') ? 'private' : 'public'; + }, + + extractVisibility(node): FieldVisibility { + const first = extractDartFieldNames(node)[0]; + return first?.startsWith('_') ? 'private' : 'public'; }, isStatic(node) { @@ -75,6 +102,8 @@ export const dartConfig: FieldExtractionConfig = { }, isReadonly(node) { + // `final` / `const` (both `final_builtin`/`const_builtin` nodes whose text + // is `final`/`const`) are read-only. return hasKeyword(node, 'final') || hasKeyword(node, 'const'); }, }; diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/go.ts b/gitnexus/src/core/ingestion/field-extractors/configs/go.ts index b51f1f046..0e37b89c4 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/go.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/go.ts @@ -3,6 +3,8 @@ import { SupportedLanguages } from 'gitnexus-shared'; import type { FieldExtractionConfig } from '../generic.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; +import type { FieldVisibility } from '../../field-types.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; /** * Go field extraction config. @@ -13,14 +15,52 @@ import { extractSimpleTypeName } from '../../type-extractors/shared.js'; * Visibility in Go is based on the first character: uppercase = exported (public), * lowercase = unexported (package). */ +function goVisibilityForName(name: string): FieldVisibility { + const first = name.charAt(0); + return first === first.toUpperCase() && first !== first.toLowerCase() ? 'public' : 'package'; +} + +function extractGoFieldNames(node: SyntaxNode): string[] { + const names: string[] = []; + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child?.type === 'field_identifier') names.push(child.text); + } + return names; +} + export const goConfig: FieldExtractionConfig = { language: SupportedLanguages.Go, - typeDeclarationNodes: ['type_declaration'], + typeDeclarationNodes: ['type_declaration', 'struct_type'], fieldNodeTypes: ['field_declaration'], bodyNodeTypes: ['field_declaration_list'], defaultVisibility: 'package', + extractOwnerName(node) { + if (node.type === 'struct_type') { + return node.parent?.type === 'type_spec' + ? node.parent.childForFieldName('name')?.text + : undefined; + } + const typeSpec = node.namedChildren.find((child) => child.type === 'type_spec'); + return typeSpec?.childForFieldName('name')?.text; + }, + + findBodyNodes(node) { + if (node.type === 'struct_type') { + const body = node.namedChildren.find((child) => child.type === 'field_declaration_list'); + return body ? [body] : []; + } + const typeSpec = node.namedChildren.find((child) => child.type === 'type_spec'); + const typeNode = typeSpec?.childForFieldName('type'); + const body = typeNode?.namedChildren.find((child) => child.type === 'field_declaration_list'); + return body ? [body] : []; + }, + extractName(node) { + const firstName = extractGoFieldNames(node)[0]; + if (firstName) return firstName; + // field_declaration > name:(field_identifier) const name = node.childForFieldName('name'); if (name) return name.text; @@ -32,6 +72,8 @@ export const goConfig: FieldExtractionConfig = { return undefined; }, + extractNames: extractGoFieldNames, + extractType(node) { // field_declaration > type:(type_identifier | pointer_type | ...) const typeNode = node.childForFieldName('type'); @@ -54,6 +96,10 @@ export const goConfig: FieldExtractionConfig = { return 'package'; }, + extractVisibilityForName(_node, name) { + return goVisibilityForName(name); + }, + isStatic(_node) { return false; // Go has no static fields }, diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/field-extractors/configs/jvm.ts index 9d6cdbf07..37015a998 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/jvm.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/jvm.ts @@ -5,6 +5,7 @@ import type { FieldExtractionConfig } from '../generic.js'; import { findVisibility, hasKeyword, hasModifier, typeFromField } from './helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { FieldVisibility } from '../../field-types.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; // --------------------------------------------------------------------------- // Java @@ -73,13 +74,49 @@ export const javaConfig: FieldExtractionConfig = { const KOTLIN_VIS = new Set(['public', 'private', 'protected', 'internal']); +/** A property_declaration is a companion-object member when its nearest + * class-body ancestor is the body of a companion_object (F52, issue #1919). + * Companion members are addressed statically through the enclosing class + * (`C.TAG`), so they are marked static. */ +function isInsideKotlinCompanion(node: SyntaxNode): boolean { + for (let cur = node.parent; cur !== null; cur = cur.parent) { + if (cur.type === 'class_body') return cur.parent?.type === 'companion_object'; + if (cur.type === 'companion_object') return true; + } + return false; +} + export const kotlinConfig: FieldExtractionConfig = { language: SupportedLanguages.Kotlin, - typeDeclarationNodes: ['class_declaration', 'object_declaration'], + // F52: include companion_object so a companion property's innermost + // class-container owner (findEnclosingClassNode returns the companion_object) + // is recognized as a type declaration and its nested class_body is walked. + // The structure query already creates the Property node and owns it on the + // ENCLOSING class for anonymous companions / on the named companion Class — + // this entry only drives field-metadata enrichment, so it does NOT change + // ownership or emit a second node (no double-count). + typeDeclarationNodes: ['class_declaration', 'object_declaration', 'companion_object'], fieldNodeTypes: ['property_declaration'], bodyNodeTypes: ['class_body'], defaultVisibility: 'public', + // F52: an anonymous `companion object { ... }` has no name child, so the + // generic factory's `childForFieldName('name')` owner lookup is empty and + // `extract()` would bail before walking the body. Supply a stable owner + // name (the named companion's identifier, else "Companion") so the body IS + // walked; the resulting FieldInfo map is keyed by field NAME only, so the + // owner name does not affect which Property node gets enriched. + extractOwnerName(node) { + const typeIdentifierText = node.namedChildren.find((c) => c.type === 'type_identifier')?.text; + if (node.type === 'companion_object') { + // Anonymous companions have no type_identifier — fall back to "Companion". + return typeIdentifierText ?? 'Companion'; + } + const name = node.childForFieldName('name'); + if (name) return name.text; + return typeIdentifierText; + }, + extractName(node) { // property_declaration > variable_declaration > simple_identifier for (let i = 0; i < node.namedChildCount; i++) { @@ -124,9 +161,11 @@ export const kotlinConfig: FieldExtractionConfig = { return findVisibility(node, KOTLIN_VIS, 'public', 'modifiers'); }, - isStatic(_node) { - // Kotlin doesn't have static; companion object members are handled separately - return false; + isStatic(node) { + // Kotlin has no `static`, but companion-object members are accessed + // statically through the enclosing class (`C.TAG`) — mark them static + // so the field metadata reflects that (F52). + return isInsideKotlinCompanion(node); }, isReadonly(node) { diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/field-extractors/configs/swift.ts index 75c70ab95..6e27ff702 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/swift.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/swift.ts @@ -2,7 +2,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import type { FieldExtractionConfig } from '../generic.js'; -import { hasKeyword, findVisibility } from './helpers.js'; +import { hasKeyword, hasModifier, findVisibility } from './helpers.js'; import { extractSimpleTypeName } from '../../type-extractors/shared.js'; import type { FieldVisibility } from '../../field-types.js'; @@ -17,18 +17,33 @@ const SWIFT_VIS = new Set([ /** * Swift field extraction config. * - * Handles property_declaration inside class_body / protocol_body. + * Handles property_declaration inside class_body / protocol_body and + * protocol_property_declaration inside protocol_body (F75 — protocol property + * requirements like "var title: String { get }"). + * * tree-sitter-swift uses property_declaration for stored/computed properties. + * A protocol property requirement parses to its own node type, + * protocol_property_declaration, whose name lives in a "name:" pattern field + * (pattern > value_binding_pattern + simple_identifier(bound_identifier)), its + * type in a sibling type_annotation, and its "{ get }" / "{ get set }" in a + * protocol_property_requirements child. Note: Swift reuses the "name:" field + * across many positions (func name, every parameter label, parameter/return + * type), so the name is synthesized from the simple_identifier inside the + * pattern rather than read blindly off "name:". */ export const swiftConfig: FieldExtractionConfig = { language: SupportedLanguages.Swift, typeDeclarationNodes: ['class_declaration', 'protocol_declaration'], - fieldNodeTypes: ['property_declaration'], + fieldNodeTypes: ['property_declaration', 'protocol_property_declaration'], bodyNodeTypes: ['class_body', 'protocol_body'], defaultVisibility: 'internal', extractName(node) { - // property_declaration > pattern > simple_identifier + // property_declaration > pattern > simple_identifier, and + // protocol_property_declaration > name: (pattern ... simple_identifier). + // For protocol_property_declaration the pattern wraps a leading + // value_binding_pattern ("var") plus the simple_identifier — the loop + // below skips the binding keyword and returns the identifier. for (let i = 0; i < node.namedChildCount; i++) { const child = node.namedChild(i); if (child?.type === 'pattern') { @@ -62,7 +77,19 @@ export const swiftConfig: FieldExtractionConfig = { }, isStatic(node) { - return hasKeyword(node, 'static') || hasKeyword(node, 'class'); + // `static`/`class` (type-level) modifiers live inside a `modifiers` + // wrapper for both property_declaration and protocol_property_declaration + // (e.g. `static var shared: P { get }`), so check the wrapper too. + // `hasKeyword` compares each direct child by `.text` equality: it matches a + // single-modifier wrapper (`modifiers.text === 'static'`) but fails for a + // multi-modifier wrapper (`private static` → `modifiers.text === 'private static'`), + // which `hasModifier` handles by descending into the wrapper's children. + return ( + hasKeyword(node, 'static') || + hasKeyword(node, 'class') || + hasModifier(node, 'modifiers', 'static') || + hasModifier(node, 'modifiers', 'class') + ); }, isReadonly(node) { diff --git a/gitnexus/src/core/ingestion/field-extractors/generic.ts b/gitnexus/src/core/ingestion/field-extractors/generic.ts index 4cb4a5b1b..68f77dc8b 100644 --- a/gitnexus/src/core/ingestion/field-extractors/generic.ts +++ b/gitnexus/src/core/ingestion/field-extractors/generic.ts @@ -33,6 +33,10 @@ export interface FieldExtractionConfig { bodyNodeTypes: string[]; /** Default visibility when no modifier is present */ defaultVisibility: FieldVisibility; + /** Extract owner type name from a type declaration node. */ + extractOwnerName?: (node: SyntaxNode) => string | undefined; + /** Find body nodes inside a type declaration node. */ + findBodyNodes?: (node: SyntaxNode) => SyntaxNode[]; /** * Extract field name from a field declaration node. * Use this for nodes that declare exactly one field. @@ -49,6 +53,8 @@ export interface FieldExtractionConfig { extractType: (node: SyntaxNode) => string | undefined; /** Extract visibility from a field declaration node */ extractVisibility: (node: SyntaxNode) => FieldVisibility; + /** Extract visibility for one field name from a multi-name declaration. */ + extractVisibilityForName?: (node: SyntaxNode, name: string) => FieldVisibility; /** Check if a field is static */ isStatic: (node: SyntaxNode) => boolean; /** Check if a field is readonly/final/const */ @@ -84,10 +90,9 @@ export function createFieldExtractor(config: FieldExtractionConfig): FieldExtrac extract(node: SyntaxNode, context: FieldExtractorContext): ExtractedFields | null { if (!this.isTypeDeclaration(node)) return null; - const nameNode = node.childForFieldName('name'); - if (!nameNode) return null; + const ownerFqn = config.extractOwnerName?.(node) ?? node.childForFieldName('name')?.text; + if (!ownerFqn) return null; - const ownerFqn = nameNode.text; const fields: FieldInfo[] = []; // Find body container(s) @@ -110,6 +115,8 @@ export function createFieldExtractor(config: FieldExtractionConfig): FieldExtrac // ------------------------------------------------------------------ private findBodies(node: SyntaxNode): SyntaxNode[] { + if (config.findBodyNodes) return config.findBodyNodes(node); + const result: SyntaxNode[] = []; // Try named 'body' field first const bodyField = node.childForFieldName('body'); @@ -179,7 +186,7 @@ export function createFieldExtractor(config: FieldExtractionConfig): FieldExtrac return { name, type, - visibility: config.extractVisibility(node), + visibility: config.extractVisibilityForName?.(node, name) ?? config.extractVisibility(node), isStatic: config.isStatic(node), isReadonly: config.isReadonly(node), sourceFile: context.filePath, diff --git a/gitnexus/src/core/ingestion/filesystem-walker.ts b/gitnexus/src/core/ingestion/filesystem-walker.ts index 9ba959ea6..0af28a958 100644 --- a/gitnexus/src/core/ingestion/filesystem-walker.ts +++ b/gitnexus/src/core/ingestion/filesystem-walker.ts @@ -6,10 +6,6 @@ import { glob } from 'glob'; import { createIgnoreFilter } from '../../config/ignore-service.js'; import { logger } from '../logger.js'; -export interface FileEntry { - path: string; - content: string; -} /** Lightweight entry — path + size from stat, no content in memory */ export interface ScannedFile { @@ -153,21 +149,3 @@ export const readFileContents = async ( return contents; }; - -/** - * Legacy API — scans and reads everything into memory. - * Used by sequential fallback path only. - */ -export const walkRepository = async ( - repoPath: string, - onProgress?: (current: number, total: number, filePath: string) => void, -): Promise => { - const scanned = await walkRepositoryPaths(repoPath, onProgress); - const contents = await readFileContents( - repoPath, - scanned.map((f) => f.path), - ); - return scanned - .filter((f) => contents.has(f.path)) - .map((f) => ({ path: f.path, content: contents.get(f.path)! })); -}; diff --git a/gitnexus/src/core/ingestion/finalize-orchestrator.ts b/gitnexus/src/core/ingestion/finalize-orchestrator.ts index 558672d7d..02717a0b9 100644 --- a/gitnexus/src/core/ingestion/finalize-orchestrator.ts +++ b/gitnexus/src/core/ingestion/finalize-orchestrator.ts @@ -51,6 +51,8 @@ import { finalize, } from 'gitnexus-shared'; import type { ScopeResolutionIndexes } from './model/scope-resolution-indexes.js'; +import { parseTruthyEnv } from './utils/env.js'; +import { TransitionalScopeTree } from '../../storage/scope-index-store.js'; // ─── Public entry point ───────────────────────────────────────────────────── @@ -114,7 +116,13 @@ export function finalizeScopeModel( moduleEntries.push({ filePath: file.filePath, moduleScopeId: file.moduleScope }); } - const scopeTree = buildScopeTree(allScopes); + // Out-of-core scope index: when enabled, build a TransitionalScopeTree + // (validated + fully resident now; sealed to disk by run.ts just before emit so + // the heavy Scope.bindings payload is reclaimed). Default off → the in-heap + // buildScopeTree result exactly, byte-identical. + const scopeTree = parseTruthyEnv(process.env.GITNEXUS_DISK_SCOPE_INDEX) + ? new TransitionalScopeTree(allScopes) + : buildScopeTree(allScopes); const defs = buildDefIndex(allDefs); const qualifiedNames = buildQualifiedNameIndex(allDefs); const moduleScopes = buildModuleScopeIndex(moduleEntries); diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index c979102e5..dd531d1f5 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -35,7 +35,10 @@ import type { MethodExtractor } from './method-types.js'; import type { VariableExtractor } from './variable-types.js'; import type { ImportResolverFn } from './import-resolvers/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; +import type { CfgVisitor } from './cfg/types.js'; import type { NodeLabel } from 'gitnexus-shared'; +import type Parser from 'tree-sitter'; +import type { ExtractedDecoratorRoute } from './workers/parse-worker.js'; // ── Shared type aliases ──────────────────────────────────────────────────── /** Tree-sitter query captures: capture name → AST node (or undefined if not captured). */ @@ -185,6 +188,12 @@ interface LanguageProviderConfig { * `undefined` when no constraints exist / the node isn't a templated * function. Languages without SFINAE / concept semantics leave this * undefined and the disambiguation is a pass-through. + * + * Cloneability contract: the returned payload crosses the worker boundary + * via structured clone, so it MUST be structured-clone-safe (no functions, + * symbols, or tree-sitter `SyntaxNode`s — only plain data). Wrap the return + * with `assertCloneable` from `workers/clone-safety.ts` so a future leak is a + * compile error at the source instead of a runtime DataCloneError (#2143). */ readonly extractTemplateConstraints?: (definitionNode: SyntaxNode) => unknown; @@ -236,6 +245,22 @@ interface LanguageProviderConfig { * Default: undefined (no route files). */ readonly isRouteFile?: (filePath: string) => boolean; + /** + * Extract decorator-style route annotations from a parsed file. + * + * When defined, the parse worker calls this after per-file capture processing + * to extract framework route definitions that require AST-level analysis beyond + * generic `@decorator` captures (e.g., Java Spring class-level prefix joining, + * multi-class handling). The returned routes are appended to `decoratorRoutes`. + * + * Default: undefined (no language-specific decorator route extraction). + */ + readonly extractDecoratorRoutes?: ( + tree: Parser.Tree, + filePath: string, + lineOffset: number, + ) => ExtractedDecoratorRoute[]; + // ── Noise filtering ──────────────────────────────────────────────── /** Built-in/stdlib names that should be filtered from the call graph for this language. * Default: undefined (no language-specific filtering). */ @@ -311,6 +336,42 @@ interface LanguageProviderConfig { }, ) => readonly CaptureMatch[]; + /** + * Snapshot the capture-time side-channel state that this provider's + * `emitScopeCaptures` just populated for `filePath` into module-level maps, + * returning a plain JSON-serializable value (or `undefined` when there is + * nothing to carry). + * + * Called in the parse worker IMMEDIATELY after `emitScopeCaptures` runs for + * a file (see `parse-worker.ts`), and the result is stored on the produced + * `ParsedFile.captureSideChannel`. Scope-resolution on the main thread reuses + * that serialized `ParsedFile` and skips re-extraction (#1983), so this hook + * is how the worker-computed marks survive the worker→main boundary and the + * disk store WITHOUT a main-thread re-parse. The main thread restores them + * via the matching `ScopeResolver.applyCaptureSideChannel` hook. + * + * Cloneability contract: MUST return plain data (objects / arrays / + * primitives — no functions, symbols, or tree-sitter `SyntaxNode`s) so it + * survives BOTH the worker→main structured clone AND `JSON.stringify` + the + * parsedfile-store interning reviver. Wrap the return with `assertCloneable` + * from `workers/clone-safety.ts` so a future non-serializable leak is a + * compile error at the source instead of a runtime DataCloneError (#2143). + * + * Default: undefined (provider has no capture-time module-level side effects). + */ + readonly collectCaptureSideChannel?: (filePath: string) => unknown; + + /** + * Per-language control-flow-graph builder (#2081 M1, PDG/taint substrate). + * Invoked IN THE PARSE WORKER (where the AST lives) for each function node, + * gated on the `--pdg` opt-in; the resulting per-function CFGs are serialized + * onto `ParsedFile.cfgSideChannel` and emitted as BasicBlock nodes + CFG + * edges during scope-resolution. `TNode` is `SyntaxNode` for the tree-sitter + * languages. Default: undefined (language has no CFG support yet — TS/JS are + * the M1 set). + */ + readonly cfgVisitor?: CfgVisitor; + /** * Interpret a raw `@import.statement` capture group into a `ParsedImport`. * The central finalize algorithm resolves `ParsedImport.targetRaw` to a diff --git a/gitnexus/src/core/ingestion/languages/c-cpp.ts b/gitnexus/src/core/ingestion/languages/c-cpp.ts index 4ce17a9c5..3d427fed8 100644 --- a/gitnexus/src/core/ingestion/languages/c-cpp.ts +++ b/gitnexus/src/core/ingestion/languages/c-cpp.ts @@ -53,6 +53,7 @@ import { cBindingScopeFor, cImportOwningScope, cReceiverBinding, + collectCStaticLinkageSideChannel, } from './c/index.js'; import { emitCppScopeCaptures, @@ -62,8 +63,13 @@ import { cppBindingScopeFor, cppImportOwningScope, cppReceiverBinding, + collectCppCaptureSideChannel, } from './cpp/index.js'; -import { extractCppTemplateConstraints } from './cpp/constraint-extractor.js'; +import { + extractCppTemplateConstraints, + type CppConstraintPayload, +} from './cpp/constraint-extractor.js'; +import { assertCloneable } from '../workers/clone-safety.js'; const C_BUILT_INS: ReadonlySet = new Set([ 'printf', @@ -395,6 +401,19 @@ export const cProvider = defineLanguage({ // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── emitScopeCaptures: emitCScopeCaptures, + // Worker-side: snapshot the module-level `static`-linkage marks + // `emitCScopeCaptures` just populated for this file (`markStaticName` → + // `staticNames`) into plain data on `ParsedFile.captureSideChannel`, so the + // main thread can restore them via `applyCaptureSideChannel` WITHOUT a + // re-parse (#1983 — the worker is the sole parse path). Without this, C + // `static` functions look non-file-local on the main thread and leak into + // cross-file global free-call resolution / wildcard imports. See + // `c/capture-side-channel.ts`. + // `assertCloneable` is a runtime identity; it makes a future non-serializable + // value in the side-channel payload a compile error here, at the source, rather + // than a DataCloneError at the worker boundary (#2143). + collectCaptureSideChannel: (filePath) => + assertCloneable(collectCStaticLinkageSideChannel(filePath)), interpretImport: interpretCImport, interpretTypeBinding: interpretCTypeBinding, bindingScopeFor: cBindingScopeFor, @@ -465,6 +484,11 @@ export const cppProvider = defineLanguage({ // ── RFC #909 Ring 3: scope-based resolution hooks (RFC §5) ────────── emitScopeCaptures: emitCppScopeCaptures, + // Worker-side: snapshot the module-level capture marks `emitCppScopeCaptures` + // just populated for this file into plain data on `ParsedFile.captureSideChannel`, + // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a + // re-parse (#1983). See `cpp/capture-side-channel.ts`. + collectCaptureSideChannel: (filePath) => assertCloneable(collectCppCaptureSideChannel(filePath)), interpretImport: interpretCppImport, interpretTypeBinding: interpretCppTypeBinding, bindingScopeFor: cppBindingScopeFor, @@ -485,7 +509,9 @@ export const cppProvider = defineLanguage({ * functions whose constraints the extractor can't model — both cases * result in no constraint suffix on the node ID. */ -function extractCppTemplateConstraintsForProvider(definitionNode: SyntaxNode): unknown { +function extractCppTemplateConstraintsForProvider( + definitionNode: SyntaxNode, +): CppConstraintPayload | undefined { // Walk up to the enclosing template_declaration. Bound the walk so we // can't accidentally land on a far-ancestor template_declaration that // wraps an unrelated function. @@ -514,5 +540,8 @@ function extractCppTemplateConstraintsForProvider(definitionNode: SyntaxNode): u } break; } - return extractCppTemplateConstraints(templateDecl, declarator); + // Guard the boundary at the source: a future non-cloneable member of the + // constraint payload becomes a compile error here, not a runtime + // DataCloneError at the worker post (#2143). + return assertCloneable(extractCppTemplateConstraints(templateDecl, declarator)); } diff --git a/gitnexus/src/core/ingestion/languages/c/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/c/capture-side-channel.ts new file mode 100644 index 000000000..2f619e5f3 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/c/capture-side-channel.ts @@ -0,0 +1,80 @@ +/** + * C capture-time side-channel serialization (#1983). + * + * `emitCScopeCaptures` populates one MODULE-LEVEL, per-file map as a side + * effect that is NOT part of the returned `ParsedFile`'s scopes/defs: + * + * - `staticNames` (static-linkage.ts) — the simple names of functions + * declared with `static` storage class (file-local / translation-unit + * linkage in C), recorded via `markStaticName` from the + * `@declaration.name` capture when the function node has a `static` + * storage-class specifier. + * + * On the worker path that map is filled in the WORKER process and lost across + * the worker→main MessageChannel (and the disk-backed parsedfile-store), + * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the + * main-thread re-extraction (the #1983 fix that avoids a main-thread + * tree-sitter re-parse / OOM on huge repos — e.g. the Linux kernel). The main + * thread then reads the map empty in `isStaticName` (consulted by + * `isFileLocalDef` in `c/scope-resolver.ts` and by `expandCWildcardNames` in + * static-linkage.ts) — so file-local `static` functions become eligible for + * cross-file global free-call resolution (false CALLS edges) and `#include` + * wildcard imports over-expose them. + * + * This module snapshots the per-file slice of that map into a plain, + * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and + * restores it on the main thread WITHOUT any parse. It mirrors the C++ pattern + * in `cpp/capture-side-channel.ts` and the Kotlin pattern in + * `kotlin/capture-side-channel.ts`. + * + * The single generic `ParsedFile.captureSideChannel` field is shared with C++ + * and Kotlin, which is safe because each file is one language (a `.c` file uses + * the C provider). The payload is self-describing (`{ kind: 'c', staticNames }`) + * so `applyCStaticLinkageSideChannel` only restores C state and ignores a + * foreign-shaped snapshot. + */ + +import type { ParsedFile } from 'gitnexus-shared'; +import { getStaticNamesForFile, markStaticName } from './static-linkage.js'; + +/** + * Plain JSON-serializable snapshot of the per-file C capture-time + * side-channel. Carried opaquely on `ParsedFile.captureSideChannel`. The + * `kind` tag makes the payload self-describing so `apply` can distinguish a C + * snapshot from another language's (C++ and Kotlin share the same field). + */ +export interface CCaptureSideChannel { + readonly kind: 'c'; + /** Simple names of `static` (file-local linkage) functions in this file. */ + readonly staticNames: readonly string[]; +} + +/** + * `LanguageProvider.collectCaptureSideChannel` implementation for C. + * Returns `undefined` when this file recorded no static names at all, so the + * produced `ParsedFile` carries the field only when there's data to ship. + */ +export function collectCStaticLinkageSideChannel( + filePath: string, +): CCaptureSideChannel | undefined { + const staticNames = getStaticNamesForFile(filePath); + if (staticNames.length === 0) return undefined; + return { kind: 'c', staticNames }; +} + +/** + * `ScopeResolver.applyCaptureSideChannel` implementation for C. Reads the + * worker-serialized snapshot from `parsed.captureSideChannel` and re-populates + * the module-level static-linkage map via `markStaticName`. Tolerant of + * `undefined` (file carried no data) and of an unexpected / foreign shape + * (defensive — the `kind` tag guards against restoring a non-C payload). + * Does NO tree-sitter parse. + */ +export function applyCStaticLinkageSideChannel(parsed: ParsedFile): void { + const data = parsed.captureSideChannel as CCaptureSideChannel | undefined; + if (data === undefined || data === null || typeof data !== 'object') return; + if (data.kind !== 'c' || !Array.isArray(data.staticNames)) return; + for (const name of data.staticNames) { + markStaticName(parsed.filePath, name); + } +} diff --git a/gitnexus/src/core/ingestion/languages/c/import-decomposer.ts b/gitnexus/src/core/ingestion/languages/c/import-decomposer.ts index 5cef27430..77b54f120 100644 --- a/gitnexus/src/core/ingestion/languages/c/import-decomposer.ts +++ b/gitnexus/src/core/ingestion/languages/c/import-decomposer.ts @@ -5,10 +5,20 @@ import { nodeToCapture, syntheticCapture, type SyntaxNode } from '../../utils/as * Decompose a `preproc_include` node into a CaptureMatch with structured * import captures. C #include maps to a wildcard import (all symbols * from the header are visible). + * + * Only literal include paths are emitted as import sources: + * #include → system_lib_string + * #include "local.h" → string_literal + * A computed include like `#include HEADER_MACRO` carries an `identifier` + * path node (the macro name, not a header path). Emitting it as an import + * source produces a garbage literal edge, so we skip it entirely — matching + * the convention in interpretCImport, which drops imports with no resolvable + * source (issue #1919 F5). */ export function splitCInclude(node: SyntaxNode): CaptureMatch | null { // node.type === 'preproc_include' // path field: (string_literal (string_content)) | (system_lib_string) + // | (identifier) ← computed macro include, NOT a header path const pathNode = node.childForFieldName?.('path') ?? null; if (pathNode === null) { // Fallback: scan children @@ -24,7 +34,13 @@ export function splitCInclude(node: SyntaxNode): CaptureMatch | null { return buildIncludeCapture(node, pathNode); } -function buildIncludeCapture(node: SyntaxNode, pathNode: SyntaxNode): CaptureMatch { +function buildIncludeCapture(node: SyntaxNode, pathNode: SyntaxNode): CaptureMatch | null { + // Skip computed includes (`#include MACRO`) — the path is an `identifier`, + // not a literal header path. Emitting it would create a garbage import. + if (pathNode.type !== 'string_literal' && pathNode.type !== 'system_lib_string') { + return null; + } + let raw: string; if (pathNode.type === 'string_literal') { // string_literal has children: `"`, string_content, `"` diff --git a/gitnexus/src/core/ingestion/languages/c/import-target.ts b/gitnexus/src/core/ingestion/languages/c/import-target.ts index 0cb9c2fb4..495846030 100644 --- a/gitnexus/src/core/ingestion/languages/c/import-target.ts +++ b/gitnexus/src/core/ingestion/languages/c/import-target.ts @@ -1,5 +1,54 @@ import { dirname, join } from 'path'; +/** + * A workspace file path pre-decomposed for the suffix-match fallback: + * `original` is returned verbatim (preserving the prior `bestMatch = filePath` + * contract); `normalized` and `depth` are precomputed so the hot path does no + * per-element regex/`split`. + */ +interface CSuffixCandidate { + original: string; + normalized: string; + depth: number; +} + +/** + * Per-pass memo: workspace paths bucketed by basename (last path segment), + * keyed on the `allFilePaths` set identity. + * + * `resolveCImportTarget` is called once per (quoted) C/C++ `#include` with the + * same `allFilePaths` set per pass (the augmented set is itself memoized in + * the C resolver). The old suffix-match fallback scanned ALL workspace paths + * per include — with a per-element `.replace`/`.split` and no early exit + * (the fewest-path-components tie-break forces a full scan) — i.e. + * O(R_suffix × (F+H)). A path can satisfy `endsWith('/'+target)` (or equal + * the target) ONLY IF its basename equals the target's last segment, so we + * pre-bucket by basename once (O(F+H), `normalized`/`depth` precomputed) and + * the fallback inspects a single small bucket → O(F+H) build + ~O(1)/include. + * `WeakMap`-keyed so it is reclaimed with the pass (no cross-pass staleness). + * Shared by C and C++ (`resolveCppImportTarget` delegates here). + */ +const suffixIndexByPaths = new WeakMap, Map>(); + +function suffixIndex(allFilePaths: ReadonlySet): Map { + let index = suffixIndexByPaths.get(allFilePaths); + if (index === undefined) { + index = new Map(); + for (const original of allFilePaths) { + const normalized = original.replace(/\\/g, '/'); + const basename = normalized.slice(normalized.lastIndexOf('/') + 1); + let bucket = index.get(basename); + if (bucket === undefined) { + bucket = []; + index.set(basename, bucket); + } + bucket.push({ original, normalized, depth: normalized.split('/').length }); + } + suffixIndexByPaths.set(allFilePaths, index); + } + return index; +} + /** * Resolve a C #include path to a file in the workspace. * @@ -41,21 +90,31 @@ export function resolveCImportTarget( // Exact match (path as-is in the workspace) if (allFilePaths.has(normalizedTarget)) return normalizedTarget; - // Suffix match: find files ending with /targetRaw or equal to targetRaw + // Suffix match: find files ending with /targetRaw or equal to targetRaw. + // A path can only match `=== normalizedTarget` or `endsWith('/'+target)` if + // its basename equals the target's last segment, so we inspect only that + // basename bucket (built once per pass) instead of scanning every workspace + // path. Match condition + tie-break (fewest path components, then + // lexicographic on the normalized path) are byte-identical to the prior scan. const suffix = '/' + normalizedTarget; + const targetBasename = normalizedTarget.slice(normalizedTarget.lastIndexOf('/') + 1); + const bucket = suffixIndex(allFilePaths).get(targetBasename); + if (bucket === undefined) return null; + let bestMatch: string | null = null; let bestDepth = Infinity; let bestNormalized = ''; - for (const filePath of allFilePaths) { - const normalized = filePath.replace(/\\/g, '/'); - if (normalized === normalizedTarget || normalized.endsWith(suffix)) { + for (const cand of bucket) { + if (cand.normalized === normalizedTarget || cand.normalized.endsWith(suffix)) { // Prefer shortest path (closest match) - const depth = normalized.split('/').length; - if (depth < bestDepth || (depth === bestDepth && normalized < bestNormalized)) { - bestDepth = depth; - bestMatch = filePath; - bestNormalized = normalized; + if ( + cand.depth < bestDepth || + (cand.depth === bestDepth && cand.normalized < bestNormalized) + ) { + bestDepth = cand.depth; + bestMatch = cand.original; + bestNormalized = cand.normalized; } } } diff --git a/gitnexus/src/core/ingestion/languages/c/index.ts b/gitnexus/src/core/ingestion/languages/c/index.ts index c6900ecba..8ebc2c98e 100644 --- a/gitnexus/src/core/ingestion/languages/c/index.ts +++ b/gitnexus/src/core/ingestion/languages/c/index.ts @@ -13,4 +13,9 @@ export { isStaticName, clearStaticNames, expandCWildcardNames, + getStaticNamesForFile, } from './static-linkage.js'; +export { + collectCStaticLinkageSideChannel, + applyCStaticLinkageSideChannel, +} from './capture-side-channel.js'; diff --git a/gitnexus/src/core/ingestion/languages/c/query.ts b/gitnexus/src/core/ingestion/languages/c/query.ts index 373e1e7a7..065603b1a 100644 --- a/gitnexus/src/core/ingestion/languages/c/query.ts +++ b/gitnexus/src/core/ingestion/languages/c/query.ts @@ -1,5 +1,15 @@ import Parser from 'tree-sitter'; -import C from 'tree-sitter-c'; +import { SupportedLanguages } from 'gitnexus-shared'; +// `tree-sitter-c` is vendored prebuild-only (#2116) and may be absent on a +// toolchain-less / `--ignore-scripts` install. It is loaded lazily + guarded via +// parser-loader rather than statically imported: this module is pulled onto the +// main thread eagerly by the scope-resolution registry and the language-provider +// index, so a top-level `import C from 'tree-sitter-c'` would throw +// ERR_MODULE_NOT_FOUND at module-load and crash `analyze` even for repos with no +// C files (#2091, #2093). The grammar is only ever needed inside the lazy getters +// below, and the main-thread `isLanguageAvailable` filter ensures they are +// reached only when the binding is present. +import { getLanguageGrammar } from '../../../tree-sitter/parser-loader.js'; const C_SCOPE_QUERY = ` ;; Scopes @@ -167,14 +177,19 @@ let _query: Parser.Query | null = null; export function getCParser(): Parser { if (_parser === null) { _parser = new Parser(); - _parser.setLanguage(C as Parameters[0]); + _parser.setLanguage( + getLanguageGrammar(SupportedLanguages.C) as Parameters[0], + ); } return _parser; } export function getCScopeQuery(): Parser.Query { if (_query === null) { - _query = new Parser.Query(C as Parameters[0], C_SCOPE_QUERY); + _query = new Parser.Query( + getLanguageGrammar(SupportedLanguages.C) as Parameters[0], + C_SCOPE_QUERY, + ); } return _query; } diff --git a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts index 90d54e072..cb4a6424f 100644 --- a/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/c/scope-resolver.ts @@ -7,6 +7,43 @@ import { cProvider } from '../c-cpp.js'; import { cArityCompatibility, cMergeBindings, resolveCImportTarget } from './index.js'; import { scanHeaderFiles } from './header-scan.js'; import { expandCWildcardNames, isStaticName, clearStaticNames } from './static-linkage.js'; +import { applyCStaticLinkageSideChannel } from './capture-side-channel.js'; + +/** + * Per-pass memo of the augmented `#include`-resolution file set + * (`allFilePaths` ∪ header `.h` paths), keyed on the two stable source sets. + * + * `resolveImportTarget` is called once per C `#include`; the old code rebuilt + * a fresh ~F-entry `Set` on EVERY call (O(R × (F+H)) inserts + GC churn) and, + * worse, defeated `resolveCImportTarget`'s own per-set suffix-index memo by + * handing it a new set identity each time. Both `allFilePaths` (built once in + * scope-resolution `run.ts`) and the header set (`loadResolutionConfig` + * result) are stable per pass, so the union is built once and reused. + * `WeakMap`-keyed → reclaimed with the pass (no cross-pass staleness). + */ +const augmentedPathsByPass = new WeakMap< + ReadonlySet, + WeakMap, ReadonlySet> +>(); + +function augmentedFilePaths( + allFilePaths: ReadonlySet, + headerPaths: ReadonlySet, +): ReadonlySet { + let byHeaders = augmentedPathsByPass.get(allFilePaths); + if (byHeaders === undefined) { + byHeaders = new WeakMap(); + augmentedPathsByPass.set(allFilePaths, byHeaders); + } + let augmented = byHeaders.get(headerPaths); + if (augmented === undefined) { + const set = new Set(allFilePaths); + for (const h of headerPaths) set.add(h); + augmented = set; + byHeaders.set(headerPaths, augmented); + } + return augmented; +} /** * C `ScopeResolver` registered in `SCOPE_RESOLVERS` and consumed by @@ -31,15 +68,34 @@ export const cScopeResolver: ScopeResolver = { return scanHeaderFiles(repoPath); }, + // Worker-boundary restore (see `ScopeResolver.applyCaptureSideChannel`). + // `emitCScopeCaptures` records per-file `static`-linkage names + // (`markStaticName` → `staticNames`) as a SIDE EFFECT — that state is NOT + // serialized onto the returned ParsedFile's scopes/defs. On the worker path + // those marks are populated in the worker process and lost across the + // MessageChannel / disk store; the main thread reuses the serialized + // ParsedFile and skips `extractParsedFile`, so `isStaticName` (read by + // `isFileLocalDef` and `expandCWildcardNames`) sees an empty map and C + // `static` functions leak into cross-file global free-call resolution + // (false CALLS edges) and `#include` wildcard imports. The worker stashed a + // plain-data snapshot on `parsed.captureSideChannel` via + // `cProvider.collectCaptureSideChannel`; this restores it into the module + // map WITHOUT any tree-sitter re-parse (the #1983 fix). The + // freshly-extracted leg never calls this — its marks were just populated in + // this process. Runs BEFORE `populateOwners`. + applyCaptureSideChannel: applyCStaticLinkageSideChannel, + resolveImportTarget: (targetRaw, fromFile, allFilePaths, resolutionConfig) => { // Augment allFilePaths with .h files discovered via loadResolutionConfig // since the phase only passes .c files to the C resolver but #include // targets .h files classified as C++ in language detection. const headerPaths = resolutionConfig as ReadonlySet | undefined; if (headerPaths !== undefined && headerPaths.size > 0) { - const augmented = new Set(allFilePaths); - for (const h of headerPaths) augmented.add(h); - return resolveCImportTarget(targetRaw, fromFile, augmented); + return resolveCImportTarget( + targetRaw, + fromFile, + augmentedFilePaths(allFilePaths, headerPaths), + ); } return resolveCImportTarget(targetRaw, fromFile, allFilePaths); }, diff --git a/gitnexus/src/core/ingestion/languages/c/static-linkage.ts b/gitnexus/src/core/ingestion/languages/c/static-linkage.ts index 5cfd166b1..2cc195205 100644 --- a/gitnexus/src/core/ingestion/languages/c/static-linkage.ts +++ b/gitnexus/src/core/ingestion/languages/c/static-linkage.ts @@ -29,11 +29,58 @@ export function isStaticName(filePath: string, name: string): boolean { return staticNames.get(filePath)?.has(name) ?? false; } +/** + * Return the `static` (file-local) names recorded for the given file as a + * plain array (empty when none). Used to snapshot the per-file slice of the + * module-level `staticNames` map into `ParsedFile.captureSideChannel` so it + * survives the worker→main boundary (#1983 — the worker is the sole parse + * path). See `c/capture-side-channel.ts`. + */ +export function getStaticNamesForFile(filePath: string): string[] { + const names = staticNames.get(filePath); + return names === undefined ? [] : [...names]; +} + /** Clear tracked static names (for testing). */ export function clearStaticNames(): void { staticNames.clear(); } +/** + * Per-pass memo: `moduleScope` → owning `ParsedFile`, keyed on the + * `parsedFiles` array identity. + * + * The shared finalize Phase-4 loop calls `expandsWildcardTo` + * (→ `expandCWildcardNames`) ONCE PER RESOLVED `#include` edge, every time + * with the SAME `parsedFiles` reference (wired at scope-resolution + * `run.ts` — `allFilePaths`/`parsedFiles` are built once per pass). The old + * `parsedFiles.find(...)` therefore did a full O(F) scan per edge → + * O(R_include × F) overall; at Linux-kernel scale (F ≈ 63k C files, tens of + * thousands of resolved includes) that is ~10^10+ comparisons on a single + * thread — the dominant term in the scope-resolution finalize grind. + * + * Building the lookup once collapses it to O(R_include + F). `WeakMap`-keyed + * on the array so the index is reclaimed with the pass — no cross-pass + * staleness (mirrors the {@link clearStaticNames} discipline for server-mode + * / multi-repo reuse), and a fresh array transparently rebuilds. + */ +const moduleScopeIndexByPass = new WeakMap>(); + +function moduleScopeIndex(parsedFiles: readonly ParsedFile[]): Map { + let index = moduleScopeIndexByPass.get(parsedFiles); + if (index === undefined) { + index = new Map(); + // First-wins to preserve `Array.find` semantics (returns the first match). + // `moduleScope` is unique per file in practice, so collisions are absent; + // the guard only formalises identical behaviour to the prior `.find`. + for (const p of parsedFiles) { + if (!index.has(p.moduleScope)) index.set(p.moduleScope, p); + } + moduleScopeIndexByPass.set(parsedFiles, index); + } + return index; +} + /** * Return the names visible through a C wildcard import (`#include`). * All module-scope defs from the target file are visible EXCEPT those @@ -43,7 +90,7 @@ export function expandCWildcardNames( targetModuleScope: ScopeId, parsedFiles: readonly ParsedFile[], ): readonly string[] { - const target = parsedFiles.find((p) => p.moduleScope === targetModuleScope); + const target = moduleScopeIndex(parsedFiles).get(targetModuleScope); if (target === undefined) return []; const seen = new Set(); diff --git a/gitnexus/src/core/ingestion/languages/cpp/adl.ts b/gitnexus/src/core/ingestion/languages/cpp/adl.ts index 565c23125..7a1bd9f66 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/adl.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/adl.ts @@ -108,6 +108,38 @@ const argInfoBySite = new Map(); const noAdlSites = new Set(); const classToNamespaceQualifiedName = new Map(); +/** + * Per-`filePath` index of the site keys this file contributed to + * `argInfoBySite` / `noAdlSites`, kept in **strict lockstep** with those two + * maps (#1983 perf). Without it, `collectCppAdlSideChannel(filePath)` had to + * scan the ENTIRE module-level maps (every site of every file the worker + * parsed in the current sub-batch) and `parseSiteKey` each entry just to pick + * out one file's slice — O(F²) per sub-batch (~100M `parseSiteKey` calls + * across the Linux kernel). These indexes turn collect into + * O(entries-for-this-file). + * + * Lockstep invariant: a key is pushed here at most once, exactly when it is + * first inserted into the corresponding map, and both indexes are cleared + * wherever `argInfoBySite` / `noAdlSites` are cleared (`clearCppAdlState` and + * the per-file restore in `applyCppAdlSideChannel`). The "first insert only" + * guard mirrors the maps' own de-dup (`Map.set` / `Set.add` are idempotent on + * the key), so iterating an index yields each of this file's keys exactly once + * — byte-identical to the old filtered full scan. + */ +const argInfoSiteKeysByFile = new Map(); +const noAdlSiteKeysByFile = new Map(); + +/** Push `key` into the per-file index `idx[filePath]` (creating the bucket on + * first use). Callers guard against duplicate keys so each key appears once. */ +function pushFileSiteKey(idx: Map, filePath: string, key: string): void { + let keys = idx.get(filePath); + if (keys === undefined) { + keys = []; + idx.set(filePath, keys); + } + keys.push(key); +} + /** * ADL candidate index — built **once** per pipeline run from * `(scopes, parsedFiles)` and reused by every call site. @@ -370,13 +402,94 @@ export function markCppAdlSiteArgs( col: number, args: readonly CppAdlArgInfo[], ): void { - argInfoBySite.set(siteKey(filePath, line, col), args); + const key = siteKey(filePath, line, col); + // Lockstep with `argInfoSiteKeysByFile`: index the key only on first insert + // (a re-mark overwrites the value but must NOT duplicate the index entry). + if (!argInfoBySite.has(key)) pushFileSiteKey(argInfoSiteKeysByFile, filePath, key); + argInfoBySite.set(key, args); } /** Mark a call site as ADL-suppressed (function child wrapped in * `parenthesized_expression`, e.g. `(f)(s)`). */ export function markCppAdlSiteNoAdl(filePath: string, line: number, col: number): void { - noAdlSites.add(siteKey(filePath, line, col)); + const key = siteKey(filePath, line, col); + // Lockstep with `noAdlSiteKeysByFile`: index the key only on first insert. + if (!noAdlSites.has(key)) pushFileSiteKey(noAdlSiteKeysByFile, filePath, key); + noAdlSites.add(key); +} + +/** + * Plain-data, JSON-serializable snapshot of the per-file ADL capture state + * (`argInfoBySite` entries for this file + `noAdlSites` keys for this file). + * Carried on `ParsedFile.captureSideChannel` across the worker→main boundary + * (#1983); the call-site key's `line:col` are stored per-entry so the full + * `filePath:line:col` key can be reconstructed without parsing. + */ +export interface CppAdlSideChannel { + /** Per-call-site arg info: `[line, col, args]` for sites in this file. */ + readonly argInfoBySite: readonly [number, number, readonly CppAdlArgInfo[]][]; + /** ADL-suppressed sites in this file: `[line, col]`. */ + readonly noAdlSites: readonly [number, number][]; +} + +const SITE_KEY_RE = /^(.*):(\d+):(\d+)$/; + +/** Split a `filePath:line:col` site key, tolerating colons in the path. */ +function parseSiteKey(key: string): { filePath: string; line: number; col: number } | undefined { + const m = SITE_KEY_RE.exec(key); + if (m === null) return undefined; + return { filePath: m[1], line: Number(m[2]), col: Number(m[3]) }; +} + +/** + * Snapshot this file's ADL capture state for the worker→main side-channel. + * + * Uses the per-file `argInfoSiteKeysByFile` / `noAdlSiteKeysByFile` indexes to + * touch only THIS file's entries — O(entries-for-this-file) — instead of the + * old O(all-entries) full scan over `argInfoBySite` / `noAdlSites` (#1983). + * The output order, and therefore the serialized JSON shape, is byte-identical + * to the old filtered scan: the index records keys in the same insertion order + * the maps' own iteration would have yielded for this file, and each key is + * indexed exactly once (mark guards on first insert), so the same per-file + * subsequence is produced. + * + * `parseSiteKey` is still used to recover `line:col` from each key, but now + * only for this file's keys (a bounded handful), never for the whole batch. + */ +export function collectCppAdlSideChannel(filePath: string): CppAdlSideChannel { + const args: [number, number, readonly CppAdlArgInfo[]][] = []; + for (const key of argInfoSiteKeysByFile.get(filePath) ?? []) { + const value = argInfoBySite.get(key); + const parsed = parseSiteKey(key); + if (value !== undefined && parsed !== undefined) { + args.push([parsed.line, parsed.col, value]); + } + } + const noAdl: [number, number][] = []; + for (const key of noAdlSiteKeysByFile.get(filePath) ?? []) { + const parsed = parseSiteKey(key); + if (parsed !== undefined) { + noAdl.push([parsed.line, parsed.col]); + } + } + return { argInfoBySite: args, noAdlSites: noAdl }; +} + +/** Restore this file's ADL capture state from the side-channel (no parse). + * Keeps the per-file site-key indexes in lockstep with `argInfoBySite` / + * `noAdlSites` (first-insert-only) so a later `collectCppAdlSideChannel` on + * the same process would still produce a correct, duplicate-free snapshot. */ +export function applyCppAdlSideChannel(filePath: string, data: CppAdlSideChannel): void { + for (const [line, col, value] of data.argInfoBySite) { + const key = siteKey(filePath, line, col); + if (!argInfoBySite.has(key)) pushFileSiteKey(argInfoSiteKeysByFile, filePath, key); + argInfoBySite.set(key, value); + } + for (const [line, col] of data.noAdlSites) { + const key = siteKey(filePath, line, col); + if (!noAdlSites.has(key)) pushFileSiteKey(noAdlSiteKeysByFile, filePath, key); + noAdlSites.add(key); + } } /** Clear ADL state. Called from `cppScopeResolver.loadResolutionConfig` @@ -385,6 +498,11 @@ export function markCppAdlSiteNoAdl(filePath: string, line: number, col: number) export function clearCppAdlState(): void { argInfoBySite.clear(); noAdlSites.clear(); + // Lockstep: the per-file site-key indexes mirror argInfoBySite/noAdlSites and + // MUST be cleared together — a stale index would resurrect a prior pass's + // (or prior file's, after a re-key) keys into the next snapshot. + argInfoSiteKeysByFile.clear(); + noAdlSiteKeysByFile.clear(); classToNamespaceQualifiedName.clear(); adlIndex = undefined; adlIndexSource = undefined; diff --git a/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts index e2afd51a5..6bd08710c 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts @@ -222,8 +222,14 @@ function findFuncDeclarator(node: SyntaxNode): SyntaxNode | null { } return null; } - // Unwrap pointer_declarator / reference_declarator - while (decl.type === 'pointer_declarator' || decl.type === 'reference_declarator') { + // Unwrap declarator wrappers. Deleted free functions are represented as + // `init_declarator(function_declarator, delete_expression)` by + // tree-sitter-cpp 0.23. + while ( + decl.type === 'pointer_declarator' || + decl.type === 'reference_declarator' || + decl.type === 'init_declarator' + ) { const next = decl.childForFieldName('declarator'); if (next === null) { // reference_declarator may not use field name diff --git a/gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts new file mode 100644 index 000000000..ad9766d46 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/cpp/capture-side-channel.ts @@ -0,0 +1,123 @@ +/** + * C++ capture-time side-channel serialization (#1983). + * + * `emitCppScopeCaptures` populates several MODULE-LEVEL maps as a side effect + * that are NOT part of the returned `ParsedFile`'s scopes/defs: + * + * - `argInfoBySite` / `noAdlSites` (adl.ts) + * - `inlineNamespaceRangesByFile` (inline-namespaces.ts) + * - `fileLocalNames` / `anonymousNamespaceRangesByFile` (file-local-linkage.ts) + * - `dependentBasesByFile` / `dependentPackBaseClassesByFile` (two-phase-lookup.ts) + * + * On the worker path those maps are filled in the WORKER process and lost + * across the worker→main MessageChannel (and the disk-backed parsedfile-store), + * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the + * main-thread re-extraction — the entire point of #1983 is to avoid a + * main-thread tree-sitter re-parse on huge `.h`/`.cpp` repos (the OOM). + * + * This module snapshots the per-file slice of those maps into a plain, + * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and + * restores it on the main thread WITHOUT any parse. It is the data-only + * replacement for the removed re-parse `replayCaptureSideChannel` hook. + * + * The derived state each `populateOwners` / `populateWorkspaceOwners` pass + * builds (resolved scope-id Sets, `dependentBaseNodeIds`, etc.) is recomputed + * on the main thread from these restored capture-time maps, so only the + * capture-time maps need to cross the boundary. + */ + +import type { ParsedFile } from 'gitnexus-shared'; +import { collectCppAdlSideChannel, applyCppAdlSideChannel, type CppAdlSideChannel } from './adl.js'; +import { + collectCppInlineNamespaceSideChannel, + applyCppInlineNamespaceSideChannel, +} from './inline-namespaces.js'; +import { + collectCppFileLocalSideChannel, + applyCppFileLocalSideChannel, + type CppFileLocalSideChannel, +} from './file-local-linkage.js'; +import { + collectCppTwoPhaseSideChannel, + applyCppTwoPhaseSideChannel, + type CppTwoPhaseSideChannel, +} from './two-phase-lookup.js'; +import { + applyCppMemberLookupSideChannel, + collectCppMemberLookupSideChannel, + type CppMemberLookupSideChannel, +} from './member-lookup.js'; + +/** + * Plain JSON-serializable composite of every C++ capture-time side-channel + * slice for one file. Carried opaquely on `ParsedFile.captureSideChannel`. + */ +export interface CppCaptureSideChannel { + /** + * Discriminant tag — the single generic `ParsedFile.captureSideChannel` + * field is shared with C (`{ kind: 'c' }`) and Kotlin (`{ kind: 'kotlin' }`). + * `applyCppCaptureSideChannel` checks this first so a foreign-language + * payload reaching the C++ apply (or vice-versa) is cleanly ignored. In + * practice apply only runs for the matching provider (one language per file), + * but the tag makes it robust and consistent with the C/Kotlin snapshots. + */ + readonly kind: 'cpp'; + readonly adl: CppAdlSideChannel; + /** Inline-namespace source-range keys recorded for this file. */ + readonly inlineNamespaceRanges: readonly string[]; + readonly fileLocal: CppFileLocalSideChannel; + readonly twoPhase: CppTwoPhaseSideChannel; + readonly memberLookup: CppMemberLookupSideChannel; +} + +/** + * `LanguageProvider.collectCaptureSideChannel` implementation for C++. + * Returns `undefined` when this file recorded no side-channel state at all, so + * the produced `ParsedFile` carries the field only when there's data to ship. + */ +export function collectCppCaptureSideChannel(filePath: string): CppCaptureSideChannel | undefined { + const adl = collectCppAdlSideChannel(filePath); + const inlineNamespaceRanges = collectCppInlineNamespaceSideChannel(filePath); + const fileLocal = collectCppFileLocalSideChannel(filePath); + const twoPhase = collectCppTwoPhaseSideChannel(filePath); + const memberLookup = collectCppMemberLookupSideChannel(filePath); + + const isEmpty = + adl.argInfoBySite.length === 0 && + adl.noAdlSites.length === 0 && + inlineNamespaceRanges.length === 0 && + fileLocal.fileLocalNames.length === 0 && + fileLocal.anonymousNamespaceRanges.length === 0 && + twoPhase.dependentBases.length === 0 && + twoPhase.dependentPackBaseClasses.length === 0 && + memberLookup.baseEdges.length === 0 && + memberLookup.memberUsings.length === 0; + if (isEmpty) return undefined; + + return { kind: 'cpp', adl, inlineNamespaceRanges, fileLocal, twoPhase, memberLookup }; +} + +/** + * `ScopeResolver.applyCaptureSideChannel` implementation for C++. Reads the + * worker-serialized snapshot from `parsed.captureSideChannel` and writes it + * back into the module-level maps. Tolerant of `undefined` (file carried no + * data) and of an unexpected shape (defensive — never throws on a malformed + * snapshot). Does NO tree-sitter parse. + */ +export function applyCppCaptureSideChannel(parsed: ParsedFile): void { + const data = parsed.captureSideChannel as CppCaptureSideChannel | undefined; + if (data === undefined || data === null || typeof data !== 'object') return; + // Discriminant guard — the generic `captureSideChannel` field is shared + // with C (`{ kind: 'c' }`) and Kotlin (`{ kind: 'kotlin' }`); cleanly + // ignore a non-C++ payload rather than mis-applying it. + if (data.kind !== 'cpp') return; + if (data.adl !== undefined) applyCppAdlSideChannel(parsed.filePath, data.adl); + if (data.inlineNamespaceRanges !== undefined) { + applyCppInlineNamespaceSideChannel(parsed.filePath, data.inlineNamespaceRanges); + } + if (data.fileLocal !== undefined) applyCppFileLocalSideChannel(parsed.filePath, data.fileLocal); + if (data.twoPhase !== undefined) applyCppTwoPhaseSideChannel(parsed.filePath, data.twoPhase); + if (data.memberLookup !== undefined) { + applyCppMemberLookupSideChannel(parsed.filePath, data.memberLookup); + } +} diff --git a/gitnexus/src/core/ingestion/languages/cpp/captures.ts b/gitnexus/src/core/ingestion/languages/cpp/captures.ts index 265db8d5d..ef39076f1 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/captures.ts @@ -20,6 +20,7 @@ import { markCppDependentBase, markCppDependentPackBase } from './two-phase-look import { markCppAdlSiteArgs, markCppAdlSiteNoAdl, type CppAdlArgInfo } from './adl.js'; import { markCppInlineNamespaceRange } from './inline-namespaces.js'; import { extractCppTemplateConstraints } from './constraint-extractor.js'; +import { captureCppMemberLookupFacts } from './member-lookup.js'; export function emitCppScopeCaptures( sourceText: string, @@ -162,6 +163,13 @@ export function emitCppScopeCaptures( 'true', ); } + if (hasDeletedMethodClause(fnNode, grouped['@declaration.name']?.text)) { + grouped['@declaration.is-deleted'] = syntheticCapture( + '@declaration.is-deleted', + fnNode, + 'true', + ); + } // Detect static storage class (file-local linkage) if (hasStaticStorageClass(fnNode)) { @@ -464,6 +472,7 @@ export function emitCppScopeCaptures( // and the resolver can suppress unqualified-call binding to those // bases per ISO C++ two-phase lookup. detectCppDependentBases(tree.rootNode, filePath); + captureCppMemberLookupFacts(tree.rootNode, filePath); return out; } @@ -1683,7 +1692,13 @@ function extractDeclaratorLeafName(node: SyntaxNode): string | null { let cur: SyntaxNode = node; let safety = 16; while (safety-- > 0) { - if (cur.type === 'identifier' || cur.type === 'type_identifier') return cur.text; + if ( + cur.type === 'identifier' || + cur.type === 'type_identifier' || + cur.type === 'operator_name' + ) { + return cur.text; + } // Common wrapper nodes — follow the 'declarator' field when present. const next = cur.childForFieldName('declarator') ?? @@ -1711,6 +1726,25 @@ function hasExplicitSpecifier(node: SyntaxNode): boolean { return /\bexplicit\b/.test(node.text.slice(0, 128)); } +function hasDeletedMethodClause(node: SyntaxNode, callableName: string | undefined): boolean { + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child?.type === 'delete_method_clause') return true; + // tree-sitter-cpp 0.23 parses a deleted free-function declaration as + // `declaration > init_declarator > delete_expression`, while class + // members use the dedicated `delete_method_clause`. + if ( + child?.type === 'init_declarator' && + child.childForFieldName('value')?.type === 'delete_expression' && + callableName !== undefined && + extractDeclaratorLeafName(child.childForFieldName('declarator') ?? child) === callableName + ) { + return true; + } + } + return false; +} + /** * Check if a C++ function_definition or declaration has `static` storage class. */ diff --git a/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts b/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts index dd5fc8c0a..c6b383bf0 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/file-local-linkage.ts @@ -95,6 +95,47 @@ export function isCppAnonymousNamespaceScope(scopeId: ScopeId): boolean { return anonymousNamespaceScopeIds.has(scopeId); } +/** + * Plain-data, JSON-serializable snapshot of the per-file capture-time + * file-local-linkage state. Carried on `ParsedFile.captureSideChannel` across + * the worker→main boundary (#1983). The derived sets (`nonGloballyVisibleNodeIds`, + * `anonymousNamespaceScopeIds`) are recomputed by `populateCppNonGloballyVisible` + * / `populateCppAnonymousNamespaceScopes` during `populateOwners`, so only the + * two capture-time maps cross the boundary. + */ +export interface CppFileLocalSideChannel { + /** File-local symbol names (static / anonymous-namespace) in this file. */ + readonly fileLocalNames: readonly string[]; + /** Anonymous-namespace source-range keys recorded for this file. */ + readonly anonymousNamespaceRanges: readonly string[]; +} + +/** Snapshot this file's file-local-linkage capture state for the side-channel. */ +export function collectCppFileLocalSideChannel(filePath: string): CppFileLocalSideChannel { + const names = fileLocalNames.get(filePath); + const anon = anonymousNamespaceRangesByFile.get(filePath); + return { + fileLocalNames: names === undefined ? [] : [...names], + anonymousNamespaceRanges: anon === undefined ? [] : [...anon], + }; +} + +/** Restore this file's file-local-linkage capture state from the side-channel. */ +export function applyCppFileLocalSideChannel( + filePath: string, + data: CppFileLocalSideChannel, +): void { + for (const name of data.fileLocalNames) markFileLocal(filePath, name); + if (data.anonymousNamespaceRanges.length > 0) { + let set = anonymousNamespaceRangesByFile.get(filePath); + if (set === undefined) { + set = new Set(); + anonymousNamespaceRangesByFile.set(filePath, set); + } + for (const r of data.anonymousNamespaceRanges) set.add(r); + } +} + /** Clear tracked file-local names (call at start of each resolution pass). */ export function clearFileLocalNames(): void { fileLocalNames.clear(); @@ -235,11 +276,37 @@ export function isCppDefGloballyVisible(filePath: string, nodeId: string): boole * does, mirror this filter or harden registration so class/namespace * members never enter `localDefs` unqualified. */ +/** + * Per-pass memo: `moduleScope` → owning `ParsedFile`, keyed on the + * `parsedFiles` array identity. The shared finalize Phase-4 loop calls + * `expandsWildcardTo` (→ this) ONCE PER RESOLVED `#include` edge with the same + * `parsedFiles` reference; the old `parsedFiles.find(...)` was therefore O(F) + * per edge → O(R·F) overall (at kernel scale the ~25–30k `.h` headers are + * classified C++, so this fires hard — the C twin in `c/static-linkage.ts`). + * Building the lookup once collapses it to O(R+F). `WeakMap`-keyed so it is + * reclaimed with the pass (no cross-pass staleness; mirrors + * {@link clearFileLocalNames}). + */ +const moduleScopeIndexByPass = new WeakMap>(); + +function moduleScopeIndex(parsedFiles: readonly ParsedFile[]): Map { + let index = moduleScopeIndexByPass.get(parsedFiles); + if (index === undefined) { + index = new Map(); + // First-wins to preserve `Array.find` semantics (returns the first match). + for (const p of parsedFiles) { + if (!index.has(p.moduleScope)) index.set(p.moduleScope, p); + } + moduleScopeIndexByPass.set(parsedFiles, index); + } + return index; +} + export function expandCppWildcardNames( targetModuleScope: ScopeId, parsedFiles: readonly ParsedFile[], ): readonly string[] { - const target = parsedFiles.find((p) => p.moduleScope === targetModuleScope); + const target = moduleScopeIndex(parsedFiles).get(targetModuleScope); if (target === undefined) return []; // Build nodeId → owning Scope map from the structural scope tree. diff --git a/gitnexus/src/core/ingestion/languages/cpp/import-decomposer.ts b/gitnexus/src/core/ingestion/languages/cpp/import-decomposer.ts index eb6b252ce..4b0820589 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/import-decomposer.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/import-decomposer.ts @@ -5,6 +5,13 @@ import { nodeToCapture, syntheticCapture, type SyntaxNode } from '../../utils/as * Decompose a `preproc_include` node into a CaptureMatch with structured * import captures. C++ #include maps to a wildcard import (all symbols * from the header are visible). Identical to C's splitCInclude. + * + * Only literal include paths are emitted as import sources: + * #include → system_lib_string + * #include "User.h" → string_literal + * A computed include like `#include HEADER_MACRO` carries an `identifier` + * path node (the macro name, not a header path); we skip it so it never + * becomes a garbage literal import source (issue #1919 F5). */ export function splitCppInclude(node: SyntaxNode): CaptureMatch | null { const pathNode = node.childForFieldName?.('path') ?? null; @@ -21,7 +28,13 @@ export function splitCppInclude(node: SyntaxNode): CaptureMatch | null { return buildIncludeCapture(node, pathNode); } -function buildIncludeCapture(node: SyntaxNode, pathNode: SyntaxNode): CaptureMatch { +function buildIncludeCapture(node: SyntaxNode, pathNode: SyntaxNode): CaptureMatch | null { + // Skip computed includes (`#include MACRO`) — the path is an `identifier`, + // not a literal header path. Emitting it would create a garbage import. + if (pathNode.type !== 'string_literal' && pathNode.type !== 'system_lib_string') { + return null; + } + let raw: string; if (pathNode.type === 'string_literal') { const content = pathNode.namedChildren.find((c) => c.type === 'string_content'); @@ -60,6 +73,12 @@ function buildIncludeCapture(node: SyntaxNode, pathNode: SyntaxNode): CaptureMat */ export function splitCppUsingDecl(node: SyntaxNode): CaptureMatch | null { if (node.type !== 'using_declaration') return null; + // A class-scope `using Base::member;` changes the derived class's member + // lookup set; it is not a namespace import. The C++ member-lookup sidecar + // captures it separately, so suppress import decomposition here. + for (let parent = node.parent; parent !== null; parent = parent.parent) { + if (parent.type === 'class_specifier' || parent.type === 'struct_specifier') return null; + } // Check for "namespace" keyword among anonymous children let hasNamespaceKeyword = false; diff --git a/gitnexus/src/core/ingestion/languages/cpp/index.ts b/gitnexus/src/core/ingestion/languages/cpp/index.ts index c4d208d76..4bc52dd32 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/index.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/index.ts @@ -14,3 +14,7 @@ export { clearFileLocalNames, expandCppWildcardNames, } from './file-local-linkage.js'; +export { + collectCppCaptureSideChannel, + applyCppCaptureSideChannel, +} from './capture-side-channel.js'; diff --git a/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts b/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts index eec44c68f..c7280bf46 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/inline-namespaces.ts @@ -61,6 +61,30 @@ export function markCppInlineNamespaceRange(filePath: string, range: RangeKey): set.add(rangeKey(range)); } +/** Snapshot this file's captured inline-namespace ranges for the worker→main + * side-channel (#1983). `populateCppInlineNamespaceScopes` (in `populateOwners`) + * later resolves these range keys to ScopeIds on the main thread, so only the + * capture-time ranges need to cross the boundary. Returns the rangeKey strings + * as a plain array (empty when this file recorded none). */ +export function collectCppInlineNamespaceSideChannel(filePath: string): readonly string[] { + const set = inlineNamespaceRangesByFile.get(filePath); + return set === undefined ? [] : [...set]; +} + +/** Restore this file's captured inline-namespace ranges from the side-channel. */ +export function applyCppInlineNamespaceSideChannel( + filePath: string, + ranges: readonly string[], +): void { + if (ranges.length === 0) return; + let set = inlineNamespaceRangesByFile.get(filePath); + if (set === undefined) { + set = new Set(); + inlineNamespaceRangesByFile.set(filePath, set); + } + for (const r of ranges) set.add(r); +} + /** Clear all inline-namespace state. Called from `clearFileLocalNames`. */ export function clearCppInlineNamespaces(): void { inlineNamespaceRangesByFile.clear(); diff --git a/gitnexus/src/core/ingestion/languages/cpp/member-lookup.ts b/gitnexus/src/core/ingestion/languages/cpp/member-lookup.ts new file mode 100644 index 000000000..a681c4952 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/cpp/member-lookup.ts @@ -0,0 +1,616 @@ +import type { ParsedFile, ReferenceSite, SymbolDefinition } from 'gitnexus-shared'; +import type { KnowledgeGraph } from '../../../graph/types.js'; +import type { GraphNodeLookup } from '../../scope-resolution/graph-bridge/node-lookup.js'; +import { resolveDefGraphId } from '../../scope-resolution/graph-bridge/ids.js'; +import type { ScopeResolutionIndexes } from '../../model/scope-resolution-indexes.js'; +import type { SemanticModel } from '../../model/semantic-model.js'; +import type { ReceiverMemberResolution } from '../../scope-resolution/contract/scope-resolver.js'; +import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; +import { + isOverloadAmbiguousAfterNormalization, + narrowOverloadCandidates, +} from '../../scope-resolution/passes/overload-narrowing.js'; +import { isClassLike } from '../../scope-resolution/scope/walkers.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { cppConstraintCompatibility } from './constraint-filter.js'; +import { cppConversionRank } from './conversion-rank.js'; + +interface CapturedBaseEdge { + readonly childName: string; + readonly childQualifiedName?: string; + readonly baseName: string; + readonly baseQualifiedName?: string; + readonly isVirtual: boolean; +} + +interface CapturedMemberUsing { + readonly childName: string; + readonly childQualifiedName?: string; + readonly baseName: string; + readonly baseQualifiedName?: string; + readonly memberName: string; +} + +export interface CppMemberLookupSideChannel { + readonly baseEdges: readonly CapturedBaseEdge[]; + readonly memberUsings: readonly CapturedMemberUsing[]; +} + +const capturedByFile = new Map(); +let directParentsByDefId = new Map(); +let virtualEdges = new Set(); +let ancestorsByDefId = new Map>(); +let memberUsingsByDefId = new Map< + string, + readonly { readonly baseDefId: string; readonly memberName: string }[] +>(); +let inheritedLookupCache = new Map(); + +const MAX_INHERITANCE_VISITS = 4096; + +type CachedInheritedLookup = + | { readonly kind: 'none' } + | { readonly kind: 'candidates'; readonly definitions: readonly SymbolDefinition[] } + | { readonly kind: 'ambiguous'; readonly candidateIds: readonly string[] }; + +export function clearCppMemberLookupState(): void { + capturedByFile.clear(); + directParentsByDefId = new Map(); + virtualEdges = new Set(); + ancestorsByDefId = new Map(); + memberUsingsByDefId = new Map(); + inheritedLookupCache = new Map(); +} + +export function captureCppMemberLookupFacts(root: SyntaxNode, filePath: string): void { + const baseEdges: CapturedBaseEdge[] = []; + const memberUsings: CapturedMemberUsing[] = []; + const stack: SyntaxNode[] = [root]; + + while (stack.length > 0) { + const node = stack.pop()!; + if (node.type === 'class_specifier' || node.type === 'struct_specifier') { + const childName = classNameOf(node); + const childQualifiedName = classQualifiedNameOf(node); + if (childName !== '') { + const baseClause = directChildOfType(node, 'base_class_clause'); + if (baseClause !== null) { + captureBaseEdges(baseClause, childName, childQualifiedName, baseEdges); + } + const body = directChildOfType(node, 'field_declaration_list'); + if (body !== null) { + for (let i = 0; i < body.namedChildCount; i++) { + const child = body.namedChild(i); + if (child?.type !== 'using_declaration') continue; + const parsed = parseMemberUsing(child, childName, childQualifiedName); + if (parsed !== undefined) memberUsings.push(parsed); + } + } + } + } + for (let i = 0; i < node.childCount; i++) { + const child = node.child(i); + if (child !== null) stack.push(child); + } + } + + if (baseEdges.length === 0 && memberUsings.length === 0) { + capturedByFile.delete(filePath); + } else { + capturedByFile.set(filePath, { baseEdges, memberUsings }); + } +} + +export function collectCppMemberLookupSideChannel(filePath: string): CppMemberLookupSideChannel { + return capturedByFile.get(filePath) ?? { baseEdges: [], memberUsings: [] }; +} + +export function applyCppMemberLookupSideChannel( + filePath: string, + data: CppMemberLookupSideChannel, +): void { + if (!Array.isArray(data.baseEdges) || !Array.isArray(data.memberUsings)) return; + if (data.baseEdges.length === 0 && data.memberUsings.length === 0) { + capturedByFile.delete(filePath); + return; + } + capturedByFile.set(filePath, { + baseEdges: data.baseEdges.slice(), + memberUsings: data.memberUsings.slice(), + }); +} + +export function buildCppMemberLookupMro( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, +): Map { + populateResolvedHierarchy(graph, parsedFiles, nodeLookup); + return buildMro(graph, parsedFiles, nodeLookup, defaultLinearize); +} + +export function resolveCppReceiverMember( + ownerDef: SymbolDefinition, + memberName: string, + callsite: ReferenceSite, + _scopes: ScopeResolutionIndexes, + model: SemanticModel, +): ReceiverMemberResolution | undefined { + if (callsite.kind !== 'call') return undefined; + const ownMethods = model.methods.lookupAllByOwner(ownerDef.nodeId, memberName); + const introduced = introducedDefinitions(ownerDef.nodeId, memberName, model); + + if (introduced.length > 0) { + return chooseOverload(uniqueDefinitions([...ownMethods, ...introduced]), callsite); + } + + // Direct declarations hide every base declaration. Let the shared path + // retain its existing overload/static filtering for this common case. + if (ownMethods.length > 0) return undefined; + + const lookup = inheritedLookupSet(ownerDef.nodeId, memberName, model); + if (lookup.kind === 'none') return undefined; + if (lookup.kind === 'ambiguous') return lookup; + return chooseOverload(lookup.definitions, callsite); +} + +interface MemberOccurrence { + readonly ownerDefId: string; + readonly definitions: readonly SymbolDefinition[]; + readonly path: readonly string[]; + readonly virtualAnchor?: string; +} + +function collectInheritedOccurrences( + ownerDefId: string, + memberName: string, + model: SemanticModel, + path: readonly string[], + virtualAnchor: string | undefined, + active: Set, + budget: { remaining: number; truncated: boolean }, +): MemberOccurrence[] { + if (budget.remaining <= 0) { + budget.truncated = true; + return []; + } + budget.remaining--; + if (active.has(ownerDefId)) return []; + const nextActive = new Set(active); + nextActive.add(ownerDefId); + + const definitions = uniqueDefinitions([ + ...model.methods.lookupAllByOwner(ownerDefId, memberName), + ...introducedDefinitions(ownerDefId, memberName, model), + ]); + if (definitions.length > 0) { + return [{ ownerDefId, definitions, path, virtualAnchor }]; + } + + const results: MemberOccurrence[] = []; + for (const parentDefId of directParentsByDefId.get(ownerDefId) ?? []) { + const edgeKey = `${ownerDefId}\0${parentDefId}`; + results.push( + ...collectInheritedOccurrences( + parentDefId, + memberName, + model, + [...path, parentDefId], + virtualEdges.has(edgeKey) ? parentDefId : virtualAnchor, + nextActive, + budget, + ), + ); + } + return results; +} + +function inheritedLookupSet( + ownerDefId: string, + memberName: string, + model: SemanticModel, +): CachedInheritedLookup { + const cacheKey = `${ownerDefId}\0${memberName}`; + const cached = inheritedLookupCache.get(cacheKey); + if (cached !== undefined) return cached; + + const budget = { remaining: MAX_INHERITANCE_VISITS, truncated: false }; + const occurrences = collectInheritedOccurrences( + ownerDefId, + memberName, + model, + [], + undefined, + new Set(), + budget, + ); + if (budget.truncated) { + const conservative: CachedInheritedLookup = { + kind: 'ambiguous', + candidateIds: uniqueDefinitions(occurrences.flatMap((entry) => entry.definitions)).map( + (definition) => definition.nodeId, + ), + }; + inheritedLookupCache.set(cacheKey, conservative); + return conservative; + } + if (occurrences.length === 0) { + const none: CachedInheritedLookup = { kind: 'none' }; + inheritedLookupCache.set(cacheKey, none); + return none; + } + + // A declaration can dominate another lookup set only when the latter is + // reached through a shared virtual subobject. Ordinary ancestry alone is + // insufficient: declarations in one non-virtual branch do not hide members + // reached through a sibling base subobject. + const undominated = occurrences.filter( + (candidate) => + !( + candidate.virtualAnchor !== undefined && + occurrences.some( + (other) => + other.ownerDefId !== candidate.ownerDefId && + isAncestor(candidate.ownerDefId, other.ownerDefId), + ) + ), + ); + const groups = new Map(); + for (const occurrence of undominated) { + const key = + occurrence.virtualAnchor !== undefined + ? `virtual:${occurrence.virtualAnchor}:${occurrence.ownerDefId}` + : `path:${occurrence.path.join('>')}:${occurrence.ownerDefId}`; + const bucket = groups.get(key); + if (bucket === undefined) groups.set(key, [occurrence]); + else bucket.push(occurrence); + } + + let result: CachedInheritedLookup; + if (groups.size !== 1) { + result = { + kind: 'ambiguous', + candidateIds: uniqueDefinitions(undominated.flatMap((entry) => entry.definitions)).map( + (definition) => definition.nodeId, + ), + }; + } else { + result = { + kind: 'candidates', + definitions: groups.values().next().value?.[0]?.definitions ?? [], + }; + } + inheritedLookupCache.set(cacheKey, result); + return result; +} + +function introducedDefinitions( + ownerDefId: string, + memberName: string, + model: SemanticModel, +): SymbolDefinition[] { + const definitions: SymbolDefinition[] = []; + for (const entry of memberUsingsByDefId.get(ownerDefId) ?? []) { + if (entry.memberName !== memberName) continue; + definitions.push(...model.methods.lookupAllByOwner(entry.baseDefId, memberName)); + } + return definitions; +} + +function uniqueDefinitions(definitions: readonly SymbolDefinition[]): SymbolDefinition[] { + return [...new Map(definitions.map((definition) => [definition.nodeId, definition])).values()]; +} + +function chooseOverload( + candidates: readonly SymbolDefinition[], + callsite: ReferenceSite, +): ReceiverMemberResolution | undefined { + if (candidates.length === 0) return undefined; + const narrowed = narrowOverloadCandidates(candidates, callsite.arity, callsite.argumentTypes, { + argumentTypeClasses: callsite.argumentTypeClasses, + conversionRankFn: cppConversionRank, + constraintCompatibility: cppConstraintCompatibility, + }); + if (narrowed.length === 1) return { kind: 'resolved', definition: narrowed[0]! }; + if (narrowed.length > 1 || isOverloadAmbiguousAfterNormalization(narrowed, callsite.arity)) { + return { + kind: 'ambiguous', + candidateIds: narrowed.map((candidate) => candidate.nodeId), + }; + } + return undefined; +} + +function populateResolvedHierarchy( + graph: KnowledgeGraph, + parsedFiles: readonly ParsedFile[], + nodeLookup: GraphNodeLookup, +): void { + const defByGraphId = new Map(); + const defById = new Map(); + const defsByFileAndName = new Map(); + + for (const parsed of parsedFiles) { + for (const def of parsed.localDefs) { + if (!isClassLike(def.type)) continue; + const graphId = resolveDefGraphId(parsed.filePath, def, nodeLookup); + if (graphId === undefined) continue; + defByGraphId.set(graphId, def); + defById.set(def.nodeId, def); + const names = new Set([simpleName(def), definitionQualifiedName(def)]); + for (const name of names) { + if (name === '') continue; + const key = `${parsed.filePath}\0${name}`; + const bucket = defsByFileAndName.get(key); + if (bucket === undefined) defsByFileAndName.set(key, [def]); + else bucket.push(def); + } + } + } + + const parents = new Map(); + for (const rel of graph.iterRelationshipsByType('EXTENDS')) { + const child = defByGraphId.get(rel.sourceId); + const parent = defByGraphId.get(rel.targetId); + if (child === undefined || parent === undefined) continue; + const bucket = parents.get(child.nodeId); + if (bucket === undefined) parents.set(child.nodeId, [parent.nodeId]); + else bucket.push(parent.nodeId); + } + directParentsByDefId = parents; + ancestorsByDefId = buildAncestorClosure(parents); + inheritedLookupCache = new Map(); + + const nextVirtualEdges = new Set(); + const nextUsings = new Map< + string, + { readonly baseDefId: string; readonly memberName: string }[] + >(); + for (const parsed of parsedFiles) { + const captured = capturedByFile.get(parsed.filePath); + if (captured === undefined) continue; + for (const edge of captured.baseEdges) { + if (!edge.isVirtual) continue; + for (const child of matchingChildren( + parsed.filePath, + edge.childName, + edge.childQualifiedName, + defsByFileAndName, + )) { + const parent = findCapturedParent( + parents.get(child.nodeId) ?? [], + edge.baseName, + edge.baseQualifiedName, + defById, + ); + if (parent !== undefined) nextVirtualEdges.add(`${child.nodeId}\0${parent.nodeId}`); + } + } + for (const using of captured.memberUsings) { + const children = matchingChildren( + parsed.filePath, + using.childName, + using.childQualifiedName, + defsByFileAndName, + ); + for (const child of children) { + const baseDef = findCapturedParent( + parents.get(child.nodeId) ?? [], + using.baseName, + using.baseQualifiedName, + defById, + ); + if (baseDef === undefined) continue; + const bucket = nextUsings.get(child.nodeId); + const entry = { baseDefId: baseDef.nodeId, memberName: using.memberName }; + if (bucket === undefined) nextUsings.set(child.nodeId, [entry]); + else bucket.push(entry); + } + } + } + virtualEdges = nextVirtualEdges; + memberUsingsByDefId = nextUsings; +} + +function captureBaseEdges( + baseClause: SyntaxNode, + childName: string, + childQualifiedName: string, + output: CapturedBaseEdge[], +): void { + let segmentStart = 0; + for (let i = 0; i < baseClause.childCount; i++) { + const child = baseClause.child(i); + if (child === null) continue; + if (child.type === ',' || child.text === ',') { + segmentStart = i + 1; + continue; + } + if ( + child.type !== 'type_identifier' && + child.type !== 'template_type' && + child.type !== 'qualified_identifier' + ) { + continue; + } + let isVirtual = false; + for (let j = segmentStart; j < i; j++) { + const modifier = baseClause.child(j); + if (modifier?.text === 'virtual') isVirtual = true; + } + const baseQualifiedName = qualifiedTypeName(child.text); + const baseName = baseQualifiedName.split('.').at(-1) ?? ''; + if (baseName !== '') { + output.push({ + childName, + ...(childQualifiedName !== childName ? { childQualifiedName } : {}), + baseName, + ...(baseQualifiedName !== baseName ? { baseQualifiedName } : {}), + isVirtual, + }); + } + } +} + +function parseMemberUsing( + node: SyntaxNode, + childName: string, + childQualifiedName: string, +): CapturedMemberUsing | undefined { + const qualified = node.namedChildren.find((child) => child.type === 'qualified_identifier'); + if (qualified === undefined) return undefined; + const parts = splitQualifiedSegments(qualified.text); + if (parts.length < 2) return undefined; + const memberName = stripTemplateSuffix(parts.at(-1) ?? ''); + const baseParts = parts.slice(0, -1).map(stripTemplateSuffix).filter(Boolean); + const baseName = baseParts.at(-1) ?? ''; + const baseQualifiedName = baseParts.join('.'); + if (baseName === '' || memberName === '') return undefined; + return { + childName, + ...(childQualifiedName !== childName ? { childQualifiedName } : {}), + baseName, + ...(baseQualifiedName !== baseName ? { baseQualifiedName } : {}), + memberName, + }; +} + +function classNameOf(node: SyntaxNode): string { + const name = node.childForFieldName?.('name'); + return name === null || name === undefined ? '' : trailingIdentifier(name.text); +} + +function classQualifiedNameOf(node: SyntaxNode): string { + const parts = [classNameOf(node)]; + let current = node.parent; + while (current !== null) { + if (current.type === 'class_specifier' || current.type === 'struct_specifier') { + const name = classNameOf(current); + if (name !== '') parts.unshift(name); + } else if (current.type === 'namespace_definition') { + const name = current.childForFieldName?.('name'); + if (name !== null && name !== undefined) { + parts.unshift( + ...splitQualifiedSegments(name.text).map(stripTemplateSuffix).filter(Boolean), + ); + } + } + current = current.parent; + } + return parts.filter(Boolean).join('.'); +} + +function directChildOfType(node: SyntaxNode, type: string): SyntaxNode | null { + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child?.type === type) return child; + } + return null; +} + +function trailingIdentifier(value: string): string { + return stripTemplateSuffix(splitQualifiedSegments(value).at(-1) ?? ''); +} + +function qualifiedTypeName(value: string): string { + return splitQualifiedSegments(value).map(stripTemplateSuffix).filter(Boolean).join('.'); +} + +function splitQualifiedSegments(value: string): string[] { + const parts: string[] = []; + let angleDepth = 0; + let segmentStart = 0; + for (let i = 0; i < value.length; i++) { + const char = value[i]; + if (char === '<') angleDepth++; + else if (char === '>' && angleDepth > 0) angleDepth--; + else if (char === ':' && value[i + 1] === ':' && angleDepth === 0) { + const segment = value.slice(segmentStart, i).trim(); + if (segment !== '') parts.push(segment); + segmentStart = i + 2; + i++; + } + } + const tail = value.slice(segmentStart).trim(); + if (tail !== '') parts.push(tail); + return parts; +} + +function stripTemplateSuffix(value: string): string { + const templateStart = value.indexOf('<'); + return (templateStart >= 0 ? value.slice(0, templateStart) : value).trim(); +} + +function simpleName(def: SymbolDefinition): string { + return def.qualifiedName?.split('.').at(-1) ?? ''; +} + +function definitionQualifiedName(def: SymbolDefinition): string { + const name = def.qualifiedName ?? ''; + if (name === '' || def.namespacePrefix === undefined || def.namespacePrefix === '') return name; + return name.startsWith(`${def.namespacePrefix}.`) ? name : `${def.namespacePrefix}.${name}`; +} + +function matchingChildren( + filePath: string, + childName: string, + childQualifiedName: string | undefined, + defsByFileAndName: ReadonlyMap, +): readonly SymbolDefinition[] { + if (childQualifiedName !== undefined) { + const qualified = defsByFileAndName.get(`${filePath}\0${childQualifiedName}`) ?? []; + if (qualified.length > 0) return qualified; + } + const simple = defsByFileAndName.get(`${filePath}\0${childName}`) ?? []; + return simple.length === 1 ? simple : []; +} + +function findCapturedParent( + parentIds: readonly string[], + baseName: string, + baseQualifiedName: string | undefined, + defById: ReadonlyMap, +): SymbolDefinition | undefined { + const candidates = parentIds + .map((id) => defById.get(id)) + .filter((definition): definition is SymbolDefinition => definition !== undefined); + if (baseQualifiedName !== undefined) { + const qualified = candidates.filter((definition) => { + const name = definitionQualifiedName(definition); + return name === baseQualifiedName || name.endsWith(`.${baseQualifiedName}`); + }); + if (qualified.length === 1) return qualified[0]; + return undefined; + } + const simple = candidates.filter((definition) => simpleName(definition) === baseName); + return simple.length === 1 ? simple[0] : undefined; +} + +function buildAncestorClosure( + parents: ReadonlyMap, +): Map> { + const closure = new Map>(); + const visiting = new Set(); + + const ancestorsOf = (defId: string): ReadonlySet => { + const cached = closure.get(defId); + if (cached !== undefined) return cached; + if (visiting.has(defId)) return new Set(); + visiting.add(defId); + const ancestors = new Set(); + for (const parent of parents.get(defId) ?? []) { + ancestors.add(parent); + for (const ancestor of ancestorsOf(parent)) ancestors.add(ancestor); + } + visiting.delete(defId); + closure.set(defId, ancestors); + return ancestors; + }; + + for (const defId of parents.keys()) ancestorsOf(defId); + return closure; +} + +function isAncestor(ancestorDefId: string, descendantDefId: string): boolean { + return ancestorsByDefId.get(descendantDefId)?.has(ancestorDefId) === true; +} diff --git a/gitnexus/src/core/ingestion/languages/cpp/query.ts b/gitnexus/src/core/ingestion/languages/cpp/query.ts index aa3202abe..b50463a7f 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/query.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/query.ts @@ -194,6 +194,29 @@ const CPP_SCOPE_QUERY = ` declarator: (function_declarator declarator: (identifier) @declaration.name)) @declaration.function +;; tree-sitter-cpp 0.23 represents a deleted free function as an +;; init_declarator whose value is a delete_expression. +(declaration + declarator: (init_declarator + declarator: (function_declarator + declarator: (identifier) @declaration.name) + value: (delete_expression))) @declaration.function + +;; Deleted free operator declaration. +(declaration + declarator: (init_declarator + declarator: (function_declarator + declarator: (operator_name) @declaration.name) + value: (delete_expression))) @declaration.function + +;; Deleted free function with a pointer return type. +(declaration + declarator: (init_declarator + declarator: (pointer_declarator + declarator: (function_declarator + declarator: (identifier) @declaration.name)) + value: (delete_expression))) @declaration.function + ;; Free operator prototype: std::ostream& operator<<(std::ostream&, T) (declaration declarator: (function_declarator diff --git a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts index 459b313c6..3ef89bc07 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/scope-resolver.ts @@ -4,7 +4,6 @@ import { findEnclosingClassDef, } from '../../scope-resolution/scope/walkers.js'; import { SupportedLanguages } from 'gitnexus-shared'; -import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; import { populateClassOwnedMembers, tagNamespacePrefixes, @@ -30,6 +29,7 @@ import { isCppDependentBaseMember, } from './two-phase-lookup.js'; import { populateCppAssociatedNamespaces, clearCppAdlState, pickCppAdlCandidates } from './adl.js'; +import { applyCppCaptureSideChannel } from './capture-side-channel.js'; import { clearCppInlineNamespaces, populateCppInlineNamespaceScopes, @@ -41,6 +41,45 @@ import { clearCppUserDefinedConversions, populateCppUserDefinedConversions, } from './user-defined-conversions.js'; +import { + buildCppMemberLookupMro, + clearCppMemberLookupState, + resolveCppReceiverMember, +} from './member-lookup.js'; + +/** + * Per-pass memo of the augmented `#include`-resolution file set + * (`allFilePaths` ∪ header paths), keyed on the two stable source sets. + * `resolveImportTarget` is called once per C++ `#include`; the old code rebuilt + * a fresh ~F-entry `Set` on every call AND defeated the shared + * `resolveCImportTarget` suffix-index memo (in `c/import-target.ts`) by handing + * it a new set identity each time. Both inputs are stable per pass, so the + * union is built once and reused. `WeakMap`-keyed → reclaimed with the pass. + * (Twin of the C resolver's `augmentedFilePaths`.) + */ +const augmentedPathsByPass = new WeakMap< + ReadonlySet, + WeakMap, ReadonlySet> +>(); + +function augmentedFilePaths( + allFilePaths: ReadonlySet, + headerPaths: ReadonlySet, +): ReadonlySet { + let byHeaders = augmentedPathsByPass.get(allFilePaths); + if (byHeaders === undefined) { + byHeaders = new WeakMap(); + augmentedPathsByPass.set(allFilePaths, byHeaders); + } + let augmented = byHeaders.get(headerPaths); + if (augmented === undefined) { + const set = new Set(allFilePaths); + for (const h of headerPaths) set.add(h); + augmented = set; + byHeaders.set(headerPaths, augmented); + } + return augmented; +} /** * C++ `ScopeResolver` registered in `SCOPE_RESOLVERS` and consumed by @@ -69,6 +108,7 @@ export const cppScopeResolver: ScopeResolver = { clearCppAdlState(); clearCppInlineNamespaces(); clearCppUserDefinedConversions(); + clearCppMemberLookupState(); return scanCppHeaderFiles(repoPath); }, @@ -78,9 +118,11 @@ export const cppScopeResolver: ScopeResolver = { // detection but are importable from .cpp files via #include. const headerPaths = resolutionConfig as ReadonlySet | undefined; if (headerPaths !== undefined && headerPaths.size > 0) { - const augmented = new Set(allFilePaths); - for (const h of headerPaths) augmented.add(h); - return resolveCppImportTarget(targetRaw, fromFile, augmented); + return resolveCppImportTarget( + targetRaw, + fromFile, + augmentedFilePaths(allFilePaths, headerPaths), + ); } return resolveCppImportTarget(targetRaw, fromFile, allFilePaths); }, @@ -100,8 +142,25 @@ export const cppScopeResolver: ScopeResolver = { // `'unknown'` keeps the candidate, preserving "degrade not lie". constraintCompatibility: cppConstraintCompatibility, - buildMro: (graph, parsedFiles, nodeLookup) => - buildMro(graph, parsedFiles, nodeLookup, defaultLinearize), + buildMro: buildCppMemberLookupMro, + + // Worker-boundary restore (see `ScopeResolver.applyCaptureSideChannel`). + // `emitCppScopeCaptures` records per-file ADL call-site arg shapes + // (`markCppAdlSiteArgs`/`markCppAdlSiteNoAdl`), inline-/anonymous-namespace + // ranges (`markCppInlineNamespaceRange`/`markCppAnonymousNamespaceRange`), + // dependent-base names (`markCppDependentBase`/`markCppDependentPackBase`), + // and file-local linkage (`markFileLocal`) into module-level maps as a SIDE + // EFFECT — none of it is serialized onto the returned ParsedFile's scopes/defs. + // On the worker path those marks are populated in the worker process and lost + // across the MessageChannel / disk store; the main thread reuses the + // serialized ParsedFile and skips `extractParsedFile`, so `populateOwners` + + // the ADL / two-phase-lookup passes would see empty maps and emit zero edges. + // The worker stashed a plain-data snapshot on `parsed.captureSideChannel` via + // `cppProvider.collectCaptureSideChannel`; this restores it into the module + // maps WITHOUT any tree-sitter re-parse (the #1983 fix — the old re-parse + // replay re-OOM'd huge `.h`/`.cpp` repos). The freshly-extracted leg never + // calls this — its marks were just populated in this process. + applyCaptureSideChannel: applyCppCaptureSideChannel, populateOwners: (parsed: ParsedFile) => { populateClassOwnedMembers(parsed); @@ -206,6 +265,7 @@ export const cppScopeResolver: ScopeResolver = { hoistTypeBindingsToModule: true, // Enable receiver-bound explicit-`this` fallback only for C++. resolveThisViaEnclosingClass: true, + resolveReceiverMember: resolveCppReceiverMember, // The `isFileLocalDef` hook on the global free-call fallback names // file-local linkage historically, but semantically gates "logically // invisible cross-file" defs. C++ extends this to also reject class- diff --git a/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts b/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts index 7ec55bc9a..739cceeb5 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/two-phase-lookup.ts @@ -104,6 +104,56 @@ export function markCppDependentPackBase(filePath: string, className: string): v perFile.add(className); } +/** + * Plain-data, JSON-serializable snapshot of the per-file capture-time + * two-phase-lookup state. Carried on `ParsedFile.captureSideChannel` across the + * worker→main boundary (#1983). The resolved `dependentBaseNodeIds` index is + * rebuilt by `populateCppDependentBases` (workspace pass) after all files have + * their `populateOwners` applied, so only the two capture-time maps cross. + * + * Nested `Map`/`Set` are flattened to arrays here so the snapshot stays plain + * JSON (avoids relying on the parsedfile-store's Map/Set replacer for nested + * structures): `dependentBases` is `[className, [baseName, qualifiers[]][]][]`. + */ +export interface CppTwoPhaseSideChannel { + readonly dependentBases: readonly [string, readonly [string, readonly string[]][]][]; + readonly dependentPackBaseClasses: readonly string[]; +} + +/** Snapshot this file's two-phase-lookup capture state for the side-channel. */ +export function collectCppTwoPhaseSideChannel(filePath: string): CppTwoPhaseSideChannel { + const perFile = dependentBasesByFile.get(filePath); + const dependentBases: [string, [string, string[]][]][] = []; + if (perFile !== undefined) { + for (const [className, bases] of perFile) { + const baseEntries: [string, string[]][] = []; + for (const [baseName, quals] of bases) { + baseEntries.push([baseName, [...quals]]); + } + dependentBases.push([className, baseEntries]); + } + } + const pack = dependentPackBaseClassesByFile.get(filePath); + return { + dependentBases, + dependentPackBaseClasses: pack === undefined ? [] : [...pack], + }; +} + +/** Restore this file's two-phase-lookup capture state from the side-channel. */ +export function applyCppTwoPhaseSideChannel(filePath: string, data: CppTwoPhaseSideChannel): void { + for (const [className, baseEntries] of data.dependentBases) { + for (const [baseName, quals] of baseEntries) { + for (const qualifier of quals) { + markCppDependentBase(filePath, className, baseName, qualifier); + } + } + } + for (const className of data.dependentPackBaseClasses) { + markCppDependentPackBase(filePath, className); + } +} + /** Clear two-phase-lookup state. Called from `clearFileLocalNames`. */ export function clearCppDependentBases(): void { dependentBasesByFile.clear(); diff --git a/gitnexus/src/core/ingestion/languages/csharp/captures.ts b/gitnexus/src/core/ingestion/languages/csharp/captures.ts index d5914780d..3f28e6be9 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/captures.ts @@ -86,10 +86,11 @@ export function emitCsharpScopeCaptures( _filePath: string, cachedTree?: unknown, ): readonly CaptureMatch[] { - // Skip the parse when the caller (parse phase's scopeTreeCache) - // already produced a Tree for this source. Cache miss = re-parse, - // same as before. The cachedTree parameter is typed as `unknown` at - // the LanguageProvider contract layer; cast here at the use site. + // Reuse a pre-parsed Tree when the caller passes one via `cachedTree`; a + // miss re-parses. (The cache is currently always empty — its only producer, + // the sequential parser, was removed — so this re-parses in practice.) The + // cachedTree parameter is typed `unknown` at the LanguageProvider contract + // layer; cast here at the use site. let tree = cachedTree as ReturnType['parse']> | undefined; if (tree === undefined) { tree = parseSourceSafe(getCsharpParser(), sourceText, undefined, { @@ -196,6 +197,33 @@ export function emitCsharpScopeCaptures( } } + // Qualified constructor calls — `new Ns.Foo()`, `new A.B.Foo()`, + // `new Ns.Box()` — bind only `@reference.call.constructor.qualified` + // with NO `@reference.name`, so the central extractor falls back to the + // whole-expression anchor and the reference name becomes the raw + // `new Ns.Foo()` text (never resolves). Derive the bare simple-name tail via + // the same `terminalTypeNameNode` helper the inheritance synth uses — it + // handles qualified_name, a generic tail (`Ns.Box` → `Box`), and + // alias_qualified_name (`global::Ns.Foo`). Mirrors Java F35 (#1928). + if ( + grouped['@reference.call.constructor.qualified'] !== undefined && + grouped['@reference.name'] === undefined + ) { + const qNode = nodeMap['@reference.call.constructor.qualified']; + const nameNode = qNode === undefined ? null : terminalTypeNameNode(qNode); + if (nameNode !== null) { + grouped['@reference.name'] = nodeToCapture('@reference.name', nameNode); + const qText = qNode.text.trim(); + if (qText.length > 0 && qText !== nameNode.text) { + grouped['@reference.qualified-name'] = syntheticCapture( + '@reference.qualified-name', + qNode, + qText, + ); + } + } + } + // Synthesize `@reference.arity` on every callsite so the // registry's arity filter can narrow overloads. Count the // `argument` named children of the backing `argument_list`. @@ -263,10 +291,99 @@ export function emitCsharpScopeCaptures( out.push(...synthesizeGenericTypeArgumentReferences(tree.rootNode)); out.push(...synthesizeCsharpInheritanceReferences(tree.rootNode)); + out.push(...synthesizeCsharpConstructorInitializerReferences(tree.rootNode)); return out; } +/** + * Synthesize `@reference.call.constructor` captures for C# constructor + * initializers — `: base(...)` and `: this(...)` (F38 analog of Java #1928). + * tree-sitter-c-sharp models these as `constructor_initializer` nodes the scope + * query never matched, so the chained-constructor CALLS edges (derived ctor → + * base ctor; ctor → sibling overload) were silently dropped. + * + * The initializer carries no constructor name (the `base`/`this` child is a bare + * keyword token), so the target is resolved structurally: + * - `this(...)` → the enclosing type's own simple name. + * - `base(...)` → the enclosing class/record's base type, reduced to its bare + * simple name via `terminalTypeNameNode`. C# requires the base class first in + * a mixed list (`class C : Base, IFoo`); interface-only lists (`class C : IFoo`) + * imply implicit `System.Object` — no `@reference` is emitted when the first + * non-builtin base would be an interface-only target (resolution also drops + * Interface-typed constructor targets in `free-call-fallback`). + * Arity is attached for overload disambiguation, mirroring `new X(...)`. + */ +function synthesizeCsharpConstructorInitializerReferences(root: SyntaxNode): CaptureMatch[] { + const out: CaptureMatch[] = []; + walkNamedTree(root, (node) => { + if (node.type !== 'constructor_initializer') return; + + let kind: 'base' | 'this' | null = null; + for (let i = 0; i < node.childCount; i++) { + const t = node.child(i)?.type; + if (t === 'base' || t === 'this') { + kind = t; + break; + } + } + if (kind === null) return; + + const enclosingType = findEnclosingTypeDeclaration(node); + if (enclosingType === null) return; + + let targetNameNode: SyntaxNode | null = null; + if (kind === 'this') { + targetNameNode = enclosingType.childForFieldName('name'); + } else { + const baseList = findNamedChild(enclosingType, 'base_list'); + if (baseList === null) return; + // Prefer the first non-builtin entry (idiomatically the base class). When + // the list is interface-only (`class C : IFoo`), do not synthesize a + // `base(...)` ref — valid C# chains to implicit Object, not IFoo (#2046). + let sawNonBuiltin = false; + for (const base of baseList.namedChildren) { + if (base === null) continue; + const n = terminalTypeNameNode(base); + if (n === null || BUILTIN_TYPE_NAMES.has(n.text)) continue; + sawNonBuiltin = true; + targetNameNode = n; + break; + } + if (!sawNonBuiltin) return; + } + if (targetNameNode === null) return; + + const argList = findNamedChild(node, 'argument_list'); + const arity = + argList === null + ? 0 + : argList.namedChildren.filter((c) => c !== null && c.type === 'argument').length; + + out.push({ + '@reference.call.constructor': nodeToCapture('@reference.call.constructor', node), + '@reference.name': nodeToCapture('@reference.name', targetNameNode), + '@reference.arity': syntheticCapture('@reference.arity', node, String(arity)), + }); + }); + return out; +} + +function findEnclosingTypeDeclaration(node: SyntaxNode): SyntaxNode | null { + let cur: SyntaxNode | null = node.parent; + while (cur !== null) { + if ( + cur.type === 'class_declaration' || + cur.type === 'struct_declaration' || + cur.type === 'record_declaration' + ) { + return cur; + } + cur = cur.parent; + } + return null; +} + /** * Synthesize `@reference.inherits` captures from C# base lists so the * registry-primary scope-resolution path emits EXTENDS / IMPLEMENTS edges diff --git a/gitnexus/src/core/ingestion/languages/csharp/index.ts b/gitnexus/src/core/ingestion/languages/csharp/index.ts index 9f06ce87d..5e8fa99ea 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/index.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/index.ts @@ -68,10 +68,9 @@ * `using static X = Y.Z;`, attributes, and preprocessor-gated * declarations are all recognized correctly. * - * Shadow-harness corpus parity is the authoritative signal for which - * of these matter in practice. The CI parity gate blocks any PR that - * regresses either the legacy or registry-primary run of - * `test/integration/resolvers/csharp.test.ts`. + * The `test/integration/resolvers/csharp.test.ts` resolver suite is the + * authoritative signal for which of these matter in practice; it runs in + * the standard CI test workflow, so a regression blocks the merge. */ export { emitCsharpScopeCaptures } from './captures.js'; diff --git a/gitnexus/src/core/ingestion/languages/csharp/interpret.ts b/gitnexus/src/core/ingestion/languages/csharp/interpret.ts index beb2594a9..ffb38a257 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/interpret.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/interpret.ts @@ -129,9 +129,20 @@ function stripGeneric(text: string): string { * receiver's generic type based on the suffix — `data.Values` → * element type of `data`'s Dictionary. */ function stripQualifier(text: string): string { - const lastDot = text.lastIndexOf('.'); - if (lastDot === -1) return text; - const tail = text.slice(lastDot + 1); + // Strip only the outermost qualifier: the last `.` at generic nesting depth 0. + // This preserves F41 (never cut inside `Dictionary`) AND + // nested types through a generic outer (`Ns.Outer.Inner` → `Inner`, not + // `Outer.Inner` — #2046 P3). + let depth = 0; + let lastDotAtDepth0 = -1; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (c === '<') depth++; + else if (c === '>') depth = Math.max(0, depth - 1); + else if (c === '.' && depth === 0) lastDotAtDepth0 = i; + } + if (lastDotAtDepth0 === -1) return text; + const tail = text.slice(lastDotAtDepth0 + 1); if (COLLECTION_ACCESSOR_SUFFIXES.has(tail)) return text; return tail; } diff --git a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts index 634afdeec..5dab83afe 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/namespace-siblings.ts @@ -356,8 +356,9 @@ function extractFileStructure(content: string, cachedTree: unknown): CsharpFileS /** Content + (optional) pre-parsed tree-sitter trees keyed by filePath. * The orchestrator builds `fileContents` from the pipeline's file list; - * `treeCache` is the same `scopeTreeCache` already populated by the - * parse phase, so cache hits avoid a second `parser.parse()`. */ + * `treeCache` is currently always empty (its only producer, the sequential + * parser, was removed), so the providers re-parse. Kept as an extension + * point that would let cache hits avoid a second `parser.parse()`. */ export interface CsharpSiblingInputs { readonly fileContents: ReadonlyMap; readonly treeCache?: { get(filePath: string): unknown }; diff --git a/gitnexus/src/core/ingestion/languages/csharp/qualified-type-names.ts b/gitnexus/src/core/ingestion/languages/csharp/qualified-type-names.ts new file mode 100644 index 000000000..a4b72e6b9 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/csharp/qualified-type-names.ts @@ -0,0 +1,66 @@ +/** + * Tag C# file-level type defs with their enclosing-namespace path on the + * sidecar `namespacePrefix` field — WITHOUT touching `qualifiedName` (mutating + * it corrupts simple-name heritage / base resolution; #2046 regression). + * + * `tagNamespacePrefixes` (shared) only reaches defs whose scope chain includes + * a Namespace scope. C# file-scoped `namespace X;` gives the Namespace scope a + * 1-line range, so top-level types land under the Module scope and are missed. + * This pass covers both block- and file-scoped namespaces so the qualified + * constructor resolver can break a same-tail collision (`new B.Foo()` with both + * `A.Foo` and `B.Foo`) by matching the explicit qualifier against the sidecar. + */ +import type { ParsedFile, ScopeId, SymbolDefinition } from 'gitnexus-shared'; +import { isClassLike } from '../../scope-resolution/scope/walkers.js'; + +function isTypeDef(def: SymbolDefinition): boolean { + return isClassLike(def.type) || def.type === 'Enum'; +} + +export function populateCsharpNamespacePrefixes(parsed: ParsedFile): void { + const scopesById = new Map(); + for (const scope of parsed.scopes) scopesById.set(scope.id, scope); + + // The file's declared namespace (file-scoped `namespace X;`). First Namespace + // scope's own def qualifiedName; undefined when the file is namespace-free. + const fileNamespace = ((): string | undefined => { + for (const scope of parsed.scopes) { + if (scope.kind !== 'Namespace') continue; + const nsDef = scope.ownedDefs.find((d) => d.type === 'Namespace'); + const q = nsDef?.qualifiedName; + if (q !== undefined && q.length > 0) return q; + } + return undefined; + })(); + + // Enclosing namespace path for a scope: nearest ancestor Namespace scope's + // full qualifiedName, else the file-scoped namespace (Module-parented types). + const namespaceOf = (scope: ParsedFile['scopes'][number]): string | undefined => { + let parentId = scope.parent; + while (parentId !== null) { + const parent = scopesById.get(parentId); + if (parent === undefined) break; + if (parent.kind === 'Namespace') { + const nsDef = parent.ownedDefs.find((d) => d.type === 'Namespace'); + const q = nsDef?.qualifiedName; + if (q !== undefined && q.length > 0) return q; + } + if (parent.kind === 'Module') return fileNamespace; + parentId = parent.parent; + } + return fileNamespace; + }; + + for (const scope of parsed.scopes) { + if (scope.kind !== 'Class') continue; + const prefix = namespaceOf(scope); + if (prefix === undefined || prefix.length === 0) continue; + for (const def of scope.ownedDefs) { + if (!isTypeDef(def)) continue; + if (def.namespacePrefix !== undefined) continue; + const q = def.qualifiedName; + if (q === prefix || (q !== undefined && q.startsWith(`${prefix}.`))) continue; + def.namespacePrefix = prefix; + } + } +} diff --git a/gitnexus/src/core/ingestion/languages/csharp/query.ts b/gitnexus/src/core/ingestion/languages/csharp/query.ts index f299aaafa..0a0bb65ca 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/query.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/query.ts @@ -41,6 +41,12 @@ const CSHARP_SCOPE_QUERY = ` (namespace_declaration) @scope.namespace (file_scoped_namespace_declaration) @scope.namespace +(namespace_declaration + name: (_) @declaration.name) @declaration.namespace + +(file_scoped_namespace_declaration + name: (_) @declaration.name) @declaration.namespace + (class_declaration) @scope.class (interface_declaration) @scope.class (struct_declaration) @scope.class @@ -482,6 +488,13 @@ const CSHARP_SCOPE_QUERY = ` (object_creation_expression type: (qualified_name) @reference.call.constructor.qualified) @reference.call.constructor +;; Alias-qualified constructor: \`new MyAlias::Foo()\`, \`new global::Foo()\`. The +;; top-level type is an alias_qualified_name (a \`global::Ns.Foo\` qualifier nests +;; under qualified_name instead, covered above). No @reference.name here — +;; captures.ts derives the simple-name tail via terminalTypeNameNode. +(object_creation_expression + type: (alias_qualified_name) @reference.call.constructor.qualified) @reference.call.constructor + ;; References — field/property writes: \`obj.Name = "x"\` emits a write ;; ACCESSES edge from the enclosing method to the field/property on ;; obj's class. diff --git a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts index 0642b6f91..19e4ae8e2 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/scope-resolver.ts @@ -10,6 +10,7 @@ import type { ParsedFile } from 'gitnexus-shared'; import { SupportedLanguages } from 'gitnexus-shared'; import { buildMro, defaultLinearize } from '../../scope-resolution/passes/mro.js'; import { populateClassOwnedMembers } from '../../scope-resolution/scope/walkers.js'; +import { populateCsharpNamespacePrefixes } from './qualified-type-names.js'; import type { ScopeResolver } from '../../scope-resolution/contract/scope-resolver.js'; import { csharpProvider } from '../csharp.js'; import { @@ -57,7 +58,13 @@ const csharpScopeResolver: ScopeResolver = { buildMro: (graph, parsedFiles, nodeLookup) => buildMro(graph, parsedFiles, nodeLookup, defaultLinearize), - populateOwners: (parsed: ParsedFile) => populateClassOwnedMembers(parsed), + populateOwners: (parsed: ParsedFile) => { + populateClassOwnedMembers(parsed); + // Sidecar-only namespace tagging (does NOT touch qualifiedName) so the + // qualified constructor resolver can break same-tail collisions like + // `new B.Foo()` by matching the explicit qualifier (#2046). + populateCsharpNamespacePrefixes(parsed); + }, // C# uses `base` for super-class dispatch, not `super`. Match as a // plain identifier (no `()` call like Python's `super(...)`) — `base` diff --git a/gitnexus/src/core/ingestion/languages/dart/query.ts b/gitnexus/src/core/ingestion/languages/dart/query.ts index c00cfbeb5..5314b0c8b 100644 --- a/gitnexus/src/core/ingestion/languages/dart/query.ts +++ b/gitnexus/src/core/ingestion/languages/dart/query.ts @@ -23,7 +23,15 @@ */ import Parser from 'tree-sitter'; -import Dart from 'tree-sitter-dart'; +import { SupportedLanguages } from 'gitnexus-shared'; +// `tree-sitter-dart` is an optional/vendored grammar that may be absent on a +// default install. Loaded lazily + guarded via parser-loader rather than +// statically imported: this module is pulled onto the main thread eagerly by +// the scope-resolution registry and the language-provider index, so a top-level +// `import Dart from 'tree-sitter-dart'` would throw ERR_MODULE_NOT_FOUND at +// module-load and crash `analyze` even for repos with no Dart files (#2091, +// #2093). The grammar is only ever needed inside the lazy getters below. +import { getLanguageGrammar } from '../../../tree-sitter/parser-loader.js'; const DART_SCOPE_QUERY = ` ; ── Scopes ─────────────────────────────────────────────────────────────────── @@ -39,6 +47,46 @@ const DART_SCOPE_QUERY = ` (extension_declaration name: (identifier) @declaration.name) @declaration.class (enum_declaration name: (identifier) @declaration.name) @declaration.enum +; ── Declarations — type aliases (old-style + new-style function typedefs) ──── +; Both forms parse as type_alias; the name position differs, and a generic +; parameter list intervenes for the generic variants. Per #1919 review CF2, +; a generic type_parameters node sits between the name and the next anchor, so +; the non-generic adjacency patterns silently drop the generic forms. Four +; standalone patterns (NOT one alternation — the tree-sitter 0.21 hazard drops +; sibling branches) keep the name capture unambiguous and single-match per form: +; non-generic old-style typedef int Cmp(int a, int b); +; children: return-type, NAME, formal_parameter_list +; generic old-style typedef int Cmp(T a, T b); (CF2) +; children: return-type, NAME, type_parameters, formal_parameter_list +; non-generic new-style typedef Pred = bool Function(int); +; children: NAME, "=", function_type +; generic new-style typedef Mapper = T Function(T); +; children: NAME, type_parameters, "=", function_type +; The alias name is the type_identifier immediately before the param list (old) +; or before "=" (new); for the generic forms it is the one immediately before +; the intervening type_parameters. Mirrors Kotlin's @declaration.type_alias +; rule; the generic scope-extractor maps "type_alias" → TypeAlias. +(type_alias + (type_identifier) @declaration.name + . + (formal_parameter_list)) @declaration.type_alias +(type_alias + (type_identifier) @declaration.name + . + (type_parameters) + . + (formal_parameter_list)) @declaration.type_alias +(type_alias + (type_identifier) @declaration.name + . + "=") @declaration.type_alias +(type_alias + (type_identifier) @declaration.name + . + (type_parameters) + . + "=") @declaration.type_alias + ; ── Declarations — top-level functions (parent is program, not method) ─────── (program (function_signature @@ -94,14 +142,19 @@ let _query: Parser.Query | null = null; export function getDartParser(): Parser { if (_parser === null) { _parser = new Parser(); - _parser.setLanguage(Dart as Parameters[0]); + _parser.setLanguage( + getLanguageGrammar(SupportedLanguages.Dart) as Parameters[0], + ); } return _parser; } export function getDartScopeQuery(): Parser.Query { if (_query === null) { - _query = new Parser.Query(Dart as Parameters[0], DART_SCOPE_QUERY); + _query = new Parser.Query( + getLanguageGrammar(SupportedLanguages.Dart) as Parameters[0], + DART_SCOPE_QUERY, + ); } return _query; } diff --git a/gitnexus/src/core/ingestion/languages/go/query.ts b/gitnexus/src/core/ingestion/languages/go/query.ts index 48387582f..4246ee3b4 100644 --- a/gitnexus/src/core/ingestion/languages/go/query.ts +++ b/gitnexus/src/core/ingestion/languages/go/query.ts @@ -53,11 +53,15 @@ const GO_SCOPE_QUERY = ` ;; Declarations — variables (var_declaration (var_spec - name: (identifier) @declaration.name)) @declaration.variable + (identifier) @declaration.name)) @declaration.variable +(var_declaration + (var_spec_list + (var_spec + (identifier) @declaration.name))) @declaration.variable (const_declaration (const_spec - name: (identifier) @declaration.name)) @declaration.const + (identifier) @declaration.name)) @declaration.const (short_var_declaration left: (expression_list (identifier) @declaration.name)) @declaration.variable diff --git a/gitnexus/src/core/ingestion/languages/java.ts b/gitnexus/src/core/ingestion/languages/java.ts index b44464ddc..f7ca23de3 100644 --- a/gitnexus/src/core/ingestion/languages/java.ts +++ b/gitnexus/src/core/ingestion/languages/java.ts @@ -13,6 +13,7 @@ import { javaClassConfig } from '../class-extractors/configs/jvm.js'; import { defineLanguage } from '../language-provider.js'; import type { AstFrameworkPatternConfig } from '../language-provider.js'; import { javaTypeConfig } from '../type-extractors/jvm.js'; +import { extractSpringRoutes } from '../route-extractors/spring.js'; import { javaExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; import { javaImportConfig } from '../import-resolvers/configs/jvm.js'; @@ -126,4 +127,7 @@ export const javaProvider = defineLanguage({ arityCompatibility: javaArityCompatibility, resolveImportTarget: resolveJavaImportTarget, orderSameNameTypeCandidates: orderJavaSameNameTypeCandidates, + + // ── Route extraction ── + extractDecoratorRoutes: extractSpringRoutes, }); diff --git a/gitnexus/src/core/ingestion/languages/java/captures.ts b/gitnexus/src/core/ingestion/languages/java/captures.ts index 85c5a9106..cd4463459 100644 --- a/gitnexus/src/core/ingestion/languages/java/captures.ts +++ b/gitnexus/src/core/ingestion/languages/java/captures.ts @@ -216,7 +216,104 @@ export function emitJavaScopeCaptures( out.push(grouped); } - return [...resolveVarTypeBindings(out), ...synthesizeJavaInheritanceReferences(tree.rootNode)]; + return [ + ...resolveVarTypeBindings(out), + ...synthesizeJavaInheritanceReferences(tree.rootNode), + ...synthesizeJavaExplicitConstructorReferences(tree.rootNode), + ]; +} + +/** + * Synthesize `@reference.call.constructor` captures for explicit constructor + * invocations — `super(...)` and `this(...)` (F38 #1928). tree-sitter-java + * models these as `explicit_constructor_invocation` nodes, which the scope + * query does not match, so the chained-constructor CALLS edges (subclass ctor → + * superclass ctor; ctor → sibling overload) were silently dropped. + * + * The grammar gives no constructor *name* at the call site (the child is a bare + * `(super)` / `(this)` token), so the target name is resolved structurally: + * - `this(...)` → the enclosing type's own simple name (constructor symbols + * are keyed by the declaring class name). + * - `super(...)` → the enclosing class's superclass simple-name tail (reusing + * `javaBaseLookupNameNode` so qualified/generic supers reduce + * to the bare class name, matching the EXTENDS synth). An + * implicit `Object` super (no `superclass` field) has no + * in-graph symbol, so it is skipped rather than emitting a + * dangling reference. + * Arity is attached so overloaded constructors disambiguate downstream, mirroring + * the call-site arity synthesized for `new X(...)`. + */ +function synthesizeJavaExplicitConstructorReferences(root: SyntaxNode): CaptureMatch[] { + const out: CaptureMatch[] = []; + const stack: SyntaxNode[] = [root]; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.type === 'explicit_constructor_invocation') { + emitJavaExplicitConstructorRef(out, node); + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child !== null) stack.push(child); + } + } + return out; +} + +const TYPE_DECL_NODE_TYPES = new Set([ + 'class_declaration', + 'enum_declaration', + 'record_declaration', +]); + +function emitJavaExplicitConstructorRef(out: CaptureMatch[], node: SyntaxNode): void { + const ctor = node.childForFieldName('constructor'); + if (ctor === null) return; + + const enclosingType = findEnclosingTypeDeclaration(node); + if (enclosingType === null) return; + + let targetNameNode: SyntaxNode | null = null; + if (ctor.type === 'this') { + targetNameNode = enclosingType.childForFieldName('name'); + } else if (ctor.type === 'super') { + // Only class_declaration carries a `superclass` field; enum/record cannot + // declare an explicit superclass, so `super(...)` there has no resolvable + // target symbol. + const superclass = enclosingType.childForFieldName('superclass'); + if (superclass === null) return; + for (const base of superclass.namedChildren) { + if (base === null) continue; + const nameNode = javaBaseLookupNameNode(base); + if (nameNode !== null) { + targetNameNode = nameNode; + break; + } + } + } + if (targetNameNode === null) return; + + const argList = node.childForFieldName('arguments'); + const args = + argList === null + ? [] + : argList.namedChildren.filter( + (c) => c !== null && c.type !== 'block_comment' && c.type !== 'line_comment', + ); + + out.push({ + '@reference.call.constructor': nodeToCapture('@reference.call.constructor', node), + '@reference.name': nodeToCapture('@reference.name', targetNameNode), + '@reference.arity': syntheticCapture('@reference.arity', node, String(args.length)), + }); +} + +function findEnclosingTypeDeclaration(node: SyntaxNode): SyntaxNode | null { + let cur: SyntaxNode | null = node.parent; + while (cur !== null) { + if (TYPE_DECL_NODE_TYPES.has(cur.type)) return cur; + cur = cur.parent; + } + return null; } /** diff --git a/gitnexus/src/core/ingestion/languages/java/interpret.ts b/gitnexus/src/core/ingestion/languages/java/interpret.ts index 87da580b6..09c911aa9 100644 --- a/gitnexus/src/core/ingestion/languages/java/interpret.ts +++ b/gitnexus/src/core/ingestion/languages/java/interpret.ts @@ -74,10 +74,14 @@ export function interpretJavaTypeBinding(captures: CaptureMatch): ParsedTypeBind const typeCap = captures['@type-binding.type']; if (nameCap === undefined || typeCap === undefined) return null; - // Strip qualifier first so that `com.example.BaseModel` becomes - // `BaseModel` before stripGeneric — the JVM-erasure fallback pattern - // requires an unqualified identifier at the start of the string. - const rawType = stripGeneric(stripQualifier(typeCap.text.trim())); + // Strip generics BEFORE the qualifier (F41 #1928). Stripping the qualifier + // first uses `lastIndexOf('.')`, which for a qualified *type argument* + // (`Map`) cuts inside the generic and yields a + // corrupted `User>`. Unwrapping generics first reduces the string to a single + // (possibly qualified) class name, then the qualifier strip leaves the bare + // simple name. `stripGeneric`'s erasure fallback is qualifier-tolerant so a + // qualified generic base (`com.example.BaseModel`) still reduces correctly. + const rawType = stripQualifier(stripGeneric(typeCap.text.trim())); // Skip `var` — tree-sitter-java parses `var` as type_identifier with // text "var". When used without a constructor initializer, there's no @@ -127,8 +131,10 @@ function stripGeneric(text: string): string { // `BaseModel` → `BaseModel`, `Builder` → `Builder`. // This mirrors JVM type erasure — the raw class name is the resolvable symbol. // The pattern matches up to the first `<` to handle nested generics safely - // (e.g. `BaseModel>` → `BaseModel`). - const fallback = text.match(/^([A-Za-z_$][A-Za-z0-9_$]*)<.+>$/s); + // (e.g. `BaseModel>` → `BaseModel`). The base is allowed to be + // qualified (`com.example.BaseModel` → `com.example.BaseModel`) since the + // caller strips the qualifier afterwards (F41 #1928). + const fallback = text.match(/^((?:[A-Za-z_$][A-Za-z0-9_$]*\.)*[A-Za-z_$][A-Za-z0-9_$]*)<.+>$/s); if (fallback !== null) return fallback[1].trim(); return text; diff --git a/gitnexus/src/core/ingestion/languages/java/query.ts b/gitnexus/src/core/ingestion/languages/java/query.ts index e1e581ad5..4821e9703 100644 --- a/gitnexus/src/core/ingestion/languages/java/query.ts +++ b/gitnexus/src/core/ingestion/languages/java/query.ts @@ -214,8 +214,23 @@ const JAVA_SCOPE_QUERY = ` type: (generic_type (type_identifier) @reference.name)) @reference.call.constructor +;; References — qualified constructor calls: new pkg.Foo(), new a.b.Foo() (F35 #1928) +;; tree-sitter-java parses \`pkg.Foo\` as a scoped_type_identifier whose final +;; child is the simple type. Bind that tail as @reference.name (trailing \`.\` +;; anchor = last child) so resolution targets \`Foo\`, not the raw \`pkg.Foo\` text. +;; Mirrors the TS/JS new-expression qualified-constructor capture. (object_creation_expression - type: (scoped_type_identifier) @reference.call.constructor.qualified) @reference.call.constructor + type: (scoped_type_identifier + (type_identifier) @reference.name .) @reference.call.constructor.qualified) @reference.call.constructor + +;; References — qualified + generic constructor calls: new pkg.Box() (F35 #1928) +;; The base is a generic_type whose first child is a scoped_type_identifier, so +;; neither the simple-generic nor the plain-scoped arm above matches it. Bind the +;; scoped tail as @reference.name. +(object_creation_expression + type: (generic_type + (scoped_type_identifier + (type_identifier) @reference.name .) @reference.call.constructor.qualified)) @reference.call.constructor ;; References — method references: User::getName, obj::method (method_reference diff --git a/gitnexus/src/core/ingestion/languages/kotlin.ts b/gitnexus/src/core/ingestion/languages/kotlin.ts index 31116663e..4cf87a76f 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin.ts @@ -11,6 +11,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { kotlinClassConfig } from '../class-extractors/configs/jvm.js'; import { defineLanguage } from '../language-provider.js'; +import { assertCloneable } from '../workers/clone-safety.js'; import { kotlinTypeConfig } from '../type-extractors/jvm.js'; import { kotlinExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; @@ -28,6 +29,7 @@ import { kotlinMethodConfig } from '../method-extractors/configs/jvm.js'; import { createVariableExtractor } from '../variable-extractors/generic.js'; import { kotlinVariableConfig } from '../variable-extractors/configs/jvm.js'; import { + collectKotlinCaptureSideChannel, emitKotlinScopeCaptures, interpretKotlinImport, interpretKotlinTypeBinding, @@ -175,6 +177,17 @@ export const kotlinProvider = defineLanguage({ // ── RFC #909 Ring 3: scope-based resolution hooks ── emitScopeCaptures: emitKotlinScopeCaptures, + // Worker-side: snapshot the module-level companion-scope marks + // `emitKotlinScopeCaptures` just populated for this file (`markCompanionScope` + // → `companionScopesByFile`) into plain data on `ParsedFile.captureSideChannel`, + // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a + // re-parse (#1983). Without this, companion/static dispatch emits no CALLS + // edges on the worker path. See `kotlin/capture-side-channel.ts`. + // `assertCloneable` is a runtime identity; it makes a future non-serializable + // value in the side-channel payload a compile error here, at the source, rather + // than a DataCloneError at the worker boundary (#2143). + collectCaptureSideChannel: (filePath) => + assertCloneable(collectKotlinCaptureSideChannel(filePath)), interpretImport: interpretKotlinImport, interpretTypeBinding: interpretKotlinTypeBinding, bindingScopeFor: kotlinBindingScopeFor, diff --git a/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts new file mode 100644 index 000000000..375e0f679 --- /dev/null +++ b/gitnexus/src/core/ingestion/languages/kotlin/capture-side-channel.ts @@ -0,0 +1,75 @@ +/** + * Kotlin capture-time side-channel serialization (#1983). + * + * `emitKotlinScopeCaptures` populates one MODULE-LEVEL, per-file map as a side + * effect that is NOT part of the returned `ParsedFile`'s scopes/defs: + * + * - `companionScopesByFile` (companion-scopes.ts) — the `ScopeId`s that came + * from a `companion_object` AST node, recorded via `markCompanionScope` + * from the `@scope.companion` marker capture. + * + * On the worker path that map is filled in the WORKER process and lost across + * the worker→main MessageChannel (and the disk-backed parsedfile-store), + * because scope-resolution reuses the serialized `ParsedFile` and SKIPS the + * main-thread re-extraction (the #1983 fix that avoids a main-thread + * tree-sitter re-parse / OOM on huge repos). The main thread then reads the map + * empty in `isKotlinStaticOnly` / `populateCompanionMembersOnEnclosingClass` + * (owners.ts) — so companion methods aren't identified as static and + * companion/static dispatch emits no CALLS edges. + * + * This module snapshots the per-file slice of that map into a plain, + * JSON-serializable object (carried on `ParsedFile.captureSideChannel`) and + * restores it on the main thread WITHOUT any parse. It mirrors the C++ pattern + * in `cpp/capture-side-channel.ts`. + * + * The single generic `ParsedFile.captureSideChannel` field is shared with C++, + * which is safe because each file is one language (a `.kt` file uses the kotlin + * provider, a `.cpp` file the cpp provider). The payload is self-describing + * (`{ kind: 'kotlin', companionScopes }`) so `applyKotlinCaptureSideChannel` + * only restores kotlin state and ignores a foreign-shaped snapshot. + */ + +import type { ParsedFile, ScopeId } from 'gitnexus-shared'; +import { getCompanionScopesForFile, markCompanionScope } from './companion-scopes.js'; + +/** + * Plain JSON-serializable snapshot of the per-file Kotlin capture-time + * side-channel. Carried opaquely on `ParsedFile.captureSideChannel`. The + * `kind` tag makes the payload self-describing so `apply` can distinguish a + * kotlin snapshot from another language's (C++ shares the same field). + */ +export interface KotlinCaptureSideChannel { + readonly kind: 'kotlin'; + /** Companion-object scope ids recorded for this file. */ + readonly companionScopes: readonly ScopeId[]; +} + +/** + * `LanguageProvider.collectCaptureSideChannel` implementation for Kotlin. + * Returns `undefined` when this file recorded no companion scopes at all, so + * the produced `ParsedFile` carries the field only when there's data to ship. + */ +export function collectKotlinCaptureSideChannel( + filePath: string, +): KotlinCaptureSideChannel | undefined { + const companionScopes = getCompanionScopesForFile(filePath); + if (companionScopes.length === 0) return undefined; + return { kind: 'kotlin', companionScopes }; +} + +/** + * `ScopeResolver.applyCaptureSideChannel` implementation for Kotlin. Reads the + * worker-serialized snapshot from `parsed.captureSideChannel` and re-populates + * the module-level companion-scope map via `markCompanionScope`. Tolerant of + * `undefined` (file carried no data) and of an unexpected / foreign shape + * (defensive — the `kind` tag guards against restoring a non-kotlin payload). + * Does NO tree-sitter parse. + */ +export function applyKotlinCaptureSideChannel(parsed: ParsedFile): void { + const data = parsed.captureSideChannel as KotlinCaptureSideChannel | undefined; + if (data === undefined || data === null || typeof data !== 'object') return; + if (data.kind !== 'kotlin' || !Array.isArray(data.companionScopes)) return; + for (const scopeId of data.companionScopes) { + markCompanionScope(parsed.filePath, scopeId); + } +} diff --git a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts index 2b14266a6..16d2a3db8 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/captures.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/captures.ts @@ -39,6 +39,7 @@ export function emitKotlinScopeCaptures( out.push(...synthesizeKotlinSmartCastBindings(tree.rootNode)); out.push(...synthesizeKotlinLambdaBindings(tree.rootNode, returnTypes)); out.push(...synthesizeKotlinInheritanceReferences(tree.rootNode)); + out.push(...synthesizeKotlinSecondaryConstructorDeclarations(tree.rootNode)); for (const match of getKotlinScopeQuery().matches(tree.rootNode)) { const grouped: Record = {}; @@ -87,6 +88,40 @@ export function emitKotlinScopeCaptures( } } + // Callable references (`::method`, `Type::new`, `obj::m`) — F47 (#1919). + // The query captures the referenced member as `@reference.name`, an + // optional receiver type as `@reference.receiver`, and the whole node as + // `@reference.callable`. Rewrite into a call reference so it participates + // in call-graph resolution: a bare `::member` resolves as a free call; + // a `Receiver::member` resolves as a member call against the receiver + // type. The function/constructor is referenced (not invoked), so no + // arity/argument metadata is attached. + if (grouped['@reference.callable'] !== undefined) { + const nameCap = grouped['@reference.name']; + const callableNode = groupedNodes['@reference.callable']; + if (nameCap !== undefined && callableNode !== undefined) { + const receiverCap = grouped['@reference.receiver']; + // The anchor Capture must carry the call-form tag as its `name` — + // the scope-extractor reads `Capture.name` (not the map key) to + // classify the reference kind, so re-wrap via nodeToCapture rather + // than reusing the `@reference.callable`-named Capture (whose head + // `callable` resolves to no ReferenceKind and silently drops it). + if (receiverCap !== undefined) { + out.push({ + '@reference.call.member': nodeToCapture('@reference.call.member', callableNode), + '@reference.name': nameCap, + '@reference.receiver': receiverCap, + }); + } else { + out.push({ + '@reference.call.free': nodeToCapture('@reference.call.free', callableNode), + '@reference.name': nameCap, + }); + } + } + continue; + } + if ( grouped['@reference.call.free'] !== undefined && grouped['@reference.receiver'] !== undefined @@ -253,6 +288,100 @@ function synthesizeKotlinInheritanceReferences(rootNode: SyntaxNode): CaptureMat return out; } +/** + * The enclosing type name for a node nested in a class/object/companion body. + * Walks up to the first `class_declaration` / `object_declaration` / + * `companion_object` ancestor and returns its `type_identifier` name node. + * Used to qualify a secondary-constructor declaration as `.constructor`. + */ +function kotlinEnclosingTypeNameNode(node: SyntaxNode): SyntaxNode | null { + for (let cur: SyntaxNode | null = node.parent; cur !== null; cur = cur.parent) { + if ( + cur.type === 'class_declaration' || + cur.type === 'object_declaration' || + cur.type === 'companion_object' + ) { + const nameNode = cur.namedChildren.find((c) => c.type === 'type_identifier'); + return nameNode ?? null; + } + } + return null; +} + +/** + * Synthesize a `@declaration.constructor` capture for each Kotlin + * `secondary_constructor` (issue #1919 review CF1). The structure phase already + * materializes a `Constructor` graph node (`Constructor:file:Class.constructor#`), + * but the registry-primary scope-resolution path had no Constructor *def* in the + * scope tree — so a call inside the constructor body resolved its caller anchor + * up to the enclosing Class def, mis-attributing the CALLS edge to the class. + * + * Paired with `(secondary_constructor) @scope.function` in query.ts: that rule + * makes the constructor body its own Function scope; this declaration places a + * Constructor def in that scope so `pickCallerCallableDef` anchors calls on the + * Constructor. The def is keyed to match the structure-phase node id: + * - `@declaration.qualified_name` = `.constructor` so the bridge's + * qualified key (`:file::Constructor::Class.constructor`) hits the node. + * - `@declaration.parameter-types` so two same-name secondary constructors are + * disambiguated by the bridge's parameter-types key (`~Int,Int`), matching + * the `#`-suffixed structure node for the overload with the same + * parameter shape. (The zero-arg overload carries no parameter types and + * resolves via the qualified/simple key to the `#0` node.) + * + * The anchor spans the whole `secondary_constructor` node — same range as the + * `@scope.function` it pairs with — so the def is owned by that Function scope + * and the constructor name auto-hoists to the enclosing class scope (exactly the + * binding shape a normal method declaration produces). + */ +function synthesizeKotlinSecondaryConstructorDeclarations(rootNode: SyntaxNode): CaptureMatch[] { + const out: CaptureMatch[] = []; + for (const ctorNode of descendantsOfType(rootNode, 'secondary_constructor')) { + const keyword = ctorNode.namedChildren.find((c) => c.type === 'constructor'); + // The `constructor` keyword is an anonymous token; fall back to the node + // itself for the name capture position when the named-child lookup misses. + const nameAnchor = keyword ?? ctorNode; + const classNameNode = kotlinEnclosingTypeNameNode(ctorNode); + const qualifiedName = + classNameNode !== null ? `${classNameNode.text}.constructor` : 'constructor'; + + const match: Record = { + '@declaration.constructor': nodeToCapture('@declaration.constructor', ctorNode), + '@declaration.name': syntheticCapture('@declaration.name', nameAnchor, 'constructor'), + '@declaration.qualified_name': syntheticCapture( + '@declaration.qualified_name', + ctorNode, + qualifiedName, + ), + }; + + const arity = computeKotlinArityMetadata(ctorNode); + if (arity.parameterCount !== undefined) { + match['@declaration.parameter-count'] = syntheticCapture( + '@declaration.parameter-count', + ctorNode, + String(arity.parameterCount), + ); + } + if (arity.requiredParameterCount !== undefined) { + match['@declaration.required-parameter-count'] = syntheticCapture( + '@declaration.required-parameter-count', + ctorNode, + String(arity.requiredParameterCount), + ); + } + if (arity.parameterTypes !== undefined) { + match['@declaration.parameter-types'] = syntheticCapture( + '@declaration.parameter-types', + ctorNode, + JSON.stringify(arity.parameterTypes), + ); + } + + out.push(match); + } + return out; +} + /** * The bare simple-name `type_identifier` of a `user_type`. Strips generic * type arguments (`Base` → `Base`) and qualifier tails (`pkg.Base` → `Base`) diff --git a/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts b/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts index ac4034620..f1c45c65c 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/companion-scopes.ts @@ -55,6 +55,17 @@ export function isCompanionScope(filePath: string, scopeId: ScopeId): boolean { return companionScopesByFile.get(filePath)?.has(scopeId) ?? false; } +/** + * Snapshot the companion-object scope ids recorded for `filePath` as a plain + * array (for the worker→main capture side-channel, #1983). Returns an empty + * array when the file recorded no companion scopes. See + * `capture-side-channel.ts`. + */ +export function getCompanionScopesForFile(filePath: string): ScopeId[] { + const scopes = companionScopesByFile.get(filePath); + return scopes === undefined ? [] : [...scopes]; +} + /** Clear all tracked companion scopes (for testing). */ export function clearCompanionScopes(): void { companionScopesByFile.clear(); diff --git a/gitnexus/src/core/ingestion/languages/kotlin/index.ts b/gitnexus/src/core/ingestion/languages/kotlin/index.ts index 206128e52..206254540 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/index.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/index.ts @@ -1,4 +1,9 @@ export { emitKotlinScopeCaptures } from './captures.js'; +export { + collectKotlinCaptureSideChannel, + applyKotlinCaptureSideChannel, + type KotlinCaptureSideChannel, +} from './capture-side-channel.js'; export { getKotlinCaptureCacheStats, resetKotlinCaptureCacheStats } from './cache-stats.js'; export { interpretKotlinImport, interpretKotlinTypeBinding } from './interpret.js'; export { kotlinArityCompatibility } from './arity.js'; diff --git a/gitnexus/src/core/ingestion/languages/kotlin/query.ts b/gitnexus/src/core/ingestion/languages/kotlin/query.ts index 9209655d8..7015e6e28 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/query.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/query.ts @@ -1,5 +1,14 @@ import Parser from 'tree-sitter'; -import Kotlin from 'tree-sitter-kotlin'; +import { SupportedLanguages } from 'gitnexus-shared'; +// `tree-sitter-kotlin` is a vendored grammar (loaded from vendor/ by absolute +// path, never node_modules — vendored-grammars.ts / #2111) that may be absent on +// a platform without a matching prebuild. Loaded lazily + guarded via parser-loader +// rather than statically imported: this module is pulled onto the main thread +// eagerly by the scope-resolution registry and the language-provider index, so +// a top-level `import Kotlin from 'tree-sitter-kotlin'` would throw +// ERR_MODULE_NOT_FOUND at module-load and crash `analyze` even for repos with no +// Kotlin files (#2091, #2093). The grammar is only ever needed in the getters. +import { getLanguageGrammar } from '../../../tree-sitter/parser-loader.js'; const KOTLIN_SCOPE_QUERY = ` ;; Scopes @@ -9,6 +18,16 @@ const KOTLIN_SCOPE_QUERY = ` (companion_object) @scope.class (function_declaration) @scope.function +;; Secondary-constructor body scope (issue #1919 review CF1). A +;; secondary constructor's "constructor(...) { ... }" body executes statements +;; just like a method body, so it must be its OWN Function scope — otherwise a +;; call inside the body resolves its caller anchor up to the enclosing Class +;; scope (the class's Class def), mis-attributing the CALLS edge to the class +;; rather than the Constructor. The matching @declaration.constructor is +;; synthesized in captures.ts (synthesizeKotlinSecondaryConstructorDeclarations) +;; so this scope owns a Constructor def keyed to the structure-phase node id. +(secondary_constructor) @scope.function + ;; Companion-object marker (issue #1756 / U4). Side-channel capture that ;; lets populateCompanionMembersOnEnclosingClass distinguish a companion ;; Class scope from a regular Class scope without inspecting ownedDefs. @@ -117,6 +136,26 @@ const KOTLIN_SCOPE_QUERY = ` (function_value_parameters) [(user_type) (nullable_type) (function_type)] @type-binding.type) @type-binding.return +;; References — callable references ("::method", "Type::new", "obj::m") — F47. +;; A "callable_reference" references a function/constructor as a value (no +;; call_suffix), so the registry-primary call path never saw it. Real-parse +;; (issue #1919) shows the canonical shape inside a function body is: +;; "::topLevelFn" -> (callable_reference :: (simple_identifier)) member only +;; "String::length" -> (callable_reference (type_identifier) :: (simple_identifier)) +;; "obj::method" -> (callable_reference (type_identifier) :: (simple_identifier)) +;; "Type::new" -> (callable_reference (type_identifier) :: (simple_identifier)) +;; The receiver (real type OR object) is always a "type_identifier"; the +;; referenced member is the LAST "simple_identifier". One rule with an +;; optional receiver and an end-anchored member covers all four forms with +;; exactly one match per callable_reference (no sibling-branch double-match). +;; (NOTE: a qualified "A.B::m" parses as a nested navigation_expression, not a +;; callable_reference, and is already captured by the read.member rule below.) +;; emitKotlinScopeCaptures rewrites this into a free/member call reference. +(callable_reference + (type_identifier)? @reference.receiver + (simple_identifier) @reference.name + .) @reference.callable + ;; References — direct calls / constructor syntax (call_expression (simple_identifier) @reference.name) @reference.call.free @@ -149,14 +188,19 @@ let query: Parser.Query | null = null; export function getKotlinParser(): Parser { if (parser === null) { parser = new Parser(); - parser.setLanguage(Kotlin as Parameters[0]); + parser.setLanguage( + getLanguageGrammar(SupportedLanguages.Kotlin) as Parameters[0], + ); } return parser; } export function getKotlinScopeQuery(): Parser.Query { if (query === null) { - query = new Parser.Query(Kotlin as Parameters[0], KOTLIN_SCOPE_QUERY); + query = new Parser.Query( + getLanguageGrammar(SupportedLanguages.Kotlin) as Parameters[0], + KOTLIN_SCOPE_QUERY, + ); } return query; } diff --git a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts index bd63c1e86..6bd48f0d1 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/scope-resolver.ts @@ -14,6 +14,7 @@ import { type KotlinResolveContext, } from './index.js'; import { clearCompanionScopes } from './companion-scopes.js'; +import { applyKotlinCaptureSideChannel } from './capture-side-channel.js'; import { isKotlinStaticOnly } from './owners.js'; /** @@ -84,6 +85,23 @@ export const kotlinScopeResolver: ScopeResolver = { buildMro: (graph, parsedFiles, nodeLookup) => buildKotlinMro(graph, parsedFiles, nodeLookup), + // Worker-boundary restore (see `ScopeResolver.applyCaptureSideChannel`). + // `emitKotlinScopeCaptures` records per-file companion-object scope ids + // (`markCompanionScope` → `companionScopesByFile`) as a SIDE EFFECT — that + // state is NOT serialized onto the returned ParsedFile's scopes/defs. On the + // worker path those marks are populated in the worker process and lost across + // the MessageChannel / disk store; the main thread reuses the serialized + // ParsedFile and skips `extractParsedFile`, so `isKotlinStaticOnly` and + // `populateCompanionMembersOnEnclosingClass` (owners.ts) would see an empty + // map and companion/static dispatch would emit zero CALLS edges. The worker + // stashed a plain-data snapshot on `parsed.captureSideChannel` via + // `kotlinProvider.collectCaptureSideChannel`; this restores it into the + // module map WITHOUT any tree-sitter re-parse (the #1983 fix). The + // freshly-extracted leg never calls this — its marks were just populated in + // this process. Runs BEFORE `populateOwners` so the restored companion map is + // visible to it. + applyCaptureSideChannel: applyKotlinCaptureSideChannel, + populateOwners: (parsed: ParsedFile) => populateKotlinOwners(parsed), isSuperReceiver: (text) => text.trim() === 'super', diff --git a/gitnexus/src/core/ingestion/languages/php/index.ts b/gitnexus/src/core/ingestion/languages/php/index.ts index 9b549bc3d..de7b6345b 100644 --- a/gitnexus/src/core/ingestion/languages/php/index.ts +++ b/gitnexus/src/core/ingestion/languages/php/index.ts @@ -54,10 +54,9 @@ * 6. **Intersection types in parameters** — `T&U $param` takes the first * named part (`T`). This matches the legacy type-extractor's behavior. * - * Shadow-harness corpus parity is the authoritative signal for which of - * these matter in practice. The CI parity gate blocks any PR that regresses - * either the legacy or registry-primary run of - * `test/integration/resolvers/php.test.ts`. + * The `test/integration/resolvers/php.test.ts` resolver suite is the + * authoritative signal for which of these matter in practice; it runs in + * the standard CI test workflow, so a regression blocks the merge. */ export { emitPhpScopeCaptures } from './captures.js'; diff --git a/gitnexus/src/core/ingestion/languages/python/captures.ts b/gitnexus/src/core/ingestion/languages/python/captures.ts index 761404ebb..756066382 100644 --- a/gitnexus/src/core/ingestion/languages/python/captures.ts +++ b/gitnexus/src/core/ingestion/languages/python/captures.ts @@ -38,9 +38,10 @@ export function emitPythonScopeCaptures( _filePath: string, cachedTree?: unknown, ): readonly CaptureMatch[] { - // Skip the parse when the caller (parse phase's ASTCache) already - // produced a Tree for this source. Cache miss = re-parse, same as - // before. The cachedTree parameter is typed as `unknown` at the + // Skip the parse when the caller (the scope-resolution orchestrator's + // `treeCache`) already produced a Tree for this source — empty under + // worker-pool runs, so cache miss = re-parse. The cachedTree parameter + // is typed as `unknown` at the // contract layer (see `LanguageProvider.emitScopeCaptures`); cast // here at the use site. let tree = cachedTree as ReturnType['parse']> | undefined; diff --git a/gitnexus/src/core/ingestion/languages/python/index.ts b/gitnexus/src/core/ingestion/languages/python/index.ts index 166c3a7ae..dc7e84f64 100644 --- a/gitnexus/src/core/ingestion/languages/python/index.ts +++ b/gitnexus/src/core/ingestion/languages/python/index.ts @@ -66,10 +66,9 @@ * site where the enclosing class can't be statically determined * is left unresolved. * - * Shadow-harness corpus parity is the authoritative signal for which - * of these matter in practice. The CI parity gate blocks any PR that - * regresses either the legacy or registry-primary run of - * `test/integration/resolvers/python.test.ts`. + * The `test/integration/resolvers/python.test.ts` resolver suite is the + * authoritative signal for which of these matter in practice; it runs in + * the standard CI test workflow, so a regression blocks the merge. */ export { emitPythonScopeCaptures } from './captures.js'; diff --git a/gitnexus/src/core/ingestion/languages/rust/query.ts b/gitnexus/src/core/ingestion/languages/rust/query.ts index d85660981..a0e75f0fa 100644 --- a/gitnexus/src/core/ingestion/languages/rust/query.ts +++ b/gitnexus/src/core/ingestion/languages/rust/query.ts @@ -134,11 +134,34 @@ const RUST_SCOPE_QUERY = ` name: (identifier) @reference.name)) @reference.call.free ;; References — constructor calls (struct literal) -;; Covers bare names (Foo {}), scoped (foo::bar::Baz {}), and turbofish -;; (Foo:: {}) — the name: field resolves to the trailing identifier -;; in all cases through tree-sitter-rust's grammar. +;; tree-sitter-rust gives struct_expression.name one of three node types +;; (type_identifier | scoped_type_identifier | generic_type_with_turbofish); +;; the turbofish form additionally nests either a type_identifier or a +;; scoped_identifier. We enumerate all four shapes below so the capture is +;; always the trailing identifier (resolved scope-aware), not the full path: +;; bare Foo {} +;; scoped foo::bar::Baz {} +;; turbofish Foo:: {} +;; scoped+turbofish foo::Bar:: {} (struct_expression - name: (_) @reference.name) @reference.call.constructor + name: (type_identifier) @reference.name) @reference.call.constructor + +;; Scoped struct (foo::bar::Baz {}) +(struct_expression + name: (scoped_type_identifier + name: (type_identifier) @reference.name)) @reference.call.constructor + +;; Turbofish struct (Foo:: {}) +(struct_expression + name: (generic_type_with_turbofish + type: (type_identifier) @reference.name)) @reference.call.constructor + +;; Scoped + turbofish struct (foo::Bar:: {}) — the turbofish wraps a +;; scoped_identifier whose tail is an identifier (not a type_identifier). +(struct_expression + name: (generic_type_with_turbofish + type: (scoped_identifier + name: (identifier) @reference.name))) @reference.call.constructor ;; References — macro invocations (disjoint namespace from functions) ;; Resolved via MacroRegistry → Macro defs only (never fn of the same name). diff --git a/gitnexus/src/core/ingestion/languages/swift/query.ts b/gitnexus/src/core/ingestion/languages/swift/query.ts index cf2699423..78c1efa41 100644 --- a/gitnexus/src/core/ingestion/languages/swift/query.ts +++ b/gitnexus/src/core/ingestion/languages/swift/query.ts @@ -43,7 +43,15 @@ */ import Parser from 'tree-sitter'; -import Swift from 'tree-sitter-swift'; +import { SupportedLanguages } from 'gitnexus-shared'; +// `tree-sitter-swift` is an optional/vendored grammar that may be absent on a +// default install. It is loaded lazily + guarded via parser-loader rather than +// statically imported: this module is pulled onto the main thread eagerly by +// the scope-resolution registry and the language-provider index, so a top-level +// `import Swift from 'tree-sitter-swift'` would throw ERR_MODULE_NOT_FOUND at +// module-load and crash `analyze` even for repos with no Swift files (#2091, +// #2093). The grammar is only ever needed inside the lazy getters below. +import { getLanguageGrammar } from '../../../tree-sitter/parser-loader.js'; const SWIFT_SCOPE_QUERY = ` ;; ── Scopes ────────────────────────────────────────────────────────── @@ -186,14 +194,19 @@ let _query: Parser.Query | null = null; export function getSwiftParser(): Parser { if (_parser === null) { _parser = new Parser(); - _parser.setLanguage(Swift as Parameters[0]); + _parser.setLanguage( + getLanguageGrammar(SupportedLanguages.Swift) as Parameters[0], + ); } return _parser; } export function getSwiftScopeQuery(): Parser.Query { if (_query === null) { - _query = new Parser.Query(Swift as Parameters[0], SWIFT_SCOPE_QUERY); + _query = new Parser.Query( + getLanguageGrammar(SupportedLanguages.Swift) as Parameters[0], + SWIFT_SCOPE_QUERY, + ); } return _query; } diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 73b33dfd8..7f41fe731 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -16,6 +16,7 @@ import { javascriptClassConfig, } from '../class-extractors/configs/typescript-javascript.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createTypeScriptCfgVisitor } from '../cfg/visitors/typescript.js'; import { typeConfig as typescriptConfig } from '../type-extractors/typescript.js'; import { tsExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; @@ -351,6 +352,8 @@ export const typescriptProvider = defineLanguage({ // canonical capture vocabulary in ./typescript/query.ts // (TYPESCRIPT_SCOPE_QUERY constant). emitScopeCaptures: emitTsScopeCaptures, + // CFG/PDG substrate (#2081 M1) — runs in the worker on a --pdg run. + cfgVisitor: createTypeScriptCfgVisitor(), interpretImport: interpretTsImport, interpretTypeBinding: interpretTsTypeBinding, bindingScopeFor: tsBindingScopeFor, @@ -412,6 +415,8 @@ export const javascriptProvider = defineLanguage({ // JSDoc type bindings) live in ./javascript/captures.ts. // See ./javascript/index.ts for the full per-module rationale. emitScopeCaptures: emitJsScopeCaptures, + // CFG/PDG substrate (#2081 M1) — TS and JS share the same grammar family. + cfgVisitor: createTypeScriptCfgVisitor(), interpretImport: interpretJsImport, interpretTypeBinding: interpretJsTypeBinding, bindingScopeFor: jsBindingScopeFor, diff --git a/gitnexus/src/core/ingestion/languages/typescript/captures.ts b/gitnexus/src/core/ingestion/languages/typescript/captures.ts index b3e338242..9d466c3f9 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/captures.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/captures.ts @@ -163,10 +163,11 @@ export function emitTsScopeCaptures( filePath: string, cachedTree?: unknown, ): readonly CaptureMatch[] { - // Skip the parse when the caller (parse phase's scopeTreeCache) already - // produced a Tree for this source. Cache miss = re-parse, same as before. - // The cachedTree parameter is typed as `unknown` at the LanguageProvider - // contract layer; cast here at the use site. + // Reuse a pre-parsed Tree when the caller passes one via `cachedTree`; a + // miss re-parses. (The cache is currently always empty — its only producer, + // the sequential parser, was removed — so this re-parses in practice.) The + // cachedTree parameter is typed `unknown` at the LanguageProvider contract + // layer; cast here at the use site. // // Grammar selection: `.tsx` files are parsed with the TSX grammar, // `.ts` files with the TypeScript grammar. The two grammars have diff --git a/gitnexus/src/core/ingestion/languages/typescript/index.ts b/gitnexus/src/core/ingestion/languages/typescript/index.ts index cb4bd4023..120566f48 100644 --- a/gitnexus/src/core/ingestion/languages/typescript/index.ts +++ b/gitnexus/src/core/ingestion/languages/typescript/index.ts @@ -80,10 +80,9 @@ * identifiers are narrowed (`user instanceof User`). Member paths * such as `user.address instanceof Address` remain unresolved. * - * Shadow-harness corpus parity on `test/integration/resolvers/ - * typescript.test.ts` is the authoritative signal for which of these - * matter in practice. The CI parity gate blocks any PR that regresses - * either the legacy or registry-primary run. + * The `test/integration/resolvers/typescript.test.ts` resolver suite is + * the authoritative signal for which of these matter in practice; it runs + * in the standard CI test workflow, so a regression blocks the merge. */ export { emitTsScopeCaptures } from './captures.js'; diff --git a/gitnexus/src/core/ingestion/languages/vue/captures.ts b/gitnexus/src/core/ingestion/languages/vue/captures.ts index b8031fcdd..b8ec7a8a4 100644 --- a/gitnexus/src/core/ingestion/languages/vue/captures.ts +++ b/gitnexus/src/core/ingestion/languages/vue/captures.ts @@ -22,6 +22,7 @@ import type { CaptureMatch } from 'gitnexus-shared'; import { extractVueScript } from '../../vue-sfc-extractor.js'; import { emitTsScopeCaptures } from '../typescript/captures.js'; +import { emitJsScopeCaptures } from '../javascript/captures.js'; /** * Emit scope captures for a Vue SFC. @@ -31,11 +32,11 @@ import { emitTsScopeCaptures } from '../typescript/captures.js'; * 1. **Full SFC content** (sequential path, <15 files): `sourceText` * contains the whole `.vue` file with `