diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 4dc2f3260..9d95ede9f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -11,7 +11,7 @@ "plugins": [ { "name": "gitnexus", - "version": "1.6.6", + "version": "1.6.7", "source": "./gitnexus-claude-plugin", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase." } diff --git a/.claude/skills/gitnexus/gitnexus-guide/SKILL.md b/.claude/skills/gitnexus/gitnexus-guide/SKILL.md index b81900b5e..cacc4e886 100644 --- a/.claude/skills/gitnexus/gitnexus-guide/SKILL.md +++ b/.claude/skills/gitnexus/gitnexus-guide/SKILL.md @@ -38,7 +38,38 @@ For any task involving code understanding, debugging, impact analysis, or refact | `detect_changes` | Git-diff impact — what do your current changes affect | | `rename` | Multi-file coordinated rename with confidence-tagged edits | | `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | +| `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | + +### Paginating `list_repos` + +`list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: + +```jsonc +{ + "repositories": [ + { "name": "...", "path": "...", "indexedAt": "...", "lastCommit": "...", "stats": { } } + ], + "pagination": { + "total": 437, + "limit": 50, + "offset": 0, + "returned": 50, + "hasMore": true, + "nextOffset": 50 + } +} +``` + +To enumerate **every** repository, keep calling with `offset` set to `pagination.nextOffset` until `hasMore` is `false`: + +```text +list_repos {} → repos 1–50, nextOffset 50, hasMore true +list_repos { offset: 50 } → repos 51–100, nextOffset 100, hasMore true +… +list_repos { offset: 400 } → repos 401–437, hasMore false (done) +``` + +Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. ## Resources Reference diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 80642a121..103a7fa44 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -310,8 +310,8 @@ VS Code's Ports panel shows forwarded ports once their listener starts. - **LadybugDB integration tests may fail in containers** (file-locking, `AGENTS.md` § Testing). Default to `npm run test:unit` inside the container; run integration tests on the host. Tracking issue: documented as a known limitation. - **Single-writer LadybugDB constraint** (`GUARDRAILS.md` § LadybugDB lock). Don't run `gitnexus analyze` on the host and inside the container against the same `.gitnexus/` directory simultaneously — the second writer will get `database busy`. -- **Native grammar builds add ~30s to first install.** Tree-sitter Dart/Proto/Swift grammars build during `gitnexus`'s `postinstall`; the `tree-sitter-kotlin` optional dependency (third-party npm package, source-only — no upstream prebuilds) compiles its native binding earlier, during npm's own dependency install, and is then probed by `build-tree-sitter-kotlin.cjs`. Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` (in your shell or `remoteEnv`, then rebuild) to skip the Dart/Proto/Swift builds and silence the Kotlin probe — but npm still compiles `tree-sitter-kotlin` unless you also pass `--omit=optional`. Each loses parsing for the affected language(s); the install still succeeds. -- **`tree-sitter-kotlin` warnings on install** are expected (per `AGENTS.md`). Ignore them. +- **Native grammar builds add ~30s to first install.** Tree-sitter Dart/Proto/Swift/Kotlin are all vendored uniformly: `node-gyp-build` picks a committed GitNexus-built prebuilt `.node` at install time (no compile), and only falls back to compiling from the vendored source during `postinstall` if no prebuild matches the host (then a toolchain is needed). Set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` (in your shell or `remoteEnv`, then rebuild) to skip all four; each loses parsing for the affected language(s), and the install still succeeds. +- **`tree-sitter-kotlin`/`tree-sitter-swift` warnings on install** only appear when no prebuild matches the platform-arch (per `AGENTS.md`); they are non-fatal — parsing for that language is simply unavailable. - **`.mcp.json` works inside the container**: `npx -y gitnexus@latest mcp` resolves cleanly because npm registry is reachable and the workspace bind mount exposes the same `.mcp.json` the host sees. - **Husky pre-commit fires inside the container** without extra setup. The root `npm install` (run automatically in `postCreateCommand`) installs the hook via `package.json` `prepare`. diff --git a/.github/scripts/update-vendored-grammars.mjs b/.github/scripts/update-vendored-grammars.mjs new file mode 100644 index 000000000..957200e77 --- /dev/null +++ b/.github/scripts/update-vendored-grammars.mjs @@ -0,0 +1,266 @@ +#!/usr/bin/env node +/** + * Vendored tree-sitter grammar update monitor. + * + * Checks each vendored grammar against its upstream source-of-origin and, for an + * available AND ABI-compatible update, re-vendors the grammar source in place so + * a PR can be opened. The version bump in vendor//package.json then triggers + * .github/workflows/build-tree-sitter-prebuilds.yml, which cross-builds + ABI- + * validates the prebuilds — so even an imperfect re-vendor can never silently + * ship: its PR's CI goes red. + * + * ABI awareness is load-bearing. Every grammar is pinned to tree-sitter@0.21.1 + * (LANGUAGE_VERSION 13–14, the #1922 gate). Most upstream grammar releases target + * a newer tree-sitter, so a blind "bump to latest" would pull an ABI-incompatible + * parser and open doomed PRs. This monitor fetches the candidate source, reads its + * parser.c `#define LANGUAGE_VERSION`, and only re-vendors when it is 13 or 14; + * incompatible updates are reported (and surfaced as a workflow notice), not + * applied. + * + * Usage: + * node update-vendored-grammars.mjs # detect only → JSON report on stdout + * node update-vendored-grammars.mjs --apply X # re-vendor grammar X in place + * + * tree-sitter-c is MONITORED but report-only (`hold`): it is ABI-pinned at 0.21.4 + * (#1242/#858) and must not auto-bump without a tree-sitter runtime upgrade, so an + * available c update is detected + reported but never auto-applied — even if it is + * ABI-13/14. A maintainer re-vendors it deliberately. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const VENDOR = path.join(REPO_ROOT, 'gitnexus', 'vendor'); + +const COMPATIBLE_ABI = new Set([13, 14]); // tree-sitter@0.21.1 LANGUAGE_VERSION range + +// Source-of-origin per grammar. npm grammars resolve `latest` via the registry; +// github grammars (no usable npm release) track the default branch HEAD. A `hold` +// reason makes a grammar report-only: updates are detected + surfaced but never +// auto-applied (c is ABI-pinned and must not move without a runtime upgrade). +const GRAMMARS = { + c: { + name: 'tree-sitter-c', + npm: 'tree-sitter-c', + hold: 'ABI-pinned at 0.21.4 (#1242/#858) — needs a tree-sitter runtime upgrade before bumping', + }, + swift: { name: 'tree-sitter-swift', npm: 'tree-sitter-swift' }, + kotlin: { name: 'tree-sitter-kotlin', npm: 'tree-sitter-kotlin' }, + dart: { name: 'tree-sitter-dart', github: 'UserNobody14/tree-sitter-dart' }, + proto: { name: 'tree-sitter-proto', github: 'coder3101/tree-sitter-proto' }, +}; + +const sh = (cmd, args, opts = {}) => + execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }).trim(); + +const clean = (v) => + String(v || '') + .replace(/^[v^~]/, '') + .trim(); + +function vendoredVersion(g) { + const p = path.join(VENDOR, g.name, 'package.json'); + return clean(JSON.parse(fs.readFileSync(p, 'utf8')).version); +} + +/** Resolve the upstream candidate: { version, ref, kind }. */ +function resolveUpstream(g) { + if (g.npm) { + const version = clean(sh('npm', ['view', g.npm, 'version'])); + return { version, ref: version, kind: 'npm' }; + } + // github: no reliable release tags here, so track the default branch HEAD sha. + const meta = JSON.parse(sh('gh', ['api', `repos/${g.github}`])); + const branch = meta.default_branch; + const sha = JSON.parse(sh('gh', ['api', `repos/${g.github}/commits/${branch}`])).sha; + // Version key: "-g" — safeRef-compatible (no `+`, + // which the build workflow's ref validator rejects) and changes on every commit. + let base = '0.0.0'; + try { + const pkg = JSON.parse( + Buffer.from( + JSON.parse(sh('gh', ['api', `repos/${g.github}/contents/package.json?ref=${sha}`])).content, + 'base64', + ).toString('utf8'), + ); + if (pkg.version) base = clean(pkg.version); + } catch { + /* no upstream package.json — base stays 0.0.0 */ + } + return { version: `${base}-g${sha.slice(0, 7)}`, ref: sha, kind: 'github' }; +} + +/** Fetch the candidate source into a temp dir; return the package root. */ +function fetchSource(g, ref) { + const work = fs.mkdtempSync( + path.join(os.tmpdir(), `revendor-${Object.keys(GRAMMARS).find((k) => GRAMMARS[k] === g)}-`), + ); + if (g.npm) { + sh('npm', ['pack', `${g.npm}@${ref}`, '--silent'], { cwd: work }); + const tgz = fs.readdirSync(work).find((f) => f.endsWith('.tgz')); + sh('tar', ['xzf', tgz], { cwd: work }); + return path.join(work, 'package'); + } + // github tarball at the resolved sha. Download + extract WITHOUT a shell + // (no `bash -c`/redirect): `gh api` writes the binary tarball to stdout, which + // we capture as a Buffer and write to a fixed path, then extract with execFile. + // Avoids the shell-command-injection surface CodeQL flags when an API-derived + // ref is interpolated into a `bash -c` string. + const tgz = path.join(work, 'src.tgz'); + fs.writeFileSync( + tgz, + execFileSync('gh', ['api', `repos/${g.github}/tarball/${ref}`], { + maxBuffer: 512 * 1024 * 1024, + }), + ); + sh('tar', ['xzf', tgz], { cwd: work }); + const dir = fs.readdirSync(work).find((f) => fs.statSync(path.join(work, f)).isDirectory()); + return path.join(work, dir); +} + +/** Read parser.c's LANGUAGE_VERSION (ABI). Prefer the ABI-14 default parser.c. */ +function readAbi(srcRoot) { + const candidates = ['src/parser.c', 'parser.c']; + for (const rel of candidates) { + const p = path.join(srcRoot, rel); + if (!fs.existsSync(p)) continue; + // Read only the head — the #define is near the top. + const head = fs.readFileSync(p, 'utf8').slice(0, 4000); + const m = head.match(/#define\s+LANGUAGE_VERSION\s+(\d+)/); + if (m) return Number(m[1]); + } + return null; // unknown (e.g. parser.c only generated at build time) +} + +function detect() { + const report = []; + for (const [key, g] of Object.entries(GRAMMARS)) { + const have = vendoredVersion(g); + let up; + try { + up = resolveUpstream(g); + } catch (err) { + report.push({ grammar: key, error: String(err.message || err) }); + continue; + } + const newer = up.kind === 'npm' ? up.version !== have : !have || up.ref.slice(0, 7) !== have; + let abi = null; + if (newer) { + try { + abi = readAbi(fetchSource(g, up.ref)); + } catch { + /* fetch/abi best-effort; null = unknown */ + } + } + report.push({ + grammar: key, + vendored: have, + upstream: up.version, + ref: up.ref, + kind: up.kind, + update: newer, + abi, + abiCompatible: abi == null ? null : COMPATIBLE_ABI.has(abi), + hold: g.hold || null, + // Auto-appliable only when there's an update, the ABI is known-compatible, + // AND the grammar is not on a policy hold (c). + applicable: newer && abi != null && COMPATIBLE_ABI.has(abi) && !g.hold, + }); + } + return report; +} + +const copyFile = (srcRoot, dest, rel) => { + const from = path.join(srcRoot, rel); + if (!fs.existsSync(from)) return false; + const to = path.join(dest, rel); + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.copyFileSync(from, to); + return true; +}; + +/** + * Re-vendor one grammar in place from its ABI-compatible upstream candidate. + * Copies ONLY the generated source-build + runtime files; deliberately KEEPS the + * GitNexus-hardened binding.gyp (Windows cflags, target_name), README (vendor + * notice), LICENSE, and prebuilds/ (the build workflow refreshes those). Bumps the + * stripped vendor package.json version + provenance — never re-introduces + * scripts/dependencies (#836/#1728). Returns the new version. + */ +function apply(key) { + const g = GRAMMARS[key]; + if (!g) { + console.error(`unknown grammar '${key}'`); + process.exit(2); + } + if (g.hold) { + console.error( + `${key}: report-only (${g.hold}); not auto-applied. Re-vendor manually if intended.`, + ); + process.exit(3); + } + const have = vendoredVersion(g); + const up = resolveUpstream(g); + const newer = up.kind === 'npm' ? up.version !== have : !have || up.version !== have; + if (!newer) { + console.error(`${key}: already current (${have}); nothing to apply.`); + process.exit(0); + } + const srcRoot = fetchSource(g, up.ref); + const abi = readAbi(srcRoot); + if (abi == null || !COMPATIBLE_ABI.has(abi)) { + console.error( + `${key}: candidate ${up.version} is ABI ${abi ?? 'unknown'} — not tree-sitter@0.21.1 ` + + `compatible (need 13/14); refusing to re-vendor. Handle manually.`, + ); + process.exit(3); + } + + const dest = path.join(VENDOR, g.name); + // The source-build inputs + runtime entrypoints that change between versions. + // binding.gyp / README / LICENSE / prebuilds are intentionally NOT touched. + for (const rel of [ + 'src/parser.c', + 'src/scanner.c', + 'src/node-types.json', + 'src/tree_sitter/alloc.h', + 'src/tree_sitter/array.h', + 'src/tree_sitter/parser.h', + 'bindings/node/binding.cc', + 'bindings/node/index.js', + 'bindings/node/index.d.ts', + ]) { + copyFile(srcRoot, dest, rel); + } + + const pkgPath = path.join(dest, 'package.json'); + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')); + pkg.version = up.version; + pkg._vendoredBy = + `gitnexus - re-vendored from ${g.npm ? `npm ${g.npm}@${up.version}` : `${g.github}@${up.ref}`} ` + + `by grammar-update-monitor on ABI ${abi}. Source-build inputs (parser.c/scanner.c/src/) refreshed; ` + + `the GitNexus-hardened binding.gyp + vendor README + prebuilds are preserved (prebuilds are ` + + `rebuilt by build-tree-sitter-prebuilds.yml on this version change). No scripts/dependencies here ` + + `(#836/#1728).`; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n'); + + console.log(`${key}: re-vendored ${g.name} → ${up.version} (ABI ${abi}).`); + return up.version; +} + +// Run the CLI only when invoked directly (not when imported by a test) — detect() +// makes live network calls, so importing must be side-effect-free. +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (isMain) { + if (process.argv[2] === '--apply') { + apply(process.argv[3]); + } else { + process.stdout.write(JSON.stringify(detect(), null, 2) + '\n'); + } +} + +export { detect, apply, resolveUpstream, readAbi, vendoredVersion, GRAMMARS, COMPATIBLE_ABI }; diff --git a/.github/workflows/build-tree-sitter-prebuilds.yml b/.github/workflows/build-tree-sitter-prebuilds.yml new file mode 100644 index 000000000..0acfca61a --- /dev/null +++ b/.github/workflows/build-tree-sitter-prebuilds.yml @@ -0,0 +1,529 @@ +name: Build tree-sitter prebuilds + +# Cross-builds the native tree-sitter prebuilds GitNexus vendors itself, so that +# grammars whose upstream packages ship SOURCE ONLY (no usable prebuilds/) never +# require a C/C++ toolchain at a user's install. This is the "no operational +# risk for any tree-sitter grammar" pipeline. +# +# Grammars covered here (the at-risk set — everything else already ships 6 +# upstream prebuilds AND stays dependency-review-tracked, so it is left alone). +# All five are vendored under gitnexus/vendor/; `kind` (below) only picks where +# the build job fetches the C source to compile: +# - tree-sitter-c (vendored prebuild-only; built from the published npm +# package — closes upstream's 4/6 ARM gap #2116 for a +# REQUIRED grammar) +# - tree-sitter-dart (vendored source; built from gitnexus/vendor/) +# - tree-sitter-proto (vendored source; built from gitnexus/vendor/) +# - tree-sitter-kotlin (vendored source; built from the published npm package — +# upstream ships source only) +# - tree-sitter-swift (vendored source; built from gitnexus/vendor/ — its +# prebuilds were originally upstream-shipped, now +# GitNexus-cross-built like the rest for uniformity) +# +# Output: gitnexus/vendor//prebuilds//.node for +# all 6 targets ({linux,darwin,win32}-{x64,arm64}). tree-sitter grammars are +# N-API, so one ABI-stable .node per platform-arch works across all Node majors. +# +# COST DISCIPLINE — this is a HEAVY native matrix (up to 3 grammars x 6 runners, +# incl. macOS + arm64). It is DELIBERATELY NOT wired into normal PR/push CI. It +# runs only: +# 1. on manual dispatch (workflow_dispatch); or +# 2. when a covered grammar's recorded version actually CHANGES — the `guard` +# job is the real gate (it diffs the recorded version vs the PR base); the +# `paths:` filter below only makes ordinary code PRs cost ZERO matrix time. +# Net effect: an ordinary code PR triggers nothing; bumping one grammar costs +# exactly one matrix run for that grammar, which opens a PR committing its rebuilt +# binaries. +# +# Concurrency convention: see CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention". +# +# NOTE: every action below is pinned to a release commit SHA (with the matching +# `# vX.Y.Z` tag comment verified against the GitHub API). If a future bump adds +# a new action, pin its real release SHA and allowlist it in .github/zizmor.yml / +# Scorecard before merge. + +on: + workflow_dispatch: + inputs: + grammars: + description: 'Comma-separated grammar shortnames to build (c,dart,proto,kotlin,swift), or "all".' + required: false + type: string + default: 'all' + ref: + description: 'Upstream version/tag/sha override (only honored when exactly one grammar is selected).' + required: false + type: string + default: '' + force: + description: 'Build even if the recorded version is unchanged (re-cut a broken prebuild).' + required: false + type: boolean + default: false + open_pr: + description: 'Open a PR with the rebuilt prebuilds (false = artifacts only).' + required: false + type: boolean + default: true + pull_request: + branches: [main] + paths: + # Vendored grammars: their version lives in the vendor snapshot package.json. + - 'gitnexus/vendor/tree-sitter-c/package.json' + - 'gitnexus/vendor/tree-sitter-dart/package.json' + - 'gitnexus/vendor/tree-sitter-proto/package.json' + - 'gitnexus/vendor/tree-sitter-kotlin/package.json' + - 'gitnexus/vendor/tree-sitter-swift/package.json' + # Transition window: kotlin's pin still lives here until it is vendored. + - 'gitnexus/package.json' + # Self-test: re-run the guard (normally a no-op) when the recipe changes. + - '.github/workflows/build-tree-sitter-prebuilds.yml' + +# Least privilege by default; only `aggregate` opts up. +permissions: + contents: read + +# One slot per ref. Collapse PR re-pushes, but never cancel a manual re-cut. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + # ── Gate: decide which grammars (if any) need a native rebuild, and emit the + # {grammar x platform-arch} matrix the build job consumes. ─────────────── + guard: + name: Decide what to build + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + outputs: + any: ${{ steps.decide.outputs.any }} + matrix: ${{ steps.decide.outputs.matrix }} + release_app: ${{ steps.relapp.outputs.configured }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 0 # need base history to diff recorded versions + persist-credentials: false + + - name: Decide + id: decide + env: + EVENT: ${{ github.event_name }} + # Untrusted dispatch inputs — read via env only, validated in JS. + INPUT_GRAMMARS: ${{ inputs.grammars }} + INPUT_REF: ${{ inputs.ref }} + FORCE: ${{ github.event_name == 'workflow_dispatch' && inputs.force || 'false' }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + set -euo pipefail + node --input-type=module - <<'NODE' + import { execSync } from 'node:child_process'; + import fs from 'node:fs'; + import { appendFileSync } from 'node:fs'; + + // Registry of the at-risk grammars this workflow owns. `kind` drives + // how the build job resolves source: 'npm' pulls the published package; + // 'vendored' builds from gitnexus/vendor/ (which carries the C + // source + binding.gyp). Extend this list to cover a new grammar. + const REGISTRY = { + // c is vendored prebuild-only but BUILT from the published npm + // package (kind 'npm'), held at 0.21.4 — it closes upstream's 4/6 + // ARM gap (#2116) for a REQUIRED grammar that otherwise hard-fails + // install on toolchain-less ARM. + c: { name: 'tree-sitter-c', kind: 'npm' }, + dart: { name: 'tree-sitter-dart', kind: 'vendored' }, + proto: { name: 'tree-sitter-proto', kind: 'vendored' }, + kotlin: { name: 'tree-sitter-kotlin', kind: 'npm' }, + // swift is vendored WITH its source (parser.c/scanner.c/binding.gyp), + // so it builds from gitnexus/vendor/ like dart/proto. Its prebuilds + // were originally upstream-shipped; rebuilding them here unifies it. + swift: { name: 'tree-sitter-swift', kind: 'vendored' }, + }; + const PLATFORMS = [ + { platform_arch: 'linux-x64', os: 'ubuntu-24.04' }, + { platform_arch: 'linux-arm64', os: 'ubuntu-24.04-arm' }, + { platform_arch: 'darwin-arm64', os: 'macos-15' }, + { platform_arch: 'darwin-x64', os: 'macos-15-intel' }, // macos-13 retired Dec-2025; Intel EOL ~Aug-2027 + { platform_arch: 'win32-x64', os: 'windows-2022' }, + { platform_arch: 'win32-arm64', os: 'windows-11-arm' }, + ]; + + const clean = (v) => (v || '').replace(/^[\^~]/, '').trim(); + const json = (p) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; } }; + + // Durable version key for a grammar at a checkout root. Prefer the + // vendor snapshot (the post-vendor source of truth); fall back to the + // optionalDependencies pin during the transition window. (A guard keyed + // on the node_modules lock entry would self-disable once a grammar is + // vendored, because that entry is deleted.) + function recordedVersion(root, name) { + const v = json(`${root}/gitnexus/vendor/${name}/package.json`); + if (v && v.version) return clean(v.version); + const pkg = json(`${root}/gitnexus/package.json`); + const od = pkg && (pkg.optionalDependencies || {}); + const d = pkg && (pkg.dependencies || {}); + return clean((od && od[name]) || (d && d[name]) || ''); + } + + const event = process.env.EVENT; + const force = process.env.FORCE === 'true'; + + // Select which grammar shortnames are in play. + let selected; + if (event === 'workflow_dispatch') { + const raw = (process.env.INPUT_GRAMMARS || 'all').trim(); + selected = raw === 'all' ? Object.keys(REGISTRY) + : raw.split(',').map((s) => s.trim()).filter(Boolean); + for (const s of selected) if (!REGISTRY[s]) throw new Error(`unknown grammar '${s}'`); + } else { + selected = Object.keys(REGISTRY); + } + + // Resolve the base-ref recorded versions (pull_request only) so we can + // diff. On dispatch, base is irrelevant (manual intent / force wins). + const baseRoot = `${process.env.RUNNER_TEMP}/base`; + if (event === 'pull_request') { + const baseSha = process.env.BASE_SHA; + for (const s of selected) { + const name = REGISTRY[s].name; + for (const rel of [`gitnexus/vendor/${name}/package.json`, `gitnexus/package.json`]) { + const dst = `${baseRoot}/${rel}`; + fs.mkdirSync(dst.slice(0, dst.lastIndexOf('/')), { recursive: true }); + try { + const buf = execSync(`git show ${baseSha}:${rel}`, { stdio: ['ignore', 'pipe', 'ignore'] }); + fs.writeFileSync(dst, buf); + } catch { /* file absent at base — fine */ } + } + } + } + + // The single-ref override is only meaningful for a one-grammar dispatch. + const refOverride = clean(process.env.INPUT_REF); + if (refOverride && !(event === 'workflow_dispatch' && selected.length === 1)) { + throw new Error('ref override requires exactly one grammar selected'); + } + const safeRef = (r) => /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(r); + + const include = []; + const built = []; + for (const short of selected) { + const { name, kind } = REGISTRY[short]; + const head = recordedVersion('.', name); + const ref = refOverride || head; + if (!ref) { console.log(`skip ${short}: no recorded version`); continue; } + if (!safeRef(ref)) throw new Error(`unsafe ref for ${short}: '${ref}'`); + + let build = false; + if (event === 'workflow_dispatch') { + build = true; // manual intent (force toggles only the unchanged-guard, which is bypassed here) + } else { + const base = recordedVersion(baseRoot, name); + build = !!head && head !== base; + console.log(`${short}: head='${head || ''}' base='${base || ''}' -> ${build ? 'BUILD' : 'skip'}`); + } + if (force) build = true; + if (!build) continue; + built.push(short); + for (const p of PLATFORMS) include.push({ grammar: short, name, kind, ref, ...p }); + } + + const out = process.env.GITHUB_OUTPUT; + appendFileSync(out, `any=${include.length > 0}\n`); + appendFileSync(out, `matrix=${JSON.stringify({ include })}\n`); + if (include.length === 0) { + console.log('::notice::No covered grammar version changed — skipping native matrix.'); + } else { + console.log(`Building: ${built.join(', ')} (${include.length} jobs)`); + } + NODE + + # The aggregate job opens a PR via a GitHub App token; without the App + # secrets it would hard-fail AFTER a full native build. Surface their + # presence as a guard output so aggregate skips cleanly (the build job's + # artifacts still upload). secrets aren't available in a job-level `if:`, + # so we compute the boolean here (a step CAN read secrets) and gate on it. + - name: Check release App secret + id: relapp + env: + HAS_APP: ${{ secrets.RELEASE_APP_ID != '' && secrets.RELEASE_APP_PRIVATE_KEY != '' }} + run: | + set -euo pipefail + echo "configured=$HAS_APP" >> "$GITHUB_OUTPUT" + if [ "$HAS_APP" != "true" ]; then + echo "::notice::Release GitHub App secrets (RELEASE_APP_ID / RELEASE_APP_PRIVATE_KEY) are not configured — prebuilds will build and upload as artifacts, but the auto-PR is skipped. Provision the App, or run with open_pr=false to suppress this notice." + fi + + # ── Build one native prebuild per (grammar, platform-arch). No cross-compile. ─ + build: + name: ${{ matrix.grammar }} ${{ matrix.platform_arch }} + needs: guard + if: needs.guard.outputs.any == 'true' + permissions: + contents: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.guard.outputs.matrix) }} + runs-on: ${{ matrix.os }} + # 45 (not 30) for headroom: the kotlin parser.c is ~23 MB and swift's ~18 MB, + # and compiling them under emulation on the arm runners is slow. + timeout-minutes: 45 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false # this job uploads artifacts (artipacked) + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + + - name: Ensure Python (arm64 Windows only) + if: matrix.platform_arch == 'win32-arm64' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Build prebuild + id: build + shell: bash + env: + GRAMMAR: ${{ matrix.grammar }} + NAME: ${{ matrix.name }} + KIND: ${{ matrix.kind }} + REF: ${{ matrix.ref }} + PLATFORM_ARCH: ${{ matrix.platform_arch }} + run: | + set -euo pipefail + work="$RUNNER_TEMP/ts-build" + rm -rf "$work"; mkdir -p "$work"; cd "$work" + npm init -y >/dev/null + + # node-addon-api must match what the grammar's binding.cc expects. + # GitNexus hoists ^8 for the vendored grammars; npm grammars declare + # their own (do NOT pin it for npm grammars — let the dep resolve it). + if [ "$KIND" = "vendored" ]; then + # Build from the vendored C source (carries parser.c + binding.gyp). + srcdir="$work/$NAME" + cp -R "$GITHUB_WORKSPACE/gitnexus/vendor/$NAME" "$srcdir" + rm -rf "$srcdir/prebuilds" "$srcdir/build" "$srcdir/node_modules" + npm install --no-audit --no-fund --ignore-scripts \ + prebuildify@^6 node-gyp@^11 node-addon-api@^8 + pkgdir="$srcdir" + export npm_config_node_gyp="$work/node_modules/node-gyp/bin/node-gyp.js" + else + # Pull the published source-only package. + npm install --no-audit --no-fund --ignore-scripts \ + "$NAME@${REF}" prebuildify@^6 node-gyp@^11 + pkgdir="$work/node_modules/$NAME" + fi + + test -f "$pkgdir/binding.gyp" || { echo "::error::no binding.gyp for $NAME@$REF"; exit 1; } + + # Drop any prebuilds the package shipped in its own tarball before we + # build. The tree-sitter-org npm grammars (e.g. tree-sitter-c) bundle + # prebuilds/ for all 6 tuples; left in place, the `find ... -print -quit` + # below would pick a non-host tuple (e.g. win32-x64 on a linux runner) + # and the assertion would wrongly fail. prebuildify rebuilds THIS host's + # tuple from the source the tarball also ships. (Vendored grammars are + # already cleaned above; this also covers the npm branch.) + rm -rf "$pkgdir/prebuilds" + + # N-API, stripped, single ABI-stable binary for THIS host's arch. No + # `-t `: an N-API prebuild is Node-version-agnostic, and + # prebuildify parses a bare `-t 22` as the NUMBER 22 and crashes + # (`v.indexOf is not a function`). prebuildify emits + # prebuilds/-/.node. + ( cd "$pkgdir" && npx --no-install prebuildify --napi --strip ) + + out=$(find "$pkgdir/prebuilds" -name '*.node' -print -quit) + test -n "$out" || { echo "::error::prebuildify produced no .node"; exit 1; } + produced=$(basename "$(dirname "$out")") + [ "$produced" = "$PLATFORM_ARCH" ] || { echo "::error::built $produced, expected $PLATFORM_ARCH"; exit 1; } + + stage="$RUNNER_TEMP/stage/$GRAMMAR/$PLATFORM_ARCH"; mkdir -p "$stage" + cp "$out" "$stage/$NAME.node" + echo "stage=$stage" >> "$GITHUB_OUTPUT" + + - name: Validate the .node loads and parses on this arch + shell: bash + env: + GRAMMAR: ${{ matrix.grammar }} + NAME: ${{ matrix.name }} + PLATFORM_ARCH: ${{ matrix.platform_arch }} + EXPECT_ARCH: ${{ contains(matrix.platform_arch, 'arm64') && 'arm64' || 'x64' }} + run: | + set -euo pipefail + probe="$RUNNER_TEMP/probe"; rm -rf "$probe" + mkdir -p "$probe/prebuilds/$PLATFORM_ARCH" + cp "$RUNNER_TEMP/stage/$GRAMMAR/$PLATFORM_ARCH/$NAME.node" \ + "$probe/prebuilds/$PLATFORM_ARCH/$NAME.node" + cd "$probe" + # Pin tree-sitter to the repo's exact runtime peer so an ABI mismatch + # fails HERE, not in a user's install (mirrors the #1922 ABI gate). + # NOT --ignore-scripts: tree-sitter@0.21.1's tarball ships prebuilds for + # the common tuples but NOT linux-arm64 / win32-arm64, so on the arm64 + # runners node-gyp-build must source-build the runtime — give it node-gyp + # + node-addon-api to do so. Where tree-sitter ships a prebuild (x64, + # darwin-arm64) node-gyp-build uses it and nothing compiles. The grammar + # .node we built is still loaded as a prebuild; only the runtime peer may + # compile. The grammar-vs-runtime ABI check still fires at setLanguage. + npm install --no-audit --no-fund \ + node-gyp-build@^4 node-gyp@^11 node-addon-api@^8 tree-sitter@0.21.1 + # The node script is single-quoted on purpose — its ${...} are JS + # template literals read from the environment, not shell expansions. + # shellcheck disable=SC2016 + GRAMMAR="$GRAMMAR" EXPECT_ARCH="$EXPECT_ARCH" node -e ' + const expect = process.env.EXPECT_ARCH; + // Catch an emulated x64 Node silently mis-passing on an arm64 runner. + if (process.arch !== expect) throw new Error(`runner arch ${process.arch} != ${expect}`); + const snippets = { + c: "int main(void) { return 0; }", + dart: "void main() { print(\"hi\"); }", + proto: "syntax = \"proto3\";\nmessage M { int32 id = 1; }", + kotlin: "fun main() { println(\"hi\") }", + swift: "func greet() { print(\"hi\") }", + }; + const lang = require("node-gyp-build")(process.cwd()); + const Parser = require("tree-sitter"); + const p = new Parser(); p.setLanguage(lang); + const tree = p.parse(snippets[process.env.GRAMMAR]); + if (!tree || !tree.rootNode || tree.rootNode.hasError) { + throw new Error("parse failed/error: " + (tree && tree.rootNode && tree.rootNode.type)); + } + console.log("OK", process.env.GRAMMAR, process.platform + "-" + process.arch, tree.rootNode.type); + ' + + - name: Upload prebuild artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ts-prebuild-${{ matrix.grammar }}-${{ matrix.platform_arch }} + path: ${{ steps.build.outputs.stage }}/${{ matrix.name }}.node + if-no-files-found: error + retention-days: 7 + + # ── Aggregate every grammar's six prebuilds, assert completeness, open a PR. ─ + aggregate: + name: Vendor prebuilds + open PR + needs: [guard, build] + # Open the prebuild PR on a non-fork pull_request that bumped a grammar + # version (the documented version-change -> prebuild-PR flow), or on a manual + # dispatch with open_pr=true. Event-gating is explicit so we never rely on + # GHA coercing a null `inputs.open_pr` on pull_request events (Codex F4): + # `inputs.open_pr` is null off-dispatch, and `null != false` is direction- + # ambiguous, so `open_pr` is only consulted on workflow_dispatch. + if: >- + needs.guard.outputs.any == 'true' && + needs.guard.outputs.release_app == 'true' && + github.event.pull_request.head.repo.fork != true && + (github.event_name == 'pull_request' || inputs.open_pr == true) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read # actual writes use a short-lived App token below + id-token: write # SLSA provenance attestation + attestations: write + steps: + - name: Mint GitHub App token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + token: ${{ steps.app-token.outputs.token }} + persist-credentials: false + + - name: Download all prebuild artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ${{ runner.temp }}/dl + pattern: ts-prebuild-* + + - name: Place prebuilds, assert each built grammar has all 6, write SHA256SUMS + id: place + shell: bash + env: + MATRIX: ${{ needs.guard.outputs.matrix }} + DL: ${{ runner.temp }}/dl + run: | + set -euo pipefail + node --input-type=module - <<'NODE' + import fs from 'node:fs'; + import { execSync } from 'node:child_process'; + const include = JSON.parse(process.env.MATRIX).include; + const dl = process.env.DL; + const byGrammar = {}; + for (const e of include) (byGrammar[e.grammar] ||= { name: e.name, archs: [] }).archs.push(e.platform_arch); + const PLATFORMS = ['linux-x64','linux-arm64','darwin-arm64','darwin-x64','win32-x64','win32-arm64']; + const changed = []; + for (const [grammar, { name }] of Object.entries(byGrammar)) { + const dest = `gitnexus/vendor/${name}/prebuilds`; + // A vendored grammar with 5/6 prebuilds silently breaks node-gyp-build + // on the 6th platform — refuse a partial result. + for (const pa of PLATFORMS) { + const art = `${dl}/ts-prebuild-${grammar}-${pa}/${name}.node`; + if (!fs.existsSync(art)) throw new Error(`missing ${grammar} prebuild for ${pa}`); + fs.mkdirSync(`${dest}/${pa}`, { recursive: true }); + fs.copyFileSync(art, `${dest}/${pa}/${name}.node`); + } + execSync(`cd ${dest} && find . -name '*.node' | sort | xargs sha256sum > SHA256SUMS`); + changed.push(name); + } + fs.appendFileSync(process.env.GITHUB_OUTPUT, `grammars=${changed.join(',')}\n`); + console.log('Vendored prebuilds for:', changed.join(', ')); + NODE + + - name: Attest build provenance (SLSA) + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 + with: + subject-path: 'gitnexus/vendor/tree-sitter-*/prebuilds/**/*.node' + + - name: Create or update PR + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + GRAMMARS: ${{ steps.place.outputs.grammars }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + with: + github-token: ${{ steps.app-token.outputs.token }} + script: | + const { execSync } = require('node:child_process'); + const run = (c) => execSync(c, { stdio: ['ignore', 'pipe', 'inherit'] }).toString().trim(); + const grammars = process.env.GRAMMARS; + const slug = grammars.replace(/[^a-z0-9]+/gi, '-'); + const branch = `chore/vendor-ts-prebuilds-${slug}-${context.runId}`; + + run('git add gitnexus/vendor/tree-sitter-*/prebuilds'); + if (!run('git status --porcelain -- gitnexus/vendor/tree-sitter-*/prebuilds')) { + core.notice('Prebuilds byte-identical to vendor; nothing to commit.'); + return; + } + run('git config user.name "gitnexus-release-bot[bot]"'); + run('git config user.email "gitnexus-release-bot[bot]@users.noreply.github.com"'); + run(`git checkout -b "${branch}"`); + run(`git commit -m "chore(vendor): rebuild native prebuilds (${grammars})\n\nBuilt by ${process.env.RUN_URL}"`); + const { owner, repo } = context.repo; + const remote = `https://x-access-token:${process.env.GH_TOKEN}@github.com/${owner}/${repo}.git`; + // Plain --force, not --force-with-lease: the branch is ephemeral and + // unique per run (keyed by context.runId), written ONLY by this job, so + // there is no concurrent writer to protect against. --force-with-lease + // would compare against a remote-tracking ref this fresh checkout never + // fetched, so re-running the SAME run (branch already pushed by attempt + // 1) fails with "stale info" instead of overwriting. + run(`git push --force "${remote}" "HEAD:${branch}"`); + const body = [ + `Rebuilt the vendored native prebuilds for: **${grammars}**.`, + '', + `Builder run: ${process.env.RUN_URL}`, + 'Each `.node` was `require()`-loaded + parsed a real snippet on its target', + 'platform-arch before upload. SLSA build-provenance attested; `SHA256SUMS`', + 'committed alongside each grammar.', + ].join('\n'); + const { data: pr } = await github.rest.pulls.create({ + owner, repo, head: branch, base: 'main', + title: `chore(vendor): tree-sitter prebuilds (${grammars})`, body, + }); + core.info(`Opened PR #${pr.number}`); diff --git a/.github/workflows/ci-devcontainer.yml b/.github/workflows/ci-devcontainer.yml index 96849b17a..d2d79f4be 100644 --- a/.github/workflows/ci-devcontainer.yml +++ b/.github/workflows/ci-devcontainer.yml @@ -36,7 +36,7 @@ jobs: # persist-credentials: false — this job only reads (tests and syntax # checks) and never pushes. The setting keeps GITHUB_TOKEN out of # .git/config, which zizmor flags as the "artipacked" issue. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 @@ -57,7 +57,7 @@ jobs: # persist-credentials: false — this is a read-only build smoke that # never pushes. The setting keeps GITHUB_TOKEN out of .git/config, # which zizmor flags as the "artipacked" issue. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 96b0a4b26..65c2e6248 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -14,7 +14,7 @@ jobs: outputs: web_changed: ${{ steps.filter.outputs.web }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v3 id: filter with: @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Configure e2e GitNexus home run: echo "GITNEXUS_HOME=${RUNNER_TEMP}/gitnexus-home" >> "$GITHUB_ENV" diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index a81876d9d..7ed4a345f 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 @@ -24,7 +24,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: node-version: 22 @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus - run: npx tsc --noEmit working-directory: gitnexus @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus-web - run: npx tsc -b --noEmit working-directory: gitnexus-web @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Validate workflow concurrency convention shell: bash run: | diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 03c933ad0..5200d3758 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -125,7 +125,7 @@ jobs: - name: Checkout (for vitest config) if: steps.meta.outputs.skip != 'true' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: sparse-checkout: gitnexus/vitest.config.ts sparse-checkout-cone-mode: false diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index b2341db36..ee0d92332 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -16,7 +16,7 @@ jobs: # test-reports artifact (if: always()). The default-persisted token in # .git/config must not be capturable through that upload (zizmor # credential-persistence / artipacked audit). The job never pushes. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -80,7 +80,7 @@ jobs: steps: # persist-credentials: false — runs tests only, never pushes (zizmor # credential-persistence / artipacked audit). - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -94,8 +94,9 @@ jobs: # 1. Static, offline: assert every grammar's compiled ABI loads on the # pinned runtime (check-tree-sitter-upgrade-readiness.py --assert-current). # 2. Dynamic: run the parser-loader ABI load-smoke on the OS matrix so an - # ABI-incompatible prebuilt (esp. the binary-only Swift vendor, which the - # static check can't introspect) fails on the platform it ships to. + # ABI-incompatible committed vendor prebuilt (e.g. Swift's — the static + # check introspects source, not the shipped .node) fails on the platform + # it ships to. abi-assert: name: tree-sitter ABI (${{ matrix.os }}) strategy: @@ -105,7 +106,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 20 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus with: build: 'true' @@ -137,7 +138,7 @@ jobs: # from a tarball and never pushes back; the token in .git/config would # be at risk of leaking through any future artifact-upload step # (zizmor artipacked audit). Disable upfront. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -245,7 +246,7 @@ jobs: # and never pushes; the default-persisted token in .git/config would be at # risk of leaking through an artifact upload (zizmor credential-persistence # / artipacked audit). Mirrors the packaged-install-smoke job below. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - uses: ./.github/actions/setup-gitnexus @@ -265,6 +266,16 @@ jobs: run: node --import tsx bench/scope-capture/measure.mjs --check working-directory: gitnexus + - name: CFG construction time / disk / memory guards (#2081 M1) + # Build-free: asserts collectFunctionCfgs output is unchanged + # (fingerprint) and that wall-time, cfgSideChannel disk bytes, AND + # retained heap all stay sub-quadratic for the straight-line / + # many-functions / branchy scenarios. Catches an O(n^2) re-regression in + # the per-function CFG builder (e.g. an extendBlock concat chain) and a + # memory/disk blow-up. --expose-gc enables the retained-heap measurement. + run: node --expose-gc --import tsx bench/cfg/measure.mjs --check + working-directory: gitnexus + - name: Cross-language pipeline benchmarks (GITNEXUS_BENCH, serial) env: GITNEXUS_BENCH: '1' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 021a56930..2f925a62b 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -129,7 +129,7 @@ jobs: core.setOutput('code_review', isCodeReview ? 'true' : 'false'); - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: repository: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.repo || github.repository }} ref: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.sha || '' }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c7a95e29a..6835403db 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Don't leave GITHUB_TOKEN in .git/config for downstream steps to read. persist-credentials: false diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index d1214f849..4341e1327 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4a73329ed..39ff10494 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -101,7 +101,7 @@ jobs: # When triggered by workflow_call the caller passes the RC tag as an input; # we check out that tag so the Dockerfile and package.json match the built image. # For tag-push events github.ref is already the tag ref — no override needed. - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ inputs.tag || github.ref }} @@ -138,7 +138,7 @@ jobs: # Required for multi-platform (linux/arm64) emulation. - name: Set up QEMU - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 diff --git a/.github/workflows/gitleaks.yml b/.github/workflows/gitleaks.yml index f5e9af170..ee5ba8ecc 100644 --- a/.github/workflows/gitleaks.yml +++ b/.github/workflows/gitleaks.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # Full history needed for the on-push full-history scan; on PRs the # action diffs against the base ref so the cost is bounded by the PR. diff --git a/.github/workflows/grammar-update-monitor.yml b/.github/workflows/grammar-update-monitor.yml new file mode 100644 index 000000000..ee14c093c --- /dev/null +++ b/.github/workflows/grammar-update-monitor.yml @@ -0,0 +1,146 @@ +name: Vendored grammar update monitor + +# Periodically checks each vendored tree-sitter grammar against its +# source-of-origin and opens a PR re-vendoring any update that is ABI-COMPATIBLE +# with the pinned tree-sitter@0.21.1 (LANGUAGE_VERSION 13–14, #1922). The version +# bump then triggers build-tree-sitter-prebuilds.yml, which cross-builds + ABI- +# validates the prebuilds — so a re-vendor that is subtly wrong can never silently +# ship: its PR's CI goes red. +# +# ABI-INCOMPATIBLE updates (the common case — upstreams move to newer tree-sitter) +# are reported as a notice + job summary, NOT applied, so the monitor never opens +# doomed PRs. tree-sitter-c is MONITORED but report-only: it is ABI-pinned at +# 0.21.4 (#1242/#858), so an available c update is surfaced (notice + summary) but +# never auto-bumped — a maintainer re-vendors it deliberately after a runtime +# upgrade. +# +# Concurrency convention: see CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention". + +on: + schedule: + - cron: '17 6 * * 1' # weekly, Monday 06:17 UTC + workflow_dispatch: + +# Least privilege; the actual writes use a short-lived App token minted below. +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + monitor: + name: Check upstreams + open update PRs + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + + # secrets aren't usable in a job/step `if:`, so compute presence here. + - name: Check release App secret + id: relapp + env: + HAS_APP: ${{ secrets.RELEASE_APP_ID != '' && secrets.RELEASE_APP_PRIVATE_KEY != '' }} + run: echo "configured=$HAS_APP" >> "$GITHUB_OUTPUT" + + - name: Mint GitHub App token + id: app-token + if: steps.relapp.outputs.configured == 'true' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - name: Detect updates, re-vendor ABI-compatible ones, open PRs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + HAS_APP: ${{ steps.relapp.outputs.configured }} + # App token writes; falls back to the read-only job token (PRs then skip). + GH_TOKEN: ${{ steps.app-token.outputs.token || github.token }} + with: + github-token: ${{ steps.app-token.outputs.token || github.token }} + script: | + const { execFileSync } = require('node:child_process'); + const SCRIPT = '.github/scripts/update-vendored-grammars.mjs'; + const run = (cmd, args, opts = {}) => + execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }); + + const report = JSON.parse(run('node', [SCRIPT])); + const { owner, repo } = context.repo; + const hasApp = process.env.HAS_APP === 'true'; + const applied = [], held = [], errors = [], skipped = []; + + run('git', ['config', 'user.name', 'gitnexus-release-bot[bot]']); + run('git', ['config', 'user.email', 'gitnexus-release-bot[bot]@users.noreply.github.com']); + const baseSha = run('git', ['rev-parse', 'HEAD']).trim(); + + for (const r of report) { + if (r.error) { errors.push(r); continue; } + if (!r.update) continue; + if (!r.applicable) { held.push(r); continue; } // ABI-incompatible / unknown + + const name = `tree-sitter-${r.grammar}`; + const branch = `chore/update-${name}-${r.upstream}`.replace(/[^a-z0-9._/-]+/gi, '-'); + + // Idempotency: don't reopen an existing PR for this exact version. + const existing = await github.rest.pulls.list({ owner, repo, head: `${owner}:${branch}`, state: 'all' }); + if (existing.data.length > 0) { skipped.push({ ...r, reason: 'PR exists' }); continue; } + + // Re-vendor in place (refuses + exits non-zero if ABI turns out wrong). + try { + run('node', [SCRIPT, '--apply', r.grammar]); + } catch (e) { + errors.push({ ...r, error: `apply failed: ${String(e.message || e).slice(0, 200)}` }); + run('git', ['checkout', '--', 'gitnexus/vendor']); + continue; + } + + if (!hasApp) { + skipped.push({ ...r, reason: 'no RELEASE_APP secret — PR not opened' }); + run('git', ['checkout', '--', 'gitnexus/vendor']); + continue; + } + + const remote = `https://x-access-token:${process.env.GH_TOKEN}@github.com/${owner}/${repo}.git`; + run('git', ['checkout', '-B', branch, baseSha]); + run('git', ['add', `gitnexus/vendor/${name}`]); + run('git', ['commit', '-m', `chore(vendor): update ${name} to ${r.upstream}`]); + run('git', ['push', '--force-with-lease', remote, `HEAD:${branch}`]); + const body = [ + `Automated re-vendor of **${name}** to \`${r.upstream}\` (from ${r.kind === 'npm' ? `npm \`${name}\`` : `\`${r.ref}\``}).`, + '', + `Verified ABI **${r.abi}** — compatible with the pinned \`tree-sitter@0.21.1\` (13–14).`, + 'Source-build inputs refreshed; the GitNexus binding.gyp / README / prebuilds are preserved.', + 'The version bump triggers `build-tree-sitter-prebuilds.yml` to rebuild + ABI-validate the', + 'prebuilds — review its result before merging.', + ].join('\n'); + const pr = await github.rest.pulls.create({ + owner, repo, head: branch, base: 'main', + title: `chore(vendor): update ${name} to ${r.upstream}`, body, + }); + applied.push({ ...r, pr: pr.data.number }); + run('git', ['checkout', '--force', baseSha]); + } + + // Summary + const s = core.summary.addHeading('Vendored grammar update monitor'); + if (applied.length) s.addRaw(`\n**Opened PRs:** ${applied.map((a) => `${a.grammar}→${a.upstream} (#${a.pr})`).join(', ')}\n`); + if (held.length) s.addRaw(`\n**Held (not auto-applied):** ${held.map((h) => `${h.grammar} ${h.upstream} (${h.hold ? 'report-only: ' + h.hold : 'ABI ' + (h.abi ?? '?') + ' — needs the tree-sitter runtime upgrade'})`).join(', ')}\n`); + if (skipped.length) s.addRaw(`\n**Skipped:** ${skipped.map((x) => `${x.grammar} (${x.reason})`).join(', ')}\n`); + if (errors.length) s.addRaw(`\n**Errors:** ${errors.map((e) => `${e.grammar}: ${e.error}`).join('; ')}\n`); + if (!applied.length && !held.length && !skipped.length && !errors.length) s.addRaw('\nAll vendored grammars are up to date. ✅\n'); + await s.write(); + + for (const h of held) core.notice(`${h.grammar}: update to ${h.upstream} available — ${h.hold ? `report-only (${h.hold})` : `ABI ${h.abi ?? 'unknown'} (need 13/14), held until the tree-sitter runtime upgrade`}.`); + if (!hasApp && (applied.length || skipped.some((x) => /secret/.test(x.reason)))) { + core.notice('RELEASE_APP_ID / RELEASE_APP_PRIVATE_KEY not configured — update PRs were not opened. Provision the App to enable auto-PRs.'); + } diff --git a/.github/workflows/pr-autofix-apply.yml b/.github/workflows/pr-autofix-apply.yml index 73bf8ceb0..9d437307b 100644 --- a/.github/workflows/pr-autofix-apply.yml +++ b/.github/workflows/pr-autofix-apply.yml @@ -336,7 +336,7 @@ jobs: # Push auth is provided inline at push time via the URL. - name: Checkout PR head if: steps.locate.outputs.found == 'true' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v5.0.4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v5.0.4 with: repository: ${{ steps.locate.outputs.head_repo }} ref: ${{ steps.locate.outputs.head_sha }} diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml index 04eb2468d..a531612f0 100644 --- a/.github/workflows/pr-autofix.yml +++ b/.github/workflows/pr-autofix.yml @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: # PR head commit (not the synthetic merge ref) — we need the # exact tree the contributor pushed so suggestions line up. diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index 0938b5507..2341f4559 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -108,7 +108,7 @@ jobs: # Pinned to v7.2.0. Verify SHA via: # gh api repos/release-drafter/release-drafter/git/refs/tags/v7.2.0 # v7 removed `disable-releaser`; use `dry-run: true` to only autolabel. - - uses: release-drafter/release-drafter@c2e2804cc59f45f57076a99af580d0fedb697927 # v7.3.0 + - uses: release-drafter/release-drafter@693d20e7c1ce1a81d3a41962f85914253b518449 # v7.3.1 with: config-name: release-drafter.yml dry-run: true diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a5265d073..942bb2bd6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -162,7 +162,7 @@ jobs: should_run: ${{ steps.decide.outputs.should_run }} head_sha: ${{ steps.decide.outputs.head_sha }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 fetch-tags: true @@ -332,7 +332,7 @@ jobs: # on the RC path. - name: Checkout (RC) if: needs.route.outputs.mode == 'rc' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: fetch-depth: 0 fetch-tags: true @@ -349,7 +349,7 @@ jobs: - name: Checkout (stable) if: needs.route.outputs.mode == 'stable' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 # No `token:` — actions/checkout uses GITHUB_TOKEN by default. Stable # path performs no git pushes; the default scope is sufficient. with: diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 1de6cd5db..229286daf 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml index 74ce72a27..1eca8861d 100644 --- a/.github/workflows/tree-sitter-upgrade-readiness.yml +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -37,7 +37,7 @@ jobs: # Needed to open/update the tracking issue on scheduled runs. issues: write steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: ./.github/actions/setup-gitnexus with: diff --git a/.github/workflows/triage-sweep.yml b/.github/workflows/triage-sweep.yml index 43d67828d..ea605eb15 100644 --- a/.github/workflows/triage-sweep.yml +++ b/.github/workflows/triage-sweep.yml @@ -59,7 +59,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: sparse-checkout: .github/scripts/triage sparse-checkout-cone-mode: false diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index a8ca6c839..fd7ade8b5 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index b74387116..678ed1a4a 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -31,7 +31,7 @@ jobs: contents: read steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false @@ -53,7 +53,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false diff --git a/AGENTS.md b/AGENTS.md index d4b09c8ce..64d264126 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -173,6 +173,6 @@ npx gitnexus serve # HTTP API on port 4747 (from any ind ### Gotchas -- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift, builds tree-sitter-proto). Native bindings need `python3`, `make`, `g++`. -- `tree-sitter-kotlin` and `tree-sitter-swift` are optional — install warnings expected. +- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (materializes the vendored grammars into `node_modules/`, then prefers a committed prebuild per platform-arch and only source-builds when none matches). A C/C++ toolchain (`python3`, `make`, `g++`) is needed only for that source-build fallback. +- The vendored grammars `tree-sitter-{c,dart,proto,swift,kotlin}` are handled uniformly: c is required; dart/proto/swift/kotlin are optional and skippable via `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`. Install warnings appear only when no prebuild matches the platform-arch and no toolchain is present, and are non-fatal — only that language's parsing is unavailable. - ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 01013c71c..b3319f172 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -204,6 +204,10 @@ Language-agnostic scope-resolution resolver. This is the resolution path for eve Orchestrator: `runScopeResolution(input, provider)` in `scope-resolution/pipeline/run.ts`. Pipeline phase: `scopeResolutionPhase` in `scope-resolution/pipeline/phase.ts` — iterates the registered `SCOPE_RESOLVERS` over the worker-serialized `ParsedFile`s. (Per-language `emitScopeCaptures` hooks may reuse a cached Tree via the orchestrator's `treeCache`, but in worker-pool runs that cache is empty — Trees can't cross MessageChannels — so they consume the pre-extracted `ParsedFile` instead; § Performance notes.) +### Optional CFG/PDG emission (`--pdg`, #2081 M1) + +On a `--pdg` run, the parse worker builds a per-function control-flow graph from the tree-sitter AST (`LanguageProvider.cfgVisitor`; TypeScript/JavaScript in M1) and serializes it onto `ParsedFile.cfgSideChannel` as plain data. Scope-resolution then emits `BasicBlock` nodes + `CFG` edges from that side-channel **inside Phase 4 of `runScopeResolution`, while the disk-backed ParsedFile store is still live** — the only window where the worker-built CFGs are loaded (the store is cleared right after the phase returns). A standalone post-`mro` phase would read an empty store, so the CFG emit deliberately lives in-phase, mirroring the `applyCaptureSideChannel` pattern. The opt-in is off by default (graph byte-identical), folded into the parse-cache key (a pdg-off warm cache is never reused on a `--pdg` run), and bounded by a per-function edge cap that logs any dropped edges. Edge *kind* (`seq`/`cond-true`/`loop-back`/…) rides in the `CFG` relationship's `reason` (CFG is a single `CodeRelation` type, not one type per kind). See `core/ingestion/cfg/`. + ### `ScopeResolver` contract Single interface a language implements to plug into the pipeline. Contract fully documented in `scope-resolution/contract/scope-resolver.ts`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b6298d78..1b34046ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +### Fixed + +- **Hook db-lock probe no longer strands unkillable `lsof`/`ps` orphans** — the probe's `lsof`/`ps` subprocesses are now wrapped in a self-tested coreutils `timeout`/`gtimeout` (`timeout -k 1 …`), so a hook SIGKILLed by the runner's 10s timeout can no longer leave `lsof` running forever (orphan lifetime bounded at ~3s); `acquireHookSlot` now also gates the probe itself, capping concurrent probes at 3 per repo. Opt out with `GITNEXUS_HOOK_TIMEOUT_PATH=disabled`. (#2163) + ### Changed - Migrated from KuzuDB to LadybugDB v0.15 (`@ladybugdb/core`, `@ladybugdb/wasm-core`) - Renamed all internal paths from `kuzu` to `lbug` (storage: `.gitnexus/kuzu` → `.gitnexus/lbug`) diff --git a/Dockerfile.cli b/Dockerfile.cli index cfba9bac4..633d23f5d 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -36,6 +36,17 @@ RUN npm ci --prefix gitnexus # Drop dev dependencies for a smaller runtime layer. RUN npm prune --omit=dev --prefix gitnexus +# `npm prune` removes anything not in package.json's dependency tree — which +# includes the VENDORED tree-sitter grammars (materialized into node_modules/ by +# postinstall, but not declared as deps) and their freshly-built native bindings. +# The `serve` image analyzes/parses uploaded repos at runtime, so those grammars +# must survive into the runtime layer. Re-run the grammar postinstall here in the +# builder (which still has python3/make/g++ and the hoisted node-addon-api / +# node-gyp-build) to re-materialize + rebuild them after the prune. This is +# load-bearing for tree-sitter-c (a core, REQUIRED grammar now vendored, #2116): +# as a former `dependency` it used to survive prune; vendored, it would not. +RUN npm run postinstall --prefix gitnexus + # -- Runtime ----------------------------------------------------------- # node:22-bookworm-slim FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime @@ -89,6 +100,22 @@ ENV HOME=/home/node \ RUN su node -s /bin/sh -c "HOME=/home/node node /app/gitnexus/scripts/install-duckdb-extension.mjs fts" \ && su node -s /bin/sh -c "HOME=/home/node node /app/gitnexus/scripts/install-duckdb-extension.mjs fts --verify-only" +# Published runtime assets (in package.json `files`). Placed AFTER the DuckDB +# FTS-extension RUN above so editing hook/skill content does not invalidate that +# network-fetching cache layer; they have no input dependency on it. +# `hooks/`: dist/cli/resolve-invocation.js does +# `require('../../hooks/claude/resolve-analyze-cmd.cjs')` at module load — the +# single source of truth for the npm-11 npx-crash invocation decision (#1939). +# Without it, `gitnexus analyze` inside the image crashes with MODULE_NOT_FOUND +# before it does any work (#2130). `skills/`: the CLI reads the bundled SKILL.md +# templates from `/skills/` for `gitnexus analyze --skills` and `gitnexus +# setup`/`uninstall`; absent, those degrade silently (placeholder content / zero +# skills installed). (The web UI bundle `web/`, also in `files`, is deliberately +# NOT shipped: this builder never builds gitnexus-web, so the image is API-only; +# the UI is the separate Dockerfile.web image / hosted app.) +COPY --from=builder --chown=node:node /app/gitnexus/hooks ./gitnexus/hooks +COPY --from=builder --chown=node:node /app/gitnexus/skills ./gitnexus/skills + USER node # The web UI defaults to http://localhost:4747 - keep that contract. diff --git a/README.md b/README.md index 44876243c..aea1079ee 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,9 @@ That's it. This indexes the codebase, installs agent skills, registers Claude Co To configure MCP for your editor, run `npx gitnexus setup` once — or set it up manually below. -> **Faster install (no C++ toolchain needed):** set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, and `tree-sitter-swift` — those three won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild. This variable does **not** control `tree-sitter-kotlin` (a third-party npm `optionalDependency` that npm compiles via its own `node-gyp-build` step regardless); to skip the Kotlin compile too, add `npm install --omit=optional` — which also drops the `node-gyp-build`/`node-addon-api` build deps and so disables the vendored builds as well. See the `tree-sitter-kotlin` note below. +> **Faster install (no C++ toolchain needed):** set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before `npm install -g gitnexus` to skip the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` — those four won't be parsed, but install completes in seconds without `python3`/`make`/`g++`. Strict `=1` only — any other value falls through to the rebuild. See the `tree-sitter-kotlin` note below. > -> **About `tree-sitter-kotlin`:** unlike the vendored grammars, Kotlin support comes from a third-party npm `optionalDependency` that ships **source only** (no upstream prebuilt binaries) and compiles via node-gyp at install time. On a host without a C/C++ toolchain its native build soft-fails: npm skips the optional dependency, the `gitnexus` install still **succeeds**, and only Kotlin (`.kt`/`.kts`) parsing is unavailable. An install-time probe surfaces a single clear warning when the binding is missing (suppressed only if you opted out with `--omit=optional`), instead of leaving raw node-gyp output as the only signal, and it honors `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`. (GitNexus does **not** yet ship prebuilt Kotlin binaries. GitNexus already vendors its own self-built Swift prebuilds and could do the same for Kotlin — that's deferred Swift-parity follow-up work tracked in [#2107](https://github.com/abhigyanpatwari/GitNexus/issues/2107), not an upstream blocker.) +> **About `tree-sitter-kotlin`:** like Dart/Proto/Swift, Kotlin is a **vendored** grammar (under `gitnexus/vendor/tree-sitter-kotlin`). Upstream `tree-sitter-kotlin` ships **source only** (no prebuilt binaries), so GitNexus builds the Kotlin platform prebuilds itself (via the `build-tree-sitter-prebuilds` GitHub Actions workflow) and vendors them — the same uniform pipeline now used for Dart, Proto, and Swift (Swift's prebuilds were originally copied from upstream; they're now GitNexus-cross-built too). `node-gyp-build` selects the right `.node` at require time, so **no C/C++ toolchain is needed**. If no prebuild matches your platform-arch, only Kotlin (`.kt`/`.kts`) parsing is unavailable; the rest of `gitnexus` is unaffected. ### MCP Setup @@ -225,6 +225,7 @@ args = ["-y", "gitnexus@latest", "mcp"] ```bash gitnexus setup # Configure MCP for your editors (one-time) +gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks (add --force to apply) gitnexus analyze [path] # Index a repository (or update stale index) gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild @@ -261,6 +262,8 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +> **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. + If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. #### Embeddings node limit @@ -330,7 +333,7 @@ Most `analyze` knobs are also CLI flags (`--workers`, `--worker-timeout`, `--max | `GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD`| `max(3, poolSize)` | Per-slot consecutive deaths before the pool's circuit breaker trips. After tripping, every subsequent dispatch rejects until a fresh pool is created. | Hosts where a SIGSEGV-prone native grammar should trip the breaker sooner; CI runners that should fail loudly. | | `GITNEXUS_CHUNK_BYTE_BUDGET` | `2097152` (2 MB) | Chunk boundary used for cache-key composition and dispatch. Smaller = finer-grained cache hits but more dispatch overhead. | Tuning incremental-analyze cache behavior on monorepos. | | `GITNEXUS_NO_GITIGNORE` | unset | When set, skips `.gitignore` parsing. `.gitnexusignore` is still honored. | Indexing a repo whose `.gitignore` excludes files you actually want indexed (e.g., generated code committed for cross-repo lookup). | -| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize/build for `tree-sitter-dart`, `tree-sitter-proto`, and `tree-sitter-swift` at install time, and silences GitNexus's `tree-sitter-kotlin` probe. It does **not** stop npm from compiling `tree-sitter-kotlin` (a third-party `optionalDependency` with its own `node-gyp-build` step) — use `npm install --omit=optional` to skip that compile too. Without a toolchain the Kotlin build soft-fails, npm skips it, the install still succeeds, and only Kotlin parsing is lost. | Installing on a host without a C++ toolchain or where Swift prebuilds don't match; willing to skip Dart/Proto/Swift parsing (and, with `--omit=optional`, Kotlin). | +| `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` | unset | When `=1` strictly, skips the vendored grammar materialize for `tree-sitter-dart`, `tree-sitter-proto`, `tree-sitter-swift`, and `tree-sitter-kotlin` at install time (and the Dart/Proto source builds). Those four won't be parsed; the install still succeeds. | Installing on a host without a C++ toolchain or where the vendored prebuilds don't match; willing to skip Dart/Proto/Swift/Kotlin parsing. | #### Publishing to understand-quickly (opt-in) @@ -344,7 +347,7 @@ It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained G | Tool | What It Does | `repo` Param | | ----------------- | ---------------------------------------------------------------- | ------------ | -| `list_repos` | Discover all indexed repositories | — | +| `list_repos` | Discover all indexed repositories (paginated — `limit`/`offset`) | — | | `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | | `context` | 360-degree symbol view — categorized refs, process participation | Optional | | `impact` | Blast radius analysis with depth grouping and confidence | Optional | @@ -700,6 +703,8 @@ GitNexus builds a complete knowledge graph of your codebase through a multi-phas **Imports** — cross-file import resolution · **Named Bindings** — `import { X as Y }` / re-export tracking · **Exports** — public/exported symbol detection · **Heritage** — class inheritance, interfaces, mixins · **Type Annotations** — explicit type extraction for receiver resolution · **Constructor Inference** — infer receiver type from constructor calls (`self`/`this` resolution included for all languages) · **Config** — language toolchain config parsing (tsconfig, go.mod, etc.) · **Frameworks** — AST-based framework pattern detection · **Entry Points** — entry point scoring heuristics +**Control flow (CFG, opt-in `--pdg`)** — per-function control-flow graphs (`BasicBlock` nodes + `CFG` edges) feeding the PDG/taint substrate, currently **TypeScript & JavaScript** (#2081 M1); other languages planned. Off by default. + --- ## Tool Examples diff --git a/gitnexus-claude-plugin/.claude-plugin/plugin.json b/gitnexus-claude-plugin/.claude-plugin/plugin.json index da6b42a7b..fe3e641ac 100644 --- a/gitnexus-claude-plugin/.claude-plugin/plugin.json +++ b/gitnexus-claude-plugin/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "gitnexus", "description": "Code intelligence powered by a knowledge graph. Provides execution flow tracing, blast radius analysis, and augmented search across your codebase.", - "version": "1.6.6", + "version": "1.6.7", "author": { "name": "GitNexus" }, diff --git a/gitnexus-claude-plugin/hooks/gitnexus-hook.js b/gitnexus-claude-plugin/hooks/gitnexus-hook.js index e3c62c769..5274ef4c0 100644 --- a/gitnexus-claude-plugin/hooks/gitnexus-hook.js +++ b/gitnexus-claude-plugin/hooks/gitnexus-hook.js @@ -110,10 +110,20 @@ function hasGitNexusServerOwner(gitNexusDir) { return hasGitNexusDbLockedByGitNexusServer(path.join(gitNexusDir, 'lbug'), process.pid); } +/** + * Whether opt-in diagnostics should be written to the hook's stderr. Strict + * hook runners (e.g. Codex `PreToolUse`) validate hook output, so normal, + * non-error skip paths must stay silent unless the operator explicitly asks + * for diagnostics via GITNEXUS_DEBUG. See issue #1913. + */ +function isDebugEnabled() { + return process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; +} + function extractAugmentContext(stderr) { const output = (stderr || '').trim(); const marker = output.indexOf('[GitNexus]'); - const debug = process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; + const debug = isDebugEnabled(); if (debug && output.length > 0) { // Emit the FULL discarded prefix (everything before the marker, or all of // it when no marker is present) so suppressed diagnostics — LadybugDB lock @@ -266,16 +276,34 @@ function handlePreToolUse(input) { const pattern = extractPattern(toolName, toolInput); if (!pattern || pattern.length < 3) return; - if (hasGitNexusServerOwner(gitNexusDir)) { - process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + + // Acquire the per-repo slot BEFORE the DB-owner probe (#2163): the probe + // itself spawns lsof/ps, so it must be bounded by the same ≤3-per-repo cap + // as the augment, or concurrent sessions fan out unbounded probe + // subprocesses. Keep the acquire right after the cheap guards above — + // moving it earlier would churn slot files on tool calls that never probe. + const release = acquireHookSlot(gitNexusDir); + if (!release) { + // Normal skip path: all per-repo hook slots are held by concurrent + // sessions. Stay silent for strict hook runners (issue #1913); surface + // the reason only when diagnostics are explicitly requested. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: hook slots saturated\n'); + } return; } - const release = acquireHookSlot(gitNexusDir); - if (!release) return; - let result = ''; try { + if (hasGitNexusServerOwner(gitNexusDir)) { + // Normal skip path: the MCP server owns the DB, so the CLI augment would + // contend on the lock. Stay silent for strict hook runners (issue #1913); + // surface the reason only when diagnostics are explicitly requested. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + } + return; + } const child = runGitNexusCli(['augment', '--', pattern], cwd, 7000); if (!child.error && child.status === 0) { result = extractAugmentContext(child.stderr || ''); @@ -366,7 +394,7 @@ function main() { const handler = handlers[input.hook_event_name || '']; if (handler) handler(input); } catch (err) { - if (process.env.GITNEXUS_DEBUG) { + if (isDebugEnabled()) { console.error('GitNexus hook error:', (err.message || '').slice(0, 200)); } } diff --git a/gitnexus-claude-plugin/hooks/hook-db-lock-probe.cjs b/gitnexus-claude-plugin/hooks/hook-db-lock-probe.cjs index 752c114a7..5c67804b1 100644 --- a/gitnexus-claude-plugin/hooks/hook-db-lock-probe.cjs +++ b/gitnexus-claude-plugin/hooks/hook-db-lock-probe.cjs @@ -11,6 +11,22 @@ * * Fail-open on most errors; fail-closed only on lsof ETIMEDOUT (Unix) or * PowerShell ETIMEDOUT (Windows), matching the hook contract. + * + * Unix subprocess containment contract (#2163): + * - lsof/ps are wrapped in coreutils `timeout`/`gtimeout` when a working + * wrapper is found (`timeout -k 1 lsof ...`). If this hook process + * is itself SIGKILLed (e.g. by the runner's 10s hook timeout) the wrapper + * survives, SIGTERMs its child at the budget (2s lsof / 1s ps) and SIGKILLs + * it 1s later — orphan lifetime is bounded at ~3s instead of unbounded. + * - GITNEXUS_HOOK_TIMEOUT_PATH: the sentinel value `disabled` switches the + * wrapper off deterministically; any other value is adopted only when it + * exists AND passes a one-shot `-k` self-test — otherwise resolution FALLS + * THROUGH to the built-in candidate list (first self-test pass wins), so + * no malformed value of any shape can silently disable orphan containment. + * - The gitnexus server is lazy-open + sticky-hold: an idle MCP server holds + * ZERO lbug fds until the repo's first MCP query, then keeps the fd open. + * A probe before that first query is therefore always false — a known, + * pre-existing race, not a bug in this probe. */ const fs = require('fs'); @@ -46,6 +62,80 @@ function resolveHookBinary(tool) { return tool; } +// Sentinel: +// undefined = not resolved yet (resolve lazily, on first lsof/ps fallback) +// string = self-tested coreutils timeout/gtimeout path (use as wrapper) +// null = no usable wrapper (disabled, none found, or self-test failed) +let unixGuardTimeoutCache; + +/** + * Resolve a coreutils `timeout`/`gtimeout` binary to wrap lsof/ps with + * (#2163). Dead code on Windows (the win32 dispatch returns earlier). + * + * GITNEXUS_HOOK_TIMEOUT_PATH semantics: the sentinel `disabled` turns the + * wrapper off; any other value is only a CANDIDATE — an existing file path + * is tried first, but it must pass the `-k` self-test to be adopted. On any + * failure (non-existent path, directory, non-executable file, wrapper + * without `-k` support, …) resolution falls through to the built-in + * candidates below, tried in order, first self-test pass wins. This is + * strictly stronger than the sibling GITNEXUS_HOOK_LSOF_PATH / + * GITNEXUS_HOOK_PS_PATH overrides (which only check existence): no bad env + * value of ANY shape can silently disable orphan containment. + * + * Lazy self-test: candidates are probed only when the lsof/ps fallback is + * first reached, and the result is memoized. A candidate is adopted only + * when `timeout -k 1 1 /bin/sh -c :` exits 0. This rejects wrappers that do + * not support the coreutils `-k` flag — busybox <1.34, toybox, broken + * symlinks — which would otherwise exit with a usage error without ever + * running lsof, silently converting the lsof-ETIMEDOUT fail-closed contract + * into fail-open (#1492 regression). Only when EVERY candidate fails does + * the probe fall back to the unwrapped status quo (memoized null). + * busybox ≥1.34 passes the test and is fully usable (capability, not + * identity, decides). + */ +function passesGuardSelfTest(guard) { + try { + const selfTest = spawnSync(guard, ['-k', '1', '1', '/bin/sh', '-c', ':'], { + encoding: 'utf-8', + timeout: 3000, + stdio: ['ignore', 'ignore', 'ignore'], + windowsHide: true, + }); + return !selfTest.error && selfTest.status === 0; + } catch { + return false; + } +} + +function resolveUnixGuardTimeout() { + if (unixGuardTimeoutCache !== undefined) return unixGuardTimeoutCache; + unixGuardTimeoutCache = null; + const fromEnv = process.env.GITNEXUS_HOOK_TIMEOUT_PATH; + const trimmed = fromEnv ? String(fromEnv).trim() : ''; + if (trimmed === 'disabled') return unixGuardTimeoutCache; + const candidates = []; + if (trimmed && fs.existsSync(trimmed)) candidates.push(trimmed); + for (const builtin of [ + '/usr/bin/timeout', + '/bin/timeout', + '/opt/homebrew/bin/gtimeout', + '/usr/local/bin/gtimeout', + ]) { + try { + if (fs.existsSync(builtin)) candidates.push(builtin); + } catch { + /* ignore */ + } + } + for (const candidate of candidates) { + if (passesGuardSelfTest(candidate)) { + unixGuardTimeoutCache = candidate; + break; + } + } + return unixGuardTimeoutCache; +} + function resolveWindowsPowerShellPath() { const fromEnv = process.env.GITNEXUS_HOOK_POWERSHELL_PATH; if (fromEnv && String(fromEnv).trim() && fs.existsSync(String(fromEnv).trim())) { @@ -188,20 +278,46 @@ function linuxProcScanFindGitNexusServer(dbPathAbs, myPid) { } function unixLsofPsFindGitNexusServer(dbPathAbs, myPid) { + const guard = resolveUnixGuardTimeout(); const lsofPath = resolveHookBinary('lsof'); - const lsof = spawnSync(lsofPath, ['-nP', '-t', '--', dbPathAbs], { + // The spawnSync timeouts below (lsof 1000ms / ps 500ms) are deliberately + // SHORTER than the wrapper budgets (2s / 1s): on the supervised path Node's + // SIGTERM always fires first, so `error.code === 'ETIMEDOUT'` and the + // fail-closed contract are untouched. The wrapper only matters once this + // hook process has been SIGKILLed and can no longer deliver that SIGTERM. + const [lsofCmd, lsofArgs] = guard + ? [guard, ['-k', '1', '2', lsofPath, '-nP', '-t', '--', dbPathAbs]] + : [lsofPath, ['-nP', '-t', '--', dbPathAbs]]; + const lsof = spawnSync(lsofCmd, lsofArgs, { encoding: 'utf-8', timeout: 1000, stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true, }); if (lsof.error) return lsof.error.code === 'ETIMEDOUT'; + // Guard-mediated deaths map to "unresponsive holder" (fail-closed). Three + // result shapes, verified against coreutils 9.1: + // - signal-death: when `-k` escalates to SIGKILL, coreutils timeout + // SELF-RAISES the signal, so spawnSync reports {status: null, signal} + // with no .error (spawnSync's own ETIMEDOUT was handled above). The + // same shape appears when this hook is frozen >2s (SIGSTOP, laptop + // suspend) and the guard expires while it sleeps. By construction, a + // guard-wrapped probe that died by signal without spawnSync ETIMEDOUT + // is a budget/kill outcome. + // - 124: budget expired and the child exited after the plain SIGTERM. + // - 137: NOT the coreutils -k path — only exit-code-propagating wrappers, + // or a child SIGKILLed externally (e.g. the OOM killer). + if (guard && lsof.status === null && lsof.signal) return true; + if (guard && (lsof.status === 124 || lsof.status === 137)) return true; const pids = (lsof.stdout || '').split(/\s+/).filter(Boolean); const psPath = resolveHookBinary('ps'); for (const pid of pids) { if (Number(pid) === myPid) continue; - const ps = spawnSync(psPath, ['-p', pid, '-o', 'command='], { + const [psCmd, psArgs] = guard + ? [guard, ['-k', '1', '1', psPath, '-p', pid, '-o', 'command=']] + : [psPath, ['-p', pid, '-o', 'command=']]; + const ps = spawnSync(psCmd, psArgs, { encoding: 'utf-8', timeout: 500, stdio: ['ignore', 'pipe', 'ignore'], @@ -211,6 +327,11 @@ function unixLsofPsFindGitNexusServer(dbPathAbs, myPid) { if (ps.error.code === 'ETIMEDOUT') return true; continue; } + // Same guard-mediated-death mapping as the lsof call above (signal-death + // from the -k escalation or a frozen hook; 124 budget expiry; 137 only + // for exit-code-propagating wrappers / external SIGKILL). + if (guard && ps.status === null && ps.signal) return true; + if (guard && (ps.status === 124 || ps.status === 137)) return true; if (isGitNexusServerCommand(ps.stdout || '')) return true; } return false; diff --git a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md index b81900b5e..cacc4e886 100644 --- a/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md +++ b/gitnexus-claude-plugin/skills/gitnexus-guide/SKILL.md @@ -38,7 +38,38 @@ For any task involving code understanding, debugging, impact analysis, or refact | `detect_changes` | Git-diff impact — what do your current changes affect | | `rename` | Multi-file coordinated rename with confidence-tagged edits | | `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | +| `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | + +### Paginating `list_repos` + +`list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: + +```jsonc +{ + "repositories": [ + { "name": "...", "path": "...", "indexedAt": "...", "lastCommit": "...", "stats": { } } + ], + "pagination": { + "total": 437, + "limit": 50, + "offset": 0, + "returned": 50, + "hasMore": true, + "nextOffset": 50 + } +} +``` + +To enumerate **every** repository, keep calling with `offset` set to `pagination.nextOffset` until `hasMore` is `false`: + +```text +list_repos {} → repos 1–50, nextOffset 50, hasMore true +list_repos { offset: 50 } → repos 51–100, nextOffset 100, hasMore true +… +list_repos { offset: 400 } → repos 401–437, hasMore false (done) +``` + +Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. ## Resources Reference diff --git a/gitnexus-shared/src/scope-resolution/parsed-file.ts b/gitnexus-shared/src/scope-resolution/parsed-file.ts index 98a327ad4..01b70f206 100644 --- a/gitnexus-shared/src/scope-resolution/parsed-file.ts +++ b/gitnexus-shared/src/scope-resolution/parsed-file.ts @@ -98,4 +98,26 @@ export interface ParsedFile { * side effects — the contract default) leave this undefined. */ readonly captureSideChannel?: unknown; + + /** + * Per-function control-flow graphs for this file (#2081 M1, PDG/taint + * substrate). A DISTINCT field from {@link captureSideChannel} — different + * producer, consumer, and lifecycle: the worker builds it from the + * tree-sitter AST via `LanguageProvider.cfgVisitor` (only on a `--pdg` run), + * and scope-resolution emits BasicBlock nodes + CFG edges from it while the + * disk-backed ParsedFile store is still live (it is NOT a capture-time + * marker the resolver restores into module maps). Kept separate so a future + * change to either channel's shape invalidates independently. + * + * Shared / ingestion code treats this as opaque (`unknown`) per AGENTS.md. + * Concretely it is a `readonly FunctionCfg[]` (see + * `core/ingestion/cfg/types.ts`) — plain JSON-serializable data (no AST + * refs, no class instances) so it round-trips through the parse cache and + * the `parsedfile-store` (whose interning reviver keys on `nodeId`, which + * these blocks/edges deliberately lack). + * + * Optional: `undefined` on non-`--pdg` runs and for languages with no + * `cfgVisitor` — the default for every run today. + */ + readonly cfgSideChannel?: unknown; } diff --git a/gitnexus-shared/src/scope-resolution/symbol-definition.ts b/gitnexus-shared/src/scope-resolution/symbol-definition.ts index 06814db3c..3bcdb43eb 100644 --- a/gitnexus-shared/src/scope-resolution/symbol-definition.ts +++ b/gitnexus-shared/src/scope-resolution/symbol-definition.ts @@ -57,6 +57,10 @@ export interface SymbolDefinition { * Currently used by C++ overload ranking to exclude explicit constructors * from implicit user-defined conversion candidates. */ isExplicit?: boolean; + /** True when the callable is declared unavailable (for example C++ `= delete`). + * Unavailable callables still participate in overload selection, but a + * selected unavailable target must suppress edge emission. */ + isDeleted?: boolean; /** Links Method/Constructor/Property to owning Class/Struct/Trait nodeId */ ownerId?: string; /** #1982/#1993: bridge-held enclosing-namespace path (e.g. `NS1`, `Outer.Inner`) diff --git a/gitnexus-web/e2e/folder-upload.spec.ts b/gitnexus-web/e2e/folder-upload.spec.ts new file mode 100644 index 000000000..ba31b752c --- /dev/null +++ b/gitnexus-web/e2e/folder-upload.spec.ts @@ -0,0 +1,112 @@ +import { test, expect } from '@playwright/test'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +/** + * E2E for the browser folder-upload flow (replaces the removed server-side + * directory picker). Mocks the backend so no live gitnexus server is needed. + */ + +const BACKEND_URL = 'http://localhost:4747'; + +let fixtureDir: string; + +test.beforeAll(() => { + // A tiny "repo" folder; Playwright sets webkitRelativePath = /. + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-upload-e2e-')); + fixtureDir = path.join(root, 'myrepo'); + fs.mkdirSync(path.join(fixtureDir, 'src'), { recursive: true }); + fs.writeFileSync(path.join(fixtureDir, 'README.md'), '# hi\n'); + fs.writeFileSync(path.join(fixtureDir, 'src', 'index.ts'), 'export const x = 1;\n'); +}); + +test.beforeEach(async ({ page }) => { + await page.route(`${BACKEND_URL}/api/repos`, (route) => route.fulfill({ json: [] })); + await page.route(`${BACKEND_URL}/api/info`, (route) => + route.fulfill({ json: { version: '1.0.0', launchContext: 'npx', nodeVersion: 'v22.0.0' } }), + ); + await page.route(`${BACKEND_URL}/api/heartbeat`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream' }, + body: ':ok\n\n', + }), + ); +}); + +test('uploading a folder posts a multipart upload and starts analysis', async ({ page }) => { + let uploadContentType = ''; + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + uploadContentType = route.request().headers()['content-type'] ?? ''; + await route.fulfill({ json: { jobId: 'job-e2e', status: 'analyzing' } }); + }); + // SSE progress → immediately complete. + await page.route(`${BACKEND_URL}/api/analyze/job-e2e/progress`, (route) => + route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }), + ); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); + + // Select the fixture folder via the hidden webkitdirectory input. + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + + // The upload endpoint should be hit with a multipart body, and the UI should + // leave the input phase (upload button no longer shown). + await expect.poll(() => uploadContentType).toContain('multipart/form-data'); + await expect(page.locator('[data-testid="upload-folder"]')).toBeHidden({ timeout: 10_000 }); +}); + +test('switching modes mid-upload aborts it and never shows progress', async ({ page }) => { + // Hold the upload response until the test releases it, so the mode switch + // happens while the POST is in flight (the review 4470339833 repro). + let releaseUpload!: () => void; + const uploadGate = new Promise((res) => (releaseUpload = res)); + let uploadAborted = false; + let progressOpened = false; + + // The client-side AbortController kills the POST at mode-switch time; that + // surfaces as a failed request (net::ERR_ABORTED), not as a response. + page.on('requestfailed', (req) => { + if (req.url().includes('/api/analyze/upload') && /ABORTED/.test(req.failure()?.errorText ?? '')) + uploadAborted = true; + }); + await page.route(`${BACKEND_URL}/api/analyze/upload`, async (route) => { + await uploadGate; + await route.fulfill({ json: { jobId: 'job-stale', status: 'analyzing' } }).catch(() => {}); // the request may already be gone — that's the point + }); + await page.route(`${BACKEND_URL}/api/analyze/job-stale/progress`, (route) => { + progressOpened = true; + return route.fulfill({ + status: 200, + headers: { 'Content-Type': 'text/event-stream', 'Cache-Control': 'no-cache' }, + body: 'event: complete\ndata: {"repoName":"myrepo"}\n\n', + }); + }); + + await page.goto('/'); + await expect(page.getByRole('tab', { name: 'Local Folder' })).toBeVisible({ timeout: 20_000 }); + await page.getByRole('tab', { name: 'Local Folder' }).click(); + await page.locator('[data-testid="folder-upload-input"]').setInputFiles(fixtureDir); + await expect(page.locator('[data-testid="upload-progress"]')).toBeVisible(); + + // Switch back to GitHub while the upload POST is still pending, then let + // the (now-stale) route handler finish. + await page.getByRole('tab', { name: 'GitHub URL' }).click(); + await expect.poll(() => uploadAborted, { timeout: 10_000 }).toBe(true); + releaseUpload(); + + // The GitHub form stays clean (no error, immediately usable), and no SSE + // progress stream is ever opened by the stale upload. + await expect(page.getByPlaceholder('https://github.com/owner/repo')).toBeEditable(); + await expect(page.locator('[data-testid="upload-progress"]')).toBeHidden(); + expect(progressOpened).toBe(false); +}); diff --git a/gitnexus-web/e2e/onboarding.spec.ts b/gitnexus-web/e2e/onboarding.spec.ts index 5b70899c6..147f55632 100644 --- a/gitnexus-web/e2e/onboarding.spec.ts +++ b/gitnexus-web/e2e/onboarding.spec.ts @@ -218,8 +218,8 @@ test.describe('Flow 3: Analyze form', () => { // Switch to Local Folder tab await page.getByRole('tab', { name: 'Local Folder' }).click(); - // Browse button should be visible - await expect(page.getByText('Browse for folder')).toBeVisible(); + // Upload-a-folder button should be visible (browser folder upload) + await expect(page.locator('[data-testid="upload-folder"]')).toBeVisible(); await page.screenshot({ path: testInfo.outputPath('local-folder-tab.png') }); }); diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index a69275159..e4efc3aa2 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -18,7 +18,7 @@ "@tailwindcss/vite": "^4.3.0", "axios": "^1.16.1", "d3": "^7.9.0", - "dompurify": "^3.4.7", + "dompurify": "^3.4.8", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -28,7 +28,7 @@ "graphology-utils": "^2.3.0", "i18next": "^26.3.0", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.4.2", + "langchain": "^1.4.4", "lru-cache": "^11.2.4", "lucide-react": "^1.16.0", "mermaid": "^11.15.0", @@ -41,7 +41,7 @@ "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", "remark-gfm": "^4.0.1", - "sigma": "^3.0.2", + "sigma": "^3.0.3", "tailwindcss": "^4.2.4", "uuid": "^14.0.0", "zod": "^4.4.3" @@ -57,9 +57,9 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.8.8", + "@vercel/node": "^5.8.12", "@vitejs/plugin-react": "^5.1.4", - "@vitest/coverage-v8": "^4.1.5", + "@vitest/coverage-v8": "^4.1.8", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", @@ -2930,9 +2930,9 @@ } }, "node_modules/@vercel/build-utils": { - "version": "13.26.4", - "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-13.26.4.tgz", - "integrity": "sha512-0g3ZxtZUJZbt4y0Vu4pkHtu1UN58FbVF9cqGT8T6jHp0EHdLGFj5TVCiME8ALeK4tjPImSmxnKZvvB5yb2hqEw==", + "version": "13.27.1", + "resolved": "https://registry.npmjs.org/@vercel/build-utils/-/build-utils-13.27.1.tgz", + "integrity": "sha512-BD9H2U8I/IPGS1c1stSIkdPxBRu6bkCQFqtRjcT2dcdnBawyHXvzTsnnBQhgB3fJVQtgJT47gVZe1oHDmv0Ktg==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2949,9 +2949,9 @@ "license": "MIT" }, "node_modules/@vercel/error-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@vercel/error-utils/-/error-utils-2.1.0.tgz", - "integrity": "sha512-DiJcXBOB9N6QM4d7hYPM9Ck/AUjzBl58XNQPxS74o7CuvIanjzrGgygP/70VsyEASeIJMazk1LrhwcNTR/eZGQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@vercel/error-utils/-/error-utils-2.2.0.tgz", + "integrity": "sha512-WFWiRxfPzoYWYifaj4thSKvAaZZwUOqD4k5GINRIgZgCiS2E3iAJbWbIsIZmkQdTecWFHcWGA6q48CjisgpOBA==", "dev": true, "license": "Apache-2.0" }, @@ -2983,9 +2983,9 @@ } }, "node_modules/@vercel/node": { - "version": "5.8.8", - "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.8.8.tgz", - "integrity": "sha512-+uRT9evnGWUE6klrJJED4fCvlSxNShbIc/UY4FeUzt2sdcy5a5b1IoYlo94RJd7tAY9Jg2lR2cVfGfsnWH81ZA==", + "version": "5.8.12", + "resolved": "https://registry.npmjs.org/@vercel/node/-/node-5.8.12.tgz", + "integrity": "sha512-XK2ML9YVdAlZ3BmGTW4jQL0D55ZHeRWKS+CLPSWReDyOBKaC4tTnTL2tp3z76bAs0pfgbT55nplMiX2mneSbLA==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2993,8 +2993,8 @@ "@edge-runtime/primitives": "4.1.0", "@edge-runtime/vm": "3.2.0", "@types/node": "20.11.0", - "@vercel/build-utils": "13.26.4", - "@vercel/error-utils": "2.1.0", + "@vercel/build-utils": "13.27.1", + "@vercel/error-utils": "2.2.0", "@vercel/nft": "1.10.0", "@vercel/static-config": "3.4.0", "async-listen": "3.0.0", @@ -3101,14 +3101,14 @@ } }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.5.tgz", - "integrity": "sha512-38C0/Ddb7HcRG0Z4/DUem8x57d2p9jYgp18mkaYswEOQBGsI1CG4f/hjm0ZCeaJfWhSZ4k7jgs29V1Zom7Ki9A==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.8.tgz", + "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.5", + "@vitest/utils": "4.1.8", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -3122,8 +3122,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.5", - "vitest": "4.1.5" + "@vitest/browser": "4.1.8", + "vitest": "4.1.8" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -3132,16 +3132,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz", - "integrity": "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", + "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -3150,13 +3150,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.5.tgz", - "integrity": "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", + "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.5", + "@vitest/spy": "4.1.8", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3187,9 +3187,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.5.tgz", - "integrity": "sha512-7I3q6l5qr03dVfMX2wCo9FxwSJbPdwKjy2uu/YPpU3wfHvIL4QHwVRp57OfGrDFeUJ8/8QdfBKIV12FTtLn00g==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", + "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", "dev": true, "license": "MIT", "dependencies": { @@ -3200,13 +3200,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.5.tgz", - "integrity": "sha512-2D+o7Pr82IEO46YPpoA/YU0neeyr6FTerQb5Ro7BUnBuv6NQtT/kmVnczngiMEBhzgqz2UZYl5gArejsyERDSQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", + "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.5", + "@vitest/utils": "4.1.8", "pathe": "^2.0.3" }, "funding": { @@ -3214,14 +3214,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.5.tgz", - "integrity": "sha512-zypXEt4KH/XgKGPUz4eC2AvErYx0My5hfL8oDb1HzGFpEk1P62bxSohdyOmvz+d9UJwanI68MKwr2EquOaOgMQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", + "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/pretty-format": "4.1.8", + "@vitest/utils": "4.1.8", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3230,9 +3230,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.5.tgz", - "integrity": "sha512-2lNOsh6+R2Idnf1TCZqSwYlKN2E/iDlD8sgU59kYVl+OMDmvldO1VDk39smRfpUNwYpNRVn3w4YfuC7KfbBnkQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", + "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", "dev": true, "license": "MIT", "funding": { @@ -3240,13 +3240,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.5.tgz", - "integrity": "sha512-76wdkrmfXfqGjueGgnb45ITPyUi1ycZ4IHgC2bhPDUfWHklY/q3MdLOAB+TF1e6xfl8NxNY0ZYaPCFNWSsw3Ug==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", + "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.5", + "@vitest/pretty-format": "4.1.8", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -4461,9 +4461,9 @@ "peer": true }, "node_modules/dompurify": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.7.tgz", - "integrity": "sha512-2jBxDJY4RR06tQNy4w5FlFH7kfxsQZlufd0sbv+chfHCxeJwrFw2baUDsSwvBISD4K4RDbd0PTfy3uNXsR6siA==", + "version": "3.4.8", + "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.8.tgz", + "integrity": "sha512-yb1cEmaOum7wFvOCSQxyfgVlv5D47Rc30iZWoMpbDIWTnJ6grDDQyu2KFJzB2k7u0pMuJcQ1zphH//fFnw2tjQ==", "license": "(MPL-2.0 OR Apache-2.0)", "optionalDependencies": { "@types/trusted-types": "^2.0.7" @@ -5752,9 +5752,9 @@ "integrity": "sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==" }, "node_modules/langchain": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.4.2.tgz", - "integrity": "sha512-SLGipy0r4nqQD0aiUOBYLMeGFfB/QiYnMndfZ8sGN89vXDCIXbYqcE7G/4QDDX3nZsM7/emQpoScmlxEX6sDnQ==", + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.4.4.tgz", + "integrity": "sha512-tepOCwUDaIZOYJ9Eo0O6o5dXEN/0KJheiFDnHHFL8Tx8rfkDLL4cOTSTln4Vpn9LpWzXYkjQ8lkHnnNDQWZPeg==", "license": "MIT", "dependencies": { "@langchain/langgraph": "^1.3.2", @@ -8153,9 +8153,9 @@ "license": "ISC" }, "node_modules/sigma": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/sigma/-/sigma-3.0.2.tgz", - "integrity": "sha512-/BUbeOwPGruiBOm0YQQ6ZMcLIZ6tf/W+Jcm7dxZyAX0tK3WP9/sq7/NAWBxPIxVahdGjCJoGwej0Gdrv0DxlQQ==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sigma/-/sigma-3.0.3.tgz", + "integrity": "sha512-5H0zFlx6/NTQpqBg4Rm569ZOpnBOXMaS25UQThIWMU3XyzI5AhmorK/gnl87BvJBLhQd0tW4C0LIp3enWzMoNw==", "license": "MIT", "dependencies": { "events": "^3.3.0", @@ -8822,19 +8822,19 @@ } }, "node_modules/vitest": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.5.tgz", - "integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", + "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.5", - "@vitest/mocker": "4.1.5", - "@vitest/pretty-format": "4.1.5", - "@vitest/runner": "4.1.5", - "@vitest/snapshot": "4.1.5", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/expect": "4.1.8", + "@vitest/mocker": "4.1.8", + "@vitest/pretty-format": "4.1.8", + "@vitest/runner": "4.1.8", + "@vitest/snapshot": "4.1.8", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -8862,12 +8862,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.5", - "@vitest/browser-preview": "4.1.5", - "@vitest/browser-webdriverio": "4.1.5", - "@vitest/coverage-istanbul": "4.1.5", - "@vitest/coverage-v8": "4.1.5", - "@vitest/ui": "4.1.5", + "@vitest/browser-playwright": "4.1.8", + "@vitest/browser-preview": "4.1.8", + "@vitest/browser-webdriverio": "4.1.8", + "@vitest/coverage-istanbul": "4.1.8", + "@vitest/coverage-v8": "4.1.8", + "@vitest/ui": "4.1.8", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index c09ad428f..d449863c6 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -28,7 +28,7 @@ "@tailwindcss/vite": "^4.3.0", "axios": "^1.16.1", "d3": "^7.9.0", - "dompurify": "^3.4.7", + "dompurify": "^3.4.8", "gitnexus-shared": "file:../gitnexus-shared", "graphology": "^0.26.0", "graphology-indices": "^0.17.0", @@ -38,7 +38,7 @@ "graphology-utils": "^2.3.0", "i18next": "^26.3.0", "i18next-browser-languagedetector": "^8.2.1", - "langchain": "^1.4.2", + "langchain": "^1.4.4", "lru-cache": "^11.2.4", "lucide-react": "^1.16.0", "mermaid": "^11.15.0", @@ -51,7 +51,7 @@ "react-syntax-highlighter": "^16.1.1", "react-zoom-pan-pinch": "^4.0.3", "remark-gfm": "^4.0.1", - "sigma": "^3.0.2", + "sigma": "^3.0.3", "tailwindcss": "^4.2.4", "uuid": "^14.0.0", "zod": "^4.4.3" @@ -67,9 +67,9 @@ "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "@types/react-syntax-highlighter": "^15.5.13", - "@vercel/node": "^5.8.8", + "@vercel/node": "^5.8.12", "@vitejs/plugin-react": "^5.1.4", - "@vitest/coverage-v8": "^4.1.5", + "@vitest/coverage-v8": "^4.1.8", "jsdom": "^29.1.1", "tree-sitter-wasms": "^0.1.13", "typescript": "^5.4.5", diff --git a/gitnexus-web/src/components/RepoAnalyzer.tsx b/gitnexus-web/src/components/RepoAnalyzer.tsx index 0b7f0abbd..613284830 100644 --- a/gitnexus-web/src/components/RepoAnalyzer.tsx +++ b/gitnexus-web/src/components/RepoAnalyzer.tsx @@ -21,9 +21,11 @@ import { startAnalyze, cancelAnalyze, streamAnalyzeProgress, + uploadFolder, type JobProgress, } from '../services/backend-client'; import { AnalyzeProgress } from './AnalyzeProgress'; +import { filterRepoFiles } from '@/lib/upload-filter'; import { useTranslation } from 'react-i18next'; // ── Helpers ────────────────────────────────────────────────────────────────── @@ -165,8 +167,11 @@ export interface RepoAnalyzerProps { export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProps) => { const { t } = useTranslation(['common', 'errors', 'onboarding']); const inputId = useId(); - const folderInputRef = useRef(null); const [mode, setMode] = useState('github'); + const [uploading, setUploading] = useState(false); + const [uploadSummary, setUploadSummary] = useState<{ count: number; dropped: number } | null>( + null, + ); const [githubUrl, setGithubUrl] = useState(''); const [gitlabUrl, setGitlabUrl] = useState(''); const [localPath, setLocalPath] = useState(''); @@ -181,28 +186,73 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const jobIdRef = useRef(null); const sseControllerRef = useRef(null); + // Owns the in-flight analyze/upload request. The controller doubles as the + // staleness token: each request captures its own controller in a closure and + // bails after the await when that controller was aborted, so a resolution + // arriving after a mode switch / cancel / unmount can never drive state. + const requestControllerRef = useRef(null); const completeTimerRef = useRef | null>(null); + const folderInputRef = useRef(null); useEffect(() => { return () => { sseControllerRef.current?.abort(); + requestControllerRef.current?.abort(); if (completeTimerRef.current) clearTimeout(completeTimerRef.current); }; }, []); + // Abort any in-flight analyze/upload request so its settlement can't drive + // state. Aborting is load-bearing: once a mode switch resets `uploading`, + // the `uploading || isLoading` re-entry guard no longer covers the stale + // request — only its aborted signal does. + const invalidateRequest = (): void => { + requestControllerRef.current?.abort(); + requestControllerRef.current = null; + }; + + // Invalidate the previous request and hand the caller a fresh controller. + const renewRequestController = (): AbortController => { + invalidateRequest(); + const controller = new AbortController(); + requestControllerRef.current = controller; + return controller; + }; + + // An upload that resolved after invalidation has still created a server-side + // job; cancel it so the single analyze slot isn't held for the job's full + // duration. Upload-path only: every upload owns a fresh job (the server + // stages each upload into a unique dir, never dedup-aliasing), whereas URL + // analyzes dedup-alias by repo — the returned jobId may belong to a job + // another session (or this user's own resubmit) is actively watching, so + // cancelling on that path could kill a live analysis. A stale URL job is + // left to finish: a same-URL resubmit re-attaches to it via dedup, and the + // server's job timeout/TTL sweep bounds the slot occupancy. + const cancelStaleUploadJob = (jobId: string): void => { + void cancelAnalyze(jobId).catch(() => {}); + }; + const handleModeChange = (m: InputMode) => { + // ModeTabs fires onChange on every click, including the already-active + // tab — never abort the user's own in-flight request for a no-op click. + if (m === mode) return; + invalidateRequest(); setMode(m); setGithubUrl(''); setGitlabUrl(''); setLocalPath(''); setValidationError(null); + setUploadSummary(null); + setUploading(false); + // An aborted request no longer resolves to move `phase` off 'starting'; + // reset so the new mode's form is immediately usable (also clears a stale + // 'error' phase). Only reachable while showInput is true. + setPhase('input'); }; - // Use the browser's native directory picker (webkitdirectory doesn't give paths, - // so we use a text input + a "Browse" button that opens a standard file input - // to let users pick files from the folder — the path is typed manually since - // browsers don't expose absolute paths for security reasons). - // For local paths, the user types or pastes the absolute path. + // Local-folder mode uploads the selected folder's files (the browser never + // exposes an absolute path, so the old typed-path/browse approach couldn't + // work — see handleFolderUpload). A typed server path is also still accepted. const canSubmit = mode === 'github' @@ -228,6 +278,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp setValidationError(null); setPhase('starting'); + // Staleness guard only (no wire abort): the POST is short-lived and + // self-terminates, but its resolution must not drive state after a mode + // switch / cancel / unmount invalidated this request. + const controller = renewRequestController(); try { const request = mode === 'github' @@ -236,8 +290,9 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp ? { url: gitlabUrl.trim() } : { path: localPath.trim() }; const { jobId } = await startAnalyze(request); - jobIdRef.current = jobId; - setPhase('analyzing'); + // Stale resolution: return without cancelling — URL jobIds may be + // dedup-aliased to a job another session owns (see cancelStaleUploadJob). + if (controller.signal.aborted) return; const nameSource = mode === 'github' @@ -245,29 +300,84 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp : mode === 'gitlab' ? gitlabUrl.trim() : localPath.trim(); - const controller = streamAnalyzeProgress( - jobId, - (p) => setProgress(p), - (data) => { - const name = - data.repoName ?? - nameSource.split(/[/\\]/).filter(Boolean).at(-1) ?? - t('onboarding:repoAnalyzer.defaultRepoName'); - setCompletedRepoName(name); - setPhase('done'); - sseControllerRef.current = null; - completeTimerRef.current = setTimeout(() => { - completeTimerRef.current = null; - onComplete(name); - }, 1200); - }, - (errMsg) => { - setValidationError(errMsg || t('errors:analysisFailed')); - setPhase('error'); - }, - ); - sseControllerRef.current = controller; + trackJob(jobId, nameSource); } catch (err) { + // Unmount aborts the controller, so this also covers the unmounted case. + if (controller.signal.aborted) return; + setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); + setPhase('error'); + } + }; + + // Drive an already-created analysis job through the SSE progress stream to + // completion. Shared by the path/URL analyze flow and the folder-upload flow. + const trackJob = (jobId: string, fallbackNameSource: string | null) => { + // Callers reach here only with a live (non-aborted) request controller, so + // the component is mounted — unmount aborts the controller. + jobIdRef.current = jobId; + setPhase('analyzing'); + const controller = streamAnalyzeProgress( + jobId, + (p) => setProgress(p), + (data) => { + const name = + data.repoName ?? + (fallbackNameSource + ? fallbackNameSource.split(/[/\\]/).filter(Boolean).at(-1) + : undefined) ?? + t('onboarding:repoAnalyzer.defaultRepoName'); + setCompletedRepoName(name); + setPhase('done'); + sseControllerRef.current = null; + completeTimerRef.current = setTimeout(() => { + completeTimerRef.current = null; + onComplete(name); + }, 1200); + }, + (errMsg) => { + setValidationError(errMsg || t('errors:analysisFailed')); + setPhase('error'); + }, + ); + sseControllerRef.current = controller; + }; + + // Upload a browser-selected folder (webkitdirectory) and start analysis. The + // upload endpoint returns a jobId, which then joins the normal SSE flow. + const handleFolderUpload = async (fileList: FileList) => { + if (uploading || isLoading) return; // guard against a concurrent upload + const { files, manifest, droppedCount } = filterRepoFiles(fileList); + if (files.length === 0) { + setValidationError(t('onboarding:repoAnalyzer.upload.empty')); + return; + } + setValidationError(null); + setUploadSummary({ count: files.length, dropped: droppedCount }); + setUploading(true); + setPhase('starting'); + // The selected folder's name (manifest entries are `/`) is a + // sensible fallback if the server's complete event omits repoName. + const folderName = manifest[0]?.split('/')[0] ?? null; + const controller = renewRequestController(); + try { + const { jobId } = await uploadFolder(files, manifest, controller.signal); + if (controller.signal.aborted) { + // The abort raced the response: the server already created the job. + // (Unmount aborts the controller, so this also covers unmounted.) + cancelStaleUploadJob(jobId); + return; + } + setUploading(false); + trackJob(jobId, folderName); + } catch (err) { + // An abort surfaces in two shapes — BackendError('Request aborted') + // when it lands during fetch, raw AbortError when it lands during the + // response-body read — so branch on the closure controller's signal, + // never on the error identity. In the second shape the server may have + // already launched a job whose id we never learn; that orphan is bounded + // by the server's job timeout and terminal-job TTL sweep. + if (controller.signal.aborted) return; + setUploading(false); setValidationError(err instanceof Error ? err.message : t('errors:startAnalysisFailed')); setPhase('error'); } @@ -276,6 +386,10 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp const handleCancel = async () => { sseControllerRef.current?.abort(); sseControllerRef.current = null; + // Defensive: no UI path can reach handleCancel while a request is in + // flight (the cancel affordance renders only at phase === 'analyzing'), + // but invalidate it anyway so the guard topology has no holes. + invalidateRequest(); if (jobIdRef.current) { try { await cancelAnalyze(jobIdRef.current); @@ -284,6 +398,8 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp } setPhase('input'); setProgress({ phase: 'queued', percent: 0, message: t('common:analyzePhases.queued') }); + setUploading(false); + setUploadSummary(null); }; const isLoading = phase === 'starting'; @@ -443,35 +559,51 @@ export const RepoAnalyzer = ({ variant, onComplete, onCancel }: RepoAnalyzerProp )} - {/* Native folder picker + Browse button — below the input */} + {/* Upload a folder from your computer — no server path or mount needed. + The browser can't expose an absolute path, so we upload the files. */} { - const files = e.target.files; - if (files && files.length > 0) { - const rel = files[0].webkitRelativePath; - const folderName = rel.split('/')[0]; - if (folderName) { - setLocalPath(folderName); - setValidationError(null); - } + if (e.target.files && e.target.files.length > 0) { + handleFolderUpload(e.target.files); } e.target.value = ''; }} /> + {uploading && ( +
+
+
+
+

+ {t('onboarding:repoAnalyzer.upload.uploading')} +

+
+ )} + {uploadSummary && !uploading && phase !== 'error' && ( +

+ {t('onboarding:repoAnalyzer.upload.selected', { + fileCount: uploadSummary.count, + dropped: uploadSummary.dropped, + })} +

+ )}
)} diff --git a/gitnexus-web/src/lib/upload-filter.test.ts b/gitnexus-web/src/lib/upload-filter.test.ts new file mode 100644 index 000000000..e97b9ec2c --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import { filterRepoFiles, MAX_FILE_BYTES } from './upload-filter'; + +type FileLike = { name: string; size: number; webkitRelativePath?: string }; + +function f(webkitRelativePath: string, size = 10): FileLike { + const name = webkitRelativePath.split('/').pop() ?? webkitRelativePath; + return { name, size, webkitRelativePath }; +} + +describe('filterRepoFiles', () => { + it('keeps source files and builds an order-aligned manifest', () => { + const input = [f('repo/src/index.ts', 100), f('repo/README.md', 50)]; + const r = filterRepoFiles(input); + expect(r.files).toHaveLength(2); + expect(r.manifest).toEqual(['repo/src/index.ts', 'repo/README.md']); + expect(r.totalBytes).toBe(150); + expect(r.droppedCount).toBe(0); + }); + + it('excludes .git / node_modules / build dirs anywhere in the path', () => { + const input = [ + f('repo/.git/HEAD'), + f('repo/node_modules/x/index.js'), + f('repo/dist/bundle.js'), + f('repo/src/app.ts'), + f('repo/.gitnexus/meta.json'), + ]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/src/app.ts']); + expect(r.droppedCount).toBe(4); + }); + + it('drops files over the per-file size cap', () => { + const input = [f('repo/big.bin', MAX_FILE_BYTES + 1), f('repo/small.ts', 10)]; + const r = filterRepoFiles(input); + expect(r.manifest).toEqual(['repo/small.ts']); + expect(r.droppedCount).toBe(1); + }); + + it('falls back to name when webkitRelativePath is absent', () => { + const r = filterRepoFiles([{ name: 'lone.ts', size: 5 }]); + expect(r.manifest).toEqual(['lone.ts']); + }); +}); diff --git a/gitnexus-web/src/lib/upload-filter.ts b/gitnexus-web/src/lib/upload-filter.ts new file mode 100644 index 000000000..a24520f74 --- /dev/null +++ b/gitnexus-web/src/lib/upload-filter.ts @@ -0,0 +1,73 @@ +/** + * Client-side pre-filter for a webkitdirectory folder upload. + * + * Drops VCS metadata, dependency/build directories, and oversized files before + * upload — `.git` alone is often larger than the working tree — so payloads + * stay small and the upload matches what the analyzer actually needs. Produces + * an order-aligned `manifest` of webkitRelativePaths (the server keys on this, + * not the multipart filename, which browsers rewrite). + */ + +/** Directory names excluded anywhere in a file's path. */ +export const EXCLUDED_DIRS = new Set([ + '.git', + '.hg', + '.svn', + 'node_modules', + 'vendor', + '.venv', + '__pycache__', + 'target', + 'dist', + 'build', + 'out', + '.next', + '.nuxt', + '.cache', + 'coverage', + '.idea', + '.gitnexus', +]); + +/** Per-file size cap; matches the server's per-file limit. */ +export const MAX_FILE_BYTES = 25 * 1024 * 1024; + +export interface FilterResult { + files: File[]; + manifest: string[]; + droppedCount: number; + totalBytes: number; +} + +type FileLike = Pick & { webkitRelativePath?: string }; + +/** + * Filter a webkitdirectory `FileList` (or array) into the files to upload plus + * their relative-path manifest. + */ +export function filterRepoFiles(input: ArrayLike): FilterResult { + const files: File[] = []; + const manifest: string[] = []; + let droppedCount = 0; + let totalBytes = 0; + + for (let i = 0; i < input.length; i++) { + const f = input[i]; + const rel = + f.webkitRelativePath && f.webkitRelativePath.length > 0 ? f.webkitRelativePath : f.name; + const segments = rel.split('/'); + if (segments.some((s) => EXCLUDED_DIRS.has(s))) { + droppedCount++; + continue; + } + if (f.size > MAX_FILE_BYTES) { + droppedCount++; + continue; + } + files.push(f as File); + manifest.push(rel); + totalBytes += f.size; + } + + return { files, manifest, droppedCount, totalBytes }; +} diff --git a/gitnexus-web/src/locales/en/onboarding.json b/gitnexus-web/src/locales/en/onboarding.json index cd12c2095..8be58efce 100644 --- a/gitnexus-web/src/locales/en/onboarding.json +++ b/gitnexus-web/src/locales/en/onboarding.json @@ -61,7 +61,12 @@ "gitlabRepositoryUrl": "GitLab Repository URL", "gitlabSupported": "Supports GitLab.com and self-hosted GitLab instances.", "localFolderPath": "Local Folder Path", - "browseForFolder": "Browse for folder", - "hideBackground": "Hide (analysis continues in background)" + "hideBackground": "Hide (analysis continues in background)", + "upload": { + "button": "Upload a folder", + "uploading": "Uploading…", + "selected": "{{fileCount}} files ready ({{dropped}} skipped: .git, node_modules, build output)", + "empty": "No analyzable files found in that folder." + } } } diff --git a/gitnexus-web/src/locales/zh-CN/onboarding.json b/gitnexus-web/src/locales/zh-CN/onboarding.json index 6199511f3..f20e60b6d 100644 --- a/gitnexus-web/src/locales/zh-CN/onboarding.json +++ b/gitnexus-web/src/locales/zh-CN/onboarding.json @@ -61,7 +61,12 @@ "gitlabRepositoryUrl": "GitLab 仓库 URL", "gitlabSupported": "支持 GitLab.com 和自托管 GitLab 实例。", "localFolderPath": "本地文件夹路径", - "browseForFolder": "浏览文件夹", - "hideBackground": "隐藏(分析继续在后台进行)" + "hideBackground": "隐藏(分析继续在后台进行)", + "upload": { + "button": "上传文件夹", + "uploading": "上传中…", + "selected": "已准备 {{fileCount}} 个文件(已跳过 {{dropped}} 个:.git、node_modules、构建产物)", + "empty": "该文件夹中未找到可分析的文件。" + } } } diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index e887e3901..286a13187 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -283,12 +283,11 @@ const fetchWithTimeout = async ( ): Promise => { // Merge the external caller signal (if any) with an // `AbortSignal.timeout()` so a timer-fired abort produces a - // `DOMException` with `name === 'TimeoutError'` — which - // `resilientFetch` correctly classifies as terminal-network (no - // retry, no breaker hit). A manual `AbortController.abort()` would - // produce `name === 'AbortError'` and route through the - // retryable-network branch, which mis-penalizes the breaker for - // user-side network slowness. + // `DOMException` with `name === 'TimeoutError'`. Both shapes are + // breaker-safe: `resilientFetch` classifies TimeoutError AND a manual + // `AbortController.abort()`'s AbortError as terminal-network (no + // retry, breaker-neutral via recordNeutral), so caller-driven + // cancellation never penalizes the breaker. const timeoutSignal = AbortSignal.timeout(timeoutMs); const externalSignal = init.signal; const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal; @@ -755,6 +754,35 @@ export const fetchClusterDetail = async (repo: string, name: string): Promise`) and start analysis. + * Sends the file blobs plus a JSON `manifest` of their relative paths — the + * multipart filename can't carry the path (browsers strip separators), so the + * manifest is the source of truth. Routed through fetchWithTimeout (the shared, + * origin-validated request path) rather than a raw XHR; returns the analysis + * jobId, which the caller drives through the normal SSE flow. + */ +export const uploadFolder = async ( + files: File[], + manifest: string[], + signal?: AbortSignal, +): Promise<{ jobId: string; status: string }> => { + const form = new FormData(); + // Manifest MUST precede the file parts (the server enforces this). + form.append('manifest', JSON.stringify(manifest)); + for (const f of files) form.append('files', f); + + const response = await fetchWithTimeout( + `${_backendUrl}/api/analyze/upload`, + { method: 'POST', body: form, signal }, + 5 * 60_000, // up to 5 min for large repos + ); + await assertOk(response); + return response.json() as Promise<{ jobId: string; status: string }>; +}; + // ── Analyze API ──────────────────────────────────────────────────────────── /** Start a server-side analysis job. */ diff --git a/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx b/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx new file mode 100644 index 000000000..094641b93 --- /dev/null +++ b/gitnexus-web/test/unit/repo-analyzer-upload-race.test.tsx @@ -0,0 +1,218 @@ +/** + * Stale-request guards in RepoAnalyzer (PR #1850 review 4470339833). + * + * An analyze request (folder upload or URL analyze) that is still in flight + * when the user switches modes, cancels, or unmounts must not drive state + * when it later settles: no SSE stream, no phase/error flip — and a + * stale-but-created server job gets a fire-and-forget cancel so the single + * analyze slot is freed. + */ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { RepoAnalyzer } from '../../src/components/RepoAnalyzer'; +import { i18nReady } from '../../src/i18n'; +import { + cancelAnalyze, + startAnalyze, + streamAnalyzeProgress, + uploadFolder, +} from '../../src/services/backend-client'; + +vi.mock('../../src/services/backend-client', () => ({ + startAnalyze: vi.fn(), + cancelAnalyze: vi.fn(), + streamAnalyzeProgress: vi.fn(), + uploadFolder: vi.fn(), +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +const JOB = { jobId: 'job-1', status: 'queued' }; + +/** Gate uploadFolder on a deferred promise and expose the signal it received. */ +function mockUploadWith(d: { promise: Promise }) { + let captured: AbortSignal | undefined; + vi.mocked(uploadFolder).mockImplementation((_files, _manifest, signal) => { + captured = signal; + return d.promise; + }); + return { signal: () => captured }; +} + +/** Render, switch to Local Folder mode, and fire a folder selection. */ +function startUpload() { + const view = render(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + fireEvent.change(screen.getByTestId('folder-upload-input'), { + target: { files: [new File(['x'], 'a.ts')] }, + }); + return view; +} + +beforeEach(async () => { + await i18nReady; + vi.clearAllMocks(); + vi.mocked(cancelAnalyze).mockResolvedValue(undefined as never); + vi.mocked(streamAnalyzeProgress).mockImplementation(() => new AbortController()); +}); + +describe('folder upload', () => { + it('a mode switch mid-upload makes the resolution inert and cancels the job', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'GitHub URL' })); + + // The wire abort happened at mode-switch time, not at resolution time. + expect(upload.signal()?.aborted).toBe(true); + + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + expect(cancelAnalyze).toHaveBeenCalledWith('job-1'); + // The GitHub form is clean and submittable (phase back to 'input'). + expect(screen.getByRole('textbox')).toBeEnabled(); + expect(screen.queryByTestId('upload-progress')).not.toBeInTheDocument(); + }); + + it.each([ + ['BackendError shape', new Error('Request aborted')], + ['raw AbortError shape', new DOMException('The operation was aborted.', 'AbortError')], + ])('an aborted rejection is silent — %s', async (_label, err) => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'GitHub URL' })); + await act(async () => { + d.reject(err); + }); + + expect(screen.queryByText('Request aborted')).not.toBeInTheDocument(); + expect(screen.queryByText('The operation was aborted.')).not.toBeInTheDocument(); + expect(screen.getByRole('textbox')).toBeEnabled(); + }); + + it('a same-tab click does not abort the in-flight upload', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + startUpload(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + + expect(upload.signal()?.aborted).toBe(false); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('an unmount mid-upload makes the resolution inert', async () => { + const d = deferred(); + const upload = mockUploadWith(d); + + const { unmount } = startUpload(); + unmount(); + + expect(upload.signal()?.aborted).toBe(true); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + }); + + it('the happy path still tracks the job', async () => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + await act(async () => { + d.resolve(JOB); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(vi.mocked(streamAnalyzeProgress).mock.calls[0][0]).toBe('job-1'); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('a genuine error still surfaces', async () => { + const d = deferred(); + vi.mocked(uploadFolder).mockReturnValue(d.promise); + + startUpload(); + await act(async () => { + d.reject(new Error('upload exploded')); + }); + + expect(screen.getByText('upload exploded')).toBeInTheDocument(); + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + }); +}); + +describe('URL analyze', () => { + function startGithubAnalyze() { + render(); + fireEvent.change(screen.getByRole('textbox'), { + target: { value: 'https://github.com/owner/repo' }, + }); + fireEvent.click(screen.getByRole('button', { name: /Analyze Repository/ })); + } + + it('a mode switch mid-analyze makes the resolution inert without cancelling', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + await act(async () => { + d.resolve({ jobId: 'job-2', status: 'queued' }); + }); + + expect(streamAnalyzeProgress).not.toHaveBeenCalled(); + // No cancel on the URL path: the jobId may be dedup-aliased to a job + // another session owns, so cancelling could kill a live analysis. + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); + + it('a stale rejection is silent', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + fireEvent.click(screen.getByRole('tab', { name: 'Local Folder' })); + await act(async () => { + d.reject(new Error('analyze exploded')); + }); + + expect(screen.queryByText('analyze exploded')).not.toBeInTheDocument(); + expect(screen.getByTestId('upload-folder')).toBeEnabled(); + }); + + it('the happy path still tracks the job', async () => { + const d = deferred(); + vi.mocked(startAnalyze).mockReturnValue(d.promise); + + startGithubAnalyze(); + await act(async () => { + d.resolve({ jobId: 'job-3', status: 'queued' }); + }); + + expect(streamAnalyzeProgress).toHaveBeenCalledTimes(1); + expect(vi.mocked(streamAnalyzeProgress).mock.calls[0][0]).toBe('job-3'); + expect(cancelAnalyze).not.toHaveBeenCalled(); + }); +}); diff --git a/gitnexus/.npmignore b/gitnexus/.npmignore index cf403314a..bf2c8b7c9 100644 --- a/gitnexus/.npmignore +++ b/gitnexus/.npmignore @@ -13,6 +13,26 @@ node_modules/ vendor/**/node_modules vendor/**/build +# ── Lean publish (FUTURE optimization — NOT done here) ───────────────────────── +# Once the build-tree-sitter-prebuilds workflow has committed 6/6 prebuilds for +# EVERY vendored grammar (c, dart, proto, kotlin, swift), the ~50 MB of generated +# source (parser.c etc.) can be dropped from the tarball — node-gyp-build never +# needs the source when a prebuild matches. +# +# IMPORTANT: this CANNOT be done from this file. package.json's `files: ["vendor"]` +# allow-list OVERRIDES .npmignore for the vendor/ subtree (verified: an active +# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). To slim +# the tarball, narrow the `files` field instead — replace the blanket "vendor" +# with the non-source subpaths only (vendor/**/prebuilds/**, +# vendor/**/bindings/node/index.*, vendor/**/src/node-types.json, +# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md). +# +# Whatever the mechanism, the prepack guard +# (scripts/assert-publish-grammar-coverage.cjs, also `npm run +# assert-publish-coverage`) inspects the EFFECTIVE `npm pack` file list and FAILS +# the publish whenever a grammar with <6 prebuilds loses a source-build input — so +# the slim can never silently ship a dead grammar. Do not bypass it. + # Package lock (consumers use their own) package-lock.json diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index 5fe3e70f2..e860e71cd 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -4,9 +4,34 @@ All notable changes to GitNexus will be documented in this file. ## [Unreleased] +## [1.6.7] - 2026-06-09 + ### Added -- **Taint/PDG substrate (M0)** — foundational schema + seams for reliable taint analysis on a PDG-expandable substrate (#2080, Epic #2087). Adds the `BasicBlock` node label and `CFG` / `REACHING_DEF` / `TAINTED` / `SANITIZES` / `TAINT_PATH` relationship types to the graph schema (round-trip through the bulk-COPY path), a phase-registry seam (`registerPhase` / `enabledWhen`) generalising the graph-phase opt-in guard, and a per-language source/sink/sanitizer config registry seam. All additive and inert — no phase emits the new nodes/edges yet, and a default `analyze` run is byte-identical to before. De-risking spikes (LadybugDB rel-property indexing, post-dominator feasibility) recorded on the issue. +- **Toolchain-free tree-sitter install** — the `c`, `dart`, `proto`, `kotlin`, and `swift` grammars now ship vendored native prebuilds (six platform/arch each — linux/darwin/win32 × x64/arm64, every `.node` load-and-parse verified with committed `SHA256SUMS` and SLSA build provenance), so a fresh install no longer requires a C/C++ toolchain; `kotlin` moved off its `optionalDependency` into the vendored path, `dart`/`proto` keep a source-build fallback when no prebuild matches, and a registry-parameterized CI workflow builds, load-validates, and vendors the binaries (#2113, #2125, #2110) +- **`gitnexus uninstall`** — reverses `gitnexus setup` target-by-target, surgically removing GitNexus MCP server entries (Cursor, Claude Code, Antigravity, OpenCode, Codex), installed skill directories, and Claude Code / Antigravity hook entries with their bundled scripts; idempotent, JSONC-preserving, dry-run by default with `--force` to apply (#2062, #2060) +- **MCP `list_repos` pagination** — bounded `limit`/`offset` paging so clients can reliably enumerate every indexed repository instead of having the unpaginated array truncated by LLM token limits; the result is now a `{ repositories, pagination }` object (page until `pagination.hasMore` is false), with deterministic `(lower-cased name, path)` ordering (#2120, #2119) +- **C++ inheritance-lattice member lookup** — receiver members now resolve through the inheritance lattice with dominance hiding, ambiguous-base suppression, virtual-diamond deduplication, and overload ranking, and class-scope `using Base::member` declarations are no longer mistaken for namespace imports (#2077, #1891) +- **Taint/PDG substrate (M0)** — foundational graph schema and pipeline seams for reliable taint analysis on a PDG-expandable substrate: the `BasicBlock` node label and `CFG` / `REACHING_DEF` / `TAINTED` / `SANITIZES` / `TAINT_PATH` relationship types (round-tripped through the bulk-COPY path), a phase-registry seam (`registerPhase` / `enabledWhen`) generalising the graph-phase opt-in guard, and a per-language source/sink/sanitizer config registry. All additive and inert — no phase emits the new nodes/edges yet and a default `analyze` run is byte-identical to before (#2092, #2080) + +### Fixed + +- **Optional grammars lazy-loaded so `analyze` never crashes when one is missing** — the swift/dart/kotlin `query.ts` modules no longer statically import their tree-sitter binding at module load, so a missing optional grammar can no longer abort `gitnexus analyze` (or the MCP server, `doctor`, and `.githooks` auto-reindex) with `ERR_MODULE_NOT_FOUND` regardless of the repo's actual languages; grammars now resolve lazily at first use inside the worker, `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` is honored at runtime, the scope-resolution phase excludes unavailable-language files, and skip diagnostics/precheck globs were corrected (#2101, #2091, #2093) +- **`tree-sitter-kotlin` optional-grammar install** — install now fails soft when no C/C++ toolchain is present, emitting one clear warning and always exiting 0 (mirroring the Swift/Dart/Proto probes) instead of breaking `gitnexus` install; optional-grammar/toolchain docs corrected to include Kotlin (#2110, #2107) +- **CLI image FTS keyword search** — the full-text-search extension is now baked into the CLI Docker image so a containerized `serve` does offline keyword search instead of silently degrading to vector-only (#2108) + +### Changed + +- **Tree-sitter prebuild CI matrix greened and made re-run-safe** — dropped the broken `-t 22` flag from the `prebuildify` invocation that crashed every matrix job (`v.indexOf is not a function`; N-API prebuilds are Node-version-agnostic, so no target is needed) (#2121), cleared npm-bundled `prebuilds/` before prebuildify so the host tuple is detected (not a stray `win32-x64`) and source-built the `tree-sitter` runtime peer on `linux-arm64` where upstream ships no prebuild (#2122), and switched the vendor-prebuilds push to `git push --force` so re-running a workflow no longer fails with a stale-lease rejection (#2123) + +### Performance + +- **MCP `query` enrichment batched** — the `query` tool now batches its per-symbol enrichment lookups (3N sequential pool round-trips collapsed to 2–3 `WHERE n.id IN $nodeIds` queries), cutting N+1 round-trips with byte-identical output (#2108) + +### Chore / Dependencies + +- **`@ladybugdb/core` bumped 0.17.0 → 0.17.1 in /gitnexus** (#2098) +- **Claude plugin manifests synced to the release version** — bumped `plugin.json` and the `gitnexus` `marketplace.json` entry to match the published npm version (stale `1.3.x` manifests had blocked marketplace updates), added a Vitest guard asserting all three manifests advertise one version, and documented the sync step in `CONTRIBUTING.md` (#2090) ## [1.6.6] - 2026-06-08 diff --git a/gitnexus/README.md b/gitnexus/README.md index 7c84087ea..4f5b27781 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -126,7 +126,7 @@ Your AI agent gets these tools automatically: | Tool | What It Does | `repo` Param | | ---------------- | ---------------------------------------------------------------- | ------------ | -| `list_repos` | Discover all indexed repositories | — | +| `list_repos` | Discover all indexed repositories (paginated — `limit`/`offset`) | — | | `query` | Process-grouped hybrid search (BM25 + semantic + RRF) | Optional | | `context` | 360-degree symbol view — categorized refs, process participation | Optional | | `impact` | Blast radius analysis with depth grouping and confidence | Optional | @@ -159,6 +159,7 @@ Your AI agent gets these tools automatically: ```bash gitnexus setup # Configure MCP for your editors (one-time) +gitnexus uninstall # Preview removal of GitNexus MCP/skills/hooks (add --force to apply) gitnexus analyze [path] # Index a repository (or update stale index) gitnexus analyze --repair-fts # Fast path: rebuild/verify only FTS indexes on existing index data gitnexus analyze --force # Full rebuild: re-parse + graph rebuild + FTS rebuild @@ -196,6 +197,8 @@ gitnexus group query # Search execution flows across all repos in a gitnexus group status # Check staleness of repos in a group ``` +> **`gitnexus uninstall`** reverses `gitnexus setup` — it removes the GitNexus MCP entries, hooks, and skill directories it added to each detected editor. Skill directories are identified **by bundled gitnexus skill name** (e.g. `gitnexus-cli/`), so if you customized files inside an installed skill directory, back them up first. It is a dry-run preview by default and prints the exact paths it would remove; pass `--force` to apply. Per-repo indexes (`gitnexus clean --all`) and the global npm package (`npm uninstall -g gitnexus`) are left for you to remove. + ## Remote Embeddings Set these env vars to use a remote OpenAI-compatible `/v1/embeddings` endpoint instead of the local model: @@ -433,6 +436,26 @@ After scope resolution, analyze prunes inert block-local value symbols (a functi Programmatic callers can pass `keepLocalValueSymbols: true` in `PipelineOptions` instead of setting the env var. +### Hook augmentation/notifications are silently skipped + +The Claude Code / Antigravity hooks intentionally stay **silent** on normal skip +paths so strict hook runners (e.g. Codex `PreToolUse`) never see unexpected +output. A search may not be augmented — or a stale-index reminder may not appear +on stderr — when the GitNexus MCP server owns the repo DB, when the DB-lock probe +times out and fails closed, or when the index is already current. + +To see why a hook skipped, set `GITNEXUS_DEBUG=1` and re-run the action — the hook +writes the reason (e.g. `[GitNexus] augment skipped: MCP server owns DB`) and the +stale-index hint to its stderr: + +```bash +GITNEXUS_DEBUG=1 # surfaces hook skip/diagnostic reasons on stderr +``` + +Only `GITNEXUS_DEBUG=1` and `GITNEXUS_DEBUG=true` enable diagnostics; every other +value (including `0` and `false`) is treated as off. Diagnostics go to stderr +only — the hook's structured stdout (the JSON the agent consumes) is unaffected. + ## Privacy - All processing happens locally on your machine diff --git a/gitnexus/bench/cfg/baselines.json b/gitnexus/bench/cfg/baselines.json new file mode 100644 index 000000000..7e3d718d8 --- /dev/null +++ b/gitnexus/bench/cfg/baselines.json @@ -0,0 +1,44 @@ +{ + "straight-line": { + "fingerprint": "792229965a726d2c6b527f9ee65440a2b3023839ee71cb51522fc30e2f2cb454", + "scaling_budget": 1.5, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "rd_scaling_budget": 2.0, + "disk_bytes_large_max": 1309481, + "_note": "#2081 M1 / #2082 M2: ONE function, N coalescing statements (extendBlock text accumulation + per-statement fact harvest). Runs at 2000->8000. M2 REWROTE the old 'output is constant 4 blocks' note: statement facts make disk/heap LINEAR in N (a free gate on the harvest payload); TIME still guards the concat path (array-join ~1.0; a genuine O(n^2) re-join accumulation is ~3.8). M2 adds rd_scaling_budget (measured ~0.74) and disk_bytes_large_max -- an ABSOLUTE ceiling ~1.35x the measured indexed-encoding bytes (969,986 at N=8000, ~121 B/stmt); a named-record encoding regression (~4x facts bytes) blows it. Re-baseline the fingerprint only on an intentional CFG/harvest-shape change (the canon now includes statements+bindings)." + }, + "many-functions": { + "fingerprint": "f3bcc5e6ef4cf58aefe4e7d801a8fea0215494b9688833e501c2afc6df029c1b", + "scaling_budget": 1.5, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "rd_scaling_budget": 2.0, + "_note": "#2081 M1 / #2082 M2: N small branchy functions (collect walk + per-function build + per-function solve). Time ~1.0, disk ~1.01, heap ~1.0, rd ~0.86 (solver is per-function; N functions scale linearly)." + }, + "branchy": { + "fingerprint": "5b5886521ab21604df8f78af98c8c28a6be8e64c24f3d67b165c2d96ba2a3d52", + "scaling_budget": 1.8, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "rd_scaling_budget": 2.0, + "_note": "#2081 M1 / #2082 M2: ONE function, N sequential ifs (block/edge growth in one CFG). Time ~1.1-1.25 (noisiest scenario; budget 1.8 absorbs noise, catches ~4.0 quadratic), disk ~1.03, heap ~1.0, rd ~0.7." + }, + "dense-bindings": { + "fingerprint": "e4d7eb3c7e8b3772423af25cef391e0e6b68067b554819e81b543439a487403f", + "scaling_budget": 1.8, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "rd_scaling_budget": 10.0, + "_note": "#2082 M2: N bindings live across ~N blocks in one loop -- bindings x blocks scale JOINTLY (the solver-lattice stressor). The overlay design measures rd ~5.2 normalized: the OUT spine copy on genning blocks is O(V) per block, which is quadratic when V scales with B (bounded in prod by maxFunctionLines; real functions have V~10-40). Budget 10 deliberately tolerates that known shape and exists to catch the repo's recurring per-item-rescan class (a per-use scan over all defs is O(n^3) here, ratio >=16). If rd drops well below 5, tighten." + }, + "fact-fanout": { + "fingerprint": "488e63e072d514a9229e21872615e32c7b099ccbd65ec8c045ba517568fd3e5d", + "scaling_budget": 1.8, + "disk_bytes_budget": 1.2, + "heap_budget": 1.3, + "rd_scaling_budget": 3.0, + "facts_large_max": 16000, + "_note": "#2082 M2: N switch-arm defs of one variable + N later uses -- facts are O(defs x uses) BY SPEC, so the gate is BOUNDEDNESS, not linearity: with the production fact limit engaged (DEFAULT_PDG_MAX_REACHING_DEF_FACTS_PER_FUNCTION=16000) the materialized fact count stays pinned at the limit as N grows (facts_large_max), and rd time stays bounded (measured ~1.4). Losing the maxFacts early-stop shows as facts_large exploding quadratically." + } +} diff --git a/gitnexus/bench/cfg/measure.mjs b/gitnexus/bench/cfg/measure.mjs new file mode 100644 index 000000000..2451b5d62 --- /dev/null +++ b/gitnexus/bench/cfg/measure.mjs @@ -0,0 +1,388 @@ +/** + * Build-free CFG-construction measurement harness (#2081 M1). + * + * Times `collectFunctionCfgs` (the per-function CFG builder the parse worker + * runs on a `--pdg` run) on synthetic TS sources at two sizes, in three + * scenarios that each stress a distinct cost dimension: + * - `straight-line`: ONE function with N coalescing statements — stresses the + * basic-block text accumulation (the `extendBlock` path); + * - `many-functions`: N small branchy functions — stresses the collect walk + + * per-function build + the tree-sitter `namedChildren` accesses; + * - `branchy`: ONE function with N sequential `if`s — stresses block/edge + * growth within a single CFG. + * + * For each scenario it reports three scaling ratios at small→large + * (`(metric_large/metric_small)/(N_large/N_small)`: ~1.0 is linear, ~4.0 is the + * O(n²) shape the M1 perf review flagged for `extendBlock`'s concat chain): + * - TIME — wall-clock of `collectFunctionCfgs` (median of reps); + * - DISK — utf8 byte size of the serialized `cfgSideChannel` (what a `--pdg` + * run writes onto every ParsedFile shard); + * - MEMORY — retained JS heap of the `cfgSideChannel` payload, by the + * release-delta method (heap held minus heap after dropping it). Requires + * `node --expose-gc`; without it the heap metric is null and its gate skips. + * It also computes an order-independent sha256 fingerprint over the emitted + * blocks/edges of a fixed-size source — the correctness gate that a structural + * speedup must leave behavior-identical. + * + * Build-free: imports the `.ts` hotpaths through tsx + * (`node --expose-gc --import tsx bench/cfg/measure.mjs`). Parsing happens ONCE + * per size and the tree is reused across reps so the time measurement isolates + * CFG build cost, not tree-sitter parse time. `maxFunctionLines` is 0 (no cap) + * here on purpose — the bench measures the algorithm; the production default cap + * is a separate safety net (and would otherwise skip the large straight-line fn). + * + * Without args: prints one JSON object per scenario. + * With `--check`: asserts each scenario's fingerprint == its committed baseline + * (baselines.json) AND each of the time / disk / heap ratios is below its + * recorded budget; exits non-zero on any drift/regression. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { fileURLToPath } from 'node:url'; + +import Parser from 'tree-sitter'; +import TypeScript from 'tree-sitter-typescript'; +import { collectFunctionCfgs } from '../../src/core/ingestion/cfg/collect.ts'; +import { computeReachingDefs } from '../../src/core/ingestion/cfg/reaching-defs.ts'; +import { DEFAULT_PDG_MAX_REACHING_DEF_FACTS_PER_FUNCTION } from '../../src/core/ingestion/cfg/emit.ts'; +import { createTypeScriptCfgVisitor } from '../../src/core/ingestion/cfg/visitors/typescript.ts'; +import { getTreeSitterBufferSize } from '../../src/core/ingestion/constants.ts'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const BASELINE_PATH = path.resolve(__dirname, 'baselines.json'); + +const visitor = createTypeScriptCfgVisitor(); +const parser = new Parser(); +parser.setLanguage(TypeScript.typescript); +// Large synthetic sources exceed tree-sitter's default read buffer; size it +// from the content exactly as the parse worker does (getTreeSitterBufferSize). +const parse = (src) => parser.parse(src, undefined, { bufferSize: getTreeSitterBufferSize(src) }); + +// ---- synthetic generators (one cost dimension each) ---- + +const SCENARIOS = [ + { + name: 'straight-line', + // One function, N coalescing simple statements → all fold into one basic + // block whose text is accumulated statement-by-statement (extendBlock). + // Uses LARGER sizes than the other scenarios: this scenario's only cost + // dimension is text accumulation (output size is constant — 4 blocks at any + // N — so the disk/heap ratios can't see it), so the TIME ratio is the sole + // guard against an extendBlock O(n²)-concat re-regression. At small N a + // quadratic is masked by V8 cons-strings + the linear tree-walk and slips + // under the budget; these larger sizes make a real quadratic separate + // cleanly (verified: a `+=` regression here exceeds the budget, the + // array-join impl stays ~1). + small: 2000, + large: 8000, + gen: (n) => { + let s = 'function f() {\n'; + for (let i = 0; i < n; i++) s += ` let v${i} = ${i} + 1;\n`; + return s + ' return v0;\n}\n'; + }, + }, + { + name: 'many-functions', + // N independent small functions with a branch + return → stresses the + // tree walk in collectFunctionCfgs and the per-function build. + gen: (n) => { + let s = ''; + for (let i = 0; i < n; i++) { + s += `function f${i}(x: number) { if (x > ${i}) { a(); } else { b(); } return x + ${i}; }\n`; + } + return s; + }, + }, + { + name: 'branchy', + // One function, N sequential `if`s → N condition blocks + 2N+ edges in a + // single CFG; stresses block/edge growth and namedChildren on the body. + gen: (n) => { + let s = 'function f(x: number) {\n'; + for (let i = 0; i < n; i++) s += ` if (x > ${i}) { s${i}(); }\n`; + return s + '}\n'; + }, + }, + { + name: 'dense-bindings', + // #2082 M2: N bindings live across ~N blocks inside one loop — bindings × + // blocks scale JOINTLY, the discriminator for solver-lattice quadratics. + // The overlay design (KTD2: sets shared by reference, OUT spine-copied + // only on gen) is expected to scale ~linearly-with-a-spine-copy here + // (normalized ratio low single digits); the regression this scenario + // exists to catch is the repo's recurring per-item-rescan shape — a + // per-use scan over all defs (O(n³) here) blows the ratio past ~16. + // rd time is the gated metric (rd_scaling_budget). + rdMaxFacts: 0, // measure the algorithm, not the cap + gen: (n) => { + let s = 'function f(c: number) {\n'; + for (let i = 0; i < n; i++) s += ` let v${i} = ${i};\n`; + s += ' while (c > 0) {\n'; + for (let i = 0; i < n; i++) s += ` if (c > ${i}) { v${i} = v${(i + 1) % n} + 1; }\n`; + return s + ' c = c - 1;\n }\n return v0;\n}\n'; + }, + }, + { + name: 'fact-fanout', + // #2082 M2: N parallel case-arm defs of one variable + N later uses — + // facts are O(defs×uses) BY SPEC, so a linearity ratio gate is the wrong + // shape. The gate here is BOUNDEDNESS: with the production fact limit + // engaged, the materialized fact count stays FLAT (== limit) as N grows + // past it (facts_large_max), and rd time stays bounded. An unbounded + // materialization regression (losing the maxFacts early-stop) shows as + // facts_large exploding quadratically. + rdMaxFacts: DEFAULT_PDG_MAX_REACHING_DEF_FACTS_PER_FUNCTION, + gen: (n) => { + let s = 'function f(c: number) {\n let x = 0;\n switch (c) {\n'; + for (let i = 0; i < n; i++) s += ` case ${i}: x = ${i}; break;\n`; + s += ' }\n'; + for (let i = 0; i < n; i++) s += ` u${i}(x);\n`; + return s + '}\n'; + }, + }, +]; + +const SMALL = 500; +const LARGE = 2000; // 4× — O(n) ⇒ ratio ~1, O(n²) ⇒ ratio ~4 +const REPS = 15; // median over more reps → stabler time signal at small absolute ms +const FP_SIZE = 15; // fixed size for the behavior fingerprint +const NO_CAP = 0; // measure the algorithm, not the production safety cap + +// ---- timing ---- + +function median(xs) { + const s = [...xs].sort((a, b) => a - b); + const m = Math.floor(s.length / 2); + return s.length % 2 ? s[m] : (s[m - 1] + s[m]) / 2; +} + +function measureCollect(src, file, reps) { + const root = parse(src).rootNode; // parse ONCE; reuse across reps + collectFunctionCfgs(root, visitor, `warmup-${file}`, NO_CAP); // warm JIT (uncounted) + const samples = []; + let out; + for (let i = 0; i < reps; i++) { + const start = process.hrtime.bigint(); + out = collectFunctionCfgs(root, visitor, file, NO_CAP); + samples.push(Number(process.hrtime.bigint() - start) / 1e6); + } + return { + ms: median(samples), + cfgs: out.cfgs, + blockCount: out.cfgs.reduce((a, c) => a + c.blocks.length, 0), + // DISK growth: utf8 byte size of the serialized cfgSideChannel — exactly + // what a --pdg run writes onto every ParsedFile shard in the durable store + // + parse cache (the field is plain JSON, so this is the on-disk delta). + // Should scale linearly with source covered; a super-linear ratio means the + // CFG duplicates text and bloats warm-cache shards at scale. + diskBytes: Buffer.byteLength(JSON.stringify(out.cfgs), 'utf8'), + }; +} + +// ---- reaching-defs solve cost (#2082 M2) ---- + +// Times computeReachingDefs over a scenario's collected CFGs (the exact work +// the scope-resolution emit loop adds per file on a --pdg run). `maxFacts` +// mirrors the per-scenario production posture: 0 (unlimited) measures the +// algorithm; the production default exercises the boundedness contract. +function measureReachingDefs(cfgs, reps, maxFacts) { + for (const c of cfgs) computeReachingDefs(c, { maxFacts }); // warm JIT + const samples = []; + let facts = 0; + for (let i = 0; i < reps; i++) { + const start = process.hrtime.bigint(); + facts = 0; + for (const c of cfgs) facts += computeReachingDefs(c, { maxFacts }).facts.length; + samples.push(Number(process.hrtime.bigint() - start) / 1e6); + } + return { ms: median(samples), facts }; +} + +// ---- memory growth: retained heap of the cfgSideChannel payload ---- + +// Needs `node --expose-gc` to force collection for a clean delta; without it the +// heap metric is reported as null and its --check gate is skipped (so a local +// run without the flag still works). +const GC = typeof global.gc === 'function' ? () => (global.gc(), global.gc()) : null; + +function retainedHeapBytes(src, file) { + if (!GC) return null; + // Retained-size-by-RELEASE: measure the heap with the CFGs held, drop them, + // GC, measure again. The drop isolates exactly the JS heap the cfgSideChannel + // payload retains (the extra RAM a --pdg run carries per file until the shard + // is flushed) — robust to pre-existing garbage, which is constant across both + // measurements. The parse tree is a temporary (its native memory isn't on the + // JS heap); block text strings are fresh copies, so they count here. + let cfgs = collectFunctionCfgs(parse(src).rootNode, visitor, file, NO_CAP).cfgs; + GC(); + const withCfgs = process.memoryUsage().heapUsed; + if (cfgs.length < 0) throw new Error('unreachable'); // keep cfgs live past withCfgs + cfgs = null; + GC(); + const withoutCfgs = process.memoryUsage().heapUsed; + return Math.max(0, withCfgs - withoutCfgs); +} + +// ---- correctness fingerprint (order-independent over blocks + edges) ---- + +function canonicalizeCfg(cfg) { + const blocks = cfg.blocks + .map( + (b) => + `B|${b.index}|${b.startLine}-${b.endLine}|${b.kind}|${b.text}|` + + // #2082 M2: statement facts join the canon so harvest drift (lost + // defs/uses, changed binding resolution) trips the fingerprint gate. + JSON.stringify(b.statements ?? null), + ) + .sort(); + const edges = cfg.edges.map((e) => `E|${e.from}->${e.to}|${e.kind}`).sort(); + const bindings = JSON.stringify(cfg.bindings ?? null); + return `${cfg.functionStartLine}:${cfg.functionStartColumn}\n${bindings}\n${blocks.join('\n')}\n${edges.join('\n')}`; +} + +function fingerprint(scenario) { + const out = collectFunctionCfgs(parse(scenario.gen(FP_SIZE)).rootNode, visitor, 'fp.ts', NO_CAP); + const canon = out.cfgs.map(canonicalizeCfg).sort().join('\n====\n'); + return { + fingerprint: crypto.createHash('sha256').update(canon).digest('hex'), + fp_cfgs: out.cfgs.length, + fp_blocks: out.cfgs.reduce((a, c) => a + c.blocks.length, 0), + fp_edges: out.cfgs.reduce((a, c) => a + c.edges.length, 0), + }; +} + +function measureScenario(scenario) { + // Per-scenario sizes (straight-line needs larger N to separate a concat + // quadratic from noise — see its comment); the rest default to the globals. + const nSmall = scenario.small ?? SMALL; + const nLarge = scenario.large ?? LARGE; + const small = measureCollect(scenario.gen(nSmall), `${scenario.name}.ts`, REPS); + const large = measureCollect(scenario.gen(nLarge), `${scenario.name}.ts`, REPS); + const sizeRatio = nLarge / nSmall; + const scalingRatio = small.ms > 0 ? large.ms / small.ms / sizeRatio : 0; + const diskRatio = small.diskBytes > 0 ? large.diskBytes / small.diskBytes / sizeRatio : 0; + + // Memory growth (only when --expose-gc gave us a forced GC). + const heapSmall = retainedHeapBytes(scenario.gen(nSmall), `${scenario.name}.ts`); + const heapLarge = retainedHeapBytes(scenario.gen(nLarge), `${scenario.name}.ts`); + const heapRatio = + heapSmall !== null && heapLarge !== null && heapSmall > 0 + ? heapLarge / heapSmall / sizeRatio + : null; + + // #2082 M2: reaching-defs solve cost over the same CFGs. + const rdMaxFacts = scenario.rdMaxFacts ?? 0; + const rdSmall = measureReachingDefs(small.cfgs, REPS, rdMaxFacts); + const rdLarge = measureReachingDefs(large.cfgs, REPS, rdMaxFacts); + // Clamp the denominator: a 0.000ms small-N median would otherwise yield + // ratio 0 and the gate would self-disable exactly when the solver is fast. + const rdRatio = rdLarge.ms / Math.max(rdSmall.ms, 0.001) / sizeRatio; + + return { + scenario: scenario.name, + elapsed_ms_small: Number(small.ms.toFixed(3)), + elapsed_ms_large: Number(large.ms.toFixed(3)), + scaling_ratio: Number(scalingRatio.toFixed(3)), + disk_bytes_small: small.diskBytes, + disk_bytes_large: large.diskBytes, + disk_bytes_ratio: Number(diskRatio.toFixed(3)), + heap_bytes_small: heapSmall, + heap_bytes_large: heapLarge, + heap_ratio: heapRatio === null ? null : Number(heapRatio.toFixed(3)), + blocks_small: small.blockCount, + blocks_large: large.blockCount, + rd_ms_small: Number(rdSmall.ms.toFixed(3)), + rd_ms_large: Number(rdLarge.ms.toFixed(3)), + rd_scaling_ratio: Number(rdRatio.toFixed(3)), + facts_small: rdSmall.facts, + facts_large: rdLarge.facts, + ...fingerprint(scenario), + }; +} + +// ---- run ---- + +const CHECK = process.argv.includes('--check'); + +// The retained-heap budget is a primary regression detector, but it can only be +// measured with a forced GC. Rather than let `--check` silently PASS with the +// heap gate skipped (a green no-op if someone drops --expose-gc), fail loudly. +if (CHECK && !GC) { + process.stderr.write( + '[cfg --check] FAIL: retained-heap gate requires --expose-gc. ' + + 'Run: node --expose-gc --import tsx bench/cfg/measure.mjs --check\n', + ); + process.exit(1); +} + +const results = SCENARIOS.map(measureScenario); + +if (!CHECK) { + for (const r of results) process.stdout.write(JSON.stringify(r) + '\n'); +} else { + const baselines = JSON.parse(fs.readFileSync(BASELINE_PATH, 'utf8')); + const failures = []; + for (const r of results) { + const base = baselines[r.scenario]; + if (base === undefined) { + failures.push(`${r.scenario}: no baseline recorded`); + continue; + } + if (r.fingerprint !== base.fingerprint) { + failures.push( + `${r.scenario}: CFG fingerprint drift (got ${r.fingerprint}, expected ${base.fingerprint})`, + ); + } + if (r.scaling_ratio >= base.scaling_budget) { + failures.push( + `${r.scenario}: scaling ratio ${r.scaling_ratio} >= budget ${base.scaling_budget} ` + + `(${SMALL}->${LARGE} stmts/fns, ms ${r.elapsed_ms_small}->${r.elapsed_ms_large})`, + ); + } + if (base.disk_bytes_budget !== undefined && r.disk_bytes_ratio >= base.disk_bytes_budget) { + failures.push( + `${r.scenario}: cfgSideChannel disk-bytes ratio ${r.disk_bytes_ratio} >= budget ` + + `${base.disk_bytes_budget} (bytes ${r.disk_bytes_small}->${r.disk_bytes_large})`, + ); + } + // #2082 M2 gates — rd solve-time scaling, fact-count boundedness, and an + // ABSOLUTE side-channel size ceiling (a ratio gate is blind to a + // constant-factor encoding bloat like named records vs indexed facts). + if (base.rd_scaling_budget !== undefined && r.rd_scaling_ratio >= base.rd_scaling_budget) { + failures.push( + `${r.scenario}: reaching-defs scaling ratio ${r.rd_scaling_ratio} >= budget ` + + `${base.rd_scaling_budget} (ms ${r.rd_ms_small}->${r.rd_ms_large})`, + ); + } + if (base.facts_large_max !== undefined && r.facts_large > base.facts_large_max) { + failures.push( + `${r.scenario}: fact materialization ${r.facts_large} > bound ${base.facts_large_max} ` + + `(the maxFacts early-stop is the boundedness contract)`, + ); + } + if (base.disk_bytes_large_max !== undefined && r.disk_bytes_large > base.disk_bytes_large_max) { + failures.push( + `${r.scenario}: cfgSideChannel absolute size ${r.disk_bytes_large} > ceiling ` + + `${base.disk_bytes_large_max} bytes (constant-factor encoding bloat)`, + ); + } + // Heap gate only when measured (--expose-gc present) AND a budget exists. + if ( + base.heap_budget !== undefined && + r.heap_ratio !== null && + r.heap_ratio >= base.heap_budget + ) { + failures.push( + `${r.scenario}: retained-heap ratio ${r.heap_ratio} >= budget ${base.heap_budget} ` + + `(heap ${r.heap_bytes_small}->${r.heap_bytes_large})`, + ); + } + process.stdout.write(JSON.stringify(r) + '\n'); + } + if (failures.length > 0) { + for (const f of failures) process.stderr.write(`[cfg --check] FAIL: ${f}\n`); + process.exit(1); + } + process.stderr.write(`[cfg --check] PASS (${results.length} scenarios)\n`); +} diff --git a/gitnexus/bench/scope-capture/baselines.json b/gitnexus/bench/scope-capture/baselines.json index bb0a3f5f4..f6f68252a 100644 --- a/gitnexus/bench/scope-capture/baselines.json +++ b/gitnexus/bench/scope-capture/baselines.json @@ -18,10 +18,10 @@ "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression)." }, "cpp": { - "fingerprint": "f56625342f73e182170e2c964d538e316c079fa6e9466a7f076bff2ebcf8aac4", + "fingerprint": "9b5b4393d158d76dcf1ef9807e0326462c45a5266310f0ae7894d017f3858219", "scaling_budget": 1.5, "_added": "#1956: cpp added to the scope-capture bench (was UNBENCHED). Heritage-bearing scale source (: public Base, public Mixin) drives emitCppInheritanceCaptures at scale. Adding it exposed + fixed a pre-existing O(n^2) findNodeAtRange root-walk in cpp/captures.ts (~12 sites, threaded c.node, byte-identical over 263 cpp-* fixtures); scaling 2.30 -> 1.12.", - "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression).", + "_rebaselined": "#1919 open-language coverage: new lang-resolution fixtures + intended capture additions (F5/F9 c-cpp, F26/F28/F29 dart, F47/F48/F49/F51/F52 kotlin, F75/F79 swift). Fingerprint-only drift; scaling_ratio ~1.0 (linear, no perf regression). #2094: deleted C++ declarations retain @declaration.is-deleted metadata; deleted operator and pointer-return shapes plus the expanded deleted-overload fixture are included. Intended capture drift; scaling remains linear (1.139 < 1.5).", "_note": "#1975: + cpp-out-of-line-class fixture, fixture_count 263->265. #1990: + cpp-adl-ns-plus-hidden-friend-same-name fixture (ADL hidden-friend + namespace-callable merge parity test). Pure fixture-corpus drift — no scope-extractor change; existing fixtures' captures byte-identical. fixture_count 265->267. #1995: + cpp-union-nested-tail-collision and cpp-anon-ns-tail-collision fixtures — pure fixture-corpus drift; fixture_count 270->272, fingerprint 538e8be->d63ded6. #1993: + cpp-cross-namespace-same-tail fixture — pure fixture-corpus drift; fixture_count 272->273, fingerprint d63ded6->6d6207ae. #2077 review follow-up: cpp-member-lattice adds cross-file, qualified-base, nested-template, inherited-using, this-receiver, and non-virtual-override regressions; fixture_count 274->275. Capture scaling remains linear (1.134 < 1.5)." }, "csharp": { diff --git a/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs b/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs index bbfccb92e..bd72c7b55 100755 --- a/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs +++ b/gitnexus/hooks/antigravity/gitnexus-antigravity-hook.cjs @@ -91,10 +91,20 @@ function hasGitNexusServerOwner(gitNexusDir) { return hasGitNexusDbLockedByGitNexusServer(path.join(gitNexusDir, 'lbug'), process.pid); } +/** + * Whether opt-in diagnostics should be written to the hook's stderr. Strict + * hook runners validate hook output, so normal, non-error skip paths must stay + * silent unless the operator explicitly asks for diagnostics via GITNEXUS_DEBUG. + * See issue #1913. + */ +function isDebugEnabled() { + return process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; +} + function extractAugmentContext(stderr) { const output = (stderr || '').trim(); const marker = output.indexOf('[GitNexus]'); - const debug = process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; + const debug = isDebugEnabled(); if (debug && output.length > 0) { // Emit the FULL discarded prefix (everything before the marker, or all of // it when no marker is present) so suppressed diagnostics — LadybugDB lock @@ -258,8 +268,14 @@ function buildAfterToolContext(input) { if (/\bgit\s+(commit|merge|rebase|cherry-pick|pull)(\s|$)/.test(command)) { const hint = buildStaleIndexHint(gitNexusDir, cwd); if (hint) { - process.stderr.write(`${hint}\n`); + // The hint always reaches the agent via additionalContext (parts). Mirror + // it to stderr (for terminal users) only under GITNEXUS_DEBUG, so strict + // hook runners see no unexpected output on this normal path (#1913). The + // claude hook never mirrored this to stderr — this aligns the two adapters. parts.push(hint); + if (isDebugEnabled()) { + process.stderr.write(`${hint}\n`); + } } } } @@ -268,14 +284,32 @@ function buildAfterToolContext(input) { } function runAugment(gitNexusDir, cwd, pattern) { - if (hasGitNexusServerOwner(gitNexusDir)) { - process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + // Acquire the per-repo slot BEFORE the DB-owner probe (#2163): the probe + // itself spawns lsof/ps, so it must be bounded by the same ≤3-per-repo cap + // as the augment, or concurrent sessions fan out unbounded probe + // subprocesses. The cheap guards (extractPattern, gitNexusDir lookup) run in + // buildAfterToolContext before this — moving the acquire any earlier would + // churn slot files on tool calls that never probe. + const release = acquireHookSlot(gitNexusDir); + if (!release) { + // Normal skip path: all per-repo hook slots are held by concurrent + // sessions. Stay silent for strict hook runners (issue #1913); surface + // the reason only under GITNEXUS_DEBUG. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: hook slots saturated\n'); + } return ''; } - const release = acquireHookSlot(gitNexusDir); - if (!release) return ''; - const cliPath = resolveCliPath(); try { + if (hasGitNexusServerOwner(gitNexusDir)) { + // Normal skip path: the MCP server owns the DB. Stay silent for strict + // hook runners (issue #1913); surface the reason only under GITNEXUS_DEBUG. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + } + return ''; + } + const cliPath = resolveCliPath(); const child = runGitNexusCli(cliPath, ['augment', '--', pattern], cwd, 7000); if (!child.error && child.status === 0) { return extractAugmentContext(child.stderr || ''); @@ -338,7 +372,7 @@ function main() { const handler = handlers[input.hook_event_name || '']; if (handler) handler(input); } catch (err) { - if (process.env.GITNEXUS_DEBUG) { + if (isDebugEnabled()) { console.error('GitNexus antigravity hook error:', (err.message || '').slice(0, 200)); } } diff --git a/gitnexus/hooks/claude/gitnexus-hook.cjs b/gitnexus/hooks/claude/gitnexus-hook.cjs index 8bfa49381..c37895f1d 100755 --- a/gitnexus/hooks/claude/gitnexus-hook.cjs +++ b/gitnexus/hooks/claude/gitnexus-hook.cjs @@ -110,10 +110,20 @@ function hasGitNexusServerOwner(gitNexusDir) { return hasGitNexusDbLockedByGitNexusServer(path.join(gitNexusDir, 'lbug'), process.pid); } +/** + * Whether opt-in diagnostics should be written to the hook's stderr. Strict + * hook runners (e.g. Codex `PreToolUse`) validate hook output, so normal, + * non-error skip paths must stay silent unless the operator explicitly asks + * for diagnostics via GITNEXUS_DEBUG. See issue #1913. + */ +function isDebugEnabled() { + return process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; +} + function extractAugmentContext(stderr) { const output = (stderr || '').trim(); const marker = output.indexOf('[GitNexus]'); - const debug = process.env.GITNEXUS_DEBUG === '1' || process.env.GITNEXUS_DEBUG === 'true'; + const debug = isDebugEnabled(); if (debug && output.length > 0) { // Emit the FULL discarded prefix (everything before the marker, or all of // it when no marker is present) so suppressed diagnostics — KuzuDB lock @@ -249,17 +259,35 @@ function handlePreToolUse(input) { const pattern = extractPattern(toolName, toolInput); if (!pattern || pattern.length < 3) return; - if (hasGitNexusServerOwner(gitNexusDir)) { - process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + + // Acquire the per-repo slot BEFORE the DB-owner probe (#2163): the probe + // itself spawns lsof/ps, so it must be bounded by the same ≤3-per-repo cap + // as the augment, or concurrent sessions fan out unbounded probe + // subprocesses. Keep the acquire right after the cheap guards above — + // moving it earlier would churn slot files on tool calls that never probe. + const release = acquireHookSlot(gitNexusDir); + if (!release) { + // Normal skip path: all per-repo hook slots are held by concurrent + // sessions. Stay silent for strict hook runners (issue #1913); surface + // the reason only when diagnostics are explicitly requested. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: hook slots saturated\n'); + } return; } - const release = acquireHookSlot(gitNexusDir); - if (!release) return; - - const cliPath = resolveCliPath(); let result = ''; try { + if (hasGitNexusServerOwner(gitNexusDir)) { + // Normal skip path: the MCP server owns the DB, so the CLI augment would + // contend on the lock. Stay silent for strict hook runners (issue #1913); + // surface the reason only when diagnostics are explicitly requested. + if (isDebugEnabled()) { + process.stderr.write('[GitNexus] augment skipped: MCP server owns DB\n'); + } + return; + } + const cliPath = resolveCliPath(); const child = runGitNexusCli(cliPath, ['augment', '--', pattern], cwd, 7000); if (!child.error && child.status === 0) { result = extractAugmentContext(child.stderr || ''); @@ -361,7 +389,7 @@ function main() { const handler = handlers[input.hook_event_name || '']; if (handler) handler(input); } catch (err) { - if (process.env.GITNEXUS_DEBUG) { + if (isDebugEnabled()) { console.error('GitNexus hook error:', (err.message || '').slice(0, 200)); } } diff --git a/gitnexus/hooks/claude/hook-db-lock-probe.cjs b/gitnexus/hooks/claude/hook-db-lock-probe.cjs index 752c114a7..5c67804b1 100644 --- a/gitnexus/hooks/claude/hook-db-lock-probe.cjs +++ b/gitnexus/hooks/claude/hook-db-lock-probe.cjs @@ -11,6 +11,22 @@ * * Fail-open on most errors; fail-closed only on lsof ETIMEDOUT (Unix) or * PowerShell ETIMEDOUT (Windows), matching the hook contract. + * + * Unix subprocess containment contract (#2163): + * - lsof/ps are wrapped in coreutils `timeout`/`gtimeout` when a working + * wrapper is found (`timeout -k 1 lsof ...`). If this hook process + * is itself SIGKILLed (e.g. by the runner's 10s hook timeout) the wrapper + * survives, SIGTERMs its child at the budget (2s lsof / 1s ps) and SIGKILLs + * it 1s later — orphan lifetime is bounded at ~3s instead of unbounded. + * - GITNEXUS_HOOK_TIMEOUT_PATH: the sentinel value `disabled` switches the + * wrapper off deterministically; any other value is adopted only when it + * exists AND passes a one-shot `-k` self-test — otherwise resolution FALLS + * THROUGH to the built-in candidate list (first self-test pass wins), so + * no malformed value of any shape can silently disable orphan containment. + * - The gitnexus server is lazy-open + sticky-hold: an idle MCP server holds + * ZERO lbug fds until the repo's first MCP query, then keeps the fd open. + * A probe before that first query is therefore always false — a known, + * pre-existing race, not a bug in this probe. */ const fs = require('fs'); @@ -46,6 +62,80 @@ function resolveHookBinary(tool) { return tool; } +// Sentinel: +// undefined = not resolved yet (resolve lazily, on first lsof/ps fallback) +// string = self-tested coreutils timeout/gtimeout path (use as wrapper) +// null = no usable wrapper (disabled, none found, or self-test failed) +let unixGuardTimeoutCache; + +/** + * Resolve a coreutils `timeout`/`gtimeout` binary to wrap lsof/ps with + * (#2163). Dead code on Windows (the win32 dispatch returns earlier). + * + * GITNEXUS_HOOK_TIMEOUT_PATH semantics: the sentinel `disabled` turns the + * wrapper off; any other value is only a CANDIDATE — an existing file path + * is tried first, but it must pass the `-k` self-test to be adopted. On any + * failure (non-existent path, directory, non-executable file, wrapper + * without `-k` support, …) resolution falls through to the built-in + * candidates below, tried in order, first self-test pass wins. This is + * strictly stronger than the sibling GITNEXUS_HOOK_LSOF_PATH / + * GITNEXUS_HOOK_PS_PATH overrides (which only check existence): no bad env + * value of ANY shape can silently disable orphan containment. + * + * Lazy self-test: candidates are probed only when the lsof/ps fallback is + * first reached, and the result is memoized. A candidate is adopted only + * when `timeout -k 1 1 /bin/sh -c :` exits 0. This rejects wrappers that do + * not support the coreutils `-k` flag — busybox <1.34, toybox, broken + * symlinks — which would otherwise exit with a usage error without ever + * running lsof, silently converting the lsof-ETIMEDOUT fail-closed contract + * into fail-open (#1492 regression). Only when EVERY candidate fails does + * the probe fall back to the unwrapped status quo (memoized null). + * busybox ≥1.34 passes the test and is fully usable (capability, not + * identity, decides). + */ +function passesGuardSelfTest(guard) { + try { + const selfTest = spawnSync(guard, ['-k', '1', '1', '/bin/sh', '-c', ':'], { + encoding: 'utf-8', + timeout: 3000, + stdio: ['ignore', 'ignore', 'ignore'], + windowsHide: true, + }); + return !selfTest.error && selfTest.status === 0; + } catch { + return false; + } +} + +function resolveUnixGuardTimeout() { + if (unixGuardTimeoutCache !== undefined) return unixGuardTimeoutCache; + unixGuardTimeoutCache = null; + const fromEnv = process.env.GITNEXUS_HOOK_TIMEOUT_PATH; + const trimmed = fromEnv ? String(fromEnv).trim() : ''; + if (trimmed === 'disabled') return unixGuardTimeoutCache; + const candidates = []; + if (trimmed && fs.existsSync(trimmed)) candidates.push(trimmed); + for (const builtin of [ + '/usr/bin/timeout', + '/bin/timeout', + '/opt/homebrew/bin/gtimeout', + '/usr/local/bin/gtimeout', + ]) { + try { + if (fs.existsSync(builtin)) candidates.push(builtin); + } catch { + /* ignore */ + } + } + for (const candidate of candidates) { + if (passesGuardSelfTest(candidate)) { + unixGuardTimeoutCache = candidate; + break; + } + } + return unixGuardTimeoutCache; +} + function resolveWindowsPowerShellPath() { const fromEnv = process.env.GITNEXUS_HOOK_POWERSHELL_PATH; if (fromEnv && String(fromEnv).trim() && fs.existsSync(String(fromEnv).trim())) { @@ -188,20 +278,46 @@ function linuxProcScanFindGitNexusServer(dbPathAbs, myPid) { } function unixLsofPsFindGitNexusServer(dbPathAbs, myPid) { + const guard = resolveUnixGuardTimeout(); const lsofPath = resolveHookBinary('lsof'); - const lsof = spawnSync(lsofPath, ['-nP', '-t', '--', dbPathAbs], { + // The spawnSync timeouts below (lsof 1000ms / ps 500ms) are deliberately + // SHORTER than the wrapper budgets (2s / 1s): on the supervised path Node's + // SIGTERM always fires first, so `error.code === 'ETIMEDOUT'` and the + // fail-closed contract are untouched. The wrapper only matters once this + // hook process has been SIGKILLed and can no longer deliver that SIGTERM. + const [lsofCmd, lsofArgs] = guard + ? [guard, ['-k', '1', '2', lsofPath, '-nP', '-t', '--', dbPathAbs]] + : [lsofPath, ['-nP', '-t', '--', dbPathAbs]]; + const lsof = spawnSync(lsofCmd, lsofArgs, { encoding: 'utf-8', timeout: 1000, stdio: ['ignore', 'pipe', 'ignore'], windowsHide: true, }); if (lsof.error) return lsof.error.code === 'ETIMEDOUT'; + // Guard-mediated deaths map to "unresponsive holder" (fail-closed). Three + // result shapes, verified against coreutils 9.1: + // - signal-death: when `-k` escalates to SIGKILL, coreutils timeout + // SELF-RAISES the signal, so spawnSync reports {status: null, signal} + // with no .error (spawnSync's own ETIMEDOUT was handled above). The + // same shape appears when this hook is frozen >2s (SIGSTOP, laptop + // suspend) and the guard expires while it sleeps. By construction, a + // guard-wrapped probe that died by signal without spawnSync ETIMEDOUT + // is a budget/kill outcome. + // - 124: budget expired and the child exited after the plain SIGTERM. + // - 137: NOT the coreutils -k path — only exit-code-propagating wrappers, + // or a child SIGKILLed externally (e.g. the OOM killer). + if (guard && lsof.status === null && lsof.signal) return true; + if (guard && (lsof.status === 124 || lsof.status === 137)) return true; const pids = (lsof.stdout || '').split(/\s+/).filter(Boolean); const psPath = resolveHookBinary('ps'); for (const pid of pids) { if (Number(pid) === myPid) continue; - const ps = spawnSync(psPath, ['-p', pid, '-o', 'command='], { + const [psCmd, psArgs] = guard + ? [guard, ['-k', '1', '1', psPath, '-p', pid, '-o', 'command=']] + : [psPath, ['-p', pid, '-o', 'command=']]; + const ps = spawnSync(psCmd, psArgs, { encoding: 'utf-8', timeout: 500, stdio: ['ignore', 'pipe', 'ignore'], @@ -211,6 +327,11 @@ function unixLsofPsFindGitNexusServer(dbPathAbs, myPid) { if (ps.error.code === 'ETIMEDOUT') return true; continue; } + // Same guard-mediated-death mapping as the lsof call above (signal-death + // from the -k escalation or a frozen hook; 124 budget expiry; 137 only + // for exit-code-propagating wrappers / external SIGKILL). + if (guard && ps.status === null && ps.signal) return true; + if (guard && (ps.status === 124 || ps.status === 137)) return true; if (isGitNexusServerCommand(ps.stdout || '')) return true; } return false; diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 3bddf0e9c..dbc61e4c9 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,12 +1,12 @@ { "name": "gitnexus", - "version": "1.6.6", + "version": "1.6.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.6.6", + "version": "1.6.7", "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { @@ -15,6 +15,7 @@ "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -28,13 +29,14 @@ "js-yaml": "^4.1.1", "jsonc-parser": "^3.3.1", "mnemonist": "^0.40.3", + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0", "onnxruntime-common": "^1.26.0", "onnxruntime-node": "^1.24.0", "pandemonium": "^2.4.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tree-sitter": "0.21.1", - "tree-sitter-c": "0.21.4", "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "0.23.2", "tree-sitter-go": "^0.23.0", @@ -51,6 +53,7 @@ "gitnexus": "dist/cli/index.js" }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", @@ -65,11 +68,6 @@ }, "engines": { "node": ">=22.0.0" - }, - "optionalDependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.0", - "tree-sitter-kotlin": "^0.3.8" } }, "../gitnexus-shared": { @@ -2036,6 +2034,16 @@ "@types/node": "*" } }, + "node_modules/@types/busboy": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/busboy/-/busboy-1.5.4.tgz", + "integrity": "sha512-kG7WrUuAKK0NoyxfQHsVE6j1m01s6kMma64E+OZenQABMQyTJop1DumUWcLwAQ2JzpefU7PDYoRDKl8uZosFjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -2143,9 +2151,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.9.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.1.tgz", - "integrity": "sha512-xfrlY7UD5rMJk3ZVJP8BNzS28J36YJg+xp+LPXV1TdWxr8uMH5A860QNxYDGQe/ylDSgjxE52Q9VnO7p75tJxg==", + "version": "25.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.2.tgz", + "integrity": "sha512-G05zqtJhcDLb8uslf5EjCxXg9G1KQxiV8OS0R26IC//Eoyitzqe8z37I7cqvnZlrlSfgocQRfSn/AHBZJJFyGw==", "license": "MIT", "dependencies": { "undici-types": ">=7.24.0 <7.24.7" @@ -2546,6 +2554,17 @@ "node": "18 || 20 || >=22" } }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3804,9 +3823,19 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", - "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.2.0.tgz", + "integrity": "sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -5208,6 +5237,14 @@ "dev": true, "license": "MIT" }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -5360,25 +5397,6 @@ "node-gyp-build": "^4.8.0" } }, - "node_modules/tree-sitter-c": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/tree-sitter-c/-/tree-sitter-c-0.21.4.tgz", - "integrity": "sha512-IahxFIhXiY15SUlrt2upBiKSBGdOaE1fjKLK1Ik5zxqGHf6T1rvr3IJrovbsE5sXhypx7Hnmf50gshsppaIihA==", - "hasInstallScript": true, - "license": "MIT", - "dependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.1" - }, - "peerDependencies": { - "tree-sitter": "^0.21.0" - }, - "peerDependenciesMeta": { - "tree_sitter": { - "optional": true - } - } - }, "node_modules/tree-sitter-c-sharp": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/tree-sitter-c-sharp/-/tree-sitter-c-sharp-0.23.1.tgz", @@ -5474,33 +5492,6 @@ } } }, - "node_modules/tree-sitter-kotlin": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/tree-sitter-kotlin/-/tree-sitter-kotlin-0.3.8.tgz", - "integrity": "sha512-A4obq6bjzmYrA+F0JLLoheFPcofFkctNaZSpnDd+GPn1SfVZLY4/GG4C0cYVBTOShuPBGGAOPLM1JWLZQV4m1g==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "dependencies": { - "node-addon-api": "^7.1.0", - "node-gyp-build": "^4.8.0" - }, - "peerDependencies": { - "tree-sitter": "^0.21.0" - }, - "peerDependenciesMeta": { - "tree_sitter": { - "optional": true - } - } - }, - "node_modules/tree-sitter-kotlin/node_modules/node-addon-api": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", - "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", - "license": "MIT", - "optional": true - }, "node_modules/tree-sitter-php": { "version": "0.23.12", "resolved": "https://registry.npmjs.org/tree-sitter-php/-/tree-sitter-php-0.23.12.tgz", diff --git a/gitnexus/package.json b/gitnexus/package.json index 0b67f42a0..7570a3bc9 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.6.6", + "version": "1.6.7", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", @@ -49,15 +49,17 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:cross-platform": "tsx scripts/run-cross-platform.ts", - "postinstall": "node scripts/materialize-vendor-grammars.cjs && node scripts/build-tree-sitter-dart.cjs && node scripts/build-tree-sitter-proto.cjs && node scripts/build-tree-sitter-swift.cjs && node scripts/build-tree-sitter-kotlin.cjs", + "postinstall": "node scripts/build-tree-sitter-grammars.cjs", + "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", - "prepack": "node scripts/build.js" + "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js" }, "dependencies": { "@huggingface/transformers": "^4.1.0", "@ladybugdb/core": "^0.17.0", "@modelcontextprotocol/sdk": "^1.0.0", "@scarf/scarf": "^1.4.0", + "busboy": "^1.6.0", "cli-progress": "^3.12.0", "commander": "^14.0.3", "cors": "^2.8.5", @@ -71,13 +73,14 @@ "js-yaml": "^4.1.1", "jsonc-parser": "^3.3.1", "mnemonist": "^0.40.3", + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0", "onnxruntime-common": "^1.26.0", "onnxruntime-node": "^1.24.0", "pandemonium": "^2.4.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tree-sitter": "0.21.1", - "tree-sitter-c": "0.21.4", "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "0.23.2", "tree-sitter-go": "^0.23.0", @@ -91,12 +94,8 @@ "uuid": "^14.0.0", "@inquirer/prompts": "^8.0.0" }, - "optionalDependencies": { - "node-addon-api": "^8.0.0", - "node-gyp-build": "^4.8.0", - "tree-sitter-kotlin": "^0.3.8" - }, "devDependencies": { + "@types/busboy": "^1.5.4", "@types/cli-progress": "^3.11.6", "@types/cors": "^2.8.17", "@types/express": "^5.0.6", diff --git a/gitnexus/scripts/assert-publish-grammar-coverage.cjs b/gitnexus/scripts/assert-publish-grammar-coverage.cjs new file mode 100644 index 000000000..b523f68e8 --- /dev/null +++ b/gitnexus/scripts/assert-publish-grammar-coverage.cjs @@ -0,0 +1,204 @@ +#!/usr/bin/env node +/** + * Publish guard: every vendored tree-sitter grammar must ship a loadable binding. + * + * The npm tarball includes gitnexus/vendor/ (package.json `files`). A grammar is + * "covered" on a platform-arch tuple if EITHER a prebuild ships for it OR the + * grammar's full source-build set ships (so the install can source-build it, + * toolchain permitting). A future lean publish — dropping the ~50 MB of generated + * source to ship prebuilds only — is safe ONLY once every grammar has all six + * prebuilds; doing it while any grammar still lacks a prebuild would ship a + * grammar with NO loadable binding (neither prebuild nor buildable source) → that + * language is silently dead for users. + * + * HOW SOURCE INCLUSION IS DECIDED. The `files` allow-list OVERRIDES `.npmignore` + * for the vendored subtree (verified: an active "vendor/(star-star)/src/parser.c" + * in .npmignore does NOT drop it from `npm pack`). So `.npmignore` can never + * exclude vendored source — the ONLY lever is the `files` field. A broad `vendor` + * ships the whole subtree (source + prebuilds); a lean publish narrows `files` to + * non-source subpaths. This guard therefore reads `files` directly rather than + * shelling out to `npm pack` (which, in prepack, would re-enter this guard and, + * on npm versions that don't honor --ignore-scripts for prepare/prepack, run the + * full build — slow enough to time out and fragile). + * + * Wired via `prepack`, so it fails `npm pack` / `npm publish` if the invariant is + * violated. + */ +const fs = require('fs'); +const path = require('path'); + +const TUPLES = [ + 'linux-x64', + 'linux-arm64', + 'darwin-x64', + 'darwin-arm64', + 'win32-x64', + 'win32-arm64', +]; + +// Source-build inputs (relative to vendor//) whose presence makes a grammar +// source-buildable. Per-grammar we only require the ones that exist on disk (e.g. +// tree-sitter-c has no external scanner.c). +const SOURCE_BUILD_REL = [ + 'binding.gyp', + 'bindings/node/binding.cc', + 'src/parser.c', + 'src/scanner.c', + 'src/tree_sitter/parser.h', +]; + +/** + * Does the package.json `files` allow-list ship the WHOLE vendor subtree (and + * therefore the vendored grammar source)? A bare `vendor` (optionally with a + * trailing slash or `/**`/`/*`) includes everything under vendor/. A lean publish + * replaces that with non-source subpaths, so this returns false and grammars must + * then rely on prebuilds. + */ +function filesShipsVendorSource(filesField) { + return (filesField || []).some((f) => { + const n = String(f) + .replace(/\\/g, '/') + .replace(/\/+$/, '') + .replace(/\/\*\*?$/, ''); + return n === 'vendor'; + }); +} + +/** The on-disk source-build inputs for a grammar (relative paths). */ +function sourceBuildSet(grammarDir) { + return SOURCE_BUILD_REL.filter((rel) => fs.existsSync(path.join(grammarDir, rel))); +} + +/** True when a grammar can be source-built from its vendored files (has gyp + parser). */ +function isBuildableFromSource(grammarDir) { + const set = sourceBuildSet(grammarDir); + return set.includes('binding.gyp') && set.includes('src/parser.c'); +} + +/** Count platform-arch tuples with a committed prebuilt .node on disk. */ +function countPrebuiltTuples(grammarDir) { + const pdir = path.join(grammarDir, 'prebuilds'); + let n = 0; + for (const t of TUPLES) { + const td = path.join(pdir, t); + try { + if (fs.statSync(td).isDirectory() && fs.readdirSync(td).some((f) => f.endsWith('.node'))) { + n++; + } + } catch { + /* tuple dir absent — not covered */ + } + } + return n; +} + +/** + * Pure core (exported for tests). `grammars` is a list of + * `{ name, prebuilt: 0..6, shipsSource: boolean }`. Returns human-readable + * problem strings; an empty array means the pack is publish-safe. + */ +function findCoverageProblems({ grammars }) { + const problems = []; + for (const g of grammars) { + if (g.prebuilt < 6 && !g.shipsSource) { + const missing = 6 - g.prebuilt; + problems.push( + `${g.name}: ${g.prebuilt}/6 prebuilds and its vendored source is not shipped ` + + `(the package.json \`files\` field excludes it, or it is not buildable) — would ship ` + + `with no loadable binding on ${missing} platform-arch tuple(s).`, + ); + } + } + return problems; +} + +/** + * Stray local source-build outputs under `vendor//build/`. These would + * ship in the tarball (`files: ["vendor"]` overrides .gitignore/.npmignore) AND + * shadow the committed prebuilds — `node-gyp-build` resolves `build/Release` + * BEFORE `prebuilds/`, so a consumer on the publisher's platform would load the + * stray (possibly stale/wrong) binding instead of the curated prebuild. The + * build dir is gitignored and only appears if a maintainer source-built locally + * (e.g. on a no-prebuild platform); refuse to publish it. (#2144 review.) + */ +function findStrayBuildArtifacts(vendorDir) { + if (!fs.existsSync(vendorDir)) return []; + return fs + .readdirSync(vendorDir) + .filter((d) => /^tree-sitter-/.test(d)) + .filter((d) => fs.existsSync(path.join(vendorDir, d, 'build'))) + .map((d) => `vendor/${d}/build`); +} + +function collectGrammars(vendorDir, shipsVendorSource) { + if (!fs.existsSync(vendorDir)) return []; + return fs + .readdirSync(vendorDir) + .filter((d) => /^tree-sitter-/.test(d)) + .map((name) => { + const dir = path.join(vendorDir, name); + return { + name, + prebuilt: countPrebuiltTuples(dir), + // Source ships when `files` includes the vendor subtree AND the grammar + // actually carries a buildable source set on disk. + shipsSource: shipsVendorSource && isBuildableFromSource(dir), + }; + }); +} + +function main() { + const gitnexusRoot = path.join(__dirname, '..'); + const vendorDir = path.join(gitnexusRoot, 'vendor'); + const pkg = JSON.parse(fs.readFileSync(path.join(gitnexusRoot, 'package.json'), 'utf8')); + const shipsVendorSource = filesShipsVendorSource(pkg.files); + + const grammars = collectGrammars(vendorDir, shipsVendorSource); + if (grammars.length === 0) { + console.error(`[publish-guard] No vendored tree-sitter grammars found under ${vendorDir}.`); + process.exit(1); + } + + const stray = findStrayBuildArtifacts(vendorDir); + if (stray.length > 0) { + console.error( + '[publish-guard] Refusing to publish — stray source-build output under vendor/ would\n' + + 'ship and shadow the committed prebuilds (node-gyp-build loads build/Release before\n' + + 'prebuilds/):', + ); + for (const s of stray) console.error(` - ${s}`); + console.error('\nFix: remove it before packing, e.g. `rm -rf gitnexus/vendor/*/build`.'); + process.exit(1); + } + + const problems = findCoverageProblems({ grammars }); + if (problems.length > 0) { + console.error('[publish-guard] Refusing to publish — a vendored grammar would ship unusable:'); + for (const p of problems) console.error(` - ${p}`); + console.error( + '\nFix: either commit the missing prebuilds (run the build-tree-sitter-prebuilds\n' + + 'workflow) or keep the vendored source in the package.json `files` field.', + ); + process.exit(1); + } + + const sourceShippers = grammars.filter((g) => g.shipsSource).length; + console.log( + `[publish-guard] OK — ${grammars.length} vendored grammar(s) covered ` + + `(${sourceShippers} shipping source, ${grammars.length - sourceShippers} prebuilds-only).`, + ); +} + +if (require.main === module) main(); + +module.exports = { + findCoverageProblems, + findStrayBuildArtifacts, + filesShipsVendorSource, + isBuildableFromSource, + sourceBuildSet, + countPrebuiltTuples, + collectGrammars, + TUPLES, + SOURCE_BUILD_REL, +}; diff --git a/gitnexus/scripts/build-tree-sitter-dart.cjs b/gitnexus/scripts/build-tree-sitter-dart.cjs deleted file mode 100644 index d7542e253..000000000 --- a/gitnexus/scripts/build-tree-sitter-dart.cjs +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env node -/** - * Build tree-sitter-dart native binding in node_modules/ after materialize-vendor-grammars.cjs. - * Vendored source lives in vendor/ only; see #836 and #1728. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: skip the native rebuild entirely. Dart parsing becomes -// unavailable but `npm install gitnexus` finishes much faster on machines -// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0' -// (read as a string), and any other value all fall through to the rebuild. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-dart] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const dartDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-dart'); -const bindingGyp = path.join(dartDir, 'binding.gyp'); -const bindingNode = path.join(dartDir, 'build', 'Release', 'tree_sitter_dart_binding.node'); - -try { - if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { - process.exit(0); - } - - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-dart] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-dart] Dart parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', - ); - process.exit(0); - } - - console.log('[tree-sitter-dart] Building native binding...'); - execSync('npx node-gyp rebuild', { - cwd: dartDir, - stdio: 'pipe', - timeout: 180000, - }); - console.log('[tree-sitter-dart] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-dart] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-dart] Dart parsing will be unavailable. Non-Dart functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-grammars.cjs b/gitnexus/scripts/build-tree-sitter-grammars.cjs new file mode 100644 index 000000000..842bd0d99 --- /dev/null +++ b/gitnexus/scripts/build-tree-sitter-grammars.cjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +/** + * Activate the vendored tree-sitter native bindings IN PLACE under `vendor/`. + * One registry-driven script replaces the former per-grammar + * build-tree-sitter-.cjs files (they were ~95% identical). + * + * The grammars (tree-sitter-c/dart/proto/swift/kotlin) are loaded from + * `vendor//` by absolute path at runtime (see + * src/core/tree-sitter/vendored-grammars.ts) and are NEVER copied into + * node_modules — an undeclared package under node_modules is "extraneous" to + * every subsequent npm/npx reify, which prunes/relocates it (Windows + * `EPERM: …, symlink` + a silent grammar deletion on the 2nd run; #2111/#1728). + * + * For each grammar the resolution order is identical: + * 1. If the vendored source is absent (no binding.gyp) or the binding is + * already built, do nothing. + * 2. Prefer a committed prebuild for this platform-arch (toolchain-free) via + * node-gyp-build — `vendor//prebuilds/` ships all six tuples, so on a + * supported platform this returns immediately and writes nothing. + * 3. Otherwise source-build from the vendored grammar source (binding.gyp + + * src/) into `vendor//build/` (gitignored) so parsing still works on + * a toolchain host that lacks a matching prebuild. + * + * HARD INVARIANT: this runs in `gitnexus`'s postinstall, so it MUST NEVER throw + * or exit non-zero — a failure for any single grammar must not break the install. + * + * Opt-out: GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (strict '1') skips the OPTIONAL + * grammars only. tree-sitter-c is REQUIRED (it backstops upstream's 4/6 ARM + * prebuild gap, #2116) and is always built. + * + * Usage: + * node build-tree-sitter-grammars.cjs # all grammars (postinstall) + * node build-tree-sitter-grammars.cjs swift c # only the named grammars + */ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +// Registry. `display`/`ext` drive the human-readable warnings; `required` +// grammars ignore the opt-out gate. Insertion order == build order (c first). +const GRAMMARS = { + c: { required: true, display: 'C', ext: '.c' }, + dart: { required: false, display: 'Dart', ext: '.dart' }, + proto: { required: false, display: 'Proto', ext: '.proto' }, + swift: { required: false, display: 'Swift', ext: '.swift' }, + kotlin: { required: false, display: 'Kotlin', ext: '.kt/.kts' }, +}; + +const skipOptional = process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1'; + +function buildGrammar(short) { + const cfg = GRAMMARS[short]; + const tag = `[tree-sitter-${short}]`; + + if (!cfg.required && skipOptional) { + console.warn( + `${tag} Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). ${cfg.display} parsing will be unavailable until reinstalled without the env var.`, + ); + return; + } + + const dir = path.join(__dirname, '..', 'vendor', `tree-sitter-${short}`); + const bindingGyp = path.join(dir, 'binding.gyp'); + const bindingNode = path.join(dir, 'build', 'Release', `tree_sitter_${short}_binding.node`); + + try { + // Not materialized (no source), or already built — nothing to do. + if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { + return; + } + + // Prefer a committed prebuild for this platform-arch (no toolchain needed). + try { + require('node-gyp-build').path(dir); + return; + } catch { + // No matching prebuild — fall through to the source build below. + } + + // The hoisted build deps must be resolvable to source-build. + try { + require.resolve('node-addon-api'); + require.resolve('node-gyp-build'); + } catch (resolveErr) { + console.warn( + `${tag} Skipping build: hoisted build deps not resolvable (${resolveErr.message}).`, + ); + console.warn( + `${tag} ${cfg.display} parsing will be unavailable until a prebuild or toolchain is present.`, + ); + return; + } + + console.log(`${tag} No prebuild for this platform — building native binding from source...`); + execSync('npx node-gyp rebuild', { cwd: dir, stdio: 'pipe', timeout: 180000 }); + console.log(`${tag} Native binding built successfully`); + } catch (err) { + console.warn(`${tag} Could not build native binding:`, err.message); + console.warn( + `${tag} ${cfg.display} (${cfg.ext}) parsing will be unavailable. Non-${cfg.display} functionality is unaffected.`, + ); + } +} + +function main() { + const args = process.argv.slice(2).filter(Boolean); + const targets = args.length > 0 ? args : Object.keys(GRAMMARS); + for (const short of targets) { + if (!GRAMMARS[short]) { + console.warn(`[tree-sitter] Unknown grammar '${short}' — skipping.`); + continue; + } + // Defensive: never let an unexpected throw escape and fail the install. + try { + buildGrammar(short); + } catch (err) { + console.warn(`[tree-sitter-${short}] Unexpected build error (ignored): ${err.message}`); + } + } + // Hard guarantee: postinstall must never exit non-zero. + process.exit(0); +} + +if (require.main === module) main(); + +module.exports = { GRAMMARS, buildGrammar }; diff --git a/gitnexus/scripts/build-tree-sitter-kotlin.cjs b/gitnexus/scripts/build-tree-sitter-kotlin.cjs deleted file mode 100644 index e21a93766..000000000 --- a/gitnexus/scripts/build-tree-sitter-kotlin.cjs +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env node -/** - * Probe tree-sitter-kotlin native-binding availability at install time. - * - * Unlike Dart/Proto/Swift (vendored under vendor/ and materialized into - * node_modules/ at postinstall), tree-sitter-kotlin is a third-party npm - * `optionalDependency`. It ships SOURCE ONLY — no upstream `prebuilds/` dir — - * and its own `install` script runs `node-gyp-build`, which compiles the - * native binding from source via node-gyp. On a host without a C/C++ toolchain - * that build soft-fails: npm skips the optional dependency and the `gitnexus` - * install still succeeds. This probe surfaces a single, friendly install-time - * warning when the Kotlin binding is unavailable — whether npm pruned the - * optional dependency after a toolchain-less build failure (its dir is gone, - * which is the common case) or the dir survives but the binding won't load — - * instead of leaving a raw node-gyp error or a first-use runtime failure as the - * only signal. A deliberate opt-out (`--omit=optional`) stays silent. The probe - * does not copy, register, or mutate anything; the runtime require() path in - * parser-loader does the actual load. This probe MUST NEVER throw or exit - * non-zero — it must never break `gitnexus` install. - */ -const fs = require('fs'); -const path = require('path'); - -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-kotlin] Skipping native-binding probe (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1).', - ); - process.exit(0); -} - -const kotlinDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-kotlin'); - -// `--omit=optional` / `--no-optional` / `.npmrc omit=optional` surface to -// lifecycle scripts as `npm_config_omit` containing `optional` (a comma- or -// space-separated list, e.g. `dev,optional`). That is a deliberate opt-out, so -// an absent package for that reason should stay silent. Any OTHER absence means -// npm attempted the optional dependency's native build and pruned the package -// after it soft-failed (the toolchain-less case) — exactly when the guidance -// below is worth surfacing. -const omitsOptional = /(^|[,\s])optional([,\s]|$)/.test(process.env.npm_config_omit || ''); - -function warnKotlinUnavailable(err) { - if (err) { - console.warn('[tree-sitter-kotlin] Native-binding probe failed:', err.message); - } - console.warn( - '[tree-sitter-kotlin] Kotlin (.kt/.kts) parsing will be unavailable. Non-Kotlin functionality is unaffected.', - ); - console.warn( - '[tree-sitter-kotlin] This is expected on hosts without a C/C++ toolchain: tree-sitter-kotlin ships source only (no upstream prebuilt binaries) and compiles via node-gyp at install. Set GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 to skip this probe.', - ); -} - -try { - if (!fs.existsSync(path.join(kotlinDir, 'bindings', 'node', 'index.js'))) { - // The package never materialized. If the user deliberately omitted optional - // dependencies, stay silent — they opted out. Otherwise npm pruned the - // package after its native build soft-failed (no toolchain), and this is the - // dominant real-world failure case: surface the guidance the raw node-gyp - // error would otherwise be the only signal of. - if (!omitsOptional) { - warnKotlinUnavailable(); - } - process.exit(0); - } - - const nodeGypBuild = require('node-gyp-build'); - nodeGypBuild(kotlinDir); -} catch (err) { - // The package is present but its native binding can't be loaded (e.g. the dir - // survived with --ignore-scripts, or a partial/ABI-mismatched build). - warnKotlinUnavailable(err); - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-proto.cjs b/gitnexus/scripts/build-tree-sitter-proto.cjs deleted file mode 100644 index eaefd14e6..000000000 --- a/gitnexus/scripts/build-tree-sitter-proto.cjs +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env node -/** - * Build tree-sitter-proto native binding. - * - * Why this script exists: - * tree-sitter-proto is vendored under gitnexus/vendor/tree-sitter-proto/ - * and copied into node_modules/ by materialize-vendor-grammars.cjs. Previously, the vendored - * package had its own `dependencies` and `install` script, which caused - * npm to create `vendor/tree-sitter-proto/node_modules/` and - * `vendor/tree-sitter-proto/build/` during install. Those directories - * blocked `rmdir` on global-install upgrade, producing: - * - * ENOTEMPTY: directory not empty, rmdir - * '.../gitnexus/vendor/tree-sitter-proto/node_modules/node-addon-api' - * - * (See https://github.com/abhigyanpatwari/GitNexus/issues/836.) - * - * We stripped `dependencies` and the `install` script from the vendored - * package.json, hoisted `node-addon-api` and `node-gyp-build` into - * gitnexus's own optionalDependencies, and moved native compilation here. - * - * What this does: - * Runs `npx node-gyp rebuild` inside `node_modules/tree-sitter-proto/`. - * Build output lands in - * `node_modules/tree-sitter-proto/build/Release/tree_sitter_proto_binding.node` - * — under npm-managed territory, safe on upgrade. - * - * Mirrors the tree-sitter-dart build helper. Best-effort: if any - * precondition fails (optional dep absent, no toolchain, --ignore-scripts), - * warn and exit 0 so gitnexus install still succeeds. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: skip the native rebuild entirely. Proto parsing becomes -// unavailable but `npm install gitnexus` finishes much faster on machines -// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0' -// (read as a string), and any other value all fall through to the rebuild. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-proto] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Proto parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto'); -const bindingGyp = path.join(protoDir, 'binding.gyp'); -const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node'); - -try { - if (!fs.existsSync(bindingGyp)) { - // tree-sitter-proto is an optionalDependency; absent when install - // skipped optional deps or the file: dep was not resolved. - process.exit(0); - } - - // Skip if the native binding already exists (idempotent re-run). - if (fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Pre-flight: the hoisted build deps must be resolvable. - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-proto] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-proto] Proto parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', - ); - process.exit(0); - } - - console.log('[tree-sitter-proto] Building native binding...'); - execSync('npx node-gyp rebuild', { - cwd: protoDir, - stdio: 'pipe', - timeout: 180000, - }); - console.log('[tree-sitter-proto] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-proto] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-proto] Proto (.proto) parsing will be unavailable. Non-proto gitnexus functionality is unaffected.', - ); - // Exit 0: optionalDependency failures must not fail the gitnexus install. - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-swift.cjs b/gitnexus/scripts/build-tree-sitter-swift.cjs deleted file mode 100644 index cbdd6eb54..000000000 --- a/gitnexus/scripts/build-tree-sitter-swift.cjs +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env node -/** - * Probe tree-sitter-swift prebuild availability at install time. - * - * The vendored package ships platform prebuilds; node-gyp-build selects the - * correct binary at require time. This script calls node-gyp-build once - * against the materialized package so a missing-prebuild failure surfaces - * as an install-time warning (with the rest of the gitnexus install - * succeeding) rather than as a runtime error the first time Swift parsing - * is requested. The result is discarded — it does not copy, register, or - * mutate anything; the runtime require() path in parser-loader does the - * actual load. Running this probe here instead of an npm `install` script - * on the vendored package preserves the #836 hygiene (no scripts.install - * inside vendor/). - */ -const fs = require('fs'); -const path = require('path'); - -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn('[tree-sitter-swift] Skipping prebuild probe (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1).'); - process.exit(0); -} - -const swiftDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-swift'); - -try { - if (!fs.existsSync(path.join(swiftDir, 'bindings', 'node', 'index.js'))) { - process.exit(0); - } - - const nodeGypBuild = require('node-gyp-build'); - nodeGypBuild(swiftDir); -} catch (err) { - console.warn('[tree-sitter-swift] Prebuild probe failed:', err.message); - console.warn( - '[tree-sitter-swift] Swift parsing will be unavailable. Non-Swift functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/scripts/materialize-vendor-grammars.cjs b/gitnexus/scripts/materialize-vendor-grammars.cjs deleted file mode 100644 index 399696f9b..000000000 --- a/gitnexus/scripts/materialize-vendor-grammars.cjs +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env node -/** - * Copy vendored tree-sitter grammars into node_modules/ using real files (fs.cpSync). - * - * Published gitnexus used to declare these as optionalDependencies with - * `file:./vendor/...`, which makes npm symlink/junction vendor → node_modules on - * install. Windows without Developer Mode often fails with EPERM (#1728). - * - * Vendor trees stay read-only in gitnexus/vendor/; build artifacts must only - * land under node_modules/ (see #836). - */ -const fs = require('fs'); -const path = require('path'); - -const ROOT = path.join(__dirname, '..'); -const VENDORED_GRAMMARS = ['tree-sitter-dart', 'tree-sitter-proto', 'tree-sitter-swift']; - -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[gitnexus] Skipping vendored grammar materialize (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart/Proto/Swift parsing will be unavailable.', - ); - process.exit(0); -} - -for (const name of VENDORED_GRAMMARS) { - const src = path.join(ROOT, 'vendor', name); - const dest = path.join(ROOT, 'node_modules', name); - - if (!fs.existsSync(src)) { - console.warn(`[gitnexus] vendor/${name} missing; skipping materialize.`); - continue; - } - - // Sequence: copy src → partial; rename dest → backup; rename partial → dest; - // remove backup. If any step fails, restore from backup so a previously- - // materialized grammar is never lost. Targets the #1728 EPERM scenario plus - // narrower failure modes (Windows AV scanner racing on rename, EBUSY mid-swap). - const partial = `${dest}.materialize-tmp`; - const backup = `${dest}.materialize-bak`; - try { - fs.mkdirSync(path.join(ROOT, 'node_modules'), { recursive: true }); - fs.rmSync(partial, { recursive: true, force: true }); - fs.rmSync(backup, { recursive: true, force: true }); - fs.cpSync(src, partial, { recursive: true, verbatim: true }); - if (fs.existsSync(dest)) { - fs.renameSync(dest, backup); - } - try { - fs.renameSync(partial, dest); - } catch (renameErr) { - // Best-effort rollback: restore the previous dest from backup. - if (fs.existsSync(backup)) { - try { - fs.renameSync(backup, dest); - } catch { - // If rollback also fails, the prior backup directory still exists on - // disk — the catch block below surfaces both errors via the warning. - } - } - throw renameErr; - } - fs.rmSync(backup, { recursive: true, force: true }); - } catch (err) { - // Fail-soft: a single locked/inaccessible file (common on Windows) must not - // abort the whole gitnexus install. Matches build-tree-sitter-*.cjs pattern. - fs.rmSync(partial, { recursive: true, force: true }); - console.warn(`[gitnexus] Could not materialize vendor/${name}: ${err.message}`); - console.warn( - `[gitnexus] ${name} parsing will be unavailable. Other functionality is unaffected.`, - ); - } -} diff --git a/gitnexus/skills/gitnexus-guide.md b/gitnexus/skills/gitnexus-guide.md index b81900b5e..a54337879 100644 --- a/gitnexus/skills/gitnexus-guide.md +++ b/gitnexus/skills/gitnexus-guide.md @@ -38,7 +38,38 @@ For any task involving code understanding, debugging, impact analysis, or refact | `detect_changes` | Git-diff impact — what do your current changes affect | | `rename` | Multi-file coordinated rename with confidence-tagged edits | | `cypher` | Raw graph queries (read `gitnexus://repo/{name}/schema` first) | -| `list_repos` | Discover indexed repos | +| `list_repos` | Discover indexed repos (paginated — `limit`/`offset`) | + +### Paginating `list_repos` + +`list_repos` is paginated so a large registry is not truncated by MCP/LLM token limits. It takes optional `limit` (default **50**, max **200**) and `offset`, and returns: + +```jsonc +{ + "repositories": [ + { "name": "...", "path": "...", "indexedAt": "...", "lastCommit": "...", "stats": { } } + ], + "pagination": { + "total": 437, + "limit": 50, + "offset": 0, + "returned": 50, + "hasMore": true, + "nextOffset": 50 + } +} +``` + +To enumerate **every** repository, keep calling with `offset` set to `pagination.nextOffset` until `hasMore` is `false`: + +```text +list_repos {} → repos 1–50, nextOffset 50, hasMore true +list_repos { offset: 50 } → repos 51–100, nextOffset 100, hasMore true +… +list_repos { offset: 400 } → repos 401–437, hasMore false (done) +``` + +Notes: `offset` ≥ `total` returns an empty page (with `total` still reported). Out-of-range or malformed `limit`/`offset` (non-integer, `limit` outside `[1, 200]`, `offset < 0`) are rejected with a clear error — `limit` above the max is rejected, not silently capped. The order is deterministic (lower-cased name, then path), so paging never skips or duplicates an entry while the registry is unchanged. ## Resources Reference diff --git a/gitnexus/src/cli/analyze-config.ts b/gitnexus/src/cli/analyze-config.ts index 8fbe40feb..63a328845 100644 --- a/gitnexus/src/cli/analyze-config.ts +++ b/gitnexus/src/cli/analyze-config.ts @@ -56,6 +56,7 @@ type ValueKind = | 'boolean' | 'boolean-negate' | 'string' + | 'string-array' | 'numeric-string' | 'embeddings' | 'branch'; @@ -84,6 +85,7 @@ const KEY_SPECS: Record = { skipContextFiles: { target: 'skipAgentsMd', kind: 'boolean' }, skipAiContext: { target: 'skipAgentsMd', kind: 'boolean' }, skipSkills: { target: 'skipSkills', kind: 'boolean' }, + pdg: { target: 'pdg', kind: 'boolean' }, indexOnly: { target: 'indexOnly', kind: 'boolean' }, stats: { target: 'stats', kind: 'boolean' }, noStats: { target: 'stats', kind: 'boolean-negate' }, @@ -99,6 +101,12 @@ const KEY_SPECS: Record = { embeddingBatchSize: { target: 'embeddingBatchSize', kind: 'numeric-string' }, embeddingSubBatchSize: { target: 'embeddingSubBatchSize', kind: 'numeric-string' }, embeddingDevice: { target: 'embeddingDevice', kind: 'string' }, + // #1589/#1852 residual — extra fetch-wrapper function names to treat as HTTP + // consumers. The auto-detector only flags functions that call the bare global + // `fetch()`; a wrapper built on axios / a custom client, or named outside the + // built-in convention set, is otherwise invisible to route_map consumers. + // Listing it here adds it to the cross-file consumer scan. + fetchWrappers: { target: 'fetchWrappers', kind: 'string-array' }, }; /** Top-level container key for the nested form; not itself an `AnalyzeOptions` field. */ @@ -230,6 +238,41 @@ const normalizeValue = (kind: ValueKind, value: unknown, key: string): unknown = } return trimmed; } + case 'string-array': { + // Generic shared validator — `source` already names the config key, so + // messages here stay key-agnostic (no fetch-wrapper coupling in the + // shared normalizer; #1589/#1852 review F7). + if (!Array.isArray(value)) { + throw new GitNexusRcError(`${source} must be an array of strings.`); + } + const names: string[] = []; + for (const item of value) { + if (typeof item !== 'string') { + throw new GitNexusRcError(`${source} entries must all be strings.`); + } + const trimmed = item.trim(); + if (!trimmed) { + throw new GitNexusRcError(`${source} entries must not be empty.`); + } + assertNoHiddenChars(trimmed, source); + // Values may be interpolated into a RegExp downstream. Restrict to + // identifier / member-access shapes so a config value can never smuggle + // regex metacharacters into a consumer. + if (!/^[A-Za-z_$][A-Za-z0-9_$.]*$/.test(trimmed)) { + throw new GitNexusRcError( + `${source} entry "${trimmed}" must be an identifier or member name ` + + `(letters, digits, _, $, . — e.g. "client.get").`, + ); + } + names.push(trimmed); + } + if (names.length === 0) { + throw new GitNexusRcError(`${source} must list at least one string.`); + } + // De-duplicate and cap to a sane bound so a pathological config cannot + // blow up the consumer scan's alternation. + return Array.from(new Set(names)).slice(0, 100); + } case 'numeric-string': { // Mirror Commander's contract: these options reach the existing CLI // validation as strings. Accept a JSON number or a string; normalize to a diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index a3bf6cd5a..e54601bab 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -599,6 +599,12 @@ export interface AnalyzeOptions { verbose?: boolean; /** Skip AGENTS.md and CLAUDE.md gitnexus block updates. */ skipAgentsMd?: boolean; + /** + * Build the control-flow-graph / PDG substrate (#2081 M1). Opt-in; off by + * default. Threaded to both the worker (CFG build) and scope-resolution + * (BasicBlock/CFG emit). + */ + pdg?: boolean; /** * Stats inclusion in AGENTS.md and CLAUDE.md. * @@ -620,6 +626,14 @@ export interface AnalyzeOptions { * before being threaded into the generated AGENTS.md / CLAUDE.md content. */ defaultBranch?: string; + /** + * Index-branch selector (#2106). From `--branch`. Distinct from + * `defaultBranch` (cosmetic base_ref): this routes the index to a per-branch + * slot. NOT sourced from `.gitnexusrc` — the `.gitnexusrc` `branch` key is an + * alias for `defaultBranch` and must not change index placement. Defaults to + * the checked-out branch inside `runFullAnalysis` when omitted. + */ + branch?: string; /** Pure index mode: skip all file injection (AGENTS.md, CLAUDE.md, skills). */ indexOnly?: boolean; /** Index the folder even when no .git directory is present. */ @@ -655,6 +669,14 @@ export interface AnalyzeOptions { embeddingBatchSize?: string; embeddingSubBatchSize?: string; embeddingDevice?: string; + /** + * Extra fetch-wrapper function names to treat as HTTP consumers (#1589/#1852 + * residual). Supplied via `.gitnexusrc` `fetchWrappers: [...]`. Threaded into + * the routes phase, where the cross-file consumer scan unions them with the + * auto-detected `fetch()` wrappers so a custom/axios-based wrapper named + * outside the built-in convention still produces `route_map` consumers. + */ + fetchWrappers?: string[]; } /** @@ -762,6 +784,21 @@ const analyzeCommandImpl = async ( } } + // Validate the index-branch selector (#2106) the same way, so a malformed + // `--branch` exits before any expensive analysis starts. Capture the TRIMMED + // return so a whitespace-padded value (e.g. " feature" from shell completion) + // normalizes before the checked-out-branch mismatch guard and slug — otherwise + // it would false-reject on-branch or create a ghost index when detached. + if (cliOptions?.branch !== undefined) { + try { + cliOptions.branch = validateBranchName(cliOptions.branch, '--branch'); + } catch (err) { + cliError(` ${err instanceof Error ? err.message : String(err)}\n`); + process.exitCode = 1; + return; + } + } + // ── Load .gitnexusrc and merge: CLI flags override config (#243) ─── // Parse/validate before the progress bar so a malformed config produces an // actionable error and exits before any expensive analysis starts. @@ -1091,9 +1128,15 @@ const analyzeCommandImpl = async ( skipGit: options.skipGit, skipAgentsMd, skipSkills, + // CFG/PDG substrate opt-in (#2081 M1) — threaded to both sinks downstream. + pdg: options.pdg === true, // Resolved default branch (CLI > .gitnexusrc > auto-detect > "main") // threaded into the generated regression-compare example (#243). defaultBranch: resolvedDefaultBranch, + // Index-branch selector (#2106). Read straight from the CLI flag (not + // the .gitnexusrc-merged options) so the cosmetic defaultBranch config + // can never change index placement. Undefined → auto-detect in pipeline. + branch: cliOptions?.branch, // commander.js `.option('--no-stats', …)` registers the flag as // `options.stats` (boolean, default true; `false` when the user // passed --no-stats). Reading `options.noStats` here returns @@ -1110,6 +1153,9 @@ const analyzeCommandImpl = async ( // GITNEXUS_WORKER_POOL_SIZE env mutation. `undefined` defers to the // env / auto-formula fallback inside the pipeline. workerPoolSize, + // Extra fetch-wrapper names from `.gitnexusrc` (#1589/#1852 residual); + // forwarded to the routes phase consumer scan. + fetchWrappers: options.fetchWrappers, }, { onProgress: (_phase, percent, message) => { @@ -1131,14 +1177,20 @@ const analyzeCommandImpl = async ( // preserving the rest of the block (incl. --skills community rows). No-op // when the value already matches, so a routine up-to-date run is silent // (#1996 tri-review P2). + // Only refresh the repo-root AGENTS.md/CLAUDE.md base_ref for the + // PRIMARY/flat index (#2106 R2). A non-primary branch's up-to-date + // analyze must not churn the committed AGENTS.md — this mirrors the + // in-pipeline `if (!placement.branch)` gate around generateAIContextFiles. let baseRefRefreshed: string[] = []; - try { - const { refreshBaseRefLine } = await import('./ai-context.js'); - baseRefRefreshed = ( - await refreshBaseRefLine(repoPath, resolvedDefaultBranch, { skipAgentsMd }) - ).files; - } catch { - /* best-effort — never fail the fast path over a context refresh */ + if (result.isPrimaryBranch !== false) { + try { + const { refreshBaseRefLine } = await import('./ai-context.js'); + baseRefRefreshed = ( + await refreshBaseRefLine(repoPath, resolvedDefaultBranch, { skipAgentsMd }) + ).files; + } catch { + /* best-effort — never fail the fast path over a context refresh */ + } } clearInterval(elapsedTimer); process.removeListener('SIGINT', sigintHandler); diff --git a/gitnexus/src/cli/clean.ts b/gitnexus/src/cli/clean.ts index ef5ab8bb8..98d881471 100644 --- a/gitnexus/src/cli/clean.ts +++ b/gitnexus/src/cli/clean.ts @@ -13,6 +13,8 @@ import { unregisterRepo, listRegisteredRepos, assertSafeStoragePath, + getStoragePaths, + removeBranchIndex, UnsafeStoragePathError, } from '../storage/repo-manager.js'; import { @@ -26,7 +28,50 @@ export const cleanCommand = async (options?: { force?: boolean; all?: boolean; lbugSidecars?: boolean; + branch?: string; }) => { + // --branch : remove a single non-primary branch's index (#2106 R7). + // Resolve against the RECORDED branches[] summary (never by slugging the + // user's raw input, which can disagree with the index-time-sanitized label). + if (options?.branch) { + const cwd = process.cwd(); + const repo = await findRepo(cwd); + if (!repo) { + console.log(t('clean.notFoundHere')); + return; + } + const entries = await listRegisteredRepos(); + const entry = entries.find((e) => path.resolve(e.path) === path.resolve(repo.repoPath)); + const summary = entry?.branches?.find((b) => b.branch === options.branch); + if (!summary) { + console.log(t('clean.branchNotIndexed', { branch: options.branch })); + return; + } + const { storagePath, lbugPath } = getStoragePaths(repo.repoPath, summary.branch); + const branchDir = path.dirname(lbugPath); + // Safety guard: the target MUST live under /.gitnexus/branches/. + // assertSafeStoragePath only validates the flat `/.gitnexus`, so this + // is a dedicated branches-sub-dir check before any destructive fs.rm. + const branchesRoot = path.join(storagePath, 'branches') + path.sep; + if (!branchDir.startsWith(branchesRoot)) { + logger.error(`Refusing to clean branch index outside .gitnexus/branches: ${branchDir}`); + return; + } + if (!options.force) { + console.log(t('clean.deleteBranch', { branch: summary.branch, path: branchDir })); + console.log(`\n${t('common.runForceConfirm')}`); + return; + } + try { + await fs.rm(branchDir, { recursive: true, force: true }); + await removeBranchIndex(repo.repoPath, summary.branch); + console.log(t('clean.deletedBranch', { branch: summary.branch })); + } catch (err) { + logger.error({ err }, 'Failed to delete branch index:'); + } + return; + } + if (options?.lbugSidecars) { const cwd = process.cwd(); const repo = await findRepo(cwd); diff --git a/gitnexus/src/cli/editor-targets.ts b/gitnexus/src/cli/editor-targets.ts new file mode 100644 index 000000000..e00cf9778 --- /dev/null +++ b/gitnexus/src/cli/editor-targets.ts @@ -0,0 +1,187 @@ +/** + * Editor targets — the single source of truth for *where* GitNexus writes its + * per-editor configuration and *how* its entries are identified. + * + * `setup` (writes these) and `uninstall` (removes them) both consume this + * module so the two stay structurally in lock-step: add or change a target + * here and both sides follow. This is declarative metadata only — file + * locations, JSON key paths, hook event names, command needles, and script + * directories, plus the shared `detectIndentation` formatting helper. The + * format-specific read/write logic (JSONC merge, TOML upsert, OpenCode's flat + * command array, Gemini's hook schema) deliberately stays in setup.ts / + * uninstall.ts. + * + * The `setup → uninstall` round-trip integration test verifies the two + * implementations remain behaviourally symmetrical on top of this shared + * structure. + */ + +import os from 'os'; +import path from 'path'; + +export type EditorId = 'cursor' | 'claude' | 'antigravity' | 'opencode' | 'codex'; + +/** An editor whose MCP config is a JSONC document (server keyed by name). */ +export interface McpJsoncTarget { + id: EditorId; + label: string; + /** Absolute path to the editor's MCP config file. */ + file: string; + /** + * JSON path of the gitnexus server entry within that file. Typed as + * `string[]` (all our keys are object keys) so it satisfies both setup's + * `mergeJsoncFile(string[])` and uninstall's `removeJsoncKey(JSONPath)` + * without either side needing a cast. + */ + keyPath: string[]; +} + +/** Codex stores MCP config as a TOML table, not JSONC. */ +export interface CodexMcpTarget { + id: 'codex'; + label: string; + /** Absolute path to ~/.codex/config.toml. */ + configFile: string; + /** The TOML table header (without brackets) setup writes / uninstall strips. */ + tomlSection: string; +} + +export interface SkillTarget { + id: EditorId; + label: string; + /** Absolute path to the editor's skills directory. */ + dir: string; +} + +export interface HookTarget { + id: EditorId; + label: string; + /** Absolute path to the editor's settings file (JSONC). */ + settingsFile: string; + /** Hook event arrays that may hold a gitnexus entry. */ + events: string[]; + /** Substring identifying the gitnexus command within a hook entry. */ + needle: string; + /** Absolute path to the bundled hook-script directory setup writes. */ + scriptDir: string; +} + +export interface EditorTargets { + /** JSONC-format MCP entries: Cursor, Claude Code, Antigravity, OpenCode. */ + mcpJsonc: McpJsoncTarget[]; + /** Codex MCP (TOML). */ + codex: CodexMcpTarget; + /** Skill install directories, one per editor that supports skills. */ + skills: SkillTarget[]; + /** Hook registrations + their bundled script directories. */ + hooks: HookTarget[]; +} + +/** + * Resolve all editor targets for the given home directory. Defaults to + * `os.homedir()`; call sites pass it through so tests can point HOME at a temp + * dir. Paths are computed at call time (not module load) so a test setting + * `process.env.HOME` before invoking sees the right locations. + */ +export function getEditorTargets(home: string = os.homedir()): EditorTargets { + const mcpJsonc: McpJsoncTarget[] = [ + { + id: 'cursor', + label: 'Cursor', + file: path.join(home, '.cursor', 'mcp.json'), + keyPath: ['mcpServers', 'gitnexus'], + }, + { + id: 'claude', + label: 'Claude Code', + file: path.join(home, '.claude.json'), + keyPath: ['mcpServers', 'gitnexus'], + }, + { + id: 'antigravity', + label: 'Antigravity', + file: path.join(home, '.gemini', 'antigravity', 'mcp_config.json'), + keyPath: ['mcpServers', 'gitnexus'], + }, + { + id: 'opencode', + label: 'OpenCode', + file: path.join(home, '.config', 'opencode', 'opencode.json'), + // OpenCode nests servers under `mcp`, not `mcpServers`. + keyPath: ['mcp', 'gitnexus'], + }, + ]; + + const codex: CodexMcpTarget = { + id: 'codex', + label: 'Codex', + configFile: path.join(home, '.codex', 'config.toml'), + tomlSection: 'mcp_servers.gitnexus', + }; + + const skills: SkillTarget[] = [ + { id: 'claude', label: 'Claude Code', dir: path.join(home, '.claude', 'skills') }, + { + id: 'antigravity', + label: 'Antigravity', + dir: path.join(home, '.gemini', 'antigravity', 'skills'), + }, + { id: 'cursor', label: 'Cursor', dir: path.join(home, '.cursor', 'skills') }, + { id: 'opencode', label: 'OpenCode', dir: path.join(home, '.config', 'opencode', 'skills') }, + // Codex reads skills from ~/.agents/skills (not ~/.codex). + { id: 'codex', label: 'Codex', dir: path.join(home, '.agents', 'skills') }, + ]; + + const hooks: HookTarget[] = [ + { + id: 'claude', + label: 'Claude Code', + settingsFile: path.join(home, '.claude', 'settings.json'), + events: ['PreToolUse', 'PostToolUse'], + needle: 'gitnexus-hook', + scriptDir: path.join(home, '.claude', 'hooks', 'gitnexus'), + }, + { + id: 'antigravity', + label: 'Antigravity', + settingsFile: path.join(home, '.gemini', 'settings.json'), + events: ['AfterTool'], + needle: 'gitnexus-antigravity-hook', + scriptDir: path.join(home, '.gemini', 'config', 'hooks', 'gitnexus'), + }, + ]; + + return { mcpJsonc, codex, skills, hooks }; +} + +/** Look up a single JSONC MCP target by editor id (throws if unknown). */ +export function mcpTarget(id: EditorId, home?: string): McpJsoncTarget { + const t = getEditorTargets(home).mcpJsonc.find((m) => m.id === id); + if (!t) throw new Error(`No JSONC MCP target for editor "${id}"`); + return t; +} + +/** Look up a single skill target by editor id (throws if unknown). */ +export function skillTarget(id: EditorId, home?: string): SkillTarget { + const t = getEditorTargets(home).skills.find((s) => s.id === id); + if (!t) throw new Error(`No skill target for editor "${id}"`); + return t; +} + +/** Look up a single hook target by editor id (throws if unknown). */ +export function hookTarget(id: EditorId, home?: string): HookTarget { + const t = getEditorTargets(home).hooks.find((h) => h.id === id); + if (!t) throw new Error(`No hook target for editor "${id}"`); + return t; +} + +/** + * Detect indentation style from file content so JSONC edits preserve the file's + * existing formatting. Shared by setup (writes) and uninstall (removes). + */ +export function detectIndentation(raw: string): { tabSize: number; insertSpaces: boolean } { + const firstIndented = raw.match(/^( +|\t)/m); + if (!firstIndented) return { tabSize: 2, insertSpaces: true }; + if (firstIndented[1] === '\t') return { tabSize: 1, insertSpaces: false }; + return { tabSize: firstIndented[1].length, insertSpaces: true }; +} diff --git a/gitnexus/src/cli/eval-server.ts b/gitnexus/src/cli/eval-server.ts index f28225e0c..a3a058264 100644 --- a/gitnexus/src/cli/eval-server.ts +++ b/gitnexus/src/cli/eval-server.ts @@ -32,7 +32,11 @@ import http from 'http'; import { isIPv4, isIPv6 } from 'node:net'; import { writeSync } from 'node:fs'; -import { LocalBackend } from '../mcp/local/local-backend.js'; +import { + LocalBackend, + type RepoListing, + type ListReposPagination, +} from '../mcp/local/local-backend.js'; import { logger } from '../core/logger.js'; import { cliInfo, cliWarn, cliError } from './cli-message.js'; import { formatDetectChangesResult } from './detect-changes-format.js'; @@ -185,7 +189,47 @@ export function formatImpactResult(result: any): string { const byDepth = result.byDepth || {}; const total = result.impactedCount || 0; + // #2129 — an ambiguous bare name must not print the "isolated / safe to + // refactor" headline. Surface the per-candidate blast radius + the maximum, + // mirroring formatContextResult, so the real impact under whichever symbol the + // caller meant is visible on the text surface, not just in the JSON. + if (result.status === 'ambiguous') { + // #2129 review F11 — report the FULL match count (`totalCandidates`), not the + // truncated `candidates[]` length; note when the candidate list is capped. + const shown = result.candidates?.length ?? 0; + const total = result.totalCandidates ?? shown; + const countPhrase = total > shown ? `${total} symbols (showing ${shown})` : `${total} symbols`; + const lines = [ + `${target?.name || '?'}: AMBIGUOUS — ${countPhrase} share this name. ` + + `Max blast radius ${result.maxImpactedCount ?? 0} (${result.maxRisk ?? 'UNKNOWN'} risk). ` + + `Disambiguate with --uid for one authoritative result:`, + ]; + for (const c of result.candidates || []) { + lines.push( + ` ${c.kind} ${c.name} → ${c.filePath}:${c.line || '?'} ` + + `[${c.impactedCount ?? 0} ${direction}, risk ${c.risk ?? 'UNKNOWN'}] (uid: ${c.uid})`, + ); + } + // #2129 review F1 — a failed per-candidate probe makes the max a lower bound. + if (result.partialProbe) { + lines.push( + ' ⚠️ One or more candidate probes failed — max blast radius / risk are lower bounds.', + ); + } + return lines.join('\n'); + } + if (total === 0) { + // #1858 — "isolated" is a confident claim. If an interface / indirection + // boundary is on the path, the true count is a lower bound, not zero; + // callers binding via DI / dynamic dispatch were not traced. Say so instead. + if (result.epistemic === 'lower-bound') { + const lines = [ + `${target?.name || '?'}: no direct ${direction} dependencies traced, but this is a LOWER BOUND — unresolved indirection on the path (actual impact may be higher):`, + ]; + for (const b of result.boundaries || []) lines.push(` • ${b}`); + return lines.join('\n'); + } return `${target?.name || '?'}: No ${direction} dependencies found. This symbol appears isolated.`; } @@ -198,6 +242,14 @@ export function formatImpactResult(result: any): string { if (result.partial) { lines.push('⚠️ Partial results — graph traversal was interrupted. Deeper impacts may exist.'); } + // #1858 — an interface / indirection boundary on the path makes this a lower + // bound; surface it so the count is not read as exhaustive. + if (result.epistemic === 'lower-bound') { + lines.push( + '⚠️ Lower bound — unresolved indirection on the path (callers binding via DI / dynamic dispatch are not traced; actual impact may be higher):', + ); + for (const b of result.boundaries || []) lines.push(` • ${b}`); + } lines.push(''); const depthLabels: Record = { @@ -265,13 +317,22 @@ export function formatCypherResult(result: any): string { return typeof result === 'string' ? result : JSON.stringify(result, null, 2); } -export function formatListReposResult(result: any): string { - if (!Array.isArray(result) || result.length === 0) { - return 'No indexed repositories.'; +export function formatListReposResult(result: { + repositories: RepoListing[]; + pagination?: ListReposPagination; +}): string { + // `list_repos` always returns the paginated { repositories, pagination } object (#2119). + const repos = result.repositories; + const pg = result.pagination; + + if (repos.length === 0) { + return pg && pg.total > 0 + ? `No repositories on this page (offset ${pg.offset} of ${pg.total} total).` + : 'No indexed repositories.'; } const lines = ['Indexed repositories:\n']; - for (const r of result) { + for (const r of repos) { const stats = r.stats || {}; lines.push( ` ${r.name} — ${stats.nodes || '?'} symbols, ${stats.edges || '?'} relationships, ${stats.processes || '?'} flows`, @@ -279,6 +340,13 @@ export function formatListReposResult(result: any): string { lines.push(` Path: ${r.path}`); lines.push(` Indexed: ${r.indexedAt}`); } + if (pg) { + lines.push(''); + lines.push( + ` Showing ${repos.length} of ${pg.total} (offset ${pg.offset}).` + + (pg.hasMore ? ` More available — re-run with offset ${pg.nextOffset}.` : ''), + ); + } return lines.join('\n'); } @@ -325,6 +393,9 @@ function getNextStepHint(toolName: string): string { case 'detect_changes': return '\n---\nNext: Run gitnexus-context "" on high-risk changed symbols to check their callers.'; + case 'list_repos': + return '\n---\nNext: READ gitnexus://repo/{name}/context for a repo above. If pagination.hasMore is true, re-run list_repos with offset set to pagination.nextOffset to page through the rest.'; + default: return ''; } diff --git a/gitnexus/src/cli/help-i18n.ts b/gitnexus/src/cli/help-i18n.ts index f342e3430..9e039023b 100644 --- a/gitnexus/src/cli/help-i18n.ts +++ b/gitnexus/src/cli/help-i18n.ts @@ -12,6 +12,7 @@ const TITLE_KEYS = { const COMMAND_DESCRIPTION_KEYS = { '': 'help.description.root', setup: 'help.command.setup.description', + uninstall: 'help.command.uninstall.description', 'ci-setup': 'help.command.ciSetup.description', analyze: 'help.command.analyze.description', index: 'help.command.index.description', @@ -79,8 +80,10 @@ const OPTION_DESCRIPTION_KEYS = { 'index|--allow-non-git': 'help.option.index.allowNonGit', 'serve|-p, --port ': 'help.option.port', 'serve|--host ': 'help.option.serve.host', + 'uninstall|-f, --force': 'help.option.uninstall.force', 'clean|-f, --force': 'help.option.force.confirmation', 'clean|--all': 'help.option.clean.all', + 'clean|--branch ': 'help.option.clean.branch', 'clean|--lbug-sidecars': 'help.option.clean.lbugSidecars', 'remove|-f, --force': 'help.option.force.confirmation', 'wiki|-f, --force': 'help.option.wiki.force', @@ -101,16 +104,19 @@ const OPTION_DESCRIPTION_KEYS = { 'publish|--id ': 'help.option.publish.id', 'publish|--skip-git': 'help.option.skipGit', 'query|-r, --repo ': 'help.option.repo.targetOmitOne', + 'query|--branch ': 'help.option.branch', 'query|-c, --context ': 'help.option.query.context', 'query|-g, --goal ': 'help.option.query.goal', 'query|-l, --limit ': 'help.option.query.limit', 'query|--content': 'help.option.content', 'context|-r, --repo ': 'help.option.repo.target', + 'context|--branch ': 'help.option.branch', 'context|-u, --uid ': 'help.option.context.uid', 'context|-f, --file ': 'help.option.context.file', 'context|--content': 'help.option.content', 'impact|-d, --direction ': 'help.option.impact.direction', 'impact|-r, --repo ': 'help.option.repo.target', + 'impact|--branch ': 'help.option.branch', 'impact|-u, --uid ': 'help.option.context.uid', 'impact|-f, --file ': 'help.option.context.file', 'impact|--kind ': 'help.option.impact.kind', @@ -120,9 +126,11 @@ const OPTION_DESCRIPTION_KEYS = { 'impact|--offset ': 'help.option.impact.offset', 'impact|--summary-only': 'help.option.impact.summaryOnly', 'cypher|-r, --repo ': 'help.option.repo.target', + 'cypher|--branch ': 'help.option.branch', 'detect-changes|-s, --scope ': 'help.option.detectChanges.scope', 'detect-changes|-b, --base-ref ': 'help.option.detectChanges.baseRef', 'detect-changes|-r, --repo ': 'help.option.repo.target', + 'detect-changes|--branch ': 'help.option.branch', 'eval-server|-p, --port ': 'help.option.port', 'eval-server|--host ': 'help.option.evalServer.host', 'eval-server|--idle-timeout ': 'help.option.evalServer.idleTimeout', diff --git a/gitnexus/src/cli/i18n/en.ts b/gitnexus/src/cli/i18n/en.ts index 78963da3f..65f66a3ca 100644 --- a/gitnexus/src/cli/i18n/en.ts +++ b/gitnexus/src/cli/i18n/en.ts @@ -10,6 +10,9 @@ export const en = { 'list.title': 'Indexed Repositories ({{count}})', 'list.indexed': 'Indexed', 'list.commit': 'Commit', + 'list.branch': 'Branch', + 'list.branchIndexes': 'Branch indexes', + 'list.branchLine': '{{branch}} ({{commit}}, {{indexed}})', 'list.stats': 'Stats', 'list.statsValue': '{{files}} files, {{symbols}} symbols, {{edges}} edges', 'list.clusters': 'Clusters', @@ -23,6 +26,10 @@ export const en = { 'status.indexed': 'Indexed', 'status.indexedCommit': 'Indexed commit', 'status.currentCommit': 'Current commit', + 'status.branch': 'Branch', + 'status.detached': '(detached HEAD)', + 'status.branchNotIndexed': + "⚠️ current branch not indexed (primary index is for '{{primary}}'; run gitnexus analyze)", 'status.status': 'Status', 'status.upToDate': '✅ up-to-date', 'status.stale': '⚠️ stale (re-run gitnexus analyze)', @@ -30,6 +37,9 @@ export const en = { 'clean.deletedRepo': 'Deleted: {{name}} ({{storagePath}})', 'clean.notFoundHere': 'No indexed repository found in this directory.', 'clean.deleteCurrent': 'This will delete the GitNexus index for: {{repoName}}', + 'clean.branchNotIndexed': 'No indexed branch named "{{branch}}" for this repository.', + 'clean.deleteBranch': 'This will delete the branch index "{{branch}}" at: {{path}}', + 'clean.deletedBranch': 'Deleted branch index: {{branch}}', 'clean.lbugSidecars.state': 'LadybugDB sidecar state: {{state}}', 'clean.lbugSidecars.none': 'No quarantined LadybugDB missing-shadow WAL sidecars found.', 'clean.lbugSidecars.preview': @@ -106,6 +116,8 @@ export const en = { 'help.option.version': 'output the version number', 'help.command.setup.description': 'One-time setup: configure MCP for Cursor, Claude Code, OpenCode, Codex', + 'help.command.uninstall.description': + 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', 'help.command.ciSetup.description': 'Generate CI/CD workflows, Docker Compose, and MCP config for a shared team GitNexus server', 'help.command.analyze.description': 'Index a repository (full analysis)', @@ -197,7 +209,9 @@ export const en = { 'help.option.port': 'Port number', 'help.option.serve.host': 'Bind address (default: 127.0.0.1, use 0.0.0.0 for remote access)', 'help.option.force.confirmation': 'Skip confirmation prompt', + 'help.option.uninstall.force': 'Apply the changes (default is a dry-run preview)', 'help.option.clean.all': 'Clean all indexed repos', + 'help.option.clean.branch': 'Delete only the named branch index (not the primary)', 'help.option.clean.lbugSidecars': 'Clean quarantined LadybugDB missing-shadow WAL sidecars', 'help.option.wiki.force': 'Force full regeneration even if up to date', 'help.option.wiki.provider': @@ -226,6 +240,7 @@ export const en = { 'help.option.query.limit': 'Max processes to return (default: 5)', 'help.option.content': 'Include full symbol source code', 'help.option.repo.target': 'Target repository', + 'help.option.branch': 'Scope to a specific branch index (multi-branch repos)', 'help.option.context.uid': 'Direct symbol UID (zero-ambiguity lookup)', 'help.option.context.file': 'File path to disambiguate common names', 'help.option.impact.kind': diff --git a/gitnexus/src/cli/i18n/zh-CN.ts b/gitnexus/src/cli/i18n/zh-CN.ts index 333fc4824..d78af436d 100644 --- a/gitnexus/src/cli/i18n/zh-CN.ts +++ b/gitnexus/src/cli/i18n/zh-CN.ts @@ -14,6 +14,9 @@ export const zhCN = { 'list.title': '已索引仓库({{count}})', 'list.indexed': '索引时间', 'list.commit': '提交', + 'list.branch': '分支', + 'list.branchIndexes': '分支索引', + 'list.branchLine': '{{branch}}({{commit}},{{indexed}})', 'list.stats': '统计', 'list.statsValue': '{{files}} 个文件,{{symbols}} 个符号,{{edges}} 条边', 'list.clusters': '聚类', @@ -27,6 +30,10 @@ export const zhCN = { 'status.indexed': '索引时间', 'status.indexedCommit': '索引提交', 'status.currentCommit': '当前提交', + 'status.branch': '分支', + 'status.detached': '(分离 HEAD)', + 'status.branchNotIndexed': + "⚠️ 当前分支未索引(主索引对应 '{{primary}}';请运行 gitnexus analyze)", 'status.status': '状态', 'status.upToDate': '✅ 已是最新', 'status.stale': '⚠️ 已过期(重新运行 gitnexus analyze)', @@ -34,6 +41,9 @@ export const zhCN = { 'clean.deletedRepo': '已删除:{{name}}({{storagePath}})', 'clean.notFoundHere': '当前目录未找到已索引仓库。', 'clean.deleteCurrent': '将删除该仓库的 GitNexus 索引:{{repoName}}', + 'clean.branchNotIndexed': '该仓库没有名为 “{{branch}}” 的已索引分支。', + 'clean.deleteBranch': '将删除分支索引 “{{branch}}”,路径:{{path}}', + 'clean.deletedBranch': '已删除分支索引:{{branch}}', 'clean.lbugSidecars.state': 'LadybugDB sidecar 状态:{{state}}', 'clean.lbugSidecars.none': '未找到已隔离的 LadybugDB missing-shadow WAL sidecar。', 'clean.lbugSidecars.preview': @@ -108,6 +118,8 @@ export const zhCN = { 'help.option.help': '显示命令帮助', 'help.option.version': '输出版本号', 'help.command.setup.description': '一次性设置:为 Cursor、Claude Code、OpenCode、Codex 配置 MCP', + 'help.command.uninstall.description': + '撤销 `setup`:从所有检测到的编辑器中移除 GitNexus 的 MCP 配置、技能和钩子', 'help.command.ciSetup.description': '为团队共享 GitNexus 服务器生成 CI/CD 工作流、Docker Compose 和 MCP 配置', 'help.command.analyze.description': '索引仓库(完整分析)', @@ -185,7 +197,9 @@ export const zhCN = { 'help.option.port': '端口号', 'help.option.serve.host': '绑定地址(默认:127.0.0.1;远程访问可用 0.0.0.0)', 'help.option.force.confirmation': '跳过确认提示', + 'help.option.uninstall.force': '应用更改(默认仅为预演预览)', 'help.option.clean.all': '清理所有已索引仓库', + 'help.option.clean.branch': '仅删除指定分支的索引(不影响主索引)', 'help.option.clean.lbugSidecars': '清理已隔离的 LadybugDB missing-shadow WAL sidecar', 'help.option.wiki.force': '即使已是最新也强制完整重新生成', 'help.option.wiki.provider': @@ -211,6 +225,7 @@ export const zhCN = { 'help.option.query.limit': '最多返回的流程数(默认:5)', 'help.option.content': '包含完整符号源码', 'help.option.repo.target': '目标仓库', + 'help.option.branch': '将查询限定到指定分支的索引(多分支仓库)', 'help.option.context.uid': '直接符号 UID(零歧义查找)', 'help.option.context.file': '用于消除常见名称歧义的文件路径', 'help.option.impact.kind': '用于消除常见名称歧义的类型过滤(如 Function、Class、Method)', diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 3dbbac42b..e2f91eb42 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -23,6 +23,14 @@ program ) .action(createLazyAction(() => import('./setup.js'), 'setupCommand')); +program + .command('uninstall') + .description( + 'Reverse `setup`: remove GitNexus MCP entries, skills, and hooks from all detected editors', + ) + .option('-f, --force', 'Apply the changes (default is a dry-run preview)') + .action(createLazyAction(() => import('./uninstall.js'), 'uninstallCommand')); + program .command('ci-setup') .description( @@ -60,11 +68,22 @@ program '(no-op when --index-only is also set).', ) .option('--skip-agents-md', 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md') + .option( + '--pdg', + 'Build the control-flow-graph / PDG substrate (BasicBlock nodes + CFG edges) ' + + 'for supported languages. Opt-in; off by default. (#2081 M1)', + ) .option( '--default-branch ', 'Default branch used in the generated regression-compare example (base_ref). ' + 'Falls back to .gitnexusrc, then auto-detected origin/HEAD, then "main".', ) + .option( + '--branch ', + 'Index the working tree under a specific branch slot (multi-branch indexing). ' + + 'Defaults to the checked-out branch; the primary/first-indexed branch keeps the ' + + 'flat index and others get their own. Distinct from --default-branch (cosmetic base_ref).', + ) .option('--no-stats', 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md') .option( '--skip-skills', @@ -153,6 +172,7 @@ program .description('Delete GitNexus index for current repo') .option('-f, --force', 'Skip confirmation prompt') .option('--all', 'Clean all indexed repos') + .option('--branch ', 'Delete only the named branch index (not the primary)') .option('--lbug-sidecars', 'Clean quarantined LadybugDB missing-shadow WAL sidecars') .action(createLazyAction(() => import('./clean.js'), 'cleanCommand')); @@ -224,6 +244,7 @@ program .command('query ') .description('Search the knowledge graph for execution flows related to a concept') .option('-r, --repo ', 'Target repository (omit if only one indexed)') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .option('-c, --context ', 'Task context to improve ranking') .option('-g, --goal ', 'What you want to find') .option('-l, --limit ', 'Max processes to return (default: 5)') @@ -234,6 +255,7 @@ program .command('context [name]') .description('360-degree view of a code symbol: callers, callees, processes') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .option('-u, --uid ', 'Direct symbol UID (zero-ambiguity lookup)') .option('-f, --file ', 'File path to disambiguate common names') .option('--content', 'Include full symbol source code') @@ -244,6 +266,7 @@ program .description('Blast radius analysis: what breaks if you change a symbol') .option('-d, --direction ', 'upstream (dependants) or downstream (dependencies)', 'upstream') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .option('-u, --uid ', 'Direct symbol UID (zero-ambiguity lookup)') .option('-f, --file ', 'File path to disambiguate common names') .option( @@ -261,6 +284,7 @@ program .command('cypher ') .description('Execute raw Cypher query against the knowledge graph') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .action(createLbugLazyAction(() => import('./tool.js'), 'cypherCommand')); program @@ -270,6 +294,7 @@ program .option('-s, --scope ', 'What to analyze: unstaged, staged, all, or compare', 'unstaged') .option('-b, --base-ref ', 'Branch/commit for compare scope (e.g. main)') .option('-r, --repo ', 'Target repository') + .option('--branch ', 'Scope to a specific branch index (multi-branch repos)') .action(createLbugLazyAction(() => import('./tool.js'), 'detectChangesCommand')); // ─── Eval Server (persistent daemon for SWE-bench) ───────────────── diff --git a/gitnexus/src/cli/list.ts b/gitnexus/src/cli/list.ts index 20214e8a8..223b662f0 100644 --- a/gitnexus/src/cli/list.ts +++ b/gitnexus/src/cli/list.ts @@ -38,6 +38,7 @@ export const listCommand = async () => { console.log(` ${t('common.path')}: ${entry.path}`); console.log(` ${t('list.indexed')}: ${indexedDate}`); console.log(` ${t('list.commit')}: ${commitShort}`); + if (entry.branch) console.log(` ${t('list.branch')}: ${entry.branch}`); console.log( ` ${t('list.stats')}: ${t('list.statsValue', { files: stats.files ?? 0, @@ -47,6 +48,18 @@ export const listCommand = async () => { ); if (stats.communities) console.log(` ${t('list.clusters')}: ${stats.communities}`); if (stats.processes) console.log(` ${t('list.processes')}: ${stats.processes}`); + // Per-branch indexes (#2106). Only rendered when extra branches were + // indexed for this path, so single-branch output is unchanged. + if (entry.branches && entry.branches.length > 0) { + console.log(` ${t('list.branchIndexes')}:`); + for (const b of entry.branches) { + const bCommit = b.lastCommit?.slice(0, 7) || t('list.unknown'); + const bIndexed = new Date(b.indexedAt).toLocaleString(); + console.log( + ` ${t('list.branchLine', { branch: b.branch, commit: bCommit, indexed: bIndexed })}`, + ); + } + } console.log(''); } }; diff --git a/gitnexus/src/cli/optional-grammars.ts b/gitnexus/src/cli/optional-grammars.ts index 994016a56..736ee6628 100644 --- a/gitnexus/src/cli/optional-grammars.ts +++ b/gitnexus/src/cli/optional-grammars.ts @@ -1,15 +1,13 @@ /** * Optional grammar availability check. * - * tree-sitter-dart, tree-sitter-proto, and tree-sitter-swift are vendored - * under vendor/ and materialized into node_modules/ at postinstall. Dart - * and Proto are built from source with node-gyp; Swift ships platform - * prebuilds activated via node-gyp-build. tree-sitter-kotlin is a declared - * optionalDependency (not vendored). All can be skipped via + * tree-sitter-dart, -proto, -swift, and -kotlin are vendored under vendor/ and + * loaded from there by absolute path (NEVER copied into node_modules — see + * core/tree-sitter/vendored-grammars.ts / #2111). Each ships committed platform + * prebuilds activated via node-gyp-build. All can be skipped via * GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (postinstall scripts), or can silently - * soft-fail when the toolchain is missing (Dart/Proto), when no prebuild - * matches the host platform (Swift), or when the optional install was - * skipped or its native build failed (Kotlin). + * soft-fail when no prebuild matches the host platform (and a source build was + * unavailable / not attempted). * * Either path produces the same observable: the .node binding is absent * at runtime. This helper detects that condition and surfaces a single @@ -17,17 +15,15 @@ * support is unavailable instead of silently getting a degraded index. */ -import { createRequire } from 'module'; import { SupportedLanguages } from 'gitnexus-shared'; import { isGrammarRuntimeSkipped } from '../core/tree-sitter/parser-loader.js'; +import { requireVendoredGrammar } from '../core/tree-sitter/vendored-grammars.js'; import { cliWarn } from './cli-message.js'; -const _require = createRequire(import.meta.url); - interface OptionalGrammar { /** Display name in warnings */ name: string; - /** Module name to require.resolve */ + /** Vendored grammar package name (directory under vendor/) */ pkg: string; /** File extensions this grammar parses */ extensions: string[]; @@ -109,7 +105,7 @@ export function detectMissingOptionalGrammars(): MissingGrammar[] { continue; } try { - _require(g.pkg); + requireVendoredGrammar(g.pkg); } catch (err) { const code = (err as NodeJS.ErrnoException | undefined)?.code; const msg = err instanceof Error ? err.message : String(err); diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index 0aa112e6a..907b7f885 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -15,6 +15,13 @@ import { promisify } from 'util'; import { fileURLToPath } from 'url'; import { parseTree, modify, applyEdits, ParseError, parse as parseJsonc } from 'jsonc-parser'; import { getGlobalDir } from '../storage/repo-manager.js'; +import { + getEditorTargets, + mcpTarget, + skillTarget, + hookTarget, + detectIndentation, +} from './editor-targets.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -162,17 +169,6 @@ function getOpenCodeMcpEntry() { return { type: 'local', command: ['npx', '-y', MCP_PINNED_REF, 'mcp'] }; } -/** - * Detect indentation style from file content. - * Returns formatting options matching the file's existing style. - */ -function detectIndentation(raw: string): { tabSize: number; insertSpaces: boolean } { - const firstIndented = raw.match(/^( +|\t)/m); - if (!firstIndented) return { tabSize: 2, insertSpaces: true }; - if (firstIndented[1] === '\t') return { tabSize: 1, insertSpaces: false }; - return { tabSize: firstIndented[1].length, insertSpaces: true }; -} - /** * Merge a key/value pair into a JSONC config file, preserving comments and formatting. * If the file is genuinely corrupt (not valid JSONC), leaves it untouched. @@ -233,9 +229,9 @@ async function setupCursor(result: SetupResult): Promise { return; } - const mcpPath = path.join(cursorDir, 'mcp.json'); + const { file: mcpPath, keyPath } = mcpTarget('cursor'); try { - const ok = await mergeJsoncFile(mcpPath, ['mcpServers', 'gitnexus'], getMcpEntry()); + const ok = await mergeJsoncFile(mcpPath, keyPath, getMcpEntry()); if (ok) { result.configured.push('Cursor'); } else { @@ -254,9 +250,9 @@ async function setupClaudeCode(result: SetupResult): Promise { } // Claude Code stores MCP config in ~/.claude.json - const mcpPath = path.join(os.homedir(), '.claude.json'); + const { file: mcpPath, keyPath } = mcpTarget('claude'); try { - const ok = await mergeJsoncFile(mcpPath, ['mcpServers', 'gitnexus'], getMcpEntry()); + const ok = await mergeJsoncFile(mcpPath, keyPath, getMcpEntry()); if (ok) { result.configured.push('Claude Code'); } else { @@ -276,7 +272,7 @@ async function installClaudeCodeSkills(result: SetupResult): Promise { const claudeDir = path.join(os.homedir(), '.claude'); if (!(await dirExists(claudeDir))) return; - const skillsDir = path.join(claudeDir, 'skills'); + const skillsDir = skillTarget('claude').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -422,13 +418,14 @@ async function installClaudeCodeHooks(result: SetupResult): Promise { const claudeDir = path.join(os.homedir(), '.claude'); if (!(await dirExists(claudeDir))) return; - const settingsPath = path.join(claudeDir, 'settings.json'); + const claudeHook = hookTarget('claude'); + const settingsPath = claudeHook.settingsFile; // Source hooks bundled within the gitnexus package (hooks/claude/) const pluginHooksPath = path.join(__dirname, '..', '..', 'hooks', 'claude'); // Copy unified hook script to ~/.claude/hooks/gitnexus/ - const destHooksDir = path.join(claudeDir, 'hooks', 'gitnexus'); + const destHooksDir = claudeHook.scriptDir; try { await fs.mkdir(destHooksDir, { recursive: true }); @@ -494,7 +491,7 @@ async function installClaudeCodeHooks(result: SetupResult): Promise { // NOTE: SessionStart hooks are broken on Windows (Claude Code bug #23576). // Session context is delivered via CLAUDE.md / skills instead. - if (!hasGitnexusHook(parsed?.hooks, 'PreToolUse')) { + if (!hasGitnexusHook(parsed?.hooks, 'PreToolUse', claudeHook.needle)) { hookEntries.push({ eventName: 'PreToolUse', value: { @@ -510,7 +507,7 @@ async function installClaudeCodeHooks(result: SetupResult): Promise { }, }); } - if (!hasGitnexusHook(parsed?.hooks, 'PostToolUse')) { + if (!hasGitnexusHook(parsed?.hooks, 'PostToolUse', claudeHook.needle)) { hookEntries.push({ eventName: 'PostToolUse', value: { @@ -566,9 +563,9 @@ async function setupAntigravity(result: SetupResult): Promise { return; } - const mcpPath = path.join(antigravityDir, 'mcp_config.json'); + const { file: mcpPath, keyPath } = mcpTarget('antigravity'); try { - const ok = await mergeJsoncFile(mcpPath, ['mcpServers', 'gitnexus'], getMcpEntry()); + const ok = await mergeJsoncFile(mcpPath, keyPath, getMcpEntry()); if (ok) { result.configured.push('Antigravity'); } else { @@ -590,7 +587,7 @@ async function installAntigravitySkills(result: SetupResult): Promise { const antigravityDir = path.join(os.homedir(), '.gemini', 'antigravity'); if (!(await dirExists(antigravityDir))) return; - const skillsDir = path.join(antigravityDir, 'skills'); + const skillsDir = skillTarget('antigravity').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -618,9 +615,9 @@ async function installAntigravityHooks(result: SetupResult): Promise { const antigravityDir = path.join(os.homedir(), '.gemini', 'antigravity'); if (!(await dirExists(antigravityDir))) return; - const geminiDir = path.join(os.homedir(), '.gemini'); - const settingsPath = path.join(geminiDir, 'settings.json'); - const destHooksDir = path.join(geminiDir, 'config', 'hooks', 'gitnexus'); + const antigravityHook = hookTarget('antigravity'); + const settingsPath = antigravityHook.settingsFile; + const destHooksDir = antigravityHook.scriptDir; // The antigravity adapter shares its lock/probe helpers with the claude // adapter — same DB, same concurrency rules — so we reuse those CJS files @@ -694,7 +691,7 @@ async function installAntigravityHooks(result: SetupResult): Promise { const hookEntries: Array<{ eventName: string; value: unknown }> = []; - if (!hasGitnexusHook(parsed?.hooks, 'AfterTool', 'gitnexus-antigravity-hook')) { + if (!hasGitnexusHook(parsed?.hooks, 'AfterTool', antigravityHook.needle)) { // Matcher follows the Gemini CLI built-in tool naming (snake_case). // search_file_content / glob cover content + filename search; run_shell_command // catches rg/grep invocations and the git commit family for stale-index hints. @@ -742,9 +739,9 @@ async function setupOpenCode(result: SetupResult): Promise { return; } - const configPath = path.join(opencodeDir, 'opencode.json'); + const { file: configPath, keyPath } = mcpTarget('opencode'); try { - const ok = await mergeJsoncFile(configPath, ['mcp', 'gitnexus'], getOpenCodeMcpEntry()); + const ok = await mergeJsoncFile(configPath, keyPath, getOpenCodeMcpEntry()); if (ok) { result.configured.push('OpenCode'); } else { @@ -764,7 +761,7 @@ function getCodexMcpTomlSection(): string { const entry = getMcpEntry(); const command = JSON.stringify(entry.command); const args = `[${entry.args.map((arg) => JSON.stringify(arg)).join(', ')}]`; - return `[mcp_servers.gitnexus]\ncommand = ${command}\nargs = ${args}\n`; + return `[${getEditorTargets().codex.tomlSection}]\ncommand = ${command}\nargs = ${args}\n`; } /** @@ -778,7 +775,7 @@ async function upsertCodexConfigToml(configPath: string): Promise { existing = ''; } - if (existing.includes('[mcp_servers.gitnexus]')) { + if (existing.includes(`[${getEditorTargets().codex.tomlSection}]`)) { return; } @@ -809,7 +806,7 @@ async function setupCodex(result: SetupResult): Promise { } try { - const configPath = path.join(codexDir, 'config.toml'); + const configPath = getEditorTargets().codex.configFile; await upsertCodexConfigToml(configPath); result.configured.push('Codex (MCP added to ~/.codex/config.toml)'); } catch (err: any) { @@ -920,7 +917,7 @@ async function installCursorSkills(result: SetupResult): Promise { const cursorDir = path.join(os.homedir(), '.cursor'); if (!(await dirExists(cursorDir))) return; - const skillsDir = path.join(cursorDir, 'skills'); + const skillsDir = skillTarget('cursor').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -938,7 +935,7 @@ async function installOpenCodeSkills(result: SetupResult): Promise { const opencodeDir = path.join(os.homedir(), '.config', 'opencode'); if (!(await dirExists(opencodeDir))) return; - const skillsDir = path.join(opencodeDir, 'skills'); + const skillsDir = skillTarget('opencode').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { @@ -958,7 +955,7 @@ async function installCodexSkills(result: SetupResult): Promise { const codexDir = path.join(os.homedir(), '.codex'); if (!(await dirExists(codexDir))) return; - const skillsDir = path.join(os.homedir(), '.agents', 'skills'); + const skillsDir = skillTarget('codex').dir; try { const installed = await installSkillsTo(skillsDir); if (installed.length > 0) { diff --git a/gitnexus/src/cli/status.ts b/gitnexus/src/cli/status.ts index c56b027d1..89ff9697d 100644 --- a/gitnexus/src/cli/status.ts +++ b/gitnexus/src/cli/status.ts @@ -4,8 +4,9 @@ * Shows the indexing status of the current repository. */ -import { findRepo, getStoragePaths, hasKuzuIndex } from '../storage/repo-manager.js'; -import { getCurrentCommit, isGitRepo, getGitRoot } from '../storage/git.js'; +import path from 'path'; +import { findRepo, getStoragePaths, loadMeta, hasKuzuIndex } from '../storage/repo-manager.js'; +import { getCurrentCommit, getCurrentBranch, isGitRepo, getGitRoot } from '../storage/git.js'; import { t } from './i18n/index.js'; export const statusCommand = async () => { @@ -32,11 +33,34 @@ export const statusCommand = async () => { } const currentCommit = getCurrentCommit(repo.repoPath); - const isUpToDate = currentCommit === repo.meta.lastCommit; + const currentBranch = getCurrentBranch(repo.repoPath); + + // Pick the index matching the checked-out branch (#2106). The flat index + // belongs to the primary branch (repo.meta.branch); when the current branch + // differs and has its own index, report that one. Legacy/no-branch metas and + // detached HEAD fall through to the flat index (unchanged behavior). + let activeMeta = repo.meta; + let currentBranchIndexed = true; + if (currentBranch && repo.meta.branch && currentBranch !== repo.meta.branch) { + const { metaPath } = getStoragePaths(repo.repoPath, currentBranch); + const branchMeta = await loadMeta(path.dirname(metaPath)); + if (branchMeta) activeMeta = branchMeta; + else currentBranchIndexed = false; + } console.log(`${t('status.repository')}: ${repo.repoPath}`); - console.log(`${t('status.indexed')}: ${new Date(repo.meta.indexedAt).toLocaleString()}`); - console.log(`${t('status.indexedCommit')}: ${repo.meta.lastCommit?.slice(0, 7)}`); + console.log(`${t('status.branch')}: ${currentBranch ?? t('status.detached')}`); + + if (!currentBranchIndexed) { + console.log( + `${t('status.status')}: ${t('status.branchNotIndexed', { primary: repo.meta.branch ?? '' })}`, + ); + return; + } + + const isUpToDate = currentCommit === activeMeta.lastCommit; + console.log(`${t('status.indexed')}: ${new Date(activeMeta.indexedAt).toLocaleString()}`); + console.log(`${t('status.indexedCommit')}: ${activeMeta.lastCommit?.slice(0, 7)}`); console.log(`${t('status.currentCommit')}: ${currentCommit?.slice(0, 7)}`); console.log(`${t('status.status')}: ${isUpToDate ? t('status.upToDate') : t('status.stale')}`); }; diff --git a/gitnexus/src/cli/tool.ts b/gitnexus/src/cli/tool.ts index 5801677b9..267482382 100644 --- a/gitnexus/src/cli/tool.ts +++ b/gitnexus/src/cli/tool.ts @@ -62,6 +62,7 @@ export async function queryCommand( queryText: string, options?: { repo?: string; + branch?: string; context?: string; goal?: string; limit?: string; @@ -81,6 +82,7 @@ export async function queryCommand( limit: options?.limit ? parseInt(options.limit) : undefined, include_content: options?.content ?? false, repo: options?.repo, + branch: options?.branch, }); output(result); } @@ -89,6 +91,7 @@ export async function contextCommand( name: string, options?: { repo?: string; + branch?: string; file?: string; uid?: string; content?: boolean; @@ -111,6 +114,7 @@ export async function contextCommand( file_path: options?.file, include_content: options?.content ?? false, repo: options?.repo, + branch: options?.branch, }); output(result); } @@ -120,6 +124,7 @@ export async function impactCommand( options?: { direction?: string; repo?: string; + branch?: string; uid?: string; file?: string; kind?: string; @@ -165,6 +170,7 @@ export async function impactCommand( maxDepth: options?.depth ? parseInt(options.depth, 10) : undefined, includeTests: options?.includeTests ?? false, repo: options?.repo, + branch: options?.branch, limit: parsedLimit, offset: parsedOffset, summaryOnly: options?.summaryOnly ?? undefined, @@ -188,6 +194,7 @@ export async function cypherCommand( query: string, options?: { repo?: string; + branch?: string; }, ): Promise { if (!query?.trim()) { @@ -199,6 +206,7 @@ export async function cypherCommand( const result = await backend.callTool('cypher', { query, repo: options?.repo, + branch: options?.branch, }); output(result); } @@ -207,12 +215,14 @@ export async function detectChangesCommand(options?: { scope?: string; baseRef?: string; repo?: string; + branch?: string; }): Promise { const backend = await getBackend(); const result = await backend.callTool('detect_changes', { scope: options?.scope || 'unstaged', base_ref: options?.baseRef, repo: options?.repo, + branch: options?.branch, }); output(formatDetectChangesResult(result)); } diff --git a/gitnexus/src/cli/uninstall.ts b/gitnexus/src/cli/uninstall.ts new file mode 100644 index 000000000..b67e9fcdc --- /dev/null +++ b/gitnexus/src/cli/uninstall.ts @@ -0,0 +1,518 @@ +/** + * Uninstall Command + * + * Reverses `gitnexus setup`: removes the GitNexus MCP server entries, + * skills, and hooks that setup writes into each detected AI editor's + * global configuration. The set of targets (paths, key paths, hook events, + * needles, script dirs) is shared with setup.ts via editor-targets.ts, so the + * two stay in lock-step. + * + * Surgical and idempotent: only gitnexus-owned keys/entries/dirs are + * removed. Unrelated user config (other MCP servers, other hooks, JSONC + * comments, indentation) is preserved. Files that are absent or that + * never contained a gitnexus entry are left untouched. + * + * Ownership is by name: skill directories are matched by the bundled gitnexus + * skill names, MCP entries by the `gitnexus` key, hooks by the gitnexus command + * needle. There is no per-install provenance marker yet (a user dir that + * happens to share a bundled skill name, or files a user added inside an + * installed skill dir, are matched purely by name) — which is why uninstall is + * a dry-run preview by default and prints the exact paths it will remove. + * Richer provenance tracking is a tracked follow-up. + * + * Intentionally NOT done here (printed as hints instead, since both are + * destructive in ways setup never caused): + * - per-repo indexes → `gitnexus clean --all` + * - the global npm package → `npm uninstall -g gitnexus` + * + * Default is a dry-run preview; pass --force to apply. + */ + +import fs from 'fs/promises'; +import path from 'path'; +import { execFile } from 'child_process'; +import { promisify } from 'util'; +import { fileURLToPath } from 'url'; +import { + parseTree, + modify, + applyEdits, + findNodeAtLocation, + parse as parseJsonc, + type ParseError, + type JSONPath, +} from 'jsonc-parser'; +import { getEditorTargets, detectIndentation } from './editor-targets.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const execFileAsync = promisify(execFile); + +interface UninstallResult { + removed: string[]; + skipped: string[]; + errors: string[]; +} + +type RemovalStatus = 'removed' | 'absent' | 'corrupt' | 'missing'; + +/** + * Remove a single key (by JSON path) from a JSONC file, preserving the + * surrounding comments and formatting. Returns: + * - 'missing': file does not exist + * - 'absent': file exists but the key isn't there (nothing to do) + * - 'corrupt': file isn't valid JSONC — left untouched on purpose + * - 'removed': the key was present (and removed unless dryRun) + */ +async function removeJsoncKey( + filePath: string, + keyPath: JSONPath, + dryRun: boolean, +): Promise { + let raw: string; + try { + raw = await fs.readFile(filePath, 'utf-8'); + } catch { + return 'missing'; + } + + if (raw.trim().length === 0) return 'absent'; + + const parseErrors: ParseError[] = []; + const tree = parseTree(raw, parseErrors); + if (!tree || tree.type !== 'object' || parseErrors.length > 0) return 'corrupt'; + + if (!findNodeAtLocation(tree, keyPath)) return 'absent'; + + if (!dryRun) { + const formattingOptions = detectIndentation(raw); + const edits = modify(raw, keyPath, undefined, { formattingOptions }); + await fs.writeFile(filePath, applyEdits(raw, edits), 'utf-8'); + } + return 'removed'; +} + +/** + * Remove the gitnexus hook command(s) — those whose command string contains + * `commandNeedle` — from the given `eventNames` arrays in a JSONC settings + * file. Mirrors the idempotency probes in setup.ts (hasGitnexusHook / + * geminiHasGitnexusHook). Returns how many event entries contained a gitnexus + * command. + * + * Removal is element-granular to honor the "other hooks are preserved" + * contract: only the matching command object inside an entry's `hooks[]` is + * deleted. The surrounding matcher entry is removed only when it becomes + * empty (i.e. it held nothing but gitnexus commands — which is exactly what + * setup creates). A user who hand-added their own command alongside ours + * keeps it. Edits are applied highest-index-first so earlier indices stay + * valid across edits. + */ +async function removeHookEntries( + filePath: string, + eventNames: string[], + commandNeedle: string, + dryRun: boolean, +): Promise<{ status: RemovalStatus; count: number }> { + let raw: string; + try { + raw = await fs.readFile(filePath, 'utf-8'); + } catch { + return { status: 'missing', count: 0 }; + } + + if (raw.trim().length === 0) return { status: 'absent', count: 0 }; + + const parseErrors: ParseError[] = []; + const tree = parseTree(raw, parseErrors); + if (!tree || tree.type !== 'object' || parseErrors.length > 0) { + return { status: 'corrupt', count: 0 }; + } + + const parsed = parseJsonc(raw); + const formattingOptions = detectIndentation(raw); + let current = raw; + let total = 0; + + const isGitnexusHook = (hh: any): boolean => + typeof hh?.command === 'string' && hh.command.includes(commandNeedle); + + for (const eventName of eventNames) { + const entries = parsed?.hooks?.[eventName]; + if (!Array.isArray(entries)) continue; + + // Walk entries high → low so removing a later one never shifts the + // index of an earlier one. + for (let entryIdx = entries.length - 1; entryIdx >= 0; entryIdx--) { + const entry = entries[entryIdx]; + if (!Array.isArray(entry?.hooks)) continue; + + const hookIdxs: number[] = []; + entry.hooks.forEach((hh: any, hi: number) => { + if (isGitnexusHook(hh)) hookIdxs.push(hi); + }); + if (hookIdxs.length === 0) continue; + + total += 1; + if (dryRun) continue; + + if (hookIdxs.length === entry.hooks.length) { + // The entry held only gitnexus command(s) — drop the whole entry. + const edits = modify(current, ['hooks', eventName, entryIdx], undefined, { + formattingOptions, + }); + current = applyEdits(current, edits); + } else { + // The entry also holds user command(s) — delete only ours, keep + // the rest. Highest hook index first to keep lower indices valid. + for (const hi of hookIdxs.reverse()) { + const edits = modify(current, ['hooks', eventName, entryIdx, 'hooks', hi], undefined, { + formattingOptions, + }); + current = applyEdits(current, edits); + } + } + } + } + + if (total === 0) return { status: 'absent', count: 0 }; + if (!dryRun) await fs.writeFile(filePath, current, 'utf-8'); + return { status: 'removed', count: total }; +} + +/** + * Remove a directory tree if it exists. Returns true when something was + * (or would be) removed. + */ +async function removeDir(dirPath: string, dryRun: boolean): Promise { + try { + await fs.access(dirPath); + } catch { + return false; + } + if (!dryRun) await fs.rm(dirPath, { recursive: true, force: true }); + return true; +} + +/** + * The exact set of skill directory names setup installs, derived from the + * bundled `skills/` source the same way installSkillsTo does (flat + * `{name}.md` and `{name}/SKILL.md` layouts). Deriving the set — rather + * than globbing `gitnexus-*` — ensures we never delete a user's own + * similarly-named skill folder. + */ +async function listGitnexusSkillNames(): Promise { + const skillsRoot = + process.env.GITNEXUS_TEST_SKILLS_ROOT ?? path.join(__dirname, '..', '..', 'skills'); + + const names = new Set(); + try { + const entries = await fs.readdir(skillsRoot, { withFileTypes: true }); + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith('.md')) { + // Guard against a bare `.md` file: basename('.md', '.md') === '', + // which would later resolve to the skills dir itself and wipe it. + const base = path.basename(entry.name, '.md'); + if (base) names.add(base); + } else if (entry.isDirectory()) { + try { + await fs.access(path.join(skillsRoot, entry.name, 'SKILL.md')); + names.add(entry.name); + } catch { + // Not a skill directory — skip. + } + } + } + } catch { + return []; + } + return [...names]; +} + +/** + * Remove the gitnexus skill directories from a target skills folder. Returns + * the absolute paths that were removed (or would be removed in dryRun) so the + * caller can show the user exactly what is affected. + */ +async function removeSkillsFrom( + targetDir: string, + skillNames: string[], + dryRun: boolean, +): Promise { + const removed: string[] = []; + for (const name of skillNames) { + // Defense in depth: an empty/relative/absolute name would resolve back to + // targetDir (or escape it) and wipe unrelated content. Only act on a + // plain child directory name. + if ( + !name || + name.includes('/') || + name.includes('\\') || + name === '.' || + name === '..' || + path.isAbsolute(name) + ) { + continue; + } + const dir = path.join(targetDir, name); + if (await removeDir(dir, dryRun)) removed.push(dir); + } + return removed; +} + +/** + * Remove the `[mcp_servers.gitnexus]` table — and any of its descendant + * sub-tables (`[mcp_servers.gitnexus.env]`, `[[mcp_servers.gitnexus.x]]`) — + * from Codex's config.toml. Used only as a fallback when the `codex` binary + * isn't on PATH; the CLI's `codex mcp remove` is preferred. + * + * Hand-rolled (no TOML dependency), but careful about the cases a naive + * line-scan gets wrong: + * - descendant sub-tables of the section are also removed (else they'd be + * left dangling, referencing a server that no longer exists); + * - `[...]`-shaped lines inside a multiline string (`"""`/`'''`) are NOT + * treated as table headers; + * - unrelated whitespace/formatting elsewhere in the file is left intact + * (no global blank-line reflow). Only a single blank separator line + * directly above the removed section is dropped. + */ +function stripTomlSection(raw: string, sectionName: string): string { + const header = `[${sectionName}]`; + const childTable = `[${sectionName}.`; + const childArray = `[[${sectionName}.`; + // Capture group 1 is the bracket token only, so a trailing inline comment + // (`[mcp_servers.gitnexus] # note`) is stripped before classification — + // otherwise an exact `=== header` check fails and the section is left behind. + const headerRe = /^(\[\[?[^[\]]+\]\]?)\s*(#.*)?$/; + + const isSectionHeader = (token: string): boolean => + token === header || token.startsWith(childTable) || token.startsWith(childArray); + + // Return the multiline-string delimiter still OPEN at the end of `line`, + // given the state at its start (null = outside any multiline string). Scans + // left→right so the delimiter that actually opens first wins — a line with an + // odd count of BOTH `"""` and `'''` (e.g. `x = '''has """ inside`) no longer + // mis-picks the wrong delimiter and desyncs the scanner. + const multilineStateAfter = (line: string, startState: string | null): string | null => { + let state = startState; + let i = 0; + while (i < line.length) { + if (state) { + const close = line.indexOf(state, i); + if (close === -1) return state; // still open at end of line + i = close + state.length; + state = null; + } else { + const a = line.indexOf('"""', i); + const b = line.indexOf("'''", i); + if (a === -1 && b === -1) return null; + const useA = b === -1 || (a !== -1 && a < b); + state = useA ? '"""' : "'''"; + i = (useA ? a : b) + 3; + } + } + return state; + }; + + const lines = raw.split(/\r?\n/); + const out: string[] = []; + let skipping = false; + let mlDelim: string | null = null; + + for (const line of lines) { + if (mlDelim) { + // Inside a multiline string: brackets here are data, not headers. + mlDelim = multilineStateAfter(line, mlDelim); + if (!skipping) out.push(line); + continue; + } + + const trimmed = line.trim(); + const headerMatch = trimmed.match(headerRe); + if (headerMatch) { + if (isSectionHeader(headerMatch[1])) { + // Drop a single blank separator line immediately above the section. + if (!skipping && out.length > 0 && out[out.length - 1].trim() === '') out.pop(); + skipping = true; + continue; + } + // A non-descendant header ends the section. + skipping = false; + out.push(line); + continue; + } + + // Track whether this (non-header) line opens a multiline string so a + // bracketed line inside it isn't mistaken for a header. + mlDelim = multilineStateAfter(line, null); + + if (!skipping) out.push(line); + } + + // Preserve the file's line endings: a CRLF (Windows) config.toml should not + // be silently rewritten to LF. Rejoin with the dominant EOL of the input. + const eol = raw.includes('\r\n') ? '\r\n' : '\n'; + let result = out.join(eol); + if (!result.endsWith(eol)) result += eol; + return result; +} + +async function uninstallCodex( + result: UninstallResult, + dryRun: boolean, + configPath: string, + tomlSection: string, +): Promise { + let raw: string; + try { + raw = await fs.readFile(configPath, 'utf-8'); + } catch { + result.skipped.push('Codex MCP (not configured)'); + return; + } + + if (!raw.includes(`[${tomlSection}]`)) { + result.skipped.push('Codex MCP (not configured)'); + return; + } + + if (dryRun) { + result.removed.push(`Codex MCP server — [${tomlSection}] in ${configPath}`); + return; + } + + // Prefer the official CLI (mirrors setup's `codex mcp add`); fall back + // to editing config.toml directly when the binary isn't on PATH. + try { + await execFileAsync('codex', ['mcp', 'remove', 'gitnexus'], { + shell: process.platform === 'win32', + windowsHide: true, + timeout: 10000, + }); + result.removed.push("Codex MCP server — via 'codex mcp remove gitnexus'"); + return; + } catch { + // Fall through to manual edit. + } + + try { + await fs.writeFile(configPath, stripTomlSection(raw, tomlSection), 'utf-8'); + result.removed.push(`Codex MCP server — [${tomlSection}] in ${configPath}`); + } catch (err: any) { + result.errors.push(`Codex: ${err.message}`); + } +} + +// ─── Main command ────────────────────────────────────────────────── + +export const uninstallCommand = async (options?: { force?: boolean }) => { + const dryRun = !options?.force; + const targets = getEditorTargets(); + + console.log(''); + console.log(' GitNexus Uninstall'); + console.log(' =================='); + console.log(''); + if (dryRun) { + console.log(' Dry run — nothing will be changed. Re-run with --force to apply.'); + console.log(''); + } + + const result: UninstallResult = { removed: [], skipped: [], errors: [] }; + + // ─── MCP server entries (JSONC editors) ────────────────────────── + for (const target of targets.mcpJsonc) { + try { + const status = await removeJsoncKey(target.file, target.keyPath, dryRun); + if (status === 'removed') + result.removed.push( + `${target.label} MCP server — ${target.keyPath.join('.')} in ${target.file}`, + ); + else if (status === 'corrupt') + result.errors.push( + `${target.label}: ${path.basename(target.file)} is corrupt — left untouched`, + ); + else result.skipped.push(`${target.label} MCP (not configured)`); + } catch (err: any) { + result.errors.push(`${target.label}: ${err.message}`); + } + } + + await uninstallCodex(result, dryRun, targets.codex.configFile, targets.codex.tomlSection); + + // ─── Hooks ─────────────────────────────────────────────────────── + for (const hook of targets.hooks) { + try { + const { status, count } = await removeHookEntries( + hook.settingsFile, + hook.events, + hook.needle, + dryRun, + ); + if (status === 'removed') + result.removed.push(`${hook.label} hooks (${count}) — ${hook.settingsFile}`); + else if (status === 'corrupt') + result.errors.push( + `${hook.label} hooks: ${path.basename(hook.settingsFile)} is corrupt — left untouched`, + ); + // Don't delete the hook script while a registered entry may still point + // at it (corrupt = we couldn't parse/remove the entry) — that would + // leave the editor invoking a missing script on every matched tool call. + if (status !== 'corrupt' && (await removeDir(hook.scriptDir, dryRun))) + result.removed.push(`${hook.label} hook scripts — ${hook.scriptDir}`); + } catch (err: any) { + result.errors.push(`${hook.label} hooks: ${err.message}`); + } + } + + // ─── Skills ────────────────────────────────────────────────────── + // Skill directories are identified by the bundled gitnexus skill names; the + // exact paths are listed below so the user can see what will be removed. + const skillNames = await listGitnexusSkillNames(); + for (const target of targets.skills) { + try { + const removedDirs = await removeSkillsFrom(target.dir, skillNames, dryRun); + for (const dir of removedDirs) result.removed.push(`${target.label} skill — ${dir}`); + } catch (err: any) { + result.errors.push(`${target.label} skills: ${err.message}`); + } + } + + // ─── Report ────────────────────────────────────────────────────── + const verb = dryRun ? 'Would remove' : 'Removed'; + if (result.removed.length > 0) { + console.log(` ${verb}:`); + for (const name of result.removed) console.log(` - ${name}`); + } else { + console.log(' Nothing to remove — GitNexus is not configured in any detected editor.'); + } + + if (result.skipped.length > 0) { + console.log(''); + console.log(' Skipped:'); + for (const name of result.skipped) console.log(` - ${name}`); + } + + if (result.errors.length > 0) { + console.log(''); + console.log(' Errors:'); + for (const err of result.errors) console.log(` ! ${err}`); + // Signal partial failure to callers/CI without aborting the remaining + // cleanup (which has already run by this point). + process.exitCode = 1; + } + + console.log(''); + console.log(' Note: skill directories are matched by bundled gitnexus skill name. If you'); + console.log(' customized files inside an installed skill dir, back them up before --force.'); + + console.log(''); + console.log(' Not removed automatically:'); + console.log(' - Per-repo indexes — run: gitnexus clean --all'); + console.log(' - The global npm package — run: npm uninstall -g gitnexus'); + + if (dryRun && result.removed.length > 0) { + console.log(''); + console.log(' Re-run with --force to apply the changes above.'); + } + console.log(''); +}; diff --git a/gitnexus/src/core/embeddings/embedder.ts b/gitnexus/src/core/embeddings/embedder.ts index d873f3a0a..3ec5f08e1 100644 --- a/gitnexus/src/core/embeddings/embedder.ts +++ b/gitnexus/src/core/embeddings/embedder.ts @@ -28,6 +28,7 @@ import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js'; import { resolveEmbeddingConfig } from './config.js'; import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js'; import { getLocalEmbeddingRuntimeBlocker } from './runtime-support.js'; +import { ensureOnnxRuntimeCommonResolvable } from './onnxruntime-common-resolver.js'; import { logger } from '../logger.js'; /** @@ -179,6 +180,9 @@ export const initEmbedder = async ( try { // Lazy-load transformers.js only after the runtime guard has passed, so // unsupported platforms never reach the native ONNX import (#1515). + // Under pnpm-strict / `pnpm dlx`, transformers' phantom `onnxruntime-common` + // import is unresolvable; register the fallback resolver first (#307). + ensureOnnxRuntimeCommonResolvable(); const { pipeline, env } = await import('@huggingface/transformers'); // Configure transformers.js environment diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index e394659f4..0405cd47c 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -36,13 +36,8 @@ import { } from './types.js'; import { resolveEmbeddingConfig } from './config.js'; import { rankExactEmbeddingRows, type ExactEmbeddingRow } from './exact-search.js'; -import { - EMBEDDING_TABLE_NAME, - EMBEDDING_INDEX_NAME, - CREATE_VECTOR_INDEX_QUERY, - STALE_HASH_SENTINEL, -} from '../lbug/schema.js'; -import { loadVectorExtension } from '../lbug/lbug-adapter.js'; +import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, STALE_HASH_SENTINEL } from '../lbug/schema.js'; +import { loadVectorExtension, createVectorIndex } from '../lbug/lbug-adapter.js'; import type { ExtensionInstallPolicy } from '../lbug/extension-loader.js'; import { getExactScanLimit } from '../platform/capabilities.js'; import { logger } from '../logger.js'; @@ -215,24 +210,36 @@ export const batchInsertEmbeddings = async ( }; /** - * Create the vector index for semantic search - - * Now indexes the separate CodeEmbedding table. - * Delegates extension loading to lbug-adapter's loadVectorExtension(), - * which owns the VECTOR extension lifecycle and state tracking. - + * Create the vector index for semantic search (indexes the CodeEmbedding table). + * + * Keeps the embedding-specific extension-install policy gate here + * (ensureVectorExtensionAvailable → resolveEmbeddingInstallPolicy, default + * `auto` for the analyze write path), then delegates the actual + * `CALL CREATE_VECTOR_INDEX(...)` to the adapter, which runs it through the + * unprepared `conn.query()` path. It must NOT go through the injected + * `executeQuery` (prepared `conn.prepare()`): LadybugDB cannot prepare that + * procedure and fails with "We do not support prepare multiple statements" — + * the silent degrade in #2114. */ -const createVectorIndex = async ( - executeQuery: (cypher: string) => Promise, -): Promise => { +const buildVectorIndex = async (): Promise => { + // This pre-check applies the embedding-specific install policy + // (resolveEmbeddingInstallPolicy, default `auto` for analyze) before reaching + // the adapter. The adapter's createVectorIndex() calls loadVectorExtension() + // again, but that's a no-op here: once this gate loads VECTOR the module-level + // `vectorExtensionLoaded` flag is set, so the adapter's second call + // short-circuits without re-resolving the policy — no double install. if (!(await ensureVectorExtensionAvailable())) return false; try { - await executeQuery(CREATE_VECTOR_INDEX_QUERY); - return true; + return await createVectorIndex(); } catch (error) { - if (isDev) { - logger.warn({ error }, 'Vector index creation warning:'); - } + // Surface this even outside dev: it silently downgrades a user-requested + // feature (semantic search) to exact scan. Log under `err` so pino's + // standard serializer captures the message/stack — logging under `error` + // serialized an Error to `{}` (the empty `{"error":{}}` reported in #2114). + logger.warn( + { err: error }, + 'Vector index creation failed; semantic search will use exact-scan fallback', + ); return false; } }; @@ -383,7 +390,7 @@ export const runEmbeddingPipeline = async ( // Ensure the vector index exists even when no new nodes need embedding. // A prior crash or first-time incremental run may have left CodeEmbedding // rows without ever reaching index creation. - const vectorIndexReady = await createVectorIndex(executeQuery); + const vectorIndexReady = await buildVectorIndex(); onProgress({ phase: 'ready', @@ -544,7 +551,7 @@ export const runEmbeddingPipeline = async ( logger.info('📇 Creating vector index...'); } - const vectorIndexReady = await createVectorIndex(executeQuery); + const vectorIndexReady = await buildVectorIndex(); onProgress({ phase: 'ready', diff --git a/gitnexus/src/core/embeddings/onnxruntime-common-resolver.ts b/gitnexus/src/core/embeddings/onnxruntime-common-resolver.ts new file mode 100644 index 000000000..fbb4f4082 --- /dev/null +++ b/gitnexus/src/core/embeddings/onnxruntime-common-resolver.ts @@ -0,0 +1,133 @@ +/** + * Make `@huggingface/transformers`' phantom `onnxruntime-common` import + * resolvable under strict package-manager layouts (#307, #2069). + * + * ## Why + * transformers' shipped `dist/transformers.node.mjs` does a bare + * `import 'onnxruntime-common'`, but transformers' `package.json` never declares + * onnxruntime-common (it lists onnxruntime-node / onnxruntime-web / sharp). With + * npm's flat `node_modules` — or pnpm with hoisting — the package is hoisted to + * a directory on transformers' resolution path and the import resolves by + * accident. Under pnpm's isolated store (and therefore `pnpm dlx` / `pnpx`), a + * package only sees its *declared* deps, so the import dies with + * `ERR_MODULE_NOT_FOUND` before `analyze --embeddings` can run. + * + * Declaring onnxruntime-common in gitnexus' own dependencies (#2074) does NOT + * fix this under pnpm: Node resolves the bare specifier from *transformers'* + * module scope, not ours, and overrides/resolutions can only re-version an + * existing edge, never add the missing one. + * + * ## What this does + * Install a synchronous, in-thread ESM resolution hook (`module.registerHooks`, + * Node >= 22.15) that redirects `onnxruntime-common` to a copy gitnexus can + * resolve — but only when the default resolver fails. The redirect target is + * preferentially the `onnxruntime-common` that `onnxruntime-node` (the native + * binding transformers actually loads) itself depends on, so the redirected copy + * is version-matched to that binding even under `pnpm dlx` — where gitnexus' + * npm-style `overrides` block does NOT apply, because it is honoured only from a + * root manifest and gitnexus is a transitive dependency there. It falls back to + * gitnexus' own direct `onnxruntime-common` dependency when that chain can't be + * walked. onnxruntime-common is a stable, pure-JS package whose `Tensor` surface + * is unchanged across 1.24–1.26, so either target is API-compatible. On working + * layouts the default resolver succeeds first and the hook never fires, so + * behaviour is unchanged. + * + * `registerHooks` (synchronous, in-thread) is preferred over the older + * `module.register` (async, off-thread, now deprecated — DEP0205, removed in + * Node 26): the redirect is a one-line conditional that needs no worker thread, + * no separate hook module, and no `data` marshalling. + * + * ## Safety + * Best-effort and idempotent. The hook is installed lazily, only on the + * local-embedding code path (after parsing), so it is never registered during + * analysis, in the parse workers, or in HTTP embedding mode. Once installed it + * is process-global: its resolve closure runs for every subsequent module + * resolution, but it passes all of them through untouched and only substitutes a + * result for the exact `onnxruntime-common` specifier when that specifier is + * genuinely absent — so it cannot mask an unrelated resolution error, and the + * per-resolution cost is a single string comparison. + * + * `module.registerHooks` is marked `@experimental` and requires Node >= 22.15 + * (the gitnexus engines floor is >= 22.0.0). On older runtimes it is absent and + * this is a graceful no-op: embeddings then resolve onnxruntime-common exactly + * as before — fine on hoisted layouts. Any failure during installation is + * swallowed. + */ +import { registerHooks, createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import { logger } from '../logger.js'; + +let attempted = false; + +/** + * Compute the file: URL the hook redirects `onnxruntime-common` to. + * + * Prefer the copy `onnxruntime-node` (the native binding transformers loads) + * depends on, so the redirected module is version-matched to the binding even + * under `pnpm dlx`, where transformers keeps its own pinned onnxruntime-node. + * The walk resolves transformers' MAIN entry — NOT `@huggingface/transformers/ + * package.json`, which transformers' `exports` map blocks + * (`ERR_PACKAGE_PATH_NOT_EXPORTED`) — then onnxruntime-node, then its + * onnxruntime-common. Falls back to gitnexus' own direct dependency (always + * resolvable from our scope) when any step fails. + */ +const resolveOnnxRuntimeCommonUrl = (): string => { + const require = createRequire(import.meta.url); + try { + const transformersMain = require.resolve('@huggingface/transformers'); + const ortNodePkg = createRequire(transformersMain).resolve('onnxruntime-node/package.json'); + const common = createRequire(ortNodePkg).resolve('onnxruntime-common'); + return pathToFileURL(common).href; + } catch { + return pathToFileURL(require.resolve('onnxruntime-common')).href; + } +}; + +/** + * Idempotently install the onnxruntime-common resolution fallback. Call once + * immediately before the dynamic `import('@huggingface/transformers')` on the + * local-embedding path. + */ +export const ensureOnnxRuntimeCommonResolvable = (): void => { + if (attempted) return; + // Mark attempted up-front: a failed attempt must not retry on every + // initEmbedder() call, and the hook is process-global — once is enough. + attempted = true; + + try { + // Node < 22.15 (the gitnexus engines floor is >= 22.0.0): no synchronous + // hooks API. Degrade gracefully — the import still works on hoisted layouts. + if (typeof registerHooks !== 'function') return; + + const redirectUrl = resolveOnnxRuntimeCommonUrl(); + + registerHooks({ + resolve(specifier, context, nextResolve) { + if (specifier !== 'onnxruntime-common') return nextResolve(specifier, context); + // Honour a real, package-manager-provided copy when one is on the path + // (npm / hoisted pnpm); only substitute ours when the specifier is + // genuinely absent. + try { + return nextResolve(specifier, context); + } catch (err) { + // The phantom import surfaces as ERR_MODULE_NOT_FOUND (or, for a + // present-but-exports-broken copy, ERR_PACKAGE_PATH_NOT_EXPORTED). + // Rethrow anything else so a genuinely broken install is not masked. + const code = (err as { code?: string } | null | undefined)?.code; + if (code === 'ERR_MODULE_NOT_FOUND' || code === 'ERR_PACKAGE_PATH_NOT_EXPORTED') { + return { url: redirectUrl, shortCircuit: true }; + } + throw err; + } + }, + }); + logger.debug({ redirectUrl }, 'Installed onnxruntime-common resolution fallback (#307)'); + } catch (err) { + // Never block embeddings on the fallback. On layouts where the package + // manager already resolves onnxruntime-common this is unnecessary anyway. + logger.debug( + { err: err instanceof Error ? err.message : String(err) }, + 'onnxruntime-common resolution fallback not installed', + ); + } +}; diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts b/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts index 3435b15d1..6309c1acc 100644 --- a/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts +++ b/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts @@ -1,4 +1,5 @@ import { createRequire } from 'node:module'; +import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.js'; import { compilePatterns, runCompiledPatterns, @@ -10,11 +11,11 @@ import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; /** * Protobuf (.proto) tree-sitter plugin for gRPC contract extraction. * - * Uses `tree-sitter-proto` (coder3101/tree-sitter-proto) as an - * optionalDependency — if the grammar is not installed (e.g. native - * compilation failed on an unusual platform), the plugin exports - * `null` and the orchestrator falls back to the existing manual - * string-sanitizing parser. + * Uses `tree-sitter-proto` (coder3101/tree-sitter-proto), loaded from + * `vendor/` by absolute path (NEVER copied into node_modules — see + * vendored-grammars.ts / #2111). If the grammar's binding cannot be loaded + * (e.g. no prebuild for an unusual platform), the plugin exports `null` and the + * orchestrator falls back to the existing manual string-sanitizing parser. * * The grammar is vendored in `vendor/tree-sitter-proto/` with * parser.c regenerated against tree-sitter-cli 0.24 (ABI version 14) @@ -22,10 +23,13 @@ import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; * (which loads ABI 13–14). */ +// Only for `tree-sitter` (a real npm dependency) in the smoke-test below; +// the vendored grammar goes through requireVendoredGrammar (never a bare +// `_require('tree-sitter-proto')`, which would force a node_modules copy — #2111). const _require = createRequire(import.meta.url); let ProtoGrammar: unknown = null; try { - ProtoGrammar = _require('tree-sitter-proto'); + ProtoGrammar = requireVendoredGrammar('tree-sitter-proto'); } catch { // Grammar not installed — PROTO_GRPC_PLUGIN will be null. } diff --git a/gitnexus/src/core/group/extractors/http-patterns/java.ts b/gitnexus/src/core/group/extractors/http-patterns/java.ts index 5d452fd1f..3ad66543e 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/java.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/java.ts @@ -6,6 +6,11 @@ import { unquoteLiteral, type LanguagePatterns, } from '../tree-sitter-scanner.js'; +import { + METHOD_ANNOTATION_TO_HTTP, + isRouteMemberKey, + findEnclosingClass, +} from '../../../ingestion/route-extractors/spring-shared.js'; import type { HttpDetection, HttpFileDetections, @@ -33,14 +38,6 @@ import type { * OkHttp, Java/Apache HttpClient) keep their own focused queries. */ -const METHOD_ANNOTATION_TO_HTTP: Record = { - GetMapping: 'GET', - PostMapping: 'POST', - PutMapping: 'PUT', - DeleteMapping: 'DELETE', - PatchMapping: 'PATCH', -}; - // Each route-defining annotation has two AST shapes — a positional argument // and a named one — that must both be matched: // @RequestMapping("/api") → (annotation_argument_list (string_literal)) @@ -361,19 +358,9 @@ const APACHE_HTTP_CLIENT_PATTERNS = compilePatterns({ } satisfies LanguagePatterns>); /** - * Find the nearest enclosing class/interface declaration ancestor for - * a node, or null if the node is top-level. Tree-sitter's - * SyntaxNode.parent walks one level at a time. + * Find the nearest enclosing interface declaration ancestor for a node, or + * null if the node is top-level. */ -function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { - let cur: Parser.SyntaxNode | null = node.parent; - while (cur) { - if (cur.type === 'class_declaration') return cur; - cur = cur.parent; - } - return null; -} - function findEnclosingInterface(node: Parser.SyntaxNode): Parser.SyntaxNode | null { let cur: Parser.SyntaxNode | null = node.parent; while (cur) { @@ -439,18 +426,6 @@ function hasAnnotation(node: Parser.SyntaxNode, names: string | readonly string[ return false; } -/** - * A named annotation argument contributes a route only when its member key is - * `path` or `value`; a positional argument (no key node) always qualifies. - * This is the JS-side replacement for the in-query `^(path|value)$` filter and - * drops Spring's non-route string attributes (`produces`, `consumes`, - * `headers`, `name`, `params`) that would otherwise be mis-read as routes. - */ -function isRouteMemberKey(keyNode: Parser.SyntaxNode | undefined): boolean { - if (!keyNode) return true; - return keyNode.text === 'path' || keyNode.text === 'value'; -} - interface MethodRouteAnnotation { methodNode: Parser.SyntaxNode; methodName: string | null; diff --git a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts index 0e56b554b..14dce0ae1 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/kotlin.ts @@ -1,5 +1,5 @@ import Parser from 'tree-sitter'; -import { createRequire } from 'node:module'; +import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.js'; import { compilePatterns, runCompiledPatterns, @@ -60,17 +60,16 @@ import type { HttpDetection, HttpLanguagePlugin } from './types.js'; * value_argument * string_literal ← the path * - * tree-sitter-kotlin is an optional npm dependency — when its native - * binding is unavailable the plugin gracefully exports `null` and - * `http-patterns/index.ts` skips registration for `.kt`/`.kts` files. + * tree-sitter-kotlin is a vendored grammar loaded from `vendor/` by absolute + * path (NEVER copied into node_modules — see vendored-grammars.ts / #2111) — + * when its native binding is unavailable the plugin gracefully exports `null` + * and `http-patterns/index.ts` skips registration for `.kt`/`.kts` files. */ -const _require = createRequire(import.meta.url); - -/** Loaded lazily; null when the grammar binding isn't installed. */ +/** Loaded lazily; null when the grammar binding isn't available. */ let Kotlin: unknown | null = null; try { - Kotlin = _require('tree-sitter-kotlin'); + Kotlin = requireVendoredGrammar('tree-sitter-kotlin'); } catch { Kotlin = null; } diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts index 98cbd371f..c8b4ee662 100644 --- a/gitnexus/src/core/group/extractors/include-extractor.ts +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -2,8 +2,22 @@ import * as path from 'node:path'; import * as fs from 'node:fs/promises'; import { glob } from 'glob'; import Parser from 'tree-sitter'; -import C from 'tree-sitter-c'; import Cpp from 'tree-sitter-cpp'; +import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js'; + +// `tree-sitter-c` is vendored (#2116), loaded from `vendor/` by absolute path +// (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Load it +// via a guarded call rather than a top-level `import C from 'tree-sitter-c'`, +// which would throw ERR_MODULE_NOT_FOUND at module-load and crash analyze +// (#2091/#2093). It may be absent on a platform without a prebuild; when the +// binding is absent, `getLanguageForFile` returns null for `.c`/`.h` so C +// include-extraction is skipped (C++ is unaffected — its binding always ships). +let C: unknown = null; +try { + C = requireVendoredGrammar('tree-sitter-c'); +} catch { + /* C grammar unavailable — C include extraction degrades to a no-op. */ +} import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; import type { ExtractedContract, RepoHandle } from '../types.js'; import { readSafe } from './fs-utils.js'; diff --git a/gitnexus/src/core/ingestion/cfg/cfg-builder.ts b/gitnexus/src/core/ingestion/cfg/cfg-builder.ts new file mode 100644 index 000000000..b6e69126b --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/cfg-builder.ts @@ -0,0 +1,175 @@ +/** + * CfgBuilder (issue #2081, M1) — the language-agnostic accumulator. + * + * A per-language `CfgVisitor` drives this: it creates blocks as it walks + * statements, wires edges (including back-edges and break/continue/return/throw + * targets resolved via {@link ControlFlowContext}), and calls {@link finish} to + * produce the serializable {@link FunctionCfg}. The builder owns the synthetic + * ENTRY (index 0) and EXIT blocks and de-duplicates identical edges so repeated + * `connect` calls (common when wiring a set of dangling exits) stay idempotent. + * + * It has no knowledge of any AST — it is exercised directly in unit tests with + * hand-built block sequences, which is how the classic CFG hazards are pinned + * before the tree-sitter visitor (U2) drives it. + */ +import type { + BasicBlockData, + BindingEntry, + CfgEdgeData, + CfgEdgeKind, + FunctionCfg, + StatementFacts, +} from './types.js'; + +interface MutableBlock { + startLine: number; + endLine: number; + /** + * Block source accumulated as fragments, joined once in {@link finish}. A + * coalescing straight-line run appends one fragment per statement; storing + * them as an array and joining at the end keeps that O(n) instead of the + * O(n²) of repeatedly concatenating onto a growing string (a long generated + * init function is the worst case — see bench/cfg). + */ + textParts: string[]; + kind: BasicBlockData['kind']; + /** + * Per-statement def/use facts in execution order (#2082 M2 U1). Parallel to + * the statements that accrued to this block — but self-describing (each + * record carries its line): facts-only attaches (ENTRY params, catch params) + * mean fact index ≠ text-fragment index. + */ + statements: StatementFacts[]; +} + +export class CfgBuilder { + private readonly blocks: MutableBlock[] = []; + private readonly edges: CfgEdgeData[] = []; + private readonly edgeKeys = new Set(); + readonly entryIndex: number; + readonly exitIndex: number; + + constructor( + private readonly filePath: string, + private readonly functionStartLine: number, + private readonly functionEndLine: number, + /** Start column of the owning function — disambiguates same-line functions + * in the BasicBlock ids (see {@link FunctionCfg.functionStartColumn}). + * Defaults to 0 for hand-built test CFGs that don't model columns. */ + private readonly functionStartColumn: number = 0, + ) { + this.entryIndex = this.newBlock(functionStartLine, functionStartLine, '', 'entry'); + this.exitIndex = this.newBlock(functionEndLine, functionEndLine, '', 'exit'); + } + + /** Create a block and return its index. */ + newBlock( + startLine: number, + endLine: number, + text: string, + kind: BasicBlockData['kind'] = 'normal', + facts?: StatementFacts, + ): number { + this.blocks.push({ + startLine, + endLine, + textParts: text ? [text] : [], + kind, + statements: facts ? [facts] : [], + }); + return this.blocks.length - 1; + } + + /** Add a single edge (idempotent on from+to+kind). */ + edge(from: number, to: number, kind: CfgEdgeKind): void { + const key = `${from}->${to}:${kind}`; + if (this.edgeKeys.has(key)) return; + this.edgeKeys.add(key); + this.edges.push({ from, to, kind }); + } + + /** Wire a set of dangling exits to a single target block with one kind. */ + connect(exits: readonly number[], to: number, kind: CfgEdgeKind = 'seq'): void { + for (const from of exits) this.edge(from, to, kind); + } + + /** Extend a block's end line as more statements accrue to it. */ + extendBlock(index: number, endLine: number, appendText?: string, facts?: StatementFacts): void { + const b = this.blocks[index]; + if (!b) return; + if (endLine > b.endLine) b.endLine = endLine; + if (appendText) b.textParts.push(appendText); + if (facts) b.statements.push(facts); + } + + /** + * Attach a facts-only statement record to a block WITHOUT touching its text + * or line span (#2082 M2 U1) — bench fingerprints and CFG snapshots include + * block text, so harvesting must never perturb it (ENTRY-block param defs + * are the canonical use; records that must precede a walked body get their + * own facts-only block instead, see the catch-param handling in visitTry). + */ + attachFacts(index: number, facts: StatementFacts): void { + const b = this.blocks[index]; + if (!b) return; + b.statements.push(facts); + } + + get blockCount(): number { + return this.blocks.length; + } + + /** Produce the serializable CFG. Caller is responsible for having wired the + * function's dangling exits to {@link exitIndex} before calling. + * + * Pass `bindings` (the function's binding table, possibly empty) to emit + * statement facts (#2082 M2 U1) — every block then carries a `statements` + * array. Omit it (hand-built test CFGs, pre-M2 producers) and both fields + * are absent, which the reaching-defs solver reports as `no-facts`. */ + finish(bindings?: readonly BindingEntry[]): FunctionCfg { + const withFacts = bindings !== undefined; + return { + filePath: this.filePath, + functionStartLine: this.functionStartLine, + functionEndLine: this.functionEndLine, + functionStartColumn: this.functionStartColumn, + entryIndex: this.entryIndex, + exitIndex: this.exitIndex, + blocks: this.blocks.map((b, index) => ({ + index, + startLine: b.startLine, + endLine: b.endLine, + text: b.textParts.join('\n'), + kind: b.kind, + ...(withFacts ? { statements: b.statements } : {}), + })), + edges: [...this.edges], + ...(withFacts ? { bindings } : {}), + }; + } +} + +/** + * Block indices reachable from `entryIndex` by following edges. Backs the + * reachability property tests (R9) over hand-built and visitor-produced CFGs. + */ +export const reachableBlocks = (cfg: FunctionCfg): Set => { + const adj = new Map(); + for (const e of cfg.edges) { + const list = adj.get(e.from); + if (list) list.push(e.to); + else adj.set(e.from, [e.to]); + } + const seen = new Set([cfg.entryIndex]); + const stack = [cfg.entryIndex]; + while (stack.length) { + const n = stack.pop() as number; + for (const next of adj.get(n) ?? []) { + if (!seen.has(next)) { + seen.add(next); + stack.push(next); + } + } + } + return seen; +}; diff --git a/gitnexus/src/core/ingestion/cfg/collect.ts b/gitnexus/src/core/ingestion/cfg/collect.ts new file mode 100644 index 000000000..890987f7d --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/collect.ts @@ -0,0 +1,63 @@ +/** + * collectFunctionCfgs (issue #2081, M1). + * + * Walks a parsed file's tree-sitter tree and builds one {@link FunctionCfg} per + * CFG-bearing function via the language's {@link CfgVisitor}. Runs IN THE PARSE + * WORKER (where the AST lives — KTD1/KTD7); the result rides on + * `ParsedFile.cfgSideChannel` across the worker→main boundary. + * + * Nested functions are enumerated independently — each gets its own CFG, and + * appears as an opaque straight-line block in its enclosing function's CFG (the + * visitor does not descend into nested function bodies). `maxFunctionLines` + * bounds per-function cost: a function whose source span exceeds the cap is + * skipped (and counted) rather than walked, so a pathological mega-function + * cannot blow up worker time/memory. A cap of `0` means no limit. + */ +import type { SyntaxNode } from '../utils/ast-helpers.js'; +import type { CfgVisitor, FunctionCfg } from './types.js'; + +/** + * Default per-function source-line cap used by the worker when the `--pdg` run + * does not specify `pdgMaxFunctionLines`. A function longer than this (almost + * always minified/generated code) is skipped rather than walked — its CFG is + * both expensive and low-value. Overridable via `PipelineOptions.pdgMaxFunctionLines`. + */ +export const DEFAULT_PDG_MAX_FUNCTION_LINES = 2000; + +export interface CollectedCfgs { + readonly cfgs: readonly FunctionCfg[]; + /** Functions skipped for exceeding `maxFunctionLines` (0 ⇒ none skipped). */ + readonly skipped: number; +} + +export function collectFunctionCfgs( + root: SyntaxNode, + visitor: CfgVisitor, + filePath: string, + maxFunctionLines = 0, +): CollectedCfgs { + const cfgs: FunctionCfg[] = []; + let skipped = 0; + const stack: SyntaxNode[] = [root]; + + while (stack.length) { + const node = stack.pop() as SyntaxNode; + if (visitor.isFunction(node)) { + const lines = node.endPosition.row - node.startPosition.row + 1; + if (maxFunctionLines > 0 && lines > maxFunctionLines) { + skipped++; + } else { + const cfg = visitor.buildFunctionCfg(node, filePath); + if (cfg) cfgs.push(cfg); + } + } + // Descend regardless (a skipped mega-function may still contain small + // nested functions that are worth a CFG of their own). + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child) stack.push(child); + } + } + + return { cfgs, skipped }; +} diff --git a/gitnexus/src/core/ingestion/cfg/control-flow-context.ts b/gitnexus/src/core/ingestion/cfg/control-flow-context.ts new file mode 100644 index 000000000..6b9bf6c1c --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/control-flow-context.ts @@ -0,0 +1,216 @@ +/** + * ControlFlowContext (issue #2081 M1; finalizer frames added by #2082 M2 U2). + * + * Resolves the targets of `break`/`continue` (plain and labeled) as the visitor + * descends through loops and switches. Loops and switches push a target frame + * on entry and pop it on exit; a labeled statement attaches its label to the + * frame of the construct it labels, so `break outer` / `continue outer` resolve + * against the right enclosing loop/switch rather than the nearest one. + * + * M2 adds FINALIZER frames, interleaved on the SAME stack as loop/switch frames + * — interleaving is load-bearing: a jump must route through exactly the + * `finally` bodies lexically BETWEEN it and its target (target-relative + * threading). A `break` whose loop lives entirely inside the `try` crosses no + * finally and must keep its direct edge; re-routing it anyway would force the + * only path to the in-try continuation through the finally, letting a finally + * redefinition falsely KILL in-loop definitions for the downstream + * reaching-defs pass (a taint false negative). A parallel stack cannot express + * that between-ness, which is why the frames live here. + */ +import type { CfgBuilder } from './cfg-builder.js'; +import type { CfgEdgeKind } from './types.js'; + +interface LoopFrame { + readonly kind: 'loop'; + /** Block a `continue` jumps to (the loop header / update). */ + readonly continueTo: number; + /** Block a `break` jumps to (the loop exit / join). */ + readonly breakTo: number; + /** All labels naming this construct (`outer: inner: for` carries both). */ + readonly labels: readonly string[]; +} + +interface SwitchFrame { + readonly kind: 'switch'; + /** Block a `break` jumps to (after the switch). `continue` is invalid here. */ + readonly breakTo: number; + readonly labels: readonly string[]; +} + +/** + * A labeled NON-loop statement (`blk: { … break blk; … }`) — break-to-label + * targets the synthesized join after the body (tri-review P1: routing such a + * break to EXIT removed the real continuation and falsely killed every def + * live at the jump for post-construct uses). Matched ONLY by a labeled break + * naming it; unlabeled breaks and continues skip it. + */ +interface BlockFrame { + readonly kind: 'block'; + readonly breakTo: number; + readonly labels: readonly string[]; +} + +/** A `finally` whose body any crossing jump must route through. */ +export interface FinalizerFrame { + readonly kind: 'finalizer'; + /** Entry block of the finally body. */ + readonly entry: number; + /** + * Completion legs registered by jumps that crossed this finally: once the + * owning try pops the frame, it wires `finally-exits → to` with `kind` for + * each entry. Mutated by the jump handlers via {@link ControlFlowContext}. + */ + readonly pending: { to: number; kind: CfgEdgeKind }[]; +} + +type Frame = LoopFrame | SwitchFrame | BlockFrame | FinalizerFrame; +type TargetFrame = LoopFrame | SwitchFrame | BlockFrame; + +/** A resolved jump: its ultimate target + the finallys it crosses (inner→outer). */ +export interface JumpResolution { + readonly target: number; + readonly finalizers: readonly FinalizerFrame[]; +} + +export class ControlFlowContext { + private readonly stack: Frame[] = []; + + pushLoop(continueTo: number, breakTo: number, labels: readonly string[] = []): void { + this.stack.push({ kind: 'loop', continueTo, breakTo, labels }); + } + + pushSwitch(breakTo: number, labels: readonly string[] = []): void { + this.stack.push({ kind: 'switch', breakTo, labels }); + } + + /** Push a labeled non-loop statement's break-target frame. */ + pushLabeledBlock(breakTo: number, labels: readonly string[]): void { + this.stack.push({ kind: 'block', breakTo, labels }); + } + + /** + * Push a finalizer frame and return it — the owning `visitTry` keeps the + * reference to wire {@link FinalizerFrame.pending} after popping it. + */ + pushFinalizer(entry: number): FinalizerFrame { + const frame: FinalizerFrame = { kind: 'finalizer', entry, pending: [] }; + this.stack.push(frame); + return frame; + } + + pop(): void { + this.stack.pop(); + } + + /** + * Resolve a `break`: the nearest enclosing loop/switch frame (or, with a + * label, the nearest frame carrying that label) plus every finalizer frame + * stacked ABOVE it — i.e. exactly the finallys the jump crosses, innermost + * first. Returns `undefined` if there is no valid target (malformed input or + * an unmodeled label) — the caller falls back to its conservative routing and + * threads nothing. + */ + resolveBreak(label?: string): JumpResolution | undefined { + return this.resolve((f) => + label === undefined + ? f.kind !== 'block' // an unlabeled break never targets a labeled block + : f.labels.includes(label), + ); + } + + /** Resolve a `continue`: like {@link resolveBreak} but only loop frames match. */ + resolveContinue(label?: string): JumpResolution | undefined { + return this.resolve( + (f) => f.kind === 'loop' && (label === undefined || f.labels.includes(label)), + (f) => (f as LoopFrame).continueTo, + ); + } + + /** Every active finalizer, innermost first — what a `return` must cross. */ + finalizersForReturn(): readonly FinalizerFrame[] { + const fins: FinalizerFrame[] = []; + for (let i = this.stack.length - 1; i >= 0; i--) { + const f = this.stack[i]; + if (f.kind === 'finalizer') fins.push(f); + } + return fins; + } + + /** + * Target block for a `break` (no finalizer info) — see {@link resolveBreak}. + * Prefer `resolveBreak` + {@link wireJumpThroughFinalizers} in visitors: a + * target-only lookup silently loses finalizer threading (the M2 soundness + * fix). Kept for target-shape assertions in tests. + */ + breakTarget(label?: string): number | undefined { + return this.resolveBreak(label)?.target; + } + + /** Target block for a `continue` — same caveat as {@link breakTarget}. */ + continueTarget(label?: string): number | undefined { + return this.resolveContinue(label)?.target; + } + + private resolve( + matches: (f: TargetFrame) => boolean, + targetOf: (f: TargetFrame) => number = (f) => f.breakTo, + ): JumpResolution | undefined { + const crossed: FinalizerFrame[] = []; + for (let i = this.stack.length - 1; i >= 0; i--) { + const f = this.stack[i]; + if (f.kind === 'finalizer') { + crossed.push(f); + continue; + } + if (matches(f)) return { target: targetOf(f), finalizers: crossed }; + } + return undefined; + } +} + +/** + * Wire a jump from `from` to `target`, routing through the finallys it + * crosses (innermost first). The first leg keeps the bare jump `kind` + * (preserving the "kind ⟹ source-block terminator" invariant in types.ts); + * each finally's completion leg is registered as pending on its frame with the + * matching `finally-*` kind and wired by the owning try via + * {@link drainFinalizerPending} once the finally's exits are known. + * + * Language-agnostic on purpose (#2082 M2): the threading protocol encodes + * three subtle invariants every future language visitor needs identically — + * keeping it here means a new visitor cannot drift on any of them. + */ +export function wireJumpThroughFinalizers( + builder: CfgBuilder, + from: number, + finalizers: readonly FinalizerFrame[], + target: number, + kind: 'return' | 'break' | 'continue', +): void { + if (finalizers.length === 0) { + builder.edge(from, target, kind); + return; + } + const completionKind = `finally-${kind}` as CfgEdgeKind; + builder.edge(from, finalizers[0].entry, kind); + for (let i = 0; i < finalizers.length; i++) { + const to = i + 1 < finalizers.length ? finalizers[i + 1].entry : target; + finalizers[i].pending.push({ to, kind: completionKind }); + } +} + +/** + * Wire a popped finalizer frame's pending completion legs from the finally's + * exit blocks. A finally that itself always jumps (`finally { return 2; }`) + * has no exits — its pending legs wire nowhere, matching JS's + * finally-override semantics. + */ +export function drainFinalizerPending( + builder: CfgBuilder, + frame: FinalizerFrame, + finallyExits: readonly number[], +): void { + for (const p of frame.pending) { + builder.connect(finallyExits, p.to, p.kind); + } +} diff --git a/gitnexus/src/core/ingestion/cfg/emit.ts b/gitnexus/src/core/ingestion/cfg/emit.ts new file mode 100644 index 000000000..3b1a1e50d --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/emit.ts @@ -0,0 +1,412 @@ +/** + * cfg/emit.ts (issue #2081, M1) — serialized side-channel → graph. + * + * Pure helper: given a file's per-function CFGs (off `ParsedFile.cfgSideChannel`, + * produced by the worker in U3), emit one persisted `BasicBlock` node per block + * and one `CFG` edge per edge into the {@link KnowledgeGraph}. Invoked from + * scope-resolution (run.ts Phase 4) while the disk-backed ParsedFile store is + * still live — the only window where the worker-built CFGs are loaded (KTD1/ + * KTD5). Default (`--pdg` off) runs never call this, so the emitted graph stays + * byte-identical to a pre-#2081 run. + * + * BasicBlock id: `BasicBlock::::` + * (KTD3). The function start line+column segments disambiguate blocks across + * multiple functions in one file — including same-line functions — since each + * function's block indices restart at 0; blocks carry no `name` (the + * BasicBlock table has no such column). The edge KIND + * (`seq`/`cond-true`/…) rides in the relationship `reason` — CFG edges are + * values of the single `CodeRelation` table's `type` column (`'CFG'`), so the + * kind cannot be its own edge type and is queried via `reason`. + */ +import type { KnowledgeGraph } from '../../graph/types.js'; +import { generateId } from '../../../lib/utils.js'; +import { computeReachingDefs } from './reaching-defs.js'; +import type { BindingEntry, FunctionCfg } from './types.js'; + +/** + * Default per-function CFG edge cap. A pathological generated function could + * otherwise emit an unbounded edge set; the cap bounds graph growth and is + * overridable via `--pdg` options. `0` (in options) means no cap (unlimited + * — see the `cap` mapping in {@link emitFileCfgs}); `undefined` means this + * default. + */ +export const DEFAULT_MAX_CFG_EDGES_PER_FUNCTION = 5000; + +/** + * Default per-function REACHING_DEF edge cap (#2082 M2 KTD9). 4000 mirrors + * Joern's per-method `maxNumberOfDefinitions` — the closest production prior + * art — but truncates-and-warns instead of silently skipping the function. + * Counts (defBlock, useBlock, binding) DEDUPED edges, not statement-level + * facts. `0` ⇒ unlimited; `undefined` ⇒ this default. + */ +export const DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION = 4000; + +/** + * Fact-materialization headroom over the edge cap (#2082 M2 U3/F3): facts are + * O(defs×uses) BY SPEC in merge-heavy code, and the edge cap alone bounds the + * GRAPH, not the per-function memory spike of materializing facts before + * dedup. {@link emitFileReachingDefs} hands `edgeCap × this` to + * `computeReachingDefs` as `maxFacts` (unlimited when the edge cap is 0) — + * single source of truth; the DEFAULT constant below is derived, never the + * mechanism. + */ +export const REACHING_DEF_FACTS_PER_EDGE_CAP = 4; + +/** Derived emit-path fact limit at the default edge cap (bench/doc anchor). */ +export const DEFAULT_PDG_MAX_REACHING_DEF_FACTS_PER_FUNCTION = + REACHING_DEF_FACTS_PER_EDGE_CAP * DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION; + +export interface CfgEmitResult { + blocks: number; + edges: number; + /** Edges dropped because a function's edge count exceeded the cap. */ + droppedEdges: number; + /** Number of functions that hit the cap. */ + cappedFunctions: number; +} + +const basicBlockId = ( + filePath: string, + functionStartLine: number, + functionStartColumn: number, + blockIndex: number, +): string => `BasicBlock:${filePath}:${functionStartLine}:${functionStartColumn}:${blockIndex}`; + +/** + * Whether an untrusted `cfgSideChannel` element is safe to feed to + * {@link emitFileCfgs}. Deliberately NOT full FunctionCfg validation — it + * checks exactly the fields whose corruption is SILENT given emit's + * mechanics: {@link basicBlockId} string-templates every id-anchor value + * (filePath, function start line/column, block index, edge endpoints) and + * the graph's addNode/addRelationship are no-throw Map inserts. Unchecked, + * a missing anchor field cross-wires same-`undefined`-id blocks across + * functions (addNode is first-writer-wins), and an edge endpoint that + * matches no block index becomes a dangling `BasicBlock:…:` edge that + * detonates much later at DB bulk-load instead of throwing here — so + * endpoints are checked for MEMBERSHIP in the block-index set, not just + * integer-ness. Lives in this module so the guard evolves with the id + * templating it defends (#2099 F4; M2 fields that join the id path must + * join this check). + */ +export const isEmitSafeCfg = (cfg: FunctionCfg | undefined | null): cfg is FunctionCfg => { + if ( + typeof cfg?.filePath !== 'string' || + !Number.isInteger(cfg.functionStartLine) || + !Number.isInteger(cfg.functionStartColumn) || + !Array.isArray(cfg.blocks) || + !Array.isArray(cfg.edges) + ) { + return false; + } + // Contiguity (index === position), not just integer-ness: every consumer — + // this module's id templating AND the reaching-defs solver's + // position-indexed adjacency arrays — assumes blocks[i].index === i. A + // membership-only check would admit a compacted channel ({index:0},{index:5}) + // whose edge 0→5 passes membership but indexes past the arrays downstream. + for (let i = 0; i < cfg.blocks.length; i++) { + if (cfg.blocks[i]?.index !== i) return false; + } + const n = cfg.blocks.length; + // entry/exit must land on real blocks — the solver feeds entryIndex straight + // into its RPO walk, where an out-of-range index throws and (worse than this + // one element) costs the whole FILE's REACHING_DEF pass (tri-review P3). + if ( + !Number.isInteger(cfg.entryIndex) || + cfg.entryIndex < 0 || + cfg.entryIndex >= n || + !Number.isInteger(cfg.exitIndex) || + cfg.exitIndex < 0 || + cfg.exitIndex >= n + ) { + return false; + } + return cfg.edges.every( + (e) => + Number.isInteger(e?.from) && + Number.isInteger(e?.to) && + e.from >= 0 && + e.from < n && + e.to >= 0 && + e.to < n, + ); +}; + +/** + * Whether a structurally-valid CFG's M2 statement facts are safe to feed to + * the reaching-defs solver + REACHING_DEF id templating (#2082 U1/U4): the + * binding table's name/declLine/declColumn template into edge ids, and + * statement def/use indices must stay IN RANGE of the table (an escaping + * index would fabricate `undefined`-keyed ids). Deliberately SEPARATE from + * {@link isEmitSafeCfg}: malformed facts must cost only the function's + * REACHING_DEF projection — degrading to M1 behavior (CFG emitted, no facts) + * — never the BasicBlock/CFG layer itself. + */ +export const hasEmitSafeFacts = (cfg: FunctionCfg): boolean => { + const bindings = cfg.bindings; + if (bindings === undefined) { + // Pre-M2 channel — statements must be absent too. + return cfg.blocks.every((b) => b.statements === undefined); + } + if (!Array.isArray(bindings)) return false; + for (const b of bindings) { + if ( + typeof b?.name !== 'string' || + !Number.isInteger(b.declLine) || + !Number.isInteger(b.declColumn) + ) { + return false; + } + } + const bindingCount = bindings.length; + const inRange = (i: number): boolean => Number.isInteger(i) && i >= 0 && i < bindingCount; + for (const b of cfg.blocks) { + const stmts = b.statements; + if (stmts === undefined) continue; + if (!Array.isArray(stmts)) return false; + for (const s of stmts) { + if (!Number.isInteger(s?.line) || !Array.isArray(s.defs) || !Array.isArray(s.uses)) { + return false; + } + if (!s.defs.every(inRange) || !s.uses.every(inRange)) return false; + if (s.mayDefs !== undefined) { + if (!Array.isArray(s.mayDefs) || !s.mayDefs.every(inRange)) return false; + } + } + } + return true; +}; + +/** + * Emit BasicBlock nodes + CFG edges for every function CFG in `cfgs`. + * + * `maxEdgesPerFunction` caps edges per function. On overflow we stop emitting + * that function's remaining edges and call `onWarn` naming the dropped count — + * no silent truncation (KTD6/R6). Block nodes are always fully emitted (their + * count is bounded by the function's statement count); only edges are capped. + */ +export function emitFileCfgs( + graph: KnowledgeGraph, + cfgs: readonly FunctionCfg[], + maxEdgesPerFunction: number = DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, + onWarn?: (message: string) => void, +): CfgEmitResult { + const result: CfgEmitResult = { blocks: 0, edges: 0, droppedEdges: 0, cappedFunctions: 0 }; + const cap = maxEdgesPerFunction > 0 ? maxEdgesPerFunction : Infinity; + + for (const cfg of cfgs) { + const { filePath, functionStartLine, functionStartColumn } = cfg; + + for (const b of cfg.blocks) { + graph.addNode({ + id: basicBlockId(filePath, functionStartLine, functionStartColumn, b.index), + label: 'BasicBlock', + properties: { + name: '', // BasicBlock has no name column; identified by id + span + filePath, + startLine: b.startLine, + endLine: b.endLine, + text: b.text, + }, + }); + result.blocks++; + } + + let emittedForFn = 0; + for (const e of cfg.edges) { + if (emittedForFn >= cap) { + const dropped = cfg.edges.length - emittedForFn; + result.droppedEdges += dropped; + result.cappedFunctions++; + onWarn?.( + `[cfg] ${filePath}:${functionStartLine}: per-function CFG edge cap ` + + `(${maxEdgesPerFunction}) reached — dropped ${dropped} of ${cfg.edges.length} edges`, + ); + break; + } + const sourceId = basicBlockId(filePath, functionStartLine, functionStartColumn, e.from); + const targetId = basicBlockId(filePath, functionStartLine, functionStartColumn, e.to); + graph.addRelationship({ + id: generateId('CFG', `${sourceId}->${targetId}:${e.kind}`), + type: 'CFG', + sourceId, + targetId, + confidence: 1.0, + reason: e.kind, // CfgEdgeKind (seq/cond-true/loop-back/…) — queryable + }); + result.edges++; + emittedForFn++; + } + } + + return result; +} + +export interface ReachingDefEmitResult { + /** Deduped (defBlock, useBlock, binding) edges persisted. */ + edges: number; + /** Deduped edges dropped by the per-function edge cap. */ + droppedEdges: number; + cappedFunctions: number; + /** Functions whose FACT materialization hit the solver's maxFacts limit. */ + truncatedFunctions: number; + /** Functions whose facts failed {@link hasEmitSafeFacts} (CFG kept, facts skipped). */ + malformedFactFunctions: number; + /** Total statement-level facts the solver produced (pre-dedup telemetry). */ + facts: number; +} + +/** + * Stable identity for a binding inside edge ids (#2082 M2 KTD3/KTD9): + * `name:declLine:declCol` for declared bindings, `name@module` for synthetic + * ones. Distinct same-name bindings never share a key; identifier characters + * cannot contain the id separators. + */ +const bindingKey = (b: BindingEntry): string => + b.synthetic ? `${b.name}@module` : `${b.name}:${b.declLine}:${b.declColumn}`; + +/** + * Compute reaching definitions per function and persist the bounded + * REACHING_DEF projection (#2082 M2 U4). + * + * Facts are DEDUPED to (defBlock, useBlock, binding) before budgeting — the + * persisted columns (`from,to,type,confidence,reason,step`; relationship ids + * are in-memory-only, the CodeRelation table has no id column) cannot + * distinguish finer rows, so statement-indexed ids would only manufacture + * byte-identical duplicate rows that burn budget. Statement granularity lives + * in the in-memory {@link computeReachingDefs} result, which the M3 taint + * engine recomputes on demand — the budget here governs only this projection + * and can never drop a taint fact. + * + * R7 (no silent truncation) covers BOTH layers: the per-function edge cap AND + * the solver's fact-materialization limit (which can fire without the edge + * cap ever being reached, since dedup is many-to-one) each produce one + * unconditional `onWarn`. The edge-cap warn names the top bindings by fact + * count — overflow is almost always one variable, which is exactly the datum + * M3 tuning wants. + */ +export function emitFileReachingDefs( + graph: KnowledgeGraph, + cfgs: readonly FunctionCfg[], + maxEdgesPerFunction: number = DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION, + onWarn?: (message: string) => void, +): ReachingDefEmitResult { + const result: ReachingDefEmitResult = { + edges: 0, + droppedEdges: 0, + cappedFunctions: 0, + truncatedFunctions: 0, + malformedFactFunctions: 0, + facts: 0, + }; + const cap = maxEdgesPerFunction > 0 ? maxEdgesPerFunction : Infinity; + const maxFacts = Number.isFinite(cap) ? (cap as number) * REACHING_DEF_FACTS_PER_EDGE_CAP : 0; // 0 ⇒ unlimited + + for (const cfg of cfgs) { + // Graceful degradation: malformed M2 facts cost only this function's + // REACHING_DEF projection — its BasicBlock/CFG layer was already emitted. + if (!hasEmitSafeFacts(cfg)) { + result.malformedFactFunctions++; + onWarn?.( + `[reaching-defs] ${cfg.filePath}:${cfg.functionStartLine}: malformed ` + + `statement facts (bad binding table or out-of-range fact indices) — ` + + `REACHING_DEF skipped for this function; its CFG is unaffected`, + ); + continue; + } + const r = computeReachingDefs(cfg, { maxFacts }); + if (r.status === 'no-facts') continue; + result.facts += r.facts.length; + + const { filePath, functionStartLine, functionStartColumn } = cfg; + if (r.status === 'truncated') { + result.truncatedFunctions++; + onWarn?.( + `[reaching-defs] ${filePath}:${functionStartLine}: fact materialization ` + + `limit (${maxFacts}) reached — facts beyond it were not computed; ` + + `the persisted REACHING_DEF projection for this function is sparse`, + ); + } else if (r.status === 'overflow') { + result.truncatedFunctions++; + onWarn?.( + `[reaching-defs] ${filePath}:${functionStartLine}: a basic block exceeds ` + + `the def-key stride (≥2^21 coalesced statements — minified/generated ` + + `code) — REACHING_DEF skipped for this function (computing any facts ` + + `would risk wrong-block aliasing); its CFG is unaffected`, + ); + continue; + } + + // Dedup to (defBlock, useBlock, binding) — facts arrive sorted, so the + // deduped order (and therefore cap truncation) is deterministic. + const seen = new Set(); + const deduped: { defBlock: number; useBlock: number; bindingIdx: number }[] = []; + for (const f of r.facts) { + const key = `${f.def.blockIndex}:${f.use.blockIndex}:${f.bindingIdx}`; + if (seen.has(key)) continue; + seen.add(key); + deduped.push({ + defBlock: f.def.blockIndex, + useBlock: f.use.blockIndex, + bindingIdx: f.bindingIdx, + }); + } + + let emittedForFn = 0; + for (const edge of deduped) { + if (emittedForFn >= cap) { + const dropped = deduped.length - emittedForFn; + result.droppedEdges += dropped; + result.cappedFunctions++; + // Tallied lazily — cap overflow is the rare path; the common uncapped + // case must not pay a per-fact counting pass. + const factsPerBinding = new Map(); + for (const f of r.facts) { + factsPerBinding.set(f.bindingIdx, (factsPerBinding.get(f.bindingIdx) ?? 0) + 1); + } + const top = [...factsPerBinding.entries()] + .sort((a, b) => b[1] - a[1] || a[0] - b[0]) + .slice(0, 2) + .map(([idx, count]) => `${r.bindings[idx]?.name ?? `#${idx}`}(${count} facts)`) + .join(', '); + onWarn?.( + `[reaching-defs] ${filePath}:${functionStartLine}: per-function ` + + `REACHING_DEF edge cap (${maxEdgesPerFunction}) reached — dropped ` + + `${dropped} of ${deduped.length} edges; top bindings: ${top}`, + ); + break; + } + const binding = r.bindings[edge.bindingIdx]; + const sourceId = basicBlockId( + filePath, + functionStartLine, + functionStartColumn, + edge.defBlock, + ); + const targetId = basicBlockId( + filePath, + functionStartLine, + functionStartColumn, + edge.useBlock, + ); + graph.addRelationship({ + // Single function anchor — the two block ids share it, so templating + // it once halves the id size (ids are in-memory-only but ~4000 of + // them per capped function is real transient heap). + id: generateId( + 'REACHING_DEF', + `${filePath}:${functionStartLine}:${functionStartColumn}:` + + `${edge.defBlock}->${edge.useBlock}:${bindingKey(binding)}`, + ), + type: 'REACHING_DEF', + sourceId, + targetId, + confidence: 1.0, + reason: binding.name, // plain source-level name (M0/S1 verdict) — queryable + }); + result.edges++; + emittedForFn++; + } + } + + return result; +} diff --git a/gitnexus/src/core/ingestion/cfg/reaching-defs.ts b/gitnexus/src/core/ingestion/cfg/reaching-defs.ts new file mode 100644 index 000000000..14c7b3745 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/reaching-defs.ts @@ -0,0 +1,448 @@ +/** + * Reaching definitions (#2082 M2 U3) — classic GEN/KILL monotone fixpoint over + * one function's CFG, plus the canonical intra-block statement sweep that + * recovers statement-granular def→use facts from M1's coalesced blocks + * WITHOUT re-splitting the CFG. + * + * PURE AND DETERMINISTIC (load-bearing contract): + * - Pure function of its inputs — no graph, no logger (warnings are the + * caller's job), importable outside the worker. The M3 taint engine calls + * this same function in-phase (facts are recomputed on demand, never + * retained run-wide — the persisted REACHING_DEF edges are a bounded + * projection, never the taint substrate). + * - Deterministic — predecessors merge in sorted block-index order, + * insertion-ordered Maps/Sets throughout, and the output fact array is + * explicitly sorted. Snapshot tests and content-derived edge ids rely on it. + * + * COMPLEXITY DISCIPLINE (the four-times-repeated repo bug shape is per-item + * re-derivation inside the loop): def-sets are SHARED BY REFERENCE, never + * deep-copied — a MUST def's kill is total per binding, so a transfer either + * aliases the incoming set or replaces it; a MAY def (conditional context — + * see StatementFacts.mayDefs) unions WITHOUT killing via a copy-on-extend. + * Single-predecessor blocks alias the predecessor's OUT map outright; + * multi-pred merges union only bindings whose incoming sets differ by + * reference. Iteration is reverse post-order, seeded with every block + * (unreachable blocks keep ⊥ IN — correct, their defs reach nothing). + * Convergence: sets grow monotonically within the finite def-site universe ⇒ + * ≤ loop-depth+1 passes in practice. + * + * `limits.maxFacts` bounds materialization: facts are O(defs×uses) BY SPEC in + * merge-heavy code (N branch-arm defs × N later uses = N² facts), and a + * 2000-line function can spike 100k+ fact objects on the main thread. The + * emit path passes DEFAULT_PDG_MAX_REACHING_DEF_FACTS_PER_FUNCTION (emit.ts); + * M3 passes its own large-but-finite limit and treats `status: 'truncated'` + * as a per-function taint-coverage gap. + */ +import type { BindingEntry, FunctionCfg } from './types.js'; + +/** A statement-granular program point within one function's CFG. */ +export interface ProgramPoint { + readonly blockIndex: number; + /** Statement index within the block's `statements` array. */ + readonly stmtIndex: number; + readonly line: number; +} + +/** One def→use fact: the definition at `def` reaches the use at `use`. */ +export interface DefUseFact { + /** Index into {@link FunctionDefUse.bindings}. */ + readonly bindingIdx: number; + readonly def: ProgramPoint; + readonly use: ProgramPoint; +} + +export interface ReachingDefsLimits { + /** + * Maximum number of facts to materialize; the sweep stops early and reports + * `status: 'truncated'`. `undefined`/0 ⇒ unlimited. + */ + readonly maxFacts?: number; +} + +export interface FunctionDefUse { + /** + * `computed` — full facts. + * `no-facts` — the CFG carries no statement facts (hand-built or pre-M2 + * side channel); empty facts, NOT an error. + * `truncated` — `limits.maxFacts` hit; `facts` is a deterministic prefix. + * `overflow` — a block's statement count breaches the def-key stride; no + * facts at all (computing any would risk key aliasing — + * wrong-block facts are strictly worse than none). Distinct + * from `truncated` so the caller's diagnostic doesn't + * misname it as the fact-materialization limit. + */ + readonly status: 'computed' | 'no-facts' | 'truncated' | 'overflow'; + /** Pass-through of the CFG's binding table (empty for `no-facts`). */ + readonly bindings: readonly BindingEntry[]; + /** Sorted by (def block, def stmt, use block, use stmt, binding). */ + readonly facts: readonly DefUseFact[]; + /** Total def / use sites seen (telemetry; independent of truncation). */ + readonly defCount: number; + readonly useCount: number; +} + +/** + * def-site key: packs (blockIndex, stmtIndex) into one number. The stride is + * a per-BLOCK statement bound, and `maxFunctionLines` caps LINES, not + * statements — a minified one-line function coalesces arbitrarily many + * statements into one block, so an overflow would silently alias + * (block b, stmt STRIDE+k) with (block b+1, stmt k) and fabricate wrong-block + * facts. computeReachingDefs therefore range-checks up front and bails to a + * sound empty `truncated` result instead of ever letting a key alias. + * 2^21 statements per block × blocks ≤ 2^32 stays inside Number's 2^53. + */ +const STMT_STRIDE = 1 << 21; +const defKey = (blockIndex: number, stmtIndex: number): number => + blockIndex * STMT_STRIDE + stmtIndex; + +type DefSet = Set; +/** bindingIdx → def-site keys reaching this program point. */ +type Lattice = Map; + +const EMPTY_LATTICE: Lattice = new Map(); + +/** + * Compute reaching definitions for one function. See the module doc for the + * purity/determinism/sharing contract. + */ +export function computeReachingDefs(cfg: FunctionCfg, limits?: ReachingDefsLimits): FunctionDefUse { + if (!cfg.bindings) { + return { status: 'no-facts', bindings: [], facts: [], defCount: 0, useCount: 0 }; + } + + const blocks = cfg.blocks; + const n = blocks.length; + + // Key-aliasing guard (see STMT_STRIDE): a block with ≥ STRIDE statements + // cannot be keyed without aliasing into the next block's def sites, which + // would fabricate wrong-block facts — strictly worse than producing none. + // Bail to a sound empty `overflow` result (the emit path warns distinctly). + for (const b of blocks) { + if ((b.statements?.length ?? 0) >= STMT_STRIDE) { + return { status: 'overflow', bindings: cfg.bindings, facts: [], defCount: 0, useCount: 0 }; + } + } + + // ── adjacency (sorted for deterministic merges) ───────────────────────── + // A `throw` edge contributes IN(from) ∪ allDefs(from) to its handler, not + // OUT: an exception can fire BEFORE the block's defs complete (the seed def + // in `let x = seed(); try { x = risky(); } catch { sink(x) }` must reach the + // sink) AND between any two defs of a multi-def coalesced block (the parse + // def in `x = parse(a); x = normalize(x);` is live exactly when normalize + // throws — OUT's last-def-wins misses it). Sound over-approximation; + // monotone, so the fixpoint absorbs it. See mergePreds. + const preds: { from: number; viaThrow: boolean }[][] = Array.from({ length: n }, () => []); + const succs: number[][] = Array.from({ length: n }, () => []); + // Handlers whose IN depends on this block's IN (throw edges) — requeued on + // IN change, since a genned binding can absorb IN growth without changing + // OUT, which would otherwise leave the handler stale. + const throwSuccs: number[][] = Array.from({ length: n }, () => []); + for (const e of cfg.edges) { + // Optional-chained pushes drop out-of-range endpoints defensively — the + // emit path validates via isEmitSafeCfg, but this pure function also runs + // on hand-built CFGs. + succs[e.from]?.push(e.to); + preds[e.to]?.push({ from: e.from, viaThrow: e.kind === 'throw' }); + if (e.kind === 'throw') throwSuccs[e.from]?.push(e.to); + } + for (const list of preds) { + list.sort((a, b) => a.from - b.from || Number(a.viaThrow) - Number(b.viaThrow)); + // duplicate (from, throw+non-throw) pairs both survive — the throw leg + // adds IN(from); the merge dedups set-wise. + } + for (const list of succs) list.sort((a, b) => a - b); + + // ── per-block GEN + def/use telemetry ──────────────────────────────────── + // gen[b]: bindingIdx → { set, kills }. A MUST def resets the accumulated + // set (kill is total); a MAY def (conditionally-evaluated context — see + // StatementFacts.mayDefs) only ADDS: the binding's incoming defs survive, + // so the transfer is out[x] = kills ? set : in[x] ∪ set. + interface GenEntry { + set: DefSet; + kills: boolean; + } + const gen: (Map | null)[] = new Array(n).fill(null); + // allDefsGen[b]: bindingIdx → EVERY def-site key in the block (must + may). + // This is what a throw edge delivers to its handler: an exception can fire + // between any two statements, so every intermediate def may be the live one + // at the handler — IN∪OUT alone misses defs overwritten later in the same + // coalesced block (`try { x = parse(a); x = normalize(x); } catch { sink(x) }` + // — parse's value is exactly what sink sees when normalize throws). + const allDefsGen: (Lattice | null)[] = new Array(n).fill(null); + const defLine = new Map(); // defKey → source line + let defCount = 0; + let useCount = 0; + for (const b of blocks) { + const stmts = b.statements; + if (!stmts || stmts.length === 0) continue; + let g: Map | null = null; + let all: Lattice | null = null; + for (let i = 0; i < stmts.length; i++) { + const s = stmts[i]; + useCount += s.uses.length; + const key = defKey(b.index, i); + const record = (d: number, kills: boolean): void => { + defCount += 1; + defLine.set(key, s.line); + if (!g) g = new Map(); + const entry = g.get(d); + if (kills || !entry) { + g.set(d, { set: new Set([key]), kills: kills || (entry?.kills ?? false) }); + } else { + entry.set.add(key); // may-def accumulates; never clears + } + if (!all) all = new Map(); + const allSet = all.get(d); + if (allSet) allSet.add(key); + else all.set(d, new Set([key])); + }; + if (s.mayDefs) for (const d of s.mayDefs) record(d, false); + for (const d of s.defs) record(d, true); + } + gen[b.index] = g; + allDefsGen[b.index] = all; + } + + // ── iteration order: RPO over reachable blocks, then the rest by index ── + const order = reversePostOrder(cfg.entryIndex, succs, n); + + // ── fixpoint ──────────────────────────────────────────────────────────── + const inSets: Lattice[] = new Array(n).fill(EMPTY_LATTICE); + const outSets: Lattice[] = new Array(n).fill(EMPTY_LATTICE); + + const inWorklist = new Array(n).fill(true); + let pending = n; + while (pending > 0) { + for (const b of order) { + if (!inWorklist[b]) continue; + inWorklist[b] = false; + pending -= 1; + + const p = preds[b]; + const inB: Lattice = + p.length === 0 + ? EMPTY_LATTICE + : p.length === 1 && !p[0].viaThrow + ? outSets[p[0].from] // alias — zero allocation on straight-line chains + : mergePreds(p, inSets, outSets, allDefsGen); + const inChanged = !latticeEquals(inSets[b], inB); + inSets[b] = inB; + + const g = gen[b]; + // OUT = overlay(IN): a KILLING gen entry replaces the binding's set; a + // may-def-only entry unions with the incoming set (never kills). When + // nothing is genned, OUT aliases IN outright. + let outB: Lattice; + if (!g) { + outB = inB; + } else { + outB = new Map(inB); // copies REFERENCES, never set contents + for (const [bindingIdx, entry] of g) { + if (entry.kills) { + outB.set(bindingIdx, entry.set); + } else { + const incoming = inB.get(bindingIdx); + outB.set(bindingIdx, incoming ? unionSets(incoming, entry.set) : entry.set); + } + } + } + + const requeue = (s: number): void => { + if (!inWorklist[s]) { + inWorklist[s] = true; + pending += 1; + } + }; + if (!latticeEquals(outSets[b], outB)) { + outSets[b] = outB; + for (const s of succs[b]) requeue(s); + } + if (inChanged) for (const s of throwSuccs[b]) requeue(s); + } + } + + // ── statement sweep: recover statement-granular def→use facts ─────────── + const maxFacts = limits?.maxFacts && limits.maxFacts > 0 ? limits.maxFacts : Infinity; + const facts: DefUseFact[] = []; + let truncated = false; + + outer: for (const b of blocks) { + const stmts = b.statements; + if (!stmts || stmts.length === 0) continue; + // Lazy overlay of IN — entries are replaced (never mutated) on def, so the + // shared sets stay intact. + let reach: Lattice | null = null; + for (let i = 0; i < stmts.length; i++) { + const s = stmts[i]; + // A use's binding that the SAME statement also defines could be a + // read-then-write (`x += 1` — sees prior defs) OR a write-then-read + // (`if ((m = re.exec(s)) && m[1])` — sees the same-statement def). + // StatementFacts carries no intra-statement order, so emit BOTH: prior + // defs ∪ the same-statement def. Sound over-approximation — the extra + // self-fact on compound assignments is harmless; missing the + // assign-and-test def→use (the most common JS idiom) would be a taint + // false negative. May-defs join the self-key set the same way. + const sameStmtDefs = + s.defs.length > 0 || s.mayDefs?.length ? new Set([...s.defs, ...(s.mayDefs ?? [])]) : null; + for (const u of s.uses) { + const reaching = (reach ?? inSets[b.index]).get(u); + const selfKey = sameStmtDefs?.has(u) ? defKey(b.index, i) : undefined; + if (!reaching && selfKey === undefined) continue; + const keys = + selfKey !== undefined && !reaching?.has(selfKey) + ? [...(reaching ?? []), selfKey] + : [...(reaching ?? [])]; + for (const key of keys) { + if (facts.length >= maxFacts) { + truncated = true; + break outer; + } + const defBlock = Math.floor(key / STMT_STRIDE); + const defStmt = key % STMT_STRIDE; + facts.push({ + bindingIdx: u, + def: { blockIndex: defBlock, stmtIndex: defStmt, line: defLine.get(key) ?? s.line }, + use: { blockIndex: b.index, stmtIndex: i, line: s.line }, + }); + } + } + if (s.mayDefs?.length) { + // Gen WITHOUT kill: the conditional def joins the binding's set. + if (!reach) reach = new Map(inSets[b.index]); + const key = defKey(b.index, i); + for (const d of s.mayDefs) { + const prior = reach.get(d); + reach.set(d, prior ? unionSets(prior, new Set([key])) : new Set([key])); + } + } + if (s.defs.length > 0) { + if (!reach) reach = new Map(inSets[b.index]); + for (const d of s.defs) reach.set(d, new Set([defKey(b.index, i)])); // kill + gen + } + } + } + + facts.sort( + (a, b) => + a.def.blockIndex - b.def.blockIndex || + a.def.stmtIndex - b.def.stmtIndex || + a.use.blockIndex - b.use.blockIndex || + a.use.stmtIndex - b.use.stmtIndex || + a.bindingIdx - b.bindingIdx, + ); + + return { + status: truncated ? 'truncated' : 'computed', + bindings: cfg.bindings, + facts, + defCount, + useCount, + }; +} + +/** RPO over blocks reachable from `entry`; unreachable blocks appended by index. */ +function reversePostOrder(entry: number, succs: readonly number[][], n: number): number[] { + const visited = new Array(n).fill(false); + const post: number[] = []; + // Iterative DFS with an explicit phase stack (children pushed in reverse so + // they pop in sorted order — determinism). + const stack: { node: number; childIdx: number }[] = [{ node: entry, childIdx: 0 }]; + visited[entry] = true; + while (stack.length) { + const top = stack[stack.length - 1]; + const children = succs[top.node]; + if (top.childIdx < children.length) { + const next = children[top.childIdx]; + top.childIdx += 1; + if (!visited[next]) { + visited[next] = true; + stack.push({ node: next, childIdx: 0 }); + } + } else { + post.push(top.node); + stack.pop(); + } + } + const order = post.reverse(); + for (let b = 0; b < n; b++) if (!visited[b]) order.push(b); + return order; +} + +/** + * Union predecessor lattices, sharing sets where possible. A normal edge + * contributes OUT(from). A THROW edge contributes IN(from) ∪ allDefs(from): + * an exception may fire before, between, or after any of the block's defs, so + * the handler can observe the incoming state OR any intermediate def — OUT + * alone (last-def-wins) misses defs overwritten later in the same block. + * IN ∪ allDefs ⊇ OUT, so the throw contribution subsumes it. + */ +function mergePreds( + preds: readonly { from: number; viaThrow: boolean }[], + inSets: readonly Lattice[], + outSets: readonly Lattice[], + allDefsGen: readonly (Lattice | null)[], +): Lattice { + const merged: Lattice = new Map(); + const mergeOne = (source: Lattice): void => { + for (const [bindingIdx, set] of source) { + const existing = merged.get(bindingIdx); + if (!existing) { + merged.set(bindingIdx, set); // share the first contributor's set + } else if (existing !== set) { + // Union only when the references differ. Copy-on-extend: `existing` + // may be a shared set from another block — never mutate it. + let target = existing; + let copied = false; + for (const key of set) { + if (!target.has(key)) { + if (!copied) { + target = new Set(existing); + copied = true; + } + target.add(key); + } + } + if (copied) merged.set(bindingIdx, target); + } + } + }; + for (const p of preds) { + if (p.viaThrow) { + mergeOne(inSets[p.from]); // exception may fire pre-defs… + const all = allDefsGen[p.from]; + if (all) mergeOne(all); // …or after ANY of the block's defs + } else { + mergeOne(outSets[p.from]); + } + } + return merged; +} + +/** Order-stable union of two def-sets (shares `a` when `b` adds nothing). */ +function unionSets(a: DefSet, b: DefSet): DefSet { + let target = a; + let copied = false; + for (const key of b) { + if (!target.has(key)) { + if (!copied) { + target = new Set(a); + copied = true; + } + target.add(key); + } + } + return target; +} + +/** Per-binding equality with a reference fast path (sets only ever grow). */ +function latticeEquals(a: Lattice, b: Lattice): boolean { + if (a === b) return true; + if (a.size !== b.size) return false; + for (const [k, bSet] of b) { + const aSet = a.get(k); + if (aSet === bSet) continue; + if (!aSet || aSet.size !== bSet.size) return false; + for (const v of bSet) if (!aSet.has(v)) return false; + } + return true; +} diff --git a/gitnexus/src/core/ingestion/cfg/traversal-result.ts b/gitnexus/src/core/ingestion/cfg/traversal-result.ts new file mode 100644 index 000000000..d26500fa5 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/traversal-result.ts @@ -0,0 +1,21 @@ +/** + * TraversalResult (issue #2081, M1). + * + * Visiting a statement (or a statement sequence) returns the block its control + * flow ENTERS through, plus the set of blocks whose **normal** control flows + * out the bottom (the "dangling exits") — to be wired to the entry of whatever + * comes next. Abnormal exits (return/break/continue/throw) are wired directly + * to their targets during the walk and are NOT part of `exits`. + * + * A statement that cannot fall through (e.g. ends in `return`/`throw`, or both + * branches of an `if` return) yields an empty `exits` array. + */ +export interface TraversalResult { + /** Block index control enters this statement/sequence through. */ + readonly entry: number; + /** Block indices whose normal control falls out the bottom (may be empty). */ + readonly exits: readonly number[]; +} + +/** A sequence of statements that produced no blocks (e.g. an empty body). */ +export const emptyTraversal = (entry: number): TraversalResult => ({ entry, exits: [entry] }); diff --git a/gitnexus/src/core/ingestion/cfg/types.ts b/gitnexus/src/core/ingestion/cfg/types.ts new file mode 100644 index 000000000..ed789ec30 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/types.ts @@ -0,0 +1,161 @@ +/** + * CFG data model — plain, JSON-serializable types (issue #2081, M1). + * + * These cross the worker→main boundary and the disk-backed/durable ParsedFile + * store, so they must contain NO tree-sitter AST references, class instances, + * or anything that does not survive `JSON.stringify` → `JSON.parse`. Block and + * edge endpoints are referenced by integer index within a function's CFG. + * + * The per-language `CfgVisitor` (built in the parse worker, where the AST + * lives — see the M1 plan KTD1/KTD7) produces a `FunctionCfg` per function; the + * array of them is what rides on `ParsedFile.cfgSideChannel`. + */ + +/** + * One distinct declared variable (binding) within a function (#2082 M2 U1). + * + * Statement facts reference bindings by integer index into + * {@link FunctionCfg.bindings} — names appear once per binding instead of once + * per occurrence (measured ~4× smaller serialized payload than named records). + * Distinct bindings of the same name (shadowing) get distinct entries, which is + * what keeps an inner `let x` from falsely killing the outer `x`'s definitions + * in the reaching-defs solver. NOTE: no field here may be named `nodeId` — the + * durable parsedfile-store reviver dedups objects keyed on that field name. + */ +export interface BindingEntry { + /** Source-level variable name (what the persisted edge's `reason` carries). */ + readonly name: string; + /** + * 1-based line/0-based column of the canonical declaration site — `var` + * multi-declarations canonicalize to the FIRST declaration in source order. + * Both 0 for synthetic bindings. + */ + readonly declLine: number; + readonly declColumn: number; + /** How the binding was introduced (param/catch matter to the M3 taint pass). */ + readonly kind: 'var' | 'let' | 'const' | 'param' | 'catch' | 'function' | 'class' | 'module'; + /** + * True when the name has no in-function declaration site (implicit global, + * import, or a variable captured from an enclosing function) — keyed + * `name@module` in edge ids instead of `name:line:col`. + */ + readonly synthetic?: boolean; +} + +/** + * Def/use facts for one harvested statement (or construct header), in + * execution order within its block (#2082 M2 U1). `defs`/`uses` are indices + * into {@link FunctionCfg.bindings}. A compound assignment / update expression + * lists its binding in BOTH. Self-describing — `line` is carried here, never + * inferred from the block's text fragments (facts-only records exist, e.g. + * params on ENTRY and catch params). + * + * `mayDefs` (tri-review P1): defs harvested inside CONDITIONALLY-EVALUATED + * subexpressions — short-circuit right operands (`a && (x = v)`, + * `c ?? (c = load())`), ternary arms, logical-assignment operators, and + * switch case-test expressions. The solver treats them as GEN WITHOUT KILL: + * treating them as must-defs would falsely kill the prior def on the + * not-taken path (a taint false negative on core JS idioms). Optional — + * absent means none. + */ +export interface StatementFacts { + readonly line: number; + readonly defs: readonly number[]; + readonly uses: readonly number[]; + readonly mayDefs?: readonly number[]; +} + +/** A basic block: a maximal straight-line run of statements between leaders. */ +export interface BasicBlockData { + /** Block index within its function. The synthetic ENTRY is always 0. */ + readonly index: number; + readonly startLine: number; + readonly endLine: number; + /** Source snippet for the block (empty for synthetic ENTRY/EXIT). */ + readonly text: string; + readonly kind: 'entry' | 'exit' | 'normal'; + /** + * Per-statement def/use facts in execution order (#2082 M2 U1). Present only + * when the producing visitor harvests (TS/JS under `--pdg`); absent on + * hand-built or pre-M2 CFGs — the reaching-defs solver reports `no-facts`. + */ + readonly statements?: readonly StatementFacts[]; +} + +/** + * Why one block flows to another — drives the `reason` on the emitted CFG edge. + * + * Kind invariant (M2): a bare jump kind (`return`/`break`/`continue`) means the + * SOURCE block's terminator is that jump statement. A `finally-*` kind marks a + * COMPLETION edge out of a `finally` body's exit — the leg that resumes a jump + * which was re-routed through the finally (issue #2082 U2). Reusing the bare + * kinds on completion edges would silently break consumers that infer the + * source block's terminator from the kind, and a single generic kind would lose + * WHICH jump each completion edge completes when a shared finally has several + * pending targets. + */ +export type CfgEdgeKind = + | 'seq' // straight-line fallthrough + | 'cond-true' // branch taken (if/while/for condition true) + | 'cond-false' // branch not taken / loop exit + | 'loop-back' // back-edge to a loop header + | 'break' // break → loop/switch exit (or the finally it must cross) + | 'continue' // continue → loop header (or the finally it must cross) + | 'return' // return → function EXIT (or the finally it must cross) + | 'throw' // throw → nearest handler / finally / EXIT + | 'switch-case' // dispatch to a case + | 'fallthrough' // switch case → next case (no break) + | 'finally-return' // finally exit → resumed return target (EXIT / outer finally) + | 'finally-break' // finally exit → resumed break target + | 'finally-continue'; // finally exit → resumed continue target + +export interface CfgEdgeData { + readonly from: number; + readonly to: number; + readonly kind: CfgEdgeKind; +} + +/** One function's control-flow graph. `cfgSideChannel` is `readonly FunctionCfg[]`. */ +export interface FunctionCfg { + readonly filePath: string; + /** Source span of the owning function — anchors the BasicBlock node ids. */ + readonly functionStartLine: number; + readonly functionEndLine: number; + /** + * Start COLUMN of the owning function. Combined with `functionStartLine` it + * disambiguates the BasicBlock node ids when two functions share a start line + * — e.g. `{ a: () => x(), b: () => y() }`, where both arrows begin on the same + * line and each restarts its block indices at 0. Without the column the ids + * collide and the graph's first-writer-wins `addNode` silently drops the + * second function's blocks and cross-wires its edges. + */ + readonly functionStartColumn: number; + readonly entryIndex: number; + readonly exitIndex: number; + readonly blocks: readonly BasicBlockData[]; + readonly edges: readonly CfgEdgeData[]; + /** + * The function's binding table (#2082 M2 U1) — referenced by index from + * {@link BasicBlockData.statements}. Present iff statement facts are. + */ + readonly bindings?: readonly BindingEntry[]; +} + +/** + * Per-language CFG strategy. Invoked **in the parse worker** for each function + * node. `TNode` is the language's AST node type (tree-sitter `SyntaxNode` for + * TS/JS) — kept generic so this module stays AST-library-agnostic. Returns + * `undefined` when the node is not a CFG-bearing function (the caller skips it). + */ +export interface CfgVisitor { + buildFunctionCfg(fnNode: TNode, filePath: string): FunctionCfg | undefined; + + /** + * Whether `node` is a CFG-bearing function this visitor handles. Lets the + * worker enumerate functions (and apply the per-function line budget) by a + * cheap node-type test, instead of attempting to build a CFG for every AST + * node. `buildFunctionCfg` still re-checks, so this is purely an optimization + * + the seam the line-budget hooks into. + */ + isFunction(node: TNode): boolean; +} diff --git a/gitnexus/src/core/ingestion/cfg/visitors/typescript-harvest.ts b/gitnexus/src/core/ingestion/cfg/visitors/typescript-harvest.ts new file mode 100644 index 000000000..81823c97a --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/visitors/typescript-harvest.ts @@ -0,0 +1,656 @@ +/** + * TS/JS def/use harvester (#2082 M2 U1). + * + * Runs in the parse worker next to the CFG visitor, extracting per-statement + * variable definition/use facts that ride the side channel for the + * reaching-defs solver (`cfg/reaching-defs.ts`). Output is the per-function + * binding table ({@link BindingEntry}[]) plus {@link StatementFacts} records + * the visitor attaches to blocks as it walks. + * + * TWO-PHASE, ORDER-INDEPENDENT (load-bearing): the CFG walk is NOT source-order + * — `visitTry` builds the finally body before the protected body, `visitFor` + * creates the init block after walking the body, `visitDoWhile` the condition + * before the body. Resolving names against a scope stack populated *during* + * that walk would mis-resolve common code (`try { var v = 1; } finally + * { use(v); }` keys the use synthetically while the def gets the real binding — + * the def→use fact silently never forms, a taint false negative). So phase 1 + * pre-scans the whole function subtree once, collecting every declaration into + * a completed lexical scope tree (also resolving `var` hoisting and multi-decl + * canonicalization order-independently, eslint-scope style); phase 2 resolves + * defs/uses against that finished tree from any walk order. + * + * v1 def-semantics scope (plan KTD4): var/let/const declarations, assignments + * (plain/compound/destructuring), update expressions, function/class + * declarations, parameters (incl. defaults/rest/destructured), catch params, + * for-in/of heads. EXCLUDED, deliberately: property/member writes (`this.x=`, + * `obj.p=` — TypeScript-CFA precedent), and BOTH directions of nested-function + * capture — writes to outer variables from nested bodies AND reads of captured + * variables inside nested bodies are invisible (nested functions are opaque + * blocks in the enclosing CFG; callback flows like `arr.forEach(() => sink(y))` + * register no use of `y` — closure/callback dataflow is M4 territory and the + * M3 consumer contract must name it). + * + * Identifiers with no in-function declaration (implicit globals, imports, + * variables captured from an enclosing function) resolve to a SYNTHETIC + * module-level binding (`name@module`), applied identically by def and use + * harvesting so `notDeclared = 1; use(notDeclared)` still forms a fact. + * + * NOTE: nothing serialized here may carry a field named `nodeId` — the durable + * parsedfile-store reviver dedups objects keyed on that field name. + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import type { BindingEntry, StatementFacts } from '../types.js'; + +/** Node types that own a nested CFG — their subtrees are opaque to harvesting. */ +const NESTED_FUNCTION_TYPES = new Set([ + 'function_declaration', + 'function_expression', + 'arrow_function', + 'method_definition', + 'generator_function_declaration', + 'generator_function', + 'async_function_declaration', + 'async_arrow_function', +]); + +/** Function-ish declaration statements whose NAME still binds in the enclosing scope. */ +const FUNCTION_DECL_TYPES = new Set([ + 'function_declaration', + 'generator_function_declaration', + 'async_function_declaration', +]); + +/** + * Nodes that open a lexical scope for `let`/`const`/`class`/catch bindings. + * A `switch` BODY is deliberately ONE scope shared by all case arms (JS + * semantics: `case 1: let x = 1; case 2: use(x)` is the same binding). + */ +const SCOPE_TYPES = new Set([ + 'statement_block', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'catch_clause', + 'switch_body', +]); + +/** Type-position subtrees — identifiers inside them are not value uses. */ +const TYPE_CONTEXT_TYPES = new Set([ + 'type_annotation', + 'type_arguments', + 'type_parameters', + 'type_predicate_annotation', + 'asserts_annotation', +]); + +interface Scope { + readonly parent: Scope | null; + /** name → binding index */ + readonly table: Map; +} + +export class TsHarvester { + private readonly bindings: BindingEntry[] = []; + /** Scope-opening node id → its scope. */ + private readonly scopeByNode = new Map(); + private readonly root: Scope = { parent: null, table: new Map() }; + /** name → synthetic binding index (implicit global / import / captured). */ + private readonly synthetic = new Map(); + private readonly fnId: number; + /** + * Innermost enclosing scope per visited node id, filled during the prescan + * (which already touches every named node once). Makes phase-2 resolution + * O(scope-chain) instead of O(AST-depth) per identifier — a deeply-chained + * single-statement expression (generated code) otherwise turns the + * parent-chain walk quadratic (tri-review perf finding). + */ + private readonly nearestScopeCache = new Map(); + /** + * >0 while walking a conditionally-evaluated subexpression (short-circuit + * right operand, ternary arm, logical-assignment target, case test). Defs + * found there are MAY-defs — gen without kill (tri-review P1: a must-def + * here falsely kills the prior def on the not-taken path). + */ + private conditionalDepth = 0; + + constructor(private readonly fnNode: SyntaxNode) { + this.fnId = fnNode.id; + this.scopeByNode.set(fnNode.id, this.root); + this.declareParams(fnNode); + const body = fnNode.childForFieldName('body'); + if (body) + this.prescan(body, body.type === 'statement_block' ? this.openScope(body) : this.root); + } + + /** The completed binding table — pass to `CfgBuilder.finish`. */ + table(): readonly BindingEntry[] { + return this.bindings; + } + + // ── phase 1: declaration pre-scan ──────────────────────────────────────── + + private openScope(node: SyntaxNode): Scope { + const existing = this.scopeByNode.get(node.id); + if (existing) return existing; + const scope: Scope = { parent: this.nearestScopeOf(node), table: new Map() }; + this.scopeByNode.set(node.id, scope); + return scope; + } + + private nearestScopeOf(node: SyntaxNode): Scope { + for (let p = node.parent; p; p = p.parent) { + const s = this.scopeByNode.get(p.id); + if (s) return s; + if (p.id === this.fnId) break; + } + return this.root; + } + + private declare( + nameNode: SyntaxNode, + kind: BindingEntry['kind'], + scope: Scope, + hoistToRoot: boolean, + ): void { + const target = hoistToRoot ? this.root : scope; + const name = nameNode.text; + // `var` multi-declaration (and a param + `var` of the same name) is ONE + // binding — first declaration in source order is canonical. The dedup is + // scoped to the single target table, so an inner `let x` shadowing a root + // `var x` still gets its own entry in its own scope. + if (target.table.has(name)) return; + target.table.set(name, this.bindings.length); + this.bindings.push({ + name, + declLine: nameNode.startPosition.row + 1, + declColumn: nameNode.startPosition.column, + kind, + }); + } + + private declareParams(fnNode: SyntaxNode): void { + const params = fnNode.childForFieldName('parameters') ?? fnNode.childForFieldName('parameter'); + if (!params) return; + if (params.type === 'identifier') { + this.declare(params, 'param', this.root, true); // `x => …` single-param arrow + return; + } + for (let i = 0; i < params.namedChildCount; i++) { + const p = params.namedChild(i); + if (!p) continue; + // TS wraps each param (required_parameter/optional_parameter, field + // `pattern`); plain JS puts the pattern directly in formal_parameters. + const pattern = p.childForFieldName('pattern') ?? p; + this.declarePattern(pattern, 'param', this.root, true); + } + } + + /** Declare every name bound by a (possibly destructuring) pattern. */ + private declarePattern( + node: SyntaxNode, + kind: BindingEntry['kind'], + scope: Scope, + hoistToRoot: boolean, + ): void { + switch (node.type) { + case 'identifier': + case 'shorthand_property_identifier_pattern': + this.declare(node, kind, scope, hoistToRoot); + return; + case 'rest_pattern': + case 'object_pattern': + case 'array_pattern': + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c) this.declarePattern(c, kind, scope, hoistToRoot); + } + return; + case 'pair_pattern': { + const value = node.childForFieldName('value'); + if (value) this.declarePattern(value, kind, scope, hoistToRoot); + return; + } + case 'assignment_pattern': + case 'object_assignment_pattern': { + const left = node.childForFieldName('left'); + if (left) this.declarePattern(left, kind, scope, hoistToRoot); + return; + } + default: + // Type annotations / unknown wrappers — descend defensively. + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c && !TYPE_CONTEXT_TYPES.has(c.type)) { + this.declarePattern(c, kind, scope, hoistToRoot); + } + } + } + } + + private prescan(node: SyntaxNode, scope: Scope): void { + this.nearestScopeCache.set(node.id, scope); + const t = node.type; + if (NESTED_FUNCTION_TYPES.has(t) && node.id !== this.fnId) { + // A nested function's NAME binds in the enclosing scope; its body is opaque. + if (FUNCTION_DECL_TYPES.has(t)) { + const name = node.childForFieldName('name'); + if (name) this.declare(name, 'function', scope, false); + } + return; + } + + let childScope = scope; + if (SCOPE_TYPES.has(t)) childScope = this.openScope(node); + + switch (t) { + case 'lexical_declaration': { + const kind = node.child(0)?.type === 'const' ? 'const' : 'let'; + this.declareDeclarators(node, kind, childScope, false); + break; + } + case 'variable_declaration': + this.declareDeclarators(node, 'var', childScope, true); + break; + case 'class_declaration': { + const name = node.childForFieldName('name'); + if (name) this.declare(name, 'class', childScope, false); + break; + } + case 'catch_clause': { + const param = node.childForFieldName('parameter'); + if (param) this.declarePattern(param, 'catch', childScope, false); + break; + } + case 'for_in_statement': + case 'for_of_statement': { + // `for (const x of xs)` — the `kind` keyword marks a declaration; a bare + // `for (x of xs)` left is an assignment, resolved at use time instead. + const kindNode = node.childForFieldName('kind'); + const left = node.childForFieldName('left'); + if (kindNode && left) { + const k = kindNode.type === 'var' ? 'var' : kindNode.type === 'const' ? 'const' : 'let'; + this.declarePattern(left, k, childScope, k === 'var'); + } + break; + } + default: + break; + } + + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c) this.prescan(c, childScope); + } + } + + private declareDeclarators( + declNode: SyntaxNode, + kind: 'var' | 'let' | 'const', + scope: Scope, + hoistToRoot: boolean, + ): void { + for (let i = 0; i < declNode.namedChildCount; i++) { + const d = declNode.namedChild(i); + if (d?.type !== 'variable_declarator') continue; + const name = d.childForFieldName('name'); + if (name) this.declarePattern(name, kind, scope, hoistToRoot); + } + } + + // ── phase 2: per-statement fact extraction ─────────────────────────────── + + /** + * Def/use facts for one statement (or construct-header expression) node. + * Safe from any walk order — resolution consults the completed scope tree. + */ + facts(node: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(node.startPosition.row + 1); + this.walkValue(node, acc); + return acc.finish(); + } + + /** + * Facts for an expression whose WHOLE evaluation is conditional (switch + * case tests, which only run when earlier cases didn't match) — every def + * inside becomes a may-def. + */ + factsConditional(node: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(node.startPosition.row + 1); + this.conditional(() => this.walkValue(node, acc)); + return acc.finish(); + } + + /** Facts for a `for (left in/of right)` head: left binds/assigns, right is used. */ + forInHeadFacts(stmt: SyntaxNode): StatementFacts { + const acc = new FactAccumulator(stmt.startPosition.row + 1); + const left = stmt.childForFieldName('left'); + const right = stmt.childForFieldName('right'); + if (left) this.walkDefPattern(left, acc); + if (right) this.walkValue(right, acc); + return acc.finish(); + } + + /** ENTRY-block facts for the function's parameters (defs + default-value uses). */ + paramFacts(): StatementFacts | undefined { + const fnNode = this.fnNode; + const params = fnNode.childForFieldName('parameters') ?? fnNode.childForFieldName('parameter'); + if (!params) return undefined; + const acc = new FactAccumulator(fnNode.startPosition.row + 1); + if (params.type === 'identifier') { + this.def(params, acc); + } else { + for (let i = 0; i < params.namedChildCount; i++) { + const p = params.namedChild(i); + if (!p) continue; + const pattern = p.childForFieldName('pattern') ?? p; + this.walkDefPattern(pattern, acc); + const dflt = p.childForFieldName('value'); + if (dflt) this.walkValue(dflt, acc); + } + } + return acc.defCount() || acc.useCount() ? acc.finish() : undefined; + } + + /** Def fact for a `catch (e)` parameter — prepend to the handler entry block. */ + catchParamFacts(catchClause: SyntaxNode): StatementFacts | undefined { + const param = catchClause.childForFieldName('parameter'); + if (!param) return undefined; + const acc = new FactAccumulator(catchClause.startPosition.row + 1); + this.walkDefPattern(param, acc); + return acc.defCount() ? acc.finish() : undefined; + } + + private resolve(nameNode: SyntaxNode): number { + const name = nameNode.text; + // Fast path: the prescan cached every visited node's innermost scope, so + // resolution walks the SCOPE chain (shallow), not the AST parent chain + // (arbitrarily deep in chained expressions). The parent-chain walk remains + // as fallback for the few nodes the prescan never visits (e.g. a nested + // function declaration's own name node). + const cached = this.nearestScopeCache.get(nameNode.id); + let startScope: Scope | null = cached ?? null; + if (!startScope) { + for (let p: SyntaxNode | null = nameNode; p; p = p.parent) { + const scope = this.scopeByNode.get(p.id) ?? this.nearestScopeCache.get(p.id); + if (scope) { + startScope = scope; + break; + } + if (p.id === this.fnId) { + startScope = this.root; + break; + } + } + } + for (let s: Scope | null = startScope; s; s = s.parent) { + const idx = s.table.get(name); + if (idx !== undefined) return idx; + } + // No in-function declaration — synthetic module-level binding, shared by + // defs and uses so `notDeclared = 1; use(notDeclared)` still forms a fact. + let idx = this.synthetic.get(name); + if (idx === undefined) { + idx = this.bindings.length; + this.synthetic.set(name, idx); + this.bindings.push({ name, declLine: 0, declColumn: 0, kind: 'module', synthetic: true }); + } + return idx; + } + + private def(nameNode: SyntaxNode, acc: FactAccumulator): void { + if (this.conditionalDepth > 0) acc.addMayDef(this.resolve(nameNode)); + else acc.addDef(this.resolve(nameNode)); + } + + /** Run `fn` with defs demoted to may-defs (conditionally-evaluated context). */ + private conditional(fn: () => void): void { + this.conditionalDepth++; + try { + fn(); + } finally { + this.conditionalDepth--; + } + } + + /** Strip wrappers that don't change the lvalue (`(x) += 1`, `x! ++`). */ + private unwrapLvalue(node: SyntaxNode): SyntaxNode { + let n = node; + while (n.type === 'parenthesized_expression' || n.type === 'non_null_expression') { + const inner = n.namedChild(0); + if (!inner) break; + n = inner; + } + return n; + } + + private use(nameNode: SyntaxNode, acc: FactAccumulator): void { + acc.addUse(this.resolve(nameNode)); + } + + /** Value-position walk: collect uses; route def positions to the pattern walk. */ + private walkValue(node: SyntaxNode, acc: FactAccumulator): void { + const t = node.type; + if (TYPE_CONTEXT_TYPES.has(t)) return; + if (NESTED_FUNCTION_TYPES.has(t) && node.id !== this.fnId) { + // Opaque nested function: its NAME (function declaration) is a def in + // the enclosing scope; captured reads/writes inside are invisible (KTD4). + if (FUNCTION_DECL_TYPES.has(t)) { + const name = node.childForFieldName('name'); + if (name) this.def(name, acc); + } + return; + } + + switch (t) { + case 'identifier': + case 'shorthand_property_identifier': + this.use(node, acc); + return; + case 'lexical_declaration': + case 'variable_declaration': + for (let i = 0; i < node.namedChildCount; i++) { + const d = node.namedChild(i); + if (d?.type !== 'variable_declarator') continue; + const name = d.childForFieldName('name'); + const value = d.childForFieldName('value'); + // A bare `var x;` mid-function is hoisted and writes NOTHING at + // runtime — harvesting it as a def would fabricate a kill of the + // live def (`x = source(); var x; sink(x)` must keep source→sink; + // tri-review P2). `let`/`const` declarators genuinely initialize. + if (name && (value || t === 'lexical_declaration')) { + this.walkDefPattern(name, acc); + } + if (value) this.walkValue(value, acc); + } + return; + case 'assignment_expression': { + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + if (left) this.walkDefPattern(this.unwrapLvalue(left), acc); + if (right) this.walkValue(right, acc); + return; + } + case 'augmented_assignment_expression': { + // `x += y` both defines and uses x. The logical-assignment operators + // (`||=`, `&&=`, `??=`) only WRITE conditionally — their def is a + // may-def (the read always happens). + const left = node.childForFieldName('left') + ? this.unwrapLvalue(node.childForFieldName('left') as SyntaxNode) + : null; + const right = node.childForFieldName('right'); + const op = node.childForFieldName('operator')?.type ?? ''; + const logical = op === '||=' || op === '&&=' || op === '??='; + if (left?.type === 'identifier') { + if (logical) this.conditional(() => this.def(left, acc)); + else this.def(left, acc); + this.use(left, acc); + } else if (left) { + this.walkValue(left, acc); // member/subscript target — uses only + } + // The RHS of a logical assignment is itself conditionally evaluated. + if (right) { + if (logical) this.conditional(() => this.walkValue(right, acc)); + else this.walkValue(right, acc); + } + return; + } + case 'update_expression': { + const rawArg = node.childForFieldName('argument'); + const arg = rawArg ? this.unwrapLvalue(rawArg) : null; + if (arg?.type === 'identifier') { + this.def(arg, acc); + this.use(arg, acc); + } else if (arg) { + this.walkValue(arg, acc); + } + return; + } + case 'binary_expression': { + // Short-circuit operators evaluate their RIGHT operand conditionally: + // a def inside it (`a && (x = clean())`, `c ?? (c = load())`) must be + // a may-def or the not-taken path's prior def is falsely killed + // (tri-review P1). Other binary operators evaluate both sides. + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + const op = node.childForFieldName('operator')?.type ?? ''; + if (left) this.walkValue(left, acc); + if (right) { + if (op === '&&' || op === '||' || op === '??') { + this.conditional(() => this.walkValue(right, acc)); + } else { + this.walkValue(right, acc); + } + } + return; + } + case 'ternary_expression': { + // Each arm is conditionally evaluated — defs inside are may-defs. + const cond = node.childForFieldName('condition'); + const consequence = node.childForFieldName('consequence'); + const alternative = node.childForFieldName('alternative'); + if (cond) this.walkValue(cond, acc); + if (consequence) this.conditional(() => this.walkValue(consequence, acc)); + if (alternative) this.conditional(() => this.walkValue(alternative, acc)); + return; + } + case 'class_declaration': { + // The class NAME is a def (prescan declared the binding) — without + // this case the default walk would record it as a bogus USE in plain + // JS (the name is an `identifier` there; in TS it's a type_identifier + // and would be silently skipped, losing the def either way). The body + // walk picks up field-initializer uses; methods are opaque nested fns. + const name = node.childForFieldName('name'); + if (name) this.def(name, acc); + const body = node.childForFieldName('body'); + if (body) this.walkValue(body, acc); + return; + } + case 'class': { + // Class EXPRESSION: its name (if any) binds only inside the class — + // not a def in the enclosing function. Walk only the body. + const body = node.childForFieldName('body'); + if (body) this.walkValue(body, acc); + return; + } + default: + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c) this.walkValue(c, acc); + } + } + } + + /** Assignment-target walk: identifiers bind; member/subscript targets are uses. */ + private walkDefPattern(node: SyntaxNode, acc: FactAccumulator): void { + switch (node.type) { + case 'identifier': + case 'shorthand_property_identifier_pattern': + this.def(node, acc); + return; + case 'rest_pattern': + case 'object_pattern': + case 'array_pattern': + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c) this.walkDefPattern(c, acc); + } + return; + case 'pair_pattern': { + const key = node.childForFieldName('key'); + const value = node.childForFieldName('value'); + if (key?.type === 'computed_property_name') this.walkValue(key, acc); + if (value) this.walkDefPattern(value, acc); + return; + } + case 'assignment_pattern': + case 'object_assignment_pattern': { + const left = node.childForFieldName('left'); + const right = node.childForFieldName('right'); + if (left) this.walkDefPattern(left, acc); + if (right) this.walkValue(right, acc); + return; + } + case 'member_expression': + case 'subscript_expression': + // Property/element write — NOT a scalar def (KTD4); its identifiers + // (object, computed key) are uses. + this.walkValue(node, acc); + return; + default: + for (let i = 0; i < node.namedChildCount; i++) { + const c = node.namedChild(i); + if (c && !TYPE_CONTEXT_TYPES.has(c.type)) this.walkDefPattern(c, acc); + } + } + } +} + +/** Ordered, deduplicating def/use collector for one statement record. */ +class FactAccumulator { + private readonly defs: number[] = []; + private readonly uses: number[] = []; + private readonly mayDefs: number[] = []; + private readonly defSeen = new Set(); + private readonly useSeen = new Set(); + private readonly mayDefSeen = new Set(); + + constructor(private readonly line: number) {} + + addDef(idx: number): void { + if (this.defSeen.has(idx)) return; + this.defSeen.add(idx); + this.defs.push(idx); + } + + /** A def that may not execute (conditional context) — gen without kill. */ + addMayDef(idx: number): void { + if (this.mayDefSeen.has(idx)) return; + this.mayDefSeen.add(idx); + this.mayDefs.push(idx); + } + + addUse(idx: number): void { + if (this.useSeen.has(idx)) return; + this.useSeen.add(idx); + this.uses.push(idx); + } + + defCount(): number { + return this.defs.length + this.mayDefs.length; + } + + useCount(): number { + return this.uses.length; + } + + finish(): StatementFacts { + return { + line: this.line, + defs: this.defs, + uses: this.uses, + // Optional field stays absent when empty — keeps the serialized + // side-channel payload lean (most statements have no may-defs). + ...(this.mayDefs.length > 0 ? { mayDefs: this.mayDefs } : {}), + }; + } +} diff --git a/gitnexus/src/core/ingestion/cfg/visitors/typescript.ts b/gitnexus/src/core/ingestion/cfg/visitors/typescript.ts new file mode 100644 index 000000000..1382fdb66 --- /dev/null +++ b/gitnexus/src/core/ingestion/cfg/visitors/typescript.ts @@ -0,0 +1,782 @@ +/** + * TS/JS CfgVisitor (issue #2081, M1). + * + * Walks a TypeScript/JavaScript function's tree-sitter AST and drives the + * language-agnostic {@link CfgBuilder} to produce a serializable + * {@link FunctionCfg}. TS and JS share a grammar family (tree-sitter-typescript + * reuses tree-sitter-javascript's statement nodes), so one visitor covers both. + * + * Design — a `visit_` dispatch over the statement taxonomy. The + * classic CFG hazards (R10) are handled explicitly: + * - loops allocate a dedicated **loop-exit** block so `break` has a concrete + * target before the loop's successor is known; `continue` targets the + * header/increment; the back-edge closes the loop. + * - `switch` cases fall through naturally: a case body that does not `break` + * yields non-empty `exits`, which we wire to the next case as `fallthrough`; + * a case that `break`s wires to the switch exit (via {@link ControlFlowContext}) + * and yields no fall-out. + * - `try/catch/finally` routes both normal completion AND a `throw` in the try + * through `finally` (the finally block post-dominates the try/catch); a + * `throw` with no catch propagates through finally to the enclosing handler. + * - EARLY EXITS THROUGH FINALLY (#2082 M2 U2, closes the M1 soundness gap): a + * `break`/`continue`/`return` whose jump CROSSES a `finally` is re-routed to + * the finally entry (keeping its bare jump kind), and the finally's exits + * gain a `finally-return`/`finally-break`/`finally-continue` completion edge + * to the resumed target. Threading is TARGET-RELATIVE via finalizer frames + * interleaved on the {@link ControlFlowContext} stack: only the finallys + * lexically between the jump and its target thread (a `break` whose loop is + * wholly inside the try keeps its direct edge — re-routing it would let a + * finally redefinition falsely kill in-loop defs for reaching-defs). Nested + * finallys chain inner→outer; finally-as-shared-join conflates exit paths + * (sound over-approximation; duplication-per-exit-path was rejected). An + * empty/comment-only finally pushes no frame — jumps keep direct edges. + * - labeled `break`/`continue` resolve against the labeled construct's frame: + * loops/switches carry their full label LIST (`outer: inner: for` resolves + * both), and a labeled NON-loop statement (`blk: { … break blk; … }`) gets + * a break-target frame whose target is a synthesized join after the body — + * the M1 route-to-EXIT fallback removed the real continuation and falsely + * killed defs for reaching-defs (tri-review P1). + * + * Known limitations: + * - A jump whose label STILL fails to resolve (malformed source) keeps the + * conservative route-to-EXIT + thread-all-finallys fallback in + * visitBreak/visitContinue — single-exit preserved, no finally bypassed, + * but the continuation path is approximate. + * - Exceptional flow stays the sound over-approximation: EVERY protected-region + * block edges to the handler (an exception may fire mid-block), which + * over-supplies reaching-defs facts into `catch` — extra facts, never false + * kills. Per-leader throw precision is deliberately deferred (M3 decides). + * - Def/use harvest scope (#2082 M2, see typescript-harvest.ts for the full + * v1 semantics table): member/property writes are not scalar defs; nested + * function bodies are opaque in BOTH directions (writes to and reads of + * captured outer variables are invisible — callback flows are M4 territory); + * `case x:` test uses attach to the switch dispatch block (sound + * over-approximation of in-order case evaluation). + * + * Block/edge accounting and reachability are pinned in + * `test/unit/cfg/cfg-builder.test.ts` (core) and + * `test/unit/cfg/typescript-visitor.test.ts` (this visitor, per hazard). + */ +import type { SyntaxNode } from '../../utils/ast-helpers.js'; +import { CfgBuilder } from '../cfg-builder.js'; +import { + ControlFlowContext, + drainFinalizerPending, + wireJumpThroughFinalizers, +} from '../control-flow-context.js'; +import type { TraversalResult } from '../traversal-result.js'; +import type { CfgVisitor, FunctionCfg } from '../types.js'; +import { TsHarvester } from './typescript-harvest.js'; + +/** TS/JS node types that own a CFG-bearing function body. */ +const TS_FUNCTION_TYPES = new Set([ + 'function_declaration', + 'function_expression', + 'arrow_function', + 'method_definition', + 'generator_function_declaration', + 'generator_function', + 'async_function_declaration', + 'async_arrow_function', +]); + +/** Statement node types that break a basic block (everything else coalesces). */ +const CONTROL_FLOW_TYPES = new Set([ + 'if_statement', + 'while_statement', + 'do_statement', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'switch_statement', + 'try_statement', + 'return_statement', + 'break_statement', + 'continue_statement', + 'throw_statement', + 'labeled_statement', + 'statement_block', +]); + +const LOOP_OR_SWITCH_TYPES = new Set([ + 'while_statement', + 'do_statement', + 'for_statement', + 'for_in_statement', + 'for_of_statement', + 'switch_statement', +]); + +const startLineOf = (n: SyntaxNode): number => n.startPosition.row + 1; +const endLineOf = (n: SyntaxNode): number => n.endPosition.row + 1; + +/** A statement sequence that produced no blocks (empty body) is "transparent". */ +type SeqResult = TraversalResult | null; + +/** + * Per-function walk state. One instance is created per function so the + * {@link ControlFlowContext}, exception-handler stack, and pending label are + * scoped to that function and never leak across functions. + */ +class TsCfgWalk { + private readonly cfc = new ControlFlowContext(); + /** Stack of exception-handler entry blocks (catch/finally) a `throw` jumps to. */ + private readonly handlers: number[] = []; + /** Labels awaiting the construct they precede (`outer: inner: for` = both). */ + private pendingLabels: string[] = []; + + constructor( + private readonly builder: CfgBuilder, + /** Def/use fact extractor (#2082 M2 U1) — phase-2 only; its scope tree is + * already complete, so any walk order resolves names correctly. */ + private readonly harvest: TsHarvester, + ) {} + + /** Statements of a block node, ignoring comments. */ + private statementsOf(block: SyntaxNode): SyntaxNode[] { + return block.namedChildren.filter((c) => c.type !== 'comment'); + } + + /** The `body` block of a node (field, or the first statement_block child). */ + private bodyBlockOf(node: SyntaxNode): SyntaxNode | undefined { + return ( + node.childForFieldName('body') ?? node.namedChildren.find((c) => c.type === 'statement_block') + ); + } + + /** Visit a body that may be a `statement_block` or a single statement. */ + private visitBody(node: SyntaxNode | undefined | null): SeqResult { + if (!node) return null; + if (node.type === 'statement_block') return this.visitSeq(this.statementsOf(node)); + return this.visitStmt(node); + } + + /** Wire a sequence of statements, coalescing straight-line runs into blocks. */ + visitSeq(stmts: SyntaxNode[]): SeqResult { + let entry: number | undefined; + let dangling: number[] = []; + let openSimple: number | undefined; + + for (const stmt of stmts) { + if (CONTROL_FLOW_TYPES.has(stmt.type)) { + openSimple = undefined; // close any open straight-line block + const res = this.visitStmt(stmt); + if (res === null) continue; // transparent (empty nested block) + if (entry === undefined) entry = res.entry; + else this.builder.connect(dangling, res.entry, 'seq'); + dangling = [...res.exits]; + } else { + // Simple statement — coalesce into the current straight-line block. + if (openSimple === undefined) { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + if (entry === undefined) entry = idx; + else this.builder.connect(dangling, idx, 'seq'); + openSimple = idx; + dangling = [idx]; + } else { + this.builder.extendBlock( + openSimple, + endLineOf(stmt), + stmt.text, + this.harvest.facts(stmt), + ); + } + } + } + + if (entry === undefined) return null; + return { entry, exits: dangling }; + } + + /** Dispatch one statement to its handler. Non-null except for empty blocks. */ + visitStmt(stmt: SyntaxNode): SeqResult { + switch (stmt.type) { + case 'if_statement': + return this.visitIf(stmt); + case 'while_statement': + return this.visitWhile(stmt); + case 'do_statement': + return this.visitDoWhile(stmt); + case 'for_statement': + return this.visitFor(stmt); + case 'for_in_statement': + case 'for_of_statement': + return this.visitForIn(stmt); + case 'switch_statement': + return this.visitSwitch(stmt); + case 'try_statement': + return this.visitTry(stmt); + case 'return_statement': + return this.visitReturn(stmt); + case 'throw_statement': + return this.visitThrow(stmt); + case 'break_statement': + return this.visitBreak(stmt); + case 'continue_statement': + return this.visitContinue(stmt); + case 'labeled_statement': + return this.visitLabeled(stmt); + case 'statement_block': + return this.visitSeq(this.statementsOf(stmt)); + default: + return this.visitSimple(stmt); + } + } + + private visitSimple(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + return { entry: idx, exits: [idx] }; + } + + private visitReturn(stmt: SyntaxNode): TraversalResult { + // Harvest the argument expression's uses — `return x` blocks live in this + // dedicated handler, not visitSeq, and were a silently-missed site once. + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + // A return crosses EVERY active finally before reaching EXIT. + wireJumpThroughFinalizers( + this.builder, + idx, + this.cfc.finalizersForReturn(), + this.builder.exitIndex, + 'return', + ); + return { entry: idx, exits: [] }; + } + + private visitThrow(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock( + startLineOf(stmt), + endLineOf(stmt), + stmt.text, + 'normal', + this.harvest.facts(stmt), + ); + this.builder.edge(idx, this.currentHandler(), 'throw'); + return { entry: idx, exits: [] }; + } + + private visitBreak(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const res = this.cfc.resolveBreak(this.labelOf(stmt)); + // An unresolved target — a label this visitor doesn't model (a stacked + // outer label like `outer: inner: for`, or a labeled non-loop block) — + // would otherwise leave this block with NO out-edge, stranding it and + // breaking the single-exit invariant a downstream post-dominator / PDG pass + // relies on. Conservatively route an unresolved jump to the function EXIT + // ("escapes the function") and thread ALL active finallys — a superset of + // the truly-crossed set (the real target is somewhere in the function, so + // execution provably runs every finally between the jump and wherever it + // lands... up to the ones the conservative EXIT routing over-includes). + // Sound for dataflow either way: extra paths, never a bypassed finally. + const { target, finalizers } = res ?? { + target: this.builder.exitIndex, + finalizers: this.cfc.finalizersForReturn(), + }; + wireJumpThroughFinalizers(this.builder, idx, finalizers, target, 'break'); + return { entry: idx, exits: [] }; + } + + private visitContinue(stmt: SyntaxNode): TraversalResult { + const idx = this.builder.newBlock(startLineOf(stmt), endLineOf(stmt), stmt.text); + const res = this.cfc.resolveContinue(this.labelOf(stmt)); + // See visitBreak: an unresolved label routes to EXIT (threading all + // active finallys) to preserve single-exit without bypassing a finally. + const { target, finalizers } = res ?? { + target: this.builder.exitIndex, + finalizers: this.cfc.finalizersForReturn(), + }; + wireJumpThroughFinalizers(this.builder, idx, finalizers, target, 'continue'); + return { entry: idx, exits: [] }; + } + + private visitLabeled(stmt: SyntaxNode): SeqResult { + const body = + stmt.childForFieldName('body') ?? stmt.namedChildren[stmt.namedChildren.length - 1]; + const label = this.labelOf(stmt); + if (body && (LOOP_OR_SWITCH_TYPES.has(body.type) || body.type === 'labeled_statement')) { + // Loop/switch consumes the accumulated labels via takeLabels(); a nested + // labeled_statement keeps accumulating (`outer: inner: for` → both + // labels land on the loop frame). + if (label) this.pendingLabels.push(label); + const res = this.visitStmt(body); + this.pendingLabels = []; // clear leftovers if the construct didn't consume + return res; + } + // Labeled NON-loop statement (`blk: { … break blk; … }`): break-to-label + // targets a synthesized join after the body. Routing it to EXIT instead + // (the M1 behavior) removed the real continuation and falsely killed + // every def live at the jump for post-construct uses (tri-review P1). + const labels = [...this.pendingLabels, ...(label ? [label] : [])]; + this.pendingLabels = []; + const join = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + this.cfc.pushLabeledBlock(join, labels); + const res = this.visitBody(body); + this.cfc.pop(); + if (res) this.builder.connect(res.exits, join, 'seq'); + return { entry: res?.entry ?? join, exits: [join] }; + } + + private visitIf(stmt: SyntaxNode): TraversalResult { + const cond = stmt.childForFieldName('condition') ?? stmt; + const condBlock = this.builder.newBlock( + startLineOf(stmt), + endLineOf(cond), + cond.text, + 'normal', + this.harvest.facts(cond), + ); + + const exits: number[] = []; + + const thenRes = this.visitBody(stmt.childForFieldName('consequence')); + if (thenRes) { + this.builder.edge(condBlock, thenRes.entry, 'cond-true'); + exits.push(...thenRes.exits); + } else { + exits.push(condBlock); // empty then — true path falls through + } + + const elseNode = this.elseBodyOf(stmt); + if (elseNode) { + const elseRes = this.visitBody(elseNode); + if (elseRes) { + this.builder.edge(condBlock, elseRes.entry, 'cond-false'); + exits.push(...elseRes.exits); + } else { + exits.push(condBlock); // empty else block + } + } else { + exits.push(condBlock); // no else — false path falls through to the join + } + + return { entry: condBlock, exits: [...new Set(exits)] }; + } + + /** The else body node (unwraps an `else_clause` wrapper if present). */ + private elseBodyOf(ifStmt: SyntaxNode): SyntaxNode | undefined { + const alt = ifStmt.childForFieldName('alternative'); + if (!alt) return undefined; + if (alt.type === 'else_clause') { + return alt.childForFieldName('body') ?? alt.namedChildren[0]; + } + return alt; + } + + private visitWhile(stmt: SyntaxNode): TraversalResult { + const labels = this.takeLabels(); + const cond = stmt.childForFieldName('condition') ?? stmt; + const header = this.builder.newBlock( + startLineOf(stmt), + endLineOf(cond), + cond.text, + 'normal', + this.harvest.facts(cond), + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, labels); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); // empty body re-tests + } + this.builder.edge(header, loopExit, 'cond-false'); + return { entry: header, exits: [loopExit] }; + } + + private visitDoWhile(stmt: SyntaxNode): TraversalResult { + const labels = this.takeLabels(); + const cond = stmt.childForFieldName('condition') ?? stmt; + const condBlock = this.builder.newBlock( + startLineOf(cond), + endLineOf(cond), + cond.text, + 'normal', + this.harvest.facts(cond), + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(condBlock, loopExit, labels); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + const backTarget = body ? body.entry : condBlock; + if (body) this.builder.connect(body.exits, condBlock, 'seq'); + this.builder.edge(condBlock, backTarget, 'loop-back'); // cond true → run body again + this.builder.edge(condBlock, loopExit, 'cond-false'); + return { entry: backTarget, exits: [loopExit] }; + } + + private visitFor(stmt: SyntaxNode): TraversalResult { + const labels = this.takeLabels(); + const init = stmt.childForFieldName('initializer'); + const cond = stmt.childForFieldName('condition'); + const incr = stmt.childForFieldName('increment'); + + const header = this.builder.newBlock( + startLineOf(stmt), + cond ? endLineOf(cond) : startLineOf(stmt), + cond ? cond.text : 'for(;;)', + 'normal', + cond ? this.harvest.facts(cond) : undefined, + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + let incrBlock = header; + if (incr) { + incrBlock = this.builder.newBlock( + startLineOf(incr), + endLineOf(incr), + incr.text, + 'normal', + this.harvest.facts(incr), + ); + this.builder.edge(incrBlock, header, 'loop-back'); + } + + this.cfc.pushLoop(incrBlock, loopExit, labels); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + // With no increment clause the body's exits ARE the back-edge — carry + // the loop-back kind on them (mirroring visitWhile/visitForIn) instead + // of a phantom header→header self-loop that models a path which never + // executes the body. With an increment, the body falls through to the + // increment (`seq`) and the increment carries the loop-back (:338). + this.builder.connect(body.exits, incrBlock, incr ? 'seq' : 'loop-back'); + } else { + this.builder.edge(header, incrBlock, 'cond-true'); + // Empty body with no increment: the header genuinely re-tests itself. + if (!incr) this.builder.edge(header, header, 'loop-back'); + } + this.builder.edge(header, loopExit, 'cond-false'); + + let entry = header; + if (init) { + const initBlock = this.builder.newBlock( + startLineOf(init), + endLineOf(init), + init.text, + 'normal', + this.harvest.facts(init), + ); + this.builder.edge(initBlock, header, 'seq'); + entry = initBlock; + } + return { entry, exits: [loopExit] }; + } + + private visitForIn(stmt: SyntaxNode): TraversalResult { + const labels = this.takeLabels(); + // Header text is SYNTHESIZED, so facts come from the left/right AST nodes + // directly (the loop variable is a def, the iterated expression a use). + const header = this.builder.newBlock( + startLineOf(stmt), + startLineOf(stmt), + this.forInHeaderText(stmt), + 'normal', + this.harvest.forInHeadFacts(stmt), + ); + const loopExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushLoop(header, loopExit, labels); + const body = this.visitBody(this.bodyBlockOf(stmt)); + this.cfc.pop(); + + if (body) { + this.builder.edge(header, body.entry, 'cond-true'); + this.builder.connect(body.exits, header, 'loop-back'); + } else { + this.builder.edge(header, header, 'loop-back'); + } + this.builder.edge(header, loopExit, 'cond-false'); + return { entry: header, exits: [loopExit] }; + } + + private forInHeaderText(stmt: SyntaxNode): string { + const left = stmt.childForFieldName('left')?.text ?? ''; + const right = stmt.childForFieldName('right')?.text ?? ''; + return left || right ? `for(${left} … ${right})` : 'for(… in/of …)'; + } + + private visitSwitch(stmt: SyntaxNode): TraversalResult { + const labels = this.takeLabels(); + const value = stmt.childForFieldName('value') ?? stmt; + const dispatch = this.builder.newBlock( + startLineOf(stmt), + endLineOf(value), + value.text, + 'normal', + this.harvest.facts(value), + ); + const switchExit = this.builder.newBlock(endLineOf(stmt), endLineOf(stmt), ''); + + this.cfc.pushSwitch(switchExit, labels); + const body = stmt.childForFieldName('body'); + const cases = body + ? body.namedChildren.filter((c) => c.type === 'switch_case' || c.type === 'switch_default') + : []; + + // `case x:` test expressions live in no block (caseStatements filters the + // value node out) — harvest their uses onto the dispatch block, one record + // per case in source order (a sound over-approximation of JS's in-order + // case evaluation). Conditionally: a later case test only evaluates when + // earlier cases didn't match, so any def inside one is a may-def — as a + // must-def on the always-executed dispatch block it would falsely kill + // prior defs for earlier-matching arms (tri-review). + for (const c of cases) { + const caseValue = c.childForFieldName('value'); + if (caseValue) this.builder.attachFacts(dispatch, this.harvest.factsConditional(caseValue)); + } + + const caseResults = cases.map((c) => this.visitSeq(this.caseStatements(c))); + const hasDefault = cases.some((c) => c.type === 'switch_default'); + + // entryOf[i] = block a dispatch/fallthrough INTO case i lands on (empty + // cases are transparent — they resolve to the next case, or the exit). + const entryOf: number[] = new Array(cases.length); + let after = switchExit; + for (let i = cases.length - 1; i >= 0; i--) { + entryOf[i] = caseResults[i]?.entry ?? after; + after = entryOf[i]; + } + + for (let i = 0; i < cases.length; i++) { + this.builder.edge(dispatch, entryOf[i], 'switch-case'); + } + if (!hasDefault) this.builder.edge(dispatch, switchExit, 'switch-case'); // no-match path + + for (let i = 0; i < cases.length; i++) { + const res = caseResults[i]; + if (!res) continue; + const fallTarget = i + 1 < cases.length ? entryOf[i + 1] : switchExit; + this.builder.connect(res.exits, fallTarget, 'fallthrough'); + } + + this.cfc.pop(); + return { entry: dispatch, exits: [switchExit] }; + } + + private caseStatements(caseNode: SyntaxNode): SyntaxNode[] { + const value = caseNode.childForFieldName('value'); + return caseNode.namedChildren.filter((c) => c.id !== value?.id && c.type !== 'comment'); + } + + private visitTry(stmt: SyntaxNode): SeqResult { + const bodyNode = stmt.childForFieldName('body'); + // Single pass over named children — tree-sitter's `namedChildren` getter + // allocates a fresh array on every access, so avoid the double `.find`. + let catchClause: SyntaxNode | undefined; + let finallyClause: SyntaxNode | undefined; + for (let i = 0; i < stmt.namedChildCount; i++) { + const c = stmt.namedChild(i); + if (c?.type === 'catch_clause') catchClause = c; + else if (c?.type === 'finally_clause') finallyClause = c; + } + + // Build finally first so its entry is known as both a normal join and a + // handler target. The finally body runs in the OUTER handler context — and + // OUTSIDE this try's finalizer frame: a return inside the finally must not + // thread itself (it threads only outer finallys, matching JS semantics). + const finallyRes = finallyClause + ? this.visitSeq(this.statementsOf(this.bodyBlockOf(finallyClause) as SyntaxNode)) + : null; + + // Finalizer frame for early-exit threading (#2082 M2 U2): active while the + // catch and protected bodies are walked, so a crossing `return`/`break`/ + // `continue` inside either routes through the finally. An empty/comment-only + // finally (`finallyRes` null — the #2099-F2 empty-catch bug shape) pushes + // NO frame: it can define nothing, so jumps soundly keep direct edges. + const finFrame = finallyRes ? this.cfc.pushFinalizer(finallyRes.entry) : null; + + // A throw inside catch propagates to finally (if any), else the outer handler. + let catchRes: SeqResult = null; + if (catchClause) { + if (finallyRes) this.handlers.push(finallyRes.entry); + catchRes = this.visitSeq(this.statementsOf(this.bodyBlockOf(catchClause) as SyntaxNode)); + if (finallyRes) this.handlers.pop(); + if (catchRes === null) { + // Empty (or comment-only) catch body — `catch {}`. The clause still + // CATCHES: handler semantics key off the syntactic clause, not the + // traversal result. Treating it as "no catch" sent the swallowed + // exception to the outer handler/EXIT and left post-try code + // unreachable when the body always throws — a hard false-negative + // for downstream taint. Synthesize one empty block spanning the + // clause (entry == sole exit) so exception flow lands in it and + // rejoins the normal continuation. Created BEFORE the protected + // region is walked, so it never receives a spurious throw edge. + const idx = this.builder.newBlock(startLineOf(catchClause), endLineOf(catchClause), ''); + catchRes = { entry: idx, exits: [idx] }; + } + // `catch (e)` has no header block — the param def gets its OWN + // facts-only block in front of the body entry. It must NOT be prepended + // into the body's entry block: when the catch body STARTS with a loop, + // that entry is the loop HEADER, re-entered on every iteration — the + // param def would re-gen there and falsely KILL loop-carried + // redefinitions of the param (`catch (e) { while (c) { e = fix(e); } + // sink(e); }` would lose the fix→sink fact, a taint false negative). + // The param block becomes the handler entry, which is also semantically + // right: the binding happens exactly once, on handler entry. + const paramFacts = this.harvest.catchParamFacts(catchClause); + if (paramFacts) { + const paramBlock = this.builder.newBlock( + startLineOf(catchClause), + startLineOf(catchClause), + '', + 'normal', + paramFacts, + ); + this.builder.edge(paramBlock, catchRes.entry, 'seq'); + catchRes = { entry: paramBlock, exits: catchRes.exits }; + } + } + + // Handler for the try body: catch if present, else finally, else outer. + const tryHandler = catchRes?.entry ?? finallyRes?.entry ?? this.currentHandler(); + const protectedStart = this.builder.blockCount; + this.handlers.push(tryHandler); + const bodyRes = bodyNode ? this.visitSeq(this.statementsOf(bodyNode)) : null; + this.handlers.pop(); + + // Conservative exceptional edges: ANY block in the protected region may raise + // to the handler — not just an explicit `throw`, and not just the body ENTRY. + // Edging every block created during the try-body walk keeps exception flow + // sound when the body BRANCHES: an `if` / nested-try / post-branch block whose + // interior blocks would otherwise have no path to the handler — i.e. a taint + // false-negative into `catch` for the downstream PDG analysis. The + // per-function edge cap bounds the count; explicit `throw`s add their own + // (idempotent) edge to the same handler. + if (catchClause || finallyClause) { + for (let b = protectedStart; b < this.builder.blockCount; b++) { + this.builder.edge(b, tryHandler, 'throw'); + } + } + + // The finalizer frame closes once the protected/catch walks are done; any + // jumps that crossed it left their completion legs on `pending`, wired + // here from the finally's exits (see drainFinalizerPending for the + // finally-override semantics of an always-jumping finally). + if (finFrame && finallyRes) { + this.cfc.pop(); + drainFinalizerPending(this.builder, finFrame, finallyRes.exits); + } + + const exits: number[] = []; + if (finallyRes) { + // Normal completion of try AND catch both flow through finally. + if (bodyRes) this.builder.connect(bodyRes.exits, finallyRes.entry, 'seq'); + if (catchRes) this.builder.connect(catchRes.exits, finallyRes.entry, 'seq'); + exits.push(...finallyRes.exits); + // No catch → an exception re-propagates out after finally runs. + if (!catchRes) this.builder.connect(finallyRes.exits, this.currentHandler(), 'throw'); + } else { + if (bodyRes) exits.push(...bodyRes.exits); + if (catchRes) exits.push(...catchRes.exits); + } + + const entry = bodyRes?.entry ?? finallyRes?.entry ?? catchRes?.entry; + if (entry === undefined) return null; + return { entry, exits: [...new Set(exits)] }; + } + + /** Nearest enclosing exception handler, or the function EXIT. */ + private currentHandler(): number { + return this.handlers.length ? this.handlers[this.handlers.length - 1] : this.builder.exitIndex; + } + + /** Consume the labels awaiting the loop/switch this call is building. */ + private takeLabels(): string[] { + const labels = this.pendingLabels; + this.pendingLabels = []; + return labels; + } + + private labelOf(stmt: SyntaxNode): string | undefined { + const id = + stmt.childForFieldName('label') ?? + stmt.namedChildren.find((c) => c.type === 'statement_identifier'); + return id?.text; + } +} + +/** Build the CFG for one TS/JS function node (or `undefined` if not a function). */ +function buildFunctionCfg(fnNode: SyntaxNode, filePath: string): FunctionCfg | undefined { + if (!TS_FUNCTION_TYPES.has(fnNode.type)) return undefined; + const startLine = startLineOf(fnNode); + const endLine = endLineOf(fnNode); + const startColumn = fnNode.startPosition.column; + const builder = new CfgBuilder(filePath, startLine, endLine, startColumn); + + const body = fnNode.childForFieldName('body'); + if (!body) return undefined; // overload signature / abstract method — no body + + // Phase-1 declaration pre-scan (#2082 M2 U1) — must complete before any + // facts are extracted; the CFG walk below is not source-order. + const harvest = new TsHarvester(fnNode); + + // Parameters define at ENTRY (facts only — never touch the entry block's + // text or span: bench fingerprints and CFG snapshots include block text). + const paramFacts = harvest.paramFacts(); + if (paramFacts) builder.attachFacts(builder.entryIndex, paramFacts); + + if (body.type !== 'statement_block') { + // Expression-bodied arrow: `() => expr` — one block whose value is returned. + // Lives outside the walk class, so it harvests explicitly. + const blk = builder.newBlock( + startLineOf(body), + endLineOf(body), + body.text, + 'normal', + harvest.facts(body), + ); + builder.edge(builder.entryIndex, blk, 'seq'); + builder.edge(blk, builder.exitIndex, 'return'); + return builder.finish(harvest.table()); + } + + const walk = new TsCfgWalk(builder, harvest); + const res = walk.visitSeq(body.namedChildren.filter((c) => c.type !== 'comment')); + if (!res) { + builder.edge(builder.entryIndex, builder.exitIndex, 'seq'); // empty body + return builder.finish(harvest.table()); + } + builder.edge(builder.entryIndex, res.entry, 'seq'); + builder.connect(res.exits, builder.exitIndex, 'seq'); // normal fall-off → EXIT + return builder.finish(harvest.table()); +} + +/** Whether a node is a TS/JS function this visitor builds a CFG for. */ +function isFunction(node: SyntaxNode): boolean { + return TS_FUNCTION_TYPES.has(node.type); +} + +/** The TS/JS CFG visitor (shared by TypeScript and JavaScript). */ +export function createTypeScriptCfgVisitor(): CfgVisitor { + return { buildFunctionCfg, isFunction }; +} + +export { TS_FUNCTION_TYPES }; diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index b375050fa..dd531d1f5 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -35,7 +35,10 @@ import type { MethodExtractor } from './method-types.js'; import type { VariableExtractor } from './variable-types.js'; import type { ImportResolverFn } from './import-resolvers/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; +import type { CfgVisitor } from './cfg/types.js'; import type { NodeLabel } from 'gitnexus-shared'; +import type Parser from 'tree-sitter'; +import type { ExtractedDecoratorRoute } from './workers/parse-worker.js'; // ── Shared type aliases ──────────────────────────────────────────────────── /** Tree-sitter query captures: capture name → AST node (or undefined if not captured). */ @@ -185,6 +188,12 @@ interface LanguageProviderConfig { * `undefined` when no constraints exist / the node isn't a templated * function. Languages without SFINAE / concept semantics leave this * undefined and the disambiguation is a pass-through. + * + * Cloneability contract: the returned payload crosses the worker boundary + * via structured clone, so it MUST be structured-clone-safe (no functions, + * symbols, or tree-sitter `SyntaxNode`s — only plain data). Wrap the return + * with `assertCloneable` from `workers/clone-safety.ts` so a future leak is a + * compile error at the source instead of a runtime DataCloneError (#2143). */ readonly extractTemplateConstraints?: (definitionNode: SyntaxNode) => unknown; @@ -236,6 +245,22 @@ interface LanguageProviderConfig { * Default: undefined (no route files). */ readonly isRouteFile?: (filePath: string) => boolean; + /** + * Extract decorator-style route annotations from a parsed file. + * + * When defined, the parse worker calls this after per-file capture processing + * to extract framework route definitions that require AST-level analysis beyond + * generic `@decorator` captures (e.g., Java Spring class-level prefix joining, + * multi-class handling). The returned routes are appended to `decoratorRoutes`. + * + * Default: undefined (no language-specific decorator route extraction). + */ + readonly extractDecoratorRoutes?: ( + tree: Parser.Tree, + filePath: string, + lineOffset: number, + ) => ExtractedDecoratorRoute[]; + // ── Noise filtering ──────────────────────────────────────────────── /** Built-in/stdlib names that should be filtered from the call graph for this language. * Default: undefined (no language-specific filtering). */ @@ -325,13 +350,28 @@ interface LanguageProviderConfig { * disk store WITHOUT a main-thread re-parse. The main thread restores them * via the matching `ScopeResolver.applyCaptureSideChannel` hook. * - * MUST return plain data (objects / arrays / primitives) so it round-trips - * through `JSON.stringify` + the parsedfile-store interning reviver. + * Cloneability contract: MUST return plain data (objects / arrays / + * primitives — no functions, symbols, or tree-sitter `SyntaxNode`s) so it + * survives BOTH the worker→main structured clone AND `JSON.stringify` + the + * parsedfile-store interning reviver. Wrap the return with `assertCloneable` + * from `workers/clone-safety.ts` so a future non-serializable leak is a + * compile error at the source instead of a runtime DataCloneError (#2143). * * Default: undefined (provider has no capture-time module-level side effects). */ readonly collectCaptureSideChannel?: (filePath: string) => unknown; + /** + * Per-language control-flow-graph builder (#2081 M1, PDG/taint substrate). + * Invoked IN THE PARSE WORKER (where the AST lives) for each function node, + * gated on the `--pdg` opt-in; the resulting per-function CFGs are serialized + * onto `ParsedFile.cfgSideChannel` and emitted as BasicBlock nodes + CFG + * edges during scope-resolution. `TNode` is `SyntaxNode` for the tree-sitter + * languages. Default: undefined (language has no CFG support yet — TS/JS are + * the M1 set). + */ + readonly cfgVisitor?: CfgVisitor; + /** * Interpret a raw `@import.statement` capture group into a `ParsedImport`. * The central finalize algorithm resolves `ParsedImport.targetRaw` to a diff --git a/gitnexus/src/core/ingestion/languages/c-cpp.ts b/gitnexus/src/core/ingestion/languages/c-cpp.ts index 84523fc70..3d427fed8 100644 --- a/gitnexus/src/core/ingestion/languages/c-cpp.ts +++ b/gitnexus/src/core/ingestion/languages/c-cpp.ts @@ -65,7 +65,11 @@ import { cppReceiverBinding, collectCppCaptureSideChannel, } from './cpp/index.js'; -import { extractCppTemplateConstraints } from './cpp/constraint-extractor.js'; +import { + extractCppTemplateConstraints, + type CppConstraintPayload, +} from './cpp/constraint-extractor.js'; +import { assertCloneable } from '../workers/clone-safety.js'; const C_BUILT_INS: ReadonlySet = new Set([ 'printf', @@ -405,7 +409,11 @@ export const cProvider = defineLanguage({ // `static` functions look non-file-local on the main thread and leak into // cross-file global free-call resolution / wildcard imports. See // `c/capture-side-channel.ts`. - collectCaptureSideChannel: collectCStaticLinkageSideChannel, + // `assertCloneable` is a runtime identity; it makes a future non-serializable + // value in the side-channel payload a compile error here, at the source, rather + // than a DataCloneError at the worker boundary (#2143). + collectCaptureSideChannel: (filePath) => + assertCloneable(collectCStaticLinkageSideChannel(filePath)), interpretImport: interpretCImport, interpretTypeBinding: interpretCTypeBinding, bindingScopeFor: cBindingScopeFor, @@ -480,7 +488,7 @@ export const cppProvider = defineLanguage({ // just populated for this file into plain data on `ParsedFile.captureSideChannel`, // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a // re-parse (#1983). See `cpp/capture-side-channel.ts`. - collectCaptureSideChannel: collectCppCaptureSideChannel, + collectCaptureSideChannel: (filePath) => assertCloneable(collectCppCaptureSideChannel(filePath)), interpretImport: interpretCppImport, interpretTypeBinding: interpretCppTypeBinding, bindingScopeFor: cppBindingScopeFor, @@ -501,7 +509,9 @@ export const cppProvider = defineLanguage({ * functions whose constraints the extractor can't model — both cases * result in no constraint suffix on the node ID. */ -function extractCppTemplateConstraintsForProvider(definitionNode: SyntaxNode): unknown { +function extractCppTemplateConstraintsForProvider( + definitionNode: SyntaxNode, +): CppConstraintPayload | undefined { // Walk up to the enclosing template_declaration. Bound the walk so we // can't accidentally land on a far-ancestor template_declaration that // wraps an unrelated function. @@ -530,5 +540,8 @@ function extractCppTemplateConstraintsForProvider(definitionNode: SyntaxNode): u } break; } - return extractCppTemplateConstraints(templateDecl, declarator); + // Guard the boundary at the source: a future non-cloneable member of the + // constraint payload becomes a compile error here, not a runtime + // DataCloneError at the worker post (#2143). + return assertCloneable(extractCppTemplateConstraints(templateDecl, declarator)); } diff --git a/gitnexus/src/core/ingestion/languages/c/query.ts b/gitnexus/src/core/ingestion/languages/c/query.ts index 373e1e7a7..065603b1a 100644 --- a/gitnexus/src/core/ingestion/languages/c/query.ts +++ b/gitnexus/src/core/ingestion/languages/c/query.ts @@ -1,5 +1,15 @@ import Parser from 'tree-sitter'; -import C from 'tree-sitter-c'; +import { SupportedLanguages } from 'gitnexus-shared'; +// `tree-sitter-c` is vendored prebuild-only (#2116) and may be absent on a +// toolchain-less / `--ignore-scripts` install. It is loaded lazily + guarded via +// parser-loader rather than statically imported: this module is pulled onto the +// main thread eagerly by the scope-resolution registry and the language-provider +// index, so a top-level `import C from 'tree-sitter-c'` would throw +// ERR_MODULE_NOT_FOUND at module-load and crash `analyze` even for repos with no +// C files (#2091, #2093). The grammar is only ever needed inside the lazy getters +// below, and the main-thread `isLanguageAvailable` filter ensures they are +// reached only when the binding is present. +import { getLanguageGrammar } from '../../../tree-sitter/parser-loader.js'; const C_SCOPE_QUERY = ` ;; Scopes @@ -167,14 +177,19 @@ let _query: Parser.Query | null = null; export function getCParser(): Parser { if (_parser === null) { _parser = new Parser(); - _parser.setLanguage(C as Parameters[0]); + _parser.setLanguage( + getLanguageGrammar(SupportedLanguages.C) as Parameters[0], + ); } return _parser; } export function getCScopeQuery(): Parser.Query { if (_query === null) { - _query = new Parser.Query(C as Parameters[0], C_SCOPE_QUERY); + _query = new Parser.Query( + getLanguageGrammar(SupportedLanguages.C) as Parameters[0], + C_SCOPE_QUERY, + ); } return _query; } diff --git a/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts b/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts index e2afd51a5..6bd08710c 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/arity-metadata.ts @@ -222,8 +222,14 @@ function findFuncDeclarator(node: SyntaxNode): SyntaxNode | null { } return null; } - // Unwrap pointer_declarator / reference_declarator - while (decl.type === 'pointer_declarator' || decl.type === 'reference_declarator') { + // Unwrap declarator wrappers. Deleted free functions are represented as + // `init_declarator(function_declarator, delete_expression)` by + // tree-sitter-cpp 0.23. + while ( + decl.type === 'pointer_declarator' || + decl.type === 'reference_declarator' || + decl.type === 'init_declarator' + ) { const next = decl.childForFieldName('declarator'); if (next === null) { // reference_declarator may not use field name diff --git a/gitnexus/src/core/ingestion/languages/cpp/captures.ts b/gitnexus/src/core/ingestion/languages/cpp/captures.ts index 883f571c5..ef39076f1 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/captures.ts @@ -163,6 +163,13 @@ export function emitCppScopeCaptures( 'true', ); } + if (hasDeletedMethodClause(fnNode, grouped['@declaration.name']?.text)) { + grouped['@declaration.is-deleted'] = syntheticCapture( + '@declaration.is-deleted', + fnNode, + 'true', + ); + } // Detect static storage class (file-local linkage) if (hasStaticStorageClass(fnNode)) { @@ -1685,7 +1692,13 @@ function extractDeclaratorLeafName(node: SyntaxNode): string | null { let cur: SyntaxNode = node; let safety = 16; while (safety-- > 0) { - if (cur.type === 'identifier' || cur.type === 'type_identifier') return cur.text; + if ( + cur.type === 'identifier' || + cur.type === 'type_identifier' || + cur.type === 'operator_name' + ) { + return cur.text; + } // Common wrapper nodes — follow the 'declarator' field when present. const next = cur.childForFieldName('declarator') ?? @@ -1713,6 +1726,25 @@ function hasExplicitSpecifier(node: SyntaxNode): boolean { return /\bexplicit\b/.test(node.text.slice(0, 128)); } +function hasDeletedMethodClause(node: SyntaxNode, callableName: string | undefined): boolean { + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (child?.type === 'delete_method_clause') return true; + // tree-sitter-cpp 0.23 parses a deleted free-function declaration as + // `declaration > init_declarator > delete_expression`, while class + // members use the dedicated `delete_method_clause`. + if ( + child?.type === 'init_declarator' && + child.childForFieldName('value')?.type === 'delete_expression' && + callableName !== undefined && + extractDeclaratorLeafName(child.childForFieldName('declarator') ?? child) === callableName + ) { + return true; + } + } + return false; +} + /** * Check if a C++ function_definition or declaration has `static` storage class. */ diff --git a/gitnexus/src/core/ingestion/languages/cpp/query.ts b/gitnexus/src/core/ingestion/languages/cpp/query.ts index aa3202abe..b50463a7f 100644 --- a/gitnexus/src/core/ingestion/languages/cpp/query.ts +++ b/gitnexus/src/core/ingestion/languages/cpp/query.ts @@ -194,6 +194,29 @@ const CPP_SCOPE_QUERY = ` declarator: (function_declarator declarator: (identifier) @declaration.name)) @declaration.function +;; tree-sitter-cpp 0.23 represents a deleted free function as an +;; init_declarator whose value is a delete_expression. +(declaration + declarator: (init_declarator + declarator: (function_declarator + declarator: (identifier) @declaration.name) + value: (delete_expression))) @declaration.function + +;; Deleted free operator declaration. +(declaration + declarator: (init_declarator + declarator: (function_declarator + declarator: (operator_name) @declaration.name) + value: (delete_expression))) @declaration.function + +;; Deleted free function with a pointer return type. +(declaration + declarator: (init_declarator + declarator: (pointer_declarator + declarator: (function_declarator + declarator: (identifier) @declaration.name)) + value: (delete_expression))) @declaration.function + ;; Free operator prototype: std::ostream& operator<<(std::ostream&, T) (declaration declarator: (function_declarator diff --git a/gitnexus/src/core/ingestion/languages/java.ts b/gitnexus/src/core/ingestion/languages/java.ts index b44464ddc..f7ca23de3 100644 --- a/gitnexus/src/core/ingestion/languages/java.ts +++ b/gitnexus/src/core/ingestion/languages/java.ts @@ -13,6 +13,7 @@ import { javaClassConfig } from '../class-extractors/configs/jvm.js'; import { defineLanguage } from '../language-provider.js'; import type { AstFrameworkPatternConfig } from '../language-provider.js'; import { javaTypeConfig } from '../type-extractors/jvm.js'; +import { extractSpringRoutes } from '../route-extractors/spring.js'; import { javaExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; import { javaImportConfig } from '../import-resolvers/configs/jvm.js'; @@ -126,4 +127,7 @@ export const javaProvider = defineLanguage({ arityCompatibility: javaArityCompatibility, resolveImportTarget: resolveJavaImportTarget, orderSameNameTypeCandidates: orderJavaSameNameTypeCandidates, + + // ── Route extraction ── + extractDecoratorRoutes: extractSpringRoutes, }); diff --git a/gitnexus/src/core/ingestion/languages/kotlin.ts b/gitnexus/src/core/ingestion/languages/kotlin.ts index 6ee7b1dda..4cf87a76f 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin.ts @@ -11,6 +11,7 @@ import { SupportedLanguages } from 'gitnexus-shared'; import { createClassExtractor } from '../class-extractors/generic.js'; import { kotlinClassConfig } from '../class-extractors/configs/jvm.js'; import { defineLanguage } from '../language-provider.js'; +import { assertCloneable } from '../workers/clone-safety.js'; import { kotlinTypeConfig } from '../type-extractors/jvm.js'; import { kotlinExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; @@ -182,7 +183,11 @@ export const kotlinProvider = defineLanguage({ // so the main thread can restore them via `applyCaptureSideChannel` WITHOUT a // re-parse (#1983). Without this, companion/static dispatch emits no CALLS // edges on the worker path. See `kotlin/capture-side-channel.ts`. - collectCaptureSideChannel: collectKotlinCaptureSideChannel, + // `assertCloneable` is a runtime identity; it makes a future non-serializable + // value in the side-channel payload a compile error here, at the source, rather + // than a DataCloneError at the worker boundary (#2143). + collectCaptureSideChannel: (filePath) => + assertCloneable(collectKotlinCaptureSideChannel(filePath)), interpretImport: interpretKotlinImport, interpretTypeBinding: interpretKotlinTypeBinding, bindingScopeFor: kotlinBindingScopeFor, diff --git a/gitnexus/src/core/ingestion/languages/kotlin/query.ts b/gitnexus/src/core/ingestion/languages/kotlin/query.ts index f3a749086..7015e6e28 100644 --- a/gitnexus/src/core/ingestion/languages/kotlin/query.ts +++ b/gitnexus/src/core/ingestion/languages/kotlin/query.ts @@ -1,7 +1,8 @@ import Parser from 'tree-sitter'; import { SupportedLanguages } from 'gitnexus-shared'; -// `tree-sitter-kotlin` is an optionalDependency that may be absent on a default -// install (or fail its native build). Loaded lazily + guarded via parser-loader +// `tree-sitter-kotlin` is a vendored grammar (loaded from vendor/ by absolute +// path, never node_modules — vendored-grammars.ts / #2111) that may be absent on +// a platform without a matching prebuild. Loaded lazily + guarded via parser-loader // rather than statically imported: this module is pulled onto the main thread // eagerly by the scope-resolution registry and the language-provider index, so // a top-level `import Kotlin from 'tree-sitter-kotlin'` would throw diff --git a/gitnexus/src/core/ingestion/languages/typescript.ts b/gitnexus/src/core/ingestion/languages/typescript.ts index 73b33dfd8..7f41fe731 100644 --- a/gitnexus/src/core/ingestion/languages/typescript.ts +++ b/gitnexus/src/core/ingestion/languages/typescript.ts @@ -16,6 +16,7 @@ import { javascriptClassConfig, } from '../class-extractors/configs/typescript-javascript.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { createTypeScriptCfgVisitor } from '../cfg/visitors/typescript.js'; import { typeConfig as typescriptConfig } from '../type-extractors/typescript.js'; import { tsExportChecker } from '../export-detection.js'; import { createImportResolver } from '../import-resolvers/resolver-factory.js'; @@ -351,6 +352,8 @@ export const typescriptProvider = defineLanguage({ // canonical capture vocabulary in ./typescript/query.ts // (TYPESCRIPT_SCOPE_QUERY constant). emitScopeCaptures: emitTsScopeCaptures, + // CFG/PDG substrate (#2081 M1) — runs in the worker on a --pdg run. + cfgVisitor: createTypeScriptCfgVisitor(), interpretImport: interpretTsImport, interpretTypeBinding: interpretTsTypeBinding, bindingScopeFor: tsBindingScopeFor, @@ -412,6 +415,8 @@ export const javascriptProvider = defineLanguage({ // JSDoc type bindings) live in ./javascript/captures.ts. // See ./javascript/index.ts for the full per-module rationale. emitScopeCaptures: emitJsScopeCaptures, + // CFG/PDG substrate (#2081 M1) — TS and JS share the same grammar family. + cfgVisitor: createTypeScriptCfgVisitor(), interpretImport: interpretJsImport, interpretTypeBinding: interpretJsTypeBinding, bindingScopeFor: jsBindingScopeFor, diff --git a/gitnexus/src/core/ingestion/method-extractors/configs/c-cpp.ts b/gitnexus/src/core/ingestion/method-extractors/configs/c-cpp.ts index dccc3f52f..1ca737793 100644 --- a/gitnexus/src/core/ingestion/method-extractors/configs/c-cpp.ts +++ b/gitnexus/src/core/ingestion/method-extractors/configs/c-cpp.ts @@ -42,18 +42,11 @@ function findFunctionDeclarator(node: SyntaxNode): SyntaxNode | null { return null; } -/** - * Detect `= delete` and `= default` special member function declarations. - * These are not callable methods and should be suppressed from extraction. - * tree-sitter-cpp ^0.23.4 emits `delete_method_clause` / `default_method_clause` - * as named children of the function_definition node. - */ -function isDeletedOrDefaulted(node: SyntaxNode): boolean { +/** Detect a C++ special member clause by its tree-sitter node type. */ +function hasSpecialMethodClause(node: SyntaxNode, clauseType: string): boolean { for (let i = 0; i < node.namedChildCount; i++) { const child = node.namedChild(i); - if (child?.type === 'delete_method_clause' || child?.type === 'default_method_clause') { - return true; - } + if (child?.type === clauseType) return true; } return false; } @@ -67,10 +60,6 @@ function extractCppMethodName(node: SyntaxNode): string | undefined { const funcDecl = findFunctionDeclarator(node); if (!funcDecl) return undefined; - // Suppress `= delete` and `= default` special members — these are not callable - // methods and should not appear in HAS_METHOD edges. - if (isDeletedOrDefaulted(node)) return undefined; - const nameNode = funcDecl.childForFieldName('declarator'); if (!nameNode) return undefined; // destructor_name: ~ClassName @@ -387,6 +376,10 @@ export const cppMethodConfig: MethodExtractionConfig = { } return false; }, + + isDeleted(node) { + return hasSpecialMethodClause(node, 'delete_method_clause'); + }, }; // --------------------------------------------------------------------------- diff --git a/gitnexus/src/core/ingestion/method-extractors/generic.ts b/gitnexus/src/core/ingestion/method-extractors/generic.ts index f02faef35..ecb25d939 100644 --- a/gitnexus/src/core/ingestion/method-extractors/generic.ts +++ b/gitnexus/src/core/ingestion/method-extractors/generic.ts @@ -252,6 +252,7 @@ function buildMethod( ...(config.isAsync?.(node) ? { isAsync: true } : {}), ...(config.isPartial?.(node) ? { isPartial: true } : {}), ...(config.isConst?.(node) ? { isConst: true } : {}), + ...(config.isDeleted?.(node) ? { isDeleted: true } : {}), annotations: config.extractAnnotations?.(node) ?? [], sourceFile: context.filePath, line: node.startPosition.row + 1, diff --git a/gitnexus/src/core/ingestion/method-types.ts b/gitnexus/src/core/ingestion/method-types.ts index e1b91eca5..d263fd5db 100644 --- a/gitnexus/src/core/ingestion/method-types.ts +++ b/gitnexus/src/core/ingestion/method-types.ts @@ -33,6 +33,7 @@ export interface MethodInfo { isAsync?: boolean; isPartial?: boolean; isConst?: boolean; + isDeleted?: boolean; annotations: string[]; sourceFile: string; line: number; @@ -84,6 +85,7 @@ export interface MethodExtractionConfig { isAsync?: (node: SyntaxNode) => boolean; isPartial?: (node: SyntaxNode) => boolean; isConst?: (node: SyntaxNode) => boolean; + isDeleted?: (node: SyntaxNode) => boolean; /** Owner node types where member functions are effectively static (e.g. * Ruby singleton_class, Kotlin companion_object / object_declaration). * When the ownerNode matches one of these types, isStatic is forced true. */ diff --git a/gitnexus/src/core/ingestion/model/symbol-table.ts b/gitnexus/src/core/ingestion/model/symbol-table.ts index 04ed8ca18..d7330a093 100644 --- a/gitnexus/src/core/ingestion/model/symbol-table.ts +++ b/gitnexus/src/core/ingestion/model/symbol-table.ts @@ -131,6 +131,7 @@ export interface AddMetadata { templateArguments?: string[]; ownerId?: string; qualifiedName?: string; + isDeleted?: boolean; } /** @@ -285,6 +286,7 @@ export const createSymbolTable = (): InternalSymbolTable => { ? { templateArguments: metadata.templateArguments } : {}), ...(metadata?.ownerId !== undefined ? { ownerId: metadata.ownerId } : {}), + ...(metadata?.isDeleted === true ? { isDeleted: true } : {}), }; // A. File Index — unconditional. diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 41c2b4d32..815099674 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -7,6 +7,7 @@ import { accumulateExportedTypesFromParsedNode, type ExportedTypeMap } from './c import type { ParsedFile } from 'gitnexus-shared'; import { WorkerPool } from './workers/worker-pool.js'; +import type { SkippedPath } from './workers/clone-safety.js'; import { logger } from '../logger.js'; import type { ParseWorkerResult, @@ -103,6 +104,7 @@ export const mergeChunkResults = ( templateArguments: sym.templateArguments, ownerId: sym.ownerId, qualifiedName: sym.qualifiedName, + isDeleted: sym.isDeleted, }); } if (exportedTypeMap) { @@ -196,6 +198,29 @@ export const dispatchChunkParse = async ( logger.warn(` Skipped unsupported languages: ${summary}`); } + // Clone-safety telemetry (#2112): files whose parse output carried a value + // the structured-clone algorithm couldn't serialize across the worker + // boundary. The worker sanitized/dropped the offending value so the run + // could complete; surface the (rare) data loss so it's visible and the + // offending extractor can be fixed at source. + const skippedPaths: SkippedPath[] = []; + for (const result of chunkResults) { + for (const entry of result.skippedPaths ?? []) skippedPaths.push(entry); + } + if (skippedPaths.length > 0) { + // Keep the per-file reason ("stripped N value(s) from nodes" / + // "dropped non-serializable parsedFiles entry") — it distinguishes a + // recoverable strip from a whole-record drop, which a path-only line loses. + const shown = skippedPaths + .slice(0, 10) + .map((e) => `${e.path} (${e.reason})`) + .join(', '); + const more = skippedPaths.length > 10 ? ` …and ${skippedPaths.length - 10} more` : ''; + logger.warn( + ` Sanitized ${skippedPaths.length} file(s) with non-serializable parse output: ${shown}${more}`, + ); + } + onFileProgress?.(total, total, 'done'); return chunkResults; }; diff --git a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts index ddc8c902e..3885df16e 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/parse-impl.ts @@ -36,6 +36,7 @@ import { restoreDurableParsedFileShard, } from '../../../storage/parsedfile-store.js'; import type { ParseWorkerResult } from '../workers/parse-worker.js'; +import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js'; import type { WorkerExtractedData } from '../parsing-processor.js'; import { processRoutesFromExtracted, @@ -461,6 +462,10 @@ export async function runChunkedParseAndResolve( // Initialized below before the chunk loop (same deferred-init pattern // as `parsedFileStorePath`); this closure only runs from the loop. durableParsedFileStoragePath: durableParsedFileDir, + // CFG/PDG opt-in (#2081 M1) — baked into each worker's workerData so the + // worker builds + attaches cfgSideChannel. Off by default. + pdg: options?.pdg === true, + pdgMaxFunctionLines: options?.pdgMaxFunctionLines, // Fan each chunk across the whole pool (#worker-idle): without this a // chunk smaller than the 8 MB sub-batch cap became a single job on a // single worker. Honors an explicit `subBatchMaxBytes` / env override. @@ -737,7 +742,21 @@ export async function runChunkedParseAndResolve( filePath: f.path, contentHash: fileContentHash(f.content), })); - chunkHash = computeChunkHash(entries); + chunkHash = computeChunkHash( + entries, + // Only worker-visible pdg config participates in the key — + // pdgMaxEdgesPerFunction is emit-time-only and deliberately + // excluded (see PdgCacheKey in parse-cache.ts; #2099 F3). The line + // cap is RESOLVED to the worker's default before folding so an + // explicit-default run shares the default run's keys (the worker + // output is byte-identical either way). + options?.pdg === true + ? { + pdg: true, + maxFunctionLines: options?.pdgMaxFunctionLines ?? DEFAULT_PDG_MAX_FUNCTION_LINES, + } + : false, + ); } const cachedRaw = diff --git a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts index 8c0a67ac5..561590e05 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/routes.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/routes.ts @@ -367,25 +367,77 @@ export const routesPhase: PipelinePhase = { // scan JS/TS consumer files for calls to those wrapper functions with // URL-like string arguments and add them to allFetchCalls so // processNextjsFetchRoutes can create FETCHES edges. - if (allFetchWrapperDefs && allFetchWrapperDefs.length > 0 && routeRegistry.size > 0) { - const wrapperNames = new Set(allFetchWrapperDefs.map((d) => d.functionName)); + // Wrapper names come from two sources: functions the parse phase + // auto-detected as calling the bare global `fetch()`, plus any names the + // user declared in `.gitnexusrc` `fetchWrappers` (#1589/#1852 residual). + // Config names let an axios/custom-client wrapper — or one named outside the + // built-in convention — still produce route_map consumers; without them it + // silently falls back to `consumers: []`. Configured names alone are enough + // to run the scan even when nothing was auto-detected. + // Configured names are already validated/trimmed/de-duped/capped by + // analyze-config.ts — trusted as-is (#1589/#1852 review F9, dropped the + // redundant re-trim/re-filter). The single filter below guards only the + // auto-detected `functionName`s, which have no shape guarantee. + const configuredWrappers = ctx.options?.fetchWrappers ?? []; + const wrapperNames = new Set( + [...(allFetchWrapperDefs ?? []).map((d) => d.functionName), ...configuredWrappers].filter( + (n): n is string => typeof n === 'string' && n.trim().length > 0, + ), + ); + if (wrapperNames.size > 0 && routeRegistry.size > 0) { const jsFiles = allPaths.filter((p) => /\.[jt]sx?$/.test(p)); - if (jsFiles.length > 0 && wrapperNames.size > 0) { - const jsContents = await readFileContents(ctx.repoPath, jsFiles); - for (const [filePath, content] of jsContents) { - for (const name of wrapperNames) { - const regex = new RegExp( - `\\b${escapeRegex(name)}\\s*\\(\\s*['"\`](/[^'"\`\\s)]+)['"\`]`, - 'g', - ); - let match; - while ((match = regex.exec(content)) !== null) { - allFetchCalls.push({ - filePath, - fetchURL: match[1], - lineNumber: content.substring(0, match.index).split('\n').length, - }); + if (jsFiles.length > 0) { + // Reuse contents already read for handler extraction; only read the + // remainder (mirrors the Expo block above). Avoids a second full read of + // files we already have in memory. + const unreadJsFiles = jsFiles.filter((p) => !handlerContents?.has(p)); + const extraContents = + unreadJsFiles.length > 0 + ? await readFileContents(ctx.repoPath, unreadJsFiles) + : new Map(); + // One alternation regex over every wrapper name per file — O(files), not + // O(files × wrappers) (#1852 review F3). Names are escaped and grouped + // non-capturing so capture group 1 stays the URL. The left boundary is a + // negative lookbehind, not `\b`: a bare configured name like `get` must + // match the free call `get('/x')` but NOT a member access `client.get(` + // (a `.get(` on an unrelated object), and `apiFetch` must not match + // `myApiFetch`. Member-style wrappers are configured with the dot + // (`client.get`), where the `.` is part of the pattern. The `u` flag + + // Unicode property classes make the boundary cover non-ASCII identifier + // characters too — ASCII `\w` would let `caféget('/x')` match `get` + // (#1852 review F10). + const alternation = [...wrapperNames].map(escapeRegex).join('|'); + const wrapperCallRegex = new RegExp( + `(? { + wrapperCallRegex.lastIndex = 0; + // 1-based line number via a running newline counter: matches arrive in + // ascending index, so accumulate newlines incrementally instead of + // re-allocating `content.substring(0, match.index).split('\n')` on + // every match (#1852 review F12). Output is identical. + let line = 1; + let scanned = 0; + let match; + while ((match = wrapperCallRegex.exec(content)) !== null) { + for (; scanned < match.index; scanned++) { + if (content.charCodeAt(scanned) === 10 /* '\n' */) line++; } + allFetchCalls.push({ + filePath, + fetchURL: match[1], + lineNumber: line, + }); + } + }; + for (const [filePath, content] of extraContents) scanContent(filePath, content); + // Also scan already-read JS/TS handler files (a handler can itself + // consume another route through a wrapper). + if (handlerContents) { + for (const p of jsFiles) { + const cached = handlerContents.get(p); + if (cached !== undefined) scanContent(p, cached); } } } diff --git a/gitnexus/src/core/ingestion/pipeline.ts b/gitnexus/src/core/ingestion/pipeline.ts index 0e610e647..f1f832632 100644 --- a/gitnexus/src/core/ingestion/pipeline.ts +++ b/gitnexus/src/core/ingestion/pipeline.ts @@ -50,6 +50,38 @@ export interface PipelineOptions { * to retain those nodes under `skipGraphPhases`. */ skipGraphPhases?: boolean; + /** + * Build the control-flow-graph / PDG substrate (#2081 M1, opt-in via `--pdg`). + * Off by default: workers skip all CFG work and emit no `cfgSideChannel`, and + * scope-resolution emits no BasicBlock nodes or CFG edges — so the default + * graph is byte-identical to a pre-#2081 run. Folded into the parse-cache key + * so a pdg-off warm cache is not reused on a `--pdg` run. + */ + pdg?: boolean; + /** + * Per-function source-line cap for worker-side CFG construction. + * `undefined` ⇒ the worker applies `DEFAULT_PDG_MAX_FUNCTION_LINES`; `0` ⇒ no + * cap (unlimited). Bounds the cost of a pathological mega-function; over-cap + * functions are skipped (no CFG emitted for them). No CLI flag in M1 — + * programmatic / server analyze-worker path only. + */ + pdgMaxFunctionLines?: number; + /** + * Per-function CFG edge cap for the scope-resolution emit step. + * `undefined` ⇒ `DEFAULT_MAX_CFG_EDGES_PER_FUNCTION`; `0` ⇒ no cap (unlimited). + * Over-cap functions stop at the cap and log a structured drop warning (no + * silent truncation). No CLI flag in M1 — programmatic / server path only. + */ + pdgMaxEdgesPerFunction?: number; + /** + * Per-function REACHING_DEF edge cap for the scope-resolution emit step + * (#2082 M2). `undefined` ⇒ `DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION` + * (4000); `0` ⇒ no cap (unlimited). Emit-time-only — NOT folded into the + * parse-cache chunk key (the worker never sees it); recorded in + * `RepoMeta.pdg` so a cap change forces a full writeback. No CLI flag — + * programmatic / server path only, like the M1 caps. + */ + pdgMaxReachingDefEdgesPerFunction?: number; /** * Request parsing with the worker pool disabled. The sequential parser was * removed — the worker pool is the sole parse path — so setting this now @@ -129,6 +161,15 @@ export interface PipelineOptions { * `process.env` state across invocations. When undefined, the env var decides. */ keepLocalValueSymbols?: boolean; + /** + * Extra fetch-wrapper function names to treat as HTTP consumers, threaded + * from `.gitnexusrc` `fetchWrappers` via `AnalyzeOptions` (#1589/#1852 + * residual). The routes phase unions these with the auto-detected `fetch()` + * wrappers when scanning for `route_map` consumers, so a wrapper named outside + * the built-in convention (or built on axios / a custom client) is still + * traced. Empty/undefined leaves behavior unchanged. + */ + fetchWrappers?: readonly string[]; } // ── Phase registry ───────────────────────────────────────────────────────── diff --git a/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts b/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts new file mode 100644 index 000000000..6d3ea1e5d --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/spring-shared.ts @@ -0,0 +1,88 @@ +/** + * Shared Spring route-annotation primitives. + * + * These are the low-level building blocks the two Spring route extractors — + * the ingestion-layer `route-extractors/spring.ts` (produces graph `Route` + * nodes) and the group-layer `group/extractors/http-patterns/java.ts` + * (produces cross-repo HTTP contracts) — would otherwise each maintain + * independently. Centralising the annotation→method map, the enclosing-class + * lookup, and the route-key filter keeps those semantics in one place so the + * two extractors can't drift apart. + * + * This module lives in `ingestion/` (the lower layer); the group layer imports + * from it, matching the existing `group → ingestion` dependency direction + * (e.g. `group/extractors/include-extractor.ts` already imports + * `ingestion/import-resolvers/utils.ts`). It MUST NOT import anything from + * `group/` to avoid a dependency cycle. + */ + +import type Parser from 'tree-sitter'; + +/** + * Spring shortcut method-annotation → HTTP verb. + * + * `@RequestMapping` is intentionally absent: on a method it carries no implicit + * verb (the verb lives in its `method = RequestMethod.X` attribute), and on a + * class it is a URL prefix rather than a route. Callers handle `@RequestMapping` + * separately. + */ +export const METHOD_ANNOTATION_TO_HTTP: Record = { + GetMapping: 'GET', + PostMapping: 'POST', + PutMapping: 'PUT', + DeleteMapping: 'DELETE', + PatchMapping: 'PATCH', +}; + +/** + * A named annotation argument contributes a route only when its member key is + * `path` or `value`; a positional argument (no key node) always qualifies. + * Drops Spring's non-route string attributes (`produces`, `consumes`, + * `headers`, `name`, `params`) that would otherwise be mis-read as routes. + */ +export function isRouteMemberKey(keyNode: Parser.SyntaxNode | undefined): boolean { + if (!keyNode) return true; + return keyNode.text === 'path' || keyNode.text === 'value'; +} + +/** + * Find the nearest enclosing `class_declaration` ancestor for a node, or null + * if the node is top-level. Tree-sitter's `SyntaxNode.parent` walks one level + * at a time. + */ +export function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { + let cur: Parser.SyntaxNode | null = node.parent; + while (cur) { + if (cur.type === 'class_declaration') return cur; + cur = cur.parent; + } + return null; +} + +/** + * Strip enclosing quotes from a tree-sitter string-literal node's text. + * Handles single / double / template (backtick) quotes and triple-quoted + * strings. Mirrors the safer semantics of the group layer's `unquoteLiteral`: + * returns `null` for empty / nullish input so callers can uniformly skip + * captures whose value is missing, and returns the text unchanged when it + * carries no recognisable surrounding quotes (some grammars expose string + * content without quotes already). + */ +export function unquoteSpringLiteral(raw: string): string | null { + if (!raw) return null; + + if ( + (raw.startsWith('"""') && raw.endsWith('"""')) || + (raw.startsWith("'''") && raw.endsWith("'''")) + ) { + return raw.slice(3, -3); + } + + const first = raw[0]; + const last = raw[raw.length - 1]; + if ((first === '"' || first === "'" || first === '`') && last === first && raw.length >= 2) { + return raw.slice(1, -1); + } + + return raw; +} diff --git a/gitnexus/src/core/ingestion/route-extractors/spring.ts b/gitnexus/src/core/ingestion/route-extractors/spring.ts new file mode 100644 index 000000000..4476e55cc --- /dev/null +++ b/gitnexus/src/core/ingestion/route-extractors/spring.ts @@ -0,0 +1,154 @@ +/** + * Spring route annotation extractor for the ingestion pipeline. + * + * Extracts `@GetMapping`, `@PostMapping`, `@PutMapping`, `@DeleteMapping`, + * `@PatchMapping`, and `@RequestMapping` annotations from Java source files + * and returns `ExtractedDecoratorRoute[]` with class-level `@RequestMapping` + * prefixes already resolved per-class. + * + * This module is the ingestion-layer counterpart of + * `group/extractors/http-patterns/java.ts` (which extracts HTTP contracts + * for cross-repo matching). It uses the same tree-sitter capture approach: + * a single predicate-free query matches all route annotations generically, + * then a for-loop discriminates class-level prefixes from method-level routes + * by reading `@node.type` and the annotation name. + * + * The query is predicate-free to avoid the tree-sitter 0.21.x hazard where + * `#match?` / `#eq?` predicates in a top-level `[...]` alternation silently + * drop sibling-branch matches (see group-layer `JAVA_ROUTE_ANNOTATION_PATTERNS` + * header comment for details). + */ + +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import type { ExtractedDecoratorRoute } from '../workers/parse-worker.js'; +import { + METHOD_ANNOTATION_TO_HTTP, + isRouteMemberKey, + findEnclosingClass, + unquoteSpringLiteral, +} from './spring-shared.js'; + +/** + * Single predicate-free tree-sitter query that captures all route annotations + * on classes and methods. Discrimination by annotation name and node type + * happens in the loop below. + * + * Captures: + * @ann → annotation name identifier (RequestMapping, GetMapping, etc.) + * @node → enclosing declaration (class_declaration | method_declaration) + * @value → the string-literal argument + * @key → the named-argument member key (absent for positional form) + */ +const ROUTE_ANNOTATION_QUERY = new Parser.Query( + Java, + ` + [ + (class_declaration + (modifiers + (annotation + name: (identifier) @ann + arguments: (annotation_argument_list (string_literal) @value)))) @node + (class_declaration + (modifiers + (annotation + name: (identifier) @ann + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key + value: (string_literal) @value))))) @node + (method_declaration + (modifiers + (annotation + name: (identifier) @ann + arguments: (annotation_argument_list (string_literal) @value)))) @node + (method_declaration + (modifiers + (annotation + name: (identifier) @ann + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key + value: (string_literal) @value))))) @node + ] +`, +); + +/** + * Extract Spring route annotations from a parsed Java file. + * + * Uses a single tree-sitter query pass to capture all annotations, then + * discriminates class-level prefixes from method-level routes in a loop. + * Handles multiple classes per file, each with its own prefix. + * + * @param tree - tree-sitter parse tree + * @param filePath - relative file path (for `ExtractedDecoratorRoute.filePath`) + * @param lineOffset - line offset for pre-processing (usually 0) + * @returns Decorator routes with prefix already set per-class + */ +export function extractSpringRoutes( + tree: Parser.Tree, + filePath: string, + lineOffset = 0, +): ExtractedDecoratorRoute[] { + const matches = ROUTE_ANNOTATION_QUERY.matches(tree.rootNode); + + // Phase 1: collect class-level @RequestMapping prefixes keyed by node id + const prefixByClassId = new Map(); + + for (const match of matches) { + const caps: Record = {}; + for (const { name, node } of match.captures) { + caps[name] = node; + } + const annNode = caps['ann']; + const node = caps['node']; + const valueNode = caps['value']; + const keyNode = caps['key']; + if (!annNode || !node || !valueNode) continue; + + if (node.type === 'class_declaration' && annNode.text === 'RequestMapping') { + if (!isRouteMemberKey(keyNode)) continue; + const prefix = unquoteSpringLiteral(valueNode.text); + if (prefix !== null) prefixByClassId.set(node.id, prefix); + } + } + + // Phase 2: collect method-level routes and resolve their class prefix + const routes: ExtractedDecoratorRoute[] = []; + + for (const match of matches) { + const caps: Record = {}; + for (const { name, node } of match.captures) { + caps[name] = node; + } + const annNode = caps['ann']; + const node = caps['node']; + const valueNode = caps['value']; + const keyNode = caps['key']; + if (!annNode || !node || !valueNode) continue; + + if (node.type !== 'method_declaration') continue; + + const ann = annNode.text; + const httpMethod = METHOD_ANNOTATION_TO_HTTP[ann]; + if (!httpMethod) continue; // skip @RequestMapping on methods (ambiguous verb) + if (!isRouteMemberKey(keyNode)) continue; + + const routePath = unquoteSpringLiteral(valueNode.text); + if (routePath === null) continue; + const enclosingClass = findEnclosingClass(node); + const classPrefix = enclosingClass ? (prefixByClassId.get(enclosingClass.id) ?? '') : ''; + + routes.push({ + filePath, + routePath, + httpMethod, + decoratorName: ann, + lineNumber: annNode.startPosition.row + lineOffset, + ...(classPrefix ? { prefix: classPrefix } : {}), + }); + } + + return routes; +} diff --git a/gitnexus/src/core/ingestion/scope-extractor.ts b/gitnexus/src/core/ingestion/scope-extractor.ts index e5b9a71ac..1977767d4 100644 --- a/gitnexus/src/core/ingestion/scope-extractor.ts +++ b/gitnexus/src/core/ingestion/scope-extractor.ts @@ -575,6 +575,7 @@ function buildDefFromDeclarationMatch( const returnType = match['@declaration.return-type']?.text; const templateConstraints = parseJsonCapture(match['@declaration.template-constraints']); const isExplicit = parseBooleanCapture(match['@declaration.is-explicit']); + const isDeleted = parseBooleanCapture(match['@declaration.is-deleted']); return { nodeId: makeDefId(filePath, anchor.range, type, nameCap.text), @@ -590,6 +591,7 @@ function buildDefFromDeclarationMatch( ...(templateArguments !== undefined ? { templateArguments } : {}), ...(templateConstraints !== undefined ? { templateConstraints } : {}), ...(isExplicit === true ? { isExplicit: true } : {}), + ...(isDeleted === true ? { isDeleted: true } : {}), }; } @@ -1158,6 +1160,7 @@ const KNOWN_SUB_TAGS: ReadonlySet = new Set([ '@declaration.return-type', '@declaration.template-constraints', '@declaration.is-explicit', + '@declaration.is-deleted', ]); /** diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts index 8c79fdb41..1a8d1d0f4 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/free-call-fallback.ts @@ -383,6 +383,18 @@ export function emitFreeCallFallback( ); } if (fnDef === undefined) continue; + if (fnDef.isDeleted === true) { + recordSuppressedOutcome(options.recordResolutionOutcome, { + phase: 'free-call-fallback', + filePath: parsed.filePath, + name: site.name, + range: site.atRange, + reason: 'selected-callable-deleted', + candidates: [fnDef], + }); + handledSites.add(siteKey(parsed.filePath, site)); + continue; + } if ( (fnDefFromImplicitThis || fnDef.type === 'Method' || fnDef.type === 'Constructor') && options.isCallableVisibleFromCaller !== undefined && diff --git a/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts b/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts index 1b7429708..f1e014772 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/passes/receiver-bound-calls.ts @@ -205,8 +205,14 @@ export function emitReceiverBoundCalls( if (impls === undefined) return 0; let n = 0; for (const implDef of impls) { - const implMember = findOwnedMember(implDef.nodeId, memberName, model); - if (implMember === undefined) continue; + const implMember = pickOverload(implDef.nodeId, memberName, site, model, provider); + if ( + implMember === undefined || + implMember === OVERLOAD_AMBIGUOUS || + implMember.isDeleted === true + ) { + continue; + } if (implMember.nodeId === primaryMemberDef.nodeId) continue; const ok = tryEmitEdge( graph, @@ -257,11 +263,46 @@ export function emitReceiverBoundCalls( ? extendsOnly(enclosingClass.nodeId) : scopes.methodDispatch.mroFor(enclosingClass.nodeId); let memberDef: SymbolDefinition | undefined; + let ambiguousOwnerId: string | undefined; for (const ownerId of ancestors) { - memberDef = findOwnedMember(ownerId, memberName, model); - if (memberDef !== undefined) break; + const picked = + site.kind === 'call' + ? pickOverload(ownerId, memberName, site, model, provider) + : findOwnedMember(ownerId, memberName, model); + if (picked === OVERLOAD_AMBIGUOUS) { + ambiguousOwnerId = ownerId; + break; + } + if (picked !== undefined) { + memberDef = picked; + break; + } + } + if (ambiguousOwnerId !== undefined) { + recordReceiverOverloadSuppression( + options.recordResolutionOutcome, + parsed.filePath, + site, + ambiguousOwnerId, + memberName, + model, + provider, + ); + handledSites.add(siteKey); + continue; } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } // Super/base calls resolve through the MRO chain, not // through imports — the ancestor method is found by // walking `methodDispatch.mroFor(enclosingClass)`, which @@ -312,9 +353,9 @@ export function emitReceiverBoundCalls( if (currentClass !== undefined) { const chain = [currentClass.nodeId, ...scopes.methodDispatch.mroFor(currentClass.nodeId)]; let memberDef: SymbolDefinition | undefined; + let ambiguousOwnerId: string | undefined; // Static-only filter (#1756 / U3): same shape as Case 4's - // chain walk (skip-and-walk-on) but without overload - // narrowing — Case 0 uses `findOwnedMember` directly. When + // overload-aware chain walk (skip-and-walk-on). When // an owner's resolved candidate is static-only (Kotlin // companion-promoted), continue to the next ancestor in // the MRO chain so a legitimate instance member can bind. @@ -326,16 +367,45 @@ export function emitReceiverBoundCalls( // shapes like `Logger.create("a")`), so there's no wrong // target to suppress. for (const ownerId of chain) { - const candidate = findOwnedMember(ownerId, memberName, model); - if (candidate === undefined) continue; - if (provider.isStaticOnly?.(candidate) === true) { - // Skip static-only candidate; walk to next ancestor. + const picked = + site.kind === 'call' + ? pickFirstNonStaticOnly(ownerId, memberName, site, model, provider) + : findOwnedMember(ownerId, memberName, model); + if (picked === OVERLOAD_AMBIGUOUS) { + ambiguousOwnerId = ownerId; + break; + } + if (picked === STATIC_ONLY_FILTERED || picked === undefined) { continue; } - memberDef = candidate; + memberDef = picked; break; } + if (ambiguousOwnerId !== undefined) { + recordReceiverOverloadSuppression( + options.recordResolutionOutcome, + parsed.filePath, + site, + ambiguousOwnerId, + memberName, + model, + provider, + ); + handledSites.add(siteKey); + continue; + } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const ok = tryEmitEdge( graph, scopes, @@ -398,6 +468,17 @@ export function emitReceiverBoundCalls( } if (languageResolution?.kind === 'resolved') { const memberDef = languageResolution.definition; + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const reason = site.kind === 'write' || site.kind === 'read' ? site.kind @@ -482,6 +563,17 @@ export function emitReceiverBoundCalls( continue; } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const reason = site.kind === 'write' || site.kind === 'read' ? site.kind @@ -509,11 +601,23 @@ export function emitReceiverBoundCalls( // ── Case 1: namespace receiver ─────────────────────────────── const targetFiles = namespaceTargets.get(receiverName); - if (targetFiles !== undefined) { + if (targetFiles !== undefined && provider.resolveQualifiedReceiverMember === undefined) { let found = false; for (const targetFile of targetFiles) { const memberDef = findExportedDef(targetFile, memberName, index); if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + found = true; + break; + } const ok = tryEmitEdge( graph, scopes, @@ -565,6 +669,17 @@ export function emitReceiverBoundCalls( continue; } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const ok = tryEmitEdge( graph, scopes, @@ -587,9 +702,18 @@ export function emitReceiverBoundCalls( if (classDef !== undefined) { const chain = [classDef.nodeId, ...scopes.methodDispatch.mroFor(classDef.nodeId)]; let memberDef: SymbolDefinition | undefined; + let ambiguousOwnerId: string | undefined; for (const ownerId of chain) { - memberDef = findOwnedMember(ownerId, memberName, model); - if (memberDef !== undefined) { + const picked = + site.kind === 'call' + ? pickOverload(ownerId, memberName, site, model, provider) + : findOwnedMember(ownerId, memberName, model); + if (picked === OVERLOAD_AMBIGUOUS) { + ambiguousOwnerId = ownerId; + break; + } + if (picked !== undefined) { + memberDef = picked; // The MRO chain is most-derived-first ([classDef, ...ancestors]). // If the most-derived definition is arity-incompatible with the // call site, PHP throws ArgumentCountError at runtime — it does @@ -605,7 +729,31 @@ export function emitReceiverBoundCalls( break; } } + if (ambiguousOwnerId !== undefined) { + recordReceiverOverloadSuppression( + options.recordResolutionOutcome, + parsed.filePath, + site, + ambiguousOwnerId, + memberName, + model, + provider, + ); + handledSites.add(siteKey); + continue; + } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const reason = site.kind === 'write' || site.kind === 'read' ? site.kind @@ -641,8 +789,38 @@ export function emitReceiverBoundCalls( for (const targetFile3 of targetFiles3) { const classDef3 = findExportedDef(targetFile3, className, index); if (classDef3 !== undefined) { - const memberDef = findOwnedMember(classDef3.nodeId, memberName, model); - if (memberDef !== undefined) { + const picked = + site.kind === 'call' + ? pickOverload(classDef3.nodeId, memberName, site, model, provider) + : findOwnedMember(classDef3.nodeId, memberName, model); + if (picked === OVERLOAD_AMBIGUOUS) { + recordReceiverOverloadSuppression( + options.recordResolutionOutcome, + parsed.filePath, + site, + classDef3.nodeId, + memberName, + model, + provider, + ); + handledSites.add(siteKey); + found3 = true; + break; + } + if (picked !== undefined) { + const memberDef = picked; + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + found3 = true; + break; + } const ok = tryEmitEdge( graph, scopes, @@ -700,8 +878,9 @@ export function emitReceiverBoundCalls( if (ownerDef !== undefined) { const chain = [ownerDef.nodeId, ...scopes.methodDispatch.mroFor(ownerDef.nodeId)]; let memberDef: SymbolDefinition | undefined; - // Static-only filter (#1756 / U3): mirrors Case 0's chain - // walk — `findOwnedMember` without overload narrowing. When + let ambiguousOwnerId: string | undefined; + // Static-only filter (#1756 / U3): mirrors Case 0's + // overload-aware chain walk. When // a static-only candidate is found at an ancestor, walk on // so a legitimate instance member can bind. If the entire // chain is static-only, no edge is emitted (Case 3b is fed @@ -709,15 +888,45 @@ export function emitReceiverBoundCalls( // `emitReferencesViaLookup` for compound shapes, so no // handled-site marker is needed for chain-only-static). for (const ownerId of chain) { - const candidate = findOwnedMember(ownerId, memberName, model); - if (candidate === undefined) continue; - if (provider.isStaticOnly?.(candidate) === true) { + const picked = + site.kind === 'call' + ? pickFirstNonStaticOnly(ownerId, memberName, site, model, provider) + : findOwnedMember(ownerId, memberName, model); + if (picked === OVERLOAD_AMBIGUOUS) { + ambiguousOwnerId = ownerId; + break; + } + if (picked === STATIC_ONLY_FILTERED || picked === undefined) { continue; } - memberDef = candidate; + memberDef = picked; break; } + if (ambiguousOwnerId !== undefined) { + recordReceiverOverloadSuppression( + options.recordResolutionOutcome, + parsed.filePath, + site, + ambiguousOwnerId, + memberName, + model, + provider, + ); + handledSites.add(siteKey); + continue; + } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const ok = tryEmitEdge( graph, scopes, @@ -790,6 +999,17 @@ export function emitReceiverBoundCalls( } if (languageResolution?.kind === 'resolved') { const memberDef = languageResolution.definition; + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } const reason = site.kind === 'write' || site.kind === 'read' ? site.kind @@ -892,6 +1112,17 @@ export function emitReceiverBoundCalls( continue; } if (memberDef !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + memberDef, + ) + ) { + handledSites.add(siteKey); + continue; + } // For read/write ACCESSES, mirror the legacy DAG's reason // convention so consumers asserting `reason === 'write'` // keep working. @@ -968,6 +1199,17 @@ export function emitReceiverBoundCalls( continue; } if (picked !== undefined) { + if ( + suppressDeletedCallTarget( + options.recordResolutionOutcome, + parsed.filePath, + site, + picked, + ) + ) { + handledSites.add(siteKey); + continue; + } // Static-only filter (#1756 / U3): unlike Case 4 there's no // MRO chain to walk here — Case 5 dispatches on a single // owner via `pickOverload`. When the picked candidate is @@ -1152,6 +1394,25 @@ function pickFirstNonStaticOnly( return candidates[0] ?? overloads[0]; } +function suppressDeletedCallTarget( + record: ResolutionOutcomeRecorder | undefined, + filePath: string, + site: ParsedFile['referenceSites'][number], + target: SymbolDefinition, +): boolean { + if (site.kind !== 'call' || target.isDeleted !== true) return false; + record?.({ + kind: 'suppressed', + phase: 'receiver-bound-calls', + filePath, + name: site.name, + range: site.atRange, + reason: 'selected-callable-deleted', + candidateIds: [target.nodeId], + }); + return true; +} + function recordReceiverOverloadSuppression( record: ResolutionOutcomeRecorder | undefined, filePath: string, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts index d686bdd43..7d0fd811e 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/phase.ts @@ -350,6 +350,10 @@ export const scopeResolutionPhase: PipelinePhase = { prebuiltNodeLookup: sharedNodeLookup, preExtractedParsedFiles: preExtractedByPath, scopeIndexStorePath: parsedFileStorePath, + // CFG/PDG emission (#2081 M1) — opt-in; off ⇒ byte-identical graph. + pdg: ctx.options?.pdg === true, + pdgMaxEdgesPerFunction: ctx.options?.pdgMaxEdgesPerFunction, + pdgMaxReachingDefEdgesPerFunction: ctx.options?.pdgMaxReachingDefEdgesPerFunction, recordResolutionOutcome: (outcome) => { resolutionOutcomes.push(outcome); }, diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/reconcile-ownership.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/reconcile-ownership.ts index 2ec47d489..a5a4ba50d 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/reconcile-ownership.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/reconcile-ownership.ts @@ -84,13 +84,41 @@ export function reconcileOwnership( for (const parsed of parsedFiles) { for (const def of parsed.localDefs) { const ownerId = (def as { ownerId?: string }).ownerId; - if (ownerId === undefined) continue; const simple = simpleQualifiedName(def); if (simple === undefined) continue; if (def.type === 'Method' || def.type === 'Function' || def.type === 'Constructor') { + if (ownerId === undefined) { + if (def.isDeleted !== true) continue; + const existingDef = model.symbols + .lookupExactAll(def.filePath, simple) + .find((candidate) => callableSignatureMatches(candidate, def)); + if (existingDef !== undefined) { + existingDef.isDeleted = true; + skippedAlreadyPresent++; + continue; + } + model.symbols.add(def.filePath, simple, def.nodeId, def.type, { + parameterCount: def.parameterCount, + requiredParameterCount: def.requiredParameterCount, + parameterTypes: def.parameterTypes, + parameterTypeClasses: def.parameterTypeClasses, + returnType: def.returnType, + qualifiedName: def.qualifiedName, + isDeleted: true, + }); + continue; + } const existing = model.methods.lookupAllByOwner(ownerId, simple); - if (existing.some((e) => e.nodeId === def.nodeId)) { + const existingDef = existing.find( + (candidate) => + candidate.nodeId === def.nodeId || + (def.isDeleted === true && callableSignatureMatches(candidate, def)), + ); + if (existingDef !== undefined) { + if (def.isDeleted === true) { + existingDef.isDeleted = true; + } skippedAlreadyPresent++; continue; } @@ -124,6 +152,24 @@ export function reconcileOwnership( return { methodsRegistered, fieldsRegistered, nestedTypesRegistered, skippedAlreadyPresent }; } +function callableSignatureMatches( + left: ParsedFile['localDefs'][number], + right: ParsedFile['localDefs'][number], +): boolean { + if (left.filePath !== right.filePath) return false; + if (left.parameterCount !== right.parameterCount) return false; + if (left.requiredParameterCount !== right.requiredParameterCount) return false; + const leftTypes = left.parameterTypes; + const rightTypes = right.parameterTypes; + if (leftTypes === undefined || rightTypes === undefined) { + return leftTypes === rightTypes; + } + return ( + leftTypes.length === rightTypes.length && + leftTypes.every((parameterType, index) => parameterType === rightTypes[index]) + ); +} + /** * Debug-mode parity validator. Runs only when * `VALIDATE_SEMANTIC_MODEL !== '0'` AND `NODE_ENV !== 'production'`. diff --git a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts index 88c9f5f70..e8d5fe6b5 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/pipeline/run.ts @@ -34,6 +34,14 @@ import { extractParsedFile } from '../../scope-extractor-bridge.js'; import { finalizeScopeModel } from '../../finalize-orchestrator.js'; import { resolveReferenceSites, type ResolveStats } from '../../resolve-references.js'; import { buildGraphNodeLookup } from '../graph-bridge/node-lookup.js'; +import { + emitFileCfgs, + emitFileReachingDefs, + isEmitSafeCfg, + DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, + DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION, +} from '../../cfg/emit.js'; +import type { FunctionCfg } from '../../cfg/types.js'; import { resolveDefGraphId } from '../graph-bridge/ids.js'; import { buildPopulatedMethodDispatch } from '../graph-bridge/method-dispatch.js'; import { propagateImportedReturnTypes } from '../passes/imported-return-types.js'; @@ -252,6 +260,19 @@ interface RunScopeResolutionInput { * cache miss is safe (the provider re-parses). */ readonly treeCache?: { get(filePath: string): unknown }; + /** + * CFG/PDG opt-in (#2081 M1). When true, emit BasicBlock nodes + CFG edges + * from each ParsedFile's worker-built `cfgSideChannel` during Phase-4 graph + * emission (while the disk store is still live). Default/false ⇒ no CFG + * nodes or edges and a byte-identical graph. + */ + readonly pdg?: boolean; + /** Per-function CFG edge cap. `undefined` ⇒ {@link DEFAULT_MAX_CFG_EDGES_PER_FUNCTION}; + * `0` ⇒ no cap (unlimited). */ + readonly pdgMaxEdgesPerFunction?: number; + /** Per-function REACHING_DEF edge cap (#2082 M2). `undefined` ⇒ + * {@link DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION}; `0` ⇒ no cap. */ + readonly pdgMaxReachingDefEdgesPerFunction?: number; /** * Optional graph-node lookup built ONCE by the caller and shared across * every language pass. `buildGraphNodeLookup` scans the whole graph and is @@ -679,6 +700,109 @@ export function runScopeResolution( }); } + // ── CFG/PDG emission (#2081 M1, opt-in via `--pdg`) ────────────────────── + // Emit BasicBlock nodes + CFG edges from each ParsedFile's worker-built + // `cfgSideChannel`, HERE — the last point inside scope-resolution where the + // ParsedFiles are still loaded (`emitParsedFiles` carries the channel; the + // disk store is cleared right after this orchestrator returns, see phase.ts). + // A post-`mro` phase would read empty data (KTD1). Off by default ⇒ zero + // BasicBlock/CFG nodes/edges and a byte-identical graph. + // Accumulated M2 reaching-defs time (solve + dedup + REACHING_DEF emit), + // reported as the PROF `pdg=` segment. It is a SUBSET of `emit=` — the M1 + // CFG emit and the M2 solve interleave per file, so a separate checkpoint + // pair can't bracket them; without this accumulator the M2 cost would + // silently disappear into `emit=` and field regressions would be invisible. + let pdgMs = 0; + if (input.pdg === true) { + let cfgBlocks = 0; + let cfgEdges = 0; + let cfgDroppedEdges = 0; + let rdEdges = 0; + let rdDropped = 0; + let rdFacts = 0; + let rdTruncated = 0; + for (const pf of emitParsedFiles) { + const cfgs = pf.cfgSideChannel; + // Defensive: cfgSideChannel is opaque (`unknown`) and crosses the cache / + // durable store. A stale or wrong-shape value (e.g. a pre-SCHEMA_BUMP + // shard that slipped the version gate) must skip emission, not throw a + // TypeError mid-graph-build and abort scope-resolution for the language. + if (!Array.isArray(cfgs) || cfgs.length === 0) continue; + try { + // Per-element emit-safety filter (mirrors the parsedfile-store + // reviver's POLICY: valid elements in a mixed array still emit; junk + // is warned and skipped). isEmitSafeCfg lives in cfg/emit.ts next to + // the id templating it defends — see its doc for why anchor-field and + // endpoint-membership checks are load-bearing. Runs INSIDE the try so + // even a predicate-time throw (e.g. a hostile getter) is isolated. + const wellFormed = (cfgs as readonly (FunctionCfg | undefined | null)[]).filter( + isEmitSafeCfg, + ); + if (wellFormed.length < cfgs.length) { + logger.warn( + `[cfg] ${pf.filePath}: skipped ${cfgs.length - wellFormed.length} malformed ` + + `cfgSideChannel element(s) (bad shape, missing id-anchor fields, or edge ` + + `endpoints matching no block) — CFG for those functions omitted`, + ); + } + if (wellFormed.length === 0) continue; + const emitted = emitFileCfgs( + graph, + wellFormed, + input.pdgMaxEdgesPerFunction ?? DEFAULT_MAX_CFG_EDGES_PER_FUNCTION, + // Log cap-overflow drops UNCONDITIONALLY (not via input.onWarn, which is + // gated behind the semantic-model validator and silent in production) so + // the per-function edge cap never truncates the CFG silently (R6/KTD6). + (message) => logger.warn(message), + ); + cfgBlocks += emitted.blocks; + cfgEdges += emitted.edges; + cfgDroppedEdges += emitted.droppedEdges; + + // M2 (#2082 U4): reaching definitions over the same validated CFGs. + // In-memory facts are computed per function and dropped after the + // bounded (defBlock, useBlock, binding) projection is persisted — + // M3 recomputes via the same pure solver in-phase (KTD8). Timing is + // PROF-gated like every other checkpoint here (zero cost when off). + const t0 = PROF ? performance.now() : 0; + const rd = emitFileReachingDefs( + graph, + wellFormed, + input.pdgMaxReachingDefEdgesPerFunction ?? + DEFAULT_PDG_MAX_REACHING_DEF_EDGES_PER_FUNCTION, + (message) => logger.warn(message), // unconditional — R7, both layers + ); + if (PROF) pdgMs += performance.now() - t0; + rdEdges += rd.edges; + rdDropped += rd.droppedEdges; + rdFacts += rd.facts; + rdTruncated += rd.truncatedFunctions; + } catch (err) { + // Last-resort isolation, mirroring the worker-side per-file try/catch: + // a shape the predicate misses must cost this one file's CFG, not + // abort the language's whole scope-resolution pass mid-graph-build. + // NOTE a mid-emit throw can leave this file's already-inserted + // BasicBlock nodes in the graph (addNode is not transactional) — + // orphaned but inert; the predicate keeps every JSON-representable + // bad shape from reaching this path at all. + logger.warn( + `[cfg] ${pf.filePath}: CFG emission failed (${err instanceof Error ? err.message : String(err)}) — ` + + `this file's CFG is partial or absent`, + ); + } + } + if (cfgBlocks > 0) { + logger.debug( + `[scope-resolution] CFG emit (lang=${provider.language}): ` + + `${cfgBlocks} BasicBlock nodes, ${cfgEdges} CFG edges` + + (cfgDroppedEdges > 0 ? `, ${cfgDroppedEdges} edges dropped (per-function cap)` : '') + + `; ${rdEdges} REACHING_DEF edges (${rdFacts} facts)` + + (rdDropped > 0 ? `, ${rdDropped} REACHING_DEF edges dropped (per-function cap)` : '') + + (rdTruncated > 0 ? `, ${rdTruncated} function(s) hit the fact limit` : ''), + ); + } + } + if (PROF) { const tEnd = process.hrtime.bigint(); const ns = (a: bigint, b: bigint): number => Number(b - a) / 1_000_000; @@ -688,6 +812,8 @@ export function runScopeResolution( ` propagate=${ns(tFinalize, tPropagate).toFixed(0)}ms` + ` resolve=${ns(tPropagate, tResolve).toFixed(0)}ms` + ` emit=${ns(tResolve, tEnd).toFixed(0)}ms` + + // pdg ⊆ emit: the M2 reaching-defs share of the emit bucket (#2082 U4). + (input.pdg === true ? ` pdg=${pdgMs.toFixed(0)}ms` : '') + ` total=${ns(tStart, tEnd).toFixed(0)}ms` + ` (${parsedFiles.length} files)`, ); diff --git a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts index b46b47f72..ad6a19370 100644 --- a/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts +++ b/gitnexus/src/core/ingestion/scope-resolution/resolution-outcome.ts @@ -5,6 +5,7 @@ export type ResolutionSuppressionReason = | 'conversion-rank-tied' | 'inline-ns-ambiguous' | 'member-lookup-ambiguous' + | 'selected-callable-deleted' | 'overload-ambiguous' | 'overload-ambiguous-normalization'; diff --git a/gitnexus/src/core/ingestion/utils/method-props.ts b/gitnexus/src/core/ingestion/utils/method-props.ts index b97b5de4b..5f8fa1a6b 100644 --- a/gitnexus/src/core/ingestion/utils/method-props.ts +++ b/gitnexus/src/core/ingestion/utils/method-props.ts @@ -213,6 +213,7 @@ export function buildMethodProps(info: MethodInfo): Record { ...(info.isAsync ? { isAsync: info.isAsync } : {}), ...(info.isPartial ? { isPartial: info.isPartial } : {}), ...(info.isConst ? { isConst: info.isConst } : {}), + ...(info.isDeleted ? { isDeleted: info.isDeleted } : {}), ...(info.annotations.length > 0 ? { annotations: info.annotations } : {}), }; } diff --git a/gitnexus/src/core/ingestion/workers/clone-safety.ts b/gitnexus/src/core/ingestion/workers/clone-safety.ts new file mode 100644 index 000000000..cf56691e3 --- /dev/null +++ b/gitnexus/src/core/ingestion/workers/clone-safety.ts @@ -0,0 +1,545 @@ +/** + * Structured-clone safety for the worker result boundary (#2112). + * + * A parse worker delivers its accumulated result to the main thread via + * `parentPort.postMessage(...)`. Node serializes that payload with the + * structured-clone algorithm SYNCHRONOUSLY on the worker thread, and it + * THROWS a `DataCloneError` the instant it meets a value it can't serialize — + * a function, a symbol, a Promise, a WeakMap, etc. The reporter of #2112 hit + * exactly this: a node record whose `properties` carried an own-enumerable + * value pointing at a native function (`function toString() { [native code] } + * could not be cloned`). One such value aborted the entire parse phase, + * because the worker re-posts the throw as `{type:'error'}` which the pool + * counts as a worker death — and under `GITNEXUS_WORKER_POOL_SIZE=1` the same + * graph re-throws on every respawn until the slot's budget is exhausted. + * + * This module is the safety net. It runs ONLY after a real clone failure on + * the fast-path post (zero overhead on healthy runs), and rewrites the + * boundary-crossing arrays so the result becomes cloneable: a non-cloneable + * value inside a plain extraction record is dropped (the record is otherwise + * kept — strictly-missing data, never wrong), and a `ParsedFile` that can't be + * made cloneable is dropped whole so scope-resolution re-derives it on the + * main thread (where there is no clone boundary) with intact edge data. + * + * Language-neutral by construction: it keys on value shape and field name + * only, never on a language (AGENTS.md shared-pipeline rule). The strip + * semantics mirror what the store path's `JSON.stringify` already silently + * drops, so store / no-store / cold / warm runs converge on the same graph. + */ + +/** A file whose parse result was sanitized or dropped at the clone boundary. */ +export interface SkippedPath { + /** Best-effort source path of the offending record (or `(unknown)`). */ + path: string; + /** Human-readable reason, e.g. "dropped 1 non-serializable value from nodes". */ + reason: string; +} + +/** + * True iff `value` survives Node's structured-clone algorithm (the same + * algorithm `postMessage` uses). This is the authoritative probe — it matches + * the real failure exactly, including Map/Set/Date/RegExp/TypedArray support, + * so it never false-positives on the `Scope` Maps that clone fine. + */ +export function isStructuredCloneable(value: unknown): boolean { + try { + structuredClone(value); + return true; + } catch { + return false; + } +} + +// ── Compile-time boundary guard (#2143) ───────────────────────────────────── +// The runtime net above is the production backstop; this is its compile-time +// complement. The worker result is plain data EXCEPT a few `unknown`-typed +// sinks (a node's `properties` bag, the provider `extractTemplateConstraints` / +// `collectCaptureSideChannel` hook returns). `unknown` lets a non-serializable +// value (a function, a leaked tree-sitter `SyntaxNode`, …) pass with no +// compile-time guard — that is the structural hole #2112 leaked through. Typing +// those producers as `Cloneable` turns such a leak into a compile error at +// the source site instead of a runtime DataCloneError far downstream. + +/** The leaf values the structured-clone algorithm copies verbatim. */ +type CloneablePrimitive = undefined | null | boolean | number | bigint | string; + +/** + * Maps `T` to itself when every value reachable from it is structured-clone + * safe, and to a type containing `never` at the first offending property + * otherwise. A function or symbol — the values `postMessage` rejects — becomes + * `never`, so a struct carrying one is no longer assignable to its own + * `Cloneable` and `assertCloneable` rejects it, naming the bad key. + * + * Implemented as a homomorphic mapped type (`{ [K in keyof T]: … }`) so it + * preserves `interface` shapes and `readonly` modifiers and works WITHOUT + * requiring the payload types to carry an index signature — sidestepping the + * "closed interface is not assignable to a recursive index-signature type" wall + * that blocked the value-typed-`Cloneable` approach (#2143). `Map`/`Set`/array + * containers recurse into their element types; `Date`/`RegExp` are clone-safe + * leaves. + */ +/** True iff `T` is `any` (the canonical `IsAny` probe: only `any` satisfies `0 extends 1 & T`). */ +type IsAny = 0 extends 1 & T ? true : false; + +export type Cloneable = + IsAny extends true + ? never // an `any`-typed member defeats the guard — reject it like `unknown` (both → never) + : T extends CloneablePrimitive | Date | RegExp + ? T + : T extends (...args: never[]) => unknown + ? never + : T extends symbol + ? never + : T extends ReadonlyMap + ? ReadonlyMap, Cloneable> + : T extends ReadonlySet + ? ReadonlySet> + : T extends readonly (infer U)[] + ? T extends unknown[] + ? Cloneable[] + : readonly Cloneable[] + : T extends object + ? { [K in keyof T]: Cloneable } + : never; + +/** + * Identity at runtime (zero cost — returns its argument unchanged); a + * compile-time assertion that `value` is structured-clone safe. Wrap a + * producer that feeds an `unknown` worker-result sink: + * + * collectCaptureSideChannel: (filePath) => assertCloneable(collectFoo(filePath)) + * + * If `collectFoo`'s return type ever gains a non-cloneable member (a function, a + * `SyntaxNode`, …) the call fails to compile, pointing at the offending key. + * + * The parameter is a conditional type rather than an `extends Cloneable` + * constraint because a self-referential constraint (`T extends Cloneable`) + * is a "circular constraint" error in TypeScript. For a clone-safe `T` the + * parameter resolves to `T` (call type-checks as a plain identity); for an + * unsafe `T` it resolves to `Cloneable` (which has `never` at the bad key), + * so the argument is rejected. + */ +export function assertCloneable(value: T extends Cloneable ? T : Cloneable): T { + return value as T; +} + +/** + * Recursion cap for the module's own traversal. An over-deep subtree is treated + * as non-cloneable rather than recursing to a stack overflow — without this, a + * deeply-nested record would throw `RangeError` inside the sanitizer and (since + * the recovery path is the safety net) re-arm the very cascade #2112 fixes. Set + * far below the observed ~3000-frame overflow and far above any real + * parse-result record (extraction records are shallow plain data). Note: this + * caps the module's recursion only; `structuredClone`'s own internal recursion + * (the `isStructuredCloneable` probe of non-plain objects) is bounded by that + * helper's catch-all, which turns a probe-side `RangeError` into a + * non-cloneable verdict — so do not narrow that catch. + */ +const MAX_CLONE_DEPTH = 200; + +/** + * True iff `key` is a canonical array-index string (`"0"`, `"1"`, … `< 2^32-1`) + * — i.e. one of the slots the numeric index loop already visits. Everything + * else returned by `Object.keys(array)` is a NON-index own-enumerable property + * (`arr.meta = …`), which the structured-clone algorithm ALSO serializes (and + * throws on if non-cloneable). The array branches of `containsNonCloneable` and + * `stripNonCloneable` use this to scan those extra keys in lockstep. + */ +function isArrayIndexKey(key: string): boolean { + const n = Number(key); + return Number.isInteger(n) && n >= 0 && n < 4294967295 && String(n) === key; +} + +/** + * Non-allocating scan: returns true on the FIRST value structured-clone would + * reject. Used to decide whether an array (or element) needs rewriting at all, + * so clean arrays keep their referential identity and pay no copy cost. + */ +function containsNonCloneable(value: unknown, seen: WeakSet, depth = 0): boolean { + const t = typeof value; + if (t === 'function' || t === 'symbol') return true; + if (value === null || t !== 'object') return false; + // Depth bound: treat an over-deep subtree as non-cloneable (the element is + // then stripped/dropped) instead of overflowing the stack. + if (depth >= MAX_CLONE_DEPTH) return true; + const obj = value as object; + // Cycles clone fine; don't recurse into one twice. + if (seen.has(obj)) return false; + // Structured-clone-native containers carry no non-cloneable payload of their + // own; their *contents* still need scanning (a Map value could be a fn). + if (obj instanceof Date || obj instanceof RegExp) return false; + // Buffers/views usually clone, but a DETACHED one is rejected by + // structuredClone — probe rather than wave it through. No byteLength + // heuristic: a legitimately empty `new Uint8Array(0)` also has byteLength 0 + // yet clones fine, so a length check would false-positive. + if (obj instanceof ArrayBuffer || ArrayBuffer.isView(obj)) return !isStructuredCloneable(obj); + seen.add(obj); + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + if (containsNonCloneable(obj[i], seen, depth + 1)) return true; + } + // structuredClone also serializes an array's NON-index own-enumerable + // properties and throws on a non-cloneable one — scan them too (lockstep + // with stripNonCloneable's array branch; see isArrayIndexKey). + for (const key of Object.keys(obj)) { + if (isArrayIndexKey(key)) continue; + let child: unknown; + try { + child = (obj as unknown as Record)[key]; + } catch { + return true; // a throwing getter can't be serialized either + } + if (containsNonCloneable(child, seen, depth + 1)) return true; + } + return false; + } + if (obj instanceof Map) { + for (const [k, v] of obj) { + if (containsNonCloneable(k, seen, depth + 1) || containsNonCloneable(v, seen, depth + 1)) + return true; + } + return false; + } + if (obj instanceof Set) { + for (const v of obj) { + if (containsNonCloneable(v, seen, depth + 1)) return true; + } + return false; + } + // A non-plain object (Promise, WeakMap, class instance with internal slots) + // that structured clone can't handle: detect via the authoritative probe. + // Plain objects fall through to a property scan (cheap, no allocation). + const proto = Object.getPrototypeOf(obj); + if (proto !== Object.prototype && proto !== null) { + if (!isStructuredCloneable(obj)) return true; + return false; + } + for (const key of Object.keys(obj)) { + let child: unknown; + try { + child = (obj as Record)[key]; + } catch { + // A getter that throws can't be serialized either — treat as non-cloneable. + return true; + } + if (containsNonCloneable(child, seen, depth + 1)) return true; + } + return false; +} + +/** + * State carried through a strip pass. `stripped` counts dropped values for the + * skip report; `seen` memoizes each visited object to its stripped COPY (not a + * bare visited-set) so a DAG-aliased subtree — the same object reached via two + * paths — is sanitized once and shared, never over-dropped, and cycles + * terminate by returning the in-progress copy. + */ +interface StripCtx { + stripped: number; + seen: Map; + /** + * Dotted key paths (relative to the element root) of every value that was + * stripped/dropped — e.g. `properties.toString`, `meta.data[3]`. Surfaced in + * the skip reason so the offending property is named precisely, which is what + * lets a still-unpinned leak be located from a single log line (#2112). + */ + keys: string[]; +} + +/** Record a strip at `path` (root → `(root)`); keeps the count + key path in sync. */ +function recordStrip(ctx: StripCtx, path: string): void { + ctx.stripped++; + ctx.keys.push(path === '' ? '(root)' : path); +} + +/** + * Deep-copy `value`, replacing any value structured-clone would reject with + * `undefined` (which clones fine). Preserves primitives, arrays, plain + * objects, and the structured-clone-native containers (Date, RegExp, Map, + * Set, ArrayBuffer, TypedArray). Rebuilds only what it must — clean leaves are + * returned by reference. `path` is the dotted key path of `value` (for the + * diagnostic record). + */ +function stripNonCloneable(value: unknown, ctx: StripCtx, depth = 0, path = ''): unknown { + const t = typeof value; + if (t === 'function' || t === 'symbol') { + recordStrip(ctx, path); + return undefined; + } + if (value === null || t !== 'object') return value; + // Depth bound (mirrors containsNonCloneable): drop an over-deep subtree to + // `undefined` (itself cloneable, and a legal property value / array element) + // rather than overflowing the stack. + if (depth >= MAX_CLONE_DEPTH) { + recordStrip(ctx, path); + return undefined; + } + const obj = value as object; + // Memoized? Return the SAME stripped copy (preserves DAG shape; terminates + // cycles by returning the in-progress copy inserted before recursing below). + if (ctx.seen.has(obj)) return ctx.seen.get(obj); + // Leaf-like values: returned by reference, but still memoize the decision so + // a second alias resolves identically. + if (obj instanceof Date || obj instanceof RegExp) { + ctx.seen.set(obj, value); + return value; + } + if (obj instanceof ArrayBuffer || ArrayBuffer.isView(obj)) { + // Keep a live buffer/view (even an empty one); drop a detached one, which + // structuredClone rejects. The probe is exact — no byteLength heuristic. + if (!isStructuredCloneable(obj)) { + recordStrip(ctx, path); + ctx.seen.set(obj, undefined); + return undefined; + } + ctx.seen.set(obj, value); + return value; + } + // Containers: allocate the empty copy, memoize it BEFORE recursing, then fill + // — so a cycle/alias that re-enters gets this in-progress copy. + if (Array.isArray(obj)) { + const out: unknown[] = []; + ctx.seen.set(obj, out); + for (let i = 0; i < obj.length; i++) + out.push(stripNonCloneable(obj[i], ctx, depth + 1, `${path}[${i}]`)); + // Carry NON-index own-enumerable props through the same strip (lockstep + // with containsNonCloneable): structuredClone serializes them, so a + // non-cloneable one must be stripped rather than left to throw on re-post. + for (const key of Object.keys(obj)) { + if (isArrayIndexKey(key)) continue; + const childPath = `${path}.${key}`; + let child: unknown; + try { + child = (obj as unknown as Record)[key]; + } catch { + recordStrip(ctx, childPath); + continue; + } + (out as unknown as Record)[key] = stripNonCloneable( + child, + ctx, + depth + 1, + childPath, + ); + } + return out; + } + if (obj instanceof Map) { + // Scope limit (acceptable): object keys aren't identity-preserved across + // stripping. Parse-result Maps are primitive-keyed, so this never bites. + const out = new Map(); + ctx.seen.set(obj, out); + for (const [k, v] of obj) + out.set( + stripNonCloneable(k, ctx, depth + 1, `${path}`), + stripNonCloneable(v, ctx, depth + 1, `${path}`), + ); + return out; + } + if (obj instanceof Set) { + const out = new Set(); + ctx.seen.set(obj, out); + for (const v of obj) out.add(stripNonCloneable(v, ctx, depth + 1, `${path}`)); + return out; + } + const proto = Object.getPrototypeOf(obj); + if (proto !== Object.prototype && proto !== null) { + // Non-plain object that the probe already flagged as non-cloneable and + // that we can't safely reconstruct (Promise, WeakMap, class instance with + // internal slots). Drop it whole — memoize the decision so aliases agree. + if (!isStructuredCloneable(obj)) { + recordStrip(ctx, path); + ctx.seen.set(obj, undefined); + return undefined; + } + ctx.seen.set(obj, value); + return value; + } + const out: Record = {}; + ctx.seen.set(obj, out); + for (const key of Object.keys(obj)) { + const childPath = path === '' ? key : `${path}.${key}`; + let child: unknown; + try { + child = (obj as Record)[key]; + } catch { + // A getter that throws is non-serializable — drop the property. + recordStrip(ctx, childPath); + continue; + } + out[key] = stripNonCloneable(child, ctx, depth + 1, childPath); + } + return out; +} + +/** Keys checked (top-level and one level deep) to attribute a record to a file. */ +const DEFAULT_PATH_KEYS = ['filePath', 'path', 'file'] as const; + +/** Read `obj[key]`, returning undefined if the access throws (throwing getter / Proxy trap). */ +function safeGet(obj: Record, key: string): unknown { + try { + return obj[key]; + } catch { + return undefined; + } +} + +/** Read a path key off a child object (one level deep); never throws. */ +function pathFromChild(child: unknown, pathKeys: readonly string[]): string | undefined { + if (child === null || typeof child !== 'object') return undefined; + const crec = child as Record; + for (const pk of pathKeys) { + const v = safeGet(crec, pk); + if (typeof v === 'string') return v; + } + return undefined; +} + +/** + * Best-effort source-path extraction for reporting; never throws. Reads are + * defensive (a throwing getter / Proxy trap on a path-attribution key must not + * escape and abandon the sanitize — it would re-arm the fail-closed cascade). + */ +function findFilePath(element: unknown, pathKeys: readonly string[]): string | undefined { + if (element === null || typeof element !== 'object') return undefined; + const rec = element as Record; + // Top level first — a ParsedFile carries `filePath` here. + for (const key of pathKeys) { + const v = safeGet(rec, key); + if (typeof v === 'string') return v; + } + // Known child next — a ParsedNode carries its path at `properties.filePath`. + // Prefer it over the generic sweep so attribution is deterministic when a + // sibling child also happens to carry a path-like key. + const fromProps = pathFromChild(safeGet(rec, 'properties'), pathKeys); + if (fromProps !== undefined) return fromProps; + // Generic one-level sweep as the fallback for other shapes. + let keys: string[]; + try { + keys = Object.keys(rec); + } catch { + return undefined; // a Proxy ownKeys trap that throws — give up on attribution + } + for (const key of keys) { + if (key === 'properties') continue; // already checked above + const fromChild = pathFromChild(safeGet(rec, key), pathKeys); + if (fromChild !== undefined) return fromChild; + } + return undefined; +} + +export interface MakeCloneSafeOptions { + /** + * Array field names whose offending elements are DROPPED whole rather than + * stripped in place (e.g. `parsedFiles` — its `captureSideChannel` drives + * edge resolution, so a stripped-and-delivered file would ship WRONG edges; + * dropping it lets scope-resolution re-derive it on the main thread). + */ + dropWholeElement: ReadonlySet; + /** Field names to skip entirely (e.g. the `skippedPaths` field itself). */ + skipFields?: ReadonlySet; + /** Keys to probe for a file path when attributing a skip. */ + pathKeys?: readonly string[]; +} + +/** + * Make a worker result's boundary-crossing array fields structured-cloneable, + * mutating `result` in place. Only arrays that actually contain a + * non-cloneable value are rewritten; everything else keeps referential + * identity. Returns the list of affected file paths for reporting. + * + * Call this after ANY failure of the fast-path post — a `DataCloneError`, OR a + * throwing getter's own error surfaced by structuredClone (the caller in + * `post-result.ts` recovers on any throw, not only `DataCloneError`). + */ +export function makeWorkerResultCloneSafe( + result: Record, + options: MakeCloneSafeOptions, +): { skipped: SkippedPath[] } { + const pathKeys = options.pathKeys ?? DEFAULT_PATH_KEYS; + const skipped: SkippedPath[] = []; + + for (const field of Object.keys(result)) { + if (options.skipFields?.has(field)) continue; + const value = result[field]; + if (!Array.isArray(value)) continue; + + const dropWhole = options.dropWholeElement.has(field); + // `out` is built lazily — only once a dirty element appears — by copying the + // clean prefix, so a fully-clean array is never rebuilt and keeps its + // referential identity (no field reassignment). A dirty element is scanned + // (containsNonCloneable) and then stripped (stripNonCloneable): two passes, + // deliberately. The non-allocating pre-scan is exactly what lets CLEAN + // elements stay by reference (zero-copy) — replacing it with an + // always-allocating strip would regress that. This whole path is + // failure-path-only (the fast post already threw), so the second pass over + // the rare dirty element is acceptable. + let out: unknown[] | null = null; + for (let i = 0; i < value.length; i++) { + const element = value[i]; + try { + if (!containsNonCloneable(element, new WeakSet())) { + if (out) out.push(element); + continue; + } + if (!out) out = value.slice(0, i); // first dirty element: copy clean prefix + const path = findFilePath(element, pathKeys) ?? '(unknown)'; + if (dropWhole) { + skipped.push({ path, reason: `dropped non-serializable ${field} entry` }); + continue; + } + const ctx: StripCtx = { stripped: 0, seen: new Map(), keys: [] }; + const cleaned = stripNonCloneable(element, ctx); + // Last-resort guard: if stripping functions/symbols still left something + // structured-clone rejects, drop the element rather than re-throw. + if (isStructuredCloneable(cleaned)) { + out.push(cleaned); + // Name the offending key path(s) so the leak is locatable from the log + // (e.g. "from nodes: properties.toString") — not just the array field. + const at = ctx.keys.slice(0, 3).join(', '); + const more = ctx.keys.length > 3 ? `, …+${ctx.keys.length - 3}` : ''; + skipped.push({ + path, + reason: `stripped ${ctx.stripped} non-serializable value(s) from ${field}: ${at}${more}`, + }); + } else { + skipped.push({ path, reason: `dropped unsalvageable ${field} entry` }); + } + } catch { + // A throw DURING this element's scan/strip — a Proxy with a throwing + // `getPrototypeOf`/`ownKeys` trap reached by Object.getPrototypeOf / + // Object.keys, or any other structural-enumeration throw. Drop the + // element rather than let the throw escape to postResultCloneSafe's + // fail-closed {type:'error'} (which under POOL_SIZE=1 re-arms the + // cascade this net prevents). One pathological element can't sink the + // whole result. + if (!out) out = value.slice(0, i); + skipped.push({ path: '(unknown)', reason: `dropped ${field} entry (sanitizer error)` }); + } + } + if (out) result[field] = out; + } + + // Final safety gate. The loop above only rewrites ARRAY fields, so a future + // non-array result sink (a nested object / Map) — or an array field whose own + // non-index property the element loop didn't reach — could still hold a + // non-cloneable value and throw on the re-post. Make "the returned result is + // structured-cloneable" a hard postcondition: strip any remaining offending + // field in place. Failure-path-only and a no-op once the result is already + // clean (the per-field probe short-circuits every clean field). + if (!isStructuredCloneable(result)) { + for (const field of Object.keys(result)) { + if (options.skipFields?.has(field)) continue; + if (isStructuredCloneable(result[field])) continue; + const ctx: StripCtx = { stripped: 0, seen: new Map(), keys: [] }; + result[field] = stripNonCloneable(result[field], ctx); + const at = ctx.keys.slice(0, 3).join(', '); + skipped.push({ + path: '(result)', + reason: `stripped ${ctx.stripped} non-serializable value(s) from ${field}${at ? `: ${at}` : ''}`, + }); + } + } + + return { skipped }; +} diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index b32d95758..f29b63c16 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -4,7 +4,6 @@ import JavaScript from 'tree-sitter-javascript'; import TypeScript from 'tree-sitter-typescript'; import Python from 'tree-sitter-python'; import Java from 'tree-sitter-java'; -import C from 'tree-sitter-c'; import CPP from 'tree-sitter-cpp'; // Explicit subpath import — see parser-loader.ts for rationale (#1013). import CSharp from 'tree-sitter-c-sharp/bindings/node/index.js'; @@ -12,7 +11,7 @@ import Go from 'tree-sitter-go'; import Rust from 'tree-sitter-rust'; import PHP from 'tree-sitter-php'; import Ruby from 'tree-sitter-ruby'; -import { createRequire } from 'node:module'; +import { requireVendoredGrammar } from '../../tree-sitter/vendored-grammars.js'; import { SupportedLanguages } from 'gitnexus-shared'; import { getProvider } from '../languages/index.js'; import { @@ -26,6 +25,9 @@ import { deriveDefaultExportHocName, } from '../ts-js-hoc-utils.js'; import { parseSourceSafe } from '../../tree-sitter/safe-parse.js'; +import type { SkippedPath } from './clone-safety.js'; +import { postResultCloneSafe } from './post-result.js'; +import { mergeResult } from './result-merge.js'; import type { SymbolTableReader } from '../model/symbol-table.js'; import type { ExtractedRouterInclude, @@ -37,8 +39,8 @@ import type { type TreeSitterLanguage = Parameters[0]; // ── Worker grammar loading — enforcement boundary (#2091/#2093, #2101) ─────── -// The worker maintains its own grammar table (the guarded `_require`s below + -// `languageMap`) and intentionally does NOT consult the runtime +// The worker maintains its own grammar table (the guarded vendored-grammar +// loads below + `languageMap`) and intentionally does NOT consult the runtime // `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` opt-out. It does not need to: the MAIN // THREAD's `parseableScanned` filter (pipeline-phases/parse-impl.ts, gated on // `parser-loader.isLanguageAvailable`, which honors the runtime opt-out and a @@ -49,23 +51,29 @@ type TreeSitterLanguage = Parameters[0]; // `isLanguageAvailable` must re-introduce the gate here. (The cleaner end-state // — routing this table through `parser-loader.getLanguageGrammar` so there is // one loader — is the deferred Tier-1 consolidation.) -// tree-sitter-swift is an optionalDependency — may not be installed -const _require = createRequire(import.meta.url); +// Swift/Dart/Kotlin/C are vendored grammars loaded from `vendor/` by absolute +// path (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Each +// may be absent on a platform without a prebuild or a toolchain-less / +// `--ignore-scripts` install, so every load is guarded so a missing binding +// cannot crash the worker at module-load (#2091/#2093, #2116). let Swift: TreeSitterLanguage | null = null; try { - Swift = _require('tree-sitter-swift'); + Swift = requireVendoredGrammar('tree-sitter-swift') as TreeSitterLanguage; } catch {} -// tree-sitter-dart is an optionalDependency — may not be installed let Dart: TreeSitterLanguage | null = null; try { - Dart = _require('tree-sitter-dart'); + Dart = requireVendoredGrammar('tree-sitter-dart') as TreeSitterLanguage; } catch {} -// tree-sitter-kotlin is an optionalDependency — may not be installed let Kotlin: TreeSitterLanguage | null = null; try { - Kotlin = _require('tree-sitter-kotlin'); + Kotlin = requireVendoredGrammar('tree-sitter-kotlin') as TreeSitterLanguage; +} catch {} + +let C: TreeSitterLanguage | null = null; +try { + C = requireVendoredGrammar('tree-sitter-c') as TreeSitterLanguage; } catch {} import { getLanguageFromFilename } from 'gitnexus-shared'; import { @@ -122,6 +130,7 @@ import { persistDurableParsedFileShardSync, } from '../../../storage/parsedfile-store.js'; import { extractLaravelRoutes, type ExtractedRoute } from '../route-extractors/laravel.js'; +import { collectFunctionCfgs, DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js'; import { logger } from '../../logger.js'; export type { ExtractedRoute } from '../route-extractors/laravel.js'; @@ -147,6 +156,19 @@ const DURABLE_PARSED_FILE_STORAGE_PATH: string | undefined = ( )?.durableParsedFileStoragePath; let shardSeq = 0; +// ── PDG/CFG opt-in (#2081 M1) ─────────────────────────────────────────────── +// Read ONCE at worker init from `workerData` (the worker never sees +// PipelineOptions — config arrives via the pool factory's `workerData`, see +// KTD7 / U5). When `pdg` is set, the worker builds a per-function control-flow +// graph from the tree-sitter AST (where it lives) and serializes it onto +// `ParsedFile.cfgSideChannel`. Off ⇒ no CFG work and no field — the default for +// every run today. `pdgMaxFunctionLines` bounds per-function CFG cost +// (0/undefined ⇒ no cap; see collectFunctionCfgs). +const PDG_ENABLED: boolean = (workerData as { pdg?: boolean } | undefined)?.pdg === true; +const PDG_MAX_FUNCTION_LINES: number = + (workerData as { pdgMaxFunctionLines?: number } | undefined)?.pdgMaxFunctionLines ?? + DEFAULT_PDG_MAX_FUNCTION_LINES; + // ── Bootstrap-stage diagnostics (#1741) ──────────────────────────────────── // When GITNEXUS_WORKER_BOOTSTRAP=1 (or --verbose sets GITNEXUS_VERBOSE), each // worker reports its startup stage timings to stderr — which the pool tees @@ -218,6 +240,7 @@ interface ParsedSymbol { isReadonly?: boolean; isAbstract?: boolean; isFinal?: boolean; + isDeleted?: boolean; annotations?: string[]; } @@ -380,6 +403,15 @@ export interface ParseWorkerResult { */ parsedFiles: ParsedFile[]; skippedLanguages: Record; + /** + * Files whose parse output carried a value the structured-clone algorithm + * couldn't serialize across the worker boundary (#2112). The clone-safety + * net stripped or dropped the offending value so the result could be + * delivered; these paths are surfaced to the operator so the (rare) data + * loss is visible. Optional for cache backward compatibility — older cache + * entries predate the field; consumers must guard with `?? []`. + */ + skippedPaths?: SkippedPath[]; fileCount: number; } @@ -404,7 +436,7 @@ const languageMap: Record = { [`${SupportedLanguages.TypeScript}:tsx`]: TypeScript.tsx, [SupportedLanguages.Python]: Python, [SupportedLanguages.Java]: Java, - [SupportedLanguages.C]: C, + ...(C ? { [SupportedLanguages.C]: C } : {}), [SupportedLanguages.CPlusPlus]: CPP, [SupportedLanguages.CSharp]: CSharp, [SupportedLanguages.Go]: Go, @@ -1026,6 +1058,7 @@ const ROUTE_DECORATOR_NAMES = new Set([ 'PostMapping', 'PutMapping', 'DeleteMapping', + 'PatchMapping', ]); // ============================================================================ @@ -1214,9 +1247,38 @@ const processFileGroup = ( // copy — scopes/defs are carried by reference) to attach the field rather // than mutate the frozen object. const sideChannel = provider.collectCaptureSideChannel?.(file.path); - result.parsedFiles.push( - sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile, - ); + let withChannels = + sideChannel !== undefined ? { ...parsedFile, captureSideChannel: sideChannel } : parsedFile; + + // CFG side-channel (#2081 M1): build the per-function control-flow graph + // here, where the tree-sitter AST is still in hand, and attach it as plain + // serializable data. Only on a --pdg run and only for languages with a + // cfgVisitor (TS/JS in M1). The same disk-store/warm-cache machinery that + // carries captureSideChannel carries this — its coherence rests on the + // SCHEMA_BUMP + the pdg-folded chunk-hash key (see parse-cache.ts). + if (PDG_ENABLED && provider.cfgVisitor) { + // Isolate the CFG build per file: a throw here (an unexpected tree-sitter + // node shape, a deep-nesting stack overflow) must NOT propagate — it + // would escape processFileGroup to the language-group catch, which treats + // any throw as "parser unavailable" and silently drops EVERY remaining + // file in the group. Skip CFG for this one file; parsing + scope + // resolution proceed unaffected (CFG is a strictly-additive opt-in). + try { + const { cfgs } = collectFunctionCfgs( + tree.rootNode, + provider.cfgVisitor, + file.path, + PDG_MAX_FUNCTION_LINES, + ); + if (cfgs.length) withChannels = { ...withChannels, cfgSideChannel: cfgs }; + } catch (err) { + const message = `CFG build failed for ${file.path}: ${err instanceof Error ? err.message : String(err)}`; + if (parentPort) parentPort.postMessage({ type: 'warning', message }); + else logger.warn(message); + } + } + + result.parsedFiles.push(withChannels); } // Build per-file type environment + constructor bindings in a single AST walk. @@ -2189,6 +2251,9 @@ const processFileGroup = ( isReadonly: methodProps.isReadonly as boolean | undefined, isAbstract: methodProps.isAbstract as boolean | undefined, isFinal: methodProps.isFinal as boolean | undefined, + ...(methodProps.isDeleted !== undefined + ? { isDeleted: methodProps.isDeleted as boolean } + : {}), ...(methodProps.isVirtual !== undefined ? { isVirtual: methodProps.isVirtual as boolean } : {}), @@ -2272,6 +2337,15 @@ const processFileGroup = ( ); } + // Language-specific decorator route extraction via provider hook. + // The provider's extractDecoratorRoutes walks the AST for framework-specific + // route patterns (e.g., Java Spring class-level prefix joining). Routes are + // appended to decoratorRoutes for the routes phase to emit as Route nodes. + if (provider.extractDecoratorRoutes) { + const frameworkRoutes = provider.extractDecoratorRoutes(tree, file.path, lineOffset); + for (const r of frameworkRoutes) result.decoratorRoutes.push(r); + } + // Vue: emit CALLS edges for components used in