diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..f3530e4d3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,75 @@ +version: 2 +updates: + # Keep third-party Actions SHA pins current. See CONTRIBUTING.md — when + # reviewing these bumps, verify the SHA corresponds to the claimed tag by + # running `gh api repos///git/refs/tags/` before merge. + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: chore + include: scope + labels: + - dependencies + - ci + + # Gitnexus npm deps — tree-sitter grammars checked daily so we catch + # new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars + # are grouped so lockstep bumps produce a single PR. The tree-sitter + # RUNTIME is pinned — upgrade deliberately via the drift check workflow. + # See .github/scripts/check-tree-sitter-upgrade-readiness.py for + # the upgrade readiness tracker. + - package-ecosystem: npm + directory: /gitnexus + schedule: + interval: daily + open-pull-requests-limit: 10 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + groups: + tree-sitter-grammars: + patterns: + - tree-sitter-* + exclude-patterns: + - tree-sitter + - tree-sitter-cli + ignore: + # Pin the tree-sitter runtime at 0.21.x until the drift check + # reports all grammars are peer-dep compatible with 0.25. + - dependency-name: tree-sitter + update-types: + - version-update:semver-major + - version-update:semver-minor + # tree-sitter-cli follows the runtime's version cadence. Bump when + # regenerating vendor/tree-sitter-proto/src/parser.c, not on a schedule. + - dependency-name: tree-sitter-cli + + # gitnexus-web (thin frontend client). + - package-ecosystem: npm + directory: /gitnexus-web + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + - frontend + + # Shared types package. + - package-ecosystem: npm + directory: /gitnexus-shared + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 000000000..378a60e2c --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,53 @@ +# release-drafter config — used only for PR autolabeling by +# `.github/workflows/pr-labeler.yml` (the workflow passes `disable-releaser: true`, +# so the draft-release side of release-drafter never runs). +# +# The labels applied here are the same ones `.github/release.yml` maps to +# categorized release-notes sections. +# +# `sync-labels: true` removes managed autolabels that no longer match the PR — +# critical for the breaking-change case: if a PR title drops the `!` or the body +# drops `BREAKING CHANGE:`, the `breaking` label is pulled off automatically. + +# Required by release-drafter; not used because releaser is disabled. +name-template: 'unused' +tag-template: 'unused' +template: | + $CHANGES + +sync-labels: true + +autolabeler: + - label: enhancement + title: + - '/^feat(\([^)]+\))?!?:/i' + - label: bug + title: + - '/^fix(\([^)]+\))?!?:/i' + - label: performance + title: + - '/^perf(\([^)]+\))?!?:/i' + - label: refactor + title: + - '/^refactor(\([^)]+\))?!?:/i' + - label: documentation + title: + - '/^docs(\([^)]+\))?!?:/i' + - label: test + title: + - '/^test(\([^)]+\))?!?:/i' + - label: ci + title: + - '/^ci(\([^)]+\))?!?:/i' + - label: dependencies + title: + - '/^(build|deps)(\([^)]+\))?!?:/i' + - label: chore + title: + - '/^(chore|revert)(\([^)]+\))?!?:/i' + # Breaking-change marker: either `!` in the type prefix or `BREAKING CHANGE:` in body. + - label: breaking + title: + - '/^[a-z]+(\([^)]+\))?!:/i' + body: + - '/BREAKING[ -]CHANGE:/i' diff --git a/.github/scripts/check-tree-sitter-upgrade-readiness.py b/.github/scripts/check-tree-sitter-upgrade-readiness.py new file mode 100644 index 000000000..df21533a8 --- /dev/null +++ b/.github/scripts/check-tree-sitter-upgrade-readiness.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Monitor tree-sitter 0.25 upgrade readiness. + +Tracks two things Dependabot cannot see: + + 1. Peer-dep compatibility. Each tree-sitter-* grammar declares a peer + dependency on the tree-sitter runtime. We want to know when every + grammar's *latest npm release* satisfies tree-sitter@0.25.0 so we + can upgrade without --legacy-peer-deps. + + 2. Vendored upstream drift. vendor/tree-sitter-proto/ is a snapshot of + coder3101/tree-sitter-proto's parser.c. When upstream moves, we want + to know whether we can pick it up. + +Invoked from .github/workflows/tree-sitter-upgrade-readiness.yml daily. +Runs locally too: + + python3 .github/scripts/check-tree-sitter-upgrade-readiness.py + +Outputs Markdown to stdout. Exit 0 when every grammar is upgrade-ready +and the vendored proto is in sync. Exit 1 when blockers remain (the +workflow uses this to open or update a tracking issue). + +No external deps -- stdlib only, so it runs on any vanilla runner. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import sys +import urllib.error +import urllib.request + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +GITNEXUS_DIR = REPO_ROOT / "gitnexus" +VENDOR_PROTO_DIR = GITNEXUS_DIR / "vendor" / "tree-sitter-proto" + +# ── Upgrade target ────────────────────────────────────────────────────── +# The runtime version we want to upgrade TO. Update this when the goal +# changes (e.g. once 0.25 lands and we target 0.26). +TARGET_RUNTIME = "0.25.0" +TARGET_RUNTIME_MAJOR_MINOR = ".".join(TARGET_RUNTIME.split(".")[:2]) + +# Tree-sitter runtime -> (min_abi, max_abi) it can load. Only the current +# and target entries matter; extend when changing TARGET_RUNTIME. +RUNTIME_ABI_RANGES: dict[str, tuple[int, int]] = { + "0.21": (13, 14), + "0.25": (13, 15), +} + +assert TARGET_RUNTIME_MAJOR_MINOR in RUNTIME_ABI_RANGES, ( + f"RUNTIME_ABI_RANGES has no entry for {TARGET_RUNTIME_MAJOR_MINOR!r}. " + f"Add the ABI range after auditing the upstream release notes." +) + +# Grammars we use. Values are the upstream GitHub repos to check for +# unreleased ABI bumps (owner/repo, branch, parser.c path). +GRAMMARS: dict[str, tuple[str, str, str]] = { + "tree-sitter-c": ("tree-sitter/tree-sitter-c", "master", "src/parser.c"), + "tree-sitter-c-sharp": ("tree-sitter/tree-sitter-c-sharp", "master", "src/parser.c"), + "tree-sitter-cpp": ("tree-sitter/tree-sitter-cpp", "master", "src/parser.c"), + "tree-sitter-dart": ("UserNobody14/tree-sitter-dart", "master", "src/parser.c"), + "tree-sitter-go": ("tree-sitter/tree-sitter-go", "master", "src/parser.c"), + "tree-sitter-java": ("tree-sitter/tree-sitter-java", "master", "src/parser.c"), + "tree-sitter-javascript": ("tree-sitter/tree-sitter-javascript", "master", "src/parser.c"), + "tree-sitter-kotlin": ("fwcd/tree-sitter-kotlin", "main", "src/parser.c"), + "tree-sitter-php": ("tree-sitter/tree-sitter-php", "master", "php/src/parser.c"), + "tree-sitter-python": ("tree-sitter/tree-sitter-python", "master", "src/parser.c"), + "tree-sitter-ruby": ("tree-sitter/tree-sitter-ruby", "master", "src/parser.c"), + "tree-sitter-rust": ("tree-sitter/tree-sitter-rust", "master", "src/parser.c"), + "tree-sitter-swift": ("alex-pinkus/tree-sitter-swift", "main", "src/parser.c"), + "tree-sitter-typescript": ("tree-sitter/tree-sitter-typescript", "master", "typescript/src/parser.c"), +} + +UPSTREAM_PROTO_OWNER = "coder3101" +UPSTREAM_PROTO_REPO = "tree-sitter-proto" +UPSTREAM_PROTO_BRANCH = "main" + + +# ── Helpers ───────────────────────────────────────────────────────────── + +def read_current_runtime() -> str: + """Return the tree-sitter runtime version pinned in package.json (e.g. '0.21').""" + pkg = json.loads((GITNEXUS_DIR / "package.json").read_text()) + raw = pkg["dependencies"]["tree-sitter"] + match = re.search(r"(\d+)\.(\d+)", raw) + if not match: + raise SystemExit(f"could not parse tree-sitter version: {raw!r}") + return f"{match.group(1)}.{match.group(2)}" + + +def npm_view_json(pkg: str) -> dict | None: + """Fetch package metadata from the npm registry via HTTPS. + + Uses the registry API directly so we don't depend on the npm CLI + being available (it's a batch file on Windows which complicates + subprocess calls). + """ + url = f"https://registry.npmjs.org/{pkg}/latest" + try: + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=8) as resp: + return json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError): + return None + + +def satisfies_target(peer_range: str | None, target: str) -> bool: + """Check if a semver range like '^0.22.4' or '^0.25.0' satisfies the target. + + Simple heuristic: extract the minimum version from the range and check + if target >= min. For caret ranges (^X.Y.Z), the upper bound is the + next major (for X>0) or next minor (for X==0). We check both bounds. + """ + if peer_range is None: + # No peer dep declared = no constraint = compatible. + return True + match = re.search(r"(\d+)\.(\d+)\.(\d+)", peer_range) + if not match: + return False + min_major, min_minor, min_patch = int(match.group(1)), int(match.group(2)), int(match.group(3)) + + t_match = re.search(r"(\d+)\.(\d+)\.(\d+)", target) + if not t_match: + return False + t_major, t_minor, t_patch = int(t_match.group(1)), int(t_match.group(2)), int(t_match.group(3)) + + # Target must be >= minimum. + target_tuple = (t_major, t_minor, t_patch) + min_tuple = (min_major, min_minor, min_patch) + if target_tuple < min_tuple: + return False + + # For caret ranges with major 0: ^0.X.Y allows [0.X.Y, 0.(X+1).0). + if peer_range.startswith("^") and min_major == 0: + if t_major != 0 or t_minor >= min_minor + 1: + return False + # For caret ranges with major >0: ^X.Y.Z allows [X.Y.Z, (X+1).0.0). + elif peer_range.startswith("^") and min_major > 0: + if t_major >= min_major + 1: + return False + + return True + + +_GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + + +def fetch_text(url: str, timeout: int = 8) -> str | None: + """Fetch a URL and return its text, or None on failure. + + Adds an Authorization header for github.com URLs when GITHUB_TOKEN is + set (raises the rate limit from 60 to 5 000 requests/hour). + """ + headers: dict[str, str] = {} + if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url): + headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}" + try: + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read().decode("utf-8", errors="ignore") + except (urllib.error.URLError, urllib.error.HTTPError): + return None + + +def extract_abi_from_text(text: str) -> int | None: + """Extract LANGUAGE_VERSION from parser.c text.""" + match = re.search(r"#define\s+LANGUAGE_VERSION\s+(\d+)", text[:4096]) + return int(match.group(1)) if match else None + + +def extract_language_version(parser_c: pathlib.Path) -> int | None: + """Return the LANGUAGE_VERSION defined in a parser.c, or None if absent.""" + if not parser_c.is_file(): + return None + with parser_c.open("r", encoding="utf-8", errors="ignore") as fh: + head = fh.read(4096) + return extract_abi_from_text(head) + + +def md_h(text: str, level: int = 2) -> str: + return f"{'#' * level} {text}\n" + + +# ── Main ──────────────────────────────────────────────────────────────── + +def main() -> int: + blockers: dict[str, str] = {} + lines: list[str] = [] + lines.append(md_h("Tree-sitter 0.25 upgrade readiness", 1)) + lines.append("") + + current_runtime = read_current_runtime() + current_abi_range = RUNTIME_ABI_RANGES.get(current_runtime, (0, 0)) + target_abi_range = RUNTIME_ABI_RANGES.get(TARGET_RUNTIME_MAJOR_MINOR, (0, 0)) + + lines.append(f"- Current runtime: `tree-sitter@{current_runtime}.x` (ABI {current_abi_range[0]}..{current_abi_range[1]})") + lines.append(f"- Target runtime: `tree-sitter@{TARGET_RUNTIME}` (ABI {target_abi_range[0]}..{target_abi_range[1]})") + lines.append("") + + # ── Grammar peer-dep compatibility ─────────────────────────────── + lines.append(md_h("Grammar compatibility", 2)) + lines.append("| Grammar | npm latest | Peer dep | Satisfies 0.25? | ABI | Upstream ABI | Status |") + lines.append("|---|---|---|---|---|---|---|") + + ready_count = 0 + total_count = len(GRAMMARS) + + for name, (upstream_repo, upstream_branch, parser_path) in sorted(GRAMMARS.items()): + # Fetch latest npm metadata. + info = npm_view_json(name) + fetch_failed = info is None + npm_version = "?" + peer_range = None + peer_optional = True + if info: + npm_version = info.get("version", "?") + peers = info.get("peerDependencies") or {} + peer_range = peers.get("tree-sitter") + meta = info.get("peerDependenciesMeta") or {} + ts_meta = meta.get("tree-sitter") or {} + peer_optional = ts_meta.get("optional", False) if peer_range else True + + if fetch_failed: + peer_display = "? (fetch failed)" + compatible = False + else: + peer_display = peer_range or "none" + if peer_range and not peer_optional: + peer_display += " (required)" + compatible = satisfies_target(peer_range, TARGET_RUNTIME) + + # Check installed ABI using the same parser_path from GRAMMARS. + installed_parser = GITNEXUS_DIR / "node_modules" / name / parser_path + if not installed_parser.is_file(): + # Fallback to default location. + installed_parser = GITNEXUS_DIR / "node_modules" / name / "src" / "parser.c" + installed_abi = extract_language_version(installed_parser) + abi_display = str(installed_abi) if installed_abi else "?" + + # Check upstream (main/master branch) ABI for unreleased work. + upstream_url = ( + f"https://raw.githubusercontent.com/{upstream_repo}/" + f"{upstream_branch}/{parser_path}" + ) + upstream_text = fetch_text(upstream_url) + upstream_abi = extract_abi_from_text(upstream_text) if upstream_text else None + upstream_abi_display = str(upstream_abi) if upstream_abi else "?" + + # Determine status. + if fetch_failed: + status = "Unknown (fetch failed)" + blockers[name] = f"`{name}`: npm registry fetch failed — could not verify peer dep" + elif compatible: + status = "Ready" + ready_count += 1 + elif upstream_abi and upstream_abi >= 15: + status = "Unreleased (ABI 15 on main)" + blockers[name] = f"`{name}`: ABI 15 on `{upstream_repo}` main but not published to npm" + else: + status = "Blocking" + blockers[name] = f"`{name}@{npm_version}`: peer `{peer_display}` incompatible with 0.25" + + # Also check upstream package.json for relaxed peer dep. + if not compatible and not fetch_failed: + upstream_pkg_url = ( + f"https://raw.githubusercontent.com/{upstream_repo}/" + f"{upstream_branch}/package.json" + ) + upstream_pkg_text = fetch_text(upstream_pkg_url) + if upstream_pkg_text: + try: + upstream_pkg = json.loads(upstream_pkg_text) + upstream_peer = (upstream_pkg.get("peerDependencies") or {}).get("tree-sitter") + if upstream_peer and satisfies_target(upstream_peer, TARGET_RUNTIME): + status = "Unreleased (peer relaxed on main)" + blockers[name] = f"`{name}`: peer dep relaxed on `{upstream_repo}` main but not published to npm" + except json.JSONDecodeError: + pass + + compat_icon = "Yes" if compatible else "**No**" + lines.append( + f"| `{name}` | {npm_version} | {peer_display} | {compat_icon} | {abi_display} | {upstream_abi_display} | {status} |" + ) + + lines.append("") + lines.append(f"**{ready_count}/{total_count}** grammars ready for `tree-sitter@{TARGET_RUNTIME}`.") + lines.append("") + + # ── Vendored proto drift ───────────────────────────────────────── + lines.append(md_h("Vendored tree-sitter-proto", 2)) + vendored_abi = extract_language_version(VENDOR_PROTO_DIR / "src" / "parser.c") + + upstream_proto_url = ( + f"https://raw.githubusercontent.com/{UPSTREAM_PROTO_OWNER}/" + f"{UPSTREAM_PROTO_REPO}/{UPSTREAM_PROTO_BRANCH}/src/parser.c" + ) + upstream_proto_text = fetch_text(upstream_proto_url) + upstream_proto_abi = extract_abi_from_text(upstream_proto_text) if upstream_proto_text else None + + sha_url = ( + f"https://api.github.com/repos/{UPSTREAM_PROTO_OWNER}/" + f"{UPSTREAM_PROTO_REPO}/commits/{UPSTREAM_PROTO_BRANCH}" + ) + sha_text = fetch_text(sha_url) + upstream_sha = "?" + if sha_text: + try: + upstream_sha = json.loads(sha_text).get("sha", "?")[:12] + except json.JSONDecodeError: + pass + + local_proto_path = VENDOR_PROTO_DIR / "src" / "parser.c" + local_proto_text = local_proto_path.read_text(encoding="utf-8", errors="ignore") if local_proto_path.is_file() else "" + in_sync = bool( + upstream_proto_text + and local_proto_text.replace("\r\n", "\n") + == upstream_proto_text.replace("\r\n", "\n") + ) + + lines.append(f"- Upstream: `{UPSTREAM_PROTO_OWNER}/{UPSTREAM_PROTO_REPO}@{UPSTREAM_PROTO_BRANCH}` (HEAD `{upstream_sha}`)") + lines.append(f"- Upstream ABI: **{upstream_proto_abi}**") + lines.append(f"- Vendored ABI: **{vendored_abi}**") + lines.append(f"- In sync: {'yes' if in_sync else 'no — upstream has diverged'}") + + if upstream_proto_abi and vendored_abi and upstream_proto_abi > vendored_abi: + can_upgrade = upstream_proto_abi <= target_abi_range[1] + lines.append(f"- Upstream ABI {upstream_proto_abi} {'is' if can_upgrade else 'is NOT'} within target runtime range ({target_abi_range[0]}..{target_abi_range[1]})") + if can_upgrade: + lines.append(f"- **Action:** after upgrading to tree-sitter@{TARGET_RUNTIME}, regenerate vendored parser.c from upstream `{upstream_sha}`") + else: + lines.append(f"- **Action:** wait for runtime upgrade beyond {TARGET_RUNTIME} that supports ABI {upstream_proto_abi}") + blockers["vendored-proto-abi"] = f"vendored tree-sitter-proto: upstream ABI {upstream_proto_abi} outside target range" + elif not in_sync: + lines.append("- **Action:** review upstream changes; vendored copy may need updating") + blockers["vendored-proto-sync"] = "vendored tree-sitter-proto: out of sync with upstream" + + # ── Summary ────────────────────────────────────────────────────── + lines.append("") + lines.append(md_h("Summary", 2)) + if blockers: + lines.append(f"**{len(blockers)} blocker(s) remaining:**\n") + for b in blockers.values(): + lines.append(f"- {b}") + lines.append("") + lines.append("Upgrade to `tree-sitter@0.25` is **blocked**.") + else: + lines.append("All grammars are compatible. Upgrade to `tree-sitter@0.25` is **ready**.") + + print("\n".join(lines)) + return 1 if blockers else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/check-workflow-concurrency.py b/.github/scripts/check-workflow-concurrency.py new file mode 100644 index 000000000..300cc7f4b --- /dev/null +++ b/.github/scripts/check-workflow-concurrency.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Enforce the GitHub Actions concurrency convention. + +See CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention" for the rules. + +Invoked from .github/workflows/ci-quality.yml. Runs locally too: + python3 .github/scripts/check-workflow-concurrency.py .github/workflows + +Rules: + 1. Every entry-point (non-reusable) workflow declares a top-level + `concurrency:` block. + 2. Reusable workflows (on: workflow_call ONLY) do NOT declare one. + 3. The `concurrency.group` expression MUST reference either + `${{ github.workflow }}` or a literal `CI-` prefix (the documented + ci.yml reusable-workflow-safe exception). This is checked by substring + containment rather than prefix match because ci.yml's group is a + conditional expression that resolves to a `CI-…` literal at runtime. + +We deliberately do not use a YAML library — keeps the script dependency-free +on any vanilla runner. `on:` block parsing is line-based and handles both the +flat (`on: workflow_call`) and mapping (`on:\n workflow_call:`) forms. +""" + +from __future__ import annotations + +import pathlib +import re +import sys + + +REQUIRED_TOKENS = ("${{ github.workflow }}", "CI-") + + +def is_reusable(lines: list[str]) -> bool: + """Return True iff the workflow's `on:` block names only `workflow_call`.""" + in_on = False + on_indent: int | None = None + keys: list[str] = [] + + for raw in lines: + # Skip blank lines and comments + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + + indent = len(raw) - len(raw.lstrip(" ")) + + if not in_on: + if raw.startswith("on:"): + remainder = raw[len("on:"):].strip() + if not remainder: + # `on:` followed by indented mapping on next lines + in_on = True + on_indent = indent + continue + if remainder.startswith("[") and remainder.endswith("]"): + # Flow-style list: on: [workflow_call] + items = [ + item.strip() for item in remainder.strip("[]").split(",") + ] + return items == ["workflow_call"] + # Scalar form: on: workflow_call (or a single other event) + return remainder == "workflow_call" + continue + + # Inside the `on:` block; stop when indentation returns to <= on_indent + if on_indent is not None and indent <= on_indent: + break + + # Only consider keys at on_indent + indentation step (anything deeper + # is nested config like `types:`) + if ":" not in stripped: + continue + # Heuristic: first-level event keys are those with indent == on_indent + 2 + # (the canonical step for a 2-space YAML doc). We collect all first-level + # keys by tracking the smallest indent seen inside the block. + keys.append((indent, stripped.split(":", 1)[0].strip())) + + if not keys: + return False + + # Take only the outermost-indented keys as the event list + min_indent = min(i for i, _ in keys) + events = [name for i, name in keys if i == min_indent] + return events == ["workflow_call"] + + +CONCURRENCY_RE = re.compile(r"^concurrency:\s*$") +GROUP_RE = re.compile(r"^\s+group:\s*(.+?)\s*$") + + +def extract_group_key(lines: list[str]) -> str | None: + """Return the `group:` value of the top-level `concurrency:` block, or None.""" + for idx, raw in enumerate(lines): + if CONCURRENCY_RE.match(raw): + # Scan forward until we leave the concurrency block (next top-level key + # is at column 0 and ends with `:`). + for follow in lines[idx + 1:]: + if follow and not follow.startswith(" ") and follow.rstrip().endswith(":"): + break + m = GROUP_RE.match(follow) + if m: + return m.group(1).strip().strip("'").strip('"') + break + return None + + +def has_top_level_concurrency(lines: list[str]) -> bool: + return any(CONCURRENCY_RE.match(raw) for raw in lines) + + +def check(workflows_dir: pathlib.Path) -> int: + fail = 0 + files = sorted( + list(workflows_dir.glob("*.yml")) + list(workflows_dir.glob("*.yaml")) + ) + for path in files: + lines = path.read_text(encoding="utf-8").splitlines() + reusable = is_reusable(lines) + has_conc = has_top_level_concurrency(lines) + + if reusable: + if has_conc: + print( + f"::error file={path}::Reusable workflow (on: workflow_call) " + "must NOT declare its own concurrency block — it inherits " + "from the caller. See CONTRIBUTING.md -> GitHub Actions — " + "Concurrency Convention." + ) + fail = 1 + continue + + if not has_conc: + print( + f"::error file={path}::Missing top-level concurrency block. " + "See CONTRIBUTING.md -> GitHub Actions — Concurrency Convention." + ) + fail = 1 + continue + + group = extract_group_key(lines) + if group is None: + print( + f"::error file={path}::concurrency block is missing a " + "`group:` key." + ) + fail = 1 + continue + + if not any(token in group for token in REQUIRED_TOKENS): + print( + f"::error file={path}::concurrency.group `{group}` must " + f"reference one of {REQUIRED_TOKENS}. See CONTRIBUTING.md -> " + "GitHub Actions — Concurrency Convention." + ) + fail = 1 + + return fail + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print(f"usage: {argv[0]} ", file=sys.stderr) + return 2 + workflows_dir = pathlib.Path(argv[1]) + if not workflows_dir.is_dir(): + print(f"not a directory: {workflows_dir}", file=sys.stderr) + return 2 + return check(workflows_dir) + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/.github/workflows/ci-e2e.yml b/.github/workflows/ci-e2e.yml index 675d9b382..a017af628 100644 --- a/.github/workflows/ci-e2e.yml +++ b/.github/workflows/ci-e2e.yml @@ -11,8 +11,8 @@ jobs: outputs: web_changed: ${{ steps.filter.outputs.web }} steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v3 id: filter with: filters: | @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup-gitnexus-web @@ -74,7 +74,7 @@ jobs: - name: Upload test results if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-results path: | diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 9a5b9fedd..5a0da5fd1 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -8,8 +8,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 cache: npm @@ -21,8 +21,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 cache: npm @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup-gitnexus - run: npx tsc --noEmit working-directory: gitnexus @@ -43,7 +43,30 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup-gitnexus-web - run: npx tsc -b --noEmit working-directory: gitnexus-web + + # Enforces the convention documented in CONTRIBUTING.md → "GitHub Actions — + # Concurrency Convention": + # 1. Every entry-point (non-reusable) workflow declares a top-level + # `concurrency:` block. + # 2. Reusable workflows (`on: workflow_call` only) do NOT declare one — + # they inherit concurrency from the caller. + # 3. The concurrency group key starts with `${{ github.workflow }}` or + # the literal `CI-` prefix (the documented ci.yml exception for + # reusable-workflow-safe grouping). + # Reusability is detected by parsing each workflow's `on:` block, not an + # allowlist, so new reusable workflows never produce false positives. + workflow-convention: + name: Workflow concurrency convention + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Validate workflow concurrency convention + shell: bash + run: | + set -euo pipefail + python3 .github/scripts/check-workflow-concurrency.py .github/workflows diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 2a6e5cea8..ab8d8362c 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -14,6 +14,16 @@ permissions: contents: read # needed for sparse checkout of vitest.config.ts pull-requests: write # needed to post sticky PR comment +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". +# Serialize sticky-comment writes per PR so two rapid CI completions don't race. +# Internal PRs surface in `pull_requests[0].number`. Fork PRs leave that array empty, +# so we fall back to `/`, which is stable across +# reruns and subsequent pushes for the same fork PR (unlike `workflow_run.id` which +# is unique per run and therefore does not serialize anything). +concurrency: + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }} + cancel-in-progress: false + jobs: pr-report: name: PR Report @@ -26,7 +36,7 @@ jobs: steps: # ── Download artifacts from the CI run ──────────────────────── - name: Download artifacts - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | const fs = require('fs'); @@ -113,7 +123,7 @@ jobs: - name: Checkout (for vitest config) if: steps.meta.outputs.skip != 'true' - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: sparse-checkout: gitnexus/vitest.config.ts sparse-checkout-cone-mode: false @@ -122,7 +132,7 @@ jobs: - name: Fetch base branch coverage if: steps.meta.outputs.skip != 'true' id: base-coverage - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | const fs = require('fs'); @@ -406,7 +416,7 @@ jobs: - name: Comment on PR if: steps.meta.outputs.skip != 'true' - uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2 + uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v2 with: header: ci-report number: ${{ steps.meta.outputs.pr_number }} diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 6a20032c6..27eb75383 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 25 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup-gitnexus with: build: 'true' @@ -43,7 +43,7 @@ jobs: - name: Upload test reports if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-reports path: | @@ -63,7 +63,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 25 steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: ./.github/actions/setup-gitnexus with: build: 'true' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba7184ace..cf0c6d5c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,9 +9,20 @@ on: paths-ignore: ['**.md', 'docs/**', 'LICENSE'] workflow_call: +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". +# Hardcoded `CI-` prefix (not `${{ github.workflow }}`) because this workflow is +# invoked as a reusable workflow from publish.yml and release-candidate.yml. In +# called-workflow context `github.workflow` evaluation is ambiguous across GitHub +# Actions versions, and a prefix that could resolve to the caller's name would +# share a concurrency group with the caller → deadlock. A literal prefix is +# immune. Direct `push`/`pull_request` invocations use `CI-`; invocations +# from a reusable-workflow caller fall into a per-run-unique group that never +# serializes with the caller. +# cancel-in-progress is event-aware: cancel superseded PR runs, queue every other +# event (push to main, workflow_call from publish.yml, etc.). concurrency: - group: ci-${{ github.ref }} - cancel-in-progress: true + group: ${{ (github.event_name == 'pull_request' || github.event_name == 'push') && format('CI-{0}', github.ref) || format('CI-nested-{0}', github.run_id) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} # ── Reusable workflow orchestration ───────────────────────────────── # Each concern lives in its own workflow file for maintainability: @@ -74,7 +85,7 @@ jobs: cp pr-meta/e2e_result pr-meta/e2e-result - name: Upload PR metadata - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: pr-meta path: pr-meta/ @@ -96,9 +107,9 @@ jobs: TESTS: ${{ needs.tests.result }} E2E: ${{ needs.e2e.result }} run: | - echo "Quality: $QUALITY" - echo "Tests: $TESTS" - echo "E2E: $E2E" + echo "Quality: $QUALITY" + echo "Tests: $TESTS" + echo "E2E: $E2E" if [[ "$QUALITY" != "success" ]] || [[ "$TESTS" != "success" ]]; then echo "::error::Quality or test jobs failed" diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 82a65f844..e5642cb3e 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -16,9 +16,10 @@ on: issue_comment: types: [created] +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". # Serialize per-PR to avoid racing review comments. concurrency: - group: claude-review-${{ github.event.issue.number || github.event.pull_request.number }} + group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number }} cancel-in-progress: false jobs: @@ -56,7 +57,7 @@ jobs: # For issue_comment triggers, resolve the PR number, head SHA, and fork repo - name: Resolve PR context id: pr - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | let pr; @@ -76,7 +77,7 @@ jobs: core.setOutput('branch', pr.head.ref); - name: Checkout PR head - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ${{ steps.pr.outputs.repo }} ref: ${{ steps.pr.outputs.sha }} diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 407b2fcf8..553d3ab0d 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -10,9 +10,10 @@ on: pull_request_review: types: [submitted] +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". # Serialize per-PR/issue to avoid racing comments. concurrency: - group: claude-code-${{ github.event.issue.number || github.event.pull_request.number || github.event.issue.id }} + group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number || github.event.issue.id }} cancel-in-progress: false jobs: @@ -58,7 +59,7 @@ jobs: # For PR-related triggers, resolve the fork repo so we can checkout correctly. - name: Resolve PR context id: pr - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | // Determine if this event is PR-related @@ -90,7 +91,7 @@ jobs: core.setOutput('branch', pr.head.ref); - name: Checkout repository - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: repository: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.repo || github.repository }} ref: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.sha || '' }} diff --git a/.github/workflows/pr-description-check.yml b/.github/workflows/pr-description-check.yml index d722dfff3..cec32de0f 100644 --- a/.github/workflows/pr-description-check.yml +++ b/.github/workflows/pr-description-check.yml @@ -8,8 +8,9 @@ on: permissions: pull-requests: write +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". concurrency: - group: pr-desc-${{ github.event.pull_request.number }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: @@ -18,7 +19,7 @@ jobs: timeout-minutes: 5 steps: - name: Check PR description quality - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const MIN_BODY_LENGTH = 50; diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml new file mode 100644 index 000000000..3c0c52725 --- /dev/null +++ b/.github/workflows/pr-labeler.yml @@ -0,0 +1,113 @@ +name: PR Conventional Labeler + +# Two workflows in one file with different triggers, matched to the minimum +# privilege each needs: +# +# validate-title (on: pull_request) +# Fork-safe. Runs with the PR-head's read-only GITHUB_TOKEN. Uses +# `amannn/action-semantic-pull-request` to fail the check when the PR +# title doesn't follow the conventional-commit format. Because the +# action only reads the event payload, no fork-controlled code runs. +# +# autolabel (on: pull_request_target) +# Needs `pull-requests: write` to apply labels, so must be +# pull_request_target. Uses `release-drafter/release-drafter` with +# `dry-run: true` to only run the autolabeler against the +# `.github/release-drafter.yml` config from the BASE ref (release- +# drafter reads the config from the repository's default branch, NOT +# the PR head — verify with `gh api repos/release-drafter/release-drafter/contents/...` +# or a fork-test PR before merging if the repo is high-value). +# `sync-labels: true` in the config removes managed autolabels that no +# longer match (e.g. when `!` or `BREAKING CHANGE:` is dropped). +# +# Title format: [(scope)][!]: +# Allowed types: feat, fix, perf, refactor, docs, test, ci, build, chore, revert, deps +# Trailing `!` on the type marks a breaking change. +# See CONTRIBUTING.md → "Pull request titles". + +on: + pull_request: + # Title-only changes fire `edited`. `opened` and `reopened` cover creation. + # `synchronize` (push to the PR branch) is intentionally excluded — titles + # don't change on push, so it only wastes CI minutes and broadens the + # privileged-token exposure window on the autolabel job. + types: [opened, edited, reopened] + pull_request_target: + types: [opened, edited, reopened] + +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". +# Include `github.event_name` so `pull_request` (validate-title) and +# `pull_request_target` (autolabel) runs for the same PR do NOT share a slot +# and therefore cannot cancel each other — a cancelled required-check would +# permanently block merge until the next title edit. +# Within each trigger the latest title edit still supersedes the prior run. +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + validate-title: + # Fork-safe job — only runs on `pull_request` (not `pull_request_target`). + # Token is read-only; writes a commit status that branch protection can + # require before merge. + name: Validate PR title + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: read + steps: + # Pinned to v6.1.1. Verify SHA via: + # gh api repos/amannn/action-semantic-pull-request/git/refs/tags/v6.1.1 + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + types: | + feat + fix + perf + refactor + docs + test + ci + build + chore + revert + deps + requireScope: false + # Subject must be non-empty. We DO allow capitalized proper nouns + # (MCP, GitHub, API, etc.) — the old `^(?![A-Z]).+$` pattern + # rejected legitimate titles like `fix: MCP tool schema`. + subjectPattern: ^\S.{2,}$ + subjectPatternError: | + The subject "{subject}" in PR title "{title}" is invalid. + Subjects must be at least 3 characters and must not start with whitespace. + wip: false + + autolabel: + # Privileged job — runs only on `pull_request_target` so it can write labels. + # Never checks out fork code, never executes fork-controlled input; only + # reads the PR metadata (title, body, labels) and calls the GitHub API. + name: Apply conventional label + if: github.event_name == 'pull_request_target' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + # `contents: read` is required — release-drafter's context.config() reads + # `.github/release-drafter.yml` from the repo's default branch via the + # repo-contents API. Without it the job silently 403s and no labels are + # applied. Job-level permissions nullify all unlisted scopes, so an + # explicit grant is necessary here. + contents: read + pull-requests: write + steps: + # Pinned to v7.2.0. Verify SHA via: + # gh api repos/release-drafter/release-drafter/git/refs/tags/v7.2.0 + # v7 removed `disable-releaser`; use `dry-run: true` to only autolabel. + - uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0 + with: + config-name: release-drafter.yml + dry-run: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8a0ee6ebc..3d883425a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -7,13 +7,22 @@ on: # No workflow-level permissions — scoped per job below. +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". +# Tag refs are unique per release, so distinct tags run in parallel. Re-pushes of the +# same tag serialize. cancel-in-progress: false — never cancel a publish mid-flight. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + jobs: ci: uses: ./.github/workflows/ci.yml permissions: contents: read actions: read - pull-requests: write + # No pull-requests:write — `ci.yml`'s save-pr-meta job is gated on + # `github.event_name == 'pull_request'`, so it never runs during a + # tag-triggered publish. Least-privilege for release-critical paths. publish: needs: ci @@ -23,8 +32,8 @@ jobs: contents: write id-token: write steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 registry-url: https://registry.npmjs.org @@ -82,7 +91,7 @@ jobs: fi - name: Create GitHub Release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 with: body_path: ${{ steps.changelog.outputs.fallback == 'false' && '/tmp/release-notes.md' || '' }} generate_release_notes: ${{ steps.changelog.outputs.fallback == 'true' }} diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml new file mode 100644 index 000000000..d4db75db0 --- /dev/null +++ b/.github/workflows/release-candidate.yml @@ -0,0 +1,366 @@ +name: Release Candidate + +on: + # Publish a release-candidate build whenever a merge/commit lands on main. + # Docs/README-only changes are filtered out so prose updates don't + # cut a release. + push: + branches: [main] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE' + workflow_dispatch: + inputs: + bump: + description: >- + Cycle policy. 'auto' (default) continues the active rc cycle on + this branch if there is one, otherwise bumps patch from latest. + Choose 'patch' / 'minor' / 'major' to explicitly start or reset + an rc cycle. + required: false + default: 'auto' + type: choice + options: + - auto + - patch + - minor + - major + force: + description: 'Publish even when HEAD already has an rc marker' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' + +# No workflow-level permissions — scoped per job below. +permissions: {} + +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". +# Serialize all runs on the same ref (push + workflow_dispatch) to prevent two publishes +# racing on the rc counter. cancel-in-progress: false — the earlier merge publishes first. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + # ── Skip when HEAD already has an rc marker (retry / duplicate dispatch) ── + # The marker is a lightweight tag `rc/` pushed *before* `npm + # publish`, so a failed publish leaves the marker in place and the guard + # refuses to re-publish. Recovery path after a partial failure: + # git push --delete origin rc/ v + # then redispatch with force=true. + guard: + name: Check if release candidate should run + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + should_run: ${{ steps.decide.outputs.should_run }} + head_sha: ${{ steps.decide.outputs.head_sha }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Decide + id: decide + shell: bash + env: + FORCE: ${{ inputs.force }} + BUMP_INPUT: ${{ inputs.bump }} + EVENT_NAME: ${{ github.event_name }} + run: | + set -euo pipefail + HEAD_SHA=$(git rev-parse HEAD) + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + if [ "$FORCE" = "true" ]; then + echo "Force flag set — running regardless of marker tag." + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # An explicit cycle reset on dispatch (bump != auto) also bypasses + # the dedup guard — the maintainer is deliberately asking for a + # new rc from the same commit. + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && [ -n "${BUMP_INPUT:-}" ] \ + && [ "${BUMP_INPUT:-auto}" != "auto" ]; then + echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup." + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Dedup: is there already an rc/ marker pointing at HEAD? + MARKER="rc/${HEAD_SHA}" + if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then + echo "HEAD already has marker $MARKER — skipping." + echo "should_run=false" >> "$GITHUB_OUTPUT" + else + echo "No marker on HEAD — proceeding." + echo "should_run=true" >> "$GITHUB_OUTPUT" + fi + + # ── Reuse the stable CI workflow ───────────────────────────────────── + ci: + needs: guard + if: needs.guard.outputs.should_run == 'true' + uses: ./.github/workflows/ci.yml + permissions: + contents: read + secrets: inherit + + # ── Publish the rc build to npm + create GitHub prerelease ─────────── + publish: + name: Publish release candidate to npm + needs: [guard, ci] + if: needs.guard.outputs.should_run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: write # push rc tag + marker + id-token: write # npm provenance + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + fetch-tags: true + + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 20 + registry-url: https://registry.npmjs.org + cache: npm + cache-dependency-path: gitnexus/package-lock.json + + - name: Build gitnexus-shared + run: npm install && npm run build + working-directory: gitnexus-shared + + - name: Install gitnexus dependencies + run: npm ci + working-directory: gitnexus + + - name: Resolve rc version + id: version + shell: bash + working-directory: gitnexus + env: + BUMP_INPUT: ${{ inputs.bump }} + EVENT_NAME: ${{ github.event_name }} + PKG_NAME: gitnexus + run: | + set -euo pipefail + + # 1. Current published `latest` — the floor for any new rc base. + # Only E404 ("never published") falls back to package.json; any + # other error (network, auth, malformed response) fails fast. + NPM_STDERR_LATEST="$(mktemp)" + if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then + : + else + if grep -q 'E404' "$NPM_STDERR_LATEST"; then + CURRENT_LATEST="$(node -p "require('./package.json').version")" + echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST" + else + echo "::error::npm registry unreachable for 'view version':" >&2 + cat "$NPM_STDERR_LATEST" >&2 + rm -f "$NPM_STDERR_LATEST" + exit 1 + fi + fi + rm -f "$NPM_STDERR_LATEST" + CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}" + + # 2. Full version list — needed for the counter and for active-cycle + # inference. Same E404-only fallback. + NPM_STDERR_VERSIONS="$(mktemp)" + if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then + : + else + if grep -q 'E404' "$NPM_STDERR_VERSIONS"; then + VERSIONS_JSON='[]' + echo "No published versions for $PKG_NAME yet (E404)." + else + echo "::error::npm registry unreachable for 'view versions':" >&2 + cat "$NPM_STDERR_VERSIONS" >&2 + rm -f "$NPM_STDERR_VERSIONS" + exit 1 + fi + fi + rm -f "$NPM_STDERR_VERSIONS" + + # 3. Base selection. + # - workflow_dispatch + bump ∈ {patch,minor,major} → explicit cycle + # reset from latest. + # - Everything else (push, or dispatch with bump=auto) → continue + # the highest active rc base > latest if one exists; else + # default to patch from latest. + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && [ -n "${BUMP_INPUT:-}" ] \ + && [ "${BUMP_INPUT:-auto}" != "auto" ]; then + BASE="$(npx --yes -p semver@7 semver -i "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")" + echo "Explicit bump=$BUMP_INPUT → BASE=$BASE" + else + cat > /tmp/active_base.mjs <<'NODESCRIPT' + const latest = process.env.LATEST; + let v; + try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } + if (!Array.isArray(v)) v = [v]; + const parse = s => s.split(".").map(n => parseInt(n, 10)); + const gt = (a, b) => { + const [A, B] = [parse(a), parse(b)]; + for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i]; + return false; + }; + const bases = new Set(); + for (const s of v) { + const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s); + if (m && gt(m[1], latest)) bases.add(m[1]); + } + if (!bases.size) { process.stdout.write(""); process.exit(0); } + const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1); + process.stdout.write(sorted[sorted.length - 1]); + NODESCRIPT + ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)" + if [ -n "$ACTIVE_BASE" ]; then + BASE="$ACTIVE_BASE" + echo "Continuing active rc cycle → BASE=$BASE" + else + BASE="$(npx --yes -p semver@7 semver -i patch "$CURRENT_LATEST_CLEAN")" + echo "No active rc cycle → patch bump from latest → BASE=$BASE" + fi + fi + + # 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1. + cat > /tmp/next_rc.mjs <<'NODESCRIPT' + const base = process.env.BASE; + const prefix = base + "-rc."; + let v; + try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } + if (!Array.isArray(v)) v = [v]; + const ns = v + .filter(s => typeof s === "string" && s.startsWith(prefix)) + .map(s => parseInt(s.slice(prefix.length), 10)) + .filter(n => Number.isInteger(n) && n >= 0); + process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1)); + NODESCRIPT + NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)" + RC_VERSION="${BASE}-rc.${NEXT_N}" + echo "Computed rc: $RC_VERSION" + + # 5. Defensive: if the exact version already exists on the registry + # (e.g., race with another run), abort before re-publishing. + # Same E404-only pattern used above — a transient network + # failure must fail loudly, not pretend the version is missing. + NPM_STDERR_EXISTS="$(mktemp)" + if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then + rm -f "$NPM_STDERR_EXISTS" + echo "::error::Version $RC_VERSION already exists on npm — aborting." + exit 1 + else + if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then + rm -f "$NPM_STDERR_EXISTS" + # Version doesn't exist — safe to proceed. + else + echo "::error::npm registry unreachable for existence check:" >&2 + cat "$NPM_STDERR_EXISTS" >&2 + rm -f "$NPM_STDERR_EXISTS" + exit 1 + fi + fi + + echo "base=$BASE" >> "$GITHUB_OUTPUT" + echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT" + echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT" + + - name: Apply rc version in-CI + shell: bash + working-directory: gitnexus + run: | + set -euo pipefail + npm version "${{ steps.version.outputs.rc_version }}" \ + --no-git-tag-version --allow-same-version + + - name: Build gitnexus + run: npm run build + working-directory: gitnexus + + - name: Dry-run publish + run: npm publish --dry-run --tag rc + working-directory: gitnexus + + # ── Acquire the "rc lock" BEFORE publishing (fixes idempotency) ───── + # We create two tags and push them atomically: + # v → annotated tag on a detached release commit + # whose tree contains the rewritten package.json + # (so the tag's source matches the npm tarball) + # rc/ → lightweight tag on HEAD; the guard's dedup key + # If this push fails, nothing is published — safe. + # If this push succeeds but npm publish fails, the marker stays on + # the remote and blocks retries until an operator manually cleans up. + - name: Create and push rc tags + id: reltag + shell: bash + working-directory: gitnexus + env: + RC_VERSION: ${{ steps.version.outputs.rc_version }} + HEAD_SHA: ${{ needs.guard.outputs.head_sha }} + run: | + set -euo pipefail + VTAG="v${RC_VERSION}" + MARKER="rc/${HEAD_SHA}" + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + + # Detached release commit with the version bump — keeps `main` + # pristine but gives the v-tag a tree that matches the published + # package contents exactly (fixes release-integrity gap). + git add package.json package-lock.json 2>/dev/null || git add package.json + git commit -m "release: ${VTAG}" --allow-empty + RELEASE_SHA="$(git rev-parse HEAD)" + echo "Detached release commit: $RELEASE_SHA" + + # Annotated release tag on the release commit. + git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG" + # Lightweight marker on the user-visible HEAD for the guard. + git tag "$MARKER" "$HEAD_SHA" + + # Atomic push of both refs. If either would clobber an existing + # remote ref, the push fails and we stop before npm publish. + git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" + + echo "vtag=$VTAG" >> "$GITHUB_OUTPUT" + echo "marker=$MARKER" >> "$GITHUB_OUTPUT" + echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT" + + - name: Publish to npm (rc dist-tag) + run: npm publish --provenance --access public --tag rc + working-directory: gitnexus + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + - name: Create GitHub prerelease + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 + with: + tag_name: ${{ steps.reltag.outputs.vtag }} + name: Release Candidate ${{ steps.reltag.outputs.vtag }} + prerelease: true + make_latest: 'false' + generate_release_notes: true + body: | + Automated release candidate build from `main`. + + **npm:** `npm install gitnexus@rc` + **Version:** `${{ steps.version.outputs.rc_version }}` + **Target base:** `${{ steps.version.outputs.base }}` (rc #${{ steps.version.outputs.rc_n }}) + **Source commit (main):** ${{ needs.guard.outputs.head_sha }} + **Release commit (versioned tree):** ${{ steps.reltag.outputs.release_sha }} + + Release candidates are pre-stable builds intended for early testing. + Stable releases remain on the `latest` dist-tag. diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml new file mode 100644 index 000000000..74ce72a27 --- /dev/null +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -0,0 +1,185 @@ +name: Tree-sitter Upgrade Readiness + +# Monitors readiness for upgrading tree-sitter to 0.25.x. Tracks: +# 1. Peer-dep compatibility — can each grammar install cleanly with +# tree-sitter@0.25.0 without --legacy-peer-deps? +# 2. Vendored proto drift — has coder3101/tree-sitter-proto moved +# ahead of our vendored snapshot? +# See .github/scripts/check-tree-sitter-upgrade-readiness.py for the logic. +# +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". + +on: + schedule: + # Daily at 09:00 UTC. Matches Dependabot's daily cadence so drift + # and dep PRs surface together. + - cron: '0 9 * * *' + workflow_dispatch: + pull_request: + paths: + - '.github/scripts/check-tree-sitter-upgrade-readiness.py' + - '.github/workflows/tree-sitter-upgrade-readiness.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + readiness: + name: Check upgrade readiness + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # Needed to open/update the tracking issue on scheduled runs. + issues: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: ./.github/actions/setup-gitnexus + with: + build: 'false' + + - name: Run upgrade readiness check + id: readiness + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + python3 .github/scripts/check-tree-sitter-upgrade-readiness.py > drift-report.md + code=$? + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + { + echo 'report<> "$GITHUB_OUTPUT" + echo "=== Report ===" + cat drift-report.md + + # On PR runs, the script validates that it runs correctly. Blockers + # are informational — the scheduled run opens a tracking issue. + - name: Annotate PR with readiness status + if: github.event_name == 'pull_request' && steps.readiness.outputs.exit_code != '0' + run: | + echo "::warning::Tree-sitter 0.25 upgrade has blockers. See job output for the full readiness report." + + - name: Upsert tracking issue on scheduled runs + if: > + github.event_name == 'schedule' && + steps.readiness.outputs.exit_code != '0' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + REPORT: ${{ steps.readiness.outputs.report }} + with: + script: | + const title = 'Tree-sitter 0.25 upgrade readiness'; + const report = process.env.REPORT; + const body = report + '\n\n' + + 'Generated daily by `.github/workflows/tree-sitter-upgrade-readiness.yml`. ' + + 'Closes automatically when all blockers are resolved.'; + const { data: open } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'tree-sitter-drift', + per_page: 10, + }); + const existing = open.find(i => i.title === title); + if (existing) { + // Extract ready/total count for the changelog comment. + const readyMatch = report.match(/\*\*(\d+)\/(\d+)\*\* grammars ready/); + const blockerMatch = report.match(/\*\*(\d+) blocker/); + const ready = readyMatch ? readyMatch[1] : '?'; + const total = readyMatch ? readyMatch[2] : '?'; + const blockers = blockerMatch ? blockerMatch[1] : '?'; + + // Find grammars whose status changed by diffing the old and + // new table rows. Each row looks like: + // | `tree-sitter-foo` | ... | Ready | + // | `tree-sitter-foo` | ... | Blocking | + const parseRows = (md) => { + const map = {}; + for (const m of md.matchAll(/\| `(tree-sitter-[^`]+)` \|.*?\| (\S+(?:\s\S+)*?) \|$/gm)) { + map[m[1]] = m[2].trim(); + } + return map; + }; + const oldRows = parseRows(existing.body || ''); + const newRows = parseRows(report); + const changes = []; + for (const [name, newStatus] of Object.entries(newRows)) { + const oldStatus = oldRows[name]; + if (oldStatus && oldStatus !== newStatus) { + changes.push(`\`${name}\`: ${oldStatus} → ${newStatus}`); + } + } + + const today = new Date().toISOString().slice(0, 10); + let comment = `**${today}:** ${ready}/${total} ready. ${blockers} blocker(s) remaining.`; + if (changes.length > 0) { + comment += '\n\nChanges:\n' + changes.map(c => `- ${c}`).join('\n'); + } else { + comment += ' No changes from previous run.'; + } + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: comment, + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body, + }); + core.info(`Updated existing issue #${existing.number}`); + } else { + const { data: created } = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: ['tree-sitter-drift', 'dependencies'], + }); + core.info(`Opened issue #${created.number}`); + } + + - name: Close tracking issue on clean scheduled runs + if: > + github.event_name == 'schedule' && + steps.readiness.outputs.exit_code == '0' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const title = 'Tree-sitter 0.25 upgrade readiness'; + const { data: open } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'tree-sitter-drift', + per_page: 10, + }); + const existing = open.find(i => i.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: 'All grammars are now compatible with tree-sitter@0.25. Upgrade is ready! Closing automatically.', + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed', + }); + core.info(`Closed issue #${existing.number}`); + } diff --git a/.github/workflows/triage-sweep.yml b/.github/workflows/triage-sweep.yml index ba5514dbf..43d67828d 100644 --- a/.github/workflows/triage-sweep.yml +++ b/.github/workflows/triage-sweep.yml @@ -47,8 +47,10 @@ permissions: issues: write pull-requests: write +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". +# Single global slot — newest manual dispatch supersedes any in-flight run. concurrency: - group: triage-sweep + group: ${{ github.workflow }} cancel-in-progress: true jobs: @@ -74,7 +76,7 @@ jobs: run: pip install -r .github/scripts/triage/requirements.txt - name: Cache FastEmbed model weights - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5 + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ${{ github.workspace }}/.fastembed_cache key: fastembed-bge-small-en-v1.5 diff --git a/.gitignore b/.gitignore index 4c2df272c..e8d4077ec 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,11 @@ GitNexus.sln # Git worktrees .worktrees/ +# Vendored tree-sitter grammar build artifacts (created at install time, +# never committed). See docs/plans/2026-04-15-002-fix-tree-sitter-proto-vendor-deps-plan.md +gitnexus/vendor/**/build/ +gitnexus/vendor/**/node_modules/ + /github/scripts/triage/__pycache__/ .claude-flow/ @@ -95,4 +100,5 @@ GitNexus.sln .swarm/ -local_docs/ \ No newline at end of file +local_docs/ + diff --git a/AGENTS.md b/AGENTS.md index 09c3eeb12..f4fbcadc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,116 +1,122 @@ - - + + -Last reviewed: 2026-03-24 +Last reviewed: 2026-04-16 **Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub) -This file uses a standard agent header (version, scope, model policy, reference docs, changelog), adapted for this **TypeScript/JavaScript monorepo**. - ## Scope -| | | -|--|--| -| **Reads** | Repository tree as needed for the task: `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/` when present, and docs. | -| **Writes** | Only paths required for the requested change; keep diffs minimal. Update lockfiles when dependencies change. | -| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/` when applicable; shell utilities for documented CI/dev workflows. | -| **Off-limits** | User secrets (e.g. real `.env`), production deployment credentials, unrelated repositories, destructive git history operations without explicit human confirmation. | +| Boundary | Rule | +|----------|------| +| **Reads** | `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/`, docs. | +| **Writes** | Only paths required for the change; keep diffs minimal. Update lockfiles when deps change. | +| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/`; documented CI/dev workflows. | +| **Off-limits** | Real `.env` / secrets, production credentials, unrelated repos, destructive git ops without confirmation. | ## Model Configuration -- **Primary:** Pin in **Cursor** (Settings → model). Use a **named** model (e.g. GPT-5.2, Claude Sonnet 4.x). Avoid relying on **Auto** when reproducibility or audit trail matters. -- **Fallback:** As configured in Cursor or your organization (do not encode `latest` or wildcards in automation configs). -- **Notes:** The open-source GitNexus CLI indexer does not call an LLM. Optional Nexus AI in the web UI uses end-user provider keys and models. +- **Primary:** Use a named model (e.g. Claude Sonnet 4.x). Avoid `Auto` or unversioned `latest` when reproducibility matters. +- **Notes:** The GitNexus CLI indexer does not call an LLM. ## Execution Sequence (complex tasks) -Long sessions dilute instructions. For **multi-step** work, state up front: - +For multi-step work, state up front: 1. Which rules in this file and **[GUARDRAILS.md](GUARDRAILS.md)** apply (and any relevant Signs). -2. Current **Scope** boundaries (Reads / Writes / Off-limits). -3. Which **validation commands** you will run (e.g. `cd gitnexus && npm test`, `npx tsc --noEmit`). +2. Current **Scope** boundaries. +3. Which **validation commands** you will run (`cd gitnexus && npm test`, `npx tsc --noEmit`). -On very long threads, the human may add *“Remember: apply all AGENTS.md rules”* to re-weight rule tokens against context dilution. +On long threads, *"Remember: apply all AGENTS.md rules"* re-weights these instructions against context dilution. ## Claude Code hooks -Hooks enforce gates that prompts cannot. In **Claude Code**, **PreToolUse** hooks can block tools such as `git_commit` until checks pass. Adapt to this repo: e.g. `cd gitnexus && npm test` before commit. +**PreToolUse** hooks can block tools (e.g. `git_commit`) until checks pass. Adapt to this repo: `cd gitnexus && npm test` before commit. -## Context budget (Cursor / standards) +## Context budget -Generic “core standards” playbooks are often long and stack-specific. For this monorepo, commands and gotchas live under **Cursor Cloud specific instructions** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split domain rules into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules live in **`.cursor/index.mdc`** (YAML frontmatter with `alwaysApply: true`). **Claude Code:** optionally load a **`STANDARDS.md`** only when needed (e.g. *“When writing new code, read STANDARDS.md”*) to save context. +Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules in `.cursor/index.mdc`. **Claude Code:** load `STANDARDS.md` only when needed. -## Reference Documentation +## Reference docs -- **This repository:** **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)**. -- **Cursor:** `.cursor/index.mdc` (always-on rules); optional `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` is deprecated — see `.cursor/index.mdc`. -- **Optional local files:** `NOTES.md` (short vendor-neutral project snapshot). For handoffs, keep notes local (e.g., a scratch file outside the repo) rather than committing `HANDOFF.md`. -- **GitNexus:** skills under `.claude/skills/gitnexus/`; machine-oriented rules in the `gitnexus:start` … `gitnexus:end` block below. +- **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)** +- **Call-resolution DAG:** See ARCHITECTURE.md § Call-Resolution DAG. Typed 6-stage DAG inside the `parse` phase; language-specific behavior behind `inferImplicitReceiver` / `selectDispatch` hooks on `LanguageProvider`. Shared code in `gitnexus/src/core/ingestion/` must not name languages. Types: `gitnexus/src/core/ingestion/call-types.ts`. +- **Cursor:** `.cursor/index.mdc` (always-on); `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` deprecated. +- **GitNexus:** skills in `.claude/skills/gitnexus/`; MCP rules in `gitnexus:start` block below. ## Changelog | Date | Version | Change | |------|---------|--------| -| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication (was inlined in Reference Docs bullet). | -| 2026-03-23 | 1.1.0 | Updated agent instructions (sections, references, Cursor layout). | -| 2026-03-22 | 1.0.0 | Added structured agent header and changelog. | +| 2026-04-16 | 1.4.0 | Fixed: web UI description, pre-commit behavior, MCP tools (7->16), added gitnexus-shared, removed stale vite-plugin-wasm gotcha. | +| 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. | +| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication. | +| 2026-03-23 | 1.1.0 | Updated agent instructions, references, Cursor layout. | +| 2026-03-22 | 1.0.0 | Initial structured header and changelog. | --- # GitNexus — Code Intelligence -This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relationships, 185 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +Indexed as **GitNexus** (4325 symbols, 10556 relationships, 300 execution flows). Use MCP tools to understand code, assess impact, and navigate safely. -> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. +> If any tool warns the index is stale, run `npx gitnexus analyze` first. ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. +- **MUST run impact analysis before editing any symbol.** `gitnexus_impact({target: "symbolName", direction: "upstream"})` — report blast radius to the user. +- **MUST run `gitnexus_detect_changes()` before committing** — verify only expected symbols and flows are affected. +- **MUST warn the user** if impact returns HIGH or CRITICAL risk. +- Explore unfamiliar code with `gitnexus_query({query: "concept"})` (process-grouped, ranked) instead of grepping. +- Full context on a symbol: `gitnexus_context({name: "symbolName"})`. ## When Debugging -1. `gitnexus_query({query: ""})` — find execution flows related to the issue -2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step -4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed +1. `gitnexus_query({query: ""})` — find related execution flows +2. `gitnexus_context({name: ""})` — callers, callees, process participation +3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace flow step by step +4. Regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` ## When Refactoring -- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. -- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. -- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. +- **Rename:** `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Graph edits are safe; text_search edits need manual review. +- **Extract/Split:** `gitnexus_context` (incoming/outgoing refs) then `gitnexus_impact` (upstream callers) before moving code. +- **After any refactor:** `gitnexus_detect_changes({scope: "all"})` to verify scope. ## Never Do -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. +- Edit a symbol without running `gitnexus_impact` first. +- Ignore HIGH/CRITICAL risk warnings. +- Rename with find-and-replace — use `gitnexus_rename`. +- Commit without `gitnexus_detect_changes()`. +- Add language-specific behavior to shared ingestion code (`gitnexus/src/core/ingestion/`) — use a `LanguageProvider` hook. Seeing `provider.mroStrategy === 'xxx'` or an import from `languages/xxx.ts` in shared code means stop and add a hook. ## Tools Quick Reference -| Tool | When to use | Command | +| Tool | When to use | Example | |------|-------------|---------| +| `list_repos` | Discover indexed repos | `gitnexus_list_repos({})` | | `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | | `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | | `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | | `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | | `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | | `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | +| `api_impact` | Pre-change API route impact | `gitnexus_api_impact({route: "/api/users", method: "GET"})` | +| `route_map` | Route → handler → consumer map | `gitnexus_route_map({})` | +| `tool_map` | MCP/RPC tool definitions | `gitnexus_tool_map({})` | +| `shape_check` | Response shape vs consumer access | `gitnexus_shape_check({route: "/api/users"})` | +| `group_list` | List repo groups | `gitnexus_group_list({})` | +| `group_query` | Cross-repo search in a group | `gitnexus_group_query({name: "myGroup", query: "auth"})` | +| `group_sync` | Rebuild group Contract Registry | `gitnexus_group_sync({name: "myGroup"})` | +| `group_contracts` | Inspect group contracts | `gitnexus_group_contracts({name: "myGroup"})` | +| `group_status` | Group staleness report | `gitnexus_group_status({name: "myGroup"})` | ## Impact Risk Levels | Depth | Meaning | Action | |-------|---------|--------| -| d=1 | WILL BREAK — direct callers/importers | MUST update these | +| d=1 | WILL BREAK — direct callers/importers | MUST update | | d=2 | LIKELY AFFECTED — indirect deps | Should test | | d=3 | MAY NEED TESTING — transitive | Test if critical path | @@ -118,87 +124,80 @@ This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relation | Resource | Use for | |----------|---------| -| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | +| `gitnexus://repo/GitNexus/context` | Codebase overview, index freshness | | `gitnexus://repo/GitNexus/clusters` | All functional areas | | `gitnexus://repo/GitNexus/processes` | All execution flows | | `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace | ## Self-Check Before Finishing -Before completing any code modification task, verify: 1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL risk warnings were ignored -3. `gitnexus_detect_changes()` confirms changes match expected scope -4. All d=1 (WILL BREAK) dependents were updated +2. No HIGH/CRITICAL warnings were ignored +3. `gitnexus_detect_changes()` confirms expected scope +4. All d=1 dependents were updated ## Keeping the Index Fresh -After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: - ```bash -npx gitnexus analyze +npx gitnexus analyze # basic refresh +npx gitnexus analyze --embeddings # preserve embeddings ``` -If the index previously included embeddings, preserve them by adding `--embeddings`: +Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). Running without `--embeddings` deletes existing vectors. -```bash -npx gitnexus analyze --embeddings -``` +> Claude Code: PostToolUse hook handles this after `git commit` and `git merge`. -To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** +## CLI Skills -> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | +| Task | Skill file | +|------|-----------| +| Architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Debugging / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Refactoring | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools/resources/schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| CLI commands (index, status, clean, wiki) | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | -## Cursor Cloud specific instructions +## Repo reference -### Repository structure +### Packages -This is a monorepo with two main products and supporting config packages: - -| Component | Path | Purpose | -|-----------|------|---------| -| **GitNexus CLI/Core** | `gitnexus/` | Main product — TypeScript CLI, indexing pipeline, MCP server. Published to npm. | -| **GitNexus Web UI** | `gitnexus-web/` | React/Vite browser app — graph explorer + AI chat. Runs entirely in WASM. | -| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace (no build). | -| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor (no build). | -| SWE-bench Eval | `eval/` | Python evaluation harness (optional; needs Docker + LLM API keys). | +| Package | Path | Purpose | +|---------|------|---------| +| **CLI/Core** | `gitnexus/` | TypeScript CLI, indexing pipeline, MCP server. Published to npm. | +| **Web UI** | `gitnexus-web/` | React/Vite thin client. All queries via `gitnexus serve` HTTP API. | +| **Shared** | `gitnexus-shared/` | Shared TypeScript types and constants. | +| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace. | +| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor. | +| Eval | `eval/` | Python evaluation harness (Docker + LLM API keys). | ### Running services -- **CLI/Core**: `cd gitnexus && npm run dev` (tsx watch mode) or `npm run build && node dist/cli/index.js ` -- **Web UI**: `cd gitnexus-web && npm run dev` (Vite on port 5173) -- **Backend mode**: `cd && node /workspace/gitnexus/dist/cli/index.js serve` (HTTP API on port 3741 by default) +```bash +cd gitnexus && npm run dev # CLI: tsx watch mode +cd gitnexus-web && npm run dev # Web UI: Vite on port 5173 +npx gitnexus serve # HTTP API on port 4747 (from any indexed repo) +``` ### Testing **CLI / Core (`gitnexus/`)** -- **Unit tests**: `cd gitnexus && npm test` (vitest, ~2000 tests) -- **Integration tests**: `cd gitnexus && npm run test:integration` (vitest, ~1850 tests). Two LadybugDB file-locking tests (`lbug-core-adapter`, `search-core`) may fail in containerized environments due to `/tmp` locking limitations — this is a known environment issue, not a code bug. -- **TypeScript check**: `cd gitnexus && npx tsc --noEmit` +- `npm test` — full vitest suite (~2000 tests) +- `npm run test:unit` — unit tests only +- `npm run test:integration` — integration (~1850 tests). LadybugDB file-locking tests may fail in containers (known env issue). +- `npx tsc --noEmit` — typecheck **Web UI (`gitnexus-web/`)** -- **Unit tests**: `cd gitnexus-web && npm test` (vitest, ~200 tests) -- **E2E tests**: `cd gitnexus-web && E2E=1 npx playwright test` (Playwright, 5 tests — requires `gitnexus serve` + `npm run dev` running) -- **TypeScript check**: `cd gitnexus-web && npx tsc -b --noEmit` +- `npm test` — vitest (~200 tests) +- `npm run test:e2e` — Playwright (7 spec files; requires `gitnexus serve` + `npm run dev`) +- `npx tsc -b --noEmit` — typecheck -No separate lint command is configured; TypeScript strict checking serves as the primary static analysis. +**Pre-commit hook** (`.husky/pre-commit`): formatting (prettier via lint-staged) + typecheck for staged packages. Tests do **not** run in pre-commit — CI only. ### Gotchas -- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift). Native tree-sitter bindings require `python3`, `make`, and `g++` to be present. -- `tree-sitter-kotlin` and `tree-sitter-swift` are optional dependencies — install warnings for these are expected and non-blocking. -- The Web UI uses `vite-plugin-wasm` and requires `Cross-Origin-Opener-Policy`/`Cross-Origin-Embedder-Policy` headers for `SharedArrayBuffer` (handled automatically by Vite dev server). -- There is no ESLint/Prettier configuration in this repo. +- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift, builds tree-sitter-proto). Native bindings need `python3`, `make`, `g++`. +- `tree-sitter-kotlin` and `tree-sitter-swift` are optional — install warnings expected. +- ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b153ff679..d934cd8b6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,123 +1,365 @@ # Architecture — GitNexus -This repository is a **monorepo** with two main products: the **CLI / MCP package** (`gitnexus/`) and the **browser UI** (`gitnexus-web/`). Supporting folders ship editor integrations and plugins without changing the core graph engine. +Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## Repository layout | Path | Role | |------|------| -| `gitnexus/` | Published npm package `gitnexus`: CLI, MCP server (stdio), local HTTP API for bridge mode, ingestion pipeline, LadybugDB graph, embeddings (optional). | -| `gitnexus-web/` | Vite + React UI: in-browser indexing (WASM), graph visualization, optional connection to `gitnexus serve`. | -| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Packaged **skills** and plugin metadata so agents discover the same workflows as documented in `AGENTS.md`. | -| `eval/` | Evaluation harnesses and docs for benchmarking tool usage. | -| `.github/` | CI workflows (quality, unit, integration, E2E) and composite actions. | +| `gitnexus/` | npm package `gitnexus`: CLI, MCP server (stdio), HTTP API, ingestion pipeline, LadybugDB graph, embeddings. | +| `gitnexus-web/` | Vite + React thin client: graph explorer + AI chat. All queries via `gitnexus serve` HTTP API. | +| `gitnexus-shared/` | Shared TypeScript types and constants (consumed by CLI and Web). | +| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Agent skills and plugin metadata. | +| `eval/` | Evaluation harnesses for benchmarking tool usage. | +| `.github/` | CI workflows + composite actions (`setup-gitnexus/`, `setup-gitnexus-web/`). | ## End-to-end flow: index → graph → tools -1. **Ingestion** (`gitnexus analyze`) - - Entry: `gitnexus/src/cli/analyze.ts` → `runPipelineFromRepo` in `gitnexus/src/core/ingestion/pipeline.ts`. - - Walks the git working tree, parses supported languages via **Tree-sitter**, resolves imports/calls/inheritance, detects **communities** and **processes** (execution flows), and builds an in-memory **knowledge graph** (`gitnexus/src/core/graph/`). - - Output is loaded into **LadybugDB** under **`.gitnexus/`** at the repo root (`lbug/`, `meta.json`, etc.). Optional **FTS** indexes and **embeddings** attach to the same store. - - The repo is registered in **`~/.gitnexus/registry.json`** so MCP can find it from any working directory. +1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 12 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. -2. **Persistence & metadata** - - `gitnexus/src/storage/repo-manager.ts` — paths, registry, cleanup of legacy Kuzu artifacts. - - `gitnexus/src/core/lbug/lbug-adapter.ts` — graph load, queries, embedding restore batches. +2. **Persistence** — `repo-manager.ts` (paths, registry, KuzuDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches). -3. **Query & agents** - - **MCP (stdio):** `gitnexus/src/cli/mcp.ts` → `startMCPServer` → `LocalBackend` (`gitnexus/src/mcp/local/local-backend.ts`) opens registered repos and serves **tools** from `gitnexus/src/mcp/tools.ts` and **resources** from `gitnexus/src/mcp/resources.ts`. - - **Bridge HTTP:** `gitnexus/src/cli/serve.ts` → Express app in `gitnexus/src/server/api.ts` (CORS-limited) exposes REST + MCP-over-HTTP for the web UI. - - **CLI tools (no MCP):** `gitnexus query`, `context`, `impact`, `cypher` in `gitnexus/src/cli/tool.ts` call the same backend for scripts and CI. +3. **Query layer** — three interfaces to the same backend: + - **MCP (stdio):** `mcp.ts` → `LocalBackend` → tools (`tools.ts`) + resources (`resources.ts`) + - **HTTP bridge:** `serve.ts` → Express (`api.ts`, `mcp-http.ts`) for web UI + - **CLI direct:** `gitnexus query|context|impact|cypher` in `tool.ts` -4. **Staleness** - - `gitnexus/src/mcp/staleness.ts` compares indexed `lastCommit` to `HEAD` and surfaces hints when the graph is behind git. +4. **Staleness** — `staleness.ts` compares indexed `lastCommit` to `HEAD`, surfaces hints. -## MCP tools (summary) +## MCP tools | Tool | Purpose | |------|---------| -| `list_repos` | Discover indexed repositories when more than one is registered. | -| `query` | Natural-language / keyword search over the graph (hybrid BM25 + optional vectors). | -| `cypher` | Ad hoc **Cypher** against the schema (see resource `gitnexus://repo/{name}/schema`). | -| `context` | Callers, callees, processes for one symbol (with disambiguation). | -| `impact` | Blast radius (upstream/downstream) with depth and risk summary. | -| `detect_changes` | Map git diffs to affected symbols and processes. | -| `rename` | Graph-assisted rename with `dry_run` preview (`graph` vs `text_search` confidence). | +| `list_repos` | Discover indexed repos | +| `query` | Hybrid BM25 + vector search over the graph | +| `cypher` | Ad hoc Cypher against the schema | +| `context` | Callers, callees, processes for one symbol | +| `impact` | Blast radius (upstream/downstream) with risk summary | +| `detect_changes` | Map git diffs to affected symbols and processes | +| `rename` | Graph-assisted multi-file rename with `dry_run` preview | +| `api_impact` | Pre-change impact report for an API route handler | +| `route_map` | API route → handler → consumer mappings | +| `tool_map` | MCP/RPC tool definitions and handlers | +| `shape_check` | Response shape vs consumer property access mismatches | +| `group_list` | List repo groups or details for one group | +| `group_query` | Cross-repo search in a group (reciprocal rank fusion) | +| `group_sync` | Rebuild group Contract Registry (`contracts.json`) | +| `group_contracts` | Inspect group contracts and cross-links | +| `group_status` | Index and Contract Registry staleness per repo in a group | ## Where to change what -| If you are changing… | Start in… | -|----------------------|-----------| -| CLI commands / flags | `gitnexus/src/cli/` (`index.ts`, per-command modules). | -| Parsing or graph construction | `gitnexus/src/core/ingestion/` (pipeline, processors, resolvers, type-extractors). | -| Graph schema / DB access | `gitnexus/src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`), `gitnexus/src/mcp/core/lbug-adapter.ts` if MCP-specific. | -| MCP protocol, tools, resources | `gitnexus/src/mcp/server.ts`, `tools.ts`, `resources.ts`. | -| Search ranking | `gitnexus/src/core/search/` (BM25, hybrid fusion). | -| Embeddings | `gitnexus/src/core/embeddings/`, phases in `analyze.ts`. | -| Wiki generation | `gitnexus/src/core/wiki/`. | -| Web UI behavior | `gitnexus-web/src/` (components, workers, graph client). | -| CI | `.github/workflows/*.yml`, `.github/actions/setup-gitnexus/`. | +| Concern | Start in | +|---------|----------| +| CLI commands/flags | `src/cli/` (`index.ts`, per-command modules) | +| Parsing/graph construction | `src/core/ingestion/pipeline-phases/` + `pipeline.ts` | +| Graph schema/DB | `src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`) | +| MCP tools/resources | `src/mcp/server.ts`, `tools.ts`, `resources.ts` | +| Search ranking | `src/core/search/` (BM25, hybrid fusion) | +| Embeddings | `src/core/embeddings/` + `src/core/run-analyze.ts` | +| Wiki generation | `src/core/wiki/` | +| Language support | `src/core/ingestion/languages/` + `tree-sitter-queries.ts` + `gitnexus-shared/src/languages.ts` | +| Import resolution | `src/core/ingestion/import-processor.ts` + `import-resolvers/configs/` + `model/resolution-context.ts` | +| Call resolution/MRO | `src/core/ingestion/call-processor.ts` + `model/resolve.ts` | +| Type extraction | `src/core/ingestion/type-extractors/` | +| Worker pool | `src/core/ingestion/workers/` | +| Web UI | `gitnexus-web/src/` | +| CI | `.github/workflows/*.yml`, `.github/actions/` | + +> Paths above are relative to `gitnexus/` unless they start with `gitnexus-web/` or `.github/`. + +--- + +## Pipeline Phase DAG + +12 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. + +``` +scan → structure → [markdown, cobol] → parse → [routes, tools, orm] + → crossFile → mro → communities → processes +``` + +| Phase | File | Deps | Output | +|-------|------|------|--------| +| `scan` | `scan.ts` | (root) | File paths + sizes | +| `structure` | `structure.ts` | `scan` | File/Folder nodes, CONTAINS edges, `allPathSet` | +| `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx | +| `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) | +| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries | +| `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators) | +| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges | +| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) | +| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order | +| `mro` | `mro.ts` | `crossFile`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | +| `communities` | `communities.ts` | `mro`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | +| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `structure` | Process nodes + STEP_IN_PROCESS edges | + +**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `orm-extraction.ts` (sequential ORM fallback), `types.ts`, `runner.ts`, `index.ts`. + +### DAG runner + +`runner.ts` — static phase graph, no plugins, compile-time type safety. + +1. **Validation** — Kahn's topological sort. Rejects on: duplicate names, missing deps, cycles (DFS traces the concrete cycle path, e.g., `A -> B -> C -> A`, plus count of transitively blocked dependents). + +2. **Execution** — sequential in topological order. Each phase receives: + - `ctx: PipelineContext` — shared mutable `KnowledgeGraph`, `repoPath`, progress callback, options + - `deps: ReadonlyMap` — **declared deps only** (runner filters the results map to prevent hidden coupling) + +3. **Error handling** — wraps phase errors with the phase name, emits terminal `error` progress event, swallows progress handler errors to preserve the original cause. + +4. **Timing** — per-phase `durationMs` in `PhaseResult`, dev-mode console logging. + +**Design patterns:** +- **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output. +- **Typed phase access** — `getPhaseOutput(deps, 'name')` for type-safe upstream results. +- **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership. +- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests). `skipWorkers` forces sequential parsing. + +### How to add a new phase + +1. Create `pipeline-phases/my-phase.ts` with a `PipelinePhase` (name, deps, execute) +2. Export from `pipeline-phases/index.ts` +3. Add to `buildPhaseList()` in `pipeline.ts` + +```typescript +import type { PipelinePhase, PhaseResult } from './types.js'; +import { getPhaseOutput } from './types.js'; +import type { ParseOutput } from './parse.js'; + +export interface MyPhaseOutput { /* ... */ } + +export const myPhase: PipelinePhase = { + name: 'myPhase', + deps: ['parse'], + async execute(ctx, deps) { + const { allPaths } = getPhaseOutput(deps, 'parse'); + // ... write to ctx.graph ... + return { /* typed output */ }; + }, +}; +``` + +--- + +## Call-Resolution DAG + +Typed 6-stage pipeline in `call-processor.ts` (inside the `parse` phase) that resolves method/function calls and emits CALLS edges. Language behavior plugs in at two `LanguageProvider` hook points (stages 3–4); shared code names no languages. Scope: call resolution only — import resolution, type extraction, heritage, and symbol-table population live in other phases. + +### Stages + +``` +extract-call ──▶ classify-form ──▶ infer-receiver ──▶ select-dispatch ──▶ resolve-target ──▶ emit-edge + (1) (2) (3) [hook] (4) [hook] (5) (6) +``` + +| Stage | Produces | Location | +|-------|----------|----------| +| **extract-call** | `ExtractedCallSite` (name, form, receiver, argCount) | `call-extractors/` (per-language); runs in worker | +| **classify-form** | callForm (`free`/`member`/`constructor`) + arity | `call-analysis.ts` → `inferCallForm`; shared, runs in worker | +| **infer-receiver** | `ReceiverEnriched` (receiver type finalized) | `call-processor.ts`; shared default chain, then `inferImplicitReceiver` hook | +| **select-dispatch** | `DispatchDecision` (primary, fallback, ancestryView) | `selectDispatch` hook, falls back to shared default | +| **resolve-target** | `TieredCandidates` | `model/resolve.ts` → `lookupMethodByOwnerWithMRO` (MRO walk) | +| **emit-edge** | CALLS edge in graph | `call-processor.ts`; writes edge with confidence tier | + +### Provider hooks + +Both hooks are optional on `LanguageProvider`. Ruby is the only current implementer. + +**`inferImplicitReceiver`** — called after shared infer-receiver defaults. Returns `ImplicitReceiverOverride | null`. + +| | | +|---|---| +| Inputs | `calledName`, `callForm`, `receiverName`, `receiverTypeName`, `callNode` (AST), `filePath` | +| Non-null fields | `callForm`, `receiverName`, `receiverTypeName` (required); `receiverSource: 'implicit-self'` (fixed); `hint?` (opaque, passed to `selectDispatch`) | +| Null | Keep existing `ReceiverEnriched` state | + +**`selectDispatch`** — called after infer-receiver (including hook). Returns `DispatchDecision | null`; null uses shared default (constructor → `primary:'constructor'`; typed receiver → `primary:'owner-scoped'`; else → `primary:'free'`). + +| | | +|---|---| +| Inputs | `calledName`, `callForm`, `receiverName`, `receiverTypeName`, `receiverSource`, `hint` | +| Non-null fields | `primary: 'owner-scoped' \| 'free' \| 'constructor'`; `fallback?: 'free-arity-narrowed'`; `ancestryView?: 'instance' \| 'singleton'`; `hint?` | + +**`DispatchDecision` field semantics:** +- `primary: 'owner-scoped'` — MRO walk from receiver's type; used when receiver type is known. +- `fallback: 'free-arity-narrowed'` — after owner-scoped miss, search free-call candidates by arity only (Ruby uses this for implicit-self calls that miss their owner's MRO). +- `ancestryView: 'singleton'` — walk singleton/class ancestry instead of instance ancestry (Ruby `def self.foo` bodies, so `extend`-ed methods are found). + +### Adding language behavior + +1. **Implicit receivers** — implement `inferImplicitReceiver`: return null if call already has a receiver; otherwise use `findEnclosingClassInfo` (`ast-helpers.ts`) to find the enclosing context, return `ImplicitReceiverOverride` with `receiverSource: 'implicit-self'`, and optionally set `hint` for `selectDispatch`. +2. **Custom dispatch** — implement `selectDispatch`: inspect `receiverSource` and `hint`, return `DispatchDecision` with `primary`, optional `fallback`, optional `ancestryView`; return null to keep shared defaults. +3. **MRO strategy** — confirm `mroStrategy` is `'first-wins'`, `'c3'`, `'ruby-mixin'`, or `'none'`; consumed by `lookupMethodByOwnerWithMRO`. + +**Ruby example** (`languages/ruby.ts` + `utils/ruby-self-call.ts`): `inferImplicitReceiver` rewrites bare-identifier calls to `self.method` and sets `hint` to `'instance'`/`'singleton'`; `selectDispatch` uses hint for `ancestryView` and adds `fallback: 'free-arity-narrowed'` for implicit-self calls. + +### Code references + +| Module | Purpose | +|--------|---------| +| `core/ingestion/call-types.ts` | DAG types: `ReceiverEnriched`, `DispatchDecision`, `ImplicitReceiverOverride` | +| `core/ingestion/language-provider.ts` | Hook signatures: `inferImplicitReceiver`, `selectDispatch` | +| `core/ingestion/call-processor.ts` | `processCalls`: stages 3–6 | +| `core/ingestion/model/resolve.ts` | `lookupMethodByOwnerWithMRO`: stage 5 MRO walk | +| `core/ingestion/languages/ruby.ts` | Both hooks + `mroStrategy: 'ruby-mixin'` | +| `core/ingestion/utils/ruby-self-call.ts` | Bare-call rewrite for `inferImplicitReceiver` | + +--- + +## Language-agnostic graph feeding + +16 languages → single unified graph. Four abstraction layers: + +``` + Unified Graph Schema (44 node types, 21 relationship types) + ↑ + Unified Resolution (3-tier name lookup + MRO walk) + ↑ + Language Providers (import semantics, type config, export checker, MRO strategy) + ↑ + Tree-Sitter Queries (per-language S-expressions, unified capture tags) +``` + +### Language providers + +Each language implements `LanguageProvider` (`language-provider.ts`). Key fields: + +| Field | Purpose | +|-------|---------| +| `id`, `extensions` | Language identity and file matching | +| `treeSitterQueries` | S-expression queries for AST extraction | +| `importSemantics` | `named` / `wildcard-leaf` / `wildcard-transitive` / `namespace` | +| `importResolver` | Language-specific path → file resolution | +| `exportChecker` | Public/exported symbol detection | +| `typeConfig` | Type annotation extraction rules | +| `mroStrategy` | `first-wins` / `c3` / `none` | + +16 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. + +### Unified capture tags + +Per-language tree-sitter queries use different AST node names but produce the **same semantic capture tags**: `@definition.class`, `@definition.function`, `@call.name`, `@import.source`, `@heritage.extends`. Downstream extraction needs no language branching. Defined in `tree-sitter-queries.ts`. + +### Import resolution + +Per-language import resolution uses the **configs + factory** pattern (like call/method/class extractors). Each language declares an `ImportResolutionConfig` in `import-resolvers/configs/`, listing an ordered chain of `ImportResolverStrategy` functions. `createImportResolver()` (in `resolver-factory.ts`) composes them: first non-null result wins. Low-level helpers shared across strategies live alongside the configs in `import-resolvers/` (e.g. `go.ts`, `rust.ts`, `python.ts`). + +Unified 3-tier algorithm (`model/resolution-context.ts`), per-language `importSemantics` controls which tier activates: + +| Tier | Confidence | Mechanism | +|------|-----------|-----------| +| 1 — same-file | 0.95 | Symbol table for caller's file | +| 2 — import-scoped | 0.9 | `NamedImportMap` chains (named) or all files in `importMap` (wildcard) | +| 3 — global | 0.5 | O(1) index lookups: class, impl, callable. Fallback only | + +| Import strategy | Languages | Behavior | +|----------------|-----------|----------| +| `named` | TS, JS, Java, C#, Rust, PHP, Kotlin | Only explicitly imported names visible | +| `wildcard-leaf` | Go, Ruby, Swift, Dart | Whole-package import, no transitive re-exports | +| `wildcard-transitive` | C, C++ | `#include` closure chains through re-exports | +| `namespace` | Python | Module aliases resolved at call site | + +### Chunked parse-and-resolve + +`parse` processes files in ~20 MB byte-budget chunks to bound memory. Per chunk: +1. Worker pool dispatches files (or sequential fallback via `skipWorkers`) +2. Each worker: detect language → load grammar → run queries → return unified `ParseWorkerResult` +3. Synthesize wildcard bindings (`wildcard-synthesis.ts`) +4. Resolve imports and heritage +5. Collect `BindingAccumulator` entries for cross-file propagation + +Workers: `workers/worker-pool.ts`, `workers/parse-worker.ts`. + +### Heritage and MRO + +All languages emit unified `ExtractedHeritage` (child, parent, `EXTENDS`/`IMPLEMENTS`). MRO phase walks the heritage graph using per-language strategy: +- **`first-wins`** — Java, C#, C++, TS, Ruby, Go +- **`c3`** — Python (C3 linearization) +- **`none`** — single-inheritance languages + +Unified walk: `lookupMethodByOwnerWithMRO()` in `model/resolve.ts`. + +--- + +## Full analysis flow + +`runFullAnalysis` in `run-analyze.ts` orchestrates everything around the pipeline: + +``` +CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) + 1. Early exit if lastCommit == HEAD (unless --force) [0%] + 2. Cache existing embeddings from prior index [0%] + 3. runPipelineFromRepo() → KnowledgeGraph [0-60%] + 4. Clean up legacy KuzuDB files [60%] + 5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%] + 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] + 7. Restore cached embeddings (batch insert) [88%] + 8. Generate new embeddings if --embeddings [90-98%] + 9. Save metadata + register repo + update .gitignore [98-100%] + 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] +``` + +**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`. + +## Storage + +``` +/.gitnexus/ + ├── lbug # LadybugDB database + ├── lbug.wal # Write-ahead log + ├── lbug.lock # Single-writer lock + └── meta.json # lastCommit, indexedAt, stats + +~/.gitnexus/ + └── registry.json # Global repo registry (MCP discovery) +``` + +Managed by `repo-manager.ts`. + +## LadybugDB schema + +Defined in `lbug/schema.ts`. Separate node tables per type, single `CodeRelation` table. + +**Node tables:** File, Folder, Function, Class, Interface, Method, Constructor, CodeElement, Struct, Enum, Macro, Typedef, Union, Namespace, Trait, Impl, TypeAlias, Const, Static, Property, Record, Delegate, Annotation, Template, Module, Community, Process, Route, Tool, Section, Embedding. + +**Relation types** (`CodeRelation.type`): CONTAINS, DEFINES, CALLS, IMPORTS, EXTENDS, IMPLEMENTS, HAS_METHOD, HAS_PROPERTY, ACCESSES, METHOD_OVERRIDES, METHOD_IMPLEMENTS, MEMBER_OF, STEP_IN_PROCESS, HANDLES_ROUTE, FETCHES, HANDLES_TOOL, ENTRY_POINT_OF. + +## Embeddings and search + +**Embeddings** (`src/core/embeddings/`): Snowflake arctic-embed-xs (384D). Embeddable: File, Function, Class, Method, Interface. Incremental via SHA1 content hash. Separate `Embedding` table. + +**Search** (`src/core/search/`): Hybrid BM25 + semantic vector, merged via Reciprocal Rank Fusion (K=60). ## Known limitations ### Overloaded method resolution -Method and Constructor node IDs include an arity suffix (`#`) to -disambiguate overloaded methods. Two overloads with different parameter counts -produce distinct graph nodes: `Method:file:Class.method#1` vs -`Method:file:Class.method#2`. +Node IDs use arity suffix (`#`): `Method:file:Class.method#1` vs `#2`. -**Same-arity overload disambiguation:** When two overloads share the same -parameter count but differ in types (e.g. `save(int)` vs `save(String)`), a -type-hash suffix `~type1,type2` is appended to produce distinct node IDs: -`Method:file:Class.save#1~int` vs `Method:file:Class.save#1~String`. The suffix -is only added when a same-arity collision is detected within a class and all -parameters have non-null type annotations. Languages without type info (Python, -Ruby, JS) fall back to arity-only IDs. TypeScript/JavaScript overload signatures -are intentionally excluded from type-hashing because they are declaration-only -contracts that should collapse to the implementation body's node ID. See issue -\#651. +**Same-arity disambiguation:** type-hash suffix `~type1,type2` when collision detected and type annotations present. Languages without types (Python, Ruby, JS) use arity-only. TS/JS overload signatures excluded (collapse to implementation body). See #651. -**C++ const-qualified overload disambiguation:** Methods overloaded by const -qualification (e.g. `begin()` vs `begin() const`) are disambiguated via an -`isConst` property and a `$const` ID suffix appended to the const-qualified -variant when a non-const collision exists. The `$const` suffix appears after the -type-hash suffix: e.g. `Method:file:Container.begin#0$const`. +**C++ const-qualified:** `$const` suffix after type-hash when non-const collision exists: `Method:file:Container.begin#0$const`. -**Generic/template type preservation in type-hash:** The type-hash suffix uses -`rawType` (full AST text including generic/template args) rather than the -simplified `type` from `extractSimpleTypeName`. This means C++ template overloads -like `process(vector)` vs `process(vector)` produce distinct IDs: -`~vector` vs `~vector`. Java generic overloads like -`process(List)` vs `process(List)` are a compile error due to -type erasure, so this gap is theoretical for Java. +**Generic/template types:** type-hash uses `rawType` (full AST text including generics): `~vector` vs `~vector`. -**ID stability on first overload:** Type and const tags are collision-only. When -a class has `save(int)` as its only `save` method, the ID is `save#1` (no tag). -Adding `save(String)` changes the original to `save#1~int`. This is correct for -fresh analysis but means IDs are not stable across overload additions. Future -incremental re-analysis should account for this. +**ID stability:** collision-only tags mean IDs change when overloads are added. `save#1` becomes `save#1~int` when `save(String)` is added. -**Variadic method matching:** When one side is variadic (`parameterCount` -undefined) and the other has a fixed count, `METHOD_IMPLEMENTS` edges are -emitted with confidence 0.7 instead of 1.0. Variadic methods like -`foo(String... args)` may superficially match `foo(String s)` by type but -are not guaranteed to be interchangeable across all languages (Java/Kotlin -accept this via varargs sugar; TypeScript, C#, Rust do not). +**Variadic matching:** confidence 0.7 when one side is variadic and the other has fixed count. -**Confidence tiering** for `METHOD_IMPLEMENTS` edges: +**METHOD_IMPLEMENTS confidence tiering:** -| Match quality | Confidence | When | -|---|---|---| -| Exact parameter types match | 1.0 | Both sides have `parameterTypes` arrays and they match | -| Arity (count) matches | 1.0 | Both sides have `parameterCount`, types unavailable | -| Variadic vs fixed | 0.7 | One side is variadic, other has fixed count | -| Lenient (insufficient info) | 0.7 | One or both sides lack type and count data | +| Match quality | Confidence | +|---|---| +| Exact parameter types match | 1.0 | +| Arity match, types unavailable | 1.0 | +| Variadic vs fixed | 0.7 | +| Insufficient info | 0.7 | ## Related docs -- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance. -- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery. -- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents. -- [TESTING.md](TESTING.md) — how to run tests. -- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage expectations for **this** repo when indexed by GitNexus. +- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance +- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery +- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents +- [TESTING.md](TESTING.md) — how to run tests +- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage diff --git a/CLAUDE.md b/CLAUDE.md index 27ab5de74..af4069fcd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,10 +1,10 @@ - + -Last reviewed: 2026-03-24 +Last reviewed: 2026-04-13 **Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub) @@ -35,12 +35,14 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g ## Reference Documentation - **This repository:** [AGENTS.md](AGENTS.md) (Cursor + monorepo notes), [ARCHITECTURE.md](ARCHITECTURE.md), [CONTRIBUTING.md](CONTRIBUTING.md), [GUARDRAILS.md](GUARDRAILS.md). +- **Call-resolution DAG:** See ARCHITECTURE.md § Call-Resolution DAG. Shared pipeline code in `gitnexus/src/core/ingestion/` must not name languages — use `LanguageProvider` hooks instead (see AGENTS.md). - **GitNexus:** `.claude/skills/gitnexus/`; MCP and indexed-repo rules live only in [AGENTS.md](AGENTS.md) (`gitnexus:start` … `gitnexus:end`). See **GitNexus rules** below. ## Changelog | Date | Version | Change | |------|---------|--------| +| 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. | | 2026-03-24 | 1.2.0 | Removed duplicated gitnexus:start block and scope table; replaced with pointers to AGENTS.md. | | 2026-03-23 | 1.1.0 | Updated agent instructions to match AGENTS.md. | | 2026-03-22 | 1.0.0 | Added structured header and changelog. | @@ -49,106 +51,4 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g ## GitNexus rules -GitNexus MCP rules are in the `` … `` block in **[AGENTS.md](AGENTS.md)** — load that section when working with MCP tools or the graph index. - - -# GitNexus — Code Intelligence - -This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relationships, 185 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. - -> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. - -## Always Do - -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. - -## When Debugging - -1. `gitnexus_query({query: ""})` — find execution flows related to the issue -2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step -4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed - -## When Refactoring - -- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. -- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. -- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. - -## Never Do - -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. - -## Tools Quick Reference - -| Tool | When to use | Command | -|------|-------------|---------| -| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | -| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | -| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | -| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | -| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | -| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | - -## Impact Risk Levels - -| Depth | Meaning | Action | -|-------|---------|--------| -| d=1 | WILL BREAK — direct callers/importers | MUST update these | -| d=2 | LIKELY AFFECTED — indirect deps | Should test | -| d=3 | MAY NEED TESTING — transitive | Test if critical path | - -## Resources - -| Resource | Use for | -|----------|---------| -| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | -| `gitnexus://repo/GitNexus/clusters` | All functional areas | -| `gitnexus://repo/GitNexus/processes` | All execution flows | -| `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace | - -## Self-Check Before Finishing - -Before completing any code modification task, verify: -1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL risk warnings were ignored -3. `gitnexus_detect_changes()` confirms changes match expected scope -4. All d=1 (WILL BREAK) dependents were updated - -## Keeping the Index Fresh - -After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: - -```bash -npx gitnexus analyze -``` - -If the index previously included embeddings, preserve them by adding `--embeddings`: - -```bash -npx gitnexus analyze --embeddings -``` - -To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** - -> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | - - +See the `` block in **[AGENTS.md](AGENTS.md)** for the canonical MCP tools, impact analysis rules, and index instructions. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 868b2a580..22104edb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,30 +21,127 @@ This project uses the [PolyForm Noncommercial License 1.0.0](https://polyformpro ## Branch and pull requests - Use short-lived branches off the default branch of the repo you are targeting. -- Prefer **conventional commits** (short prefix + description), for example: - - ```text - feat: add graph export option - fix: correct MCP tool schema for query - test: cover cluster merge edge case - docs: clarify analyze flags - ``` - -- **PR title:** `[area] Short description` (e.g. `[cli] Fix index refresh race`). +- **PR titles MUST follow the conventional-commit format** — `pr-labeler.yml` enforces this on every PR and auto-applies the matching label so release notes group the change correctly. - **PR description:** what changed, why, how to verify (commands), and any risk or rollback notes. +### Pull request titles + +Format: `[(scope)][!]: ` + +Allowed types and the release-notes section each one lands in (defined in `.github/release.yml`): + +| Type | Label applied | Release-notes section | +|------|---------------|-----------------------| +| `feat` | `enhancement` | 🚀 Features | +| `fix` | `bug` | 🐛 Bug Fixes | +| `perf` | `performance` | 🏎️ Performance | +| `refactor` | `refactor` | 🔄 Refactoring | +| `test` | `test` | 🧪 Tests | +| `ci` | `ci` | 👷 CI/CD | +| `build` / `deps` | `dependencies` | 📦 Dependencies | +| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) | +| `chore` / `revert` | `chore` | (excluded from release notes) | + +Append `!` to the type (e.g. `feat(api)!: drop /v1 endpoint`) or include `BREAKING CHANGE:` in the PR body to flag a breaking change — the labeler then adds the `breaking` label and the 💥 Breaking Changes section is rendered first. + +Examples: + +```text +feat(web): add smart chat scroll +fix(extractors): resolve silent contract mis-resolution +perf: avoid O(n²) traversal in heritage walker +chore(deps): bump vitest to 3.0.0 +ci: standardize workflow concurrency +``` + +Commits within a PR may use any style — only the **merged PR title** shows up in release notes, so that's the one the convention applies to. + ## Before you open a PR - [ ] Tests pass for the packages you touched (`gitnexus` and/or `gitnexus-web`). - [ ] Typecheck passes: `npx tsc --noEmit` in `gitnexus/` and `npx tsc -b --noEmit` in `gitnexus-web/`. - [ ] No secrets, tokens, or machine-specific paths committed. - [ ] Documentation updated if behavior or public CLI/MCP contract changes. -- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — typecheck + unit tests for staged packages). +- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — formatting via lint-staged + typecheck for staged packages; tests run in CI only). ## Code review Maintainers may request changes for correctness, tests, performance, or consistency with existing patterns. Keeping diffs focused makes review faster. +## GitHub Actions — Concurrency Convention + +Every workflow under `.github/workflows/` MUST declare a top-level `concurrency:` block using this convention: + +- **Group key** starts with `${{ github.workflow }}` so no two workflows can collide on the same group name. The discriminator that follows is chosen per event shape: + - Branch/tag scope: `${{ github.workflow }}-${{ github.ref }}` + - Per-PR scope (for `issue_comment`, `pull_request_review*`, `pull_request` meta events): `${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}` + - `workflow_run` scope (e.g. `ci-report.yml`): `${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}` — the fork fallback must be stable across reruns (never `workflow_run.id`, which is per-run-unique and defeats serialization). + - Global single-slot (manual dispatch utilities): `${{ github.workflow }}` + - **Reusable workflows invoked via `workflow_call`:** do NOT use `${{ github.workflow }}` in the group key — in called-workflow context its evaluation is ambiguous and can resolve to the caller's name, which would deadlock against the caller's own group. Use a hardcoded literal prefix and a `github.event_name`-aware expression that falls through to `github.run_id` for reusable invocations (see `ci.yml` for the canonical form). + - **Merge queue (`merge_group`)**: when this event is added, use `${{ github.workflow }}-${{ github.event.merge_group.head_ref }}` with `cancel-in-progress: false` (every queue entry is a distinct ref; never cancel). +- **`cancel-in-progress` policy:** + + | Event | `cancel-in-progress` | Why | + |-------|----------------------|-----| + | `pull_request` CI run | `true` | New push supersedes old run | + | `push` to `main` | `false` | Every main commit gets validated | + | Tag push (`v*` publish) | `false` | Never cancel mid-publish | + | `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish | + | `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional | + | `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race | + | Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR | + | PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins | + | Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes | + +- For workflows that serve multiple events at once (e.g. `ci.yml` handles `pull_request`, `push`, and `workflow_call`), make `cancel-in-progress` event-aware: + + ```yaml + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + ``` + +- When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape. + ## AI-assisted contributions If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes. + +## Releases + +Two publish workflows ship `gitnexus` to npm: + +- **Stable** (`.github/workflows/publish.yml`) — triggered by pushing any `v*` + tag. Publishes to the `latest` dist-tag with a changelog-backed GitHub + release. Maintainers are expected to tag from `main` as a convention; the + workflow itself does not enforce branch reachability. +- **Release Candidate** (`.github/workflows/release-candidate.yml`) — runs on + every push to `main` (typically a merged PR) plus manual dispatch. Docs-only + changes are skipped via `paths-ignore`. Publishes to the `rc` dist-tag with + version `X.Y.Z-rc.N` and a GitHub prerelease, where: + - `X.Y.Z` is selected automatically. On push (and on dispatch with + `bump: auto`, the default) the workflow **continues the active rc cycle**: + if the registry already has `X.Y.Z-rc.*` versions with `X.Y.Z` > current + `latest`, it reuses the highest such base; otherwise it patch-bumps + from `latest`. Dispatching with `bump: patch|minor|major` **resets** + the cycle from `latest`. + - `N` is auto-incremented against existing `X.Y.Z-rc.*` entries on the + registry. First rc for a given base is `rc.1`. + + Idempotency: the workflow pushes an `rc/` marker tag and a + `v` release tag **atomically, before** calling `npm publish`. The guard + refuses to re-run once the marker exists, so a post-publish failure will + not mint a duplicate rc for the same commit. The `v` tag points at a + detached release commit whose `package.json` matches the npm tarball + exactly (traceable releases). Recovery after a partial failure: + + ```bash + git push --delete origin rc/ v + # then redispatch the workflow with force: true + ``` + +The rc workflow never moves `latest`. To verify after a change, inspect dist-tags: + +```bash +npm view gitnexus dist-tags +``` diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 7401c79c5..ac48ab906 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -1,72 +1,69 @@ -# Guardrails — GitNexus (repo + agents) +# Guardrails — GitNexus -Rules for **human contributors** and **AI agents** working on this codebase or publishing artifacts. These complement `AGENTS.md` / `CLAUDE.md` (which focus on GitNexus-in-GitNexus workflows). +Rules for **human contributors** and **AI agents**. Complements `AGENTS.md` (workflows) and `CONTRIBUTING.md` (PR process). -## Scope (typical agent session) +## Scope (least privilege) -When automating changes in this repository, treat scope as **least privilege**: +- **Read:** Source, tests, docs, public config as needed. +- **Write:** Only files required for the fix or feature; no unrelated formatting or refactors. +- **Execute:** Tests, typecheck, documented CLI commands. No destructive commands on user data without approval. +- **Off-limits:** Other people's machines, production deployments you don't own, credentials you lack permission to use. -- **Read:** Source, tests, docs, public config as needed for the task. -- **Write:** Only files required for the requested fix or feature; avoid unrelated formatting or refactors. -- **Execute:** Tests, typecheck, and documented CLI commands; do not run destructive commands on user data outside the repo without explicit approval. -- **Off-limits:** Other people’s machines, production deployments you don’t own, and credentials you didn’t receive permission to use. - -Adjust explicitly if the maintainer defines a different scope for a task. +Maintainer may widen scope per task. --- ## Non-negotiables -1. **Never commit secrets** — API keys, tokens, `.env` with real values, private URLs, or session cookies. Use `.env.example` with placeholders only. -2. **Never rename symbols with blind find-and-replace** when working in a GitNexus-indexed project — use the **`rename` MCP tool** with **`dry_run: true` first**, then review `graph` vs `text_search` edits. (There is no separate `gitnexus rename` CLI; renaming goes through MCP or editor integration.) -3. **Run impact analysis before editing shared symbols** — use **`impact`** (upstream) for functions/classes/methods others call; do not ignore **HIGH** / **CRITICAL** risk without maintainer sign-off. -4. **Prefer `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. -5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, run `npx gitnexus analyze --embeddings` when refreshing the index; plain `analyze` can drop them. +1. **Never commit secrets** — API keys, tokens, real `.env` values, private URLs, session cookies. Use `.env.example` with placeholders. +2. **Never rename with find-and-replace** in GitNexus-indexed projects — use `rename` MCP tool with `dry_run: true` first, review `graph` vs `text_search` edits. No separate `gitnexus rename` CLI exists. +3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. +4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. +5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, use `npx gitnexus analyze --embeddings`; plain `analyze` drops them. --- ## Signs (recurring failure patterns) -Use this format: **Trigger → Instruction → Reason**. -Append new Signs here when the same mistake repeats (e.g. CI broken twice the same way). +Format: **Trigger → Instruction → Reason**. Append new Signs when the same mistake repeats. -### Sign: Stale graph after edits +### Stale graph after edits -- **Trigger:** MCP or resources warn the index is behind `HEAD`, or code search doesn’t match latest commit. -- **Instruction:** Run `npx gitnexus analyze` from the repo root (plus `--embeddings` if the project used them). -- **Reason:** Tools query LadybugDB built at last analyze; git changes are invisible until re-indexed. +- **Trigger:** MCP warns index is behind `HEAD`, or search doesn't match latest commit. +- **Do:** `npx gitnexus analyze` (plus `--embeddings` if used). +- **Why:** Tools query LadybugDB from last analyze; git changes are invisible until re-indexed. -### Sign: Embeddings vanished after analyze +### Embeddings vanished after analyze -- **Trigger:** Semantic search quality drops; `stats.embeddings` in `.gitnexus/meta.json` is 0 after a refresh. -- **Instruction:** Re-run `npx gitnexus analyze --embeddings` and confirm `meta.json` reflects stored embeddings. -- **Reason:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors. +- **Trigger:** Semantic search quality drops; `stats.embeddings` in `meta.json` is 0 after refresh. +- **Do:** `npx gitnexus analyze --embeddings`, confirm `meta.json` reflects stored embeddings. +- **Why:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors. -### Sign: MCP lists no repos +### MCP lists no repos -- **Trigger:** MCP stderr says no indexed repos. -- **Instruction:** Run `npx gitnexus analyze` in the target repository; verify `npx gitnexus list` shows it. -- **Reason:** The MCP server discovers repos via `~/.gitnexus/registry.json`, populated by analyze. +- **Trigger:** MCP stderr says no indexed repos. +- **Do:** `npx gitnexus analyze` in the target repo; verify `npx gitnexus list` shows it. +- **Why:** MCP discovers repos via `~/.gitnexus/registry.json`, populated by analyze. -### Sign: Wrong repo in multi-repo setups +### Wrong repo in multi-repo setups -- **Trigger:** Query/impact results clearly belong to another project. -- **Instruction:** Call `list_repos`, then pass **`repo`** on subsequent tools (or use per-workspace MCP config). -- **Reason:** Default target may be ambiguous when multiple repos are registered. +- **Trigger:** Query/impact results belong to another project. +- **Do:** Call `list_repos`, then pass `repo` on subsequent tools. +- **Why:** Default target is ambiguous when multiple repos are registered. -### Sign: LadybugDB lock / “database busy” +### LadybugDB lock / "database busy" -- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run. -- **Instruction:** Stop overlapping processes; one writer at a time. Retry analyze or restart MCP. -- **Reason:** Embedded DB expects single-process ownership of the store. +- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run. +- **Do:** Stop overlapping processes (one writer at a time). Retry analyze or restart MCP. +- **Why:** Embedded DB expects single-process ownership. --- ## Publishing & supply chain -- **npm:** Do not publish from unreviewed automation; follow maintainer release process. Bump version intentionally; tag releases to match `package.json`. -- **Dependencies:** Prefer minimal, auditable changes to `package.json`; run tests and CI after lockfile updates. -- **License:** This project ships under **PolyForm Noncommercial 1.0.0** — do not relicense or imply a different license in docs or metadata without maintainer approval. +- **npm:** Do not publish from unreviewed automation. Bump version intentionally; tag releases to match `package.json`. +- **Dependencies:** Minimal, auditable `package.json` changes; run tests and CI after lockfile updates. +- **License:** PolyForm Noncommercial 1.0.0 — do not relicense without maintainer approval. --- @@ -74,15 +71,15 @@ Append new Signs here when the same mistake repeats (e.g. CI broken twice the sa Stop and ask a **human maintainer** when: -- Impact analysis shows **HIGH** / **CRITICAL** risk and the task still requires the change. -- You need to alter **CI**, **release**, or **security-sensitive** config. -- Requirements conflict (e.g. “speed up analyze” vs “must keep all embeddings on huge repo”). +- Impact analysis shows HIGH/CRITICAL risk and the task still requires the change. +- You need to alter CI, release, or security-sensitive config. +- Requirements conflict (e.g. "speed up analyze" vs "must keep all embeddings on huge repo"). - You are unsure whether data loss is acceptable (`clean`, forced migrations, schema changes). --- ## Related docs -- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow. -- [RUNBOOK.md](RUNBOOK.md) — commands for recovery. -- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations. +- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow +- [RUNBOOK.md](RUNBOOK.md) — commands for recovery +- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations diff --git a/TESTING.md b/TESTING.md index 8d267983a..cf481d32b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -20,9 +20,9 @@ From repository root, unless noted: cd gitnexus npm install npm run build -npm test # unit: vitest run test/unit +npm test # full suite: vitest run +npm run test:unit # unit only: vitest run test/unit npm run test:integration # integration suite -npm run test:all npm run test:coverage npx tsc --noEmit # typecheck (matches CI) ``` @@ -42,8 +42,11 @@ npm run test:e2e # Playwright (requires gitnexus serve + npm run dev) A husky pre-commit hook (`.husky/pre-commit`) runs automatically on every `git commit`: -- **`gitnexus-web/` files staged** → `tsc -b --noEmit` + `vitest run` -- **`gitnexus/` files staged** → `tsc --noEmit` + `vitest run --project default` +1. **Formatting** — `lint-staged` runs prettier on staged files +2. **`gitnexus-web/` files staged** → `tsc -b --noEmit` +3. **`gitnexus/` files staged** → `tsc --noEmit` + +Tests do **not** run in the pre-commit hook — they run in CI (`ci-tests.yml`) only. Skip with `git commit --no-verify` (use sparingly). @@ -77,7 +80,7 @@ Re-run the full relevant suite when: GitHub Actions (`.github/workflows/ci.yml`) orchestrate: -- **`ci-quality.yml`** — `tsc --noEmit` for `gitnexus/` + `tsc -b --noEmit` for `gitnexus-web/` +- **`ci-quality.yml`** — prettier format check, eslint lint, `tsc --noEmit` for `gitnexus/`, `tsc -b --noEmit` for `gitnexus-web/` - **`ci-tests.yml`** — `vitest run` with coverage (ubuntu) + cross-platform (macOS, Windows) - **`ci-e2e.yml`** — Playwright E2E tests, gated on `gitnexus-web/**` changes diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index bd89dfc62..4024bf070 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -19,6 +19,7 @@ export type { NodeTableName, RelType } from './lbug/schema-constants.js'; // Language support export { SupportedLanguages } from './languages.js'; export { getLanguageFromFilename, getSyntaxLanguageFromFilename } from './language-detection.js'; +export type { MroStrategy } from './mro-strategy.js'; // Pipeline progress export type { PipelinePhase, PipelineProgress } from './pipeline.js'; diff --git a/gitnexus-shared/src/lbug/schema-constants.ts b/gitnexus-shared/src/lbug/schema-constants.ts index 0eca57286..656ffe552 100644 --- a/gitnexus-shared/src/lbug/schema-constants.ts +++ b/gitnexus-shared/src/lbug/schema-constants.ts @@ -30,6 +30,7 @@ export const NODE_TABLES = [ 'TypeAlias', 'Const', 'Static', + 'Variable', 'Property', 'Record', 'Delegate', diff --git a/gitnexus-shared/src/mro-strategy.ts b/gitnexus-shared/src/mro-strategy.ts new file mode 100644 index 000000000..7ace9306f --- /dev/null +++ b/gitnexus-shared/src/mro-strategy.ts @@ -0,0 +1,46 @@ +/** + * MRO (Method Resolution Order) strategy — shared canonical definition. + * + * Lives in `gitnexus-shared` so `model/resolve.ts` and `mro-processor.ts` share + * the type without importing the language registry (avoids circular coupling). + * + * `first-wins` (default, Java/C#/Kotlin/Go/Swift/Dart): + * BFS ancestor walk in declaration order; first match wins. + * + * `leftmost-base` (C++): + * BFS walk; HeritageMap preserves source insertion order, so BFS naturally + * picks the leftmost base in diamond inheritance. + * + * `c3` (Python): + * C3-linearization; falls back to BFS on cyclic/inconsistent hierarchy. + * See model/resolve.ts § c3Linearize. + * + * `implements-split` (Java/C#/Kotlin): + * Low-level lookup is BFS; graph-level mro-processor detects and warns on + * interface-default method ambiguity. + * + * `qualified-syntax` (Rust): + * No auto-resolution — `lookupMethodByOwnerWithMRO` returns undefined immediately. + * Rust requires explicit `::method` syntax. + * + * `ruby-mixin` (Ruby): + * Kind-aware walk that does NOT short-circuit on direct owner first (`prepend` + * must beat the class's own method). Walk order: + * 1. Prepend providers (reverse declaration — last-prepended wins) + * 2. Direct owner's own methods + * 3. Include providers (reverse declaration) + * 4. Transitive ancestors (BFS fallback) + * Singleton dispatch: caller passes `ancestryOverride` (extend providers only); + * becomes a simple left-to-right scan. Miss NEVER falls through to file-scoped + * lookup — null-routes or honors `fallback`. + * + * @see model/resolve.ts § lookupMethodByOwnerWithMRO + * @see languages/ruby.ts § selectDispatch + */ +export type MroStrategy = + | 'first-wins' + | 'c3' + | 'leftmost-base' + | 'implements-split' + | 'qualified-syntax' + | 'ruby-mixin'; diff --git a/gitnexus-web/e2e/repo-switching.spec.ts b/gitnexus-web/e2e/repo-switching.spec.ts new file mode 100644 index 000000000..4cf6bf8c8 --- /dev/null +++ b/gitnexus-web/e2e/repo-switching.spec.ts @@ -0,0 +1,168 @@ +import { test, expect } from '@playwright/test'; + +/** + * E2E tests for the repo-switching and false-404 fixes. + * + * Most tests use the live backend (same pattern as multi-repo-scoping.spec.ts). + * The 503 hold-queue test uses route interception to simulate a slow analysis. + */ + +const BACKEND_URL = process.env.BACKEND_URL ?? 'http://localhost:4747'; +const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:5173'; + +let firstRepoName: string; + +test.beforeAll(async () => { + if (process.env.E2E) { + try { + const res = await fetch(`${BACKEND_URL}/api/repos`); + const repos = await res.json(); + firstRepoName = repos[0]?.name ?? ''; + } catch { + firstRepoName = ''; + } + return; + } + try { + const [backendRes, frontendRes] = await Promise.allSettled([ + fetch(`${BACKEND_URL}/api/repos`), + fetch(FRONTEND_URL), + ]); + if ( + backendRes.status === 'rejected' || + (backendRes.status === 'fulfilled' && !backendRes.value.ok) + ) { + test.skip(true, 'gitnexus serve not available'); + return; + } + if ( + frontendRes.status === 'rejected' || + (frontendRes.status === 'fulfilled' && !frontendRes.value.ok) + ) { + test.skip(true, 'Vite dev server not available'); + return; + } + if (backendRes.status === 'fulfilled') { + const repos = await backendRes.value.json(); + if (!repos.length) { + test.skip(true, 'No indexed repos'); + return; + } + firstRepoName = repos[0].name; + } + } catch { + test.skip(true, 'servers not available'); + } +}); + +// ── 1. Hold-queue: 503 → descriptive user message ──────────────────────────── + +test.describe('Hold-queue timeout error', () => { + test('shows descriptive message when /api/repo returns 503', async ({ page }, testInfo) => { + // Intercept only /api/repo (singular) — not /api/repos — to return a 503 + // regex: /api/repo followed by end, ?, or # — NOT /api/repos + await page.route(/\/api\/repo(?!s)(\?.*)?$/, (route) => + route.fulfill({ + status: 503, + contentType: 'application/json', + body: JSON.stringify({ + error: `Repository analysis for "${firstRepoName}" is taking longer than expected. Please try again in a moment.`, + }), + }), + ); + + await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`); + + // UI should show the 503 error message + await expect(page.getByText(/taking longer than expected/i)).toBeVisible({ + timeout: 20_000, + }); + + await page.screenshot({ path: testInfo.outputPath('hold-queue-503.png') }); + }); +}); + +// ── 2. ?project= URL persistence ───────────────────────────────────────────── + +test.describe('?project= URL persistence', () => { + test('?project= is set in URL after connecting via ?server=', async ({ page }) => { + await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`); + + await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 }); + + const url = new URL(page.url()); + const project = url.searchParams.get('project'); + expect(project).toBeTruthy(); + // first repo returned by the live backend + if (firstRepoName) expect(project).toBe(firstRepoName); + }); + + test('?project= is still present after F5 reload', async ({ page }) => { + await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`); + await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 }); + + // After connect, URL has ?server=&project= — F5 re-uses both params + await page.reload(); + await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 }); + + const url = new URL(page.url()); + expect(url.searchParams.get('project')).toBeTruthy(); + }); +}); + +// ── 3. ?project= + ?server= combined auto-connect ──────────────────────────── + +test.describe('?project= auto-connect', () => { + test('navigating with ?server=&project= connects to the correct repo', async ({ + page, + }, testInfo) => { + if (!firstRepoName) test.skip(true, 'no repo name available'); + + await page.goto( + `/?server=${encodeURIComponent(BACKEND_URL)}&project=${encodeURIComponent(firstRepoName)}`, + ); + + await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 }); + + // ?project= in URL should match what we passed in + const url = new URL(page.url()); + expect(url.searchParams.get('project')).toBe(firstRepoName); + + await page.screenshot({ path: testInfo.outputPath('project-param-connect.png') }); + }); +}); + +// ── 4. Windows path normalization ───────────────────────────────────────────── + +test.describe('Windows path normalization', () => { + test('project name uses basename when /api/repo returns a Windows-style repoPath', async ({ + page, + }) => { + const repoName = firstRepoName || 'test-repo'; + const windowsPath = `C:\\Users\\LENOVO\\.gitnexus\\repos\\${repoName}`; + + // Mock /api/repo to return a Windows backslash path while keeping name correct + await page.route(/\/api\/repo(?!s)(\?.*)?$/, (route) => + route.fulfill({ + contentType: 'application/json', + body: JSON.stringify({ + // intentionally omit `name` to force path-based extraction + path: windowsPath, + repoPath: windowsPath, + }), + }), + ); + + await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`); + + await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 }); + + // URL ?project= must be the short basename, NOT the full Windows path + const url = new URL(page.url()); + const project = url.searchParams.get('project'); + expect(project).toBeTruthy(); + expect(project).not.toContain('\\'); + expect(project).not.toContain('LENOVO'); + expect(project).toBe(repoName); + }); +}); diff --git a/gitnexus-web/e2e/server-connect.spec.ts b/gitnexus-web/e2e/server-connect.spec.ts index eb241da10..0705b3a27 100644 --- a/gitnexus-web/e2e/server-connect.spec.ts +++ b/gitnexus-web/e2e/server-connect.spec.ts @@ -1,4 +1,4 @@ -import { test, expect, type TestInfo } from '@playwright/test'; +import { test, expect } from '@playwright/test'; /** * E2E tests for the GitNexus web UI — exploring view features. @@ -58,36 +58,41 @@ test.beforeAll(async () => { * For these tests we require at least one indexed repo, so pick the first * landing card when present and then wait for the exploring view. */ -async function waitForGraphLoaded(page: import('@playwright/test').Page, testInfo: TestInfo) { +async function waitForGraphLoaded(page: import('@playwright/test').Page) { await page.goto('/'); - const landingCard = page.locator('[data-testid="landing-repo-card"]').first(); + const landingCards = page.locator('[data-testid="landing-repo-card"]'); + const preferredLandingCard = landingCards + .filter({ hasText: /GitNexus|local-integration/ }) + .first(); try { - await landingCard.waitFor({ state: 'visible', timeout: 15_000 }); + await landingCards.first().waitFor({ state: 'visible', timeout: 15_000 }); + const landingCard = + (await preferredLandingCard.count()) > 0 ? preferredLandingCard : landingCards.first(); await landingCard.click(); } catch { // Landing screen may not appear (e.g. ?server auto-connect) } - await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 }); - await expect(page.getByText(/\d+ nodes/).first()).toBeVisible(); - await page.screenshot({ path: testInfo.outputPath('graph-loaded.png') }); + const statusBar = page.getByRole('contentinfo'); + await expect(statusBar.getByText('Ready', { exact: true })).toBeVisible({ timeout: 45_000 }); + await expect(statusBar).toContainText(/nodes/, { + timeout: 20_000, + }); } test.describe('Server Connection & Graph Loading', () => { - test('selects a repo from landing and loads graph', async ({ page }, testInfo) => { - await waitForGraphLoaded(page, testInfo); - await page.screenshot({ path: testInfo.outputPath('graph-loaded-full.png'), fullPage: true }); + test('selects a repo from landing and loads graph', async ({ page }) => { + await waitForGraphLoaded(page); }); }); test.describe('Nexus AI', () => { - test('panel opens and agent initializes without error', async ({ page }, testInfo) => { - await waitForGraphLoaded(page, testInfo); + test('panel opens and agent initializes without error', async ({ page }) => { + await waitForGraphLoaded(page); await page.getByRole('button', { name: 'Nexus AI' }).click(); await expect(page.getByText('Ask me anything')).toBeVisible({ timeout: 15_000 }); - await page.screenshot({ path: testInfo.outputPath('nexus-ai-panel.png'), fullPage: true }); const errorBanner = page.getByText('Database not ready'); expect(await errorBanner.isVisible().catch(() => false)).toBe(false); @@ -95,8 +100,8 @@ test.describe('Nexus AI', () => { }); test.describe('Processes Panel', () => { - test('shows process list and View button works', async ({ page }, testInfo) => { - await waitForGraphLoaded(page, testInfo); + test('shows process list and View button works', async ({ page }) => { + await waitForGraphLoaded(page); await page.getByRole('button', { name: 'Nexus AI' }).click(); await page.getByText('Processes').click(); @@ -104,7 +109,6 @@ test.describe('Processes Panel', () => { await expect(page.locator('[data-testid="process-list-loaded"]')).toBeVisible({ timeout: 15_000, }); - await page.screenshot({ path: testInfo.outputPath('processes-panel.png'), fullPage: true }); const processRow = page.locator('[data-testid="process-row"]').first(); await expect(processRow).toBeVisible({ timeout: 10_000 }); @@ -114,14 +118,10 @@ test.describe('Processes Panel', () => { await viewBtn.waitFor({ state: 'visible', timeout: 5_000 }); await viewBtn.click(); await expect(page.locator('[data-testid="process-modal"]')).toBeVisible({ timeout: 5_000 }); - await page.screenshot({ - path: testInfo.outputPath('process-view-clicked.png'), - fullPage: true, - }); }); - test('lightbulb highlights nodes in graph', async ({ page }, testInfo) => { - await waitForGraphLoaded(page, testInfo); + test('lightbulb highlights nodes in graph', async ({ page }) => { + await waitForGraphLoaded(page); await page.getByRole('button', { name: 'Nexus AI' }).click(); await page.getByText('Processes').click(); @@ -137,13 +137,12 @@ test.describe('Processes Panel', () => { await lightbulb.waitFor({ state: 'visible', timeout: 5_000 }); await lightbulb.click(); await expect(processRow).toHaveClass(/bg-amber-950/, { timeout: 5_000 }); - await page.screenshot({ path: testInfo.outputPath('after-highlight.png'), fullPage: true }); }); }); test.describe('Turn Off All Highlights', () => { - test('selecting a node dims others, button clears it', async ({ page }, testInfo) => { - await waitForGraphLoaded(page, testInfo); + test('selecting a node dims others, button clears it', async ({ page }) => { + await waitForGraphLoaded(page); await expect(page.locator('canvas').first()).toBeVisible({ timeout: 10_000 }); @@ -160,6 +159,5 @@ test.describe('Turn Off All Highlights', () => { await expect(highlightToggle).toHaveAttribute('title', 'Turn on AI highlights', { timeout: 5_000, }); - await page.screenshot({ path: testInfo.outputPath('highlights-cleared.png'), fullPage: true }); }); }); diff --git a/gitnexus-web/src/App.tsx b/gitnexus-web/src/App.tsx index 97af8d14c..2ee3569a8 100644 --- a/gitnexus-web/src/App.tsx +++ b/gitnexus-web/src/App.tsx @@ -56,16 +56,14 @@ const AppContent = () => { // backend calls (queries, search, grep, readFile) scope to this repo. const repoName = result.repoInfo.name; const repoPath = result.repoInfo.repoPath ?? result.repoInfo.path; + // Normalize both Windows (\) and Unix (/) path separators before splitting const projectName = - repoName || repoPath?.split('/').filter(Boolean).pop() || 'server-project'; + result.repoInfo.name || + (repoPath || '').replace(/\\/g, '/').split('/').filter(Boolean).pop() || + 'server-project'; setProjectName(projectName); setCurrentRepo(projectName); - // Update URL so F5 / bookmarks preserve which repo is open - const url = new URL(window.location.href); - url.searchParams.set('project', projectName); - window.history.replaceState(null, '', url.toString()); - // Build KnowledgeGraph from server data for visualization const graph = createKnowledgeGraph(); for (const node of result.nodes) { @@ -76,6 +74,11 @@ const AppContent = () => { } setGraph(graph); + // Persist the active project in the URL for bookmarkability and F5 refresh resilience + const urlObj = new URL(window.location.href); + urlObj.searchParams.set('project', projectName); + window.history.replaceState(null, '', urlObj.toString()); + // Transition directly to exploring view setViewMode('exploring'); @@ -99,22 +102,17 @@ const AppContent = () => { ], ); - // Auto-connect when ?server query param is present (bookmarkable shortcut). - // Also reads ?project= to connect to a specific repo. + // Auto-connect when ?server or ?project query param is present (bookmarkable shortcut) const autoConnectRan = useRef(false); useEffect(() => { if (autoConnectRan.current) return; const params = new URLSearchParams(window.location.search); - if (!params.has('server')) return; + const serverUrlParam = params.get('server'); + const projectParam = params.get('project'); + + if (!serverUrlParam && !projectParam) return; autoConnectRan.current = true; - const serverUrl = params.get('server') || window.location.origin; - const projectParam = params.get('project') || undefined; - - // Keep ?server= in the URL so F5 reconnects to the same server. - // autoConnectRan.current prevents re-trigger within the same session. - // handleServerConnect() will add/update ?project= after connecting. - setProgress({ phase: 'extracting', percent: 0, @@ -123,39 +121,45 @@ const AppContent = () => { }); setViewMode('loading'); + const serverUrl = serverUrlParam || window.location.origin; const baseUrl = normalizeServerUrl(serverUrl); - connectToServer( - serverUrl, - (phase, downloaded, total) => { - if (phase === 'validating') { - setProgress({ - phase: 'extracting', - percent: 5, - message: 'Connecting to server...', - detail: 'Validating server', - }); - } else if (phase === 'downloading') { - const pct = total ? Math.round((downloaded / total) * 90) + 5 : 50; - const mb = (downloaded / (1024 * 1024)).toFixed(1); - setProgress({ - phase: 'extracting', - percent: pct, - message: 'Downloading graph...', - detail: `${mb} MB downloaded`, - }); - } else if (phase === 'extracting') { - setProgress({ - phase: 'extracting', - percent: 97, - message: 'Processing...', - detail: 'Extracting file contents', - }); - } - }, - undefined, - projectParam, - ) + const tryConnect = async () => { + return await connectToServer( + serverUrl, + (phase, downloaded, total) => { + if (phase === 'validating') { + setProgress({ + phase: 'extracting', + percent: 5, + message: 'Connecting to server...', + detail: 'Validating server', + }); + } else if (phase === 'downloading') { + const pct = total ? Math.round((downloaded / total) * 90) + 5 : 50; + const mb = (downloaded / (1024 * 1024)).toFixed(1); + setProgress({ + phase: 'extracting', + percent: pct, + message: 'Downloading graph...', + detail: `${mb} MB downloaded`, + }); + } else if (phase === 'extracting') { + setProgress({ + phase: 'extracting', + percent: 97, + message: 'Processing...', + detail: 'Extracting file contents', + }); + } + }, + undefined, + projectParam || undefined, + { awaitAnalysis: true }, // enable backend hold-queue for repos still being analyzed + ); + }; + + tryConnect() .then(async (result) => { await handleServerConnect(result); setProgress(null); diff --git a/gitnexus-web/src/components/CodeReferencesPanel.tsx b/gitnexus-web/src/components/CodeReferencesPanel.tsx index d70847599..1b7a0c5b0 100644 --- a/gitnexus-web/src/components/CodeReferencesPanel.tsx +++ b/gitnexus-web/src/components/CodeReferencesPanel.tsx @@ -231,7 +231,7 @@ export const CodeReferencesPanel = ({ onFocusNode }: CodeReferencesPanelProps) = repo: projectName, }; - readFile(selectedFilePath, options) + readFile(selectedFilePath, { ...options, repo: projectName || undefined }) .then((result) => { if (!cancelled) { setFileResult(result); diff --git a/gitnexus-web/src/components/RightPanel.tsx b/gitnexus-web/src/components/RightPanel.tsx index 44c7ed7ab..3e063a168 100644 --- a/gitnexus-web/src/components/RightPanel.tsx +++ b/gitnexus-web/src/components/RightPanel.tsx @@ -8,8 +8,10 @@ import { Loader2, AlertTriangle, GitBranch, + ArrowDown, } from '@/lib/lucide-icons'; import { useAppState } from '../hooks/useAppState'; +import { useAutoScroll } from '../hooks/useAutoScroll'; import { ToolCallCard } from './ToolCallCard'; import { isProviderConfigured } from '../core/llm/settings-service'; import { MarkdownRenderer } from './MarkdownRenderer'; @@ -35,14 +37,11 @@ export const RightPanel = () => { const [chatInput, setChatInput] = useState(''); const [activeTab, setActiveTab] = useState<'chat' | 'processes'>('chat'); const textareaRef = useRef(null); - const messagesEndRef = useRef(null); - - // Auto-scroll to bottom when messages update or while streaming - useEffect(() => { - if (messagesEndRef.current) { - messagesEndRef.current.scrollIntoView({ behavior: 'smooth' }); - } - }, [chatMessages, isChatLoading]); + // Keep streamed replies pinned unless the user intentionally scrolls away from the bottom. + const { scrollContainerRef, messagesContainerRef, isAtBottom, scrollToBottom } = useAutoScroll( + chatMessages, + isChatLoading, + ); const resolveFilePathForUI = useCallback((_requestedPath: string): string | null => { return null; @@ -265,7 +264,7 @@ export const RightPanel = () => { {/* Chat Content - only show when chat tab is active */} {activeTab === 'chat' && ( -
+
{/* Status bar */}
@@ -291,7 +290,7 @@ export const RightPanel = () => { )} {/* Messages */} -
+
{chatMessages.length === 0 ? (
@@ -315,7 +314,7 @@ export const RightPanel = () => {
) : ( -
+
{chatMessages.map((message) => (
{/* User message - compact label style */} @@ -391,10 +390,22 @@ export const RightPanel = () => { ))}
)} - {/* Scroll anchor for auto-scroll */} -
+ {/* Scroll to bottom */} + + {/* Input */}
diff --git a/gitnexus-web/src/components/StatusBar.tsx b/gitnexus-web/src/components/StatusBar.tsx index a618c3c1c..7468072fa 100644 --- a/gitnexus-web/src/components/StatusBar.tsx +++ b/gitnexus-web/src/components/StatusBar.tsx @@ -64,7 +64,7 @@ export const StatusBar = () => { {/* Right - Stats */} -
+
{graph && ( <> {nodeCount} nodes diff --git a/gitnexus-web/src/hooks/useAppState.tsx b/gitnexus-web/src/hooks/useAppState.tsx index 2fa5218ba..25c57767f 100644 --- a/gitnexus-web/src/hooks/useAppState.tsx +++ b/gitnexus-web/src/hooks/useAppState.tsx @@ -579,6 +579,13 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { try { const effectiveProjectName = overrideProjectName || projectName || 'project'; + + // Sync repoRef so all agent backend calls target the correct repo. + // initializeAgent can be called from App.tsx (handleServerConnect) which + // never sets repoRef.current directly — without this, queries default to repo[0]. + if (overrideProjectName) { + repoRef.current = overrideProjectName; + } const repo = repoRef.current; // Build backend interface for Graph RAG tools @@ -610,7 +617,8 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { setIsAgentInitializing(false); } }, - [projectName], + // eslint-disable-next-line react-hooks/exhaustive-deps + [], // repoRef is a stable ref — we sync it explicitly on entry; no state deps needed ); const sendChatMessage = useCallback( @@ -1042,6 +1050,9 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { setCodePanelOpen(false); setCodeReferenceFocus(null); + let connectedRepo: BackendRepo | undefined; + let pNameStr = repoName || 'server-project'; + try { const result: ConnectResult = await connectToServer( serverBaseUrl, @@ -1073,44 +1084,28 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { }, undefined, repoName, + { awaitAnalysis: true }, // enable backend hold-queue for repos still being analyzed ); // Build graph for visualization const repoPath = result.repoInfo.repoPath ?? result.repoInfo.path; + // Prefer the registry name, then normalize Windows \ and Unix / paths const pName = - repoName || result.repoInfo.name || repoPath?.split('/').pop() || 'server-project'; + repoName || + result.repoInfo.name || + (repoPath || '').replace(/\\/g, '/').split('/').filter(Boolean).pop() || + 'server-project'; setProjectName(pName); repoRef.current = pName; - // Update URL so F5 / bookmarks open the correct repo - const url = new URL(window.location.href); - url.searchParams.set('project', pName); - window.history.replaceState(null, '', url.toString()); + connectedRepo = result.repoInfo; + pNameStr = pName; const newGraph = createKnowledgeGraph(); for (const node of result.nodes) newGraph.addNode(node); for (const rel of result.relationships) newGraph.addRelationship(rel); setGraph(newGraph); - - // No fileContents needed — grep/read tools use backend HTTP - - // Initialize agent with backend queries, then start embeddings - try { - if (getActiveProviderConfig()) { - await initializeAgent(pName); - } - setViewMode('exploring'); - startEmbeddingsWithFallback(); - setProgress(null); - } catch (err) { - console.warn('Failed to initialize agent:', err); - setIsAgentReady(false); - agentRef.current = null; - setAgentError('Failed to initialize agent'); - setViewMode('exploring'); - setProgress(null); - } - } catch (err) { + } catch (err: unknown) { console.error('Repo switch failed:', err); setProgress({ phase: 'error', @@ -1124,6 +1119,36 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { setViewMode('exploring'); setProgress(null); }, ERROR_RESET_DELAY_MS); + return; // Abort the whole switchRepo process + } + + if (pNameStr) { + // Persist the selected project in the URL so a refresh re-opens it + const urlObj = new URL(window.location.href); + urlObj.searchParams.set('project', pNameStr); + window.history.replaceState(null, '', urlObj.toString()); + } + + // Reset the agent and clear chat history so the AI starts fresh for the new repo + agentRef.current = null; + setIsAgentReady(false); + setChatMessages([]); + + // Re-initialize agent with the new repo's graph context + try { + if (getActiveProviderConfig()) { + await initializeAgent(pNameStr); + } + setViewMode('exploring'); + startEmbeddingsWithFallback(); + setProgress(null); + } catch (err) { + console.warn('Failed to initialize agent:', err); + setIsAgentReady(false); + agentRef.current = null; + setAgentError('Failed to initialize agent'); + setViewMode('exploring'); + setProgress(null); } }, [ @@ -1143,6 +1168,7 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => { setCodeReferences, setCodePanelOpen, setCodeReferenceFocus, + setChatMessages, ], ); diff --git a/gitnexus-web/src/hooks/useAutoScroll.ts b/gitnexus-web/src/hooks/useAutoScroll.ts new file mode 100644 index 000000000..55c2946f7 --- /dev/null +++ b/gitnexus-web/src/hooks/useAutoScroll.ts @@ -0,0 +1,145 @@ +import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'; + +const DEFAULT_BOTTOM_THRESHOLD = 100; +const USER_SCROLL_EPSILON = 5; + +export interface UseAutoScrollResult { + scrollContainerRef: React.RefObject; + messagesContainerRef: React.RefObject; + isAtBottom: boolean; + scrollToBottom: (behavior?: ScrollBehavior) => void; +} + +function isNearBottom(element: HTMLElement, threshold: number): boolean { + return element.scrollHeight - element.scrollTop - element.clientHeight <= threshold; +} + +export function useAutoScroll( + chatMessages: T[], + isChatLoading: boolean, + bottomThreshold = DEFAULT_BOTTOM_THRESHOLD, +): UseAutoScrollResult { + const scrollContainerRef = useRef(null); + const messagesContainerRef = useRef(null); + + const [isAtBottom, setIsAtBottom] = useState(true); + + const shouldStickToBottomRef = useRef(true); + const lastScrollTopRef = useRef(0); + const scrollFrameIdRef = useRef(null); + + const syncScrollState = useCallback(() => { + const element = scrollContainerRef.current; + if (!element) return; + + const currentScrollTop = element.scrollTop; + const nearBottom = isNearBottom(element, bottomThreshold); + + if (nearBottom) { + shouldStickToBottomRef.current = true; + } else if (currentScrollTop < lastScrollTopRef.current - USER_SCROLL_EPSILON) { + shouldStickToBottomRef.current = false; + } + + lastScrollTopRef.current = currentScrollTop; + setIsAtBottom(nearBottom); + }, [bottomThreshold]); + + const scrollToBottom = useCallback( + (behavior: ScrollBehavior = 'smooth') => { + const element = scrollContainerRef.current; + if (!element) return; + + shouldStickToBottomRef.current = true; + + if (behavior === 'auto') { + element.scrollTop = element.scrollHeight; + lastScrollTopRef.current = element.scrollTop; + setIsAtBottom(isNearBottom(element, bottomThreshold)); + return; + } + + element.scrollTo({ + top: element.scrollHeight, + behavior, + }); + }, + [bottomThreshold], + ); + + useEffect(() => { + const element = scrollContainerRef.current; + if (!element) return; + + lastScrollTopRef.current = element.scrollTop; + + const handleScroll = () => { + if (scrollFrameIdRef.current !== null) { + cancelAnimationFrame(scrollFrameIdRef.current); + } + + scrollFrameIdRef.current = requestAnimationFrame(() => { + scrollFrameIdRef.current = null; + syncScrollState(); + }); + }; + + element.addEventListener('scroll', handleScroll, { passive: true }); + syncScrollState(); + + return () => { + element.removeEventListener('scroll', handleScroll); + + if (scrollFrameIdRef.current !== null) { + cancelAnimationFrame(scrollFrameIdRef.current); + scrollFrameIdRef.current = null; + } + }; + }, [syncScrollState]); + + useEffect(() => { + const content = messagesContainerRef.current; + const scrollEl = scrollContainerRef.current; + if (!content || !scrollEl || typeof ResizeObserver === 'undefined') return; + + let resizeFrameId: number | null = null; + + const observer = new ResizeObserver(() => { + if (shouldStickToBottomRef.current) { + if (resizeFrameId !== null) { + cancelAnimationFrame(resizeFrameId); + } + + resizeFrameId = requestAnimationFrame(() => { + resizeFrameId = null; + scrollToBottom('auto'); + }); + } else { + syncScrollState(); + } + }); + + observer.observe(content); + + return () => { + observer.disconnect(); + + if (resizeFrameId !== null) { + cancelAnimationFrame(resizeFrameId); + resizeFrameId = null; + } + }; + }, [chatMessages.length, scrollToBottom, syncScrollState]); + + useLayoutEffect(() => { + if (!shouldStickToBottomRef.current) return; + scrollToBottom('auto'); + }, [chatMessages.length, isChatLoading, scrollToBottom]); + + return { + scrollContainerRef, + messagesContainerRef, + isAtBottom, + scrollToBottom, + }; +} diff --git a/gitnexus-web/src/lib/lucide-icons.tsx b/gitnexus-web/src/lib/lucide-icons.tsx index 7ec9d4122..c6a4b565b 100644 --- a/gitnexus-web/src/lib/lucide-icons.tsx +++ b/gitnexus-web/src/lib/lucide-icons.tsx @@ -9,6 +9,7 @@ export { AlertCircle, AlertTriangle, + ArrowDown, ArrowRight, AtSign, Brain, diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index 4ef6adf34..6dc8c6555 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -222,7 +222,7 @@ export function normalizeServerUrl(input: string): string { // ── Internal Helpers ─────────────────────────────────────────────────────── -const DEFAULT_TIMEOUT_MS = 10_000; +const DEFAULT_TIMEOUT_MS = 30_000; const PROBE_TIMEOUT_MS = 2_000; const fetchWithTimeout = async ( @@ -264,11 +264,13 @@ const fetchWithTimeout = async ( const assertOk = async (response: Response): Promise => { if (response.ok) return; - let message = `Backend returned ${response.status} ${response.statusText}`; + let message = response.statusText; try { const body = await response.json(); if (body && typeof body.error === 'string') { message = body.error; + } else if (body && typeof body.message === 'string') { + message = body.message; } } catch { // Response body was not JSON @@ -386,10 +388,22 @@ export const fetchRepos = async (): Promise => { return response.json() as Promise; }; -/** Fetch repo metadata. */ -export const fetchRepoInfo = async (repo?: string): Promise => { +/** Fetch repo metadata. + * Pass `awaitAnalysis: true` when connecting to a repo that may still be cloning/analyzing — + * this enables the backend's hold-queue and uses a 5-minute timeout to match. + * Normal calls (e.g. repo switching between already-indexed repos) use the default 10s timeout. + * + * Must stay in sync with HOLD_QUEUE_TIMEOUT_SECS in gitnexus/src/server/api.ts. + */ +const HOLD_QUEUE_TIMEOUT_MS = 300_000; // 5 minutes — matches backend HOLD_QUEUE_TIMEOUT_SECS + +export const fetchRepoInfo = async ( + repo?: string, + opts?: { awaitAnalysis?: boolean }, +): Promise => { const url = `${_backendUrl}/api/repo${repo ? `?${repoParam(repo)}` : ''}`; - const response = await fetchWithTimeout(url); + const timeout = opts?.awaitAnalysis ? HOLD_QUEUE_TIMEOUT_MS : undefined; + const response = await fetchWithTimeout(url, {}, timeout); await assertOk(response); const data = await response.json(); return { ...data, repoPath: data.repoPath ?? data.path }; @@ -404,13 +418,19 @@ export const fetchGraph = async ( onProgress?: (downloaded: number, total: number | null) => void; }, ): Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }> => { - const params = [repoParam(repo), opts?.includeContent ? 'includeContent=true' : ''] + const params = [repoParam(repo), opts?.includeContent ? 'includeContent=true' : '', 'stream=true'] .filter(Boolean) .join('&'); const url = `${_backendUrl}/api/graph${params ? `?${params}` : ''}`; + // Large repos can take a while to serialize the graph — use an elevated timeout const response = await fetchWithTimeout(url, { signal: opts?.signal }, 300_000); await assertOk(response); + const contentType = response.headers.get('Content-Type') || ''; + if (contentType.includes('application/x-ndjson')) { + return parseNdjsonGraphResponse(response, opts?.onProgress); + } + if (!opts?.onProgress || !response.body) { return response.json() as Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }>; } @@ -439,6 +459,66 @@ export const fetchGraph = async ( return JSON.parse(new TextDecoder().decode(combined)); }; +const parseNdjsonGraphResponse = async ( + response: Response, + onProgress?: (downloaded: number, total: number | null) => void, +): Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }> => { + if (!response.body) { + throw new BackendError('No response body', response.status, 'server'); + } + + const contentLength = response.headers.get('Content-Length'); + const total = contentLength ? parseInt(contentLength, 10) : null; + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + const nodes: GraphNode[] = []; + const relationships: GraphRelationship[] = []; + let buffer = ''; + let downloaded = 0; + + const parseLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed) return; + + const record = JSON.parse(trimmed) as + | { type: 'node'; data: GraphNode } + | { type: 'relationship'; data: GraphRelationship } + | { type: 'error'; error: string }; + + if (record.type === 'node') { + nodes.push(record.data); + return; + } + if (record.type === 'relationship') { + relationships.push(record.data); + return; + } + if (record.type === 'error') { + throw new BackendError(record.error, response.status || 500, 'server'); + } + }; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + downloaded += value.length; + onProgress?.(downloaded, total); + buffer += decoder.decode(value, { stream: true }); + + const lines = buffer.split('\n'); + buffer = lines.pop() || ''; + for (const line of lines) { + parseLine(line); + } + } + + buffer += decoder.decode(); + parseLine(buffer); + + return { nodes, relationships }; +}; + /** Execute a Cypher query. Returns rows. */ export const runQuery = async ( cypher: string, @@ -670,18 +750,21 @@ export interface ConnectResult { /** * Connect to a server: validate, fetch repo info, download graph. * Content is NOT included (use readFile/grep for file access). + * Pass `awaitAnalysis: true` when the repo may still be cloning/analyzing — + * this enables the backend hold-queue and a 5-minute fetch timeout. */ export async function connectToServer( url: string, onProgress?: (phase: string, downloaded: number, total: number | null) => void, signal?: AbortSignal, repoName?: string, + opts?: { awaitAnalysis?: boolean }, ): Promise { const baseUrl = normalizeServerUrl(url); setBackendUrl(baseUrl); onProgress?.('validating', 0, null); - const repoInfo = await fetchRepoInfo(repoName); + const repoInfo = await fetchRepoInfo(repoName, { awaitAnalysis: opts?.awaitAnalysis }); onProgress?.('downloading', 0, null); const { nodes, relationships } = await fetchGraph(repoName, { diff --git a/gitnexus-web/test/unit/server-connection.test.ts b/gitnexus-web/test/unit/server-connection.test.ts index a818adb13..f5ee43c53 100644 --- a/gitnexus-web/test/unit/server-connection.test.ts +++ b/gitnexus-web/test/unit/server-connection.test.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from 'vitest'; -import { normalizeServerUrl } from '../../src/services/backend-client'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { fetchGraph, normalizeServerUrl, setBackendUrl } from '../../src/services/backend-client'; describe('normalizeServerUrl', () => { it('adds http:// to localhost', () => { @@ -31,3 +31,137 @@ describe('normalizeServerUrl', () => { expect(normalizeServerUrl('https://gitnexus.example.com')).toBe('https://gitnexus.example.com'); }); }); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('fetchGraph', () => { + it('requests streamed graph responses from the backend', async () => { + setBackendUrl('http://localhost:4747'); + + const fetchMock = vi.fn().mockResolvedValue( + new Response('{"nodes":[],"relationships":[]}', { + status: 200, + headers: { + 'Content-Type': 'application/json', + }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + await fetchGraph('big-repo'); + + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining('/api/graph?repo=big-repo&stream=true'), + expect.any(Object), + ); + }); + + it('parses NDJSON graph streams incrementally', async () => { + setBackendUrl('http://localhost:4747'); + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + [ + '{"type":"node","data":{"id":"File:src/app.ts","label":"File","properties":{"name":"app.ts","filePath":"src/app.ts"}}}\n', + '{"type":"relationship","data":{"id":"File:src/app.ts_CONTAINS_Function:src/app.ts:main","type":"CONTAINS","sourceId":"File:src/app.ts","targetId":"Function:src/app.ts:main"}}\n', + ].join(''), + ), + ); + controller.close(); + }, + }); + + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(stream, { + status: 200, + headers: { + 'Content-Type': 'application/x-ndjson', + }, + }), + ), + ); + + const progress = vi.fn(); + const result = await fetchGraph('big-repo', { onProgress: progress }); + + expect(result.nodes).toHaveLength(1); + expect(result.relationships).toHaveLength(1); + expect(result.nodes[0].id).toBe('File:src/app.ts'); + expect(result.relationships[0].type).toBe('CONTAINS'); + expect(progress).toHaveBeenCalled(); + }); + + it('parses NDJSON graph lines split across chunks', async () => { + setBackendUrl('http://localhost:4747'); + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue( + encoder.encode( + '{"type":"node","data":{"id":"File:src/app.ts","label":"File","properties":{"name":"app.ts"', + ), + ); + controller.enqueue( + encoder.encode( + ',"filePath":"src/app.ts"}}}\n{"type":"relationship","data":{"id":"File:src/app.ts_CONTAINS_Function:src/app.ts:main","type":"CONTAINS","sourceId":"File:src/app.ts","targetId":"Function:src/app.ts:main"}}\n', + ), + ); + controller.close(); + }, + }); + + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(stream, { + status: 200, + headers: { + 'Content-Type': 'application/x-ndjson', + }, + }), + ), + ); + + const result = await fetchGraph('big-repo'); + + expect(result.nodes).toHaveLength(1); + expect(result.relationships).toHaveLength(1); + expect(result.nodes[0].properties.filePath).toBe('src/app.ts'); + }); + + it('throws backend errors emitted in the NDJSON stream', async () => { + setBackendUrl('http://localhost:4747'); + + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('{"type":"error","error":"stream failed"}\n')); + controller.close(); + }, + }); + + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue( + new Response(stream, { + status: 200, + headers: { + 'Content-Type': 'application/x-ndjson', + }, + }), + ), + ); + + await expect(fetchGraph('big-repo')).rejects.toMatchObject({ + message: 'stream failed', + }); + }); +}); diff --git a/gitnexus-web/test/unit/use-auto-scroll.test.tsx b/gitnexus-web/test/unit/use-auto-scroll.test.tsx new file mode 100644 index 000000000..e58227d67 --- /dev/null +++ b/gitnexus-web/test/unit/use-auto-scroll.test.tsx @@ -0,0 +1,287 @@ +import { act, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { useAutoScroll } from '../../src/hooks/useAutoScroll'; + +interface HarnessProps { + messages: unknown[]; + isChatLoading: boolean; +} + +function AutoScrollHarness({ messages, isChatLoading }: HarnessProps) { + const { scrollContainerRef, messagesContainerRef, isAtBottom, scrollToBottom } = useAutoScroll( + messages, + isChatLoading, + ); + + return ( + <> +
{String(isAtBottom)}
+
+ {messages.length > 0 ? ( +
+ {messages.map((message, index) => ( +
{String(message)}
+ ))} +
+ ) : null} +
+ + + ); +} + +function setScrollMetrics( + element: HTMLDivElement, + metrics: { scrollTop?: number; scrollHeight?: number; clientHeight?: number }, +) { + if (metrics.scrollTop !== undefined) { + Object.defineProperty(element, 'scrollTop', { + configurable: true, + writable: true, + value: metrics.scrollTop, + }); + } + + if (metrics.scrollHeight !== undefined) { + Object.defineProperty(element, 'scrollHeight', { + configurable: true, + value: metrics.scrollHeight, + }); + } + + if (metrics.clientHeight !== undefined) { + Object.defineProperty(element, 'clientHeight', { + configurable: true, + value: metrics.clientHeight, + }); + } +} + +async function flushAnimationFrame() { + await act(async () => { + vi.runAllTimers(); + }); +} + +async function scrollContainer(element: HTMLDivElement, scrollTop: number) { + setScrollMetrics(element, { scrollTop }); + fireEvent.scroll(element); + await flushAnimationFrame(); +} + +const resizeObserverInstances: ResizeObserverMock[] = []; + +class ResizeObserverMock { + callback: ResizeObserverCallback; + observedElements: Element[] = []; + observe = vi.fn((element: Element) => { + this.observedElements.push(element); + }); + unobserve = vi.fn(); + disconnect = vi.fn(); + + constructor(callback: ResizeObserverCallback) { + this.callback = callback; + resizeObserverInstances.push(this); + } +} + +async function triggerResize(instance: ResizeObserverMock) { + await act(async () => { + instance.callback([], instance as unknown as ResizeObserver); + }); + await flushAnimationFrame(); +} + +describe('useAutoScroll', () => { + beforeEach(() => { + vi.useFakeTimers(); + resizeObserverInstances.length = 0; + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + return window.setTimeout(() => callback(performance.now()), 0); + }), + ); + vi.stubGlobal( + 'cancelAnimationFrame', + vi.fn((frameId: number) => { + clearTimeout(frameId); + }), + ); + Object.defineProperty(HTMLElement.prototype, 'scrollTo', { + configurable: true, + value: function (options: ScrollToOptions) { + if (options.top !== undefined) { + Object.defineProperty(this, 'scrollTop', { + configurable: true, + writable: true, + value: options.top, + }); + } + }, + }); + vi.stubGlobal('ResizeObserver', ResizeObserverMock); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + }); + + it('starts with isAtBottom true and auto-scrolls the very first message', () => { + const { rerender } = render(); + + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true'); + + const container = screen.getByTestId('container') as HTMLDivElement; + setScrollMetrics(container, { scrollTop: 0, scrollHeight: 500, clientHeight: 200 }); + + rerender(); + + expect(container.scrollTop).toBe(500); + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true'); + }); + + it('follows streaming updates while the view stays pinned to the bottom', () => { + const { rerender } = render(); + const container = screen.getByTestId('container') as HTMLDivElement; + + setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 }); + + rerender(); + + expect(container.scrollTop).toBe(1000); + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true'); + }); + + it('stops auto-scroll after the user scrolls up', async () => { + const { rerender } = render(); + const container = screen.getByTestId('container') as HTMLDivElement; + + setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 }); + await scrollContainer(container, 700); + await scrollContainer(container, 250); + + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false'); + + setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1400, clientHeight: 200 }); + rerender(); + + expect(container.scrollTop).toBe(250); + }); + + it('re-enables auto-scroll once the user returns near the bottom', async () => { + const { rerender } = render(); + const container = screen.getByTestId('container') as HTMLDivElement; + + setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 }); + await scrollContainer(container, 700); + await scrollContainer(container, 250); + + setScrollMetrics(container, { scrollTop: 1120, scrollHeight: 1400, clientHeight: 200 }); + await scrollContainer(container, 1120); + + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true'); + + setScrollMetrics(container, { scrollTop: 1120, scrollHeight: 1800, clientHeight: 200 }); + rerender(); + + expect(container.scrollTop).toBe(1800); + }); + + it('scrollToBottom re-engages auto-scroll and scrolls to the container bottom', async () => { + const { rerender } = render(); + const container = screen.getByTestId('container') as HTMLDivElement; + const scrollTo = vi.spyOn(container, 'scrollTo'); + + setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 }); + await scrollContainer(container, 700); + await scrollContainer(container, 250); + + fireEvent.click(screen.getByRole('button', { name: 'Scroll to bottom' })); + + expect(scrollTo).toHaveBeenCalledWith({ top: 1000, behavior: 'smooth' }); + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false'); + + setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1600, clientHeight: 200 }); + rerender(); + + expect(container.scrollTop).toBe(1600); + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true'); + }); + + it('re-pins to the latest bottom when inner content grows asynchronously', async () => { + render(); + const container = screen.getByTestId('container') as HTMLDivElement; + + setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 }); + await scrollContainer(container, 700); + + setScrollMetrics(container, { scrollTop: 1000, scrollHeight: 1450, clientHeight: 200 }); + await triggerResize(resizeObserverInstances[0]); + + expect(container.scrollTop).toBe(1450); + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true'); + }); + + it('does not auto-scroll on async growth after user intentionally scrolls away', async () => { + render(); + const container = screen.getByTestId('container') as HTMLDivElement; + + setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 }); + await scrollContainer(container, 700); + await scrollContainer(container, 250); + + setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1400, clientHeight: 200 }); + await triggerResize(resizeObserverInstances[0]); + + expect(container.scrollTop).toBe(250); + expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false'); + }); + + it('cancels the pending ResizeObserver rAF when the component unmounts', () => { + const cancelRAF = vi.mocked(cancelAnimationFrame); + + const { unmount } = render(); + const container = screen.getByTestId('container') as HTMLDivElement; + + setScrollMetrics(container, { scrollTop: 950, scrollHeight: 1000, clientHeight: 200 }); + + const callsBefore = cancelRAF.mock.calls.length; + + act(() => { + resizeObserverInstances[0].callback( + [], + resizeObserverInstances[0] as unknown as ResizeObserver, + ); + }); + + unmount(); + + expect(cancelRAF.mock.calls.length).toBeGreaterThan(callsBefore); + + expect(() => vi.runAllTimers()).not.toThrow(); + }); + + it('attaches the observer when the messages wrapper first appears and disconnects on unmount', () => { + const { rerender, unmount } = render(); + + expect(screen.queryByTestId('messages-container')).toBeNull(); + expect(resizeObserverInstances).toHaveLength(0); + + rerender(); + + const messagesContainer = screen.getByTestId('messages-container'); + const resizeObserver = resizeObserverInstances[0]; + + expect(resizeObserverInstances).toHaveLength(1); + expect(resizeObserver.observe).toHaveBeenCalledWith(messagesContainer); + + unmount(); + + expect(resizeObserver.disconnect).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus/.npmignore b/gitnexus/.npmignore index 6a4cfb118..cf403314a 100644 --- a/gitnexus/.npmignore +++ b/gitnexus/.npmignore @@ -9,6 +9,10 @@ tsconfig.json .gitignore node_modules/ +# Vendor build artifacts (created during install, not shipped) +vendor/**/node_modules +vendor/**/build + # Package lock (consumers use their own) package-lock.json diff --git a/gitnexus/CHANGELOG.md b/gitnexus/CHANGELOG.md index 081eb9b26..80096345d 100644 --- a/gitnexus/CHANGELOG.md +++ b/gitnexus/CHANGELOG.md @@ -2,6 +2,81 @@ All notable changes to GitNexus will be documented in this file. +## [1.6.1] - 2026-04-13 + +### Added +- **Service group extractor expansion** — manifest extractor and broader extractor coverage (2/4 of #606 split) (#796) +- **Dart call patterns** for `await`, cascade, lambda, and widget-tree contexts (#801) + +### Fixed +- **Stack overflow and memory exhaustion** on large repository analysis (#814) +- **`tree-sitter-dart` install crash** — switched from git URL to npm tarball (#811) +- **Generic TypeScript awaited function calls** missing from the call graph (#804) +- **Runtime dependency on `file:../gitnexus-shared`** removed from the published package (#803) +- **Ruby `singleton_class` context** preserved during sequential parsing (#774) + +### Changed +- **DAG-based ingestion pipeline architecture** — pipeline phases now declare typed dependencies and run via a topologically sorted DAG; container-node logic extracted to `LanguageProvider`. Includes hardened lifecycle (try/finally cleanup, error wrapping, cycle reporting), tightened `ParseOutput.exportedTypeMap` immutability, and corrected phase dependencies (#809) + +## [1.6.0] - 2026-04-12 + +### Added +- **SemanticModel architecture refactor (SM-8 through SM-19)** — extracted registries into `model/` module with ISP-compliant interfaces: TypeRegistry, MethodRegistry, FieldRegistry, RegistrationTable, ResolutionContext (#786) + - HeritageMap built from accumulated `ExtractedHeritage[]` for MRO-aware resolution (#739) + - `lookupMethodByOwnerWithMRO` using HeritageMap for cross-class method dispatch (#740) + - MRO fast path before D2 fuzzy widening in call resolution (#741) + - BindingAccumulator for cross-file return type propagation (#743, #763) + - Restructured `resolveUncached` replacing `lookupFuzzy` data source for all tiers (#764) + - Deleted `lookupFuzzy`, `lookupFuzzyCallable`, `globalIndex`, `callableIndex` — replaced with structured lookups (#769) + - Deleted `resolveCallTarget` god-method — replaced with thin dispatcher delegating to `resolveMemberCall` (#744), `resolveStaticCall` (#754), `resolveFreeCall` (#756) (#770) +- **Service group infrastructure** — service boundary detection, contract extractors, sync pipeline, CLI/MCP tools, monorepo fixture; bridge.lbug storage and contract matching expansion (#795) +- **C# interface-to-interface heritage** capture (#789) +- **Vue SFC support** with destructured call result tracking (#604) +- **Java method reference** resolution — `obj::method` as call sites (#622) +- **C/C++ MethodExtractor** config with pure virtual detection (#617) +- **MethodExtractor configs** for Python, PHP, Swift, Dart, Rust, Ruby (#624) +- **METHOD_IMPLEMENTS edges** with overload disambiguation and MethodExtractor unification (#642) +- **Same-arity overload disambiguation** via type-hash suffix (#658) +- **`GITNEXUS_HOME` env var** to customize global directory (#746) +- **Verbose analyze output** prints skipped large file paths (#745) +- **Class name lookup index** for O(1) qualified lookups (#707, #716) +- **`lookupMethodByOwner` index** for O(1) cross-class chain resolution (#665) +- **Fuzzy lookup counters** for performance visibility (#708) + +### Fixed +- **Stack overflow on large PHP files** — iterative AST traversal (#783) +- **Large repository graph loading** failure (#732) +- **Windows multi-repo switching** — false 404 errors and stale repo context (#633) +- **`detect_changes` diff mapping** — map diff hunks to symbol line ranges (#779) +- **HTTP client vs Express route detection** and Spring interface attribution (#780) +- **VECTOR extension** not loaded during DB init for semantic search (#782) +- **tree-sitter-swift** postinstall patch for macOS ARM64 (#788) +- **tree-sitter-c** peer dependency conflict pinned (#723) +- **Constructor indexing** in methodByOwner (#694, #753) +- **Named binding processor** — `lookupExact` replaced with `lookupExactAll` (#755) +- **`.gitnexusignore` negation patterns** now respected (#654) +- **MCP setup** prefers global gitnexus binary over npx (#653) +- **CORS rejection** returns clean error instead of 500 (#646) +- **Array.push stack overflow** — replaced spread with loop (#650) +- **MCP stdout silencing** prevents embedder/pool-adapter conflicts (#645) +- **Web heartbeat** — graceful reconnection replaces aggressive disconnect (#643) +- **Web repo scoping** — backend calls scoped to active repo (#644) +- **OpenCode config path** and FTS extension load order (#781) +- **OnboardingGuide** dev-mode serve command corrected (#725) +- **Security issues** and critical bugs from code review (#709) + +### Changed +- Replaced class-type fuzzy lookups with structured indices in type-env (#733, #734, #736) +- Extracted `CLASS_LIKE_TYPES` constant (#693) + +## [1.5.3] - 2026-04-01 + +### Added +- **TypeScript/JavaScript MethodExtractor** config (#588) + +### Fixed +- **Wiki Azure OpenAI** compat and HTML viewer script injection (#618) + ## [1.5.2] - 2026-04-01 ### Fixed diff --git a/gitnexus/README.md b/gitnexus/README.md index 7e87c93b4..ed27bf728 100644 --- a/gitnexus/README.md +++ b/gitnexus/README.md @@ -234,6 +234,79 @@ Installed automatically by both `gitnexus analyze` (per-repo) and `gitnexus setu - Node.js >= 18 - Git repository (uses git for commit tracking) +## Release candidates + +Stable releases publish to the default `latest` dist-tag. When a pull request +with non-documentation changes merges into `main`, an automated workflow also +publishes a prerelease build under the `rc` dist-tag, so early adopters can +try in-flight fixes without waiting for the next stable cut. (Docs-only +merges are skipped.) + +```bash +# Try the latest release candidate (pre-stable — may change at any time) +npm install -g gitnexus@rc +# — or — +npx gitnexus@rc analyze +``` + +Release-candidate versions follow the standard semver prerelease format +`X.Y.Z-rc.N`, where `X.Y.Z` is the next stable target (bumped from the +current `latest` by patch by default; `minor` or `major` when kicking off a +bigger cycle) and `N` increments per published rc. Example sequence: +`1.6.2-rc.1`, `1.6.2-rc.2`, …, then once `1.6.2` ships stable, +`1.6.3-rc.1`. See the [Releases page](https://github.com/abhigyanpatwari/GitNexus/releases) +for the full list; stable `latest` is unaffected. + +## Troubleshooting + +### `Cannot destructure property 'package' of 'node.target' as it is null` + +This crash was caused by a dependency URL format that is incompatible with +certain npm/arborist versions ([npm/cli#8126](https://github.com/npm/cli/issues/8126)). +It is fixed in **gitnexus v1.6.2+**. Upgrade to the latest version: + +```bash +npx gitnexus@latest analyze # always uses the newest release +# — or — +npm install -g gitnexus@latest # upgrade a global install +``` + +If you still hit npm install issues after upgrading, these generic workarounds +may help: + +```bash +npm install -g npm@latest # update npm itself +npm cache clean --force # clear a possibly corrupt cache +``` + +### Installation fails with native module errors + +Some optional language grammars (Dart, Kotlin, Swift) require native compilation. If they fail, GitNexus still works — those languages will be skipped. + +If `npm install -g gitnexus` fails on native modules: + +```bash +# Ensure build tools are available (Linux/macOS) +# Ubuntu/Debian: sudo apt install python3 make g++ +# macOS: xcode-select --install + +# Retry installation +npm install -g gitnexus +``` + +### Analysis runs out of memory + +For very large repositories: + +```bash +# Increase Node.js heap size +NODE_OPTIONS="--max-old-space-size=16384" npx gitnexus analyze + +# Exclude large directories +echo "vendor/" >> .gitnexusignore +echo "dist/" >> .gitnexusignore +``` + ## Privacy - All processing happens locally on your machine diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index 9abd15b0e..def88c93c 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -1,12 +1,13 @@ { "name": "gitnexus", - "version": "1.5.3", + "version": "1.6.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "gitnexus", - "version": "1.5.3", + "version": "1.6.1", + "hasInstallScript": true, "license": "PolyForm-Noncommercial-1.0.0", "dependencies": { "@huggingface/transformers": "^3.0.0", @@ -29,7 +30,7 @@ "pandemonium": "^2.4.0", "tree-sitter": "^0.21.1", "tree-sitter-c": "0.23.2", - "tree-sitter-c-sharp": "^0.23.1", + "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "^0.23.4", "tree-sitter-go": "^0.23.0", "tree-sitter-java": "^0.23.5", @@ -61,8 +62,11 @@ "node": ">=20.0.0" }, "optionalDependencies": { - "tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4", + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0", + "tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4", "tree-sitter-kotlin": "^0.3.8", + "tree-sitter-proto": "file:./vendor/tree-sitter-proto", "tree-sitter-swift": "^0.6.0" } }, @@ -1224,6 +1228,12 @@ "win32" ] }, + "node_modules/@ladybugdb/core/node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "license": "MIT" + }, "node_modules/@modelcontextprotocol/sdk": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.28.0.tgz", @@ -4116,10 +4126,13 @@ } }, "node_modules/node-addon-api": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", - "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", - "license": "MIT" + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", + "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } }, "node_modules/node-api-headers": { "version": "1.8.0", @@ -5067,24 +5080,6 @@ } } }, - "node_modules/tree-sitter-c-sharp/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, - "node_modules/tree-sitter-c/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-cli": { "version": "0.23.2", "resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.23.2.tgz", @@ -5119,15 +5114,6 @@ } } }, - "node_modules/tree-sitter-cpp/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-dart": { "version": "1.0.0", "resolved": "git+ssh://git@github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4", @@ -5174,15 +5160,6 @@ } } }, - "node_modules/tree-sitter-go/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-java": { "version": "0.23.5", "resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.23.5.tgz", @@ -5202,15 +5179,6 @@ } } }, - "node_modules/tree-sitter-java/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-javascript": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz", @@ -5230,15 +5198,6 @@ } } }, - "node_modules/tree-sitter-javascript/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-kotlin": { "version": "0.3.8", "resolved": "https://registry.npmjs.org/tree-sitter-kotlin/-/tree-sitter-kotlin-0.3.8.tgz", @@ -5285,14 +5244,9 @@ } } }, - "node_modules/tree-sitter-php/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } + "node_modules/tree-sitter-proto": { + "resolved": "vendor/tree-sitter-proto", + "link": true }, "node_modules/tree-sitter-python": { "version": "0.23.4", @@ -5313,15 +5267,6 @@ } } }, - "node_modules/tree-sitter-python/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-ruby": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/tree-sitter-ruby/-/tree-sitter-ruby-0.23.1.tgz", @@ -5341,15 +5286,6 @@ } } }, - "node_modules/tree-sitter-ruby/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-rust": { "version": "0.23.1", "resolved": "https://registry.npmjs.org/tree-sitter-rust/-/tree-sitter-rust-0.23.1.tgz", @@ -5369,15 +5305,6 @@ } } }, - "node_modules/tree-sitter-rust/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-swift": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tree-sitter-swift/-/tree-sitter-swift-0.6.0.tgz", @@ -5407,16 +5334,6 @@ "license": "ISC", "optional": true }, - "node_modules/tree-sitter-swift/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "optional": true, - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tree-sitter-swift/node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -5453,24 +5370,6 @@ } } }, - "node_modules/tree-sitter-typescript/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, - "node_modules/tree-sitter/node_modules/node-addon-api": { - "version": "8.7.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz", - "integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==", - "license": "MIT", - "engines": { - "node": "^18 || ^20 || >= 21" - } - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -5875,6 +5774,14 @@ "peerDependencies": { "zod": "^3.25.28 || ^4" } + }, + "vendor/tree-sitter-proto": { + "version": "0.4.1", + "license": "MIT", + "optional": true, + "peerDependencies": { + "tree-sitter": ">=0.21.0" + } } } } diff --git a/gitnexus/package.json b/gitnexus/package.json index 07723f3e3..ad075041d 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -1,6 +1,6 @@ { "name": "gitnexus", - "version": "1.5.3", + "version": "1.6.1", "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.", "author": "Abhigyan Patwari", "license": "PolyForm-Noncommercial-1.0.0", @@ -46,6 +46,7 @@ "test:integration": "vitest run test/integration", "test:watch": "vitest", "test:coverage": "vitest run --coverage", + "postinstall": "node scripts/patch-tree-sitter-swift.cjs && node scripts/build-tree-sitter-proto.cjs", "prepare": "node scripts/build.js", "prepack": "node scripts/build.js" }, @@ -58,7 +59,6 @@ "commander": "^12.0.0", "cors": "^2.8.5", "express": "^4.19.2", - "gitnexus-shared": "file:../gitnexus-shared", "glob": "^11.0.0", "graphology": "^0.25.4", "graphology-indices": "^0.17.0", @@ -71,7 +71,7 @@ "pandemonium": "^2.4.0", "tree-sitter": "^0.21.1", "tree-sitter-c": "0.23.2", - "tree-sitter-c-sharp": "^0.23.1", + "tree-sitter-c-sharp": "0.23.1", "tree-sitter-cpp": "^0.23.4", "tree-sitter-go": "^0.23.0", "tree-sitter-java": "^0.23.5", @@ -84,8 +84,11 @@ "uuid": "^13.0.0" }, "optionalDependencies": { - "tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4", + "node-addon-api": "^8.0.0", + "node-gyp-build": "^4.8.0", + "tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4", "tree-sitter-kotlin": "^0.3.8", + "tree-sitter-proto": "file:./vendor/tree-sitter-proto", "tree-sitter-swift": "^0.6.0" }, "devDependencies": { diff --git a/gitnexus/scripts/build-tree-sitter-proto.cjs b/gitnexus/scripts/build-tree-sitter-proto.cjs new file mode 100644 index 000000000..d2828d5ba --- /dev/null +++ b/gitnexus/scripts/build-tree-sitter-proto.cjs @@ -0,0 +1,82 @@ +#!/usr/bin/env node +/** + * Build tree-sitter-proto native binding. + * + * Why this script exists: + * tree-sitter-proto is vendored under gitnexus/vendor/tree-sitter-proto/ + * and declared as a `file:` optionalDependency. Previously, the vendored + * package had its own `dependencies` and `install` script, which caused + * npm to create `vendor/tree-sitter-proto/node_modules/` and + * `vendor/tree-sitter-proto/build/` during install. Those directories + * blocked `rmdir` on global-install upgrade, producing: + * + * ENOTEMPTY: directory not empty, rmdir + * '.../gitnexus/vendor/tree-sitter-proto/node_modules/node-addon-api' + * + * (See https://github.com/abhigyanpatwari/GitNexus/issues/836.) + * + * We stripped `dependencies` and the `install` script from the vendored + * package.json, hoisted `node-addon-api` and `node-gyp-build` into + * gitnexus's own optionalDependencies, and moved native compilation here. + * + * What this does: + * Runs `npx node-gyp rebuild` inside `node_modules/tree-sitter-proto/` + * (which npm creates as a copy of vendor/tree-sitter-proto/ when + * resolving the file: dep). Build output lands in + * `node_modules/tree-sitter-proto/build/Release/tree_sitter_proto_binding.node` + * — under npm-managed territory, safe on upgrade. + * + * Mirrors scripts/patch-tree-sitter-swift.cjs. Best-effort: if any + * precondition fails (optional dep absent, no toolchain, --ignore-scripts), + * warn and exit 0 so gitnexus install still succeeds. + */ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto'); +const bindingGyp = path.join(protoDir, 'binding.gyp'); +const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node'); + +try { + if (!fs.existsSync(bindingGyp)) { + // tree-sitter-proto is an optionalDependency; absent when install + // skipped optional deps or the file: dep was not resolved. + process.exit(0); + } + + // Skip if the native binding already exists (idempotent re-run). + if (fs.existsSync(bindingNode)) { + process.exit(0); + } + + // Pre-flight: the hoisted build deps must be resolvable. + try { + require.resolve('node-addon-api'); + require.resolve('node-gyp-build'); + } catch (resolveErr) { + console.warn( + '[tree-sitter-proto] Skipping build: hoisted build deps not resolvable (%s).', + resolveErr.message, + ); + console.warn( + '[tree-sitter-proto] Proto parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', + ); + process.exit(0); + } + + console.log('[tree-sitter-proto] Building native binding...'); + execSync('npx node-gyp rebuild', { + cwd: protoDir, + stdio: 'pipe', + timeout: 180000, + }); + console.log('[tree-sitter-proto] Native binding built successfully'); +} catch (err) { + console.warn('[tree-sitter-proto] Could not build native binding:', err.message); + console.warn( + '[tree-sitter-proto] Proto (.proto) parsing will be unavailable. Non-proto gitnexus functionality is unaffected.', + ); + // Exit 0: optionalDependency failures must not fail the gitnexus install. + process.exit(0); +} diff --git a/gitnexus/scripts/patch-tree-sitter-swift.cjs b/gitnexus/scripts/patch-tree-sitter-swift.cjs new file mode 100644 index 000000000..6580b00e7 --- /dev/null +++ b/gitnexus/scripts/patch-tree-sitter-swift.cjs @@ -0,0 +1,78 @@ +#!/usr/bin/env node +/** + * WORKAROUND: tree-sitter-swift@0.6.0 binding.gyp build failure + * + * Background: + * tree-sitter-swift@0.6.0's binding.gyp contains an "actions" array that + * invokes `tree-sitter generate` to regenerate parser.c from grammar.js. + * This is intended for grammar developers, but the published npm package + * already ships pre-generated parser files (parser.c, scanner.c), so the + * actions are unnecessary for consumers. Since consumers don't have + * tree-sitter-cli installed, the actions always fail during `npm install`. + * + * Why we can't just upgrade: + * tree-sitter-swift@0.7.1 fixes this (removes postinstall, ships prebuilds), + * but it requires tree-sitter@^0.22.1. The upstream project pins tree-sitter + * to ^0.21.0 and all other grammar packages depend on that version. + * Upgrading tree-sitter would be a separate breaking change. + * + * How this workaround works: + * 1. tree-sitter-swift's own postinstall fails (npm warns but continues) + * 2. This script runs as gitnexus's postinstall + * 3. It removes the "actions" array from binding.gyp + * 4. It rebuilds the native binding with the cleaned binding.gyp + * + * TODO: Remove this script when tree-sitter is upgraded to ^0.22.x, + * which allows using tree-sitter-swift@0.7.1+ directly. + */ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +const swiftDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-swift'); +const bindingPath = path.join(swiftDir, 'binding.gyp'); + +try { + if (!fs.existsSync(bindingPath)) { + process.exit(0); + } + + const content = fs.readFileSync(bindingPath, 'utf8'); + let needsRebuild = false; + + if (content.includes('"actions"')) { + // Strip Python-style comments (#) and trailing commas before JSON parsing + const cleaned = content + .replace(/#[^\n]*/g, '') // Remove # comments + .replace(/,(\s*[\]}])/g, '$1'); // Remove trailing commas before ] or } + const gyp = JSON.parse(cleaned); + + if (gyp.targets && gyp.targets[0] && gyp.targets[0].actions) { + delete gyp.targets[0].actions; + fs.writeFileSync(bindingPath, JSON.stringify(gyp, null, 2) + '\n'); + console.log('[tree-sitter-swift] Patched binding.gyp (removed actions array)'); + needsRebuild = true; + } + } + + // Check if native binding exists + const bindingNode = path.join(swiftDir, 'build', 'Release', 'tree_sitter_swift_binding.node'); + if (!fs.existsSync(bindingNode)) { + needsRebuild = true; + } + + if (needsRebuild) { + console.log('[tree-sitter-swift] Rebuilding native binding...'); + execSync('npx node-gyp rebuild', { + cwd: swiftDir, + stdio: 'pipe', + timeout: 120000, + }); + console.log('[tree-sitter-swift] Native binding built successfully'); + } +} catch (err) { + console.warn('[tree-sitter-swift] Could not build native binding:', err.message); + console.warn( + '[tree-sitter-swift] You may need to manually run: cd node_modules/tree-sitter-swift && npx node-gyp rebuild', + ); +} diff --git a/gitnexus/src/cli/ai-context.ts b/gitnexus/src/cli/ai-context.ts index 1c7a95d7a..ae7f984fb 100644 --- a/gitnexus/src/cli/ai-context.ts +++ b/gitnexus/src/cli/ai-context.ts @@ -26,6 +26,7 @@ interface RepoStats { export interface AIContextOptions { skipAgentsMd?: boolean; + noStats?: boolean; } const GITNEXUS_START_MARKER = ''; @@ -64,6 +65,7 @@ function generateGitNexusContent( stats: RepoStats, generatedSkills?: GeneratedSkillInfo[], groupNames?: string[], + noStats?: boolean, ): string { const generatedRows = generatedSkills && generatedSkills.length > 0 @@ -87,7 +89,7 @@ function generateGitNexusContent( return `${GITNEXUS_START_MARKER} # GitNexus — Code Intelligence -This project is indexed by GitNexus as **${projectName}** (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows)`}. Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. > If any GitNexus tool warns the index is stale, run \`npx gitnexus analyze\` in terminal first. @@ -332,7 +334,13 @@ export async function generateAIContextFiles( options?: AIContextOptions, ): Promise<{ files: string[] }> { const groupNames = await findGroupsContainingRegistryName(projectName); - const content = generateGitNexusContent(projectName, stats, generatedSkills, groupNames); + const content = generateGitNexusContent( + projectName, + stats, + generatedSkills, + groupNames, + options?.noStats, + ); const createdFiles: string[] = []; if (!options?.skipAgentsMd) { diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index c77903de0..1e75ea675 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -20,8 +20,11 @@ import fs from 'fs/promises'; const HEAP_MB = 8192; const HEAP_FLAG = `--max-old-space-size=${HEAP_MB}`; +/** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */ +const STACK_KB = 4096; +const STACK_FLAG = `--stack-size=${STACK_KB}`; -/** Re-exec the process with an 8GB heap if we're currently below that. */ +/** Re-exec the process with an 8GB heap and larger stack if we're currently below that. */ function ensureHeap(): boolean { const nodeOpts = process.env.NODE_OPTIONS || ''; if (nodeOpts.includes('--max-old-space-size')) return false; @@ -29,8 +32,13 @@ function ensureHeap(): boolean { const v8Heap = v8.getHeapStatistics().heap_size_limit; if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false; + // --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+, + // so pass it only as a direct CLI argument, not via the environment. + const cliFlags = [HEAP_FLAG]; + if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG); + try { - execFileSync(process.execPath, [HEAP_FLAG, ...process.argv.slice(1)], { + execFileSync(process.execPath, [...cliFlags, ...process.argv.slice(1)], { stdio: 'inherit', env: { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim() }, }); @@ -47,6 +55,8 @@ export interface AnalyzeOptions { verbose?: boolean; /** Skip AGENTS.md and CLAUDE.md gitnexus block updates. */ skipAgentsMd?: boolean; + /** Omit volatile symbol/relationship counts from AGENTS.md and CLAUDE.md. */ + noStats?: boolean; /** Index the folder even when no .git directory is present. */ skipGit?: boolean; } @@ -137,9 +147,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption const origLog = console.log.bind(console); const origWarn = console.warn.bind(console); const origError = console.error.bind(console); + let barCurrentValue = 0; const barLog = (...args: any[]) => { process.stdout.write('\x1b[2K\r'); origLog(args.map((a) => (typeof a === 'string' ? a : String(a))).join(' ')); + bar.update(barCurrentValue); }; console.log = barLog; console.warn = barLog; @@ -150,6 +162,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption let phaseStart = Date.now(); const updateBar = (value: number, phaseLabel: string) => { + barCurrentValue = value; if (phaseLabel !== lastPhaseLabel) { lastPhaseLabel = phaseLabel; phaseStart = Date.now(); @@ -177,6 +190,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption embeddings: options?.embeddings, skipGit: options?.skipGit, skipAgentsMd: options?.skipAgentsMd, + noStats: options?.noStats, }, { onProgress: (_phase, percent, message) => { @@ -240,7 +254,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption processes: s.processes, }, skillResult.skills, - { skipAgentsMd: options?.skipAgentsMd }, + { skipAgentsMd: options?.skipAgentsMd, noStats: options?.noStats }, ); } } catch { @@ -282,7 +296,51 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption console.warn = origWarn; console.error = origError; bar.stop(); - console.error(`\n Analysis failed: ${err.message}\n`); + + const msg = err.message || String(err); + console.error(`\n Analysis failed: ${msg}\n`); + + // Provide helpful guidance for known failure modes + if ( + msg.includes('Maximum call stack size exceeded') || + msg.includes('call stack') || + msg.includes('Map maximum size') || + msg.includes('Invalid array length') || + msg.includes('Invalid string length') || + msg.includes('allocation failed') || + msg.includes('heap out of memory') || + msg.includes('JavaScript heap') + ) { + console.error(' This error typically occurs on very large repositories.'); + console.error(' Suggestions:'); + console.error(' 1. Add large vendored/generated directories to .gitnexusignore'); + console.error(' 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"'); + console.error(' 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"'); + console.error(''); + } else if (msg.includes('ERESOLVE') || msg.includes('Could not resolve dependency')) { + // Note: the original arborist "Cannot destructure property 'package' of + // 'node.target'" crash happens inside npm *before* gitnexus code runs, + // so it can't be caught here. This branch handles dependency-resolution + // errors that surface at runtime (e.g. dynamic require failures). + console.error(' This looks like an npm dependency resolution issue.'); + console.error(' Suggestions:'); + console.error(' 1. Clear the npm cache: npm cache clean --force'); + console.error(' 2. Update npm: npm install -g npm@latest'); + console.error(' 3. Reinstall gitnexus: npm install -g gitnexus@latest'); + console.error(' 4. Or try npx directly: npx gitnexus@latest analyze'); + console.error(''); + } else if ( + msg.includes('MODULE_NOT_FOUND') || + msg.includes('Cannot find module') || + msg.includes('ERR_MODULE_NOT_FOUND') + ) { + console.error(' A required module could not be loaded. The installation may be corrupt.'); + console.error(' Suggestions:'); + console.error(' 1. Reinstall: npm install -g gitnexus@latest'); + console.error(' 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze'); + console.error(''); + } + process.exitCode = 1; return; } diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index 75940bcbf..02581ae47 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -26,6 +26,7 @@ program .option('--embeddings', 'Enable embedding generation for semantic search (off by default)') .option('--skills', 'Generate repo-specific skill files from detected communities') .option('--skip-agents-md', 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md') + .option('--no-stats', 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md') .option('--skip-git', 'Index a folder without requiring a .git directory') .option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)') .addHelpText( diff --git a/gitnexus/src/cli/setup.ts b/gitnexus/src/cli/setup.ts index ed941e6a4..8263405a5 100644 --- a/gitnexus/src/cli/setup.ts +++ b/gitnexus/src/cli/setup.ts @@ -265,7 +265,7 @@ async function setupOpenCode(result: SetupResult): Promise { return; } - const configPath = path.join(opencodeDir, 'config.json'); + const configPath = path.join(opencodeDir, 'opencode.json'); try { const existing = await readJsonFile(configPath); const config = existing || {}; diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts new file mode 100644 index 000000000..fb51ac052 --- /dev/null +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -0,0 +1,112 @@ +/** + * Shared AST utilities for the embedding pipeline. + * Centralizes parser caching and tree-sitter node lookups + * used by both chunker.ts and structural-extractor.ts. + */ + +import { getLanguageFromFilename } from 'gitnexus-shared'; +import { + createParserForLanguage, + isLanguageAvailable, + resolveLanguageKey, +} from '../tree-sitter/parser-loader.js'; + +const parserCache = new Map(); + +/** + * Ensure parser is initialized and language is loaded, then parse content. + * Returns null if language is unavailable or parsing fails. + */ +export const ensureAndParse = async (content: string, filePath: string): Promise => { + const language = getLanguageFromFilename(filePath); + if (!language) return null; + if (!isLanguageAvailable(language)) return null; + + const parserKey = resolveLanguageKey(language, filePath); + let parserInstance = parserCache.get(parserKey); + if (!parserInstance) { + parserInstance = await createParserForLanguage(language, filePath); + parserCache.set(parserKey, parserInstance); + } + + return parserInstance.parse(content); +}; + +const FUNCTION_LIKE_TYPES = new Set([ + 'function_declaration', + 'function_definition', + 'method_declaration', + 'method_definition', + 'function_item', + 'function_signature_item', + 'arrow_function', + 'function_expression', + 'generator_function_declaration', + 'generator_function', + 'async_function_declaration', + 'async_arrow_function', + 'constructor_declaration', + 'constructor_definition', + 'compact_constructor_declaration', + 'short_function_declaration', + 'proc_declaration', + 'func_literal', + 'local_function_statement', + 'anonymous_function', + 'lambda_literal', + 'init_declaration', + 'deinit_declaration', +]); + +/** + * Find the first function/method-like declaration in a snippet AST. + * Used by the chunker when parsing node.content where absolute line + * numbers don't apply. + */ +export const findFunctionNode = (root: any): any | null => { + if (FUNCTION_LIKE_TYPES.has(root.type)) return root; + + for (let i = 0; i < root.namedChildCount; i++) { + const child = root.namedChild(i); + if (!child) continue; + if (FUNCTION_LIKE_TYPES.has(child.type)) return child; + const found = findFunctionNode(child); + if (found) return found; + } + + return null; +}; + +/** + * Find the first class/struct/interface/enum-like declaration in an AST. + * Used when parsing node.content (a snippet, not a full file) where + * absolute line numbers don't apply. + */ +export const findDeclarationNode = (root: any): any | null => { + const CLASS_LIKE_TYPES = new Set([ + 'class_declaration', + 'class_definition', + 'struct_declaration', + 'struct_item', + 'interface_declaration', + 'interface_definition', + 'enum_declaration', + 'enum_item', + 'type_declaration', // Go: type X struct + 'declaration', // Go: type X struct + 'object_declaration', // Kotlin: object + 'impl_item', // Rust: impl + ]); + + if (CLASS_LIKE_TYPES.has(root.type)) return root; + + for (let i = 0; i < root.namedChildCount; i++) { + const child = root.namedChild(i); + if (!child) continue; + if (CLASS_LIKE_TYPES.has(child.type)) return child; + const found = findDeclarationNode(child); + if (found) return found; + } + + return null; +}; diff --git a/gitnexus/src/core/embeddings/character-chunk.ts b/gitnexus/src/core/embeddings/character-chunk.ts new file mode 100644 index 000000000..e4f6f949c --- /dev/null +++ b/gitnexus/src/core/embeddings/character-chunk.ts @@ -0,0 +1,63 @@ +/** + * Character-based sliding window chunking (pure, no tree-sitter dependency) + */ + +import { buildLineIndex, resolveChunkLines } from './line-index.js'; + +export interface Chunk { + text: string; + chunkIndex: number; + startOffset: number; + endOffset: number; + startLine: number; + endLine: number; +} + +export const characterChunk = ( + content: string, + startLine: number, + endLine: number, + chunkSize: number = 1200, + overlap: number = 120, +): Chunk[] => { + if (content.length <= chunkSize) { + return [ + { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }, + ]; + } + + const chunks: Chunk[] = []; + let offset = 0; + const lineOffsets = buildLineIndex(content); + + while (offset < content.length) { + const end = Math.min(offset + chunkSize, content.length); + const chunkText = content.slice(offset, end); + const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine); + + chunks.push({ + text: chunkText, + chunkIndex: chunks.length, + startOffset: offset, + endOffset: end, + startLine: lineRange.startLine, + endLine: lineRange.endLine, + }); + + offset = end - overlap; + if (offset >= content.length) break; + if (end >= content.length) break; + if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) { + offset = end; + } + } + + return chunks; +}; diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts new file mode 100644 index 000000000..114a5f68c --- /dev/null +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -0,0 +1,363 @@ +/** + * Chunker Module + * + * Splits code nodes into chunks for embedding. + * - Function/Method: AST-aware chunking by statement boundaries + * - Other types: character-based sliding window fallback + * - Short content (≤ chunkSize): no chunking + */ + +export { type Chunk, characterChunk } from './character-chunk.js'; + +import { characterChunk } from './character-chunk.js'; +import type { Chunk } from './character-chunk.js'; +import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js'; +import { buildLineIndex, resolveChunkLines } from './line-index.js'; + +/** + * Main chunkNode function: dispatches by label + */ +export const chunkNode = async ( + label: string, + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number = 1200, + overlap: number = 120, +): Promise => { + // Content fits in one chunk — no splitting needed + if (content.length <= chunkSize) { + return [ + { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }, + ]; + } + + // Only function-like labels get AST chunking + if (label === 'Function' || label === 'Method' || label === 'Constructor') { + try { + const astChunks = await astChunk(content, filePath, startLine, endLine, chunkSize, overlap); + if (astChunks.length > 0) return astChunks; + } catch { + // AST parsing failed — fall through to character fallback + } + } + + if (label === 'Class' || label === 'Interface') { + try { + const declarationChunks = await declarationChunk( + label, + content, + filePath, + startLine, + endLine, + chunkSize, + overlap, + ); + if (declarationChunks.length > 0) return declarationChunks; + } catch { + // AST parsing failed — fall through to character fallback + } + } + + // Character-based fallback for everything else + return characterChunk(content, startLine, endLine, chunkSize, overlap); +}; + +/** + * AST-based chunking for Function/Method + * Parse snippet content, locate the function declaration node, + * split body by statement boundaries. + */ +const astChunk = async ( + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number, + overlap: number, +): Promise => { + const tree = await ensureAndParse(content, filePath); + if (!tree) return []; + + const root = tree.rootNode; + const lineOffsets = buildLineIndex(content); + + // Find the function/method declaration in the snippet AST. + // tree-sitter parses node.content (a snippet), so rows are relative (0-based). + const targetNode = findFunctionNode(root); + if (!targetNode) return []; + + // Get the body (statements) via childForFieldName('body') + const bodyNode = targetNode.childForFieldName('body'); + if (!bodyNode) return []; + + // Extract individual statements + const statements: Array<{ startIndex: number; endIndex: number }> = []; + for (let i = 0; i < bodyNode.namedChildCount; i++) { + const child = bodyNode.namedChild(i); + if (!child) continue; + statements.push({ + startIndex: child.startIndex, + endIndex: child.endIndex, + }); + } + + if (statements.length === 0) return []; + + return chunkByUnits( + content, + lineOffsets, + startLine, + chunkSize, + overlap, + statements, + targetNode.startIndex, + targetNode.endIndex, + true, + true, + ); +}; + +const DECLARATION_BODY_NODE_TYPES = new Set([ + 'class_body', + 'object_type', + 'declaration_list', + 'interface_body', +]); + +const FIELD_LIKE_MEMBER_TYPES = new Set([ + 'field_definition', + 'public_field_definition', + 'property_definition', + 'property_signature', + 'variable_declarator', + 'lexical_declaration', + 'pair', + 'enum_assignment', +]); + +const declarationChunk = async ( + label: 'Class' | 'Interface', + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number, + overlap: number, +): Promise => { + const tree = await ensureAndParse(content, filePath); + if (!tree) return []; + + const targetNode = findDeclarationNode(tree.rootNode); + if (!targetNode) return []; + + const bodyNode = getDeclarationBodyNode(targetNode); + if (!bodyNode) return []; + + const members = collectDeclarationUnits(bodyNode, label); + if (members.length === 0) return []; + + return chunkByUnits( + content, + buildLineIndex(content), + startLine, + chunkSize, + overlap, + members, + targetNode.startIndex, + targetNode.endIndex, + false, + false, + ); +}; + +const buildChunk = ( + content: string, + lineOffsets: Int32Array, + chunkIndex: number, + startOffset: number, + endOffset: number, + baseStartLine: number, +): Chunk => { + const lineRange = resolveChunkLines(lineOffsets, startOffset, endOffset, baseStartLine); + return { + text: content.slice(startOffset, endOffset), + chunkIndex, + startOffset, + endOffset, + startLine: lineRange.startLine, + endLine: lineRange.endLine, + }; +}; + +const chunkByUnits = ( + content: string, + lineOffsets: Int32Array, + baseStartLine: number, + chunkSize: number, + overlap: number, + units: Array<{ startIndex: number; endIndex: number }>, + containerStartOffset: number, + containerEndOffset: number, + includeContainerPrefixOnFirstChunk: boolean, + includeContainerSuffixOnLastChunk: boolean, +): Chunk[] => { + const chunks: Chunk[] = []; + let chunkStartUnitIdx = 0; + + while (chunkStartUnitIdx < units.length) { + const chunkStartOffset = + chunkStartUnitIdx === 0 && includeContainerPrefixOnFirstChunk + ? containerStartOffset + : units[chunkStartUnitIdx].startIndex; + + let chunkEndUnitIdx = chunkStartUnitIdx; + let candidateEndOffset = + chunkEndUnitIdx === units.length - 1 && includeContainerSuffixOnLastChunk + ? containerEndOffset + : units[chunkEndUnitIdx].endIndex; + + while (chunkEndUnitIdx + 1 < units.length) { + const nextEndOffset = + chunkEndUnitIdx + 1 === units.length - 1 && includeContainerSuffixOnLastChunk + ? containerEndOffset + : units[chunkEndUnitIdx + 1].endIndex; + if (nextEndOffset - chunkStartOffset > chunkSize) break; + chunkEndUnitIdx += 1; + candidateEndOffset = nextEndOffset; + } + + if (candidateEndOffset - chunkStartOffset > chunkSize) { + const oversizedUnit = units[chunkStartUnitIdx]; + const oversizedLineRange = resolveChunkLines( + lineOffsets, + oversizedUnit.startIndex, + oversizedUnit.endIndex, + baseStartLine, + ); + const oversizedChunks = characterChunk( + content.slice(oversizedUnit.startIndex, oversizedUnit.endIndex), + oversizedLineRange.startLine, + oversizedLineRange.endLine, + chunkSize, + overlap, + ).map((chunk, offsetIdx) => ({ + ...chunk, + chunkIndex: chunks.length + offsetIdx, + startOffset: chunk.startOffset + oversizedUnit.startIndex, + endOffset: chunk.endOffset + oversizedUnit.startIndex, + })); + chunks.push(...oversizedChunks); + chunkStartUnitIdx += 1; + continue; + } + + chunks.push( + buildChunk( + content, + lineOffsets, + chunks.length, + chunkStartOffset, + candidateEndOffset, + baseStartLine, + ), + ); + + if (chunkEndUnitIdx === units.length - 1) { + break; + } + + const nextChunkStartUnitIdx = findOverlapStartIndex( + units, + chunkStartUnitIdx, + chunkEndUnitIdx, + overlap, + ); + if (nextChunkStartUnitIdx <= chunkStartUnitIdx) { + chunkStartUnitIdx = chunkEndUnitIdx + 1; + } else { + chunkStartUnitIdx = nextChunkStartUnitIdx; + } + } + + return chunks; +}; + +const findOverlapStartIndex = ( + statements: Array<{ startIndex: number; endIndex: number }>, + chunkStartStmtIdx: number, + chunkEndStmtIdx: number, + overlapSize: number, +): number => { + if (overlapSize <= 0) return chunkEndStmtIdx + 1; + + let overlapStartIdx = chunkEndStmtIdx; + while (overlapStartIdx > chunkStartStmtIdx) { + const overlapLength = + statements[chunkEndStmtIdx].endIndex - statements[overlapStartIdx - 1].startIndex; + if (overlapLength > overlapSize) break; + overlapStartIdx -= 1; + } + + return overlapStartIdx; +}; + +const getDeclarationBodyNode = (node: any): any | null => { + const bodyNode = node.childForFieldName?.('body'); + if (bodyNode) return bodyNode; + + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child) continue; + if (DECLARATION_BODY_NODE_TYPES.has(child.type)) return child; + } + + return null; +}; + +const collectDeclarationUnits = ( + bodyNode: any, + label: 'Class' | 'Interface', +): Array<{ startIndex: number; endIndex: number }> => { + const members: Array<{ startIndex: number; endIndex: number; groupable: boolean }> = []; + + for (let i = 0; i < bodyNode.namedChildCount; i++) { + const child = bodyNode.namedChild(i); + if (!child) continue; + members.push({ + startIndex: child.startIndex, + endIndex: child.endIndex, + groupable: label === 'Class' && FIELD_LIKE_MEMBER_TYPES.has(child.type), + }); + } + + if (members.length === 0) return []; + + const grouped: Array<{ startIndex: number; endIndex: number }> = []; + let current = members[0]; + + for (let i = 1; i < members.length; i++) { + const next = members[i]; + if (current.groupable && next.groupable) { + current = { + startIndex: current.startIndex, + endIndex: next.endIndex, + groupable: true, + }; + continue; + } + grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex }); + current = next; + } + + grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex }); + return grouped; +}; diff --git a/gitnexus/src/core/embeddings/embedder.ts b/gitnexus/src/core/embeddings/embedder.ts index e2d26c9e7..d27718ce5 100644 --- a/gitnexus/src/core/embeddings/embedder.ts +++ b/gitnexus/src/core/embeddings/embedder.ts @@ -157,6 +157,11 @@ export const initEmbedder = async ( try { // Configure transformers.js environment env.allowLocalModels = false; + // Default cache to user-writable location. transformers.js defaults to + // ./node_modules/.cache inside its own install dir, which is unwritable + // when gitnexus is installed globally (e.g. /usr/lib/node_modules/). + // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface. + env.cacheDir = process.env.HF_HOME ?? `${process.env.HOME}/.cache/huggingface`; const isDev = process.env.NODE_ENV === 'development'; if (isDev) { diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index d3dc0854e..302903f8b 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -3,12 +3,13 @@ * * Orchestrates the background embedding process: * 1. Query embeddable nodes from LadybugDB - * 2. Generate text representations - * 3. Batch embed using transformers.js - * 4. Update LadybugDB with embeddings + * 2. Generate text representations with enriched metadata + * 3. Chunk long nodes, batch embed + * 4. Update LadybugDB with chunk-aware embeddings * 5. Create vector index for semantic search */ +import { createHash } from 'crypto'; import { initEmbedder, embedBatch, @@ -16,19 +17,54 @@ import { embeddingToArray, isEmbedderReady, } from './embedder.js'; -import { generateBatchEmbeddingTexts } from './text-generator.js'; +import { generateEmbeddingText } from './text-generator.js'; +import { chunkNode, characterChunk } from './chunker.js'; +import { extractStructuralNames } from './structural-extractor.js'; import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult, type ModelProgress, + type EmbeddingContext, DEFAULT_EMBEDDING_CONFIG, EMBEDDABLE_LABELS, + isShortLabel, + LABELS_WITH_EXPORTED, + STRUCTURAL_LABELS, + collectBestChunks, } from './types.js'; +import { + EMBEDDING_TABLE_NAME, + EMBEDDING_INDEX_NAME, + CREATE_VECTOR_INDEX_QUERY, + STALE_HASH_SENTINEL, +} from '../lbug/schema.js'; +import { loadVectorExtension } from '../lbug/lbug-adapter.js'; const isDev = process.env.NODE_ENV === 'development'; +/** + * Compute a stable content fingerprint for an embeddable node. + * Used to detect when the underlying text has changed so stale vectors + * can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for + * vector-indexed rows). + */ +export const contentHashForNode = ( + node: EmbeddableNode, + config: Partial = {}, +): string => { + // Hash must be deterministic across runs, so exclude methodNames/fieldNames + // which are populated during the batch loop via AST extraction. + // Using only node.content ensures the hash stays stable. + const text = generateEmbeddingText( + { ...node, methodNames: undefined, fieldNames: undefined }, + node.content, + config, + ); + return createHash('sha1').update(text).digest('hex'); +}; + /** * Progress callback type */ @@ -36,37 +72,50 @@ export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void; /** * Query all embeddable nodes from LadybugDB - * Uses table-specific queries (File has different schema than code elements) + * Uses table-specific queries for different label types */ const queryEmbeddableNodes = async ( executeQuery: (cypher: string) => Promise, ): Promise => { const allNodes: EmbeddableNode[] = []; - // Query each embeddable table with table-specific columns for (const label of EMBEDDABLE_LABELS) { try { let query: string; - if (label === 'File') { - // File nodes don't have startLine/endLine + if (label === 'Method') { + // Method has parameterCount and returnType query = ` - MATCH (n:File) - RETURN n.id AS id, n.name AS name, 'File' AS label, - n.filePath AS filePath, n.content AS content + MATCH (n:Method) + RETURN n.id AS id, n.name AS name, 'Method' AS label, + n.filePath AS filePath, n.content AS content, + n.startLine AS startLine, n.endLine AS endLine, + n.isExported AS isExported, n.description AS description, + n.parameterCount AS parameterCount, n.returnType AS returnType + `; + } else if (LABELS_WITH_EXPORTED.has(label)) { + // Function, Class, Interface have isExported and description + query = ` + MATCH (n:\`${label}\`) + RETURN n.id AS id, n.name AS name, '${label}' AS label, + n.filePath AS filePath, n.content AS content, + n.startLine AS startLine, n.endLine AS endLine, + n.isExported AS isExported, n.description AS description `; } else { - // Code elements have startLine/endLine + // Multi-language tables (Struct, Enum, etc.) — have description but no isExported query = ` - MATCH (n:${label}) - RETURN n.id AS id, n.name AS name, '${label}' AS label, + MATCH (n:\`${label}\`) + RETURN n.id AS id, n.name AS name, '${label}' AS label, n.filePath AS filePath, n.content AS content, - n.startLine AS startLine, n.endLine AS endLine + n.startLine AS startLine, n.endLine AS endLine, + n.description AS description `; } const rows = await executeQuery(query); for (const row of rows) { + const hasExportedColumn = label === 'Method' || LABELS_WITH_EXPORTED.has(label); allNodes.push({ id: row.id ?? row[0], name: row.name ?? row[1], @@ -75,10 +124,17 @@ const queryEmbeddableNodes = async ( content: row.content ?? row[4] ?? '', startLine: row.startLine ?? row[5], endLine: row.endLine ?? row[6], + isExported: hasExportedColumn ? (row.isExported ?? row[7]) : undefined, + description: row.description ?? (hasExportedColumn ? row[8] : row[7]), + ...(label === 'Method' + ? { + parameterCount: row.parameterCount ?? row[9], + returnType: row.returnType ?? row[10], + } + : {}), }); } } catch (error) { - // Table might not exist or be empty, continue if (isDev) { console.warn(`Query for ${label} nodes failed:`, error); } @@ -89,52 +145,52 @@ const queryEmbeddableNodes = async ( }; /** - * Batch INSERT embeddings into separate CodeEmbedding table - * Using a separate lightweight table avoids copy-on-write overhead - * that occurs when UPDATEing nodes with large content fields + * Batch INSERT chunk-aware embeddings into CodeEmbedding table */ -const batchInsertEmbeddings = async ( +export const batchInsertEmbeddings = async ( executeWithReusedStatement: ( cypher: string, paramsList: Array>, ) => Promise, - updates: Array<{ id: string; embedding: number[] }>, + updates: Array<{ + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + embedding: number[]; + contentHash?: string; + }>, ): Promise => { - // INSERT into separate embedding table - much more memory efficient! - const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`; - const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding })); + const cypher = `CREATE (e:${EMBEDDING_TABLE_NAME} {id: $id, nodeId: $nodeId, chunkIndex: $chunkIndex, startLine: $startLine, endLine: $endLine, embedding: $embedding, contentHash: $contentHash})`; + const paramsList = updates.map((u) => ({ + id: `${u.nodeId}:${u.chunkIndex}`, + nodeId: u.nodeId, + chunkIndex: u.chunkIndex, + startLine: u.startLine, + endLine: u.endLine, + embedding: u.embedding, + contentHash: u.contentHash ?? STALE_HASH_SENTINEL, + })); await executeWithReusedStatement(cypher, paramsList); }; /** * Create the vector index for semantic search - * Now indexes the separate CodeEmbedding table - */ -let vectorExtensionLoaded = false; + * Now indexes the separate CodeEmbedding table. + * Delegates extension loading to lbug-adapter's loadVectorExtension(), + * which owns the VECTOR extension lifecycle and state tracking. + + */ const createVectorIndex = async ( executeQuery: (cypher: string) => Promise, ): Promise => { - // LadybugDB v0.15+ requires explicit VECTOR extension loading (once per session) - if (!vectorExtensionLoaded) { - try { - await executeQuery('INSTALL VECTOR'); - await executeQuery('LOAD EXTENSION VECTOR'); - vectorExtensionLoaded = true; - } catch { - // Extension may already be loaded — CREATE_VECTOR_INDEX will fail clearly if not - vectorExtensionLoaded = true; - } - } - - const cypher = ` - CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine') - `; + // Delegate to the adapter which tracks loaded state and handles DB reconnect resets + await loadVectorExtension(); try { - await executeQuery(cypher); + await executeQuery(CREATE_VECTOR_INDEX_QUERY); } catch (error) { - // Index might already exist if (isDev) { console.warn('Vector index creation warning:', error); } @@ -149,6 +205,11 @@ const createVectorIndex = async ( * @param onProgress - Callback for progress updates * @param config - Optional configuration override * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode) + * @param context - Optional repo/server context for metadata enrichment + * @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode. + * Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd + * and re-embedded; nodes not in the map are embedded fresh. + */ export const runEmbeddingPipeline = async ( executeQuery: (cypher: string) => Promise, @@ -159,6 +220,8 @@ export const runEmbeddingPipeline = async ( onProgress: EmbeddingProgressCallback, config: Partial = {}, skipNodeIds?: Set, + context?: EmbeddingContext, + existingEmbeddings?: Map, ): Promise => { const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config }; @@ -194,13 +257,65 @@ export const runEmbeddingPipeline = async ( // Phase 2: Query embeddable nodes let nodes = await queryEmbeddableNodes(executeQuery); - // Incremental mode: filter out nodes that already have embeddings - if (skipNodeIds && skipNodeIds.size > 0) { + // Apply context metadata + if (context?.repoName) { + for (const node of nodes) { + node.repoName = context.repoName; + node.serverName = context.serverName; + } + } + + // Incremental mode: compare content hashes, delete stale rows, skip fresh ones. + // Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them + // (avoids double computation). + const computedStaleHashes = new Map(); + if (existingEmbeddings && existingEmbeddings.size > 0) { const beforeCount = nodes.length; - nodes = nodes.filter((n) => !skipNodeIds.has(n.id)); + const staleNodeIds: string[] = []; + nodes = nodes.filter((n) => { + const existingHash = existingEmbeddings.get(n.id); + if (existingHash === undefined) { + // New node — needs embedding + return true; + } + const currentHash = contentHashForNode(n, finalConfig); + if (currentHash !== existingHash) { + // Content changed — cache hash for reuse during insert, mark for DELETE + re-embed + computedStaleHashes.set(n.id, currentHash); + staleNodeIds.push(n.id); + return true; + } + // Hash matches — skip (fresh); no need to cache hash for skipped nodes + return false; + }); + + // DELETE stale embedding rows so they can be re-inserted + // (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern) + if (staleNodeIds.length > 0) { + if (isDev) { + console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`); + } + try { + await executeWithReusedStatement( + `MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, + staleNodeIds.map((nodeId) => ({ nodeId })), + ); + } catch (err) { + // "does not exist" = rows already gone — safe to proceed. + // All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT + // for vector-indexed properties) — propagate so the pipeline aborts cleanly. + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes('does not exist')) { + throw new Error( + `[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`, + ); + } + } + } + if (isDev) { console.log( - `📦 Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`, + `📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`, ); } } @@ -212,6 +327,11 @@ export const runEmbeddingPipeline = async ( } if (totalNodes === 0) { + // Ensure the vector index exists even when no new nodes need embedding. + // A prior crash or first-time incremental run may have left CodeEmbedding + // rows without ever reaching index creation. + await createVectorIndex(executeQuery); + onProgress({ phase: 'ready', percent: 100, @@ -221,10 +341,12 @@ export const runEmbeddingPipeline = async ( return; } - // Phase 3: Batch embed nodes + // Phase 3: Chunk + embed nodes const batchSize = finalConfig.batchSize; - const totalBatches = Math.ceil(totalNodes / batchSize); + const chunkSize = finalConfig.chunkSize; + const overlap = finalConfig.overlap; let processedNodes = 0; + let totalChunks = 0; onProgress({ phase: 'embedding', @@ -232,39 +354,116 @@ export const runEmbeddingPipeline = async ( nodesProcessed: 0, totalNodes, currentBatch: 0, - totalBatches, + totalBatches: Math.ceil(totalNodes / batchSize), }); - for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { - const start = batchIndex * batchSize; - const end = Math.min(start + batchSize, totalNodes); - const batch = nodes.slice(start, end); + // Process in batches of nodes + for (let batchIndex = 0; batchIndex < totalNodes; batchIndex += batchSize) { + const batch = nodes.slice(batchIndex, batchIndex + batchSize); - // Generate texts for this batch - const texts = generateBatchEmbeddingTexts(batch, finalConfig); + // Chunk each node and generate text + const allTexts: string[] = []; + const allUpdates: Array<{ + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + contentHash: string; + }> = []; - // Embed the batch - const embeddings = await embedBatch(texts); + for (const node of batch) { + const isShort = isShortLabel(node.label); + const startLine = node.startLine ?? 0; + const endLine = node.endLine ?? 0; - // Update LadybugDB with embeddings - const updates = batch.map((node, i) => ({ - id: node.id, - embedding: embeddingToArray(embeddings[i]), - })); + // Extract structural names for class-like nodes via AST extractors + if (!isShort && STRUCTURAL_LABELS.has(node.label)) { + try { + const names = await extractStructuralNames(node.content, node.filePath); + node.methodNames = names.methodNames; + node.fieldNames = names.fieldNames; + } catch { + // AST extraction failed — names stay undefined, text-generator handles gracefully + } + } - await batchInsertEmbeddings(executeWithReusedStatement, updates); + // Compute content hash once per node (re-use cached value for stale nodes) + const hash = computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig); + + let chunks: Array<{ text: string; chunkIndex: number; startLine: number; endLine: number }>; + if (isShort) { + chunks = [{ text: node.content, chunkIndex: 0, startLine, endLine }]; + } else { + try { + chunks = await chunkNode( + node.label, + node.content, + node.filePath, + startLine, + endLine, + chunkSize, + overlap, + ); + } catch (chunkErr) { + if (isDev) { + console.warn( + `⚠️ AST chunking failed for ${node.label} "${node.name}" (${node.filePath}), falling back to character-based chunking:`, + chunkErr, + ); + } + chunks = characterChunk(node.content, startLine, endLine, chunkSize, overlap); + } + } + + for (const chunk of chunks) { + const text = generateEmbeddingText(node, chunk.text, finalConfig); + allTexts.push(text); + allUpdates.push({ + nodeId: node.id, + chunkIndex: chunk.chunkIndex, + startLine: chunk.startLine, + endLine: chunk.endLine, + contentHash: hash, + }); + } + } + + // Embed chunk texts in sub-batches to control memory + const EMBED_SUB_BATCH = 8; + for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) { + const subTexts = allTexts.slice(si, si + EMBED_SUB_BATCH); + const subUpdates = allUpdates.slice(si, si + EMBED_SUB_BATCH); + + let embeddings: Float32Array[]; + try { + embeddings = await embedBatch(subTexts); + } catch (embedErr) { + console.error( + `❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`, + embedErr, + ); + throw embedErr; + } + + const dbUpdates = subUpdates.map((u, i) => ({ + ...u, + embedding: embeddingToArray(embeddings[i]), + })); + + await batchInsertEmbeddings(executeWithReusedStatement, dbUpdates); + } processedNodes += batch.length; + totalChunks += allUpdates.length; - // Report progress (20-90% for embedding phase) const embeddingProgress = 20 + (processedNodes / totalNodes) * 70; onProgress({ phase: 'embedding', percent: Math.round(embeddingProgress), nodesProcessed: processedNodes, totalNodes, - currentBatch: batchIndex + 1, - totalBatches, + currentBatch: Math.floor(batchIndex / batchSize) + 1, + totalBatches: Math.ceil(totalNodes / batchSize), }); } @@ -282,7 +481,6 @@ export const runEmbeddingPipeline = async ( await createVectorIndex(executeQuery); - // Complete onProgress({ phase: 'ready', percent: 100, @@ -291,7 +489,9 @@ export const runEmbeddingPipeline = async ( }); if (isDev) { - console.log('✅ Embedding pipeline complete!'); + console.log( + `✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`, + ); } } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; @@ -311,15 +511,7 @@ export const runEmbeddingPipeline = async ( }; /** - * Perform semantic search using the vector index - * - * Uses CodeEmbedding table and queries each node table to get metadata - * - * @param executeQuery - Function to execute Cypher queries - * @param query - Search query text - * @param k - Number of results to return (default: 10) - * @param maxDistance - Maximum distance threshold (default: 0.5) - * @returns Array of search results ordered by relevance + * Perform semantic search using the vector index with chunk deduplication */ export const semanticSearch = async ( executeQuery: (cypher: string) => Promise, @@ -331,37 +523,46 @@ export const semanticSearch = async ( throw new Error('Embedding model not initialized. Run embedding pipeline first.'); } - // Embed the query const queryEmbedding = await embedText(query); const queryVec = embeddingToArray(queryEmbedding); const queryVecStr = `[${queryVec.join(',')}]`; - // Query the vector index on CodeEmbedding to get nodeIds and distances - const vectorQuery = ` - CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', - CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${k}) - YIELD node AS emb, distance - WITH emb, distance - WHERE distance < ${maxDistance} - RETURN emb.nodeId AS nodeId, distance - ORDER BY distance - `; + const bestChunks = await collectBestChunks(k, async (fetchLimit) => { + const vectorQuery = ` + CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', + CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit}) + YIELD node AS emb, distance + WITH emb, distance + WHERE distance < ${maxDistance} + RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex, + emb.startLine AS startLine, emb.endLine AS endLine, distance + ORDER BY distance + `; - const embResults = await executeQuery(vectorQuery); + const embResults = await executeQuery(vectorQuery); + return embResults.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + distance: row.distance ?? row[4], + })); + }); - if (embResults.length === 0) { + if (bestChunks.size === 0) { return []; } // Group results by label for batched metadata queries - const byLabel = new Map>(); - for (const embRow of embResults) { - const nodeId = embRow.nodeId ?? embRow[0]; - const distance = embRow.distance ?? embRow[1]; + const byLabel = new Map< + string, + Array<{ nodeId: string; distance: number } & Record> + >(); + for (const [nodeId, chunk] of Array.from(bestChunks.entries()).slice(0, k)) { const labelEndIdx = nodeId.indexOf(':'); const label = labelEndIdx > 0 ? nodeId.substring(0, labelEndIdx) : 'Unknown'; if (!byLabel.has(label)) byLabel.set(label, []); - byLabel.get(label)!.push({ nodeId, distance }); + byLabel.get(label)!.push({ nodeId, ...chunk }); } // Batch-fetch metadata per label @@ -370,19 +571,11 @@ export const semanticSearch = async ( for (const [label, items] of byLabel) { const idList = items.map((i) => `'${i.nodeId.replace(/'/g, "''")}'`).join(', '); try { - let nodeQuery: string; - if (label === 'File') { - nodeQuery = ` - MATCH (n:File) WHERE n.id IN [${idList}] - RETURN n.id AS id, n.name AS name, n.filePath AS filePath - `; - } else { - nodeQuery = ` - MATCH (n:${label}) WHERE n.id IN [${idList}] - RETURN n.id AS id, n.name AS name, n.filePath AS filePath, - n.startLine AS startLine, n.endLine AS endLine - `; - } + const nodeQuery = ` + MATCH (n:\`${label}\`) WHERE n.id IN [${idList}] + RETURN n.id AS id, n.name AS name, n.filePath AS filePath, + n.startLine AS startLine, n.endLine AS endLine + `; const nodeRows = await executeQuery(nodeQuery); const rowMap = new Map(); for (const row of nodeRows) { @@ -398,8 +591,8 @@ export const semanticSearch = async ( label, filePath: nodeRow.filePath ?? nodeRow[2] ?? '', distance: item.distance, - startLine: label !== 'File' ? (nodeRow.startLine ?? nodeRow[3]) : undefined, - endLine: label !== 'File' ? (nodeRow.endLine ?? nodeRow[4]) : undefined, + startLine: item.startLine, + endLine: item.endLine, }); } } @@ -408,7 +601,6 @@ export const semanticSearch = async ( } } - // Re-sort by distance since batch queries may have mixed order results.sort((a, b) => a.distance - b.distance); return results; @@ -416,16 +608,6 @@ export const semanticSearch = async ( /** * Semantic search with graph expansion (flattened results) - * - * Note: With multi-table schema, graph traversal is simplified. - * Returns semantic matches with their metadata. - * For full graph traversal, use execute_vector_cypher tool directly. - * - * @param executeQuery - Function to execute Cypher queries - * @param query - Search query text - * @param k - Number of initial semantic matches (default: 5) - * @param _hops - Unused (kept for API compatibility). - * @returns Semantic matches with metadata */ export const semanticSearchWithContext = async ( executeQuery: (cypher: string) => Promise, @@ -433,8 +615,6 @@ export const semanticSearchWithContext = async ( k: number = 5, _hops: number = 1, ): Promise => { - // For multi-table schema, just return semantic search results - // Graph traversal is complex with separate tables - use execute_vector_cypher instead const results = await semanticSearch(executeQuery, query, k, 0.5); return results.map((r) => ({ diff --git a/gitnexus/src/core/embeddings/line-index.ts b/gitnexus/src/core/embeddings/line-index.ts new file mode 100644 index 000000000..2dc12de3d --- /dev/null +++ b/gitnexus/src/core/embeddings/line-index.ts @@ -0,0 +1,50 @@ +export interface ResolvedLineRange { + startLine: number; + endLine: number; +} + +export const buildLineIndex = (content: string): Int32Array => { + const offsets: number[] = [0]; + for (let i = 0; i < content.length; i++) { + if (content.charCodeAt(i) === 10) offsets.push(i + 1); + } + return new Int32Array(offsets); +}; + +const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => { + if (lineOffsets.length === 0) return 0; + const maxOffset = lineOffsets[lineOffsets.length - 1]; + if (charOffset < 0) return 0; + if (charOffset > maxOffset) return maxOffset; + return charOffset; +}; + +export const lineFromOffset = (lineOffsets: Int32Array, charOffset: number): number => { + if (lineOffsets.length === 0) return 0; + + const clamped = clampOffset(lineOffsets, charOffset); + let lo = 0; + let hi = lineOffsets.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (lineOffsets[mid] <= clamped) lo = mid; + else hi = mid - 1; + } + return lo; +}; + +export const resolveChunkLines = ( + lineOffsets: Int32Array, + startOffset: number, + endOffset: number, + baseStartLine: number, +): ResolvedLineRange => { + const relativeStartLine = lineFromOffset(lineOffsets, startOffset); + const effectiveEndOffset = endOffset > startOffset ? endOffset - 1 : startOffset; + const relativeEndLine = lineFromOffset(lineOffsets, effectiveEndOffset); + + return { + startLine: baseStartLine + relativeStartLine, + endLine: baseStartLine + relativeEndLine, + }; +}; diff --git a/gitnexus/src/core/embeddings/server-mapping.ts b/gitnexus/src/core/embeddings/server-mapping.ts new file mode 100644 index 000000000..fa685ac8f --- /dev/null +++ b/gitnexus/src/core/embeddings/server-mapping.ts @@ -0,0 +1,37 @@ +/** + * Server Mapping Configuration + * + * Reads ~/.gitnexus/server-mapping.json to map repo names to service names. + * Used in embedding text to enrich metadata with microservice context. + */ + +import fs from 'fs/promises'; +import path from 'path'; +import os from 'os'; + +const MAPPING_FILE = path.join(os.homedir(), '.gitnexus', 'server-mapping.json'); + +let cachedMapping: Record | null = null; + +/** + * Read the server mapping file and return the serverName for a given repoName. + * Returns undefined if no mapping exists. + */ +export const readServerMapping = async (repoName: string): Promise => { + try { + if (!cachedMapping) { + const raw = await fs.readFile(MAPPING_FILE, 'utf-8'); + cachedMapping = JSON.parse(raw); + } + return cachedMapping[repoName]; + } catch { + return undefined; + } +}; + +/** + * Clear the cached mapping (useful for testing or after file changes) + */ +export const clearServerMappingCache = (): void => { + cachedMapping = null; +}; diff --git a/gitnexus/src/core/embeddings/structural-extractor.ts b/gitnexus/src/core/embeddings/structural-extractor.ts new file mode 100644 index 000000000..7035ee569 --- /dev/null +++ b/gitnexus/src/core/embeddings/structural-extractor.ts @@ -0,0 +1,89 @@ +/** + * Structural Extractor Module + * + * Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor + * to extract method and field names for embedding text generation. + */ + +import { getProviderForFile } from '../ingestion/languages/index.js'; +import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js'; +import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js'; +import type { LanguageProvider } from '../ingestion/language-provider.js'; +import { buildTypeEnv } from '../ingestion/type-env.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { ensureAndParse, findDeclarationNode } from './ast-utils.js'; + +export interface StructuralNames { + methodNames: string[]; + fieldNames: string[]; +} + +const NOOP_SYMBOL_TABLE = { + lookupExactAll: () => [], + lookupExact: () => undefined, + lookupExactFull: () => undefined, +} as any; + +/** + * Extract method and field names from a class/struct/interface node + * using the ingestion pipeline's AST extractors. + */ +export const extractStructuralNames = async ( + content: string, + filePath: string, +): Promise => { + const provider = getProviderForFile(filePath); + if (!provider) return { methodNames: [], fieldNames: [] }; + + const tree = await ensureAndParse(content, filePath); + if (!tree) return { methodNames: [], fieldNames: [] }; + + // Parse node.content (a snippet) — find declaration directly, not by range + const classNode = findDeclarationNode(tree.rootNode); + if (!classNode) return { methodNames: [], fieldNames: [] }; + + const language = provider.id; + + const methodNames = extractMethodNames(classNode, provider, filePath, language); + const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language); + + return { methodNames, fieldNames }; +}; + +function extractMethodNames( + classNode: any, + provider: LanguageProvider, + filePath: string, + language: SupportedLanguages, +): string[] { + if (!provider.methodExtractor) return []; + + const context: MethodExtractorContext = { filePath, language }; + const result: ExtractedMethods | null = provider.methodExtractor.extract(classNode, context); + if (!result?.methods?.length) return []; + + return result.methods.map((m) => m.name); +} + +function extractFieldNames( + classNode: any, + provider: LanguageProvider, + tree: any, + filePath: string, + language: SupportedLanguages, +): string[] { + if (!provider.fieldExtractor) return []; + + const typeEnv = buildTypeEnv(tree, language); + + const context: FieldExtractorContext = { + typeEnv, + symbolTable: NOOP_SYMBOL_TABLE, + filePath, + language, + }; + const result: ExtractedFields | null = provider.fieldExtractor.extract(classNode, context); + if (!result?.fields?.length) return []; + + return result.fields.map((f) => f.name); +} diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index d74a9f894..5b96f6b5e 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -1,206 +1,253 @@ /** * Text Generator Module * - * Pure functions to generate embedding text from code nodes. - * Combines node metadata with code snippets for semantic matching. + * Generates enriched embedding text from code nodes with metadata. + * Supports chunkable labels (Function/Method with AST chunking), + * Class-specific structural text, and short-node direct embed. + * + * Method/field names for Class nodes are extracted by the ingestion + * pipeline's AST extractors and passed via node.methodNames/node.fieldNames. */ import type { EmbeddableNode, EmbeddingConfig } from './types.js'; -import { DEFAULT_EMBEDDING_CONFIG } from './types.js'; +import { DEFAULT_EMBEDDING_CONFIG, isShortLabel } from './types.js'; /** - * Extract the filename from a file path + * Truncate description to max length at sentence/word boundary */ -const getFileName = (filePath: string): string => { - const parts = filePath.split('/'); - return parts[parts.length - 1] || filePath; -}; +const truncateDescription = (text: string, maxLength: number): string => { + if (text.length <= maxLength) return text; -/** - * Extract the directory path from a file path - */ -const getDirectory = (filePath: string): string => { - const parts = filePath.split('/'); - parts.pop(); - return parts.join('/') || ''; -}; + const truncated = text.slice(0, maxLength); -/** - * Truncate content to max length, preserving word boundaries - */ -const truncateContent = (content: string, maxLength: number): string => { - if (content.length <= maxLength) { - return content; + // Try sentence boundary (. ! ?) + const sentenceEnd = Math.max( + truncated.lastIndexOf('. '), + truncated.lastIndexOf('! '), + truncated.lastIndexOf('? '), + ); + if (sentenceEnd > maxLength * 0.5) { + return truncated.slice(0, sentenceEnd + 1); } - // Find last space before maxLength to avoid cutting words - const truncated = content.slice(0, maxLength); + // Try word boundary const lastSpace = truncated.lastIndexOf(' '); - - if (lastSpace > maxLength * 0.8) { - return truncated.slice(0, lastSpace) + '...'; + if (lastSpace > maxLength * 0.5) { + return truncated.slice(0, lastSpace); } - return truncated + '...'; + return truncated; }; /** * Clean code content for embedding - * Removes excessive whitespace while preserving structure */ const cleanContent = (content: string): string => { - return ( - content - // Normalize line endings - .replace(/\r\n/g, '\n') - // Remove excessive blank lines (more than 2) - .replace(/\n{3,}/g, '\n\n') - // Trim each line - .split('\n') - .map((line) => line.trimEnd()) - .join('\n') - .trim() - ); + return content + .replace(/\r\n/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .split('\n') + .map((line) => line.trimEnd()) + .join('\n') + .trim(); }; /** - * Generate embedding text for a Function node + * Build metadata header for a node */ -const generateFunctionText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Function: ${node.name}`, `File: ${getFileName(node.filePath)}`]; +const buildMetadataHeader = (node: EmbeddableNode, config: Partial): string => { + const parts: string[] = []; - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); + // Label + name + parts.push(`${node.label}: ${node.name}`); + + // Repo name + if (node.repoName) { + parts.push(`Repo: ${node.repoName}`); } - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); + // Server name (optional) + if (node.serverName) { + parts.push(`Server: ${node.serverName}`); + } + + // Full file path + parts.push(`Path: ${node.filePath}`); + + // Export status + if (node.isExported !== undefined) { + parts.push(`Export: ${node.isExported}`); + } + + // Description (truncated) + if (node.description) { + const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength; + const truncated = truncateDescription(node.description, maxLen); + if (truncated) { + parts.push(truncated); + } } return parts.join('\n'); }; -/** - * Generate embedding text for a Class node - */ -const generateClassText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Class: ${node.name}`, `File: ${getFileName(node.filePath)}`]; +const generateCodeBodyText = ( + node: EmbeddableNode, + codeBody: string, + config: Partial, +): string => { + const header = buildMetadataHeader(node, config); + const cleaned = cleanContent(codeBody); + return `${header}\n\n${cleaned}`; +}; - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); +/** + * Generate embedding text for Class nodes + * Signature + properties + method name list only (no method bodies) + * Method/field names come from AST extractors via node.methodNames/node.fieldNames. + */ +const generateClassText = ( + node: EmbeddableNode, + codeBody: string, + config: Partial, +): string => { + return generateStructuralTypeText(node, codeBody, config); +}; + +const generateStructuralTypeText = ( + node: EmbeddableNode, + codeBody: string, + config: Partial, +): string => { + const header = buildMetadataHeader(node, config); + const parts: string[] = [header]; + + if (node.methodNames?.length) { + parts.push(`Methods: ${node.methodNames.join(', ')}`); + } + if (node.fieldNames?.length) { + parts.push(`Properties: ${node.fieldNames.join(', ')}`); } - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); + const declarationOnly = extractDeclarationOnly(cleanContent(node.content)); + if (declarationOnly) { + parts.push('', declarationOnly); + } + + const cleanedChunk = cleanContent(codeBody); + if (cleanedChunk && cleanedChunk !== cleanContent(node.content)) { + parts.push('', cleanedChunk); } return parts.join('\n'); }; -/** - * Generate embedding text for a Method node - */ -const generateMethodText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Method: ${node.name}`, `File: ${getFileName(node.filePath)}`]; - - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); - } - - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); - } - - return parts.join('\n'); -}; +const DECL_START_RE = + /^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/; /** - * Generate embedding text for an Interface node + * Extract class/interface/struct declaration lines, skipping method bodies. + * - Brace-based languages: detects method signatures (lines with `(` and `{`) + * and skips until depth returns to class body level. + * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction). */ -const generateInterfaceText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Interface: ${node.name}`, `File: ${getFileName(node.filePath)}`]; +export const extractDeclarationOnly = (content: string): string => { + const lines = content.split('\n'); + const declLines: string[] = []; + let depth = 0; + let started = false; + let classDepth = 0; + let skipDepth = 0; - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); + for (const [idx, line] of lines.entries()) { + const trimmed = line.trim(); + + if (!started) { + if (DECL_START_RE.test(trimmed)) { + // Non-brace language check: current line or next 3 lines must have `{` + const nextLines = lines.slice(idx + 1, idx + 4); + if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) { + return ''; + } + started = true; + declLines.push(trimmed); + for (const ch of trimmed) { + if (ch === '{') depth++; + else if (ch === '}') depth--; + } + if (depth > 0) classDepth = depth; + } + continue; + } + + // Always update depth (even when skipping) + const opens = (trimmed.match(/{/g) || []).length; + const closes = (trimmed.match(/}/g) || []).length; + const prevDepth = depth; + depth += opens - closes; + + if (skipDepth > 0) { + if (depth <= classDepth) { + skipDepth = 0; + // Closing brace of class + if (depth <= 0) { + declLines.push(trimmed); + break; + } + } + continue; + } + + // Detect method signature: line has both `(` and `{` and goes deeper than class body + const hasParens = trimmed.includes('('); + const hasOpenBrace = opens > 0; + if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) { + if (opens === closes && trimmed.endsWith(';')) { + // Property with function/object initializer like `config = { timeout: 5000 };` — keep + declLines.push(trimmed); + } + // else: single-line or multi-line method — skip entirely + if (opens !== closes) { + skipDepth = classDepth; + } + continue; + } + + declLines.push(trimmed); + + if (depth <= 0 && declLines.length > 1) break; } - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); - } - - return parts.join('\n'); -}; - -/** - * Generate embedding text for a File node - * Uses file name and first N characters of content - */ -const generateFileText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`File: ${node.name}`, `Path: ${node.filePath}`]; - - if (node.content) { - const cleanedContent = cleanContent(node.content); - // For files, use a shorter snippet since they can be very long - const snippet = truncateContent(cleanedContent, Math.min(maxSnippetLength, 300)); - parts.push('', snippet); - } - - return parts.join('\n'); + return declLines.join('\n').trim(); }; /** * Generate embedding text for any embeddable node * Dispatches to the appropriate generator based on node label - * - * @param node - The node to generate text for - * @param config - Optional configuration for max snippet length - * @returns Text suitable for embedding */ export const generateEmbeddingText = ( node: EmbeddableNode, + codeBody: string, config: Partial = {}, ): string => { - const maxSnippetLength = config.maxSnippetLength ?? DEFAULT_EMBEDDING_CONFIG.maxSnippetLength; - - switch (node.label) { - case 'Function': - return generateFunctionText(node, maxSnippetLength); - case 'Class': - return generateClassText(node, maxSnippetLength); - case 'Method': - return generateMethodText(node, maxSnippetLength); - case 'Interface': - return generateInterfaceText(node, maxSnippetLength); - case 'File': - return generateFileText(node, maxSnippetLength); - default: - // Fallback for any other embeddable type - return `${node.label}: ${node.name}\nPath: ${node.filePath}`; + if (isShortLabel(node.label)) { + const header = buildMetadataHeader(node, config); + const cleaned = cleanContent(node.content); + return `${header}\n\n${cleaned}`; } + + if (node.label === 'Class') { + return generateClassText(node, codeBody, config); + } + + if (node.label === 'Interface') { + return generateStructuralTypeText(node, codeBody, config); + } + + return generateCodeBodyText(node, codeBody, config); }; /** - * Generate embedding texts for a batch of nodes - * - * @param nodes - Array of nodes to generate text for - * @param config - Optional configuration - * @returns Array of texts in the same order as input nodes + * Export truncation helper for testing */ -export const generateBatchEmbeddingTexts = ( - nodes: EmbeddableNode[], - config: Partial = {}, -): string[] => { - return nodes.map((node) => generateEmbeddingText(node, config)); -}; +export { truncateDescription }; diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index 599faa3a9..c24dcdf40 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -5,10 +5,40 @@ */ /** - * Node labels that should be embedded for semantic search - * These are code elements that benefit from semantic matching + * Node labels that need chunking (have code body, potentially long) */ -export const EMBEDDABLE_LABELS = ['Function', 'Class', 'Method', 'Interface', 'File'] as const; +export const CHUNKABLE_LABELS = [ + 'Function', + 'Method', + 'Constructor', + 'Class', + 'Interface', + 'Struct', + 'Enum', + 'Trait', + 'Impl', + 'Macro', + 'Namespace', +] as const; + +/** + * Node labels that are short (no chunking needed, embed directly) + */ +export const SHORT_LABELS = [ + 'TypeAlias', + 'Typedef', + 'Const', + 'Property', + 'Record', + 'Union', + 'Static', + 'Variable', +] as const; + +/** + * All embeddable labels (union of CHUNKABLE + SHORT) + */ +export const EMBEDDABLE_LABELS = [...CHUNKABLE_LABELS, ...SHORT_LABELS] as const; export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number]; @@ -18,6 +48,39 @@ export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number]; export const isEmbeddableLabel = (label: string): label is EmbeddableLabel => EMBEDDABLE_LABELS.includes(label as EmbeddableLabel); +/** + * Check if a label needs chunking + */ +export const isChunkableLabel = (label: string): boolean => + (CHUNKABLE_LABELS as readonly string[]).includes(label); + +/** + * Check if a label is a short type (no chunking) + */ +export const isShortLabel = (label: string): boolean => + (SHORT_LABELS as readonly string[]).includes(label); + +/** + * Node labels that have structural names (methods/fields) extractable via AST + */ +export const STRUCTURAL_LABELS: ReadonlySet = new Set([ + 'Class', + 'Struct', + 'Interface', + 'Enum', +]); + +/** + * Node labels that have isExported column in their schema + */ +export const LABELS_WITH_EXPORTED = new Set([ + 'Function', + 'Class', + 'Interface', + 'Method', + 'CodeElement', +]) as ReadonlySet; + /** * Embedding pipeline phases */ @@ -57,6 +120,12 @@ export interface EmbeddingConfig { device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm'; /** Maximum characters of code snippet to include */ maxSnippetLength: number; + /** Maximum code chunk size in characters (for chunking long code) */ + chunkSize: number; + /** Overlap between chunks in characters */ + overlap: number; + /** Maximum description length in characters */ + maxDescriptionLength: number; } /** @@ -70,6 +139,9 @@ export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = { dimensions: 384, device: 'auto', maxSnippetLength: 500, + chunkSize: 1200, + overlap: 120, + maxDescriptionLength: 150, }; /** @@ -96,6 +168,34 @@ export interface EmbeddableNode { content: string; startLine?: number; endLine?: number; + isExported?: boolean; + description?: string; + parameterCount?: number; + returnType?: string; + repoName?: string; + serverName?: string; + methodNames?: string[]; + fieldNames?: string[]; +} + +/** + * Cached embedding entry restored from LadybugDB before a graph rebuild + */ +export interface CachedEmbedding { + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + embedding: number[]; + contentHash?: string; +} + +/** + * Context info for embedding pipeline (repo/server metadata enrichment) + */ +export interface EmbeddingContext { + repoName?: string; + serverName?: string; } /** @@ -108,3 +208,75 @@ export interface ModelProgress { loaded?: number; total?: number; } + +export interface ChunkSearchRow { + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + distance: number; +} + +export interface BestChunkMatch { + chunkIndex: number; + startLine: number; + endLine: number; + distance: number; +} + +/** + * Deduplicate vector search chunk results by nodeId, + * keeping the chunk with smallest distance for each node. + */ +export const dedupBestChunks = ( + rows: ChunkSearchRow[], + limit?: number, +): Map => { + const best = new Map(); + for (const row of rows) { + const existing = best.get(row.nodeId); + if (!existing || row.distance < existing.distance) { + best.set(row.nodeId, { + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + distance: row.distance, + }); + } + if (limit !== undefined && best.size >= limit) break; + } + return best; +}; + +const DEFAULT_FETCH_MULTIPLIER = 4; +const DEFAULT_FETCH_BUFFER = 8; +const DEFAULT_MAX_FETCH = 200; + +/** + * Fetch vector-search chunks until we have enough unique nodeIds + * or can tell the result set is exhausted. + */ +export const collectBestChunks = async ( + limit: number, + fetchRows: (fetchLimit: number) => Promise, + maxFetch: number = DEFAULT_MAX_FETCH, +): Promise> => { + if (limit <= 0) return new Map(); + + let fetchLimit = Math.max(limit * DEFAULT_FETCH_MULTIPLIER, limit + DEFAULT_FETCH_BUFFER); + let previousFetchLimit = 0; + + while (fetchLimit > previousFetchLimit) { + const rows = await fetchRows(fetchLimit); + const bestChunks = dedupBestChunks(rows, limit); + + if (bestChunks.size >= limit || rows.length < fetchLimit) { + return bestChunks; + } + + previousFetchLimit = fetchLimit; + fetchLimit = fetchLimit >= maxFetch ? fetchLimit * 2 : Math.min(maxFetch, fetchLimit * 2); + } + + return new Map(); +}; diff --git a/gitnexus/src/core/group/PIPELINE.md b/gitnexus/src/core/group/PIPELINE.md new file mode 100644 index 000000000..7730b48e7 --- /dev/null +++ b/gitnexus/src/core/group/PIPELINE.md @@ -0,0 +1,139 @@ +# Group Analysis Pipeline + +Flow chart of the cross-repo contract extraction + matching pipeline. +This covers what runs **inside this PR** (extractors + manifest) and +the downstream handoff to the bridge storage (PR #795) and +cross-impact query (PR #606). + +## High-level overview + +```mermaid +flowchart TD + A[group.yaml] --> B[GroupConfig parser] + B --> C{For each repo
in group} + C --> D[Per-repo LadybugDB
indexed by main pipeline] + + D --> E1[TopicExtractor] + D --> E2[HttpRouteExtractor] + D --> E3[GrpcExtractor] + + E1 --> F[ExtractedContract array
per repo] + E2 --> F + E3 --> F + + B --> M[ManifestExtractor] + M --> G[Manifest contracts
+ cross-links] + + F --> H[Contract matching
exact + wildcard] + G --> H + + H --> I[(bridge.lbug
#795)] + + I --> J[runGroupImpact
#606] + J --> K[CrossRepoImpact] +``` + +## Per-repo extractor pipeline + +Each extractor under `src/core/group/extractors/` follows the same +two-strategy shape: + +```mermaid +flowchart TD + R[RepoHandle + CypherExecutor
for this repo] --> S{Graph-assisted
Strategy A
available?} + + S -->|yes| A1[Cypher query against
per-repo LadybugDB] + A1 --> A2{non-empty
result?} + A2 -->|yes| OUT[ExtractedContract array] + A2 -->|no| B1 + + S -->|no| B1[Source-scan Strategy B] + B1 --> B2[glob repo source files] + B2 --> B3{ext in registry?} + B3 -->|yes| B4[Per-language plugin
scan parsed tree] + B3 -->|no| SKIP[skip file] + B4 --> OUT + + SKIP --> B2 +``` + +**Strategy A** (graph-assisted) uses Cypher over edges already produced +by the main ingestion pipeline: +- HTTP: `HANDLES_ROUTE` / `FETCHES` edges from `(File)-[]->(Route)` +- topic: none (pipeline doesn't yet produce topic nodes — Strategy B only) +- gRPC: none (Strategy B + proto map only) + +**Strategy B** (source-scan) is 100% tree-sitter based after this PR. +Each `*-patterns/.ts` plugin owns its grammar + S-expression +queries; the top-level orchestrator imports neither. + +## Plugin architecture + +```mermaid +flowchart LR + O[Orchestrator
topic|http|grpc-extractor.ts] --> REG[REGISTRY
*-patterns/index.ts] + REG --> P1[java.ts
tree-sitter-java] + REG --> P2[go.ts
tree-sitter-go] + REG --> P3[python.ts
tree-sitter-python] + REG --> P4[node.ts
JS + TS + TSX] + REG --> P5[php.ts
tree-sitter-php
HTTP only] + REG --> P6[proto.ts
tree-sitter-proto
gRPC only, optional] + + P1 --> SCAN[tree-sitter-scanner.ts
compilePatterns + runCompiledPatterns] + P2 --> SCAN + P3 --> SCAN + P4 --> SCAN + P5 --> SCAN + P6 --> SCAN + + SCAN --> DET[Detection objects
TopicMeta / HttpDetection / GrpcDetection] + DET --> O + O --> CT[ExtractedContract array] +``` + +The orchestrator never imports a grammar. Adding a new language / +framework = drop one file in `*-patterns/`, register it in +`index.ts`. No orchestrator edits required. + +## Manifest extraction + +```mermaid +flowchart TD + Y[group.yaml links] --> ME[ManifestExtractor] + ME --> LOOP{for each link} + LOOP --> RES[resolveSymbol
label-scoped Cypher] + RES --> OK{found?} + OK -->|yes| REF[real symbol uid + ref] + OK -->|no| SYN[synthetic uid
manifest::repo::cid] + + REF --> EMIT[emit provider + consumer
Contract objects
+ CrossLink] + SYN --> EMIT + + EMIT --> BRIDGE[(bridge.lbug
#795)] +``` + +Label-scoped queries in `resolveSymbol` keep accidental cross-matches +out: +- `topic` → `(n:Function|Method|Class|Interface)` +- `grpc` method → `(n:Function|Method)`, service → `(n:Class|Interface)` +- `lib` → `(n:Package|Module)` + +## Cross-impact query (PR #606) + +```mermaid +flowchart TD + U[User changes symbol S
in repo R] --> LI[Local impact engine
per-repo uid expansion] + LI --> IDS[Affected uid set] + + IDS --> BR[Bridge query
MATCH Contract WHERE uid IN ids] + BR --> CL[CrossLink traversal] + CL --> OTHER[Matching contract in
other repo] + + OTHER --> FE[Fan-out impact
to consuming repo] + FE --> OUT[CrossRepoImpact
per affected repo] +``` + +The bridge stores every extracted contract keyed by `symbolUid`. +Manifest-sourced contracts use the synthetic uid form so both sides +of the `(local impact) ↔ (bridge query)` join derive the same uid +without coordinating through any shared state. diff --git a/gitnexus/src/core/group/bridge-db.ts b/gitnexus/src/core/group/bridge-db.ts new file mode 100644 index 000000000..864a79599 --- /dev/null +++ b/gitnexus/src/core/group/bridge-db.ts @@ -0,0 +1,588 @@ +import fsp from 'node:fs/promises'; +import path from 'node:path'; +import { createHash } from 'node:crypto'; +import lbug from '@ladybugdb/core'; +import type { LbugValue } from '@ladybugdb/core'; +import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js'; +import { BRIDGE_SCHEMA_QUERIES, BRIDGE_SCHEMA_VERSION } from './bridge-schema.js'; +import { dedupeContracts, dedupeCrossLinks } from './normalization.js'; + +export function contractNodeId( + repo: string, + contractId: string, + role: string, + filePath: string, +): string { + return createHash('sha256').update(`${repo}\0${contractId}\0${role}\0${filePath}`).digest('hex'); +} + +/* ------------------------------------------------------------------ */ +/* ContractLookupIndex — in-memory lookup for findContractNode */ +/* ------------------------------------------------------------------ */ + +/** + * In-memory index of contract node IDs keyed three ways, mirroring the + * three-tier fallback lookup in {@link findContractNode}. Built once per + * `writeBridge` call after all contracts are successfully inserted, then + * consulted for every cross-link — which eliminates the former N+1 query + * pattern (up to `6 × cross-links` DB round-trips) and turns cross-link + * resolution into constant-time per link. + * + * Keys are deliberately flat strings (not tuples) so `Map` + * works; the separator `\0` can't occur in any legal repo path / file + * path / symbol identifier, which makes the encoding injection-safe. + */ +export interface ContractLookupIndex { + /** tier 1: `repo + role + symbolUid` → contract node id */ + byUid: Map; + /** tier 2: `repo + role + filePath + symbolName` → contract node id */ + byRef: Map; + /** tier 3: `repo + role + filePath` → list of contract node ids in that file */ + byFile: Map; +} + +export function createContractLookupIndex(): ContractLookupIndex { + return { + byUid: new Map(), + byRef: new Map(), + byFile: new Map(), + }; +} + +function uidKey(repo: string, role: string, symbolUid: string): string { + return `${repo}\0${role}\0${symbolUid}`; +} + +function refKey(repo: string, role: string, filePath: string, symbolName: string): string { + return `${repo}\0${role}\0${filePath}\0${symbolName}`; +} + +function fileKey(repo: string, role: string, filePath: string): string { + return `${repo}\0${role}\0${filePath}`; +} + +/** + * Add a successfully-inserted contract to the lookup index. Must be called + * AFTER the DB insert succeeds (not before) so failed inserts don't poison + * the index and cause cross-links to point at non-existent rows. + */ +export function indexContract( + index: ContractLookupIndex, + contract: StoredContract, + nodeId: string, +): void { + if (contract.symbolUid) { + index.byUid.set(uidKey(contract.repo, contract.role, contract.symbolUid), nodeId); + } + index.byRef.set( + refKey(contract.repo, contract.role, contract.symbolRef.filePath, contract.symbolRef.name), + nodeId, + ); + const fk = fileKey(contract.repo, contract.role, contract.symbolRef.filePath); + const existing = index.byFile.get(fk); + if (existing) { + existing.push(nodeId); + } else { + index.byFile.set(fk, [nodeId]); + } +} + +/** + * Resolve a cross-link endpoint (consumer or provider reference) to an + * already-inserted contract node id. Returns `null` if no match — the + * caller is expected to count that as a dropped link in `WriteBridgeReport`. + * + * The resolution order matches the pre-cache DB-query behavior: + * 1. exact `symbolUid` match in the same `(repo, role)` scope + * 2. exact `(filePath, symbolName)` match + * 3. if exactly one contract lives in the file → that one (fallback for + * legacy graph-assisted extractors that couldn't resolve a symbol name) + * + * This is a pure function — no I/O, no DB — so it's trivial to unit-test + * in isolation (which was the reviewer's main clean-code concern on the + * original 35-line inner closure in `writeBridge`). + */ +export function findContractNode( + index: ContractLookupIndex, + repo: string, + role: 'consumer' | 'provider', + symbolUid: string, + filePath: string, + symbolName: string, +): string | null { + if (symbolUid) { + const uidHit = index.byUid.get(uidKey(repo, role, symbolUid)); + if (uidHit !== undefined) return uidHit; + } + + const refHit = index.byRef.get(refKey(repo, role, filePath, symbolName)); + if (refHit !== undefined) return refHit; + + const fileCandidates = index.byFile.get(fileKey(repo, role, filePath)); + if (fileCandidates && fileCandidates.length === 1) return fileCandidates[0]; + + return null; +} + +export async function openBridgeDb(dbPath: string): Promise { + const parentDir = path.dirname(dbPath); + await fsp.mkdir(parentDir, { recursive: true }); + const db = new lbug.Database(dbPath, 0, false, false); // writable + const conn = new lbug.Connection(db); + return { _db: db, _conn: conn, groupDir: parentDir } as BridgeHandle; +} + +/** + * LadybugDB returns an error whose message contains this substring when a + * CREATE NODE TABLE or CREATE REL TABLE statement hits an already-existing + * table. LadybugDB DDL doesn't support IF NOT EXISTS, and its JS driver + * doesn't expose typed error codes, so we match on the message substring — + * the same pattern used by `core/lbug/lbug-adapter.ts`. If a future + * LadybugDB release changes the wording, update this constant. + */ +const LBUG_ALREADY_EXISTS_MSG = 'already exists'; + +export async function ensureBridgeSchema(handle: BridgeHandle): Promise { + const conn = handle._conn as lbug.Connection; + for (const q of BRIDGE_SCHEMA_QUERIES) { + try { + await conn.query(q); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes(LBUG_ALREADY_EXISTS_MSG)) throw err; + } + } +} + +export async function queryBridge( + handle: BridgeHandle, + cypher: string, + params?: Record, +): Promise { + const conn = handle._conn as lbug.Connection; + if (params && Object.keys(params).length > 0) { + const stmt = await conn.prepare(cypher); + if (!stmt.isSuccess()) { + const errMsg = await stmt.getErrorMessage(); + throw new Error(`Bridge query prepare failed: ${errMsg}`); + } + const queryResult = await conn.execute(stmt, params); + const result = unwrapQueryResult(queryResult); + return (await result.getAll()) as T[]; + } + const queryResult = await conn.query(cypher); + const result = unwrapQueryResult(queryResult); + return (await result.getAll()) as T[]; +} + +/** + * LadybugDB's `conn.query` / `conn.execute` can return either a single + * `QueryResult` (for a single statement) or an array of them (when a + * multi-statement script is dispatched). We always pass a single statement, + * so the array form is a wrapper we unwrap here — but an empty top-level + * array would cause `.getAll()` on `undefined` and crash with a confusing + * stack. Throwing an explicit error makes a driver-contract regression + * visible immediately instead of masking it. + */ +function unwrapQueryResult(queryResult: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult { + if (Array.isArray(queryResult)) { + if (queryResult.length === 0) { + throw new Error('Bridge query returned an empty QueryResult array'); + } + return queryResult[0]; + } + return queryResult; +} + +export async function closeBridgeDb(handle: BridgeHandle): Promise { + try { + await (handle._conn as lbug.Connection).close(); + } catch { + /* ignore */ + } + try { + await (handle._db as lbug.Database).close(); + } catch { + /* ignore */ + } +} + +/* ------------------------------------------------------------------ */ +/* retryRename — handles transient EBUSY/EPERM/EACCES on Windows */ +/* ------------------------------------------------------------------ */ + +const RETRY_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']); + +export async function retryRename(src: string, dst: string, attempts = 3): Promise { + for (let i = 1; i <= attempts; i++) { + try { + await fsp.rename(src, dst); + return; + } catch (err: unknown) { + const code = (err as NodeJS.ErrnoException).code; + if (!code || !RETRY_CODES.has(code) || i === attempts) throw err; + await new Promise((r) => setTimeout(r, 100 * Math.pow(2, i - 1))); + } + } +} + +/* ------------------------------------------------------------------ */ +/* writeBridgeMeta / readBridgeMeta */ +/* ------------------------------------------------------------------ */ + +export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise { + const target = path.join(groupDir, 'meta.json'); + const tmp = `${target}.tmp.${Date.now()}`; + await fsp.writeFile(tmp, JSON.stringify(meta, null, 2), 'utf-8'); + // Use retryRename for consistency with writeBridge's atomic swap — on + // Windows a concurrent reader can cause EBUSY/EPERM even on a tiny + // meta.json, and we don't want meta write to be less robust than the + // bridge.lbug swap it accompanies. + await retryRename(tmp, target); +} + +export async function readBridgeMeta(groupDir: string): Promise { + try { + const content = await fsp.readFile(path.join(groupDir, 'meta.json'), 'utf-8'); + return JSON.parse(content) as BridgeMeta; + } catch { + return { version: 0, generatedAt: '', missingRepos: [] }; + } +} + +/* ------------------------------------------------------------------ */ +/* writeBridge — atomic write-to-temp-then-rename */ +/* ------------------------------------------------------------------ */ + +export interface WriteBridgeInput { + contracts: StoredContract[]; + crossLinks: CrossLink[]; + repoSnapshots: Record; + missingRepos: string[]; +} + +/** + * Non-fatal issues encountered during writeBridge. Callers can log these to + * surface partial-success state without aborting the whole sync. + * `sampleErrors` is capped at MAX_SAMPLE_ERRORS per category to bound memory. + */ +export interface WriteBridgeReport { + contractsInserted: number; + contractsFailed: number; + snapshotsInserted: number; + snapshotsFailed: number; + linksInserted: number; + linksFailed: number; + /** Cross-links skipped because their from/to contract nodes weren't found. */ + linksDroppedMissingNode: number; + sampleErrors: Array<{ + kind: 'contract' | 'snapshot' | 'link'; + id: string; + message: string; + }>; +} + +const MAX_SAMPLE_ERRORS = 10; + +function errMessage(err: unknown): string { + if (err instanceof Error) return err.message; + try { + return String(err); + } catch { + return 'unknown error'; + } +} + +export async function writeBridge( + groupDir: string, + input: WriteBridgeInput, +): Promise { + await fsp.mkdir(groupDir, { recursive: true }); + const contracts = dedupeContracts(input.contracts); + const crossLinks = dedupeCrossLinks(input.crossLinks); + + const finalPath = path.join(groupDir, 'bridge.lbug'); + const tmpPath = path.join(groupDir, 'bridge.lbug.tmp'); + const bakPath = path.join(groupDir, 'bridge.lbug.bak'); + + const report: WriteBridgeReport = { + contractsInserted: 0, + contractsFailed: 0, + snapshotsInserted: 0, + snapshotsFailed: 0, + linksInserted: 0, + linksFailed: 0, + linksDroppedMissingNode: 0, + sampleErrors: [], + }; + + const recordError = (kind: 'contract' | 'snapshot' | 'link', id: string, err: unknown) => { + if (report.sampleErrors.length < MAX_SAMPLE_ERRORS) { + report.sampleErrors.push({ kind, id, message: errMessage(err) }); + } + }; + + // Clean up any leftover tmp + try { + await fsp.rm(tmpPath, { recursive: true, force: true }); + } catch { + /* ignore */ + } + + // 1. Create temp DB, insert all data. + // + // Everything after `openBridgeDb` must run inside a try/finally so that + // if ANY step before the explicit `closeBridgeDb` throws — schema + // creation, a contract insert loop that rethrows, a snapshot write, the + // cross-link loop, or anything else — the handle is still released. A + // leaked handle holds the native LadybugDB file lock on tmpPath, which + // (a) leaks a FD and (b) prevents the next writeBridge call from + // reusing the same tmp slot. + const handle = await openBridgeDb(tmpPath); + let handleClosed = false; + try { + await ensureBridgeSchema(handle); + + // Build the lookup index incrementally as contracts are inserted, so + // failed inserts are never in the index (and therefore never resolved + // by the cross-link loop below). This replaces a previous N+1 query + // pattern where each link made up to 6 DB round-trips to find its + // endpoints — see ContractLookupIndex. + const lookupIndex = createContractLookupIndex(); + + // Insert contracts — tolerate individual failures (e.g., a corrupt meta + // that can't be serialized). The whole sync must not fail because one + // contract is broken. + for (const c of contracts) { + const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath); + try { + await queryBridge( + handle, + `CREATE (n:Contract { + id: $id, + contractId: $contractId, + type: $type, + role: $role, + repo: $repo, + service: $service, + symbolUid: $symbolUid, + filePath: $filePath, + symbolName: $symbolName, + confidence: $confidence, + meta: $meta + })`, + { + id, + contractId: c.contractId, + type: c.type, + role: c.role, + repo: c.repo, + service: c.service ?? '', + symbolUid: c.symbolUid, + filePath: c.symbolRef.filePath, + symbolName: c.symbolName, + confidence: c.confidence, + meta: JSON.stringify(c.meta), + }, + ); + report.contractsInserted++; + // Only index on successful insert — the cross-link loop must never + // resolve to a row that isn't actually in the DB. + indexContract(lookupIndex, c, id); + } catch (err) { + report.contractsFailed++; + recordError('contract', id, err); + } + } + + // Insert repo snapshots + for (const [repoId, snap] of Object.entries(input.repoSnapshots)) { + try { + await queryBridge( + handle, + `CREATE (s:RepoSnapshot { + id: $id, + indexedAt: $indexedAt, + lastCommit: $lastCommit + })`, + { + id: repoId, + indexedAt: snap.indexedAt, + lastCommit: snap.lastCommit, + }, + ); + report.snapshotsInserted++; + } catch (err) { + report.snapshotsFailed++; + recordError('snapshot', repoId, err); + } + } + + // Insert cross-links (tolerating missing nodes). + // + // `findContractNode` consults the in-memory lookup index built above, + // not the DB — that's an O(1) pure-function lookup per endpoint instead + // of the previous 2-3 DB queries. For M cross-links, the previous code + // issued up to 6M round-trips; this version issues zero. + // + // `link.contractId` may differ between the consumer and provider sides + // (e.g. wildcard consumer `grpc::Service/*` → method-level provider + // `grpc::Service/Method`) — that's why we resolve each endpoint + // independently via its own `(repo, role, symbolUid, filePath, symbolName)` + // tuple rather than matching on contractId. + for (const link of crossLinks) { + const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`; + try { + const fromId = findContractNode( + lookupIndex, + link.from.repo, + 'consumer', + link.from.symbolUid, + link.from.symbolRef.filePath, + link.from.symbolRef.name, + ); + const toId = findContractNode( + lookupIndex, + link.to.repo, + 'provider', + link.to.symbolUid, + link.to.symbolRef.filePath, + link.to.symbolRef.name, + ); + if (!fromId || !toId) { + report.linksDroppedMissingNode++; + continue; + } + await queryBridge( + handle, + ` + MATCH (a:Contract), (b:Contract) + WHERE a.id = $fromId AND b.id = $toId + CREATE (a)-[:ContractLink { + matchType: $matchType, + confidence: $confidence, + contractId: $contractId, + fromRepo: $fromRepo, + toRepo: $toRepo + }]->(b) + `, + { + fromId, + toId, + matchType: link.matchType, + confidence: link.confidence, + contractId: link.contractId, + fromRepo: link.from.repo, + toRepo: link.to.repo, + }, + ); + report.linksInserted++; + } catch (err) { + report.linksFailed++; + recordError('link', linkId, err); + } + } + + // 2. Close temp DB (happy path). The finally block also calls + // closeBridgeDb if we threw above; `handleClosed` prevents a + // double-close on the native handle. + await closeBridgeDb(handle); + handleClosed = true; + } finally { + if (!handleClosed) { + await closeBridgeDb(handle).catch(() => { + /* ignore: cleanup path, best effort */ + }); + } + } + + // 3. Atomic swap: old→.bak, tmp→final, rm .bak + try { + await fsp.access(finalPath); + await retryRename(finalPath, bakPath); + } catch { + /* no existing db */ + } + await retryRename(tmpPath, finalPath); + try { + await fsp.rm(bakPath, { recursive: true, force: true }); + } catch { + /* ignore */ + } + + // 4. Write meta.json + await writeBridgeMeta(groupDir, { + version: BRIDGE_SCHEMA_VERSION, + generatedAt: new Date().toISOString(), + missingRepos: input.missingRepos, + }); + + return report; +} + +/* ------------------------------------------------------------------ */ +/* openBridgeDbReadOnly */ +/* ------------------------------------------------------------------ */ + +export async function openBridgeDbReadOnly(groupDir: string): Promise { + const dbPath = path.join(groupDir, 'bridge.lbug'); + try { + await fsp.access(dbPath); + } catch { + // Check for .bak recovery. Use `retryRename` (not `fsp.rename`) for the + // exact same reason the rest of this file does: the scenario that + // triggers bak recovery is an interrupted writer, which on Windows may + // still be holding an open handle on `.bak` for a few milliseconds when + // a reader races in. EBUSY/EPERM retries recover that case silently. + const bakPath = path.join(groupDir, 'bridge.lbug.bak'); + try { + await fsp.access(bakPath); + await retryRename(bakPath, dbPath); + } catch { + return null; + } + } + // Version gate: check meta.json version compatibility + const meta = await readBridgeMeta(groupDir); + if (meta.version > 0 && meta.version !== BRIDGE_SCHEMA_VERSION) { + return null; // incompatible schema version — fallback to JSON or re-sync + } + + // Open the native handle. If Connection construction throws AFTER + // Database was successfully allocated, we'd leak the native Database + // object. Wrap each step separately and tear down the partial handle. + let db: lbug.Database | undefined; + let conn: lbug.Connection | undefined; + try { + db = new lbug.Database(dbPath, 0, false, true); // readOnly + conn = new lbug.Connection(db); + return { _db: db, _conn: conn, groupDir } as BridgeHandle; + } catch { + if (conn) { + try { + await conn.close(); + } catch { + /* ignore */ + } + } + if (db) { + try { + await db.close(); + } catch { + /* ignore */ + } + } + return null; + } +} + +/* ------------------------------------------------------------------ */ +/* bridgeExists */ +/* ------------------------------------------------------------------ */ + +export async function bridgeExists(groupDir: string): Promise { + const handle = await openBridgeDbReadOnly(groupDir); + if (!handle) return false; + await closeBridgeDb(handle); + return true; +} diff --git a/gitnexus/src/core/group/bridge-schema.ts b/gitnexus/src/core/group/bridge-schema.ts new file mode 100644 index 000000000..d61680390 --- /dev/null +++ b/gitnexus/src/core/group/bridge-schema.ts @@ -0,0 +1,60 @@ +/** + * Bridge LadybugDB schema for cross-repo Contract Registry. + * Separate from per-repo schema in lbug/schema.ts. + */ + +/** + * Version of the bridge.lbug schema below. `openBridgeDbReadOnly` compares + * this against `meta.json`'s version field and returns `null` on mismatch, + * which trips the caller into either the JSON fallback path or a fresh + * `group sync` that rebuilds `bridge.lbug` from scratch. + * + * Migration contract for contributors bumping this constant: + * 1. Bump the number (e.g. `1` → `2`). + * 2. Update the DDL below to match the new schema. + * 3. DO NOT attempt an online migration in this file — the version gate + * is intentionally a "discard and re-sync" strategy for V1. An old + * bridge.lbug whose version doesn't match is treated as opaque and + * rebuilt by the next `group sync`. + * 4. If online migration becomes necessary (e.g. when groups accumulate + * large amounts of embedding data), add a migration path as a + * separate `bridge-migrations.ts` module rather than bloating this + * file — keep schema and migration concerns separate. + */ +export const BRIDGE_SCHEMA_VERSION = 1; + +export const CONTRACT_SCHEMA = ` +CREATE NODE TABLE Contract ( + id STRING, + contractId STRING, + type STRING, + role STRING, + repo STRING, + service STRING DEFAULT '', + symbolUid STRING DEFAULT '', + filePath STRING DEFAULT '', + symbolName STRING DEFAULT '', + confidence DOUBLE DEFAULT 0.0, + meta STRING DEFAULT '{}', + PRIMARY KEY (id) +)`; + +export const REPO_SNAPSHOT_SCHEMA = ` +CREATE NODE TABLE RepoSnapshot ( + id STRING, + indexedAt STRING DEFAULT '', + lastCommit STRING DEFAULT '', + PRIMARY KEY (id) +)`; + +export const CONTRACT_LINK_SCHEMA = ` +CREATE REL TABLE ContractLink ( + FROM Contract TO Contract, + matchType STRING, + confidence DOUBLE, + contractId STRING, + fromRepo STRING, + toRepo STRING +)`; + +export const BRIDGE_SCHEMA_QUERIES = [CONTRACT_SCHEMA, REPO_SNAPSHOT_SCHEMA, CONTRACT_LINK_SCHEMA]; diff --git a/gitnexus/src/core/group/extractors/fs-utils.ts b/gitnexus/src/core/group/extractors/fs-utils.ts new file mode 100644 index 000000000..384f63203 --- /dev/null +++ b/gitnexus/src/core/group/extractors/fs-utils.ts @@ -0,0 +1,23 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +/** + * Safely read a file inside a repo, rejecting any path that escapes + * `repoPath` via `..` traversal or absolute segments. Returns `null` if + * the path is outside the repo or the file can't be read. + * + * Used by every source-scan extractor under this directory. Kept as a + * single shared implementation so the path-traversal guard (security- + * sensitive) lives in exactly one place. + */ +export function readSafe(repoPath: string, rel: string): string | null { + const abs = path.resolve(repoPath, rel); + const base = path.resolve(repoPath); + const relToBase = path.relative(base, abs); + if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; + try { + return fs.readFileSync(abs, 'utf-8'); + } catch { + return null; + } +} diff --git a/gitnexus/src/core/group/extractors/grpc-extractor.ts b/gitnexus/src/core/group/extractors/grpc-extractor.ts index b4cefadc5..b379a4dbd 100644 --- a/gitnexus/src/core/group/extractors/grpc-extractor.ts +++ b/gitnexus/src/core/group/extractors/grpc-extractor.ts @@ -1,20 +1,38 @@ -import * as fs from 'node:fs'; import * as path from 'node:path'; import { glob } from 'glob'; +import Parser from 'tree-sitter'; import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafe } from './fs-utils.js'; +import { + GRPC_SCAN_GLOB, + getPluginForFile, + hasProtoPlugin, + type GrpcDetection, +} from './grpc-patterns/index.js'; -function readSafe(repoPath: string, rel: string): string | null { - const abs = path.resolve(repoPath, rel); - const base = path.resolve(repoPath); - const relToBase = path.relative(base, abs); - if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; - try { - return fs.readFileSync(abs, 'utf-8'); - } catch { - return null; - } -} +/** + * Language-agnostic orchestrator for gRPC (provider + consumer) contract + * extraction. + * + * Two parts: + * + * 1. **`.proto` parsing** — tree-sitter when `tree-sitter-proto` is + * installed (optionalDependency vendored in `vendor/tree-sitter-proto/`), + * via the `.proto` entry in `grpc-patterns/` and `hasProtoPlugin`. + * When the grammar isn't available (platform incompatibility, native + * build failure) the orchestrator falls back to the in-process + * string-sanitizing parser defined below (`stripProtoCommentsAndStrings` + * + `extractServiceBlocks`). The fallback preserves offsets so any + * downstream regex scans run against a sanitized copy without + * affecting line numbers of the original. + * + * 2. **Source-scan providers / consumers** — delegated to per-language + * plugins in `./grpc-patterns/`. The orchestrator imports NO + * tree-sitter grammars or query strings — each plugin owns its own. + */ + +// ─── .proto fallback parser (used only when tree-sitter-proto is absent) ─── function contractId(pkg: string, service: string, method: string): string { const prefix = pkg ? `${pkg}.${service}` : service; @@ -25,20 +43,110 @@ function serviceOnlyContractId(serviceName: string): string { return `grpc::${serviceName}/*`; } +/** + * Replace all .proto comments and string literals with spaces, preserving the + * original length and character offsets of the input. This lets downstream + * regex / brace-depth parsers run on a "sanitized" copy without having to + * understand proto syntax, while any RegExp.exec/index-based lookups that + * were already positional against `content` continue to work against the + * original string. + * + * Supported comment forms: `// line comment`, `/* block comment * /`. + * Supported strings: double-quoted ("…") and single-quoted ('…') with `\` + * escape handling. Raw/unterminated strings are not supported — we stop + * on a line break for line-style comments and on EOF for unterminated + * strings/blocks, which matches how most real proto files parse. + */ +function stripProtoCommentsAndStrings(content: string): string { + const out = new Array(content.length); + let i = 0; + while (i < content.length) { + const ch = content[i]; + const next = content[i + 1]; + + // Line comment: // ... \n + if (ch === '/' && next === '/') { + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + while (i < content.length && content[i] !== '\n') { + out[i] = content[i] === '\r' ? '\r' : ' '; + i++; + } + continue; + } + + // Block comment: /* ... */ + if (ch === '/' && next === '*') { + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + while (i < content.length) { + if (content[i] === '*' && content[i + 1] === '/') { + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + break; + } + // Preserve newlines so line numbers stay stable for downstream code. + out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' '; + i++; + } + continue; + } + + // String literal: "..." or '...' + if (ch === '"' || ch === "'") { + const quote = ch; + out[i] = ' '; // replace opening quote + i++; + while (i < content.length) { + const c = content[i]; + if (c === '\\' && i + 1 < content.length) { + // Skip escaped pair (e.g. \" \n \\) + out[i] = ' '; + out[i + 1] = ' '; + i += 2; + continue; + } + if (c === quote) { + out[i] = ' '; + i++; + break; + } + // Preserve newlines; proto technically disallows unescaped newlines + // inside strings, but real files occasionally have them. + out[i] = c === '\n' || c === '\r' ? c : ' '; + i++; + } + continue; + } + + out[i] = ch; + i++; + } + return out.join(''); +} + function extractServiceBlocks(content: string): Array<{ name: string; body: string }> { const results: Array<{ name: string; body: string }> = []; - // v1: brace-depth only — braces inside comments or string literals are not filtered (see spec Fix 2) + // Sanitize comments and string literals so braces inside them don't + // throw off the depth counter. The sanitized copy has the same length + // and offsets as the original, so we use it ONLY to scan for service + // headers and braces; the service body we return is sliced from the + // ORIGINAL content to preserve exact source text for downstream use. + const sanitized = stripProtoCommentsAndStrings(content); const headerRe = /service\s+(\w+)\s*\{/g; let headerMatch: RegExpExecArray | null; - while ((headerMatch = headerRe.exec(content)) !== null) { + while ((headerMatch = headerRe.exec(sanitized)) !== null) { const serviceName = headerMatch[1]; const bodyStart = headerMatch.index + headerMatch[0].length; let depth = 1; let pos = bodyStart; - while (pos < content.length && depth > 0) { - const ch = content[pos]; + while (pos < sanitized.length && depth > 0) { + const ch = sanitized[pos]; if (ch === '{') depth++; else if (ch === '}') depth--; pos++; @@ -75,6 +183,177 @@ function makeContract( }; } +export interface ProtoServiceInfo { + package: string; + serviceName: string; + methods: string[]; + protoPath: string; +} + +function normalizeProtoPath(rel: string): string { + return rel.replace(/\\/g, '/'); +} + +function extractProtoImports(content: string): string[] { + const imports: string[] = []; + const re = /^\s*import\s+"([^"]+)"\s*;/gm; + let match: RegExpExecArray | null; + while ((match = re.exec(content)) !== null) { + imports.push(match[1]); + } + return imports; +} + +function longestSharedSegmentRun(aPath: string, bPath: string): number { + const a = aPath.split('/').filter(Boolean); + const b = bPath.split('/').filter(Boolean); + let best = 0; + + for (let i = 0; i < a.length; i++) { + for (let j = 0; j < b.length; j++) { + let run = 0; + while (a[i + run] && b[j + run] && a[i + run] === b[j + run]) { + run++; + } + if (run > best) best = run; + } + } + + return best; +} + +async function buildProtoContext(repoPath: string): Promise<{ + packagesByProto: Map; + servicesByName: Map; +}> { + const servicesByName = new Map(); + const protoFiles = await glob('**/*.proto', { + cwd: repoPath, + absolute: false, + nodir: true, + ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'], + }); + const contents = new Map(); + + for (const rel of protoFiles) { + const content = readSafe(repoPath, rel); + if (!content) continue; + contents.set(normalizeProtoPath(rel), content); + } + + const packagesByProto = new Map(); + + const resolvePackage = (protoPath: string, seen = new Set()): string => { + if (packagesByProto.has(protoPath)) return packagesByProto.get(protoPath) ?? ''; + if (seen.has(protoPath)) return ''; + + const content = contents.get(protoPath); + if (!content) return ''; + + seen.add(protoPath); + const pkgMatch = content.match(/^\s*package\s+([\w.]+)\s*;/m); + if (pkgMatch?.[1]) { + packagesByProto.set(protoPath, pkgMatch[1]); + return pkgMatch[1]; + } + + for (const importPath of extractProtoImports(content)) { + const normalizedImport = normalizeProtoPath(importPath); + const candidates = [ + normalizeProtoPath( + path.posix.normalize(path.posix.join(path.posix.dirname(protoPath), normalizedImport)), + ), + normalizedImport, + ]; + for (const candidate of candidates) { + if (!contents.has(candidate)) continue; + const inheritedPackage = resolvePackage(candidate, seen); + if (inheritedPackage) { + packagesByProto.set(protoPath, inheritedPackage); + return inheritedPackage; + } + } + } + + packagesByProto.set(protoPath, ''); + return ''; + }; + + for (const rel of protoFiles) { + const normalizedRel = normalizeProtoPath(rel); + const content = contents.get(normalizedRel); + if (!content) continue; + const pkg = resolvePackage(normalizedRel); + + const serviceBlocks = extractServiceBlocks(content); + for (const block of serviceBlocks) { + const rpcRe = /rpc\s+(\w+)\s*\(/g; + const methods: string[] = []; + let m: RegExpExecArray | null; + while ((m = rpcRe.exec(block.body)) !== null) { + methods.push(m[1]); + } + const info: ProtoServiceInfo = { + package: pkg, + serviceName: block.name, + methods, + protoPath: normalizedRel, + }; + const existing = servicesByName.get(block.name) ?? []; + existing.push(info); + servicesByName.set(block.name, existing); + } + } + + return { packagesByProto, servicesByName }; +} + +export async function buildProtoMap(repoPath: string): Promise> { + const { servicesByName } = await buildProtoContext(repoPath); + return servicesByName; +} + +export function resolveProtoConflict( + serviceName: string, + sourceFilePath: string, + candidates: ProtoServiceInfo[], +): ProtoServiceInfo | null { + if (candidates.length === 0) return null; + if (candidates.length === 1) return candidates[0]; + + const sourceDir = normalizeProtoPath(path.dirname(sourceFilePath)); + const scored = candidates.map((c) => { + const protoDir = normalizeProtoPath(path.dirname(c.protoPath)); + return { candidate: c, score: longestSharedSegmentRun(sourceDir, protoDir) }; + }); + + let maxScore = -1; + for (const s of scored) { + if (s.score > maxScore) maxScore = s.score; + } + const winners = scored.filter((s) => s.score === maxScore); + + // Path heuristic cannot uniquely identify a winner — refuse to guess. + // Ties (including all-zero ties) would otherwise silently merge unrelated + // services under a fabricated package-qualified contract id. + if (winners.length !== 1) { + const paths = candidates.map((c) => c.protoPath).join(', '); + console.warn( + `[grpc-extractor] Ambiguous proto resolution for service "${serviceName}" from ${sourceFilePath}: ${winners.length} candidates tied at score ${maxScore} among [${paths}] — skipping canonical contract`, + ); + return null; + } + + return winners[0].candidate; +} + +export function serviceContractId(pkg: string, serviceName: string): string { + const prefix = pkg ? `${pkg}.${serviceName}` : serviceName; + return `grpc::${prefix}/*`; +} + +// ─── Orchestrator ──────────────────────────────────────────────────── + export class GrpcExtractor implements ContractExtractor { type = 'grpc' as const; @@ -88,270 +367,116 @@ export class GrpcExtractor implements ContractExtractor { _repo: RepoHandle, ): Promise { const out: ExtractedContract[] = []; + const protoContext = await buildProtoContext(repoPath); + const protoMap = protoContext.servicesByName; - // Proto files — definitive provider source - const protoFiles = await glob('**/*.proto', { - cwd: repoPath, - ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'], - nodir: true, - }); - for (const rel of protoFiles) { - const content = readSafe(repoPath, rel); - if (content) out.push(...this.parseProtoFile(content, rel)); + // ─── Proto files — definitive provider source ───────────────── + // When tree-sitter-proto is available, .proto files are handled by + // the plugin loop below (they're in GRPC_SCAN_GLOB). Otherwise + // emit provider contracts directly from the proto map that + // `buildProtoContext` already built — no second glob / parse pass. + if (!hasProtoPlugin) { + for (const infos of protoMap.values()) { + for (const info of infos) { + for (const methodName of info.methods) { + const cid = contractId(info.package, info.serviceName, methodName); + out.push( + makeContract( + cid, + 'provider', + info.protoPath, + `${info.serviceName}.${methodName}`, + 0.85, + { + package: info.package, + service: info.serviceName, + method: methodName, + source: 'proto', + }, + ), + ); + } + } + } } - // Source files — server/client detection - const sourceFiles = await glob('**/*.{go,java,py,ts,tsx,js,jsx}', { + // ─── Source files (+ .proto when plugin available) ──────────── + const sourceFiles = await glob(GRPC_SCAN_GLOB, { cwd: repoPath, ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'], nodir: true, }); + + const parser = new Parser(); for (const rel of sourceFiles) { + const plugin = getPluginForFile(rel); + if (!plugin) continue; const content = readSafe(repoPath, rel); if (!content) continue; - const ext = path.extname(rel).toLowerCase(); - - if (ext === '.go') { - out.push(...this.scanGoProviders(content, rel)); - out.push(...this.scanGoConsumers(content, rel)); - } else if (ext === '.java') { - out.push(...this.scanJavaProviders(content, rel)); - out.push(...this.scanJavaConsumers(content, rel)); - } else if (ext === '.py') { - out.push(...this.scanPythonProviders(content, rel)); - out.push(...this.scanPythonConsumers(content, rel)); - } else if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) { - out.push(...this.scanTsProviders(content, rel)); + let detections: GrpcDetection[] = []; + try { + parser.setLanguage(plugin.language); + const tree = parser.parse(content); + detections = plugin.scan(tree); + } catch { + continue; + } + for (const d of detections) { + const contract = this.detectionToContract(d, rel, protoMap); + if (contract) out.push(contract); } } return this.dedupe(out); } - private parseProtoFile(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - - const pkgMatch = content.match(/^package\s+([\w.]+)\s*;/m); - const pkg = pkgMatch ? pkgMatch[1] : ''; - - for (const { name: serviceName, body } of extractServiceBlocks(content)) { - const rpcRe = /rpc\s+(\w+)\s*\(/g; - let rpcMatch: RegExpExecArray | null; - while ((rpcMatch = rpcRe.exec(body)) !== null) { - const methodName = rpcMatch[1]; - const cid = contractId(pkg, serviceName, methodName); - out.push( - makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.85, { - package: pkg, - service: serviceName, - method: methodName, - source: 'proto', - }), - ); - } - } - - return out; - } - - private scanGoProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - - // pb.RegisterXxxServer( - const registerRe = /\w+\.Register(\w+)Server\s*\(/g; - let m: RegExpExecArray | null; - while ((m = registerRe.exec(content)) !== null) { - const serviceName = m[1]; - out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'provider', - filePath, - `Register${serviceName}Server`, - 0.8, - { service: serviceName, source: 'go_register' }, - ), - ); - } - - // pb.UnimplementedXxxServer - const unimplRe = /\w+\.Unimplemented(\w+)Server\b/g; - while ((m = unimplRe.exec(content)) !== null) { - const serviceName = m[1]; - out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'provider', - filePath, - `Unimplemented${serviceName}Server`, - 0.8, - { service: serviceName, source: 'go_unimplemented' }, - ), - ); - } - - return out; - } - - private scanGoConsumers(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - const re = /\w+\.New(\w+)Client\s*\(/g; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const serviceName = m[1]; - out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'consumer', - filePath, - `New${serviceName}Client`, - 0.7, - { service: serviceName, source: 'go_client' }, - ), - ); - } - return out; - } - - private scanJavaProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - - // @GrpcService - if (content.includes('@GrpcService')) { - const implBaseRe = /extends\s+(\w+)Grpc\.(\w+)ImplBase/; - const m = content.match(implBaseRe); - if (m) { - out.push( - makeContract(serviceOnlyContractId(m[1]), 'provider', filePath, m[2], 0.8, { - service: m[1], - source: 'java_grpc_service', - }), - ); - } else { - // Try extracting service name from class name - const classRe = - /class\s+(\w*?)(?:Grpc)?(?:Service)?\s+extends\s+(\w+)(?:Grpc\.(\w+))?ImplBase/; - const cm = content.match(classRe); - if (cm) { - const svcName = cm[2].replace(/Grpc$/, ''); - out.push( - makeContract(serviceOnlyContractId(svcName), 'provider', filePath, cm[1], 0.8, { - service: svcName, - source: 'java_grpc_service', - }), - ); - } - } - } - - // extends XxxImplBase (without @GrpcService) - if (!content.includes('@GrpcService')) { - const implRe = /extends\s+(\w+?)(?:Grpc\.(\w+))?ImplBase/; - const m = content.match(implRe); - if (m) { - const svcName = m[2] || m[1].replace(/Grpc$/, ''); - out.push( - makeContract(serviceOnlyContractId(svcName), 'provider', filePath, svcName, 0.8, { - service: svcName, - source: 'java_impl_base', - }), - ); - } - } - - return out; - } - - private scanJavaConsumers(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - // XxxGrpc.newBlockingStub( or XxxGrpc.newStub( - const re = /(\w+)Grpc\.new(?:Blocking)?Stub\s*\(/g; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const serviceName = m[1]; - out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'consumer', - filePath, - `${serviceName}Stub`, - 0.7, - { service: serviceName, source: 'java_stub' }, - ), - ); - } - return out; - } - - private scanPythonProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - // add_XxxServicer_to_server( - const re = /add_(\w+?)Servicer_to_server\s*\(/g; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const serviceName = m[1]; - out.push( - makeContract( - serviceOnlyContractId(serviceName), - 'provider', - filePath, - `add_${serviceName}Servicer_to_server`, - 0.8, - { service: serviceName, source: 'python_servicer' }, - ), - ); - } - return out; - } - - private scanPythonConsumers(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - // XxxStub( - const re = /(\w+)Stub\s*\(/g; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const name = m[1]; - // Filter out common false positives - if (['Mock', 'Test', 'Fake', 'Stub'].includes(name)) continue; - out.push( - makeContract(serviceOnlyContractId(name), 'consumer', filePath, `${name}Stub`, 0.7, { - service: name, - source: 'python_stub', - }), - ); - } - return out; - } - - private scanTsProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - // @GrpcMethod('ServiceName', 'MethodName') - const re = /@GrpcMethod\s*\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)/g; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const serviceName = m[1]; - const methodName = m[2]; - const cid = contractId('', serviceName, methodName); - out.push( - makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.8, { - service: serviceName, - method: methodName, - source: 'ts_grpc_method', - }), - ); - } - return out; + /** + * Convert a plugin `GrpcDetection` into a concrete `ExtractedContract` + * by resolving the short service name against the proto map, building + * either a service-level (`grpc::pkg.Svc/*`) or method-level + * (`grpc::pkg.Svc/Method`) contract id, and selecting confidence + * based on whether the proto map had an entry. + */ + private detectionToContract( + d: GrpcDetection, + filePath: string, + protoMap: Map, + ): ExtractedContract | null { + const candidates = protoMap.get(d.serviceName) ?? []; + const proto = resolveProtoConflict(d.serviceName, filePath, candidates); + // If there were proto candidates but resolution was ambiguous, skip + // contract emission rather than fabricating a package-qualified id from + // an arbitrary candidate. resolveProtoConflict already warned. + if (candidates.length > 0 && proto === null) return null; + const pkg = proto?.package ?? ''; + const cid = d.methodName + ? contractId(pkg, d.serviceName, d.methodName) + : proto + ? serviceContractId(pkg, d.serviceName) + : serviceOnlyContractId(d.serviceName); + const confidence = proto ? d.confidenceWithProto : d.confidenceWithoutProto; + const meta: Record = { + service: d.serviceName, + source: d.source, + }; + if (d.methodName) meta.method = d.methodName; + return makeContract(cid, d.role, filePath, d.symbolName, confidence, meta); } private dedupe(items: ExtractedContract[]): ExtractedContract[] { - const seen = new Set(); - const out: ExtractedContract[] = []; + const byKey = new Map(); for (const c of items) { const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; - if (seen.has(k)) continue; - seen.add(k); - out.push(c); + const existing = byKey.get(k); + if ( + !existing || + c.confidence > existing.confidence || + (c.confidence === existing.confidence && + String(c.meta.source) < String(existing.meta.source)) + ) { + byKey.set(k, c); + } } - return out; + return Array.from(byKey.values()); } } diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/go.ts b/gitnexus/src/core/group/extractors/grpc-patterns/go.ts new file mode 100644 index 000000000..b1abbaeb7 --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/go.ts @@ -0,0 +1,109 @@ +import Go from 'tree-sitter-go'; +import { + compilePatterns, + runCompiledPatterns, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; + +/** + * Go gRPC plugin. Detects: + * - Provider: `pb.RegisterXxxServer(...)` calls + * - Provider: `pb.UnimplementedXxxServer` embedded in a struct + * - Consumer: `pb.NewXxxClient(conn)` calls + */ + +const REGISTER_RE = /^Register(\w+)Server$/; +const UNIMPLEMENTED_RE = /^Unimplemented(\w+)Server$/; +const NEW_CLIENT_RE = /^New(\w+)Client$/; + +// Any `xxx.(...)` call — plugin filters the field identifier text. +const SELECTOR_CALL_PATTERNS = compilePatterns({ + name: 'go-grpc-selector-call', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (call_expression + function: (selector_expression + field: (field_identifier) @fn)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// Any `qualified_type` used as a struct field — for `pb.UnimplementedXxxServer`. +const STRUCT_EMBEDDING_PATTERNS = compilePatterns({ + name: 'go-grpc-struct-embedding', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (struct_type + (field_declaration_list + (field_declaration + type: (qualified_type + name: (type_identifier) @field_type)))) + `, + }, + ], +} satisfies LanguagePatterns>); + +export const GO_GRPC_PLUGIN: GrpcLanguagePlugin = { + name: 'go-grpc', + language: Go, + scan(tree) { + const out: GrpcDetection[] = []; + + for (const match of runCompiledPatterns(SELECTOR_CALL_PATTERNS, tree)) { + const fnNode = match.captures.fn; + if (!fnNode) continue; + const fnText = fnNode.text; + + const registerMatch = REGISTER_RE.exec(fnText); + if (registerMatch) { + out.push({ + role: 'provider', + serviceName: registerMatch[1], + symbolName: fnText, + source: 'go_register', + confidenceWithProto: 0.8, + confidenceWithoutProto: 0.65, + }); + continue; + } + + const newClientMatch = NEW_CLIENT_RE.exec(fnText); + if (newClientMatch) { + out.push({ + role: 'consumer', + serviceName: newClientMatch[1], + symbolName: fnText, + source: 'go_client', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + continue; + } + } + + for (const match of runCompiledPatterns(STRUCT_EMBEDDING_PATTERNS, tree)) { + const fieldNode = match.captures.field_type; + if (!fieldNode) continue; + const unimpl = UNIMPLEMENTED_RE.exec(fieldNode.text); + if (!unimpl) continue; + out.push({ + role: 'provider', + serviceName: unimpl[1], + symbolName: fieldNode.text, + source: 'go_unimplemented', + confidenceWithProto: 0.8, + confidenceWithoutProto: 0.65, + }); + } + + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/index.ts b/gitnexus/src/core/group/extractors/grpc-patterns/index.ts new file mode 100644 index 000000000..617c14beb --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/index.ts @@ -0,0 +1,53 @@ +import * as path from 'node:path'; +import type { GrpcLanguagePlugin } from './types.js'; +import { GO_GRPC_PLUGIN } from './go.js'; +import { JAVA_GRPC_PLUGIN } from './java.js'; +import { PYTHON_GRPC_PLUGIN } from './python.js'; +import { JAVASCRIPT_GRPC_PLUGIN, TYPESCRIPT_GRPC_PLUGIN, TSX_GRPC_PLUGIN } from './node.js'; +import { PROTO_GRPC_PLUGIN } from './proto.js'; + +export type { GrpcDetection, GrpcLanguagePlugin, GrpcRole } from './types.js'; +export { PROTO_GRPC_PLUGIN, extractPackageFromTree } from './proto.js'; + +/** + * File-extension → gRPC language plugin registry. Mirrors the shape + * of `http-patterns/index.ts` and `topic-patterns/index.ts`. + * + * `.proto` files are registered only when `tree-sitter-proto` is + * available (it's an optionalDependency). When absent, the orchestrator + * falls back to the built-in manual proto parser. + */ +const REGISTRY: Record = { + '.go': GO_GRPC_PLUGIN, + '.java': JAVA_GRPC_PLUGIN, + '.py': PYTHON_GRPC_PLUGIN, + '.js': JAVASCRIPT_GRPC_PLUGIN, + '.jsx': JAVASCRIPT_GRPC_PLUGIN, + '.ts': TYPESCRIPT_GRPC_PLUGIN, + '.tsx': TSX_GRPC_PLUGIN, + ...(PROTO_GRPC_PLUGIN ? { '.proto': PROTO_GRPC_PLUGIN } : {}), +}; + +/** + * Glob for source files worth scanning for gRPC server/client patterns. + * Includes `.proto` when the grammar is available. + */ +export const GRPC_SCAN_GLOB = PROTO_GRPC_PLUGIN + ? '**/*.{go,java,py,ts,tsx,js,jsx,proto}' + : '**/*.{go,java,py,ts,tsx,js,jsx}'; + +/** + * Whether the tree-sitter proto plugin is available. The orchestrator + * uses this to decide between the tree-sitter path and the fallback + * manual parser for `.proto` files. + */ +export const hasProtoPlugin = PROTO_GRPC_PLUGIN !== null; + +/** + * Return the gRPC plugin registered for the given file's extension, + * or `undefined` if the extension is not registered. + */ +export function getPluginForFile(rel: string): GrpcLanguagePlugin | undefined { + const ext = path.extname(rel).toLowerCase(); + return REGISTRY[ext]; +} diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/java.ts b/gitnexus/src/core/group/extractors/grpc-patterns/java.ts new file mode 100644 index 000000000..bf1cf4816 --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/java.ts @@ -0,0 +1,179 @@ +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { + compilePatterns, + runCompiledPatterns, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; + +/** + * Java gRPC plugin. Detects: + * - Provider: classes extending `XxxServiceGrpc.XxxServiceImplBase` + * (with or without a `@GrpcService` annotation; the annotation + * only affects confidence labelling in the original regex version + * — here we emit a single detection per class and pick the source + * label based on whether the annotation is present). + * - Consumer: `XxxServiceGrpc.newBlockingStub(ch)` / + * `XxxServiceGrpc.newStub(ch)` calls. + */ + +const IMPL_BASE_RE = /^(\w+)ImplBase$/; +const GRPC_SUFFIX_RE = /^(\w+)Grpc$/; + +// Classes extending `ScopedType.ScopedType` where the inner name ends +// in ImplBase. Covers `XxxServiceGrpc.XxxServiceImplBase`. +// Note: tree-sitter-java's `scoped_type_identifier` exposes its two +// segments as positional `type_identifier` children, NOT as named +// `scope:`/`name:` fields. We match positionally here and rely on the +// grammar's left-to-right ordering: first child = outer, second = inner. +const SCOPED_IMPL_BASE_PATTERNS = compilePatterns({ + name: 'java-grpc-scoped-impl-base', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + name: (identifier) @class_name + superclass: (superclass + (scoped_type_identifier + (type_identifier) @outer + (type_identifier) @inner (#match? @inner "ImplBase$")))) @class + `, + }, + ], +} satisfies LanguagePatterns>); + +// Classes extending a simple `XxxImplBase` identifier (no scope). +const PLAIN_IMPL_BASE_PATTERNS = compilePatterns({ + name: 'java-grpc-plain-impl-base', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + name: (identifier) @class_name + superclass: (superclass + (type_identifier) @plain_type (#match? @plain_type "ImplBase$"))) @class + `, + }, + ], +} satisfies LanguagePatterns>); + +// gRPC stub factories: `XxxGrpc.newStub(ch)` / `XxxGrpc.newBlockingStub(ch)`. +const STUB_PATTERNS = compilePatterns({ + name: 'java-grpc-stub', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (method_invocation + object: (identifier) @grpc_cls + name: (identifier) @method (#match? @method "^new(Blocking)?Stub$")) + `, + }, + ], +} satisfies LanguagePatterns>); + +/** + * Check whether a `class_declaration` node has a `@GrpcService` + * annotation in its modifiers list. In tree-sitter-java, class-level + * annotations live under `(class_declaration (modifiers (marker_annotation|annotation)))`. + */ +function hasGrpcServiceAnnotation(classNode: Parser.SyntaxNode): boolean { + for (let i = 0; i < classNode.namedChildCount; i++) { + const child = classNode.namedChild(i); + if (!child || child.type !== 'modifiers') continue; + for (let j = 0; j < child.namedChildCount; j++) { + const mod = child.namedChild(j); + if (!mod) continue; + if (mod.type !== 'marker_annotation' && mod.type !== 'annotation') continue; + const nameNode = mod.childForFieldName('name'); + if (nameNode?.text === 'GrpcService') return true; + } + } + return false; +} + +/** + * Given the inner type_identifier text like `AuthServiceImplBase`, + * return the service name (`AuthService`), or null if the text + * doesn't end in `ImplBase`. + */ +function extractServiceFromImplBase(text: string): string | null { + const m = IMPL_BASE_RE.exec(text); + if (!m) return null; + // Strip a trailing `Grpc` on the service name too — the original + // regex replaces `Grpc$` on the extracted prefix. + return m[1].replace(/Grpc$/, ''); +} + +export const JAVA_GRPC_PLUGIN: GrpcLanguagePlugin = { + name: 'java-grpc', + language: Java, + scan(tree) { + const out: GrpcDetection[] = []; + const emittedClassIds = new Set(); + + // ─── Providers: scoped form (`...Grpc.XxxImplBase`) ───────────── + for (const match of runCompiledPatterns(SCOPED_IMPL_BASE_PATTERNS, tree)) { + const classNode = match.captures.class; + const innerNode = match.captures.inner; + if (!classNode || !innerNode) continue; + const serviceName = extractServiceFromImplBase(innerNode.text); + if (!serviceName) continue; + emittedClassIds.add(classNode.id); + const annotated = hasGrpcServiceAnnotation(classNode); + out.push({ + role: 'provider', + serviceName, + symbolName: serviceName, + source: annotated ? 'java_grpc_service' : 'java_impl_base', + confidenceWithProto: 0.8, + confidenceWithoutProto: 0.65, + }); + } + + // ─── Providers: plain form (`XxxImplBase`) ────────────────────── + for (const match of runCompiledPatterns(PLAIN_IMPL_BASE_PATTERNS, tree)) { + const classNode = match.captures.class; + const plainNode = match.captures.plain_type; + if (!classNode || !plainNode) continue; + if (emittedClassIds.has(classNode.id)) continue; + const serviceName = extractServiceFromImplBase(plainNode.text); + if (!serviceName) continue; + emittedClassIds.add(classNode.id); + const annotated = hasGrpcServiceAnnotation(classNode); + out.push({ + role: 'provider', + serviceName, + symbolName: serviceName, + source: annotated ? 'java_grpc_service' : 'java_impl_base', + confidenceWithProto: 0.8, + confidenceWithoutProto: 0.65, + }); + } + + // ─── Consumers: `XxxGrpc.newBlockingStub(...)` / `newStub(...)` ─ + for (const match of runCompiledPatterns(STUB_PATTERNS, tree)) { + const grpcClsNode = match.captures.grpc_cls; + if (!grpcClsNode) continue; + const grpcMatch = GRPC_SUFFIX_RE.exec(grpcClsNode.text); + if (!grpcMatch) continue; + const serviceName = grpcMatch[1]; + out.push({ + role: 'consumer', + serviceName, + symbolName: `${serviceName}Stub`, + source: 'java_stub', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + } + + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/node.ts b/gitnexus/src/core/group/extractors/grpc-patterns/node.ts new file mode 100644 index 000000000..033962206 --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/node.ts @@ -0,0 +1,314 @@ +import Parser from 'tree-sitter'; +import JavaScript from 'tree-sitter-javascript'; +import TypeScript from 'tree-sitter-typescript'; +import { + compilePatterns, + runCompiledPatterns, + unquoteLiteral, + type CompiledPatterns, + type LanguagePatterns, + type PatternSpec, +} from '../tree-sitter-scanner.js'; +import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; + +/** + * Node.js / TypeScript gRPC plugin family. Detects: + * - Provider: NestJS `@GrpcMethod('Service', 'Method')` decorators + * - Consumer: NestJS `@GrpcClient(...) readonly x!: XxxServiceClient` + * - Consumer: `client.getService('AuthService')` + * - Consumer: `new XxxServiceClient(...)` (generated client constructor) + * - Consumer: `new foo.bar.Xxx(...)` when the file uses + * `loadPackageDefinition` (gRPC dynamic proto loader) + * + * As with the HTTP `node.ts`, pattern sources are defined once and + * compiled against three grammar variants (JS / TS / TSX) because + * `Parser.Query` is not portable across grammar objects. + */ + +const SERVICE_CLIENT_RE = /^(\w+Service)Client$/; +const CAPITALIZED_SERVICE_RE = /^[A-Z]\w+$/; + +// @GrpcMethod('Service', 'Method') +const GRPC_METHOD_SPEC: PatternSpec> = { + meta: {}, + query: ` + (decorator + (call_expression + function: (identifier) @dec (#eq? @dec "GrpcMethod") + arguments: (arguments + . [(string) (template_string)] @service + . [(string) (template_string)] @method))) + `, +}; + +// @GrpcClient(...) standalone decorator — the plugin walks to the next +// sibling (a field definition) to read its type annotation. +const GRPC_CLIENT_SPEC: PatternSpec> = { + meta: {}, + query: ` + (decorator + (call_expression + function: (identifier) @dec (#eq? @dec "GrpcClient"))) @grpc_client_decorator + `, +}; + +// `.getService('AuthService')` / `.getService('AuthService')` +const GET_SERVICE_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + property: (property_identifier) @method (#eq? @method "getService")) + arguments: (arguments . [(string) (template_string)] @service)) + `, +}; + +// `new XxxServiceClient(...)` — bare identifier constructor. +const NEW_SIMPLE_CTOR_SPEC: PatternSpec> = { + meta: {}, + query: ` + (new_expression + constructor: (identifier) @ctor) + `, +}; + +// `new foo.bar.XxxService(...)` — qualified constructor. +const NEW_QUALIFIED_CTOR_SPEC: PatternSpec> = { + meta: {}, + query: ` + (new_expression + constructor: (member_expression + property: (property_identifier) @ctor)) + `, +}; + +// Detect whether the file uses `loadPackageDefinition` (gRPC dynamic +// proto loader). Matches either a bare call or an `obj.loadPackageDefinition(...)` +// call. Plugin gates the qualified-constructor consumer on this — +// structural check avoids materializing `tree.rootNode.text` for every file. +const LOAD_PACKAGE_DEFINITION_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: [ + (identifier) @fn (#eq? @fn "loadPackageDefinition") + (member_expression property: (property_identifier) @fn (#eq? @fn "loadPackageDefinition")) + ]) + `, +}; + +interface NodeGrpcPatternBundle { + grpcMethod: CompiledPatterns>; + grpcClient: CompiledPatterns>; + getService: CompiledPatterns>; + newSimpleCtor: CompiledPatterns>; + newQualifiedCtor: CompiledPatterns>; + loadPackageDefinition: CompiledPatterns>; +} + +function compileBundle(language: unknown, name: string): NodeGrpcPatternBundle { + const mk = (spec: PatternSpec>, suffix: string) => + compilePatterns({ + name: `${name}-${suffix}`, + language, + patterns: [spec], + } satisfies LanguagePatterns>); + return { + grpcMethod: mk(GRPC_METHOD_SPEC, 'grpc-method'), + grpcClient: mk(GRPC_CLIENT_SPEC, 'grpc-client'), + getService: mk(GET_SERVICE_SPEC, 'get-service'), + newSimpleCtor: mk(NEW_SIMPLE_CTOR_SPEC, 'new-simple-ctor'), + newQualifiedCtor: mk(NEW_QUALIFIED_CTOR_SPEC, 'new-qualified-ctor'), + loadPackageDefinition: mk(LOAD_PACKAGE_DEFINITION_SPEC, 'load-package-definition'), + }; +} + +const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-grpc'); +const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-grpc'); +const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-grpc'); + +/** + * Given a `@GrpcClient(...)` decorator node, find the type annotation + * text of the field it decorates (e.g. `AuthServiceClient`). + * + * In tree-sitter-typescript, decorators on class fields can appear in + * two configurations: + * - As a CHILD of `public_field_definition` alongside the field's + * type annotation (the common case for NestJS `@GrpcClient`). + * - As a SIBLING of the field in `class_body` (for method + * decorators, but kept for resilience against grammar variants). + * We walk the parent container and search for a type annotation. + */ +function resolveGrpcClientFieldType(decoratorNode: Parser.SyntaxNode): string | null { + const parent = decoratorNode.parent; + if (!parent) return null; + + // Case 1: decorator is a child of the field definition — search + // the parent itself (which is the field definition) for a + // type_annotation child. + if (parent.type === 'public_field_definition' || parent.type.endsWith('field_definition')) { + return findFirstTypeAnnotationText(parent); + } + + // Case 2: decorator is a sibling of the field in a class_body — walk + // forward through subsequent siblings until we find a node containing + // a type annotation. + for (let i = 0; i < parent.namedChildCount; i++) { + const child = parent.namedChild(i); + if (child && child.id === decoratorNode.id) { + for (let j = i + 1; j < parent.namedChildCount; j++) { + const next = parent.namedChild(j); + if (!next) continue; + if (next.type === 'decorator') continue; + const typeText = findFirstTypeAnnotationText(next); + if (typeText) return typeText; + return null; + } + return null; + } + } + return null; +} + +/** + * Recursively search `node` for the first `type_annotation` child and + * return the text of its inner `type_identifier`, or null. Handles + * both `public_field_definition` and its variants. + */ +function findFirstTypeAnnotationText(node: Parser.SyntaxNode): string | null { + if (node.type === 'type_annotation') { + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child) continue; + if (child.type === 'type_identifier') return child.text; + } + return null; + } + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child) continue; + const found = findFirstTypeAnnotationText(child); + if (found) return found; + } + return null; +} + +function scanBundle(bundle: NodeGrpcPatternBundle, tree: Parser.Tree): GrpcDetection[] { + const out: GrpcDetection[] = []; + + // ─── Provider: @GrpcMethod('Service', 'Method') ────────────────── + for (const match of runCompiledPatterns(bundle.grpcMethod, tree)) { + const svcNode = match.captures.service; + const methodNode = match.captures.method; + if (!svcNode || !methodNode) continue; + const svc = unquoteLiteral(svcNode.text); + const mth = unquoteLiteral(methodNode.text); + if (!svc || !mth) continue; + out.push({ + role: 'provider', + serviceName: svc, + symbolName: `${svc}.${mth}`, + source: 'ts_grpc_method', + methodName: mth, + // @GrpcMethod hard-coded confidence 0.8 in the original code + // regardless of whether the proto map has a match. + confidenceWithProto: 0.8, + confidenceWithoutProto: 0.8, + }); + } + + // ─── Consumer: @GrpcClient() field with XxxServiceClient type ──── + for (const match of runCompiledPatterns(bundle.grpcClient, tree)) { + const decoratorNode = match.captures.grpc_client_decorator; + if (!decoratorNode) continue; + const typeText = resolveGrpcClientFieldType(decoratorNode); + if (!typeText) continue; + const svcMatch = SERVICE_CLIENT_RE.exec(typeText); + if (!svcMatch) continue; + const serviceName = svcMatch[1]; + out.push({ + role: 'consumer', + serviceName, + symbolName: `${serviceName}Client`, + source: 'ts_grpc_client_decorator', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + } + + // ─── Consumer: client.getService('Service') ─────────────────── + for (const match of runCompiledPatterns(bundle.getService, tree)) { + const svcNode = match.captures.service; + if (!svcNode) continue; + const svc = unquoteLiteral(svcNode.text); + if (!svc) continue; + out.push({ + role: 'consumer', + serviceName: svc, + symbolName: `${svc}Client`, + source: 'ts_client_grpc_get_service', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + } + + // ─── Consumer: new XxxServiceClient(...) ───────────────────────── + for (const match of runCompiledPatterns(bundle.newSimpleCtor, tree)) { + const ctorNode = match.captures.ctor; + if (!ctorNode) continue; + const svcMatch = SERVICE_CLIENT_RE.exec(ctorNode.text); + if (!svcMatch) continue; + const serviceName = svcMatch[1]; + out.push({ + role: 'consumer', + serviceName, + symbolName: `${serviceName}Client`, + source: 'ts_generated_client', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + } + + // ─── Consumer: loadPackageDefinition dynamic proto loader ──────── + // Only emit when the file uses loadPackageDefinition, otherwise a + // generic `new foo.bar.Something()` in unrelated code would falsely + // register as a gRPC consumer. Check structurally via a dedicated + // query — avoids materializing `tree.rootNode.text` for the whole + // file (expensive on large files). + const usesLoadPackage = runCompiledPatterns(bundle.loadPackageDefinition, tree).length > 0; + if (usesLoadPackage) { + for (const match of runCompiledPatterns(bundle.newQualifiedCtor, tree)) { + const ctorNode = match.captures.ctor; + if (!ctorNode) continue; + if (!CAPITALIZED_SERVICE_RE.test(ctorNode.text)) continue; + out.push({ + role: 'consumer', + serviceName: ctorNode.text, + symbolName: `${ctorNode.text}Client`, + source: 'ts_load_package_definition', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + } + } + + return out; +} + +export const JAVASCRIPT_GRPC_PLUGIN: GrpcLanguagePlugin = { + name: 'javascript-grpc', + language: JavaScript, + scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree), +}; + +export const TYPESCRIPT_GRPC_PLUGIN: GrpcLanguagePlugin = { + name: 'typescript-grpc', + language: TypeScript.typescript, + scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree), +}; + +export const TSX_GRPC_PLUGIN: GrpcLanguagePlugin = { + name: 'tsx-grpc', + language: TypeScript.tsx, + scan: (tree) => scanBundle(TSX_BUNDLE, tree), +}; diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts b/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts new file mode 100644 index 000000000..69b446e55 --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/proto.ts @@ -0,0 +1,147 @@ +import { createRequire } from 'node:module'; +import { + compilePatterns, + runCompiledPatterns, + type CompiledPatterns, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; + +/** + * Protobuf (.proto) tree-sitter plugin for gRPC contract extraction. + * + * Uses `tree-sitter-proto` (coder3101/tree-sitter-proto) as an + * optionalDependency — if the grammar is not installed (e.g. native + * compilation failed on an unusual platform), the plugin exports + * `null` and the orchestrator falls back to the existing manual + * string-sanitizing parser. + * + * The grammar is vendored in `vendor/tree-sitter-proto/` with + * parser.c regenerated against tree-sitter-cli 0.24 (ABI version 14) + * so it is compatible with the project's tree-sitter 0.25 runtime. + */ + +const _require = createRequire(import.meta.url); +let ProtoGrammar: unknown = null; +try { + ProtoGrammar = _require('tree-sitter-proto'); +} catch { + // Grammar not installed — PROTO_GRPC_PLUGIN will be null. +} + +let PACKAGE_PATTERNS: CompiledPatterns> | null = null; +let SERVICE_PATTERNS: CompiledPatterns> | null = null; + +if (ProtoGrammar) { + try { + // Validate that the grammar actually loads end-to-end: compile queries + // AND parse + walk a trivial proto file. tree-sitter's internal + // `initializeLanguageNodeClasses` can fail with a TDZ error in some + // test runners (vitest forks) when SyntaxNode isn't fully initialized + // yet. Catching that here ensures `PROTO_GRPC_PLUGIN` stays null and + // the orchestrator falls back to the manual parser. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const _Parser = _require('tree-sitter') as any; + // Smoke-test: parse + setLanguage to verify the grammar is + // end-to-end compatible with this tree-sitter runtime. + const _testParser = new _Parser(); + _testParser.setLanguage(ProtoGrammar); + _testParser.parse('service X { rpc Y (R) returns (R); }'); + + PACKAGE_PATTERNS = compilePatterns({ + name: 'proto-package', + language: ProtoGrammar, + patterns: [ + { + meta: {}, + query: `(package (full_ident) @pkg)`, + }, + ], + } satisfies LanguagePatterns>); + + SERVICE_PATTERNS = compilePatterns({ + name: 'proto-service', + language: ProtoGrammar, + patterns: [ + { + meta: {}, + query: ` + (service + (service_name) @service_name + (rpc + (rpc_name) @rpc_name)) + `, + }, + ], + } satisfies LanguagePatterns>); + } catch { + // Compilation failed (grammar ABI mismatch?) — fall back to null. + PACKAGE_PATTERNS = null; + SERVICE_PATTERNS = null; + ProtoGrammar = null; + } +} + +function buildPlugin(): GrpcLanguagePlugin | null { + if (!ProtoGrammar || !PACKAGE_PATTERNS || !SERVICE_PATTERNS) return null; + const pkgPatterns = PACKAGE_PATTERNS; + const svcPatterns = SERVICE_PATTERNS; + + return { + name: 'proto-grpc', + language: ProtoGrammar, + scan(tree) { + const out: GrpcDetection[] = []; + + // Extract `package` declaration (first match wins). + let pkg = ''; + for (const match of runCompiledPatterns(pkgPatterns, tree)) { + const pkgNode = match.captures.pkg; + if (pkgNode) { + pkg = pkgNode.text; + break; + } + } + + // Extract `service → rpc` pairs. The query returns one match per + // (service, rpc) combination thanks to the nested structure. + for (const match of runCompiledPatterns(svcPatterns, tree)) { + const serviceNode = match.captures.service_name; + const rpcNode = match.captures.rpc_name; + if (!serviceNode || !rpcNode) continue; + const serviceName = serviceNode.text; + const methodName = rpcNode.text; + out.push({ + role: 'provider', + serviceName, + symbolName: `${serviceName}.${methodName}`, + source: 'proto', + methodName, + // Proto definitions are the canonical source of truth — always + // high confidence regardless of cross-referencing. + confidenceWithProto: 0.85, + confidenceWithoutProto: 0.85, + }); + } + + return out; + }, + }; +} + +/** + * The proto plugin, or `null` if tree-sitter-proto is not available. + * The orchestrator checks this at import time and decides whether to + * use the tree-sitter path or the fallback manual parser. + */ +export const PROTO_GRPC_PLUGIN: GrpcLanguagePlugin | null = buildPlugin(); + +/** The package declaration text from a proto file's tree. */ +export function extractPackageFromTree(tree: import('tree-sitter').Tree): string { + if (!PACKAGE_PATTERNS) return ''; + for (const match of runCompiledPatterns(PACKAGE_PATTERNS, tree)) { + const pkgNode = match.captures.pkg; + if (pkgNode) return pkgNode.text; + } + return ''; +} diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/python.ts b/gitnexus/src/core/group/extractors/grpc-patterns/python.ts new file mode 100644 index 000000000..a19896c1f --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/python.ts @@ -0,0 +1,77 @@ +import Python from 'tree-sitter-python'; +import { + compilePatterns, + runCompiledPatterns, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { GrpcDetection, GrpcLanguagePlugin } from './types.js'; + +/** + * Python gRPC plugin. Detects: + * - Provider: `add_XxxServicer_to_server(...)` calls (bare identifier + * or qualified attribute form `auth_pb2_grpc.add_XxxServicer_to_server`) + * - Consumer: `XxxStub(channel)` calls (bare or `auth_pb2_grpc.XxxStub`) + */ + +const ADD_SERVICER_RE = /^add_(\w+)Servicer_to_server$/; +const STUB_RE = /^(\w+)Stub$/; +/** Reserved names that would produce garbage service names. */ +const STUB_IGNORE = new Set(['Mock', 'Test', 'Fake', 'Stub']); + +// Any call whose target is either a bare identifier or an attribute +// access (`obj.method`). The plugin filters the function name in JS. +const CALL_PATTERNS = compilePatterns({ + name: 'python-grpc-call', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (call + function: [ + (identifier) @fn + (attribute attribute: (identifier) @fn) + ]) + `, + }, + ], +} satisfies LanguagePatterns>); + +export const PYTHON_GRPC_PLUGIN: GrpcLanguagePlugin = { + name: 'python-grpc', + language: Python, + scan(tree) { + const out: GrpcDetection[] = []; + for (const match of runCompiledPatterns(CALL_PATTERNS, tree)) { + const fnNode = match.captures.fn; + if (!fnNode) continue; + const fnText = fnNode.text; + + const addServicer = ADD_SERVICER_RE.exec(fnText); + if (addServicer) { + out.push({ + role: 'provider', + serviceName: addServicer[1], + symbolName: fnText, + source: 'python_servicer', + confidenceWithProto: 0.8, + confidenceWithoutProto: 0.65, + }); + continue; + } + + const stubMatch = STUB_RE.exec(fnText); + if (stubMatch && !STUB_IGNORE.has(stubMatch[1])) { + out.push({ + role: 'consumer', + serviceName: stubMatch[1], + symbolName: fnText, + source: 'python_stub', + confidenceWithProto: 0.75, + confidenceWithoutProto: 0.55, + }); + } + } + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/grpc-patterns/types.ts b/gitnexus/src/core/group/extractors/grpc-patterns/types.ts new file mode 100644 index 000000000..606d9629b --- /dev/null +++ b/gitnexus/src/core/group/extractors/grpc-patterns/types.ts @@ -0,0 +1,54 @@ +import type Parser from 'tree-sitter'; + +/** + * Shared types for the grpc-extractor language plugins. + * + * Each plugin lives in its own file (java.ts, go.ts, ...) and owns the + * tree-sitter grammar import + query sources. The top-level + * `grpc-extractor.ts` orchestrator only knows about this type module + * and the plugin registry (`./index.ts`). It MUST NOT import any + * grammar or query text directly. + */ + +export type GrpcRole = 'provider' | 'consumer'; + +/** + * One raw gRPC detection produced by a plugin's `scan()` function. The + * orchestrator uses the proto map to resolve the full package-qualified + * contract id and choose a confidence based on whether the proto was + * found. + * + * Most patterns produce service-level detections; `TS @GrpcMethod` is + * the only pattern that captures an explicit `methodName`, producing + * a method-level contract (`grpc::pkg.Service/Method`). + */ +export interface GrpcDetection { + role: GrpcRole; + /** Short service name, e.g. `"AuthService"`. */ + serviceName: string; + /** Symbol name emitted into the contract's symbolRef. */ + symbolName: string; + /** Metadata source label (goes into `meta.source`). */ + source: string; + /** Explicit method name; set only by TS `@GrpcMethod`. */ + methodName?: string; + /** Confidence when the proto map resolves the service. */ + confidenceWithProto: number; + /** Confidence when the proto map has no entry. */ + confidenceWithoutProto: number; +} + +/** + * One language-scoped gRPC plugin. Plugins own the tree-sitter grammar + * and a `scan(tree)` function that returns zero or more + * `GrpcDetection`s. The plugin is free to run multiple compiled query + * bundles and walk the AST to cross-reference captures. + * + * `language` is typed `unknown` for the same reason as in + * `tree-sitter-scanner.ts`. + */ +export interface GrpcLanguagePlugin { + name: string; + language: unknown; + scan(tree: Parser.Tree): GrpcDetection[]; +} diff --git a/gitnexus/src/core/group/extractors/http-patterns/go.ts b/gitnexus/src/core/group/extractors/http-patterns/go.ts new file mode 100644 index 000000000..afbfaad56 --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/go.ts @@ -0,0 +1,224 @@ +import Go from 'tree-sitter-go'; +import { + compilePatterns, + runCompiledPatterns, + unquoteLiteral, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { HttpDetection, HttpLanguagePlugin } from './types.js'; + +/** + * Go HTTP plugin. Handles: + * - gin / echo / chi framework routing — `r.GET("/path", handler)` + * - net/http stdlib — `http.HandleFunc("/path", handler)` + * - net/http consumer — `http.Get(...)`, `http.NewRequest("METHOD", ...)` + * - resty consumer — `client.R().Delete("/path")` + */ + +// ─── Provider: framework routing ────────────────────────────────────── +// Matches `\w+\.GET(...)` etc. (gin, echo, chi all share this shape). +// Captures the HTTP method (field name), path literal, and handler +// identifier passed as the second argument. +const FRAMEWORK_ROUTE_PATTERNS = compilePatterns({ + name: 'go-framework-route', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (call_expression + function: (selector_expression + field: (field_identifier) @http_method (#match? @http_method "^(GET|POST|PUT|DELETE|PATCH)$")) + arguments: (argument_list + (interpreted_string_literal) @path + (identifier) @handler)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Provider: net/http `http.HandleFunc("/p", handler)` ───────────── +const HANDLE_FUNC_PATTERNS = compilePatterns({ + name: 'go-handle-func', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (call_expression + function: (selector_expression + operand: (identifier) @pkg (#eq? @pkg "http") + field: (field_identifier) @fn (#eq? @fn "HandleFunc")) + arguments: (argument_list + (interpreted_string_literal) @path + (identifier) @handler)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: net/http stdlib Get / Post / Head ───────────────────── +const HTTP_CLIENT_METHOD_TO_HTTP: Record = { + Get: 'GET', + Post: 'POST', + Head: 'GET', // HEAD has no body semantics we care about — treat as GET for contract matching +}; + +const HTTP_CLIENT_PATTERNS = compilePatterns({ + name: 'go-http-client', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (call_expression + function: (selector_expression + operand: (identifier) @pkg (#eq? @pkg "http") + field: (field_identifier) @fn (#match? @fn "^(Get|Post|Head)$")) + arguments: (argument_list . (interpreted_string_literal) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: net/http `http.NewRequest("METHOD", "/path", ...)` ──── +const NEW_REQUEST_PATTERNS = compilePatterns({ + name: 'go-new-request', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (call_expression + function: (selector_expression + operand: (identifier) @pkg (#eq? @pkg "http") + field: (field_identifier) @fn (#eq? @fn "NewRequest")) + arguments: (argument_list + . + (interpreted_string_literal) @http_method + (interpreted_string_literal) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: resty `client.R().Delete("/path")` ───────────────────── +// Matches any chained call whose receiver is `something.R()` and whose +// method name is an HTTP verb. This is how go-resty's fluent API looks. +const RESTY_PATTERNS = compilePatterns({ + name: 'go-resty', + language: Go, + patterns: [ + { + meta: {}, + query: ` + (call_expression + function: (selector_expression + operand: (call_expression + function: (selector_expression + field: (field_identifier) @r (#eq? @r "R"))) + field: (field_identifier) @http_method (#match? @http_method "^(Get|Post|Put|Delete|Patch)$")) + arguments: (argument_list . (interpreted_string_literal) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +export const GO_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'go-http', + language: Go, + scan(tree) { + const out: HttpDetection[] = []; + + // Framework providers: r.GET/POST/... with handler identifier + for (const match of runCompiledPatterns(FRAMEWORK_ROUTE_PATTERNS, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + const handlerNode = match.captures.handler; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'provider', + framework: 'go-framework', + method: methodNode.text.toUpperCase(), + path, + name: handlerNode?.text ?? null, + confidence: 0.8, + }); + } + + // net/http HandleFunc: default method GET + for (const match of runCompiledPatterns(HANDLE_FUNC_PATTERNS, tree)) { + const pathNode = match.captures.path; + const handlerNode = match.captures.handler; + if (!pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'provider', + framework: 'go-stdlib', + method: 'GET', + path, + name: handlerNode?.text ?? null, + confidence: 0.8, + }); + } + + // net/http client: http.Get/Post/Head + for (const match of runCompiledPatterns(HTTP_CLIENT_PATTERNS, tree)) { + const fnNode = match.captures.fn; + const pathNode = match.captures.path; + if (!fnNode || !pathNode) continue; + const httpMethod = HTTP_CLIENT_METHOD_TO_HTTP[fnNode.text]; + if (!httpMethod) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'go-stdlib', + method: httpMethod, + path, + name: null, + confidence: 0.7, + }); + } + + // net/http NewRequest + for (const match of runCompiledPatterns(NEW_REQUEST_PATTERNS, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const method = unquoteLiteral(methodNode.text); + const path = unquoteLiteral(pathNode.text); + if (method === null || path === null) continue; + out.push({ + role: 'consumer', + framework: 'go-stdlib', + method: method.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // resty + for (const match of runCompiledPatterns(RESTY_PATTERNS, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'go-resty', + method: methodNode.text.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/http-patterns/index.ts b/gitnexus/src/core/group/extractors/http-patterns/index.ts new file mode 100644 index 000000000..e33d32a79 --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/index.ts @@ -0,0 +1,50 @@ +import * as path from 'node:path'; +import type { HttpLanguagePlugin } from './types.js'; +import { JAVA_HTTP_PLUGIN } from './java.js'; +import { GO_HTTP_PLUGIN } from './go.js'; +import { PYTHON_HTTP_PLUGIN } from './python.js'; +import { PHP_HTTP_PLUGIN } from './php.js'; +import { JAVASCRIPT_HTTP_PLUGIN, TYPESCRIPT_HTTP_PLUGIN, TSX_HTTP_PLUGIN } from './node.js'; + +export type { HttpDetection, HttpLanguagePlugin, HttpRole } from './types.js'; + +/** + * File-extension → HTTP language plugin registry. The top-level + * orchestrator (`http-route-extractor.ts`) looks up the plugin for each + * file it visits and delegates the tree-sitter scanning to the plugin. + * + * Keys are lowercase extensions including the leading dot. To add a + * new language, drop a `http-patterns/.ts` that exports a + * `HttpLanguagePlugin`, import it here and register the extension(s). + * No edits to `http-route-extractor.ts` are required. + */ +const REGISTRY: Record = { + '.java': JAVA_HTTP_PLUGIN, + '.go': GO_HTTP_PLUGIN, + '.py': PYTHON_HTTP_PLUGIN, + '.php': PHP_HTTP_PLUGIN, + '.js': JAVASCRIPT_HTTP_PLUGIN, + '.jsx': JAVASCRIPT_HTTP_PLUGIN, + '.ts': TYPESCRIPT_HTTP_PLUGIN, + '.tsx': TSX_HTTP_PLUGIN, +}; + +/** + * Glob for files worth scanning for HTTP routes. Kept alongside the + * registry so adding a new language widens the glob in one edit. + * + * `.vue` / `.svelte` files are intentionally omitted for the source-scan + * path — they need their own grammar-aware extraction and the existing + * regex fallback for them was never very accurate. The graph-assisted + * Strategy A still handles them via the ingestion pipeline. + */ +export const HTTP_SCAN_GLOB = '**/*.{ts,tsx,js,jsx,java,go,py,php}'; + +/** + * Return the HTTP plugin registered for the given file's extension, + * or `undefined` if the extension is not registered. + */ +export function getPluginForFile(rel: string): HttpLanguagePlugin | undefined { + const ext = path.extname(rel).toLowerCase(); + return REGISTRY[ext]; +} diff --git a/gitnexus/src/core/group/extractors/http-patterns/java.ts b/gitnexus/src/core/group/extractors/http-patterns/java.ts new file mode 100644 index 000000000..484f74fb2 --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/java.ts @@ -0,0 +1,267 @@ +import Parser from 'tree-sitter'; +import Java from 'tree-sitter-java'; +import { + compilePatterns, + runCompiledPatterns, + unquoteLiteral, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { HttpDetection, HttpLanguagePlugin } from './types.js'; + +/** + * Java HTTP plugin. Handles: + * - Spring `@RequestMapping` class prefixes + `@(Get|Post|...)Mapping` method annotations + * - Spring `RestTemplate.getForObject/...`, `WebClient.method(HttpMethod.X, ...)` + * - OkHttp `new Request.Builder().url("...")` + * + * The plugin runs two pattern bundles: one to collect class-level + * `@RequestMapping` prefixes keyed by the enclosing class node, and a + * second to match method-level annotations. The `scan` function walks + * up from each matched annotation to find its enclosing class and + * combines the prefix with the method path. + */ + +const METHOD_ANNOTATION_TO_HTTP: Record = { + GetMapping: 'GET', + PostMapping: 'POST', + PutMapping: 'PUT', + DeleteMapping: 'DELETE', + PatchMapping: 'PATCH', +}; + +// ─── Provider: Spring class-level @RequestMapping prefix ────────────── +const SPRING_CLASS_PREFIX_PATTERNS = compilePatterns({ + name: 'java-spring-class-prefix', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (class_declaration + (modifiers + (annotation + name: (identifier) @ann (#eq? @ann "RequestMapping") + arguments: (annotation_argument_list (string_literal) @prefix)))) @class + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Provider: Spring @(Get|Post|...)Mapping method annotations ─────── +const SPRING_METHOD_ROUTE_PATTERNS = compilePatterns({ + name: 'java-spring-method-route', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (method_declaration + (modifiers + (annotation + name: (identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$") + arguments: (annotation_argument_list (string_literal) @path))) + name: (identifier) @method_name) @method + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: Spring RestTemplate (object-named + method-named) ────── +// RestTemplate.getForObject / getForEntity → GET +// RestTemplate.postForObject / postForEntity → POST +// RestTemplate.put → PUT +// RestTemplate.delete → DELETE +// RestTemplate.patchForObject → PATCH +const REST_TEMPLATE_TO_HTTP: Record = { + getForObject: 'GET', + getForEntity: 'GET', + postForObject: 'POST', + postForEntity: 'POST', + put: 'PUT', + delete: 'DELETE', + patchForObject: 'PATCH', +}; + +interface RestTemplateMeta { + framework: 'spring-rest-template'; +} + +const REST_TEMPLATE_PATTERNS = compilePatterns({ + name: 'java-rest-template', + language: Java, + patterns: [ + { + meta: { framework: 'spring-rest-template' }, + query: ` + (method_invocation + object: (identifier) @obj (#eq? @obj "restTemplate") + name: (identifier) @method + arguments: (argument_list . (string_literal) @path)) + `, + }, + ], +} satisfies LanguagePatterns); + +// ─── Consumer: Spring WebClient — webClient.method(HttpMethod.X, "path") ─ +const WEB_CLIENT_PATTERNS = compilePatterns({ + name: 'java-web-client', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (method_invocation + object: (identifier) @obj (#eq? @obj "webClient") + name: (identifier) @method (#eq? @method "method") + arguments: (argument_list + (field_access + object: (identifier) @httpMethodCls (#eq? @httpMethodCls "HttpMethod") + field: (identifier) @http_method) + (string_literal) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: OkHttp `new Request.Builder().url("path")` ───────────── +// Note: `Request.Builder` is a `scoped_type_identifier` whose text includes +// the dot, so `#eq?` against the literal string matches cleanly (no need +// to escape a regex dot). +const OK_HTTP_PATTERNS = compilePatterns({ + name: 'java-okhttp', + language: Java, + patterns: [ + { + meta: {}, + query: ` + (method_invocation + object: (object_creation_expression + type: (scoped_type_identifier) @type (#eq? @type "Request.Builder")) + name: (identifier) @method (#eq? @method "url") + arguments: (argument_list . (string_literal) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +/** + * Find the nearest enclosing class_declaration ancestor for a node, or + * null if the node is top-level. Tree-sitter's SyntaxNode.parent walks + * one level at a time. + */ +function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { + let cur: Parser.SyntaxNode | null = node.parent; + while (cur) { + if (cur.type === 'class_declaration') return cur; + cur = cur.parent; + } + return null; +} + +/** + * Join a class-level prefix and a method-level path into a single URL + * path. Mirrors the semantics of the original regex implementation: + * strip trailing slashes on the prefix, then ensure a single slash + * between prefix and method path. + */ +function joinPath(prefix: string, methodPath: string): string { + const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, ''); + const cleanSub = methodPath.replace(/^\/+/, ''); + if (!cleanPrefix) return `/${cleanSub}`; + return `/${cleanPrefix}/${cleanSub}`; +} + +export const JAVA_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'java-http', + language: Java, + scan(tree) { + const out: HttpDetection[] = []; + + // ─── Providers: Spring class prefix + method annotations ──────── + const prefixByClassId = new Map(); + for (const match of runCompiledPatterns(SPRING_CLASS_PREFIX_PATTERNS, tree)) { + const prefixNode = match.captures.prefix; + const classNode = match.captures.class; + if (!prefixNode || !classNode) continue; + const prefix = unquoteLiteral(prefixNode.text); + if (prefix !== null) prefixByClassId.set(classNode.id, prefix); + } + + for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) { + const annNode = match.captures.ann; + const pathNode = match.captures.path; + const nameNode = match.captures.method_name; + const methodNode = match.captures.method; + if (!annNode || !pathNode || !methodNode) continue; + const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text]; + if (!httpMethod) continue; + const rawPath = unquoteLiteral(pathNode.text); + if (rawPath === null) continue; + const enclosingClass = findEnclosingClass(methodNode); + const prefix = enclosingClass ? (prefixByClassId.get(enclosingClass.id) ?? '') : ''; + const fullPath = joinPath(prefix, rawPath); + out.push({ + role: 'provider', + framework: 'spring', + method: httpMethod, + path: fullPath, + name: nameNode?.text ?? null, + confidence: 0.8, + }); + } + + // ─── Consumers: RestTemplate ──────────────────────────────────── + for (const match of runCompiledPatterns(REST_TEMPLATE_PATTERNS, tree)) { + const methodNode = match.captures.method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const httpMethod = REST_TEMPLATE_TO_HTTP[methodNode.text]; + if (!httpMethod) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'spring-rest-template', + method: httpMethod, + path, + name: null, + confidence: 0.7, + }); + } + + // ─── Consumers: WebClient.method(HttpMethod.X, "path") ────────── + for (const match of runCompiledPatterns(WEB_CLIENT_PATTERNS, tree)) { + const httpMethodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!httpMethodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'spring-web-client', + method: httpMethodNode.text.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // ─── Consumers: OkHttp Request.Builder().url("path") ──────────── + for (const match of runCompiledPatterns(OK_HTTP_PATTERNS, tree)) { + const pathNode = match.captures.path; + if (!pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'okhttp', + method: 'GET', + path, + name: null, + confidence: 0.7, + }); + } + + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/http-patterns/node.ts b/gitnexus/src/core/group/extractors/http-patterns/node.ts new file mode 100644 index 000000000..fbf988665 --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/node.ts @@ -0,0 +1,502 @@ +import Parser from 'tree-sitter'; +import JavaScript from 'tree-sitter-javascript'; +import TypeScript from 'tree-sitter-typescript'; +import { + compilePatterns, + runCompiledPatterns, + unquoteLiteral, + type CompiledPatterns, + type LanguagePatterns, + type PatternSpec, +} from '../tree-sitter-scanner.js'; +import type { HttpDetection, HttpLanguagePlugin } from './types.js'; + +/** + * Node.js / TypeScript HTTP plugin family. Handles: + * - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods + * - Express `router.get(...)` / `app.post(...)` providers + * - `fetch(url)` / `fetch(url, { method: 'POST' })` consumers + * - `axios.get(url)` / `axios.delete(url)` consumers + * - `axios({ method, url })` object-form consumers + * - jQuery `$.get(url)` / `$.post(url, ...)` shorthand consumers + * - jQuery `$.ajax({ url, method | type })` consumers + * + * Because the JavaScript and TypeScript tree-sitter grammars share + * node type names for every construct we query, pattern sources are + * defined once and compiled against each grammar variant. The plugin + * exports three `HttpLanguagePlugin`s (JS, TS, TSX) that share the + * same `scan` function but bind to different grammars. + */ + +// ─── Provider: NestJS — class-level @Controller('prefix') ──────────── +// In tree-sitter-typescript decorators are NOT children of +// class_declaration / method_definition — they're siblings in the +// surrounding class_body / program node. We therefore match the +// decorator standalone and walk to its related class/method in JS. +const NEST_CONTROLLER_SPEC: PatternSpec> = { + meta: {}, + query: ` + (decorator + (call_expression + function: (identifier) @dec (#eq? @dec "Controller") + arguments: (arguments . [(string) (template_string)] @prefix))) @ctrl_decorator + `, +}; + +// ─── Provider: NestJS — method-level @Get/@Post/... decorators ─────── +// Matches either `@Get('path')` or `@Get()`. The `@path` capture is +// optional — when the first argument isn't a string, the plugin falls +// back to '/' for the method-level path. +const NEST_METHOD_SPEC: PatternSpec> = { + meta: {}, + query: ` + (decorator + (call_expression + function: (identifier) @dec (#match? @dec "^(Get|Post|Put|Delete|Patch)$") + arguments: (arguments) @args)) @method_decorator + `, +}; + +// ─── Provider: Express — router.get/app.post/... ───────────────────── +const EXPRESS_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#match? @obj "^(router|app)$") + property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$")) + arguments: (arguments . [(string) (template_string)] @path)) + `, +}; + +// ─── Consumer: fetch(url) with NO options ───────────────────────────── +const FETCH_NO_OPTIONS_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (identifier) @fn (#eq? @fn "fetch") + arguments: (arguments . [(string) (template_string)] @path .)) + `, +}; + +// ─── Consumer: fetch(url, { method: 'X', ... }) ────────────────────── +const FETCH_WITH_OPTIONS_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (identifier) @fn (#eq? @fn "fetch") + arguments: (arguments + . [(string) (template_string)] @path + (object + (pair + key: (property_identifier) @key (#eq? @key "method") + value: (string) @http_method)))) + `, +}; + +// ─── Consumer: axios.get/post/... ──────────────────────────────────── +const AXIOS_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "axios") + property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$")) + arguments: (arguments . [(string) (template_string)] @path)) + `, +}; + +// ─── Consumer: jQuery shorthand $.get(url) / $.post(url, ...) ──────── +// `$` is a valid JS identifier, so tree-sitter parses `$.get(...)` as a +// call_expression whose function is a member_expression on identifier `$`. +const JQUERY_SHORTHAND_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "$") + property: (property_identifier) @http_method (#match? @http_method "^(get|post)$")) + arguments: (arguments . [(string) (template_string)] @path)) + `, +}; + +// ─── Consumer: jQuery $.ajax({ url, method|type }) ─────────────────── +// The query captures the options object only; key/value pairs are read +// programmatically via `readStringProp` below, which tolerates any key +// order and accepts either `method:` or `type:` (jQuery supports both). +const JQUERY_AJAX_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "$") + property: (property_identifier) @fn (#eq? @fn "ajax")) + arguments: (arguments (object) @options)) + `, +}; + +// ─── Consumer: axios({ method, url }) object form ──────────────────── +// Distinct from AXIOS_SPEC above because the call target is an identifier +// (`axios`) rather than a member expression (`axios.get`). As with the +// jQuery ajax form, option keys are resolved programmatically. +const AXIOS_OBJECT_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (identifier) @fn (#eq? @fn "axios") + arguments: (arguments (object) @options)) + `, +}; + +interface NodePatternBundle { + controller: CompiledPatterns>; + methodDecorator: CompiledPatterns>; + express: CompiledPatterns>; + fetchNoOptions: CompiledPatterns>; + fetchWithOptions: CompiledPatterns>; + axios: CompiledPatterns>; + jqueryShorthand: CompiledPatterns>; + jqueryAjax: CompiledPatterns>; + axiosObject: CompiledPatterns>; +} + +function compileBundle(language: unknown, name: string): NodePatternBundle { + const mk = (spec: PatternSpec>, suffix: string) => + compilePatterns({ + name: `${name}-${suffix}`, + language, + patterns: [spec], + } satisfies LanguagePatterns>); + return { + controller: mk(NEST_CONTROLLER_SPEC, 'nest-controller'), + methodDecorator: mk(NEST_METHOD_SPEC, 'nest-method-decorator'), + express: mk(EXPRESS_SPEC, 'express'), + fetchNoOptions: mk(FETCH_NO_OPTIONS_SPEC, 'fetch-no-options'), + fetchWithOptions: mk(FETCH_WITH_OPTIONS_SPEC, 'fetch-with-options'), + axios: mk(AXIOS_SPEC, 'axios'), + jqueryShorthand: mk(JQUERY_SHORTHAND_SPEC, 'jquery-shorthand'), + jqueryAjax: mk(JQUERY_AJAX_SPEC, 'jquery-ajax'), + axiosObject: mk(AXIOS_OBJECT_SPEC, 'axios-object'), + }; +} + +const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-http'); +const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-http'); +const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-http'); + +const NEST_DECORATOR_TO_HTTP: Record = { + Get: 'GET', + Post: 'POST', + Put: 'PUT', + Delete: 'DELETE', + Patch: 'PATCH', +}; + +/** + * Find the nearest enclosing class_declaration for a node, or null. + */ +function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null { + let cur: Parser.SyntaxNode | null = node.parent; + while (cur) { + if (cur.type === 'class_declaration') return cur; + cur = cur.parent; + } + return null; +} + +function joinPath(prefix: string, sub: string): string { + const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, ''); + const cleanSub = sub.replace(/^\/+/, ''); + if (!cleanPrefix) return `/${cleanSub}`; + return `/${cleanPrefix}/${cleanSub}`; +} + +/** + * Walk `pair` children of an `object` literal and return the unquoted + * string/template_string value for the first pair whose key matches one + * of `keyNames`. Returns null when no matching pair is present or the + * value is not a string literal. Used by the jQuery ajax / axios object + * consumers to resolve `url` / `method` / `type` keys in any order. + */ +function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string[]): string | null { + for (let i = 0; i < objectNode.namedChildCount; i++) { + const pair = objectNode.namedChild(i); + if (!pair || pair.type !== 'pair') continue; + const keyNode = pair.childForFieldName('key'); + const valueNode = pair.childForFieldName('value'); + if (!keyNode || !valueNode) continue; + if (!keyNames.includes(keyNode.text)) continue; + if (valueNode.type !== 'string' && valueNode.type !== 'template_string') continue; + const lit = unquoteLiteral(valueNode.text); + if (lit !== null) return lit; + } + return null; +} + +/** + * For a standalone `decorator` node (child of class_body / program), + * find the related `class_declaration` node that it decorates. In + * tree-sitter-typescript the decorator is placed before the class + * declaration as a sibling (when decorating a class) or inside the + * class_body before a method_definition (when decorating a method); + * we walk the parent chain until we find the enclosing class. + */ +function findDecoratedClass(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null { + const parent = decoratorNode.parent; + if (!parent) return null; + // Case 1: decorator is a sibling of the class_declaration at program / + // export_statement level. Walk forward through siblings until we find + // the class_declaration this decorator belongs to. + for (let i = 0; i < parent.namedChildCount; i++) { + const child = parent.namedChild(i); + if (child && child.id === decoratorNode.id) { + for (let j = i + 1; j < parent.namedChildCount; j++) { + const next = parent.namedChild(j); + if (!next) continue; + if (next.type === 'decorator') continue; // adjacent decorators stack + if (next.type === 'class_declaration') return next; + if (next.type === 'export_statement') { + // `export class Foo { ... }` wraps the declaration. + for (let k = 0; k < next.namedChildCount; k++) { + const inner = next.namedChild(k); + if (inner?.type === 'class_declaration') return inner; + } + } + break; + } + break; + } + } + // Case 2: decorator is inside a class_body (decorating a method) — + // walk up to the enclosing class_declaration. + return findEnclosingClass(decoratorNode); +} + +/** + * For a method-level decorator node (child of class_body before a + * method_definition), find the method_definition it decorates. + */ +function findDecoratedMethod(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null { + const parent = decoratorNode.parent; + if (!parent || parent.type !== 'class_body') return null; + for (let i = 0; i < parent.namedChildCount; i++) { + const child = parent.namedChild(i); + if (child && child.id === decoratorNode.id) { + for (let j = i + 1; j < parent.namedChildCount; j++) { + const next = parent.namedChild(j); + if (!next) continue; + if (next.type === 'decorator') continue; + if (next.type === 'method_definition') return next; + return null; + } + return null; + } + } + return null; +} + +function scanBundle(bundle: NodePatternBundle, tree: Parser.Tree): HttpDetection[] { + const out: HttpDetection[] = []; + + // NestJS: collect `@Controller('prefix')` class decorators, keyed by + // the `class_declaration` they decorate. + const prefixByClassId = new Map(); + for (const match of runCompiledPatterns(bundle.controller, tree)) { + const prefixNode = match.captures.prefix; + const decoratorNode = match.captures.ctrl_decorator; + if (!prefixNode || !decoratorNode) continue; + const prefix = unquoteLiteral(prefixNode.text); + if (prefix === null) continue; + const classNode = findDecoratedClass(decoratorNode); + if (!classNode) continue; + prefixByClassId.set(classNode.id, prefix); + } + + // NestJS: method-level @Get/@Post/... decorators. The decorator's + // arguments list may be empty (`@Get()`), a string (`@Get('path')`), + // or something else (which we skip). + for (const match of runCompiledPatterns(bundle.methodDecorator, tree)) { + const decNode = match.captures.dec; + const argsNode = match.captures.args; + const decoratorNode = match.captures.method_decorator; + if (!decNode || !argsNode || !decoratorNode) continue; + const httpMethod = NEST_DECORATOR_TO_HTTP[decNode.text]; + if (!httpMethod) continue; + const methodNode = findDecoratedMethod(decoratorNode); + if (!methodNode) continue; + const enclosingClass = findEnclosingClass(methodNode); + // Only emit NestJS detections when the class actually has a + // @Controller decorator — without it, the match is almost certainly + // something else (e.g. an unrelated library using similar names). + if (!enclosingClass || !prefixByClassId.has(enclosingClass.id)) continue; + const prefix = prefixByClassId.get(enclosingClass.id) ?? ''; + + let rawPath = '/'; + const firstArg = argsNode.namedChild(0); + if (firstArg && (firstArg.type === 'string' || firstArg.type === 'template_string')) { + const unquoted = unquoteLiteral(firstArg.text); + if (unquoted !== null) rawPath = unquoted; + } + + // Get the method name from the decorated method_definition. + const methodNameNode = methodNode.childForFieldName('name'); + const name = methodNameNode?.text ?? null; + + out.push({ + role: 'provider', + framework: 'nest', + method: httpMethod, + path: joinPath(prefix, rawPath), + name, + confidence: 0.8, + }); + } + + // Express: router/app.(...) + for (const match of runCompiledPatterns(bundle.express, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'provider', + framework: 'express', + method: methodNode.text.toUpperCase(), + path, + name: 'handler', + confidence: 0.8, + }); + } + + // Consumer: fetch with options { method: 'X' } + const fetchSeen = new Set(); + for (const match of runCompiledPatterns(bundle.fetchWithOptions, tree)) { + const pathNode = match.captures.path; + const methodNode = match.captures.http_method; + if (!pathNode || !methodNode) continue; + const path = unquoteLiteral(pathNode.text); + const method = unquoteLiteral(methodNode.text); + if (path === null || method === null) continue; + fetchSeen.add(pathNode.id); + out.push({ + role: 'consumer', + framework: 'fetch', + method: method.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: plain fetch(path) — default GET. Skip path nodes we already + // matched with the options variant so we don't double-emit. + for (const match of runCompiledPatterns(bundle.fetchNoOptions, tree)) { + const pathNode = match.captures.path; + if (!pathNode) continue; + if (fetchSeen.has(pathNode.id)) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'fetch', + method: 'GET', + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: axios.(url) + for (const match of runCompiledPatterns(bundle.axios, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'axios', + method: methodNode.text.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: jQuery shorthand $.get(url) / $.post(url, ...) + for (const match of runCompiledPatterns(bundle.jqueryShorthand, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'jquery', + method: methodNode.text.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: jQuery $.ajax({ url, method|type }). jQuery accepts either + // `method:` or `type:`; both default to GET when absent. + for (const match of runCompiledPatterns(bundle.jqueryAjax, tree)) { + const optionsNode = match.captures.options; + if (!optionsNode) continue; + const path = readStringProp(optionsNode, ['url']); + if (path === null) continue; + const rawMethod = readStringProp(optionsNode, ['method', 'type']); + const method = (rawMethod ?? 'GET').toUpperCase(); + out.push({ + role: 'consumer', + framework: 'jquery', + method, + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: axios({ method, url }) object form. Structurally distinct + // from axios.(url) (identifier vs member_expression call), so no + // dedup against the member-form loop above is required. + for (const match of runCompiledPatterns(bundle.axiosObject, tree)) { + const optionsNode = match.captures.options; + if (!optionsNode) continue; + const path = readStringProp(optionsNode, ['url']); + if (path === null) continue; + const rawMethod = readStringProp(optionsNode, ['method']); + const method = (rawMethod ?? 'GET').toUpperCase(); + out.push({ + role: 'consumer', + framework: 'axios', + method, + path, + name: null, + confidence: 0.7, + }); + } + + return out; +} + +export const JAVASCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'javascript-http', + language: JavaScript, + scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree), +}; + +export const TYPESCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'typescript-http', + language: TypeScript.typescript, + scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree), +}; + +export const TSX_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'tsx-http', + language: TypeScript.tsx, + scan: (tree) => scanBundle(TSX_BUNDLE, tree), +}; diff --git a/gitnexus/src/core/group/extractors/http-patterns/php.ts b/gitnexus/src/core/group/extractors/http-patterns/php.ts new file mode 100644 index 000000000..ae91c141b --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/php.ts @@ -0,0 +1,79 @@ +import PHP from 'tree-sitter-php'; +import { + compilePatterns, + runCompiledPatterns, + unquoteLiteral, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { HttpDetection, HttpLanguagePlugin } from './types.js'; + +/** + * PHP HTTP plugin — Laravel `Route::get/post/...` declarations. + * + * The pipeline already uses `PHP.php_only` for ingesting plain `.php` + * files (see `core/tree-sitter/parser-loader.ts`), and we do the same + * here so Laravel route files are parsed with the right grammar dialect. + */ + +const LARAVEL_PATTERNS = compilePatterns({ + name: 'php-laravel', + language: PHP.php_only, + patterns: [ + { + meta: {}, + query: ` + (scoped_call_expression + scope: (name) @scope (#eq? @scope "Route") + name: (name) @method (#match? @method "^(get|post|put|delete|patch)$") + arguments: (arguments . (argument (string) @path))) + `, + }, + ], +} satisfies LanguagePatterns>); + +/** + * Extract the inner text of a PHP `string` node. The tree-sitter-php + * grammar wraps single / double-quoted literals differently depending + * on content; we try both the raw `text` (with quotes) through + * `unquoteLiteral`, and a fallback via the `string_value` / `string_content` + * child nodes. + */ +function phpStringText(node: import('tree-sitter').SyntaxNode): string | null { + // Most single-quoted strings expose their inner content through the + // full node text (including quotes), which unquoteLiteral strips. + const direct = unquoteLiteral(node.text); + if (direct !== null && direct !== node.text) return direct; + // Fall back to child string_content / string_value node if present. + for (const child of node.children) { + if (child.type === 'string_content' || child.type === 'string_value') { + return child.text; + } + } + return direct; +} + +export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'php-http', + language: PHP.php_only, + scan(tree) { + const out: HttpDetection[] = []; + + for (const match of runCompiledPatterns(LARAVEL_PATTERNS, tree)) { + const methodNode = match.captures.method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = phpStringText(pathNode); + if (path === null) continue; + out.push({ + role: 'provider', + framework: 'laravel', + method: methodNode.text.toUpperCase(), + path, + name: 'route', + confidence: 0.8, + }); + } + + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/http-patterns/python.ts b/gitnexus/src/core/group/extractors/http-patterns/python.ts new file mode 100644 index 000000000..27ddf6633 --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/python.ts @@ -0,0 +1,142 @@ +import Python from 'tree-sitter-python'; +import { + compilePatterns, + runCompiledPatterns, + unquoteLiteral, + type LanguagePatterns, +} from '../tree-sitter-scanner.js'; +import type { HttpDetection, HttpLanguagePlugin } from './types.js'; + +/** + * Python HTTP plugin. Handles: + * - FastAPI `@app.get("/path")` provider decorators + * - `requests.get/post/...("url")` consumer calls + * - Generic `requests.request("METHOD", "url")` consumer calls + */ + +const FASTAPI_VERBS: Record = { + get: 'GET', + post: 'POST', + put: 'PUT', + delete: 'DELETE', + patch: 'PATCH', +}; + +// ─── Provider: FastAPI @app.get/... ────────────────────────────────── +const FASTAPI_PATTERNS = compilePatterns({ + name: 'python-fastapi', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (decorator + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "app") + attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$")) + arguments: (argument_list . (string) @path))) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: requests.get/post/... ────────────────────────────────── +const REQUESTS_VERB_PATTERNS = compilePatterns({ + name: 'python-requests-verb', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "requests") + attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$")) + arguments: (argument_list . (string) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +// ─── Consumer: requests.request("METHOD", "url") ───────────────────── +const REQUESTS_GENERIC_PATTERNS = compilePatterns({ + name: 'python-requests-generic', + language: Python, + patterns: [ + { + meta: {}, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "requests") + attribute: (identifier) @method (#eq? @method "request")) + arguments: (argument_list . (string) @http_method (string) @path)) + `, + }, + ], +} satisfies LanguagePatterns>); + +export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = { + name: 'python-http', + language: Python, + scan(tree) { + const out: HttpDetection[] = []; + + // Providers: FastAPI + for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) { + const methodNode = match.captures.method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const httpMethod = FASTAPI_VERBS[methodNode.text]; + if (!httpMethod) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'provider', + framework: 'fastapi', + method: httpMethod, + path, + name: null, + confidence: 0.8, + }); + } + + // Consumers: requests. + for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) { + const methodNode = match.captures.method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'python-requests', + method: methodNode.text.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // Consumers: requests.request("METHOD", "url") + for (const match of runCompiledPatterns(REQUESTS_GENERIC_PATTERNS, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const methodRaw = unquoteLiteral(methodNode.text); + const path = unquoteLiteral(pathNode.text); + if (methodRaw === null || path === null) continue; + out.push({ + role: 'consumer', + framework: 'python-requests', + method: methodRaw.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + return out; + }, +}; diff --git a/gitnexus/src/core/group/extractors/http-patterns/types.ts b/gitnexus/src/core/group/extractors/http-patterns/types.ts new file mode 100644 index 000000000..6df0ede28 --- /dev/null +++ b/gitnexus/src/core/group/extractors/http-patterns/types.ts @@ -0,0 +1,65 @@ +import type Parser from 'tree-sitter'; + +/** + * Shared types for the http-route-extractor language plugins. + * + * Each plugin lives in its own file (java.ts, node.ts, ...) and owns + * the tree-sitter grammar import + queries. The top-level + * `http-route-extractor.ts` orchestrator only knows about this type + * module and the plugin registry (`./index.ts`). It MUST NOT import + * any grammar or query text directly — language-specific knowledge + * belongs in the plugins. + */ + +export type HttpRole = 'provider' | 'consumer'; + +/** + * One raw HTTP detection produced by a plugin's `scan()` function. The + * orchestrator converts this into a full `ExtractedContract` by running + * path normalization and building the contract id. + * + * `path` is the raw literal string as it appeared in source (with + * `${...}` template placeholders still in place); the orchestrator + * runs the appropriate normalizer for provider vs. consumer paths. + */ +export interface HttpDetection { + role: HttpRole; + /** Short framework label, e.g. `'spring'`, `'nest'`, `'express'`. */ + framework: string; + /** HTTP method in upper case (`'GET'`, `'POST'`, ...). */ + method: string; + /** Raw path literal as seen in source (template placeholders intact). */ + path: string; + /** + * Symbol name of the handler (for providers) or calling function + * (for consumers) when the plugin can determine it structurally. + * Null when no good candidate is available. + */ + name: string | null; + /** Confidence in (0, 1]. Source-scan plugins typically use 0.7–0.8. */ + confidence: number; +} + +/** + * One language-scoped HTTP plugin. The plugin owns the tree-sitter + * grammar and the `scan` function that translates a parsed tree into + * zero or more `HttpDetection`s. Plugins are free to run multiple + * compiled pattern bundles internally (see the shared scanner's + * `runCompiledPatterns` helper). + * + * `language` is typed as `unknown` for the same reason as + * `LanguagePatterns.language` in `tree-sitter-scanner.ts` — the + * grammar modules export different shapes. + */ +export interface HttpLanguagePlugin { + /** Human-readable plugin name for diagnostics. */ + name: string; + /** tree-sitter grammar object (passed to the shared parser). */ + language: unknown; + /** + * Scan a parsed tree and return zero or more HTTP detections. Plugins + * must not throw — they should swallow per-match errors so a single + * malformed construct does not abort the whole file. + */ + scan(tree: Parser.Tree): HttpDetection[]; +} diff --git a/gitnexus/src/core/group/extractors/http-route-extractor.ts b/gitnexus/src/core/group/extractors/http-route-extractor.ts index 8dfb242bf..f2914613d 100644 --- a/gitnexus/src/core/group/extractors/http-route-extractor.ts +++ b/gitnexus/src/core/group/extractors/http-route-extractor.ts @@ -1,8 +1,34 @@ -import * as fs from 'node:fs'; import * as path from 'node:path'; import { glob } from 'glob'; +import Parser from 'tree-sitter'; import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafe } from './fs-utils.js'; +import { getPluginForFile, HTTP_SCAN_GLOB, type HttpDetection } from './http-patterns/index.js'; + +/** + * Language-agnostic orchestrator for HTTP route (provider + consumer) + * contract extraction. Two strategies, in order of preference per role: + * + * 1. **Graph-assisted (Strategy A)** — if a per-repo LadybugDB executor + * is available, read `HANDLES_ROUTE` / `FETCHES` Cypher edges that + * the ingestion pipeline already produced via tree-sitter. This is + * the preferred path because the graph has richer symbol metadata + * (real uids, class/method structure, etc.). + * + * 2. **Source-scan fallback (Strategy B)** — parse files directly with + * the per-language plugin registry in `./http-patterns/`. Used when + * the graph has no routes/fetches for this repo (e.g. a repo that + * hasn't been indexed yet, or whose indexer doesn't know the + * framework). Each plugin owns its tree-sitter grammar and query + * sources — this orchestrator imports NO grammars or query strings. + * + * Adding a new language for Strategy B is a one-file edit in + * `http-patterns/index.ts`: register a new `HttpLanguagePlugin` and + * widen `HTTP_SCAN_GLOB` if needed. + */ + +// ─── Graph-assisted queries ────────────────────────────────────────── const HANDLES_ROUTE_QUERY = ` MATCH (handlerFile:File)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route) @@ -23,14 +49,56 @@ WHERE sym.startLine IS NOT NULL RETURN sym.id AS uid, sym.name AS name, sym.filePath AS filePath, labels(sym) AS labels ORDER BY sym.startLine`; +// ─── Path normalization (shared between provider / consumer paths) ── + +/** + * Canonicalize a provider-side HTTP path for contract-id generation: + * - strip query string + * - lower-case + * - drop trailing slash + * - collapse `:id`, `{id}`, `[id]` path params into a single `{param}` + */ export function normalizeHttpPath(p: string): string { let s = p.trim().split('?')[0].toLowerCase().replace(/\/+$/, ''); s = s.replace(/:\w+/g, '{param}'); s = s.replace(/\{[^}]+\}/g, '{param}'); s = s.replace(/\[[^\]]+\]/g, '{param}'); - return s; + // Preserve root: after stripping trailing slashes, the root "/" + // collapses to "" which would produce malformed contract ids like + // `http::GET::`. Restore a single slash for the root case. + return s === '' ? '/' : s; } +/** + * Consumer-side normalization is more aggressive: + * - template literals (`${x}`) → `{param}` + * - strip protocol + host if the URL is absolute + * - numeric segments → `{param}` (so `/api/orders/42` → `/api/orders/{param}`) + */ +function normalizeConsumerPath(url: string): string { + const templated = url.replace(/\$\{[^}]+\}/g, '{param}').trim(); + let pathOnly = templated; + if (/^https?:\/\//i.test(templated)) { + try { + pathOnly = new URL(templated).pathname; + } catch { + pathOnly = templated.replace(/^https?:\/\/[^/]+/i, ''); + } + } + const normalized = normalizeHttpPath(pathOnly || '/'); + const segments = normalized + .split('/') + .filter(Boolean) + .map((segment) => (/^\d+$/.test(segment) ? '{param}' : segment)); + return `/${segments.join('/')}`.replace(/\/+$/, '') || '/'; +} + +function contractIdFor(method: string, pathNorm: string): string { + return `http::${method.toUpperCase()}::${pathNorm}`; +} + +// ─── Graph row helpers ─────────────────────────────────────────────── + function methodFromRouteReason(reason: string): string | null { const r = reason || ''; if (/GetMapping|decorator-Get/i.test(r)) return 'GET'; @@ -41,50 +109,6 @@ function methodFromRouteReason(reason: string): string | null { return null; } -function contractIdFor(method: string, pathNorm: string): string { - return `http::${method.toUpperCase()}::${pathNorm}`; -} - -function readSafe(repoPath: string, rel: string): string | null { - const abs = path.resolve(repoPath, rel); - const base = path.resolve(repoPath); - const relToBase = path.relative(base, abs); - if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; - try { - return fs.readFileSync(abs, 'utf-8'); - } catch { - return null; - } -} - -function pickJavaHandlerName( - content: string, - routePath: string, - httpMethod: string, -): string | null { - const tail = routePath.split('/').filter(Boolean).pop() || ''; - const mapNames: Record = { - GET: 'GetMapping', - POST: 'PostMapping', - PUT: 'PutMapping', - DELETE: 'DeleteMapping', - PATCH: 'PatchMapping', - }; - const ann = mapNames[httpMethod] || 'GetMapping'; - const lines = content.split(/\r?\n/); - for (let i = 0; i < lines.length; i++) { - const line = lines[i]; - if (!line.includes(`@${ann}`)) continue; - if (!line.includes(`"${tail}"`) && !line.includes(`'${tail}'`) && tail && !line.includes(tail)) - continue; - for (let j = i + 1; j < Math.min(i + 8, lines.length); j++) { - const m = lines[j].match(/(?:public|protected|private)\s+[\w<>,\s\[\]]+\s+(\w+)\s*\(/); - if (m) return m[1]; - } - } - return null; -} - function pickSymbolUid( rows: Record[], preferredName: string | null, @@ -114,6 +138,8 @@ function pickSymbolUid( }; } +// ─── Orchestrator ──────────────────────────────────────────────────── + export class HttpRouteExtractor implements ContractExtractor { type = 'http' as const; @@ -124,20 +150,76 @@ export class HttpRouteExtractor implements ContractExtractor { async extract( dbExecutor: CypherExecutor | null, repoPath: string, - repo: RepoHandle, + _repo: RepoHandle, ): Promise { - const graphP = dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, repoPath) : []; - const providers = graphP.length > 0 ? graphP : await this.extractProvidersSourceScan(repoPath); + // Parse each file at most once and reuse the plugin results across + // both graph-assisted enrichment and source-scan emission. + const parser = new Parser(); + const cachedDetections = new Map(); + const getDetections = (rel: string): HttpDetection[] => { + const cached = cachedDetections.get(rel); + if (cached) return cached; + const plugin = getPluginForFile(rel); + if (!plugin) { + cachedDetections.set(rel, []); + return []; + } + const content = readSafe(repoPath, rel); + if (!content) { + cachedDetections.set(rel, []); + return []; + } + try { + parser.setLanguage(plugin.language); + const tree = parser.parse(content); + const detections = plugin.scan(tree); + cachedDetections.set(rel, detections); + return detections; + } catch { + cachedDetections.set(rel, []); + return []; + } + }; - const graphC = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, repoPath) : []; - const consumers = graphC.length > 0 ? graphC : await this.extractConsumersSourceScan(repoPath); + // Glob the source-scan file list at most once per extract() — + // both provider and consumer fallback paths share the same list. + let scannedFiles: string[] | null = null; + const getScannedFiles = async (): Promise => { + if (scannedFiles) return scannedFiles; + scannedFiles = await this.scanFiles(repoPath); + return scannedFiles; + }; + + const graphProviders = + dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, getDetections) : []; + const providers = + graphProviders.length > 0 + ? graphProviders + : this.extractProvidersSourceScan(await getScannedFiles(), getDetections); + + const graphConsumers = + dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : []; + const consumers = + graphConsumers.length > 0 + ? graphConsumers + : this.extractConsumersSourceScan(await getScannedFiles(), getDetections); return [...providers, ...consumers]; } + private async scanFiles(repoPath: string): Promise { + return glob(HTTP_SCAN_GLOB, { + cwd: repoPath, + ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/vendor/**'], + nodir: true, + }); + } + + // ─── Graph-assisted providers ────────────────────────────────────── + private async extractProvidersGraph( db: CypherExecutor, - repoPath: string, + getDetections: (rel: string) => HttpDetection[], ): Promise { const out: ExtractedContract[] = []; let rows: Record[]; @@ -152,22 +234,55 @@ export class HttpRouteExtractor implements ContractExtractor { const routePath = String(row.routePath ?? ''); const routeSource = String(row.routeSource ?? row.routeReason ?? ''); let method = methodFromRouteReason(routeSource); - const content = readSafe(repoPath, filePath); - if (!method && content) { - method = this.inferMethodFromFileScan(content, routePath, 'provider'); + + // Look up handler name (and backfill method if missing) from the + // plugin's scan of the handler file. This replaces the old + // regex-based `inferMethodFromFileScan` and `pickJavaHandlerName` + // helpers — tree-sitter gives both pieces of information + // structurally. Always run the lookup: even when method is set by + // `methodFromRouteReason`, we still need the handler name. + const detections = filePath ? getDetections(filePath) : []; + const providerDetections = detections.filter((d) => d.role === 'provider'); + let handlerName: string | null = null; + const normalizedRoute = normalizeHttpPath(routePath); + // Candidates share the same normalized path. When multiple + // detections at the same path exist (e.g. GET + POST /api/orders + // in one router), a blind `.find()` silently returned the first + // verb — attaching the wrong handler and, when method was not + // already pinned by the route reason, the wrong method too. + // Disambiguate by method when we know it; refuse to guess when + // we don't. + const candidates = providerDetections.filter( + (d) => normalizeHttpPath(d.path) === normalizedRoute, + ); + let match: (typeof candidates)[number] | undefined; + const ambiguousCandidates = !method && candidates.length > 1; + if (method) { + match = candidates.find((d) => d.method === method); + } else if (candidates.length === 1) { + match = candidates[0]; + } + // else: multiple candidates + unknown method → leave match + // undefined so handlerName stays null and skip symbol + // enrichment below, keeping the file-basename fallback instead + // of letting pickSymbolUid silently pick the first Function / + // Method in the file (which reintroduces the mis-attribution + // we were trying to avoid). Method stays at the conservative + // 'GET' default set below. + if (match) { + if (!method) method = match.method; + handlerName = match.name; } if (!method) method = 'GET'; const pathNorm = normalizeHttpPath(routePath); const cid = contractIdFor(method, pathNorm); - const handlerName = - content && routePath ? pickJavaHandlerName(content, routePath, method) : null; let symbolUid = ''; let symbolName = path.basename(filePath) || 'handler'; let symPath = filePath; const fileId = row.fileId ?? row[0]; - if (fileId) { + if (fileId && !ambiguousCandidates) { try { const syms = await db(CONTAINS_QUERY, { fileId }); if (syms.length > 0) { @@ -201,145 +316,44 @@ export class HttpRouteExtractor implements ContractExtractor { return out; } - private inferMethodFromFileScan( - content: string, - routePath: string, - _role: string, - ): string | null { - const tail = routePath.split('/').filter(Boolean).pop() || ''; - for (const m of ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'] as const) { - const mapNames: Record = { - GET: 'GetMapping', - POST: 'PostMapping', - PUT: 'PutMapping', - DELETE: 'DeleteMapping', - PATCH: 'PatchMapping', - }; - if ( - content.includes(`@${mapNames[m]}`) && - (content.includes(tail) || routePath.includes(tail)) - ) { - return m; - } - } - return null; - } + // ─── Source-scan providers ───────────────────────────────────────── - private async extractProvidersSourceScan(repoPath: string): Promise { - const files = await glob('**/*.{ts,tsx,js,jsx,java,vue,svelte,php,py}', { - cwd: repoPath, - ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'], - nodir: true, - }); + private extractProvidersSourceScan( + files: string[], + getDetections: (rel: string) => HttpDetection[], + ): ExtractedContract[] { const out: ExtractedContract[] = []; for (const rel of files) { - const content = readSafe(repoPath, rel); - if (!content) continue; - out.push(...this.scanSpringProviders(content, rel)); - out.push(...this.scanExpressProviders(content, rel)); - out.push(...this.scanLaravelProviders(content, rel)); - out.push(...this.scanFastApiProviders(content, rel)); + const detections = getDetections(rel); + for (const d of detections) { + if (d.role !== 'provider') continue; + const pathNorm = normalizeHttpPath(d.path); + out.push({ + contractId: contractIdFor(d.method, pathNorm), + type: 'http', + role: 'provider', + symbolUid: '', + symbolRef: { filePath: rel, name: d.name ?? 'handler' }, + symbolName: d.name ?? 'handler', + confidence: d.confidence, + meta: { + method: d.method, + path: pathNorm, + pathSegments: pathNorm.split('/').filter(Boolean), + extractionStrategy: 'source_scan', + framework: d.framework, + }, + }); + } } return this.dedupeContracts(out); } - private dedupeContracts(items: ExtractedContract[]): ExtractedContract[] { - const seen = new Set(); - const out: ExtractedContract[] = []; - for (const c of items) { - const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`; - if (seen.has(k)) continue; - seen.add(k); - out.push(c); - } - return out; - } - - private scanSpringProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - let classPrefix = ''; - const classRm = content.match(/@RequestMapping\s*\(\s*"([^"]+)"/); - if (classRm) classPrefix = classRm[1].replace(/\/+$/, ''); - - const re = /@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*"([^"]+)"/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const method = m[1].toUpperCase(); - let p = m[2]; - if (classPrefix) p = `${classPrefix}/${p.replace(/^\//, '')}`; - const pathNorm = normalizeHttpPath(p); - const sub = content.slice(m.index); - const nameM = sub.match(/(?:public|protected|private)\s+[\w<>,\s\[\]]+\s+(\w+)\s*\(/); - const name = nameM ? nameM[1] : m[0]; - out.push(this.makeProvider(filePath, method, pathNorm, name, 0.8)); - } - return out; - } - - private scanExpressProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - const re = /(?:router|app)\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const method = m[1].toUpperCase(); - const pathNorm = normalizeHttpPath(m[2]); - out.push(this.makeProvider(filePath, method, pathNorm, 'handler', 0.8)); - } - return out; - } - - private scanLaravelProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - const re = /Route::(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const method = m[1].toUpperCase(); - const pathNorm = normalizeHttpPath(m[2]); - out.push(this.makeProvider(filePath, method, pathNorm, 'route', 0.8)); - } - return out; - } - - private scanFastApiProviders(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - const re = /@app\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const method = m[1].toUpperCase(); - const pathNorm = normalizeHttpPath(m[2]); - out.push(this.makeProvider(filePath, method, pathNorm, 'handler', 0.8)); - } - return out; - } - - private makeProvider( - filePath: string, - method: string, - pathNorm: string, - name: string, - confidence: number, - ): ExtractedContract { - const cid = contractIdFor(method, pathNorm); - return { - contractId: cid, - type: 'http', - role: 'provider', - symbolUid: '', - symbolRef: { filePath, name }, - symbolName: name, - confidence, - meta: { - method, - path: pathNorm, - pathSegments: pathNorm.split('/').filter(Boolean), - extractionStrategy: 'source_scan', - }, - }; - } + // ─── Graph-assisted consumers ────────────────────────────────────── private async extractConsumersGraph( db: CypherExecutor, - repoPath: string, + getDetections: (rel: string) => HttpDetection[], ): Promise { const out: ExtractedContract[] = []; let rows: Record[]; @@ -353,11 +367,23 @@ export class HttpRouteExtractor implements ContractExtractor { const routePath = String(row.routePath ?? ''); const pathNorm = normalizeHttpPath(routePath); let method = 'GET'; - const content = readSafe(repoPath, filePath); - if (content) { - const inferred = this.inferFetchMethod(content, pathNorm); - if (inferred) method = inferred; + // Prefer the plugin's detected method if we can find a matching + // fetch/axios call in the same file. + const detections = filePath ? getDetections(filePath) : []; + // Symmetric to the provider path: if multiple consumer calls in + // the same file share the same normalized path (e.g. a GET + // fetch AND a POST fetch to `/api/orders`), `.find()` silently + // picked the first verb and keyed the contract id on the wrong + // method. With no upstream method signal here, refuse to guess + // when candidates are ambiguous — leave `method` at its + // conservative 'GET' default. + const consumerCandidates = detections.filter( + (d) => d.role === 'consumer' && normalizeConsumerPath(d.path) === pathNorm, + ); + if (consumerCandidates.length === 1) { + method = consumerCandidates[0].method; } + const cid = contractIdFor(method, pathNorm); let symbolUid = ''; let symbolName = 'fetch'; @@ -395,81 +421,47 @@ export class HttpRouteExtractor implements ContractExtractor { return out; } - private inferFetchMethod(content: string, pathNorm: string): string | null { - const esc = pathNorm.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); - const fetchRe = new RegExp( - `fetch\\s*\\(\\s*['"\`]([^'"\`]*${esc}[^'"\`]*)['"\`]\\s*,\\s*\\{[^}]*method:\\s*['"](\\w+)['"]`, - 'i', - ); - const m = content.match(fetchRe); - if (m) return m[2].toUpperCase(); - return null; - } + // ─── Source-scan consumers ───────────────────────────────────────── - private async extractConsumersSourceScan(repoPath: string): Promise { - const files = await glob('**/*.{ts,tsx,js,jsx,vue,svelte}', { - cwd: repoPath, - ignore: ['**/node_modules/**', '**/.git/**'], - nodir: true, - }); + private extractConsumersSourceScan( + files: string[], + getDetections: (rel: string) => HttpDetection[], + ): ExtractedContract[] { const out: ExtractedContract[] = []; for (const rel of files) { - const content = readSafe(repoPath, rel); - if (!content) continue; - out.push(...this.scanFetchConsumers(content, rel)); - out.push(...this.scanAxiosConsumers(content, rel)); + const detections = getDetections(rel); + for (const d of detections) { + if (d.role !== 'consumer') continue; + const pathNorm = normalizeConsumerPath(d.path); + out.push({ + contractId: contractIdFor(d.method, pathNorm), + type: 'http', + role: 'consumer', + symbolUid: '', + symbolRef: { filePath: rel, name: 'fetch' }, + symbolName: 'fetch', + confidence: d.confidence, + meta: { + method: d.method, + path: pathNorm, + extractionStrategy: 'source_scan', + framework: d.framework, + }, + }); + } } return this.dedupeContracts(out); } - private scanFetchConsumers(content: string, filePath: string): ExtractedContract[] { + private dedupeContracts(items: ExtractedContract[]): ExtractedContract[] { + const seen = new Set(); const out: ExtractedContract[] = []; - const re = - /fetch\s*\(\s*['"`]([^'"`]+)['"`](?:\s*,\s*\{[^}]*method:\s*['"](\w+)['"][^}]*\})?\s*\)/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const pathNorm = normalizeHttpPath(this.templateToPattern(m[1])); - const method = (m[2] || 'GET').toUpperCase(); - out.push(this.makeConsumer(filePath, method, pathNorm, 0.7)); + for (const c of items) { + const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`; + if (seen.has(k)) continue; + seen.add(k); + out.push(c); } return out; } - - private templateToPattern(url: string): string { - return url.replace(/\$\{[^}]+\}/g, '{param}'); - } - - private scanAxiosConsumers(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - const re = /axios\.(get|post|put|delete|patch)\s*\(\s*[`'"]([^`'"]+)[`'"]/gi; - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const method = m[1].toUpperCase(); - const pathNorm = normalizeHttpPath(this.templateToPattern(m[2])); - out.push(this.makeConsumer(filePath, method, pathNorm, 0.7)); - } - return out; - } - - private makeConsumer( - filePath: string, - method: string, - pathNorm: string, - confidence: number, - ): ExtractedContract { - return { - contractId: contractIdFor(method, pathNorm), - type: 'http', - role: 'consumer', - symbolUid: '', - symbolRef: { filePath, name: 'fetch' }, - symbolName: 'fetch', - confidence, - meta: { - method, - path: pathNorm, - extractionStrategy: 'source_scan', - }, - }; - } } diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts new file mode 100644 index 000000000..83f5cab5e --- /dev/null +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -0,0 +1,344 @@ +import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js'; +import type { CypherExecutor } from '../contract-extractor.js'; + +export interface ManifestExtractResult { + contracts: StoredContract[]; + crossLinks: CrossLink[]; +} + +/** + * Canonicalize an HTTP path for matching against Route.name in the graph. + * Mirrors core/ingestion/pipeline.ts ensureSlash semantics: + * - Ensures a leading slash. + * - Strips trailing slashes (except the root "/"). + * - Normalizes consecutive slashes. + * - Does NOT lowercase (route matching is case-sensitive). + */ +function normalizeRoutePath(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) return '/'; + const withLeading = trimmed.startsWith('/') ? trimmed : `/${trimmed}`; + const collapsed = withLeading.replace(/\/+/g, '/'); + if (collapsed === '/') return '/'; + return collapsed.replace(/\/+$/, ''); +} + +/** + * Split a manifest HTTP contract into its optional `METHOD::` prefix and + * its path portion. + * + * `buildContractId` recommends the explicit-method form `GET::/api/orders` + * in group.yaml; if we hand that raw string to `normalizeRoutePath` we get + * `/GET::/api/orders`, which can never match `Route.name = "/api/orders"` + * in the graph. This helper extracts the path portion so the Cypher + * lookup uses the canonical route name. + * + * The method prefix regex mirrors `buildContractId` (line ~251) for + * symmetry: case-insensitive `[A-Za-z]+` followed by `::`. The captured + * method is upper-cased for downstream use; method-constrained matching + * against `HANDLES_ROUTE` is a future enhancement (not yet wired). + * + * Edge cases: + * - `"::/api/orders"` — empty method portion, no alpha prefix match, so + * the whole string is treated as a bare path (matches buildContractId + * which also requires `[A-Za-z]+`). + * - `"GET::"` — method with empty path, returns `{ method: 'GET', path: '' }`; + * `normalizeRoutePath('')` resolves to `/` for caller. + */ +function parseHttpContract(raw: string): { method: string | null; path: string } { + const match = raw.match(/^([A-Za-z]+)::/); + if (!match) return { method: null, path: raw }; + return { method: match[1].toUpperCase(), path: raw.slice(match[0].length) }; +} + +/** + * Stable synthetic symbolUid for a manifest-declared contract whose target + * symbol could not be resolved against the per-repo graph (resolveSymbol + * returned null). Two reasons we don't leave the uid empty: + * + * 1. The bridge stores Contract nodes keyed in part by symbolUid; an empty + * uid means downstream Cypher queries that anchor on `provider.symbolUid` + * can't tell two different unresolved manifest contracts apart. + * 2. The cross-impact bridge query in cross-impact.ts joins local impact + * results to bridge contracts via `WHERE provider.symbolUid IN $localUids`. + * If the local impact engine produces a deterministic identifier for the + * unresolved target, it must agree with the value the bridge stored. A + * synthetic uid keyed off (repo, contractId) is the only thing both sides + * can derive without knowing about each other. + * + * Format: `manifest::::`. Stable across syncs, scoped to a + * single repo within a group, and never collides with real indexer uids + * (which never start with `manifest::`). + */ +export function manifestSymbolUid(repo: string, contractId: string): string { + return `manifest::${repo}::${contractId}`; +} + +export class ManifestExtractor { + async extractFromManifest( + links: GroupManifestLink[], + dbExecutors?: Map, + ): Promise { + // Resolve all (repo, link) pairs in parallel. The previous sequential + // await-per-link produced 2N round-trips; parallel resolution uses the + // per-repo executor pool directly and scales linearly with manifest size. + // + // Memoization: a manifest can list the same contract multiple times + // (e.g. a consumer and provider declaration, or cross-referenced groups). + // Key on (repo, type, contract) — the canonical input to the Cypher + // query — so duplicate links resolve to one DB hit. + type ResolvedSymbol = { filePath: string; name: string; uid: string } | null; + const resolveCache = new Map>(); + const resolveOnce = (repo: string, link: GroupManifestLink): Promise => { + const key = `${repo}\u0000${link.type}\u0000${link.contract}`; + let pending = resolveCache.get(key); + if (!pending) { + pending = this.resolveSymbol(repo, link, dbExecutors); + resolveCache.set(key, pending); + } + return pending; + }; + + const perLink = await Promise.all( + links.map(async (link) => { + const contractId = this.buildContractId(link.type, link.contract); + const providerRepo = link.role === 'provider' ? link.from : link.to; + const consumerRepo = link.role === 'provider' ? link.to : link.from; + const [providerSymbol, consumerSymbol] = await Promise.all([ + resolveOnce(providerRepo, link), + resolveOnce(consumerRepo, link), + ]); + return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol }; + }), + ); + + const contracts: StoredContract[] = []; + const crossLinks: CrossLink[] = []; + + for (const { + link, + contractId, + providerRepo, + consumerRepo, + providerSymbol, + consumerSymbol, + } of perLink) { + const providerRef = providerSymbol || { filePath: '', name: link.contract }; + const consumerRef = consumerSymbol || { filePath: '', name: link.contract }; + // When the resolver finds a real graph symbol we keep its uid, otherwise + // fall back to the deterministic synthetic uid (see manifestSymbolUid). + const providerUid = providerSymbol?.uid || manifestSymbolUid(providerRepo, contractId); + const consumerUid = consumerSymbol?.uid || manifestSymbolUid(consumerRepo, contractId); + + contracts.push({ + contractId, + type: link.type, + role: 'provider', + symbolUid: providerUid, + symbolRef: providerRef, + symbolName: link.contract, + confidence: 1.0, + meta: { source: 'manifest' }, + repo: providerRepo, + }); + + contracts.push({ + contractId, + type: link.type, + role: 'consumer', + symbolUid: consumerUid, + symbolRef: consumerRef, + symbolName: link.contract, + confidence: 1.0, + meta: { source: 'manifest' }, + repo: consumerRepo, + }); + + crossLinks.push({ + from: { repo: consumerRepo, symbolUid: consumerUid, symbolRef: consumerRef }, + to: { repo: providerRepo, symbolUid: providerUid, symbolRef: providerRef }, + type: link.type, + contractId, + matchType: 'manifest', + confidence: 1.0, + }); + } + + return { contracts, crossLinks }; + } + + private async resolveSymbol( + repoPathKey: string, + link: GroupManifestLink, + dbExecutors?: Map, + ): Promise<{ filePath: string; name: string; uid: string } | null> { + const executor = dbExecutors?.get(repoPathKey); + if (!executor) return null; + + // NOTE: All lookups use EXACT equality on the relevant name field and + // deterministic ORDER BY before LIMIT 1. Previous versions used CONTAINS + // for fuzzy matching (plus an unconditional ".proto" fallback for gRPC) + // which produced silent false positives: e.g. manifest "/orders" would + // match "/suborders", and a gRPC manifest entry in a repo with any + // .proto file would attach to a random proto symbol. + // + // If resolveSymbol returns null, the extractor falls back to a + // deterministic synthetic uid via `manifestSymbolUid(repo, contractId)` + // (see the function's docstring for why synthetic rather than empty). + // Cross-impact still works: the bridge query joins on the synthetic + // uid, and the local impact engine derives the same uid for the + // unresolved symbol — name-based hints are the additional safety net. + try { + let rows: Record[]; + if (link.type === 'http') { + // Route.name is the canonicalized URL path (see + // core/ingestion/pipeline.ts ensureSlash + generateId('Route', ...)). + // Normalize the manifest contract the same way so a user-written + // "/api/orders" matches "api/orders" in the graph. + // + // The contract may also use the explicit-method form "GET::/api/orders" + // recommended by buildContractId. Strip the METHOD:: prefix before + // normalizing — otherwise `normalizeRoutePath('GET::/api/orders')` + // returns `/GET::/api/orders` and never matches Route.name. The + // captured method is not yet used to constrain the Cypher query + // (method-aware HANDLES_ROUTE matching is a future enhancement). + const parsed = parseHttpContract(link.contract); + const normalized = normalizeRoutePath(parsed.path); + rows = await executor( + `MATCH (handler)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route) + WHERE route.name = $normalized + RETURN handler.id AS uid, handler.name AS name, handler.filePath AS filePath + ORDER BY handler.filePath ASC + LIMIT 1`, + { normalized }, + ); + } else if (link.type === 'topic') { + // Topic names aren't a first-class NodeLabel in the graph — + // topics are referenced by function/method symbols (Kafka + // listeners, publishers). Restrict to symbol-like labels to + // avoid cross-matching Files/Variables/Imports that happen to + // share the topic name. + rows = await executor( + `MATCH (n:Function|Method|Class|Interface) WHERE n.name = $contract + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { contract: link.contract }, + ); + } else if (link.type === 'grpc') { + // Contract is "Service/Method" or just "Service" (or package.Service + // variants). Prefer matching by method name when present, otherwise + // by service name. NO .proto path fallback — that's guaranteed to + // return a wrong symbol in any repo with more than one proto file. + // Label filters scope lookups: methods → Function|Method, services + // → Class|Interface (no label match = no silent wrong hits on + // File/Variable nodes that happen to share the name). + const parts = link.contract.split('/'); + const serviceName = parts[0]?.trim() ?? ''; + const methodName = parts[1]?.trim() ?? ''; + if (methodName) { + rows = await executor( + `MATCH (n:Function|Method) WHERE n.name = $methodName + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { methodName }, + ); + } else if (serviceName) { + rows = await executor( + `MATCH (n:Class|Interface) WHERE n.name = $serviceName + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { serviceName }, + ); + } else { + rows = []; + } + } else if (link.type === 'lib') { + // Only exact match on the symbol's name. Previous fallback to + // CONTAINS on n.filePath would promote "react" to "react-native" + // or "@types/react" — silent wrong attribution. Restrict to + // package-level labels so we don't return arbitrary symbols + // named after a library. + rows = await executor( + `MATCH (n:Package|Module) WHERE n.name = $contract + RETURN n.id AS uid, n.name AS name, n.filePath AS filePath + ORDER BY n.filePath ASC + LIMIT 1`, + { contract: link.contract }, + ); + } else { + return null; + } + if (rows.length > 0) { + return { + filePath: rows[0].filePath as string, + name: rows[0].name as string, + uid: String(rows[0].uid ?? ''), + }; + } + } catch (err) { + // Log but don't throw: a broken graph query in one repo shouldn't + // fail the whole manifest extraction. Unresolved contracts still + // get a synthetic symbolUid below, so cross-impact can proceed. + const message = err instanceof Error ? err.message : String(err); + console.warn( + `[manifest-extractor] resolveSymbol failed for ${link.type}:${link.contract} ` + + `in ${repoPathKey}: ${message}`, + ); + } + return null; + } + + /** + * Build a canonical contract id for a manifest link. + * + * HTTP is the only type with two valid forms: + * - Explicit method: `"GET::/api/orders"` → `"http::GET::/api/orders"` + * (matches exactly against `HttpRouteExtractor` provider/consumer + * contracts, which are also keyed by `http::::`). + * - Method-agnostic: `"/api/orders"` → `"http::*::/api/orders"` + * — the `*` is a wildcard and is intended to match any concrete + * HTTP method on that path. Wildcard-aware matching is the + * responsibility of the sync / cross-impact layer (see #793); + * downstream code should treat `http::*::` as matching + * every `http::::` for the same path. + * + * Recommend the explicit-method form in group.yaml whenever the + * manifest author knows the method — it round-trips through exact + * equality matching without requiring wildcard logic downstream. + * + * NOTE on exhaustiveness: the switch covers every current + * `ContractType` variant and falls through to a `never` assertion so + * TypeScript fails the build if a new variant is added without a + * corresponding case. + */ + private buildContractId(type: ContractType, contract: string): string { + switch (type) { + case 'http': { + // Canonicalize method casing and path separators so logically + // equivalent inputs (`get::/api/orders` vs `GET::/api/orders`, + // or trailing-slash variants) produce the same contractId and + // matching `manifestSymbolUid` fallback. Without this, raw + // user casing leaks into cross-impact join keys and fragments + // matches across repos. + const { method, path: rawPath } = parseHttpContract(contract); + const normalizedPath = normalizeRoutePath(rawPath); + return method ? `http::${method}::${normalizedPath}` : `http::*::${normalizedPath}`; + } + case 'grpc': + return `grpc::${contract}`; + case 'topic': + return `topic::${contract}`; + case 'lib': + return `lib::${contract}`; + case 'custom': + return `custom::${contract}`; + default: { + const _exhaustive: never = type; + throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`); + } + } + } +} diff --git a/gitnexus/src/core/group/extractors/topic-extractor.ts b/gitnexus/src/core/group/extractors/topic-extractor.ts index c27b419bb..1fbccac8a 100644 --- a/gitnexus/src/core/group/extractors/topic-extractor.ts +++ b/gitnexus/src/core/group/extractors/topic-extractor.ts @@ -1,214 +1,49 @@ -import * as fs from 'node:fs'; -import * as path from 'node:path'; import { glob } from 'glob'; +import Parser from 'tree-sitter'; import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafe } from './fs-utils.js'; +import { scanFile, unquoteLiteral } from './tree-sitter-scanner.js'; +import { + TOPIC_SCAN_GLOB, + getProviderForFile, + type Broker, + type TopicMeta, +} from './topic-patterns/index.js'; -type Broker = 'kafka' | 'rabbitmq' | 'nats'; +/** + * Language-agnostic orchestrator for topic (message broker) contract + * extraction. All grammar-specific knowledge lives in `topic-patterns/*` + * — this file must not import any tree-sitter grammar directly. + * + * Flow per file: + * 1. `getProviderForFile(rel)` → compiled plugin (or `undefined` if the + * file's extension isn't registered, in which case we skip it). + * 2. `scanFile(parser, provider, content)` → list of `{meta, valueText}` + * pairs, one per matched literal. + * 3. `unquoteLiteral(valueText)` → the raw topic string. + * 4. `makeContract(topic, meta, relPath)` → `ExtractedContract`. + * + * Adding a new language is a one-file edit in `topic-patterns/index.ts`. + */ -function readSafe(repoPath: string, rel: string): string | null { - const abs = path.resolve(repoPath, rel); - const base = path.resolve(repoPath); - const relToBase = path.relative(base, abs); - if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null; - try { - return fs.readFileSync(abs, 'utf-8'); - } catch { - return null; - } -} - -function makeContract( - topicName: string, - role: 'provider' | 'consumer', - filePath: string, - symbolName: string, - confidence: number, - broker: Broker, -): ExtractedContract { +function makeContract(topicName: string, meta: TopicMeta, filePath: string): ExtractedContract { return { contractId: `topic::${topicName}`, type: 'topic', - role, + role: meta.role, symbolUid: '', - symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: symbolName }, - symbolName, - confidence, + symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: meta.symbolName }, + symbolName: meta.symbolName, + confidence: meta.confidence, meta: { - broker, + broker: meta.broker satisfies Broker, topicName, - extractionStrategy: 'source_scan', + extractionStrategy: 'tree_sitter', }, }; } -interface PatternDef { - regex: RegExp; - role: 'provider' | 'consumer'; - broker: Broker; - confidence: number; - topicGroup: number; - symbolName: string; -} - -// --- Kafka patterns --- -const KAFKA_PATTERNS: PatternDef[] = [ - // Java: @KafkaListener(topics = "xxx") - { - regex: /@KafkaListener\s*\(\s*topics\s*=\s*"([^"]+)"/g, - role: 'consumer', - broker: 'kafka', - confidence: 0.8, - topicGroup: 1, - symbolName: 'kafkaListener', - }, - // Java: kafkaTemplate.send("xxx" - { - regex: /kafkaTemplate\.send\s*\(\s*"([^"]+)"/gi, - role: 'provider', - broker: 'kafka', - confidence: 0.8, - topicGroup: 1, - symbolName: 'kafkaTemplate.send', - }, - // Node: producer.send({ topic: 'xxx' - { - regex: /producer\.send\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g, - role: 'provider', - broker: 'kafka', - confidence: 0.8, - topicGroup: 1, - symbolName: 'producer.send', - }, - // Node: consumer.subscribe({ topic: 'xxx' - { - regex: /consumer\.subscribe\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g, - role: 'consumer', - broker: 'kafka', - confidence: 0.8, - topicGroup: 1, - symbolName: 'consumer.subscribe', - }, - // Go: consumer.ConsumePartition("xxx" - { - regex: /\.ConsumePartition\s*\(\s*"([^"]+)"/g, - role: 'consumer', - broker: 'kafka', - confidence: 0.7, - topicGroup: 1, - symbolName: 'ConsumePartition', - }, - // Python: KafkaConsumer('xxx' - { - regex: /KafkaConsumer\s*\(\s*['"]([^'"]+)['"]/g, - role: 'consumer', - broker: 'kafka', - confidence: 0.7, - topicGroup: 1, - symbolName: 'KafkaConsumer', - }, - // Python: producer.send('xxx' or producer.produce('xxx' - { - regex: /producer\.(?:send|produce)\s*\(\s*['"]([^'"]+)['"]/g, - role: 'provider', - broker: 'kafka', - confidence: 0.7, - topicGroup: 1, - symbolName: 'producer.send', - }, -]; - -// --- RabbitMQ patterns --- -const RABBITMQ_PATTERNS: PatternDef[] = [ - // Java: @RabbitListener(queues = "xxx") - { - regex: /@RabbitListener\s*\(\s*queues\s*=\s*"([^"]+)"/g, - role: 'consumer', - broker: 'rabbitmq', - confidence: 0.8, - topicGroup: 1, - symbolName: 'rabbitListener', - }, - // Java: rabbitTemplate.convertAndSend("xxx" - { - regex: /rabbitTemplate\.convertAndSend\s*\(\s*"([^"]+)"/gi, - role: 'provider', - broker: 'rabbitmq', - confidence: 0.8, - topicGroup: 1, - symbolName: 'rabbitTemplate.convertAndSend', - }, - // Node: channel.consume("xxx" - { - regex: /channel\.consume\s*\(\s*"([^"]+)"/g, - role: 'consumer', - broker: 'rabbitmq', - confidence: 0.8, - topicGroup: 1, - symbolName: 'channel.consume', - }, - // Node: channel.publish("xxx" - { - regex: /channel\.publish\s*\(\s*"([^"]+)"/g, - role: 'provider', - broker: 'rabbitmq', - confidence: 0.8, - topicGroup: 1, - symbolName: 'channel.publish', - }, - // Node: channel.sendToQueue("xxx" - { - regex: /channel\.sendToQueue\s*\(\s*"([^"]+)"/g, - role: 'provider', - broker: 'rabbitmq', - confidence: 0.8, - topicGroup: 1, - symbolName: 'channel.sendToQueue', - }, - // Python: channel.basic_consume(queue='xxx' - { - regex: /channel\.basic_consume\s*\(\s*queue\s*=\s*['"]([^'"]+)['"]/g, - role: 'consumer', - broker: 'rabbitmq', - confidence: 0.7, - topicGroup: 1, - symbolName: 'basic_consume', - }, - // Python: channel.basic_publish(exchange='xxx' - { - regex: /channel\.basic_publish\s*\([^)]*exchange\s*=\s*['"]([^'"]+)['"]/g, - role: 'provider', - broker: 'rabbitmq', - confidence: 0.7, - topicGroup: 1, - symbolName: 'basic_publish', - }, -]; - -// --- NATS patterns --- -const NATS_PATTERNS: PatternDef[] = [ - // Go/Node: nc.Subscribe("xxx" or nc.subscribe("xxx" - { - regex: /nc\.(?:S|s)ubscribe\s*\(\s*"([^"]+)"/g, - role: 'consumer', - broker: 'nats', - confidence: 0.8, - topicGroup: 1, - symbolName: 'nc.Subscribe', - }, - // Go/Node: nc.Publish("xxx" or nc.publish("xxx" - { - regex: /nc\.(?:P|p)ublish\s*\(\s*"([^"]+)"/g, - role: 'provider', - broker: 'nats', - confidence: 0.8, - topicGroup: 1, - symbolName: 'nc.Publish', - }, -]; - -const ALL_PATTERNS: PatternDef[] = [...KAFKA_PATTERNS, ...RABBITMQ_PATTERNS, ...NATS_PATTERNS]; - export class TopicExtractor implements ContractExtractor { type = 'topic' as const; @@ -221,46 +56,48 @@ export class TopicExtractor implements ContractExtractor { repoPath: string, _repo: RepoHandle, ): Promise { - const files = await glob('**/*.{ts,tsx,js,jsx,java,go,py}', { + const files = await glob(TOPIC_SCAN_GLOB, { cwd: repoPath, - ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'], + ignore: [ + '**/node_modules/**', + '**/.git/**', + '**/vendor/**', + '**/dist/**', + '**/build/**', + // Language-level test file conventions. Go test files + // `*_test.go` live next to source; other languages either use + // separate test directories (Python's `tests/`, Java's + // `src/test/`) or are already covered by the dist/build ignores. + // Pushed to the glob level so the orchestrator stays + // language-agnostic. + '**/*_test.go', + ], nodir: true, }); + // One parser reused across files; the scanner calls `setLanguage` per + // file based on which plugin the registry returns. + const parser = new Parser(); const out: ExtractedContract[] = []; + for (const rel of files) { + const provider = getProviderForFile(rel); + if (!provider) continue; + const content = readSafe(repoPath, rel); if (!content) continue; - out.push(...this.scanFile(content, rel)); - } - return this.dedupe(out); - } - - private scanFile(content: string, filePath: string): ExtractedContract[] { - const out: ExtractedContract[] = []; - - for (const pattern of ALL_PATTERNS) { - // Reset regex state for each file - const re = new RegExp(pattern.regex.source, pattern.regex.flags); - let m: RegExpExecArray | null; - while ((m = re.exec(content)) !== null) { - const topicName = m[pattern.topicGroup]; + const matches = scanFile(parser, provider, content); + for (const match of matches) { + const valueNode = match.captures.value; + if (!valueNode) continue; + const topicName = unquoteLiteral(valueNode.text); if (!topicName) continue; - out.push( - makeContract( - topicName, - pattern.role, - filePath, - pattern.symbolName, - pattern.confidence, - pattern.broker, - ), - ); + out.push(makeContract(topicName, match.meta, rel)); } } - return out; + return this.dedupe(out); } private dedupe(items: ExtractedContract[]): ExtractedContract[] { diff --git a/gitnexus/src/core/group/extractors/topic-patterns/go.ts b/gitnexus/src/core/group/extractors/topic-patterns/go.ts new file mode 100644 index 000000000..df3bab095 --- /dev/null +++ b/gitnexus/src/core/group/extractors/topic-patterns/go.ts @@ -0,0 +1,123 @@ +import Go from 'tree-sitter-go'; +import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js'; +import type { TopicMeta } from './types.js'; + +/** + * Go topic extraction patterns. + * + * Detects Sarama, segmentio/kafka-go and nats.go producer/consumer APIs: + * - `X.ConsumePartition("topic", ...)` + * - `sarama.ProducerMessage{Topic: "xxx"}` + * - `kafka.Writer{Topic: "xxx"}` / `kafka.WriterConfig{Topic: ...}` + * - `kafka.Reader{Topic: "xxx"}` / `kafka.ReaderConfig{Topic: ...}` + * - `nc.Subscribe("topic", ...)` / `js.Subscribe("topic", ...)` + * - `nc.Publish("topic", ...)` / `js.Publish("topic", ...)` + * + * Every query MUST bind `@value` to the topic literal node. + */ +const GO_TOPIC_SPEC: LanguagePatterns = { + name: 'go-topic', + language: Go, + patterns: [ + { + meta: { + role: 'consumer', + broker: 'kafka', + confidence: 0.7, + symbolName: 'ConsumePartition', + }, + query: ` + (call_expression + function: (selector_expression + field: (field_identifier) @method (#eq? @method "ConsumePartition")) + arguments: (argument_list . (interpreted_string_literal) @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'kafka', + confidence: 0.75, + symbolName: 'sarama.ProducerMessage', + }, + query: ` + (composite_literal + type: (qualified_type + package: (package_identifier) @pkg (#eq? @pkg "sarama") + name: (type_identifier) @ty (#eq? @ty "ProducerMessage")) + body: (literal_value + (keyed_element + (literal_element (identifier) @field (#eq? @field "Topic")) + (literal_element (interpreted_string_literal) @value)))) + `, + }, + { + meta: { + role: 'provider', + broker: 'kafka', + confidence: 0.75, + symbolName: 'kafka.Writer', + }, + query: ` + (composite_literal + type: (qualified_type + package: (package_identifier) @pkg (#eq? @pkg "kafka") + name: (type_identifier) @ty (#match? @ty "^(Writer|WriterConfig)$")) + body: (literal_value + (keyed_element + (literal_element (identifier) @field (#eq? @field "Topic")) + (literal_element (interpreted_string_literal) @value)))) + `, + }, + { + meta: { + role: 'consumer', + broker: 'kafka', + confidence: 0.75, + symbolName: 'kafka.Reader', + }, + query: ` + (composite_literal + type: (qualified_type + package: (package_identifier) @pkg (#eq? @pkg "kafka") + name: (type_identifier) @ty (#match? @ty "^(Reader|ReaderConfig)$")) + body: (literal_value + (keyed_element + (literal_element (identifier) @field (#eq? @field "Topic")) + (literal_element (interpreted_string_literal) @value)))) + `, + }, + { + meta: { + role: 'consumer', + broker: 'nats', + confidence: 0.8, + symbolName: 'nc.Subscribe', + }, + query: ` + (call_expression + function: (selector_expression + operand: (identifier) @obj (#match? @obj "^(nc|js)$") + field: (field_identifier) @method (#match? @method "^[Ss]ubscribe$")) + arguments: (argument_list . (interpreted_string_literal) @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'nats', + confidence: 0.8, + symbolName: 'nc.Publish', + }, + query: ` + (call_expression + function: (selector_expression + operand: (identifier) @obj (#match? @obj "^(nc|js)$") + field: (field_identifier) @method (#match? @method "^[Pp]ublish$")) + arguments: (argument_list . (interpreted_string_literal) @value)) + `, + }, + ], +}; + +export const GO_TOPIC_PROVIDER = compilePatterns(GO_TOPIC_SPEC); diff --git a/gitnexus/src/core/group/extractors/topic-patterns/index.ts b/gitnexus/src/core/group/extractors/topic-patterns/index.ts new file mode 100644 index 000000000..b6e1b8c1f --- /dev/null +++ b/gitnexus/src/core/group/extractors/topic-patterns/index.ts @@ -0,0 +1,49 @@ +import * as path from 'node:path'; +import type { CompiledPatterns } from '../tree-sitter-scanner.js'; +import type { TopicMeta } from './types.js'; +import { JAVA_TOPIC_PROVIDER } from './java.js'; +import { GO_TOPIC_PROVIDER } from './go.js'; +import { PYTHON_TOPIC_PROVIDER } from './python.js'; +import { + JAVASCRIPT_TOPIC_PROVIDER, + TYPESCRIPT_TOPIC_PROVIDER, + TSX_TOPIC_PROVIDER, +} from './node.js'; + +export type { TopicMeta, Broker } from './types.js'; + +/** + * File-extension → compiled-plugin registry for topic extraction. The + * top-level orchestrator (`topic-extractor.ts`) looks up the plugin for + * each file it visits and delegates the scanning to `tree-sitter-scanner`. + * + * Keys are lowercase extensions including the leading dot. To add a new + * language, drop a `topic-patterns/.ts` that exports a compiled + * provider, import it here and register the extension(s). No edits to + * `topic-extractor.ts` are required. + */ +const REGISTRY: Record> = { + '.java': JAVA_TOPIC_PROVIDER, + '.go': GO_TOPIC_PROVIDER, + '.py': PYTHON_TOPIC_PROVIDER, + '.js': JAVASCRIPT_TOPIC_PROVIDER, + '.jsx': JAVASCRIPT_TOPIC_PROVIDER, + '.ts': TYPESCRIPT_TOPIC_PROVIDER, + '.tsx': TSX_TOPIC_PROVIDER, +}; + +/** + * Glob pattern for files worth scanning. Kept here so adding a new + * language to the registry also widens the glob automatically via a + * single edit. + */ +export const TOPIC_SCAN_GLOB = '**/*.{ts,tsx,js,jsx,java,go,py}'; + +/** + * Return the compiled provider registered for the given file's + * extension, or `undefined` if the extension is not registered. + */ +export function getProviderForFile(rel: string): CompiledPatterns | undefined { + const ext = path.extname(rel).toLowerCase(); + return REGISTRY[ext]; +} diff --git a/gitnexus/src/core/group/extractors/topic-patterns/java.ts b/gitnexus/src/core/group/extractors/topic-patterns/java.ts new file mode 100644 index 000000000..d126f25ce --- /dev/null +++ b/gitnexus/src/core/group/extractors/topic-patterns/java.ts @@ -0,0 +1,83 @@ +import Java from 'tree-sitter-java'; +import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js'; +import type { TopicMeta } from './types.js'; + +/** + * Java topic extraction patterns. + * + * Detects Kafka and RabbitMQ (Spring conventions) producer/consumer APIs: + * - `@KafkaListener(topics = "xxx")` + * - `@RabbitListener(queues = "xxx")` + * - `kafkaTemplate.send("xxx", ...)` + * - `rabbitTemplate.convertAndSend("xxx", ...)` + * + * Every query MUST bind `@value` to the topic literal node. + */ +const JAVA_TOPIC_SPEC: LanguagePatterns = { + name: 'java-topic', + language: Java, + patterns: [ + { + meta: { + role: 'consumer', + broker: 'kafka', + confidence: 0.8, + symbolName: 'kafkaListener', + }, + query: ` + (annotation + name: (identifier) @name (#eq? @name "KafkaListener") + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key (#eq? @key "topics") + value: (string_literal) @value))) + `, + }, + { + meta: { + role: 'consumer', + broker: 'rabbitmq', + confidence: 0.8, + symbolName: 'rabbitListener', + }, + query: ` + (annotation + name: (identifier) @name (#eq? @name "RabbitListener") + arguments: (annotation_argument_list + (element_value_pair + key: (identifier) @key (#eq? @key "queues") + value: (string_literal) @value))) + `, + }, + { + meta: { + role: 'provider', + broker: 'kafka', + confidence: 0.8, + symbolName: 'kafkaTemplate.send', + }, + query: ` + (method_invocation + object: (identifier) @obj (#eq? @obj "kafkaTemplate") + name: (identifier) @method (#eq? @method "send") + arguments: (argument_list . (string_literal) @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'rabbitmq', + confidence: 0.8, + symbolName: 'rabbitTemplate.convertAndSend', + }, + query: ` + (method_invocation + object: (identifier) @obj (#eq? @obj "rabbitTemplate") + name: (identifier) @method (#eq? @method "convertAndSend") + arguments: (argument_list . (string_literal) @value)) + `, + }, + ], +}; + +export const JAVA_TOPIC_PROVIDER = compilePatterns(JAVA_TOPIC_SPEC); diff --git a/gitnexus/src/core/group/extractors/topic-patterns/node.ts b/gitnexus/src/core/group/extractors/topic-patterns/node.ts new file mode 100644 index 000000000..68f3a4ef8 --- /dev/null +++ b/gitnexus/src/core/group/extractors/topic-patterns/node.ts @@ -0,0 +1,165 @@ +import JavaScript from 'tree-sitter-javascript'; +import TypeScript from 'tree-sitter-typescript'; +import { + compilePatterns, + type LanguagePatterns, + type PatternSpec, +} from '../tree-sitter-scanner.js'; +import type { TopicMeta } from './types.js'; + +/** + * Node.js / TypeScript topic extraction patterns. + * + * Detects kafkajs, amqplib (RabbitMQ), and nats.js producer/consumer APIs: + * - `producer.send({ topic: 'xxx', ... })` (kafkajs) + * - `consumer.subscribe({ topic: 'xxx', ... })` (kafkajs) + * - `channel.consume("queue", ...)` / `channel.publish(...)` / `channel.sendToQueue(...)` + * - `nc.subscribe("topic")` / `js.subscribe("topic")` + * - `nc.publish("topic", ...)` / `js.publish("topic", ...)` + * + * The JavaScript and TypeScript tree-sitter grammars share node type + * names for every construct we query here, so the pattern sources are + * defined once and compiled against each grammar variant. We export three + * providers because Parser.Query objects are NOT portable across grammar + * instances — `.js` files use the JavaScript grammar, `.ts` uses + * TypeScript.typescript, and `.tsx` uses TypeScript.tsx. + * + * Every query MUST bind `@value` to the topic literal node. + */ +const NODE_TOPIC_PATTERNS: PatternSpec[] = [ + { + meta: { + role: 'provider', + broker: 'kafka', + confidence: 0.8, + symbolName: 'producer.send', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "producer") + property: (property_identifier) @prop (#eq? @prop "send")) + arguments: (arguments + (object + (pair + key: (property_identifier) @key (#eq? @key "topic") + value: [(string) (template_string)] @value)))) + `, + }, + { + meta: { + role: 'consumer', + broker: 'kafka', + confidence: 0.8, + symbolName: 'consumer.subscribe', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "consumer") + property: (property_identifier) @prop (#eq? @prop "subscribe")) + arguments: (arguments + (object + (pair + key: (property_identifier) @key (#eq? @key "topic") + value: [(string) (template_string)] @value)))) + `, + }, + { + meta: { + role: 'consumer', + broker: 'rabbitmq', + confidence: 0.8, + symbolName: 'channel.consume', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "channel") + property: (property_identifier) @prop (#eq? @prop "consume")) + arguments: (arguments . [(string) (template_string)] @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'rabbitmq', + confidence: 0.8, + symbolName: 'channel.publish', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "channel") + property: (property_identifier) @prop (#eq? @prop "publish")) + arguments: (arguments . [(string) (template_string)] @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'rabbitmq', + confidence: 0.8, + symbolName: 'channel.sendToQueue', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "channel") + property: (property_identifier) @prop (#eq? @prop "sendToQueue")) + arguments: (arguments . [(string) (template_string)] @value)) + `, + }, + { + meta: { + role: 'consumer', + broker: 'nats', + confidence: 0.8, + symbolName: 'nc.subscribe', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#match? @obj "^(nc|js)$") + property: (property_identifier) @prop (#match? @prop "^[Ss]ubscribe$")) + arguments: (arguments . [(string) (template_string)] @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'nats', + confidence: 0.8, + symbolName: 'nc.publish', + }, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#match? @obj "^(nc|js)$") + property: (property_identifier) @prop (#match? @prop "^[Pp]ublish$")) + arguments: (arguments . [(string) (template_string)] @value)) + `, + }, +]; + +const JAVASCRIPT_TOPIC_SPEC: LanguagePatterns = { + name: 'javascript-topic', + language: JavaScript, + patterns: NODE_TOPIC_PATTERNS, +}; + +const TYPESCRIPT_TOPIC_SPEC: LanguagePatterns = { + name: 'typescript-topic', + language: TypeScript.typescript, + patterns: NODE_TOPIC_PATTERNS, +}; + +const TSX_TOPIC_SPEC: LanguagePatterns = { + name: 'tsx-topic', + language: TypeScript.tsx, + patterns: NODE_TOPIC_PATTERNS, +}; + +export const JAVASCRIPT_TOPIC_PROVIDER = compilePatterns(JAVASCRIPT_TOPIC_SPEC); +export const TYPESCRIPT_TOPIC_PROVIDER = compilePatterns(TYPESCRIPT_TOPIC_SPEC); +export const TSX_TOPIC_PROVIDER = compilePatterns(TSX_TOPIC_SPEC); diff --git a/gitnexus/src/core/group/extractors/topic-patterns/python.ts b/gitnexus/src/core/group/extractors/topic-patterns/python.ts new file mode 100644 index 000000000..d84cae999 --- /dev/null +++ b/gitnexus/src/core/group/extractors/topic-patterns/python.ts @@ -0,0 +1,119 @@ +import Python from 'tree-sitter-python'; +import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js'; +import type { TopicMeta } from './types.js'; + +/** + * Python topic extraction patterns. + * + * Detects kafka-python, pika (RabbitMQ), and nats-py producer/consumer APIs: + * - `KafkaConsumer('topic', ...)` + * - `producer.send('topic', ...)` / `producer.produce('topic', ...)` + * - `channel.basic_consume(queue='xxx', ...)` + * - `channel.basic_publish(exchange='xxx', ...)` + * - `await nc.subscribe('topic')` + * - `await nc.publish('topic', ...)` + * + * Every query MUST bind `@value` to the topic literal node. + */ +const PYTHON_TOPIC_SPEC: LanguagePatterns = { + name: 'python-topic', + language: Python, + patterns: [ + { + meta: { + role: 'consumer', + broker: 'kafka', + confidence: 0.7, + symbolName: 'KafkaConsumer', + }, + query: ` + (call + function: (identifier) @func (#eq? @func "KafkaConsumer") + arguments: (argument_list . (string) @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'kafka', + confidence: 0.7, + symbolName: 'producer.send', + }, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "producer") + attribute: (identifier) @method (#match? @method "^(send|produce)$")) + arguments: (argument_list . (string) @value)) + `, + }, + { + meta: { + role: 'consumer', + broker: 'rabbitmq', + confidence: 0.7, + symbolName: 'basic_consume', + }, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "channel") + attribute: (identifier) @method (#eq? @method "basic_consume")) + arguments: (argument_list + (keyword_argument + name: (identifier) @kw (#eq? @kw "queue") + value: (string) @value))) + `, + }, + { + meta: { + role: 'provider', + broker: 'rabbitmq', + confidence: 0.7, + symbolName: 'basic_publish', + }, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "channel") + attribute: (identifier) @method (#eq? @method "basic_publish")) + arguments: (argument_list + (keyword_argument + name: (identifier) @kw (#eq? @kw "exchange") + value: (string) @value))) + `, + }, + { + meta: { + role: 'consumer', + broker: 'nats', + confidence: 0.75, + symbolName: 'nc.subscribe', + }, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "nc") + attribute: (identifier) @method (#eq? @method "subscribe")) + arguments: (argument_list . (string) @value)) + `, + }, + { + meta: { + role: 'provider', + broker: 'nats', + confidence: 0.75, + symbolName: 'nc.publish', + }, + query: ` + (call + function: (attribute + object: (identifier) @obj (#eq? @obj "nc") + attribute: (identifier) @method (#eq? @method "publish")) + arguments: (argument_list . (string) @value)) + `, + }, + ], +}; + +export const PYTHON_TOPIC_PROVIDER = compilePatterns(PYTHON_TOPIC_SPEC); diff --git a/gitnexus/src/core/group/extractors/topic-patterns/types.ts b/gitnexus/src/core/group/extractors/topic-patterns/types.ts new file mode 100644 index 000000000..3a27f21d3 --- /dev/null +++ b/gitnexus/src/core/group/extractors/topic-patterns/types.ts @@ -0,0 +1,27 @@ +/** + * Shared types for the topic-extractor language plugins. + * + * Each plugin lives in its own file (java.ts, go.ts, ...) and owns the + * tree-sitter grammar import + query sources. The top-level + * `topic-extractor.ts` orchestrator only knows about this type module and + * the plugin registry (`./index.ts`). It MUST NOT import any grammar or + * query text directly — that's the whole point of the split. + */ + +export type Broker = 'kafka' | 'rabbitmq' | 'nats'; + +/** + * Per-pattern payload every topic plugin attaches to its query. Whatever + * the pattern matches, the orchestrator receives this object verbatim + * and uses it to build an `ExtractedContract`. + * + * Plugins produce one `TopicMeta` per pattern (not per match) because a + * single query uniquely identifies its broker/role/confidence triple. + */ +export interface TopicMeta { + role: 'provider' | 'consumer'; + broker: Broker; + confidence: number; + /** Short human-readable label of the API being detected. */ + symbolName: string; +} diff --git a/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts b/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts new file mode 100644 index 000000000..cd50456aa --- /dev/null +++ b/gitnexus/src/core/group/extractors/tree-sitter-scanner.ts @@ -0,0 +1,193 @@ +import Parser from 'tree-sitter'; + +/** + * Shared, language-agnostic tree-sitter scanning utilities used by group + * extractors (topic, http, grpc, ...). + * + * Design goals: + * - The top-level extractors must not import any tree-sitter grammar. + * - Per-language plugins own their grammar import, their query sources, + * and the mapping from capture → meta. + * - This module provides the plumbing: compile queries once per plugin, + * parse a file with a given grammar, run all patterns, and return the + * captured `string_literal`-style nodes together with the plugin's meta. + */ + +/** + * One pattern owned by a language plugin. Each pattern owns a tree-sitter + * S-expression query. Plugins can freely choose which capture names to + * use — the scanner exposes every capture in the returned `captures` + * map and does not privilege any particular name. + * + * `TMeta` is the plugin-specific payload the orchestrator receives back + * when this pattern matches — e.g. for topic extraction it carries the + * broker name, role, confidence, symbol name. + */ +export interface PatternSpec { + /** Tree-sitter S-expression. */ + query: string; + /** Plugin-specific payload returned on every match. */ + meta: TMeta; +} + +/** + * A set of patterns owned by one language plugin, bound to a specific + * tree-sitter grammar. + * + * `language` is typed as `unknown` because tree-sitter's TypeScript + * declarations use `any` for the grammar object, and the grammar modules + * export different shapes (plain grammar vs. namespace with `typescript` + * / `tsx` members). Callers pass the concrete grammar object; this + * module forwards it to `parser.setLanguage` / `new Parser.Query`. + */ +export interface LanguagePatterns { + /** Human-readable plugin name for diagnostics. */ + name: string; + /** tree-sitter grammar object. */ + language: unknown; + /** Patterns authored against `language`. */ + patterns: PatternSpec[]; +} + +/** + * Compiled form of a `LanguagePatterns` bundle. Queries are compiled + * eagerly at module load time so a broken grammar/query pair fails + * loudly the first time the plugin is imported, instead of silently + * at scan time when no contract is produced. + */ +export interface CompiledPatterns { + name: string; + language: unknown; + patterns: CompiledPattern[]; +} + +export interface CompiledPattern { + query: Parser.Query; + meta: TMeta; +} + +/** + * Map from capture name → syntax node. Every named capture the query + * binds is exposed as an entry. If a query captures the same name more + * than once (unusual), the first occurrence wins — plugins that need + * all occurrences should use distinct capture names or fall back to + * `match.captures` array directly by iterating `query.matches()` + * themselves. + */ +export type CaptureMap = Record; + +/** + * One match returned by `scanFile` / `runCompiledPatterns`. The caller + * receives the full capture map plus the plugin meta, and is + * responsible for turning it into a domain object. + */ +export interface ScanMatch { + meta: TMeta; + captures: CaptureMap; +} + +/** + * Compile a LanguagePatterns bundle. Call this once per plugin, at + * module load time, and export the result. Throws if any pattern + * fails to compile against the grammar — that's a bug in the plugin + * author's query, not a runtime condition. + */ +export function compilePatterns(bundle: LanguagePatterns): CompiledPatterns { + const compiled: CompiledPattern[] = []; + for (const spec of bundle.patterns) { + try { + const query = new Parser.Query(bundle.language, spec.query); + compiled.push({ query, meta: spec.meta }); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + `[tree-sitter-scanner] Failed to compile pattern in ${bundle.name}: ${message}\n` + + `Query source:\n${spec.query}`, + ); + } + } + return { name: bundle.name, language: bundle.language, patterns: compiled }; +} + +/** + * Run every compiled pattern in `plugin` against an already-parsed + * tree. Use this when a plugin needs multiple query bundles against + * the same file (e.g. one query for class-level prefixes and another + * for method-level annotations) and wants to avoid re-parsing. + */ +export function runCompiledPatterns( + plugin: CompiledPatterns, + tree: Parser.Tree, +): ScanMatch[] { + const out: ScanMatch[] = []; + for (const compiled of plugin.patterns) { + let matches: Parser.QueryMatch[]; + try { + matches = compiled.query.matches(tree.rootNode); + } catch { + continue; + } + for (const match of matches) { + const captures: CaptureMap = {}; + for (const cap of match.captures) { + if (!(cap.name in captures)) captures[cap.name] = cap.node; + } + out.push({ meta: compiled.meta, captures }); + } + } + return out; +} + +/** + * Parse `content` with the plugin's grammar and run every compiled + * pattern against the AST. Returns one `ScanMatch` per matched query + * occurrence, carrying the plugin's meta payload. + * + * Errors are swallowed at the file level (malformed file must not abort + * the whole extract). Individual pattern failures are swallowed too so + * a single unusable query doesn't block the rest of the plugin. + */ +export function scanFile( + parser: Parser, + plugin: CompiledPatterns, + content: string, +): ScanMatch[] { + let tree: Parser.Tree; + try { + parser.setLanguage(plugin.language); + tree = parser.parse(content); + } catch { + return []; + } + return runCompiledPatterns(plugin, tree); +} + +/** + * Strip enclosing quotes from a tree-sitter string literal node's text. + * Handles single / double / template quotes, Python triple-quoted strings, + * and Go raw string literals (backticks). + * + * Returns null for empty/nullish input so callers can uniformly skip + * captures whose value is missing. + */ +export function unquoteLiteral(raw: string): string | null { + if (!raw) return null; + + // Python triple-quoted + if ( + (raw.startsWith('"""') && raw.endsWith('"""')) || + (raw.startsWith("'''") && raw.endsWith("'''")) + ) { + return raw.slice(3, -3); + } + + const first = raw[0]; + const last = raw[raw.length - 1]; + if ((first === '"' || first === "'" || first === '`') && last === first && raw.length >= 2) { + return raw.slice(1, -1); + } + + // Some grammars expose the string content without quotes already (e.g. + // Python `string_content` child). Return as-is. + return raw; +} diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index 6d39f4ce4..ec793968b 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -5,6 +5,15 @@ export interface MatchResult { unmatched: StoredContract[]; } +export interface WildcardMatchResult { + matched: CrossLink[]; + remaining: StoredContract[]; +} + +function isGrpcWildcard(cid: string): boolean { + return cid.startsWith('grpc::') && cid.endsWith('/*'); +} + export function normalizeContractId(id: string): string { const colonIdx = id.indexOf('::'); if (colonIdx === -1) return id; @@ -24,6 +33,22 @@ export function normalizeContractId(id: string): string { return id; } case 'grpc': { + // Canonical form: `grpc::[/]`. + // + // The package/service segment is lowercased because gRPC package + // names are effectively case-insensitive across language bindings + // (`auth.AuthService`, `auth.authservice`, `AUTH.AUTHSERVICE` all + // describe the same wire protocol service). The RPC method segment + // is preserved as-is because the HTTP/2 path used on the wire is + // case-sensitive per the gRPC spec (`/Service/MethodName`), and + // method names in generated clients match the proto source exactly. + // + // A package-only id (no slash) and a package/method id are treated + // as DISTINCT canonical forms: `grpc::userservice` does not match + // `grpc::userservice/Login`. That's by design — callers that want + // service-level manifest matching against method-level providers + // should use the gRPC wildcard form `grpc::UserService/*` which is + // handled by runWildcardMatch below. const slashIdx = rest.indexOf('/'); if (slashIdx > 0) { const pkg = rest.substring(0, slashIdx).toLowerCase(); @@ -31,12 +56,12 @@ export function normalizeContractId(id: string): string { return `grpc::${pkg}${method}`; } if (slashIdx === 0) { - // Malformed "package/method" with leading slash — do not lowercase the whole string - // (method segment is case-sensitive per spec). + // Malformed "/method" with leading slash — keep as-is so two + // equally malformed ids can still match each other. return `grpc::${rest}`; } - // No slash: spec is ambiguous (package-only vs full service.method). MVP: lowercase - // the whole token; differs from pkg/method split above where RPC method keeps case. + // No slash: package/service only. Lowercase to match the package + // segment produced by the pkg/method branch above. return `grpc::${rest.toLowerCase()}`; } case 'topic': @@ -66,27 +91,36 @@ function findMatchingKeys(contractId: string, index: Map { const providers = contracts.filter((c) => c.role === 'provider'); - const consumers = contracts.filter((c) => c.role === 'consumer'); - - const providerIndex = new Map(); + const index = new Map(); for (const p of providers) { const key = normalizeContractId(p.contractId); - const list = providerIndex.get(key) || []; + const list = index.get(key) || []; list.push(p); - providerIndex.set(key, list); + index.set(key, list); } + return index; +} + +export function runExactMatch( + contracts: StoredContract[], + providerIndex?: Map, +): MatchResult { + const index = providerIndex ?? buildProviderIndex(contracts); + + // Skip gRPC wildcard consumers — they go to wildcard pass only + const consumers = contracts.filter((c) => c.role === 'consumer' && !isGrpcWildcard(c.contractId)); const matched: CrossLink[] = []; const matchedConsumerIds = new Set(); const matchedProviderIds = new Set(); for (const consumer of consumers) { - const matchingKeys = findMatchingKeys(consumer.contractId, providerIndex); + const matchingKeys = findMatchingKeys(consumer.contractId, index); if (matchingKeys.length === 0) continue; - const allMatchingProviders = matchingKeys.flatMap((k) => providerIndex.get(k) || []); + const allMatchingProviders = matchingKeys.flatMap((k) => index.get(k) || []); for (const provider of allMatchingProviders) { if (provider.repo === consumer.repo) { if (!provider.service || !consumer.service || provider.service === consumer.service) { @@ -118,10 +152,86 @@ export function runExactMatch(contracts: StoredContract[]): MatchResult { } } - const unmatched = contracts.filter((c) => { + // normalUnmatched: contracts that weren't matched in exact pass + const normalUnmatched = contracts.filter((c) => { + if (isGrpcWildcard(c.contractId)) return false; // excluded from exact, handled separately const id = `${c.repo}::${c.contractId}`; return c.role === 'provider' ? !matchedProviderIds.has(id) : !matchedConsumerIds.has(id); }); + // Re-add gRPC wildcard contracts — they were never in exact matching + const grpcWildcards = contracts.filter((c) => isGrpcWildcard(c.contractId)); + const unmatched = [...normalUnmatched, ...grpcWildcards]; + return { matched, unmatched }; } + +export function runWildcardMatch( + unmatched: StoredContract[], + providerIndex: Map, +): WildcardMatchResult { + const wildcardConsumers = unmatched.filter( + (c) => c.role === 'consumer' && isGrpcWildcard(c.contractId), + ); + const matched: CrossLink[] = []; + const matchedConsumerIds = new Set(); + + for (const consumer of wildcardConsumers) { + const normalized = normalizeContractId(consumer.contractId); + // "grpc::com.example.userservice/*" → "com.example.userservice" + // "grpc::userservice/*" → "userservice" + const fqService = normalized.slice(normalized.indexOf('::') + 2, -2); // strip "grpc::" and "/*" + + for (const [key, providers] of providerIndex) { + // Only match against non-wildcard gRPC providers (method-level IDs) + if (!key.startsWith('grpc::') || key.endsWith('/*')) continue; + const afterPrefix = key.slice(6); // strip "grpc::" + const slashIdx = afterPrefix.indexOf('/'); + if (slashIdx < 0) continue; + const providerFqService = afterPrefix.slice(0, slashIdx); + + // Match: exact FQ service, or bare-name match when consumer has no package + const isMatch = + providerFqService === fqService || + (!fqService.includes('.') && providerFqService.endsWith('.' + fqService)); + + if (!isMatch) continue; + + for (const provider of providers) { + // Skip same-repo same-service (same logic as runExactMatch) + if (provider.repo === consumer.repo) { + if (!provider.service || !consumer.service || provider.service === consumer.service) { + continue; + } + } + + matched.push({ + from: { + repo: consumer.repo, + service: consumer.service, + symbolUid: consumer.symbolUid, + symbolRef: consumer.symbolRef, + }, + to: { + repo: provider.repo, + service: provider.service, + symbolUid: provider.symbolUid, + symbolRef: provider.symbolRef, + }, + type: consumer.type, + contractId: consumer.contractId, // consumer's wildcard ID + matchType: 'wildcard', + confidence: Math.min(provider.confidence, consumer.confidence), + }); + matchedConsumerIds.add(`${consumer.repo}::${consumer.contractId}`); + } + } + } + + const remaining = unmatched.filter((c) => { + if (c.role !== 'consumer' || !isGrpcWildcard(c.contractId)) return true; + return !matchedConsumerIds.has(`${c.repo}::${c.contractId}`); + }); + + return { matched, remaining }; +} diff --git a/gitnexus/src/core/group/normalization.ts b/gitnexus/src/core/group/normalization.ts new file mode 100644 index 000000000..c99d36850 --- /dev/null +++ b/gitnexus/src/core/group/normalization.ts @@ -0,0 +1,124 @@ +import type { CrossLink, CrossLinkEndpoint, StoredContract } from './types.js'; + +function contractKey(contract: StoredContract): string { + return [contract.repo, contract.contractId, contract.role, contract.symbolRef.filePath].join( + '\0', + ); +} + +function endpointKey(endpoint: CrossLinkEndpoint): string { + return [ + endpoint.repo, + endpoint.service ?? '', + endpoint.symbolRef.filePath, + endpoint.symbolRef.name, + ].join('\0'); +} + +/** + * Score a contract by how much information it carries, so `dedupeContracts` + * can prefer the "richer" record when two contracts collide on the same + * `(repo, contractId, role, filePath)` key. + * + * Weights express a priority ordering, not calibrated probabilities: + * +3 — `symbolUid` resolved (tier 1 of the downstream lookup — highest + * signal because it's the strongest anchor for cross-impact traversal + * and the only one that's robust to renames) + * +2 — any of `filePath`, `symbolRef.name`, or `symbolName` that's more + * specific than the contractId itself (tier 2 signal — resolves + * uniquely in most cases and survives across syncs) + * +1 — `service` tag (monorepo attribution — useful but not sufficient + * on its own) or non-manifest origin (auto-extracted contracts are + * preferred over manifest-declared synthetic ones because the former + * are grounded in real source code) + * + * The absolute numbers don't matter, only their relative ordering. + */ +function contractRichness(contract: StoredContract): number { + let score = 0; + if (contract.symbolUid) score += 3; + if (contract.symbolRef.filePath) score += 2; + if (contract.symbolRef.name && contract.symbolRef.name !== contract.contractId) score += 2; + if (contract.symbolName && contract.symbolName !== contract.contractId) score += 2; + if (contract.service) score += 1; + if (contract.meta.source !== 'manifest') score += 1; + return score; +} + +function mergeContracts(existing: StoredContract, incoming: StoredContract): StoredContract { + const [primary, secondary] = + contractRichness(incoming) > contractRichness(existing) + ? [incoming, existing] + : [existing, incoming]; + const symbolRefName = primary.symbolRef.name || secondary.symbolRef.name; + return { + ...secondary, + ...primary, + symbolUid: primary.symbolUid || secondary.symbolUid, + symbolRef: { + filePath: primary.symbolRef.filePath || secondary.symbolRef.filePath, + name: symbolRefName, + }, + symbolName: primary.symbolName || secondary.symbolName || symbolRefName, + confidence: Math.max(existing.confidence, incoming.confidence), + service: primary.service ?? secondary.service, + meta: { ...secondary.meta, ...primary.meta }, + }; +} + +function mergeEndpoints( + existing: CrossLinkEndpoint, + incoming: CrossLinkEndpoint, +): CrossLinkEndpoint { + return { + repo: existing.repo, + service: existing.service ?? incoming.service, + symbolUid: existing.symbolUid || incoming.symbolUid, + symbolRef: { + filePath: existing.symbolRef.filePath || incoming.symbolRef.filePath, + name: existing.symbolRef.name || incoming.symbolRef.name, + }, + }; +} + +function crossLinkKey(link: CrossLink): string { + return [ + link.type, + link.contractId, + link.matchType, + endpointKey(link.from), + endpointKey(link.to), + ].join('\0'); +} + +export function dedupeContracts(items: StoredContract[]): StoredContract[] { + const deduped = new Map(); + for (const contract of items) { + const key = contractKey(contract); + const existing = deduped.get(key); + deduped.set(key, existing ? mergeContracts(existing, contract) : contract); + } + return [...deduped.values()]; +} + +export function dedupeCrossLinks(items: CrossLink[]): CrossLink[] { + const deduped = new Map(); + for (const link of items) { + const key = crossLinkKey(link); + const existing = deduped.get(key); + if (!existing) { + deduped.set(key, link); + continue; + } + const keepIncoming = link.confidence > existing.confidence; + const primary = keepIncoming ? link : existing; + const secondary = keepIncoming ? existing : link; + deduped.set(key, { + ...primary, + confidence: Math.max(existing.confidence, link.confidence), + from: mergeEndpoints(primary.from, secondary.from), + to: mergeEndpoints(primary.to, secondary.to), + }); + } + return [...deduped.values()]; +} diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 92cd9fe5f..af7c3e686 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -7,6 +7,7 @@ import type { GroupConfig, RepoHandle, RepoSnapshot, StoredContract, CrossLink } import { HttpRouteExtractor } from './extractors/http-route-extractor.js'; import { GrpcExtractor } from './extractors/grpc-extractor.js'; import { TopicExtractor } from './extractors/topic-extractor.js'; +import { ManifestExtractor } from './extractors/manifest-extractor.js'; import { runExactMatch } from './matching.js'; import { detectServiceBoundaries, assignService } from './service-boundary-detector.js'; import type { CypherExecutor } from './contract-extractor.js'; @@ -60,10 +61,28 @@ function defaultResolveHandle(allEntries: RegistryEntry[]) { }; } +/** + * Dedupe cross-links that point from the same consumer endpoint to the same + * provider endpoint for the same contract. Preserves first-seen order so the + * caller controls precedence (e.g., pass manifest links first). + */ +function dedupeCrossLinks(links: CrossLink[]): CrossLink[] { + const seen = new Set(); + const out: CrossLink[] = []; + for (const link of links) { + const key = `${link.from.repo}::${link.from.symbolUid}|${link.to.repo}::${link.to.symbolUid}|${link.type}|${link.contractId}`; + if (seen.has(key)) continue; + seen.add(key); + out.push(link); + } + return out; +} + export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promise { const missingRepos: string[] = []; const repoSnapshots: Record = {}; let autoContracts: StoredContract[] = []; + let manifestCrossLinks: CrossLink[] = []; let dbExecutors: Map | undefined; const eo = opts?.extractorOverride; @@ -158,8 +177,44 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis } } + // Process manifest links declared in group.yaml. + // ManifestExtractor is fully implemented but was never wired into this + // pipeline — config.links were parsed and validated but silently dropped. + // Placed after the DB try/finally: resolveSymbol falls back to synthetic + // UIDs when dbExecutors is undefined or a pool is closed, so cross-links + // are always generated regardless of whether real DB executors are available. + if (config.links.length > 0) { + // Warn about dangling links that reference repos not declared in config.repos. + // They still generate cross-links via synthetic UIDs (determinism is preserved), + // but the operator probably meant something that now silently does nothing useful. + const knownRepos = new Set(Object.keys(config.repos)); + for (const link of config.links) { + const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r)); + if (dangling.length > 0) { + console.warn( + `[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`, + ); + } + } + + const manifestEx = new ManifestExtractor(); + const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors); + autoContracts.push(...manifestResult.contracts); + manifestCrossLinks = manifestResult.crossLinks; + if (opts?.verbose) { + console.log( + ` manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`, + ); + } + } + const { matched, unmatched } = runExactMatch(autoContracts); - const crossLinks: CrossLink[] = matched; + + // Dedupe cross-links. Manifest contracts participate in runExactMatch, so a + // manifest-declared link can also emit a matchType:'exact' CrossLink with the + // same endpoints. Prefer the manifest version — it reflects operator intent + // and carries matchType:'manifest' which downstream consumers may rely on. + const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]); const allContracts: StoredContract[] = autoContracts; const registry: ContractRegistry = { diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index 7ab0f071a..b9ba97582 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,5 +1,5 @@ export type ContractType = 'http' | 'grpc' | 'topic' | 'lib' | 'custom'; -export type MatchType = 'exact' | 'manifest' | 'bm25' | 'embedding'; +export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding'; export type ContractRole = 'provider' | 'consumer'; export interface GroupConfig { @@ -131,3 +131,17 @@ export interface OutOfScopeLink { contractId: string; confidence: number; } + +/** Opaque handle to an open bridge LadybugDB. */ +export interface BridgeHandle { + /** Internal — do not access directly. */ + readonly _db: unknown; + readonly _conn: unknown; + readonly groupDir: string; +} + +export interface BridgeMeta { + version: number; + generatedAt: string; + missingRepos: string[]; +} diff --git a/gitnexus/src/core/ingestion/binding-accumulator.ts b/gitnexus/src/core/ingestion/binding-accumulator.ts new file mode 100644 index 000000000..adea3a202 --- /dev/null +++ b/gitnexus/src/core/ingestion/binding-accumulator.ts @@ -0,0 +1,391 @@ +/** + * BindingAccumulator — read-append-only accumulator that collects TypeEnv + * bindings across files in the GitNexus analyzer pipeline. + * + * **Current behavior (both execution paths):** The accumulator carries only + * file-scope (`scope = ''`) entries. Function-scope bindings are stripped + * at both write sites: + * + * - **Worker path**: `parse-worker.ts` serializes only + * `typeEnv.fileScope()` entries across the IPC boundary. + * - **Sequential path**: `type-env.ts::flush()` iterates only the FILE_SCOPE + * entry of the env map and writes `BindingEntry` records with + * `scope: ''` hardcoded. + * + * The narrowing exists because function-scope bindings have zero downstream + * consumers today and were previously costing ~4.9 MB of heap + IPC on + * every pipeline run. See `type-env.ts::flush()` and the `FileScopeBindings` + * JSDoc in `parse-worker.ts` for the paired Phase 9 reversion checklist. + * + * **Historical quality asymmetry (Phase 9 consideration):** Even though + * both paths now carry only file-scope data, the two paths were built + * under different resolution capabilities, and a future Phase 9 reverter + * that widens them back to all scopes will inherit that asymmetry: + * + * - **Sequential path** had (and would regain) access to the full + * `SymbolTable` and `importedBindings`, so its bindings benefit from + * Tier 2 cross-file propagation. + * - **Worker path** runs without `SymbolTable` / `importedBindings` and + * can only produce Tier 0 (annotation-declared) and local Tier 1 + * (same-file constructor inference) bindings. + * + * Phase 9 consumers that trust every entry equally will silently produce + * worse results for large repos (worker-dominant) than small ones + * (sequential-dominant). If Phase 9 needs homogeneous quality, either + * (a) tag entries with their tier at insert time so consumers can filter, + * or (b) post-process worker-path entries through a follow-up resolution + * pass after the main-thread `SymbolTable` is complete. + * + * **Lifecycle contract**: single-use — `append* → finalize → consume → dispose`. + * After `dispose()` the accumulator is permanently dead: any mutating call + * (`appendFile`) throws, and read methods return empty/undefined as if the + * accumulator had never been appended to. The instance is not recyclable; + * construct a new one for a new pipeline run. Finalization and disposal are + * orthogonal state dimensions and may be invoked in either order. + */ + +export interface BindingEntry { + readonly scope: string; // '' for file-level, 'funcName@startIndex' for function-local + readonly varName: string; + readonly typeName: string; +} + +/** + * Minimal graph-node shape required by `enrichExportedTypeMap()`. Intentionally + * narrower than the full `GraphNode` type in `graph/types.ts` so tests can + * construct a minimal mock without depending on the full graph module, and + * so the enrichment logic is a pure function over this contract. + * + * Matches the shape of the real `KnowledgeGraph` node's `properties.isExported` + * access path — tests that use a different shape silently pass while + * production fails. + */ +export interface EnrichmentGraphNode { + readonly id: string; + readonly properties?: { readonly isExported?: boolean } | undefined; +} + +/** + * Minimal graph lookup interface used by `enrichExportedTypeMap()`. + * Consumes only the method the enrichment loop actually calls. + */ +export interface EnrichmentGraphLookup { + getNode(id: string): EnrichmentGraphNode | undefined; +} + +/** + * Merge file-scope bindings from a (finalized) `BindingAccumulator` into an + * `exportedTypeMap` for symbols whose graph nodes are marked as exported. + * + * This is the single source of truth for the worker-path ExportedTypeMap + * enrichment loop. Previously the logic lived inline in `pipeline.ts` and + * the test suite reimplemented it as a `runEnrichmentLoop` helper — a + * drift-prone pattern that meant tests could pass while production regressed. + * Extracting it here makes the production code call the same function the + * tests call. + * + * **Node ID candidate order**: `Function:{filePath}:{name}` → + * `Variable:{filePath}:{name}` → `Const:{filePath}:{name}`. First match wins. + * + * **Tier 0 priority**: if `exportedTypeMap` already has an entry for a + * `(filePath, name)` pair, the accumulator entry does NOT overwrite it — + * the SymbolTable tier-0 pass is authoritative. Without this guard, a + * worker-path binding could clobber a higher-quality type from SymbolTable. + * + * **Finalize precondition**: the accumulator should be finalized before + * calling this function. The lifecycle contract is + * `append → finalize → enrich → dispose`. Finalization is not asserted + * here (the test suite and pipeline both honor it separately), but any + * append happening concurrently with this enrichment would be a lifecycle + * bug at the caller level. + * + * @returns The number of new entries written into `exportedTypeMap` + * (0 on empty accumulator or when every candidate was filtered + * out by the export check or the Tier 0 guard). + */ +export function enrichExportedTypeMap( + bindingAccumulator: BindingAccumulator, + graph: EnrichmentGraphLookup, + exportedTypeMap: Map>, +): number { + if (bindingAccumulator.fileCount === 0) return 0; + let enriched = 0; + for (const filePath of bindingAccumulator.files()) { + for (const [name, type] of bindingAccumulator.fileScopeEntries(filePath)) { + // Three-candidate-ID lookup mirrors the sequential-path export check + // in `collectExportedBindings()` (call-processor.ts). + const functionNodeId = `Function:${filePath}:${name}`; + const variableNodeId = `Variable:${filePath}:${name}`; + const constNodeId = `Const:${filePath}:${name}`; + const node = + graph.getNode(functionNodeId) ?? + graph.getNode(variableNodeId) ?? + graph.getNode(constNodeId); + if (!node?.properties?.isExported) continue; + + let fileExports = exportedTypeMap.get(filePath); + if (!fileExports) { + fileExports = new Map(); + exportedTypeMap.set(filePath, fileExports); + } + // Tier 0 priority: SymbolTable-populated entries are authoritative. + if (!fileExports.has(name)) { + fileExports.set(name, type); + enriched++; + } + } + } + return enriched; +} + +const ENTRY_OVERHEAD = 64; // bytes per entry (object overhead + property refs) +const MAP_ENTRY_OVERHEAD = 80; // bytes per file entry in the map + +export class BindingAccumulator { + // Storage is split into two parallel maps so file-scope reads are fast. + // - _allByFile holds every BindingEntry (used by getFile, memory estimate). + // - _fileScopeByFile is a nested Map> for + // O(1) point-lookup via fileScopeGet(). For iteration-based consumers + // (enrichExportedTypeMap), fileScopeEntries() iterates the inner Map. + // Both maps carry the same key set modulo the `scope === ''` precondition: + // _allByFile has a key as soon as any entry is appended; _fileScopeByFile + // only has a key once a file-scope entry arrives. Code that iterates via + // files() uses _allByFile so files with only function-scope entries + // remain visible. + // + // Note: Map.set semantics mean a duplicate varName for the same file + // overwrites the previous value (last-write-wins). This is the correct + // behavior — duplicate top-level bindings in the same file shouldn't + // happen in well-formed source, and if they do the last declaration + // is typically the one the compiler sees. + private readonly _allByFile = new Map(); + private readonly _fileScopeByFile = new Map>(); + private _totalBindings = 0; + private _finalized = false; + private _disposed = false; + + /** + * Append bindings for a file. Safe to call multiple times for the same file. + * Throws if the accumulator has been finalized. Skips if entries is empty. + * + * The `entries` parameter is `readonly` — this method never mutates the + * caller's array. Internally, the first `appendFile` call per filePath + * makes a defensive copy (`slice()`), and subsequent calls push into the + * accumulator's own storage. + */ + appendFile(filePath: string, entries: readonly BindingEntry[]): void { + if (this._finalized) { + throw new Error( + '[BindingAccumulator] appendFile after finalize — no further appends allowed', + ); + } + // Single-use lifecycle: once disposed, the accumulator is dead. A + // post-dispose append almost always indicates a missed wiring step + // (the consumer is reading state that was supposed to be released), + // so convert the silent use-after-dispose into a loud failure. + if (this._disposed) { + throw new Error('BindingAccumulator: use after dispose'); + } + if (entries.length === 0) { + return; + } + // Note on the file-scope-only invariant: + // The accumulator does NOT reject function-scope entries at this + // boundary. The narrowing contract is enforced by the two production + // write sites — `parse-worker.ts` (which uses `typeEnv.fileScope()` + // and hardcodes `scope: ''` in the pipeline adapter) and + // `type-env.ts::flush()` (which iterates only `env.get(FILE_SCOPE)`). + // The class JSDoc documents the invariant and the Phase 9 reversion + // path. Making `appendFile` runtime-reject non-file-scope entries + // would break the accumulator's own storage-split tests which + // legitimately exercise mixed-scope entries. If a future write path + // violates the invariant, tests should fail via missing exports in + // the enrichment loop, not via an assertion here. + // All-scope store. + const existingAll = this._allByFile.get(filePath); + if (existingAll !== undefined) { + for (const e of entries) { + existingAll.push(e); + } + } else { + this._allByFile.set(filePath, entries.slice()); + } + // File-scope fast-path store (nested Map for O(1) point-lookup via fileScopeGet). + // Populated lazily on first file-scope entry per file. + let fileScopeMap = this._fileScopeByFile.get(filePath); + for (const e of entries) { + if (e.scope === '') { + if (fileScopeMap === undefined) { + fileScopeMap = new Map(); + this._fileScopeByFile.set(filePath, fileScopeMap); + } + fileScopeMap.set(e.varName, e.typeName); + } + } + this._totalBindings += entries.length; + } + + /** Lock the accumulator — no further appends. Idempotent. */ + finalize(): void { + // Dev-mode invariant: verify the parallel storage split is consistent. + // `_fileScopeByFile` must be a proper projection of `_allByFile` + // where the outer key is a subset and the inner entries are exactly + // the `scope === ''` subset of `_allByFile[key]`. A drift would + // indicate a bug in `appendFile()` where one map was updated but + // not the other. + if (process.env.NODE_ENV !== 'production' && !this._finalized) { + for (const [filePath, fileScopeMap] of this._fileScopeByFile) { + const allEntries = this._allByFile.get(filePath); + if (allEntries === undefined) { + throw new Error( + `[BindingAccumulator] storage split drift: file ${filePath} has file-scope entries ` + + `but no _allByFile entry`, + ); + } + // Count unique file-scope varNames in _allByFile (to match Map dedup + // semantics in _fileScopeByFile where Map.set deduplicates same-name). + const projectedNames = new Set( + allEntries.filter((e) => e.scope === '').map((e) => e.varName), + ); + if (projectedNames.size !== fileScopeMap.size) { + throw new Error( + `[BindingAccumulator] storage split drift: file ${filePath} has ` + + `${fileScopeMap.size} file-scope names in Map but ${projectedNames.size} unique ` + + `file-scope varNames in _allByFile`, + ); + } + } + } + this._finalized = true; + } + + /** + * Release the accumulator's heap footprint. Clears both internal storage + * maps and resets `_totalBindings` to zero. Idempotent — calling twice + * is a no-op. Orthogonal to `finalize()` — calling `dispose()` does not + * change the finalized state. + * + * **Single-use lifecycle.** This is a one-way terminal transition: the + * accumulator is not recyclable. Any subsequent `appendFile` call throws + * (`'BindingAccumulator: use after dispose'`), regardless of whether + * `finalize()` was called first. Post-dispose reads do not throw — + * they return empty/undefined state matching a never-appended-to + * accumulator: + * - `fileCount === 0` + * - `totalBindings === 0` + * - `files()` yields an empty iterator + * - `getFile(x)` returns `undefined` for all `x` + * - `fileScopeEntries(x)` returns `[]` for all `x` + * - `fileScopeGet(x, y)` returns `undefined` for all `x, y` + * - `estimateMemoryBytes()` returns `0` + * + * Lifecycle note: the pipeline disposes the accumulator inside the + * `finally` of the `crossFile` phase, which is scheduled after every + * other accumulator consumer (Phase 9 call/assignment processing and + * the ExportedTypeMap enrichment loop). The dispose call therefore + * runs once, on both the happy path and the throw path of the + * crossFile phase. + */ + dispose(): void { + this._allByFile.clear(); + this._fileScopeByFile.clear(); + this._totalBindings = 0; + this._disposed = true; + } + + /** Get all bindings for a file, or undefined if the file is unknown. */ + getFile(filePath: string): readonly BindingEntry[] | undefined { + return this._allByFile.get(filePath); + } + + /** + * Get only scope='' (file-level) entries as [varName, typeName] tuples. + * For iteration-based consumers (e.g., `enrichExportedTypeMap`). + * Returns an empty array for an unknown file. + * + * O(1) map lookup + O(n_file_scope) tuple reconstruction from the inner + * Map. Does NOT walk function-scope entries. + * + * For point-lookup consumers (e.g., Phase 9 fallback), prefer + * `fileScopeGet(filePath, name)` — O(1) with no allocation. + */ + fileScopeEntries(filePath: string): readonly (readonly [string, string])[] { + const map = this._fileScopeByFile.get(filePath); + return map ? [...map.entries()] : []; + } + + /** + * O(1) point-lookup for a single file-scope binding by (filePath, name). + * Returns the typeName if found, `undefined` otherwise. + * + * This is the preferred lookup path for Phase 9 consumers that resolve + * a single callee's return type — avoids the O(n_file_scope) iteration + * and defensive-copy allocation of `fileScopeEntries()`. + */ + fileScopeGet(filePath: string, name: string): string | undefined { + return this._fileScopeByFile.get(filePath)?.get(name); + } + + /** Iterate over all file paths in insertion order. */ + files(): IterableIterator { + return this._allByFile.keys(); + } + + /** Number of distinct files with at least one binding. */ + get fileCount(): number { + return this._allByFile.size; + } + + /** Total number of binding entries across all files. */ + get totalBindings(): number { + return this._totalBindings; + } + + /** Whether the accumulator has been finalized. */ + get finalized(): boolean { + return this._finalized; + } + + /** + * Whether the accumulator has been disposed. Exposed for symmetry with + * `finalized` so debug tooling and future Phase 9 consumers can detect a + * disposed accumulator without inspecting empty state heuristically. + * + * Disposal and finalization are orthogonal: a disposed accumulator may or + * may not be finalized, and vice versa. See `dispose()` for the full + * lifecycle contract. + */ + get disposed(): boolean { + return this._disposed; + } + + /** + * Rough memory estimate in bytes (intentionally pessimistic). + * Formula: sum of (ENTRY_OVERHEAD + char bytes of scope+varName+typeName) per entry + * + MAP_ENTRY_OVERHEAD + char bytes of filePath per file. + * + * Note: V8 stores all-ASCII strings as Latin-1 (1 byte/char) and only upgrades + * to UCS-2 (2 bytes/char) for non-Latin-1 code points. Source paths and type names + * are typically all-ASCII, so actual heap cost is roughly half what this returns. + * The pessimistic factor is intentional — better to over-budget than under-budget. + * + * **⚠ Cost profile**: O(totalBindings) — iterates every entry in + * `_allByFile` and reads three string `.length` properties per entry. + * At a typical repo scale (10k files × ~20 file-scope bindings) this is + * ~200k property reads per call. Call at most once per pipeline run, + * NOT per file, per chunk, or per progress tick. The current single + * call site is the dev-mode telemetry log at the pipeline finalize + * seam. Adding a per-file-progress caller would silently make it + * quadratic in repo size. + */ + estimateMemoryBytes(): number { + let total = 0; + for (const [filePath, entries] of this._allByFile) { + total += MAP_ENTRY_OVERHEAD + filePath.length * 2; + for (const e of entries) { + total += ENTRY_OVERHEAD + (e.scope.length + e.varName.length + e.typeName.length) * 2; + } + } + return total; + } +} diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts b/gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts new file mode 100644 index 000000000..02a6ed60f --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts @@ -0,0 +1,12 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const cCallConfig: CallExtractionConfig = { + language: SupportedLanguages.C, +}; + +export const cppCallConfig: CallExtractionConfig = { + language: SupportedLanguages.CPlusPlus, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts new file mode 100644 index 000000000..e2c0415c2 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts @@ -0,0 +1,9 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const csharpCallConfig: CallExtractionConfig = { + language: SupportedLanguages.CSharp, + typeAsReceiverHeuristic: true, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/call-extractors/configs/dart.ts new file mode 100644 index 000000000..9d3c08def --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/dart.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/dart.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const dartCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Dart, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/go.ts b/gitnexus/src/core/ingestion/call-extractors/configs/go.ts new file mode 100644 index 000000000..870fc88e0 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/go.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/go.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const goCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Go, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts new file mode 100644 index 000000000..51de04228 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts @@ -0,0 +1,59 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig, ExtractedCallSite } from '../../call-types.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; + +// --------------------------------------------------------------------------- +// Java method_reference (::) parsing — absorbs call-sites/java.ts +// --------------------------------------------------------------------------- + +/** + * Parse Java `method_reference` nodes (`expr::method`, `Type::new`, + * `this::m`, `super::m`). + */ +function parseJavaMethodReference(callNode: SyntaxNode): ExtractedCallSite | null { + if (callNode.type !== 'method_reference') return null; + + const recv = callNode.namedChild(0); + if (!recv) return null; + + // Type::new → constructor call + for (const c of callNode.children) { + if (c.type === 'new') { + if (recv.type !== 'identifier') return null; + return { calledName: recv.text, callForm: 'constructor' }; + } + } + + // expr::method → member call with receiver + const rhs = callNode.child(callNode.childCount - 1); + if (!rhs || rhs.type !== 'identifier') return null; + const methodName = rhs.text; + + if (recv.type === 'identifier') { + return { calledName: methodName, callForm: 'member', receiverName: recv.text }; + } + if (recv.type === 'this') { + return { calledName: methodName, callForm: 'member', receiverName: 'this' }; + } + if (recv.type === 'super') { + return { calledName: methodName, callForm: 'member', receiverName: 'super' }; + } + return null; +} + +// --------------------------------------------------------------------------- +// Configs +// --------------------------------------------------------------------------- + +export const javaCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Java, + extractLanguageCallSite: parseJavaMethodReference, + typeAsReceiverHeuristic: true, +}; + +export const kotlinCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Kotlin, + typeAsReceiverHeuristic: true, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/php.ts b/gitnexus/src/core/ingestion/call-extractors/configs/php.ts new file mode 100644 index 000000000..25ed0b9ab --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/php.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/php.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const phpCallConfig: CallExtractionConfig = { + language: SupportedLanguages.PHP, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/python.ts b/gitnexus/src/core/ingestion/call-extractors/configs/python.ts new file mode 100644 index 000000000..35ab87305 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/python.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/python.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const pythonCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Python, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts b/gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts new file mode 100644 index 000000000..d829c5c89 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const rubyCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Ruby, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/rust.ts b/gitnexus/src/core/ingestion/call-extractors/configs/rust.ts new file mode 100644 index 000000000..03c48f781 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/rust.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/rust.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const rustCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Rust, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/call-extractors/configs/swift.ts new file mode 100644 index 000000000..28f2c180c --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/swift.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/swift.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const swiftCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Swift, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts new file mode 100644 index 000000000..20a63cda9 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts @@ -0,0 +1,12 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const typescriptCallConfig: CallExtractionConfig = { + language: SupportedLanguages.TypeScript, +}; + +export const javascriptCallConfig: CallExtractionConfig = { + language: SupportedLanguages.JavaScript, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/generic.ts b/gitnexus/src/core/ingestion/call-extractors/generic.ts new file mode 100644 index 000000000..82e38457b --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/generic.ts @@ -0,0 +1,86 @@ +// gitnexus/src/core/ingestion/call-extractors/generic.ts + +/** + * Generic table-driven call extractor factory. + * + * Mirrors method-extractors/generic.ts and field-extractors/generic.ts — + * define a config per language and generate extractors from configs. + * + * The factory converts a declarative {@link CallExtractionConfig} into a + * runtime {@link CallExtractor} whose `extract()` method: + * 1. Tries `config.extractLanguageCallSite(callNode)` for non-standard shapes. + * 2. Falls through to the generic path using shared utilities from + * `utils/call-analysis.ts` (`inferCallForm`, `extractReceiverName`, etc.). + */ + +import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { + inferCallForm, + extractReceiverName, + extractReceiverNode, + extractMixedChain, + countCallArguments, +} from '../utils/call-analysis.js'; +import type { CallExtractor, CallExtractionConfig, ExtractedCallSite } from '../call-types.js'; + +/** + * Create a CallExtractor from a declarative config. + */ +export function createCallExtractor(config: CallExtractionConfig): CallExtractor { + return { + language: config.language, + + extract(callNode: SyntaxNode, callNameNode: SyntaxNode | undefined): ExtractedCallSite | null { + // ── Path 1: Language-specific call site ────────────────────────── + // Non-standard call shapes (e.g. Java `::` method references) are + // handled entirely by the config hook. When it returns a result, + // the generic path is skipped — no argCount, no mixed chain. + // + // Note: `extractLanguageCallSite` is called on every `extract()` + // invocation — both `extract(callNode, undefined)` (parse-worker + // Path 1) and `extract(callNode, callNameNode)` (Path 2). + // Language hooks must therefore be idempotent and cheap (e.g. a + // single node-type check). + if (config.extractLanguageCallSite) { + const seed = config.extractLanguageCallSite(callNode); + if (seed) { + return { + ...seed, + ...(config.typeAsReceiverHeuristic ? { typeAsReceiverHeuristic: true } : {}), + }; + } + } + + // ── Path 2: Generic extraction via @call.name ──────────────────── + if (!callNameNode) return null; + + const calledName = callNameNode.text; + const callForm = inferCallForm(callNode, callNameNode); + let receiverName = callForm === 'member' ? extractReceiverName(callNameNode) : undefined; + let receiverMixedChain: ExtractedCallSite['receiverMixedChain']; + + // When the receiver is a complex expression (call chain, field chain, + // or mixed), extractReceiverName returns undefined. Walk the receiver + // node to build a unified mixed chain for deferred resolution. + if (callForm === 'member' && receiverName === undefined) { + const receiverNode = extractReceiverNode(callNameNode); + if (receiverNode) { + const extracted = extractMixedChain(receiverNode); + if (extracted && extracted.chain.length > 0) { + receiverMixedChain = extracted.chain; + receiverName = extracted.baseReceiverName; + } + } + } + + return { + calledName, + ...(callForm !== undefined ? { callForm } : {}), + ...(receiverName !== undefined ? { receiverName } : {}), + argCount: countCallArguments(callNode), + ...(receiverMixedChain !== undefined ? { receiverMixedChain } : {}), + ...(config.typeAsReceiverHeuristic ? { typeAsReceiverHeuristic: true } : {}), + }; + }, + }; +} diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 2a47d5f39..c30d83848 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -1,11 +1,42 @@ import { KnowledgeGraph } from '../graph/types.js'; import { ASTCache } from './ast-cache.js'; -import type { SymbolDefinition, SymbolTable } from './symbol-table.js'; -import { CLASS_TYPES } from './symbol-table.js'; +import type { + SymbolDefinition, + SymbolTableReader, + HeritageMap, + ExtractedHeritage, +} from './model/index.js'; +import { CLASS_TYPES, CALL_TARGET_TYPES, lookupMethodByOwnerWithMRO } from './model/index.js'; +import type { DispatchDecision, ReceiverEnriched } from './call-types.js'; + +/** Shorthand for the receiver-source discriminant shared across the DAG. */ +type ReceiverSource = ReceiverEnriched['receiverSource']; + +/** + * DAG stage 4 fallback: used when `selectDispatch` is absent or returns null. + * Preserves pre-DAG dispatch semantics: + * - 'constructor' → constructor branch + * - 'free' → free branch (admits Swift/Kotlin class-target fast path) + * - 'member' or undefined → owner-scoped branch + * + * `undefined` callForm MUST route through owner-scoped (not free) so bare + * identifiers without a classified shape do NOT trigger `resolveFreeCall`'s + * class-target fast path. Without a `receiverTypeName`, the owner-scoped + * branch falls through to `resolveModuleAliasedCall` + `singleCandidate`, + * matching legacy behavior where non-callable symbols (Class, Interface) + * null-route instead of producing spurious Constructor edges. + */ +const defaultDispatchDecision = ( + callForm: 'free' | 'member' | 'constructor' | undefined, +): DispatchDecision => { + if (callForm === 'constructor') return { primary: 'constructor' }; + if (callForm === 'free') return { primary: 'free' }; + return { primary: 'owner-scoped' }; +}; import Parser from 'tree-sitter'; -import type { ResolutionContext } from './resolution-context.js'; -import { TIER_CONFIDENCE, type ResolutionTier } from './resolution-context.js'; -import type { TieredCandidates } from './resolution-context.js'; +import type { ResolutionContext } from './model/resolution-context.js'; +import { TIER_CONFIDENCE, type ResolutionTier } from './model/resolution-context.js'; +import type { TieredCandidates } from './model/resolution-context.js'; import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { getProvider } from './languages/index.js'; import { generateId } from '../../lib/utils.js'; @@ -32,13 +63,11 @@ import { } from './utils/call-analysis.js'; import { buildTypeEnv, isSubclassOf } from './type-env.js'; import type { ConstructorBinding, TypeEnvironment } from './type-env.js'; -import type { HeritageMap } from './heritage-map.js'; -import { c3Linearize } from './mro-processor.js'; +import type { BindingAccumulator } from './binding-accumulator.js'; import { getTreeSitterBufferSize } from './constants.js'; import type { ExtractedCall, ExtractedAssignment, - ExtractedHeritage, ExtractedRoute, ExtractedFetchCall, FileConstructorBindings, @@ -48,7 +77,6 @@ import { extractTemplateComponents } from './vue-sfc-extractor.js'; import { extractReturnTypeName, stripNullable } from './type-extractors/shared.js'; import type { LiteralTypeInferrer } from './type-extractors/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; -import { extractParsedCallSite } from './call-sites/extract-language-call-site.js'; /** Per-file resolved type bindings for exported symbols. * Populated during call processing, consumed by Phase 14 re-resolution pass. */ @@ -104,7 +132,13 @@ const MAX_EXPORTS_PER_FILE = 500; const MAX_TYPE_NAME_LENGTH = 256; /** Build a map of imported callee names → return types for cross-file call-result binding. - * Consulted ONLY when SymbolTable has no unambiguous local match (local-first principle). */ + * Consulted ONLY when SymbolTable has no unambiguous local match (local-first principle). + * + * Overlapping mechanism (1 of 3): this is the SymbolTable-backed path. + * See also: + * 2. collectExportedBindings (~line 168) / enrichExportedTypeMap — TypeEnv + graph isExported + * 3. Phase 9 fallback in verifyConstructorBindings (~line 563) — namedImportMap + BindingAccumulator + * A future cleanup should merge these into a single resolution pass. */ export function buildImportedReturnTypes( filePath: string, namedImportMap: ReadonlyMap< @@ -155,7 +189,20 @@ export function buildImportedRawReturnTypes( } /** Collect resolved type bindings for exported file-scope symbols. - * Uses graph node isExported flag — does NOT require isExported on SymbolDefinition. */ + * Uses graph node isExported flag — does NOT require isExported on SymbolDefinition. + * + * **Counterpart**: the worker path populates `exportedTypeMap` via the + * accumulator enrichment loop in `pipeline.ts` (search for "Worker path + * quality enrichment"). Both sites populate the same map with subtly + * different export-check semantics — this site uses SymbolTable + + * graph lookup, the worker loop uses three-candidate-ID graph lookup. + * They must stay in sync until unified. If you edit one, check the other. + * + * Overlapping mechanism (2 of 3): this is the TypeEnv + graph isExported path. + * See also: + * 1. buildImportedReturnTypes (~line 109) — namedImportMap + SymbolTable + * 3. Phase 9 fallback in verifyConstructorBindings (~line 563) — namedImportMap + BindingAccumulator + * A future cleanup should merge these into a single resolution pass. */ function collectExportedBindings( typeEnv: { fileScope(): ReadonlyMap }, filePath: string, @@ -184,7 +231,7 @@ function collectExportedBindings( * exported symbols that have callables with known return types. */ export function buildExportedTypeMapFromGraph( graph: KnowledgeGraph, - symbolTable: SymbolTable, + symbolTable: SymbolTableReader, ): ExportedTypeMap { const result: ExportedTypeMap = new Map(); graph.forEachNode((node) => { @@ -503,6 +550,7 @@ const verifyConstructorBindings = ( filePath: string, ctx: ResolutionContext, graph?: KnowledgeGraph, + bindingAccumulator?: BindingAccumulator, ): Map => { const verified = new Map(); @@ -539,12 +587,60 @@ const verifyConstructorBindings = ( } } + let typeName: string | undefined; if (callableDefs && callableDefs.length === 1 && callableDefs[0].returnType) { - const typeName = extractReturnTypeName(callableDefs[0].returnType); - if (typeName) { - verified.set(receiverKey(scope, varName), typeName); + typeName = extractReturnTypeName(callableDefs[0].returnType); + } + + // Phase 9: BindingAccumulator fallback for cross-file return types. + // Used when the SymbolTable has no return type for a cross-file callee + // (e.g., a return type that TypeEnv resolved via fixpoint in the source + // file but was not stored as a SymbolTable returnType annotation). + // namedImportMap tells us which source file exported the callee so we + // can look up its file-scope binding via the O(1) fileScopeGet method. + // + // Tier gating: only fall back to the accumulator when resolution is + // unambiguously import-scoped or global. When tiered.tier is 'same-file', + // the local definition is authoritative even without a return type + // annotation — using the accumulator here would let an imported callee + // with the same name shadow the local one, producing false CALLS edges. + // When multiple callable candidates exist, the accumulator would pick + // arbitrarily — skip to avoid fabricated edges. + // + // Quality note: worker-path accumulator entries are Tier 0/1 only + // (annotation-declared + same-file constructor inference) — see the + // BindingAccumulator class JSDoc. For large repos where the worker + // path dominates, Phase 9 binding accuracy is structurally lower + // than for sequential-path repos where Tier 2 cross-file propagation + // is available. + // + // Overlapping mechanism note: this is one of three cross-file + // return-type resolution paths in the codebase: + // 1. buildImportedReturnTypes (~line 109) — namedImportMap + + // SymbolTable.lookupExactFull (structure-processor captured) + // 2. collectExportedBindings (~line 168) / enrichExportedTypeMap + // — TypeEnv + graph isExported flag + // 3. This fallback — namedImportMap + BindingAccumulator + // A future cleanup should merge these into a single resolution pass. + const shouldFallback = + tiered?.tier !== 'same-file' && (!callableDefs || callableDefs.length <= 1); + if (!typeName && bindingAccumulator && shouldFallback) { + const namedImports = ctx.namedImportMap.get(filePath); + const importBinding = namedImports?.get(calleeName); + if (importBinding) { + const rawType = bindingAccumulator.fileScopeGet( + importBinding.sourcePath, + importBinding.exportedName, + ); + if (rawType) { + typeName = extractReturnTypeName(rawType); + } } } + + if (typeName) { + verified.set(receiverKey(scope, varName), typeName); + } } } @@ -583,7 +679,7 @@ function findInterfaceDispatchTargets( const results: ResolveResult[] = []; for (const implFile of implFiles) { - const methods = ctx.symbols.lookupExactAll(implFile, calledName); + const methods = ctx.model.symbols.lookupExactAll(implFile, calledName); for (const method of methods) { if (method.nodeId !== primaryNodeId) { results.push({ @@ -612,6 +708,7 @@ export const processCalls = async ( /** Phase 14 E3: cross-file RAW return types for for-loop element extraction. Keyed by filePath → Map. */ importedRawReturnTypesMap?: ReadonlyMap>, heritageMap?: HeritageMap, + bindingAccumulator?: BindingAccumulator, ): Promise => { const parser = await loadParser(); const collectedHeritage: ExtractedHeritage[] = []; @@ -630,10 +727,29 @@ export const processCalls = async ( const logSkipped = isVerboseIngestionEnabled(); const skippedByLang = logSkipped ? new Map() : null; + // ── Prepare-then-resolve: single preparation loop, deferred resolution ── + // All files are prepared (parse → query → heritage → TypeEnv) in one loop, + // then resolved (verifyConstructorBindings → call edges) in a second loop. + // This ensures: + // 1. When bindingAccumulator is present, ALL files flush their TypeEnv + // bindings before ANY verifyConstructorBindings reads — fixing the + // consumer-before-provider ordering bug on the sequential path. + // 2. globalParentMap is fully populated before resolution, improving + // cross-file isSubclassOf accuracy regardless of file order. + // For the sequential path (<15 files), buffering per-file state is negligible. + interface PreparedFile { + file: { path: string; content: string }; + language: SupportedLanguages; + provider: ReturnType; + tree: ReturnType; + matches: ReturnType; + parentMap: ReadonlyMap; + typeEnv: ReturnType; + } + const prepared: PreparedFile[] = []; + for (let i = 0; i < files.length; i++) { const file = files[i]; - enclosingFnExtractCache.clear(); - onProgress?.(i + 1, files.length); if (i % 20 === 0) await yieldToEventLoop(); const language = getLanguageFromFilename(file.path); @@ -663,41 +779,43 @@ export const processCalls = async ( astCache.set(file.path, tree); } - let query; let matches; try { - const language = parser.getLanguage(); - query = new Parser.Query(language, queryStr); + const lang = parser.getLanguage(); + const query = new Parser.Query(lang, queryStr); matches = query.matches(tree.rootNode); } catch (queryError) { console.warn(`Query error for ${file.path}:`, queryError); continue; } - // Pre-pass: extract heritage from query matches to build parentMap for buildTypeEnv. + // Extract heritage from query matches to build parentMap for buildTypeEnv. // Heritage-processor runs in PARALLEL, so graph edges don't exist when buildTypeEnv runs. const fileParentMap = new Map(); - for (const match of matches) { - const captureMap: Record = {}; - match.captures.forEach((c) => (captureMap[c.name] = c.node)); - if (captureMap['heritage.class'] && captureMap['heritage.extends']) { - const className: string = captureMap['heritage.class'].text; - const parentName: string = captureMap['heritage.extends'].text; - const extendsNode = captureMap['heritage.extends']; - const fieldDecl = extendsNode.parent; - if (fieldDecl?.type === 'field_declaration' && fieldDecl.childForFieldName('name')) - continue; - let parents = fileParentMap.get(className); - if (!parents) { - parents = []; - fileParentMap.set(className, parents); + if (provider.heritageExtractor) { + for (const match of matches) { + const captureMap: Record = {}; + match.captures.forEach((c) => (captureMap[c.name] = c.node)); + if (captureMap['heritage.class']) { + const heritageItems = provider.heritageExtractor.extract(captureMap, { + filePath: file.path, + language, + }); + for (const item of heritageItems) { + if (item.kind === 'extends') { + let parents = fileParentMap.get(item.className); + if (!parents) { + parents = []; + fileParentMap.set(item.className, parents); + } + if (!parents.includes(item.parentName)) parents.push(item.parentName); + } + } } - if (!parents.includes(parentName)) parents.push(parentName); } } const parentMap: ReadonlyMap = fileParentMap; // Merge per-file heritage into globalParentMap for cross-file isSubclassOf lookups. - // Uses a parallel Set (globalParentSeen) for O(1) deduplication instead of O(n) includes(). for (const [cls, parents] of fileParentMap) { let global = globalParentMap.get(cls); let seen = globalParentSeen.get(cls); @@ -721,7 +839,7 @@ export const processCalls = async ( const importedReturnTypes = importedReturnTypesMap?.get(file.path); const importedRawReturnTypes = importedRawReturnTypesMap?.get(file.path); const typeEnv = buildTypeEnv(tree, language, { - symbolTable: ctx.symbols, + model: ctx.model, parentMap, importedBindings, importedReturnTypes, @@ -730,14 +848,38 @@ export const processCalls = async ( extractFunctionName: provider?.methodExtractor?.extractFunctionName, }); if (typeEnv && exportedTypeMap) { - const fileExports = collectExportedBindings(typeEnv, file.path, ctx.symbols, graph); + const fileExports = collectExportedBindings(typeEnv, file.path, ctx.model.symbols, graph); if (fileExports) exportedTypeMap.set(file.path, fileExports); } + if (bindingAccumulator) { + typeEnv.flush(file.path, bindingAccumulator); + } + + prepared.push({ file, language, provider, tree, matches, parentMap, typeEnv }); + } + + // ── Resolution loop: verify constructor bindings and resolve calls ── + // The accumulator (if present) is now fully populated from the preparation + // loop above, so verifyConstructorBindings sees all provider bindings + // regardless of file processing order. + for (let i = 0; i < prepared.length; i++) { + const { file, language, provider, tree, matches, parentMap, typeEnv } = prepared[i]; + + enclosingFnExtractCache.clear(); + onProgress?.(i + 1, files.length); + if (i % 20 === 0) await yieldToEventLoop(); + const callRouter = provider.callRouter; const verifiedReceivers = typeEnv.constructorBindings.length > 0 - ? verifyConstructorBindings(typeEnv.constructorBindings, file.path, ctx) + ? verifyConstructorBindings( + typeEnv.constructorBindings, + file.path, + ctx, + undefined, // graph not available on the sequential path here + bindingAccumulator, // Phase 9 fallback — same as worker path (R3 parity) + ) : new Map(); const receiverIndex = buildReceiverTypeIndex(verifiedReceivers); @@ -799,74 +941,79 @@ export const processCalls = async ( if (!captureMap['call']) return; const callNode = captureMap['call']; - const languageSeed = extractParsedCallSite(language, callNode); - if (languageSeed) { - if (provider.isBuiltInName(languageSeed.calledName)) return; + const callExtractor = provider.callExtractor; - const sourceId = - findEnclosingFunction(callNode, file.path, ctx, provider) || - generateId('File', file.path); - const receiverName = - languageSeed.callForm === 'member' ? languageSeed.receiverName : undefined; - let receiverTypeName = - receiverName && typeEnv ? typeEnv.lookup(receiverName, callNode) : undefined; + // ── Language-specific call site (e.g. Java :: method references) ── + if (callExtractor) { + const langCallSite = callExtractor.extract(callNode, undefined); + if (langCallSite) { + if (provider.isBuiltInName(langCallSite.calledName)) return; - if ( - receiverName !== undefined && - receiverTypeName === undefined && - languageSeed.callForm === 'member' && - (language === 'java' || language === 'csharp' || language === 'kotlin') - ) { - const c0 = receiverName.charCodeAt(0); - if (c0 >= 65 && c0 <= 90) receiverTypeName = receiverName; - } + const sourceId = + findEnclosingFunction(callNode, file.path, ctx, provider) || + generateId('File', file.path); + const receiverName = + langCallSite.callForm === 'member' ? langCallSite.receiverName : undefined; + let receiverTypeName = + receiverName && typeEnv ? typeEnv.lookup(receiverName, callNode) : undefined; - const resolved = resolveCallTarget( - { - calledName: languageSeed.calledName, - callForm: languageSeed.callForm, - ...(receiverTypeName !== undefined ? { receiverTypeName } : {}), - ...(receiverName !== undefined ? { receiverName } : {}), - }, - file.path, - ctx, - undefined, - widenCache, - undefined, - heritageMap, - ); + if ( + langCallSite.typeAsReceiverHeuristic && + receiverName !== undefined && + receiverTypeName === undefined && + langCallSite.callForm === 'member' + ) { + const c0 = receiverName.charCodeAt(0); + if (c0 >= 65 && c0 <= 90) receiverTypeName = receiverName; + } - if (!resolved) return; - graph.addRelationship({ - id: generateId('CALLS', `${sourceId}:${languageSeed.calledName}->${resolved.nodeId}`), - sourceId, - targetId: resolved.nodeId, - type: 'CALLS', - confidence: resolved.confidence, - reason: resolved.reason, - }); - - if (heritageMap && languageSeed.callForm === 'member' && receiverTypeName) { - const implTargets = findInterfaceDispatchTargets( - languageSeed.calledName, - receiverTypeName, + const resolved = resolveCallTarget( + { + calledName: langCallSite.calledName, + callForm: langCallSite.callForm, + ...(receiverTypeName !== undefined ? { receiverTypeName } : {}), + ...(receiverName !== undefined ? { receiverName } : {}), + }, file.path, ctx, + undefined, + widenCache, + undefined, heritageMap, - resolved.nodeId, ); - for (const impl of implTargets) { - graph.addRelationship({ - id: generateId('CALLS', `${sourceId}:${languageSeed.calledName}->${impl.nodeId}`), - sourceId, - targetId: impl.nodeId, - type: 'CALLS', - confidence: impl.confidence, - reason: impl.reason, - }); + + if (!resolved) return; + graph.addRelationship({ + id: generateId('CALLS', `${sourceId}:${langCallSite.calledName}->${resolved.nodeId}`), + sourceId, + targetId: resolved.nodeId, + type: 'CALLS', + confidence: resolved.confidence, + reason: resolved.reason, + }); + + if (heritageMap && langCallSite.callForm === 'member' && receiverTypeName) { + const implTargets = findInterfaceDispatchTargets( + langCallSite.calledName, + receiverTypeName, + file.path, + ctx, + heritageMap, + resolved.nodeId, + ); + for (const impl of implTargets) { + graph.addRelationship({ + id: generateId('CALLS', `${sourceId}:${langCallSite.calledName}->${impl.nodeId}`), + sourceId, + targetId: impl.nodeId, + type: 'CALLS', + confidence: impl.confidence, + reason: impl.reason, + }); + } } + return; } - return; } const nameNode = captureMap['call.name']; @@ -874,6 +1021,28 @@ export const processCalls = async ( const calledName = nameNode.text; + // Check heritage extractor for call-based heritage (e.g., Ruby include/extend/prepend) + if (provider.heritageExtractor?.extractFromCall) { + const heritageItems = provider.heritageExtractor.extractFromCall( + calledName, + captureMap['call'], + { filePath: file.path, language }, + ); + if (heritageItems !== null) { + for (const item of heritageItems) { + collectedHeritage.push({ + filePath: file.path, + className: item.className, + parentName: item.parentName, + kind: item.kind, + }); + } + return; + } + } + + // Dispatch: route language-specific calls (properties, imports) + // Heritage routing is handled by heritageExtractor.extractFromCall above. const routed = callRouter?.(calledName, captureMap['call']); if (routed) { switch (routed.kind) { @@ -881,17 +1050,6 @@ export const processCalls = async ( case 'import': return; - case 'heritage': - for (const item of routed.items) { - collectedHeritage.push({ - filePath: file.path, - className: item.enclosingClass, - parentName: item.mixinName, - kind: item.heritageKind, - }); - } - return; - case 'properties': { const fileId = generateId('File', file.path); const propEnclosingClassId = findEnclosingClassId(captureMap['call'], file.path); @@ -910,7 +1068,7 @@ export const processCalls = async ( description: item.accessorType, }, }); - ctx.symbols.add(file.path, item.propName, nodeId, 'Property', { + ctx.model.symbols.add(file.path, item.propName, nodeId, 'Property', { ...(propEnclosingClassId ? { ownerId: propEnclosingClassId } : {}), ...(item.declaredType ? { declaredType: item.declaredType } : {}), }); @@ -944,10 +1102,17 @@ export const processCalls = async ( if (provider.isBuiltInName(calledName)) return; - const callForm = inferCallForm(callNode, nameNode); - const receiverName = callForm === 'member' ? extractReceiverName(nameNode) : undefined; + // --- DAG stage 2-3: classify-form + infer-receiver (shared defaults) --- + // These stages run the shared inference chain. Language providers can + // customize infer-receiver (stage 3) via the inferImplicitReceiver hook + // which runs AFTER this default chain (typed-binding → constructor-map → + // module-alias → class-as-receiver → mixed-chain), and selectDispatch + // (stage 4) which picks the resolver branch. + let callForm = inferCallForm(callNode, nameNode); + let receiverName = callForm === 'member' ? extractReceiverName(nameNode) : undefined; let receiverTypeName = receiverName && typeEnv ? typeEnv.lookup(receiverName, callNode) : undefined; + let receiverSource: ReceiverSource = receiverTypeName ? 'typed-binding' : 'none'; // Phase P: virtual dispatch override — when the declared type is a base class but // the constructor created a known subclass, prefer the more specific type. // Checks per-file parentMap first, then falls back to globalParentMap for @@ -982,10 +1147,11 @@ export const processCalls = async ( if ( isSubclassOf(ctorType, receiverTypeName, parentMap) || isSubclassOf(ctorType, receiverTypeName, globalParentMap) || - (ctx.symbols.lookupClassByName(ctorType).length > 0 && - ctx.symbols.lookupClassByName(receiverTypeName).length > 0) + (ctx.model.types.lookupClassByName(ctorType).length > 0 && + ctx.model.types.lookupClassByName(receiverTypeName).length > 0) ) { receiverTypeName = ctorType; + receiverSource = 'constructor-map'; } } } @@ -994,10 +1160,14 @@ export const processCalls = async ( const enclosingFunc = findEnclosingFunction(callNode, file.path, ctx, provider); const funcName = enclosingFunc ? extractFuncNameFromSourceId(enclosingFunc) : ''; receiverTypeName = lookupReceiverType(receiverIndex, funcName, receiverName); + if (receiverTypeName) receiverSource = 'constructor-map'; } - // Fall back to class-as-receiver for static method calls (e.g. UserService.find_user()). - // When the receiver name is not a variable in TypeEnv but resolves to a Class/Struct/Interface - // through the standard tiered resolution, use it directly as the receiver type. + // Fall back to class-as-receiver for static method calls (e.g. UserService.find_user(), + // Greetable.format()). When the receiver name is not a variable in TypeEnv but + // resolves to a class-like symbol (Class / Interface / Struct / Enum / Trait) via + // tiered resolution, use it directly as the receiver type. `Trait` is included so + // Ruby module class-method calls flow through the class-as-receiver path and reach + // the `selectDispatch` hook's singleton branch. if (!receiverTypeName && receiverName && callForm === 'member') { const typeResolved = ctx.resolve(receiverName, file.path); if ( @@ -1007,10 +1177,12 @@ export const processCalls = async ( d.type === 'Class' || d.type === 'Interface' || d.type === 'Struct' || - d.type === 'Enum', + d.type === 'Enum' || + d.type === 'Trait', ) ) { receiverTypeName = receiverName; + receiverSource = 'class-as-receiver'; } } // Hoist sourceId so it's available for ACCESSES edge emission during chain walk. @@ -1056,11 +1228,51 @@ export const processCalls = async ( makeAccessEmitter(graph, sourceId), heritageMap, ); + if (receiverTypeName) receiverSource = 'mixed-chain'; } } } } + // --- DAG stage 3: infer-receiver (provider hook) --- + // Synthesize implicit receivers for languages that omit them (e.g., Ruby bare-call). + // This hook runs AFTER the shared inference chain so explicit receivers / + // typed bindings always take precedence. Output (if non-null) overlays onto + // the ReceiverEnriched for the next stage. + let dispatchHint: string | undefined; + if (provider.inferImplicitReceiver) { + const override = provider.inferImplicitReceiver({ + calledName, + callForm, + receiverName, + receiverTypeName, + callNode, + filePath: file.path, + }); + if (override) { + callForm = override.callForm; + receiverName = override.receiverName; + receiverTypeName = override.receiverTypeName; + receiverSource = override.receiverSource; + dispatchHint = override.hint; + } + } + + // --- DAG stage 4: select-dispatch (provider hook + default fallback) --- + // Decide which resolver path to try first (primary) and fallback strategy. + // Language providers can customize dispatch via selectDispatch hook; all + // others use the shared defaultDispatchDecision. Always non-null after this + // block so downstream resolvers are table-driven. + const dispatchDecision: DispatchDecision = + provider.selectDispatch?.({ + calledName, + callForm, + receiverName, + receiverTypeName, + receiverSource, + hint: dispatchHint, + }) ?? defaultDispatchDecision(callForm); + // Build overload hints for languages with inferLiteralType (Java/Kotlin/C#/C++). // Only used when multiple candidates survive arity filtering — ~1-3% of calls. const langConfig = provider.typeConfig; @@ -1082,6 +1294,7 @@ export const processCalls = async ( widenCache, undefined, heritageMap, + dispatchDecision, ); if (!resolved) return; @@ -1188,10 +1401,13 @@ export const processCalls = async ( return collectedHeritage; }; -const CALLABLE_SYMBOL_TYPES = new Set(['Function', 'Method', 'Constructor', 'Macro', 'Delegate']); +// FREE_CALLABLE_TYPES imported from symbol-table.ts — single source of truth. const CONSTRUCTOR_TARGET_TYPES = new Set(['Constructor', 'Class', 'Struct', 'Record']); +/** Per-file cache for module-alias widening. Cleared between files. */ +type WidenCache = Map; + const filterCallableCandidates = ( candidates: readonly SymbolDefinition[], argCount?: number, @@ -1206,10 +1422,14 @@ const filterCallableCandidates = ( } else { const types = candidates.filter((c) => CONSTRUCTOR_TARGET_TYPES.has(c.type)); kindFiltered = - types.length > 0 ? types : candidates.filter((c) => CALLABLE_SYMBOL_TYPES.has(c.type)); + types.length > 0 ? types : candidates.filter((c) => CALL_TARGET_TYPES.has(c.type)); } } else { - kindFiltered = candidates.filter((c) => CALLABLE_SYMBOL_TYPES.has(c.type)); + // CALL_TARGET_TYPES (not FREE_CALLABLE_TYPES) — the post-A4 filter must + // also admit Method and Constructor candidates, which are now unioned + // into the pool from `model.methods.lookupMethodByName` rather than + // `symbols.lookupCallableByName`. + kindFiltered = candidates.filter((c) => CALL_TARGET_TYPES.has(c.type)); } if (kindFiltered.length === 0) return []; @@ -1228,6 +1448,40 @@ const filterCallableCandidates = ( ); }; +/** + * Count callable candidates matching the kind + arity filter without + * allocating an intermediate array. Short-circuits once count exceeds + * `threshold` (default 1) — used by the dispatcher's `skipMember` check + * where we only need to know "more than one survivor". + */ +const countCallableCandidates = ( + candidates: readonly SymbolDefinition[], + argCount?: number, + callForm?: 'free' | 'member' | 'constructor', + threshold = 1, +): number => { + let count = 0; + for (const c of candidates) { + // Kind filter (mirrors filterCallableCandidates) + const typeOk = + callForm === 'constructor' + ? CONSTRUCTOR_TARGET_TYPES.has(c.type) + : CALL_TARGET_TYPES.has(c.type); + if (!typeOk) continue; + // Arity filter + if ( + argCount !== undefined && + c.parameterCount !== undefined && + (argCount < (c.requiredParameterCount ?? c.parameterCount) || argCount > c.parameterCount) + ) { + continue; + } + count++; + if (count > threshold) return count; // early exit + } + return count; +}; + const toResolveResult = (definition: SymbolDefinition, tier: ResolutionTier): ResolveResult => ({ nodeId: definition.nodeId, confidence: TIER_CONFIDENCE[tier], @@ -1318,19 +1572,232 @@ const tryOverloadDisambiguation = ( }; /** - * Resolve a function call to its target node ID using priority strategy: - * A. Narrow candidates by scope tier via ctx.resolve() - * B. Filter to callable symbol kinds (constructor-aware when callForm is set) - * C. Apply arity filtering when parameter metadata is available - * D. Apply receiver-type filtering for member calls with typed receivers - * E. Apply overload disambiguation via argument literal types (when available) + * Apply overload-hint or arg-type disambiguation to a pre-filtered candidate + * pool. Returns the unique survivor, or null when neither signal is present, + * neither can disambiguate, or the pool remains ambiguous. * - * If filtering still leaves multiple candidates, refuse to emit a CALLS edge. + * Precedence rule: `overloadHints` wins over `preComputedArgTypes` when both + * are supplied. The AST-based disambiguator has access to live type inference + * hooks, whereas `preComputedArgTypes` is a worker-path pre-computation that + * may be coarser-grained. + * + * Single source of truth for the narrowing-signal precedence used by member + * and constructor resolution paths. Add a new narrowing signal here once, not + * at each call site. */ -/** Per-file cache for the widen path's lookupFuzzy calls. Cleared between files. */ -type WidenCache = Map; +const disambiguateByOverloadOrArgTypes = ( + pool: SymbolDefinition[], + overloadHints: OverloadHints | undefined, + preComputedArgTypes: (string | undefined)[] | undefined, +): SymbolDefinition | null => { + if (!overloadHints && !preComputedArgTypes) return null; + if (overloadHints) return tryOverloadDisambiguation(pool, overloadHints); + if (preComputedArgTypes) return matchCandidatesByArgTypes(pool, preComputedArgTypes); + return null; +}; -/** @internal Exported for unit tests of D0 skip conditions (SM-11). Do not use outside tests. */ +/** + * Collapse Swift-extension duplicate Class/Struct candidates to the primary + * definition, preferring the shortest file path. + * + * Swift extensions (`extension User { ... }` in a separate file) create + * multiple `Class` nodes sharing the same symbol name — one for the primary + * declaration and one per extension file. When overload disambiguation and + * receiver narrowing both fail to converge on a single candidate, this + * heuristic picks the primary definition based on the assumption that it + * lives at the shortest file path (e.g. `User.swift` over `UserExtensions.swift`). + * + * Intentionally narrower than {@link INSTANTIABLE_CLASS_TYPES}: only `Class` + * and `Struct` are considered, not `Record`. Swift extensions only produce + * `Class` duplicates in practice, and C#/Kotlin records do not exhibit the + * same multi-file-definition pattern, so widening this set risks accidental + * dedup of legitimately distinct record types. + * + * Returns a `ResolveResult` when the heuristic fires, `null` when the + * candidate pool does not match the shape (mixed types, non-Class/Struct + * kinds, or `length <= 1`). Callers should fall through to their own null + * return when this helper returns `null`. + * + * Used by `resolveFreeCall`. Having a single source of truth prevents + * duplication if the heuristic is ever tuned. + */ +const dedupSwiftExtensionCandidates = ( + candidates: readonly SymbolDefinition[], + tier: ResolutionTier, +): ResolveResult | null => { + if (candidates.length <= 1) return null; + const allSameType = candidates.every((c) => c.type === candidates[0].type); + if (!allSameType) return null; + if (candidates[0].type !== 'Class' && candidates[0].type !== 'Struct') return null; + const sorted = [...candidates].sort((a, b) => a.filePath.length - b.filePath.length); + return toResolveResult(sorted[0], tier); +}; + +/** + * Thin dispatcher that routes a call to the appropriate specialized resolver. + * + * - `free` → {@link resolveFreeCall} + * - `constructor` → {@link resolveStaticCall} (with pre-resolved tiered pool) + * - `member` with a known receiver type → {@link resolveMemberCall}, with + * file-based fallback for traits/interfaces + * - `member` without receiver type → module-alias check, then tiered lookup + * + * Replaces the former 200+ line function (SM-19: fuzzy-free call resolution). + */ +/** + * Module-alias resolution for member calls without a receiver type. + * + * Handles Python/Ruby `import mod; mod.Symbol()` patterns where the receiver + * is a module name, not a typed variable. Uses `moduleAliasMap` to scope + * candidates to the correct module file. + */ +const resolveModuleAliasedCall = ( + call: Pick, + currentFile: string, + ctx: ResolutionContext, + widenCache?: WidenCache, + tieredOverride?: TieredCandidates, +): ResolveResult | null => { + if (!call.receiverName) return null; + const aliasMap = ctx.moduleAliasMap?.get(currentFile); + if (!aliasMap) return null; + const moduleFile = aliasMap.get(call.receiverName); + if (!moduleFile) return null; + + // Reuse the caller's pre-computed tiered result when available — + // the dispatcher already called ctx.resolve(call.calledName, currentFile). + const tiered = tieredOverride ?? ctx.resolve(call.calledName, currentFile); + if (!tiered) return null; + + // Try member-form, then constructor-form (for `module.ClassName()` patterns) + let filtered = filterCallableCandidates(tiered.candidates, call.argCount, call.callForm).filter( + (c) => c.filePath === moduleFile, + ); + if (filtered.length === 0) { + filtered = filterCallableCandidates(tiered.candidates, call.argCount, 'constructor').filter( + (c) => c.filePath === moduleFile, + ); + } + if (filtered.length === 0) { + // Widen to global callable+method indexes scoped to the aliased module + // file. Function+ownerId (Python/Rust/Kotlin) is still routed to both + // indexes until Unit 5 unblocks, so dedup by nodeId. + const cacheKey = `${call.calledName}\0${moduleFile}`; + let defs = widenCache?.get(cacheKey); + if (!defs) { + const rawCallable = ctx.model.symbols.lookupCallableByName(call.calledName); + const rawMethods = ctx.model.methods.lookupMethodByName(call.calledName); + const widenCombined: SymbolDefinition[] = []; + const widenSeen = new Set(); + for (const d of rawCallable) { + if (widenSeen.has(d.nodeId)) continue; + widenSeen.add(d.nodeId); + widenCombined.push(d); + } + for (const d of rawMethods) { + if (widenSeen.has(d.nodeId)) continue; + widenSeen.add(d.nodeId); + widenCombined.push(d); + } + defs = widenCombined; + widenCache?.set(cacheKey, defs); + } + filtered = filterCallableCandidates(defs, call.argCount, call.callForm).filter( + (c) => c.filePath === moduleFile, + ); + if (filtered.length === 0) { + filtered = filterCallableCandidates(defs, call.argCount, 'constructor').filter( + (c) => c.filePath === moduleFile, + ); + } + } + return filtered.length === 1 ? toResolveResult(filtered[0], tiered.tier) : null; +}; + +/** + * File-based fallback for member calls where owner-scoped resolution fails. + * + * Resolves the receiver type via `ctx.resolve()` and narrows all callable + * symbols with the method name to the receiver type's defining file(s), + * then applies ownerId filtering and overload disambiguation. + * + * Handles Rust trait dispatch (`repo.find()` where `find` is on a trait impl), + * cross-file overloaded methods, and similar patterns where ownerId + * relationships may not be established on all candidates. + */ +const resolveMemberCallByFile = ( + calledName: string, + receiverTypeName: string, + currentFile: string, + ctx: ResolutionContext, + argCount?: number, + callForm?: 'free' | 'member' | 'constructor', + overloadHints?: OverloadHints, + preComputedArgTypes?: (string | undefined)[], +): ResolveResult | null => { + const typeResolved = ctx.resolve(receiverTypeName, currentFile); + if (!typeResolved || typeResolved.candidates.length === 0) return null; + const typeNodeIds = new Set(typeResolved.candidates.map((d) => d.nodeId)); + const typeFiles = new Set(typeResolved.candidates.map((d) => d.filePath)); + + // A4 (plan 006, Unit 4): consult both indexes. Strictly-labeled + // Method/Constructor are disjoint, but Function+ownerId (Python/Rust/ + // Kotlin) is routed into BOTH indexes by `wrappedAdd` until Unit 5 + // unblocks — dedup by nodeId so overload disambiguation doesn't see + // phantom duplicates. + const rawCallablePool = ctx.model.symbols.lookupCallableByName(calledName); + const rawMethodPool = ctx.model.methods.lookupMethodByName(calledName); + const combinedPool: SymbolDefinition[] = []; + const combinedSeen = new Set(); + for (const def of rawCallablePool) { + if (combinedSeen.has(def.nodeId)) continue; + combinedSeen.add(def.nodeId); + combinedPool.push(def); + } + for (const def of rawMethodPool) { + if (combinedSeen.has(def.nodeId)) continue; + combinedSeen.add(def.nodeId); + combinedPool.push(def); + } + const methodPool = filterCallableCandidates(combinedPool, argCount, callForm); + const fileFiltered = methodPool.filter((c) => typeFiles.has(c.filePath)); + if (fileFiltered.length === 1) { + return toResolveResult(fileFiltered[0], typeResolved.tier); + } + + // ownerId fallback: narrow by ownerId matching the type's nodeId + const pool = fileFiltered.length > 0 ? fileFiltered : methodPool; + const ownerFiltered = pool.filter((c) => c.ownerId && typeNodeIds.has(c.ownerId)); + if (ownerFiltered.length === 1) return toResolveResult(ownerFiltered[0], typeResolved.tier); + + // Overload disambiguation on the narrowed pool + if (fileFiltered.length > 1 || ownerFiltered.length > 1) { + const overloadPool = ownerFiltered.length > 1 ? ownerFiltered : fileFiltered; + const disambiguated = disambiguateByOverloadOrArgTypes( + overloadPool, + overloadHints, + preComputedArgTypes, + ); + if (disambiguated) return toResolveResult(disambiguated, typeResolved.tier); + } + + // Zero-match null-route: receiver type resolved but no candidate matched + // after file-based and owner-based narrowing. Refuse to emit a CALLS edge + // rather than guess — matches the SM-10 R3 null-route contract. + return null; +}; + +/** Return the sole survivor from a tiered pool after callable + arity filtering, or null. */ +const singleCandidate = ( + tiered: TieredCandidates, + argCount?: number, + callForm?: 'free' | 'member' | 'constructor', +): ResolveResult | null => { + const filtered = filterCallableCandidates(tiered.candidates, argCount, callForm); + return filtered.length === 1 ? toResolveResult(filtered[0], tiered.tier) : null; +}; + +/** @internal Exported for unit tests. Do not use outside tests. */ export const _resolveCallTargetForTesting = ( call: Pick< ExtractedCall, @@ -1366,270 +1833,151 @@ const resolveCallTarget = ( widenCache?: WidenCache, preComputedArgTypes?: (string | undefined)[], heritageMap?: HeritageMap, + dispatchDecision?: DispatchDecision, ): ResolveResult | null => { const tiered = ctx.resolve(call.calledName, currentFile); if (!tiered) return null; - let filteredCandidates = filterCallableCandidates( - tiered.candidates, - call.argCount, - call.callForm, - ); + // DAG dispatch: use decision.primary to pick the resolver branch. + // Callers that own the DAG (processCalls + crossFile deferred paths) + // pass a decision; other callers use the shared default ladder. + // Language-specific primary / fallback / ancestryView overrides come from + // the provider's `selectDispatch` hook. + const decision = dispatchDecision ?? defaultDispatchDecision(call.callForm); + const primary = decision.primary; - // S0. Constructor/static fast path (SM-12): O(1) class + constructor lookup - // via lookupClassByName + lookupMethodByOwner before falling back to the - // existing filtering + fuzzy-widening path. Falls back to the class node - // itself when no Constructor symbol is indexed for the type. - // - // Handles: - // (a) callForm === 'constructor' — explicit `new User()` in Java/TS/C#/etc. - // (b) callForm === 'free' with class target — implicit `User()` in Swift/Kotlin - // - // Known gaps (handled by the existing tail fallback at the bottom of - // this function, not S0): - // - `callForm === 'member'` constructor patterns (e.g. Python - // `models.User()` after `import models`, Ruby `User.new`). Extending - // S0 to cover them would require threading receiver-type resolution - // through the module-alias logic; revisit if it shows up as a hot - // spot. - // - // The `.some()` trigger below must stay aligned with - // `INSTANTIABLE_CLASS_TYPES` — any type admitted here that is not in - // that set will cause S0 → `resolveStaticCall` to run and return null, - // wasting two lookup passes per call. `Enum` is deliberately excluded - // (same rationale as `INSTANTIABLE_CLASS_TYPES`); `Record` is included - // so C# records and Kotlin data classes reach the fast path. - const freeFormHasClassTarget = - call.callForm === 'free' && - filteredCandidates.length === 0 && - tiered.candidates.some((c) => c.type === 'Class' || c.type === 'Struct' || c.type === 'Record'); - if (call.callForm === 'constructor' || freeFormHasClassTarget) { - // Reuse the pre-computed `tiered` result — resolveStaticCall's class name - // is identical to `call.calledName` here, so re-running ctx.resolve would - // duplicate the tiered-lookup work performed at the top of this function. - const staticResult = resolveStaticCall( + if (primary === 'free') { + return resolveFreeCall( call.calledName, currentFile, ctx, call.argCount, tiered, + overloadHints, + preComputedArgTypes, ); - if (staticResult) return staticResult; } - - // Swift/Kotlin: constructor calls look like free function calls (no `new` keyword). - // If free-form filtering found no callable candidates but the symbol resolves to a - // Class/Struct, retry with constructor form so CONSTRUCTOR_TARGET_TYPES applies. - if (filteredCandidates.length === 0 && call.callForm === 'free') { - // `freeFormHasClassTarget` was already computed for the S0 fast path - // above under the same `callForm === 'free' && filteredCandidates.length === 0` - // precondition. Reuse it to avoid a second `.some()` scan on the same pool. - if (freeFormHasClassTarget) { - filteredCandidates = filterCallableCandidates( - tiered.candidates, - call.argCount, - 'constructor', - ); - } - } - - // Module-qualified constructor pattern: e.g. Python `import models; models.User()`. - // The attribute access gives callForm='member', but the callee may be a Class — a valid - // constructor target. Re-try with constructor-form filtering so that `module.ClassName()` - // emits a CALLS edge to the class node. - if (filteredCandidates.length === 0 && call.callForm === 'member') { - filteredCandidates = filterCallableCandidates(tiered.candidates, call.argCount, 'constructor'); - } - - // Module-alias disambiguation: Python `import auth; auth.User()` — receiverName='auth' - // selects auth.py via moduleAliasMap. Runs for ALL member calls with a known module alias, - // not just ambiguous ones — same-file tier may shadow the correct cross-module target when - // the caller defines a function with the same name as the callee (Issue #417). - // - // Tracks `aliasNarrowed` so the D2 widening step below does NOT undo the alias filtering - // by calling lookupFuzzy again (which would re-introduce homonym candidates from other files). - let aliasNarrowed = false; - if (call.callForm === 'member' && call.receiverName) { - const aliasMap = ctx.moduleAliasMap?.get(currentFile); - if (aliasMap) { - const moduleFile = aliasMap.get(call.receiverName); - if (moduleFile) { - const aliasFiltered = filteredCandidates.filter((c) => c.filePath === moduleFile); - if (aliasFiltered.length > 0) { - filteredCandidates = aliasFiltered; - aliasNarrowed = true; - } else { - // Same-file tier returned a local match, but the alias points elsewhere. - // Widen to global candidates and filter to the aliased module's file. - // Use per-file widenCache to avoid repeated lookupFuzzy for the same - // calledName+moduleFile from multiple call sites in the same file. - const cacheKey = `${call.calledName}\0${moduleFile}`; - let fuzzyDefs = widenCache?.get(cacheKey); - if (!fuzzyDefs) { - fuzzyDefs = ctx.symbols.lookupFuzzy(call.calledName); - widenCache?.set(cacheKey, fuzzyDefs); - } - const widened = filterCallableCandidates(fuzzyDefs, call.argCount, call.callForm).filter( - (c) => c.filePath === moduleFile, - ); - if (widened.length > 0) { - filteredCandidates = widened; - aliasNarrowed = true; - } - } - } - } - } - - // D. Receiver-type filtering: for member calls with a known receiver type, - // resolve the type through the same tiered import infrastructure, then - // filter method candidates to the type's defining file. Fall back to - // fuzzy ownerId matching only when file-based narrowing is inconclusive. - // - // Applied regardless of candidate count — the sole same-file candidate may - // belong to the wrong class (e.g. super.save() should hit the parent's save, - // not the child's own save method in the same file). - if (call.callForm === 'member' && call.receiverTypeName) { - // D0. Delegate to resolveMemberCall (SM-11): owner-scoped + MRO lookup - // before falling back to the expensive D1-D4 fuzzy widening. - // Skip conditions: - // (a) overloadHints or preComputedArgTypes present — the MRO lookup may - // pick the wrong overload for same-return-type overloads since it - // does not consider argument types. D1-D4+E handles those correctly. - // (b) A module alias on call.receiverName is active for this file — the - // alias block above already narrowed `filteredCandidates` to a - // specific file. resolveMemberCall re-resolves `receiverTypeName` - // from scratch via `ctx.resolve`, which ignores that narrowing and - // could pick a homonymous class from the wrong file. Fall through to - // D1-D4 which respects the alias-filtered candidate pool. - // D0 skip for overload disambiguation: only fires when the name actually - // has multiple candidates in the tiered pool. The sequential path sets - // `overloadHints` for every call regardless of whether the method is - // overloaded — skipping D0 unconditionally would make this fast path - // dead code for the sequential pipeline. By gating on - // `filteredCandidates.length > 1`, we preserve the original intent - // (let D1-D4+E pick the right overload when there are multiple) while - // allowing D0 to fire for the common single-candidate case. - const hasOverloadConcern = - (!!overloadHints || !!preComputedArgTypes) && filteredCandidates.length > 1; - // D0 skip for active module alias: only fires when the alias block above - // actually narrowed filteredCandidates. In Python, a local variable can - // shadow an imported module name (e.g. `from models.c import C; c = C()` - // creates both a module alias `c → models/c.py` AND a typed local `c`). - // Checking `aliasNarrowed` rather than `ctx.moduleAliasMap.has(receiverName)` - // ensures D0 still runs when the method isn't in the aliased module — - // which means the receiver is a typed local variable, not a module reference. - if (!hasOverloadConcern && !aliasNarrowed) { - const memberResult = resolveMemberCall( - call.receiverTypeName, + if (primary === 'constructor') { + return ( + resolveStaticCall( call.calledName, currentFile, ctx, - heritageMap, call.argCount, - ); - if (memberResult) return memberResult; + tiered, + overloadHints, + preComputedArgTypes, + ) ?? singleCandidate(tiered, call.argCount, 'constructor') + ); + } + // primary === 'owner-scoped' + if (call.receiverTypeName) { + // Skip the owner-scoped MRO path when the tiered pool has genuine + // overload ambiguity that needs D1-D4+E handling, not D0. + const skipMember = + (!!overloadHints || !!preComputedArgTypes) && + countCallableCandidates(tiered.candidates, call.argCount, call.callForm) > 1; + // Try owner-scoped (resolveMemberCall) then file-scoped (resolveMemberCallByFile). + // DAG: dispatchDecision.ancestryView selects instance vs singleton ancestry + // for kind-aware MRO strategies. Ruby `Account.log` flows via 'singleton'. + // + // Singleton-ancestry miss MUST NOT degrade to the file-scoped fallback: + // resolveMemberCallByFile matches by ownerId and would happily pick an + // instance method defined on the same class, leaking instance dispatch + // onto what was declared a class-method call. For singleton dispatch, + // a miss either null-routes or falls through to `decision.fallback`. + const singletonDispatch = decision.ancestryView === 'singleton'; + const memberResult = + (!skipMember + ? resolveMemberCall( + call.receiverTypeName, + call.calledName, + currentFile, + ctx, + heritageMap, + call.argCount, + decision.ancestryView, + ) + : null) ?? + (singletonDispatch + ? null + : resolveMemberCallByFile( + call.calledName, + call.receiverTypeName, + currentFile, + ctx, + call.argCount, + call.callForm, + overloadHints, + preComputedArgTypes, + )); + if (memberResult) return memberResult; + + // Module-alias narrowing runs as a FALLBACK, after owner/file-scoped + // resolvers have returned null. This ordering is load-bearing: placing + // alias narrowing first would short-circuit unique owner-scoped answers + // when a local variable coincidentally matches an alias name, leaking + // unrelated homonyms from the aliased file onto the wrong receiver type. + // + // The type-file verification guard is load-bearing for SM-10 R3: an + // alias is only a VALID narrowing signal when the alias target file is + // among the receiver type's defining files. If the alias points at a + // file that does not hold `receiverTypeName`, any candidate we would + // pick from there would belong to an unrelated class — a cross-type + // false positive. ctx.resolve is cached per (name, file), so resolving + // the receiver type a second time here is free. + const typeResolves = ctx.resolve(call.receiverTypeName, currentFile); + const aliasMap = ctx.moduleAliasMap?.get(currentFile); + const aliasTargetFile = + call.receiverName && aliasMap ? aliasMap.get(call.receiverName) : undefined; + if ( + aliasTargetFile && + typeResolves && + typeResolves.candidates.some((c) => c.filePath === aliasTargetFile) + ) { + const aliasResult = resolveModuleAliasedCall(call, currentFile, ctx, widenCache, tiered); + if (aliasResult) return aliasResult; } - // D1. Resolve the receiver type - const typeResolved = ctx.resolve(call.receiverTypeName, currentFile); - if (typeResolved && typeResolved.candidates.length > 0) { - const typeNodeIds = new Set(typeResolved.candidates.map((d) => d.nodeId)); - const typeFiles = new Set(typeResolved.candidates.map((d) => d.filePath)); - - // D2. Widen candidates: same-file tier may miss the parent's method when - // it lives in another file. Query the symbol table directly for all - // global methods with this name, then apply arity/kind filtering. - // - // When the candidate set was already narrowed by module-alias - // disambiguation, do NOT widen back to the full fuzzy pool — that - // would undo the alias narrowing and reintroduce homonym candidates - // from other files. - const methodPool = - filteredCandidates.length <= 1 && !aliasNarrowed - ? filterCallableCandidates( - ctx.symbols.lookupFuzzy(call.calledName), - call.argCount, - call.callForm, - ) - : filteredCandidates; - - // D3. File-based: prefer candidates whose filePath matches the resolved type's file - const fileFiltered = methodPool.filter((c) => typeFiles.has(c.filePath)); - if (fileFiltered.length === 1) { - return toResolveResult(fileFiltered[0], tiered.tier); - } - - // D4. ownerId fallback: narrow by ownerId matching the type's nodeId - const pool = fileFiltered.length > 0 ? fileFiltered : methodPool; - const ownerFiltered = pool.filter((c) => c.ownerId && typeNodeIds.has(c.ownerId)); - if (ownerFiltered.length === 1) { - return toResolveResult(ownerFiltered[0], tiered.tier); - } - // E. Try overload disambiguation on the narrowed pool - if (fileFiltered.length > 1 || ownerFiltered.length > 1) { - const overloadPool = ownerFiltered.length > 1 ? ownerFiltered : fileFiltered; - const disambiguated = overloadHints - ? tryOverloadDisambiguation(overloadPool, overloadHints) - : preComputedArgTypes - ? matchCandidatesByArgTypes(overloadPool, preComputedArgTypes) - : null; - if (disambiguated) return toResolveResult(disambiguated, tiered.tier); - return null; - } - - // Zero-match null-route: we committed to receiver narrowing (D1 succeeded) - // but both file-based (D3) and owner-based (D4) filters produced zero - // matches. The lone candidate in `filteredCandidates` does not belong to - // this receiver type — refuse to emit a CALLS edge rather than fall - // through to the permissive single-candidate tail return. - // - // Addresses Codex review finding R3 (PR #744): member calls where - // fuzzy fallback picked a globally-matching symbol that has no - // relationship to the receiver's class hierarchy were silently - // producing false-positive edges. Example: Rust `c.trait_only()` where - // `trait_only` is captured as a Function node with no ownerId — it - // matches the name but fails both file and owner narrowing, so the - // old tail return would pick it incorrectly. - if (fileFiltered.length === 0 && ownerFiltered.length === 0) { - return null; - } - } - } - - // E. Overload disambiguation: when multiple candidates survive arity + receiver filtering, - // try matching argument types against parameter types (Phase P). - // Sequential path uses AST-based hints; worker path uses pre-computed argTypes. - if (filteredCandidates.length > 1) { - const disambiguated = overloadHints - ? tryOverloadDisambiguation(filteredCandidates, overloadHints) - : preComputedArgTypes - ? matchCandidatesByArgTypes(filteredCandidates, preComputedArgTypes) - : null; - if (disambiguated) return toResolveResult(disambiguated, tiered.tier); - } - - if (filteredCandidates.length !== 1) { - // Deduplicate: Swift extensions create multiple Class nodes with the same name. - // When all candidates share the same type and differ only by file (extension vs - // primary definition), they represent the same symbol. Prefer the primary - // definition (shortest file path: Product.swift over ProductExtension.swift). - if (filteredCandidates.length > 1) { - const allSameType = filteredCandidates.every((c) => c.type === filteredCandidates[0].type); - if ( - allSameType && - (filteredCandidates[0].type === 'Class' || filteredCandidates[0].type === 'Struct') - ) { - const sorted = [...filteredCandidates].sort( - (a, b) => a.filePath.length - b.filePath.length, + // SM-10 R3 null-route: when the receiver type resolves to indexed types + // but no scoped resolver (nor the guarded alias fallback) produced a + // match, that's a genuine miss — refuse to emit a CALLS edge rather + // than guess via an unscoped singleCandidate that ignores the class + // hierarchy. When the type is NOT in the index (PHP `mixed`, dynamic + // types, unresolvable aliases), the scoped resolvers had nothing to + // work with and singleCandidate is the correct last resort. + // + // DAG fallback override: when `select-dispatch` returned + // `fallback: 'free-arity-narrowed'` (today: Ruby implicit-self bare + // calls whose enclosing class doesn't define the method), fall through + // to free-call resolution instead of null-routing. This preserves + // existing free-call arity-narrowing heuristics for bare calls that + // happen to target methods on unrelated classes. + if (typeResolves && typeResolves.candidates.length > 0) { + if (decision.fallback === 'free-arity-narrowed') { + const free = resolveFreeCall( + call.calledName, + currentFile, + ctx, + call.argCount, + tiered, + overloadHints, + preComputedArgTypes, ); - return toResolveResult(sorted[0], tiered.tier); + if (free) return free; } + return null; // null-route: type resolved, no candidate matched } - return null; + return singleCandidate(tiered, call.argCount, call.callForm); } - - return toResolveResult(filteredCandidates[0], tiered.tier); + // Member call with no inferred receiver type — e.g. Python `mod.fn()` + // where `mod` is a module alias. Module-alias narrowing is the primary + // disambiguation signal here. Also consulted from the typed-member + // branch above as a guarded fallback after owner/file-scoped resolvers. + return ( + resolveModuleAliasedCall(call, currentFile, ctx, widenCache, tiered) ?? + singleCandidate(tiered, call.argCount, call.callForm) + ); }; // ── Scope key helpers ──────────────────────────────────────────────────── @@ -1643,9 +1991,6 @@ const resolveCallTarget = ( // classes (e.g. User.save@100 and Repo.save@200 are distinct keys). // Lookup uses a secondary funcName-only index built in lookupReceiverType. -/** Extract the function name from a scope key ("funcName@startIndex" → "funcName"). */ -const extractFuncNameFromScope = (scope: string): string => scope.slice(0, scope.indexOf('@')); - /** Extract the bare function name from a sourceId. * Handles both unqualified ("Function:filepath:funcName" → "funcName") * and qualified ("Function:filepath:ClassName.funcName" → "funcName"). @@ -1784,7 +2129,7 @@ const resolveFieldOwnership = ( const classDef = typeResolved.candidates.find((d) => CLASS_LIKE_TYPES.has(d.type)); if (!classDef) return undefined; - return ctx.symbols.lookupFieldByOwner(classDef.nodeId, fieldName) ?? undefined; + return ctx.model.fields.lookupFieldByOwner(classDef.nodeId, fieldName) ?? undefined; }; /** @@ -1801,21 +2146,13 @@ const resolveFieldOwnership = ( * * After deduplication: * - * - 0 unique matches → `undefined` (owner-scoped path has no answer; D1-D4 fuzzy - * fallback in `resolveCallTarget` may still find something via lookupFuzzy) + * - 0 unique matches → `undefined` (owner-scoped path has no answer) * - 1 unique match → return it * - ≥2 unique matches → `undefined` (genuine homonym ambiguity; don't silently pick one) * - * This absorbs what was previously D4's job inside `resolveCallTarget` — "filter fuzzy - * candidates to those whose ownerId is in the receiver type's nodeId set" — into the - * owner-scoped path, aligning with the plan's target: - * - * `resolveCallTarget` D2 widening → `model.lookupMethodWithMRO(ownerNodeId, name)` - * * The returned `tier` reflects how the owner TYPE was resolved (not the method name). * Threaded out here so callers don't need a second `ctx.resolve(ownerType, ...)` call — - * this decouples callers from `ctx.resolve`'s per-file caching contract, which SM-16 - * will restructure when it replaces the `lookupFuzzy` data source. + * this decouples callers from `ctx.resolve`'s per-file caching contract. */ const resolveMethodByOwner = ( receiverTypeName: string, @@ -1824,14 +2161,23 @@ const resolveMethodByOwner = ( ctx: ResolutionContext, heritageMap?: HeritageMap, argCount?: number, + /** + * DAG-sourced ancestry selector. `'singleton'` routes through + * `heritageMap.getSingletonAncestry(owner)` for class-method dispatch + * (Ruby `Account.log` via `extend LoggerMixin`). Default / undefined + * uses the walker's instance-dispatch behavior. + */ + ancestryView?: 'instance' | 'singleton', ): { def: SymbolDefinition; tier: ResolutionTier } | undefined => { const typeResolved = ctx.resolve(receiverTypeName, filePath); if (!typeResolved) return undefined; - // MRO walking needs a language hint; compute once and reuse for every candidate. - // Unknown extension → fall back to plain direct lookup (D1-D4 still runs on miss). + // MRO walking needs a language hint so we can derive the per-language + // strategy; compute it once and reuse for every candidate. Unknown + // extension → fall back to plain direct lookup (D1-D4 still runs on miss). const language = heritageMap ? getLanguageFromFilename(filePath) : null; - const canWalkMRO = heritageMap != null && language != null; + const mroStrategy = language != null ? getProvider(language).mroStrategy : null; + const canWalkMRO = heritageMap != null && mroStrategy != null; // Iterate all class-like candidates tracking the first unambiguous hit. // Zero-allocation fast path: the common case is exactly one class candidate, @@ -1850,16 +2196,25 @@ const resolveMethodByOwner = ( let ambiguous = false; for (const candidate of typeResolved.candidates) { if (!CLASS_LIKE_TYPES.has(candidate.type)) continue; + // Singleton dispatch: when the DAG decision requested the singleton + // ancestry view, pass `heritageMap.getSingletonAncestry` as the walker's + // ancestry override. Kind-aware strategies (e.g. MroStrategy 'ruby-mixin') + // honor the override by scanning it linearly in place of their default walk. + const singletonOverride = + ancestryView === 'singleton' && canWalkMRO && heritageMap + ? heritageMap.getSingletonAncestry(candidate.nodeId).map((e) => e.parentId) + : undefined; const def = canWalkMRO ? lookupMethodByOwnerWithMRO( candidate.nodeId, methodName, heritageMap, - ctx.symbols, - language, + ctx.model, + mroStrategy, argCount, + singletonOverride, ) - : ctx.symbols.lookupMethodByOwner(candidate.nodeId, methodName, argCount); + : ctx.model.methods.lookupMethodByOwner(candidate.nodeId, methodName, argCount); if (!def) continue; if (!firstDef) { firstDef = def; @@ -1885,14 +2240,10 @@ const resolveMethodByOwner = ( * method lookup and, when a {@link HeritageMap} is provided, walks the MRO chain * via {@link lookupMethodByOwnerWithMRO}. * - * {@link resolveCallTarget} delegates here for member calls before falling back - * to the more expensive fuzzy-widening path (D1-D4). + * {@link resolveCallTarget} delegates here for member calls. * - * **SEMANTIC CHANGE (2026-04-09):** The confidence tier now reflects how the - * owner TYPE was resolved, not how the method NAME was resolved globally. The - * previous D0 fast path in `resolveCallTarget` used `tiered.tier` from - * `ctx.resolve(calledName, ...)` — a name-based tier that matched what D1-D4 - * fuzzy widening would produce. The new tier is owner-type-based, which is + * **SEMANTIC CHANGE (2026-04-09):** The confidence tier reflects how the + * owner TYPE was resolved, not how the method NAME was resolved globally. * more accurate for owner-scoped resolution (the discriminant IS the class, * not the method name). Downstream consumers that filter CALLS edges by * confidence threshold may see shifted values on otherwise-unchanged code. @@ -1916,6 +2267,7 @@ export const resolveMemberCall = ( ctx: ResolutionContext, heritageMap?: HeritageMap, argCount?: number, + ancestryView?: 'instance' | 'singleton', ): ResolveResult | null => { const resolved = resolveMethodByOwner( ownerType, @@ -1924,11 +2276,132 @@ export const resolveMemberCall = ( ctx, heritageMap, argCount, + ancestryView, ); if (!resolved) return null; return toResolveResult(resolved.def, resolved.tier); }; +// --------------------------------------------------------------------------- +// SM-13: Free-function call resolution +// --------------------------------------------------------------------------- + +/** + * Resolve a free-function call using `lookupExact` (same-file) + import-scoped + * resolution via `ctx.resolve()`. + * + * Used for `foo()`, `doStuff()` — unqualified calls with no receiver. + * Also handles Swift/Kotlin implicit constructors (`User()` without `new`) + * by delegating to {@link resolveStaticCall} when the tiered pool contains + * class-like targets. + * + * {@link resolveCallTarget} delegates here for `callForm === 'free'`. + * + * `resolveFreeCall` does not take a `widenCache` parameter. Free calls + * have no receiver type and rely exclusively on the tiered pool + * from `ctx.resolve()`. + * + * @param calledName - The called function name (e.g. 'doStuff') + * @param filePath - File path of the call site + * @param ctx - Resolution context + * @param argCount - Optional argument count for arity filtering + * @param tieredOverride - Pre-computed tiered candidates from an upstream + * `ctx.resolve` call. When provided, skips the redundant + * lookup inside this function. + * @param overloadHints - Optional AST-based overload disambiguation hints + * @param preComputedArgTypes - Optional pre-computed argument types (worker path) + */ +export const resolveFreeCall = ( + calledName: string, + filePath: string, + ctx: ResolutionContext, + argCount?: number, + tieredOverride?: TieredCandidates, + overloadHints?: OverloadHints, + preComputedArgTypes?: (string | undefined)[], +): ResolveResult | null => { + const tiered = tieredOverride ?? ctx.resolve(calledName, filePath); + if (!tiered) return null; + + let filteredCandidates = filterCallableCandidates(tiered.candidates, argCount, 'free'); + + // Class-target fast path: Swift/Kotlin `User()` — free-form call targeting a + // class. Delegates to resolveStaticCall for O(1) class + constructor lookup. + // The `.some()` trigger must stay aligned with `INSTANTIABLE_CLASS_TYPES` — + // any type admitted here that is not in that set will cause resolveStaticCall + // to return null, wasting two lookup passes per call. `Enum` is deliberately + // excluded; `Record` is included so C# records and Kotlin data classes reach + // the fast path. + // Align with INSTANTIABLE_CLASS_TYPES by reusing the set directly rather + // than enumerating literal strings. This converts an invariant that was + // previously enforced by a comment ("keep this list aligned with + // INSTANTIABLE_CLASS_TYPES") into one enforced structurally — any future + // extension of the set (e.g. Kotlin `object`) propagates here automatically. + // The `dedupSwiftExtensionCandidates` helper used in the tail of this + // function deliberately uses a narrower literal `'Class' | 'Struct'` check + // — Swift extensions only produce Class duplicates in practice, so Record + // is excluded there by design. Do not collapse that helper into + // INSTANTIABLE_CLASS_TYPES. + const hasClassTarget = + filteredCandidates.length === 0 && + tiered.candidates.some((c) => INSTANTIABLE_CLASS_TYPES.has(c.type)); + if (hasClassTarget) { + const staticResult = resolveStaticCall(calledName, filePath, ctx, argCount, tiered); + if (staticResult) return staticResult; + // Retry with constructor form: Swift/Kotlin constructor calls look like + // free function calls (no `new` keyword). If resolveStaticCall didn't + // match, re-filter with constructor form so CONSTRUCTOR_TARGET_TYPES + // applies. + // + // The retry fires for every null return from `resolveStaticCall`, which + // can happen for three distinct reasons — all three are handled below: + // + // (a) No explicit `Constructor` node found and zero instantiable + // class candidates (e.g. Interface/Trait/Impl only — the SM-12 + // null-route contract). `filterCallableCandidates` with + // `'constructor'` form will also return nothing → we fall + // through to the final null return. Correct. + // + // (b) Homonym ambiguity — two or more instantiable class candidates + // share the name (e.g. `User` in two files, same tier). The + // retry repopulates `filteredCandidates` with both Classes and + // they flow into `dedupSwiftExtensionCandidates` below, which + // either picks the shortest-path primary or null-routes. + // Covered by the R7 Swift-extension dedup test. + // + // (c) `resolveStaticCall` step 4 bailed because the tiered pool + // contains ownerless `Constructor` nodes (some extractors emit + // constructors without `ownerId`). Those `Constructor` nodes + // survive the constructor-form filter below and reach overload + // disambiguation, giving the existing filter path a chance to + // pick the right one. Correct but currently uncovered by a + // dedicated test — the R5 `preComputedArgTypes` path exercises + // overload disambiguation for Functions, which is structurally + // the same code. + filteredCandidates = filterCallableCandidates(tiered.candidates, argCount, 'constructor'); + } + + // E. Overload disambiguation + if (filteredCandidates.length > 1) { + const disambiguated = overloadHints + ? tryOverloadDisambiguation(filteredCandidates, overloadHints) + : preComputedArgTypes + ? matchCandidatesByArgTypes(filteredCandidates, preComputedArgTypes) + : null; + if (disambiguated) return toResolveResult(disambiguated, tiered.tier); + } + + if (filteredCandidates.length !== 1) { + // See `dedupSwiftExtensionCandidates` — shared helper, single source of + // truth for the Swift-extension same-name collision heuristic. + const deduped = dedupSwiftExtensionCandidates(filteredCandidates, tiered.tier); + if (deduped) return deduped; + return null; + } + + return toResolveResult(filteredCandidates[0], tiered.tier); +}; + // --------------------------------------------------------------------------- // SM-12: Constructor/static call resolution (no fuzzy lookup) // --------------------------------------------------------------------------- @@ -1937,11 +2410,10 @@ export const resolveMemberCall = ( * Resolve a constructor or static call using class-scoped lookup (no fuzzy lookup). * Used for `new User()` / `User()` calls where the calledName targets a class. * - * Uses {@link SymbolTable.lookupClassByName} for O(1) class lookup and - * {@link SymbolTable.lookupMethodByOwner} for constructor resolution. + * Uses {@link TypeRegistry.lookupClassByName} for O(1) class lookup and + * {@link MethodRegistry.lookupMethodByOwner} for constructor resolution. * {@link resolveCallTarget} delegates here for constructor and free-form calls - * that target a class, before falling back to the more expensive fuzzy-widening - * path (D1-D4). + * that target a class. * * Resolution strategy: * 1. `lookupClassByName(className)` — O(1) pre-check; bail early if no class exists. @@ -1982,6 +2454,8 @@ export const resolveStaticCall = ( ctx: ResolutionContext, argCount?: number, tieredOverride?: TieredCandidates, + overloadHints?: OverloadHints, + preComputedArgTypes?: (string | undefined)[], ): ResolveResult | null => { // 1. Pre-check: does a class with this name exist at all? (O(1)) // This guards against the expensive `ctx.resolve` walk when the name @@ -1989,7 +2463,7 @@ export const resolveStaticCall = ( // is supplied, the caller has already paid for the tiered lookup, so this // pre-check still prevents the class-candidate filter + lookupMethodByOwner // loop from running on obviously non-class targets. - const allClasses = ctx.symbols.lookupClassByName(className); + const allClasses = ctx.model.types.lookupClassByName(className); if (allClasses.length === 0) return null; // 2. Scope via ctx.resolve for import-tier information. Reuse the caller's @@ -2020,7 +2494,7 @@ export const resolveStaticCall = ( let firstDef: SymbolDefinition | undefined; let ambiguous = false; for (const candidate of classCandidates) { - const def = ctx.symbols.lookupMethodByOwner(candidate.nodeId, className, argCount); + const def = ctx.model.methods.lookupMethodByOwner(candidate.nodeId, className, argCount); if (!def || def.type !== 'Constructor') continue; if (!firstDef) { firstDef = def; @@ -2043,10 +2517,30 @@ export const resolveStaticCall = ( // with two distinct Constructor nodes across multiple class candidates): // the same Constructor nodes are indexed under the class name in the // tiered pool, so `.some(Constructor)` is true here and we defer to - // `filterCallableCandidates` downstream rather than guess which overload - // to pick. Do not remove this check without also handling the ambiguous - // step-3 path explicitly. + // step 4.5 (overload/arg-type disambiguation) or the caller's fallback. + // Do not remove this check without also handling the ambiguous step-3 + // path explicitly. if (typeResolved.candidates.some((c) => c.type === 'Constructor')) { + // 4.5. Overload / arg-type disambiguation for ambiguous or ownerless + // Constructor pools. When the caller supplied a narrowing signal + // (AST-based overload hints from the sequential path, or pre- + // computed arg types from the worker path), give disambiguation a + // chance before null-routing. Symmetric with resolveMemberCallByFile's + // disambiguation pass — both resolvers now share the same signal + // precedence via disambiguateByOverloadOrArgTypes. Only fires when + // at least one narrowing signal is present; preserves SM-10 R3 for + // genuinely ambiguous cases with no disambiguating input. + if (overloadHints || preComputedArgTypes) { + const ctorPool = filterCallableCandidates(typeResolved.candidates, argCount, 'constructor'); + if (ctorPool.length > 1) { + const disambiguated = disambiguateByOverloadOrArgTypes( + ctorPool, + overloadHints, + preComputedArgTypes, + ); + if (disambiguated) return toResolveResult(disambiguated, typeResolved.tier); + } + } return null; } @@ -2076,138 +2570,6 @@ export const resolveStaticCall = ( return null; }; -// --------------------------------------------------------------------------- -// MRO-aware method resolution via HeritageMap (SM-9) -// --------------------------------------------------------------------------- - -/** - * Per-HeritageMap cache of C3 linearization results keyed by owner nodeId. - * - * HeritageMap instances are immutable after construction, so C3 output is - * stable for the lifetime of a HeritageMap. WeakMap lets the cache auto-drain - * when the HeritageMap is garbage collected (end of ingestion run), so we - * never need to manually invalidate it. - * - * `null` is a sentinel for "C3 failed for this owner" (cyclic or inconsistent - * hierarchy) so we don't re-run the expensive linearization repeatedly. - */ -const c3LinearizationCache = new WeakMap>(); - -const getCachedC3Linearization = ( - ownerNodeId: string, - heritageMap: HeritageMap, -): readonly string[] | null => { - let perHmCache = c3LinearizationCache.get(heritageMap); - if (!perHmCache) { - perHmCache = new Map(); - c3LinearizationCache.set(heritageMap, perHmCache); - } - const cached = perHmCache.get(ownerNodeId); - if (cached !== undefined) return cached; - const parentMap = buildParentMapFromHeritage(ownerNodeId, heritageMap); - const result = c3Linearize(ownerNodeId, parentMap, new Map()) ?? null; - perHmCache.set(ownerNodeId, result); - return result; -}; - -/** - * Build a parentMap from HeritageMap for use with c3Linearize. - * Traverses the parent chain starting from startNodeId, collecting all - * parent→children relationships into a Map. - */ -const buildParentMapFromHeritage = ( - startNodeId: string, - heritageMap: HeritageMap, -): Map => { - const parentMap = new Map(); - const visited = new Set(); - const queue = [startNodeId]; - - while (queue.length > 0) { - const nodeId = queue.shift()!; - if (visited.has(nodeId)) continue; - visited.add(nodeId); - const parents = heritageMap.getParents(nodeId); - if (parents.length > 0) { - parentMap.set(nodeId, parents); - for (const p of parents) { - if (!visited.has(p)) queue.push(p); - } - } - } - - return parentMap; -}; - -/** - * Look up a method on an owner class, walking the parent chain via HeritageMap - * when the method isn't found on the direct owner. - * - * Respects the 5 per-language MRO strategies: - * - `first-wins`: BFS ancestor walk, first match wins (default) - * - `leftmost-base`: BFS ancestor walk, leftmost base in declaration order wins (C++); - * HeritageMap preserves insertion order matching source declaration, - * so BFS order is equivalent to leftmost-base semantics - * - `c3`: C3-linearized ancestor order, first match wins (Python) - * - `implements-split`: BFS ancestor walk, first match wins (Java/C#) — - * full ambiguity detection for multiple interface defaults - * is handled by computeMRO at graph level - * - `qualified-syntax`: No auto-resolution (Rust) — returns undefined - * - * Delegates to mro-processor.ts c3Linearize for C3 strategy. - * - * @internal Exported only to enable unit testing in isolation. The proper - * entry point for callers outside this module is {@link resolveMethodByOwner}, - * which handles receiver-type resolution before delegating here. - */ -export const lookupMethodByOwnerWithMRO = ( - ownerNodeId: string, - methodName: string, - heritageMap: HeritageMap, - symbols: SymbolTable, - language: SupportedLanguages, - argCount?: number, -): SymbolDefinition | undefined => { - // Direct lookup first (child override — no walk needed). - // argCount is threaded through so arity-differing overloads on the direct - // owner can be disambiguated before the MRO walk starts. - const direct = symbols.lookupMethodByOwner(ownerNodeId, methodName, argCount); - if (direct) return direct; - - const strategy = getProvider(language).mroStrategy; - - // Rust: requires qualified syntax (::method), no auto-resolution - if (strategy === 'qualified-syntax') return undefined; - - // Determine ancestor walk order based on MRO strategy. - // readonly to accept the cached (frozen) c3 linearization without copying. - let ancestors: readonly string[]; - if (strategy === 'c3') { - // Delegate to mro-processor.ts C3 linearization (memoized per HeritageMap - // so repeated calls for the same owner within an ingestion run reuse the - // linearization instead of rebuilding the parent map and re-running C3). - // c3Linearize returns ancestors only (excludes the owner itself), - // matching heritageMap.getAncestors() semantics. - const c3Result = getCachedC3Linearization(ownerNodeId, heritageMap); - // Fall back to BFS order if C3 fails (cyclic or inconsistent hierarchy). - // Note: BFS order may not preserve Python MRO semantics in these edge - // cases, but cyclic/inconsistent hierarchies are invalid in Python anyway. - ancestors = c3Result ?? heritageMap.getAncestors(ownerNodeId); - } else { - // first-wins, leftmost-base, implements-split: BFS order via HeritageMap - ancestors = heritageMap.getAncestors(ownerNodeId); - } - - // Walk ancestors in MRO order — first match wins. - // argCount narrows overloaded ancestors the same way as the direct lookup. - for (const ancestorId of ancestors) { - const method = symbols.lookupMethodByOwner(ancestorId, methodName, argCount); - if (method) return method; - } - - return undefined; -}; - /** * Create a deduplicated ACCESSES edge emitter for a single source node. * Each (sourceId, fieldNodeId) pair is emitted at most once per source. @@ -2285,7 +2647,7 @@ const walkMixedChain = ( continue; } } - // Fallback: fuzzy resolution via resolveCallTarget (cross-file, inherited, etc.) + // Fallback: resolve via resolveCallTarget dispatcher (delegates to resolveMemberCall) const resolved = resolveCallTarget( { calledName: step.name, callForm: 'member', receiverTypeName: currentType }, filePath, @@ -2319,6 +2681,12 @@ const walkMixedChain = ( /** * Fast path: resolve pre-extracted call sites from workers. * No AST parsing — workers already extracted calledName + sourceId. + * + * @param bindingAccumulator Phase 9: optional accumulator carrying file-scope + * TypeEnv bindings from all worker-processed files. When the SymbolTable has + * no return type for a cross-file callee, `verifyConstructorBindings` falls + * back to the accumulator via `namedImportMap` to bind the variable to the + * callee's resolved type (e.g. `var x = getUser()` → `x: User`). */ export const processCallsFromExtracted = async ( graph: KnowledgeGraph, @@ -2327,6 +2695,7 @@ export const processCallsFromExtracted = async ( onProgress?: (current: number, total: number) => void, constructorBindings?: FileConstructorBindings[], heritageMap?: HeritageMap, + bindingAccumulator?: BindingAccumulator, ) => { // Scope-aware receiver types: keyed by filePath → "funcName\0varName" → typeName. // The scope dimension prevents collisions when two functions in the same file @@ -2334,7 +2703,13 @@ export const processCallsFromExtracted = async ( const fileReceiverTypes = new Map(); if (constructorBindings) { for (const { filePath, bindings } of constructorBindings) { - const verified = verifyConstructorBindings(bindings, filePath, ctx, graph); + const verified = verifyConstructorBindings( + bindings, + filePath, + ctx, + graph, + bindingAccumulator, + ); if (verified.size > 0) { fileReceiverTypes.set(filePath, buildReceiverTypeIndex(verified)); } @@ -2532,12 +2907,19 @@ export const processAssignmentsFromExtracted = ( assignments: ExtractedAssignment[], ctx: ResolutionContext, constructorBindings?: FileConstructorBindings[], + bindingAccumulator?: BindingAccumulator, ): void => { // Build per-file receiver type indexes from verified constructor bindings const fileReceiverTypes = new Map(); if (constructorBindings) { for (const { filePath, bindings } of constructorBindings) { - const verified = verifyConstructorBindings(bindings, filePath, ctx, graph); + const verified = verifyConstructorBindings( + bindings, + filePath, + ctx, + graph, + bindingAccumulator, + ); if (verified.size > 0) { fileReceiverTypes.set(filePath, buildReceiverTypeIndex(verified)); } diff --git a/gitnexus/src/core/ingestion/call-routing.ts b/gitnexus/src/core/ingestion/call-routing.ts index 3922c0c89..f9e60df10 100644 --- a/gitnexus/src/core/ingestion/call-routing.ts +++ b/gitnexus/src/core/ingestion/call-routing.ts @@ -1,10 +1,14 @@ /** * Shared Ruby call routing logic. * - * Ruby expresses imports, heritage (mixins), and property definitions as - * method calls rather than syntax-level constructs. This module provides a - * routing function used by the CLI call-processor, CLI parse-worker, and - * the web call-processor so that the classification logic lives in one place. + * Ruby expresses imports and property definitions as method calls rather + * than syntax-level constructs. This module provides a routing function + * used by the CLI call-processor, CLI parse-worker, and the web + * call-processor so that the classification logic lives in one place. + * + * Heritage (mixins: include/extend/prepend) was previously routed here + * but is now handled by heritageExtractor.extractFromCall before the + * call router runs. The router still returns 'skip' for these calls. * * NOTE: This file is intentionally duplicated in gitnexus-web/ because the * two packages have separate build targets (Node native vs WASM/browser). @@ -30,17 +34,10 @@ export type CallRouter = (calledName: string, callNode: SyntaxNode) => CallRouti export type RubyCallRouting = | { kind: 'import'; importPath: string; isRelative: boolean } - | { kind: 'heritage'; items: RubyHeritageItem[] } | { kind: 'properties'; items: RubyPropertyItem[] } | { kind: 'call' } | { kind: 'skip' }; -export interface RubyHeritageItem { - enclosingClass: string; - mixinName: string; - heritageKind: 'include' | 'extend' | 'prepend'; -} - export type RubyAccessorType = 'attr_accessor' | 'attr_reader' | 'attr_writer'; export interface RubyPropertyItem { @@ -56,9 +53,6 @@ export interface RubyPropertyItem { const CALL_RESULT: RubyCallRouting = { kind: 'call' }; const SKIP_RESULT: RubyCallRouting = { kind: 'skip' }; -/** Max depth for parent-walking loops to prevent pathological AST traversals */ -const MAX_PARENT_DEPTH = 50; - // ── Routing function ──────────────────────────────────────────────────────── /** @@ -88,35 +82,12 @@ export function routeRubyCall(calledName: string, callNode: SyntaxNode): RubyCal return { kind: 'import', importPath, isRelative }; } - // ── include / extend / prepend → heritage (mixin) ────────────────────── + // ── include / extend / prepend — heritage (now handled by heritageExtractor) ─ + // Call-based heritage is intercepted by heritageExtractor.extractFromCall + // before the call router runs. Return SKIP_RESULT so these calls don't + // fall through to normal call processing. if (calledName === 'include' || calledName === 'extend' || calledName === 'prepend') { - let enclosingClass: string | null = null; - let current = callNode.parent; - let depth = 0; - while (current && ++depth <= MAX_PARENT_DEPTH) { - if (current.type === 'class' || current.type === 'module') { - const nameNode = current.childForFieldName?.('name'); - if (nameNode) { - enclosingClass = nameNode.text; - break; - } - } - current = current.parent; - } - if (!enclosingClass) return SKIP_RESULT; - - const items: RubyHeritageItem[] = []; - const argList = callNode.childForFieldName?.('arguments'); - for (const arg of argList?.children ?? []) { - if (arg.type === 'constant' || arg.type === 'scope_resolution') { - items.push({ - enclosingClass, - mixinName: arg.text, - heritageKind: calledName as 'include' | 'extend' | 'prepend', - }); - } - } - return items.length > 0 ? { kind: 'heritage', items } : SKIP_RESULT; + return SKIP_RESULT; } // ── attr_accessor / attr_reader / attr_writer → property definitions ─── diff --git a/gitnexus/src/core/ingestion/call-sites/extract-language-call-site.ts b/gitnexus/src/core/ingestion/call-sites/extract-language-call-site.ts deleted file mode 100644 index feed2cd70..000000000 --- a/gitnexus/src/core/ingestion/call-sites/extract-language-call-site.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** Non-generic @call shapes → { calledName, callForm, receiverName? } (used from call-processor / parse-worker). */ - -import { SupportedLanguages } from '../../../config/supported-languages.js'; -import type { SyntaxNode } from '../utils/ast-helpers.js'; -import { parseJavaMethodReference } from './java.js'; - -export type ParsedCallSite = { - calledName: string; - callForm: 'free' | 'member' | 'constructor'; - receiverName?: string; -}; - -/** Non-null → seed replaces @call.name; null → use @call.name + inferCallForm / extractReceiverName. */ -export function extractParsedCallSite( - language: SupportedLanguages, - callNode: SyntaxNode, -): ParsedCallSite | null { - switch (language) { - case SupportedLanguages.Java: - if (callNode.type === 'method_reference') { - const parsed = parseJavaMethodReference(callNode); - if (!parsed) return null; - return { - calledName: parsed.calledName, - callForm: parsed.callForm, - ...(parsed.receiverName !== undefined ? { receiverName: parsed.receiverName } : {}), - }; - } - return null; - default: - return null; - } -} diff --git a/gitnexus/src/core/ingestion/call-sites/java.ts b/gitnexus/src/core/ingestion/call-sites/java.ts deleted file mode 100644 index e22c71cca..000000000 --- a/gitnexus/src/core/ingestion/call-sites/java.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** Java `method_reference` (`::`) nodes (tree-sitter-java). `super::` still lacks TypeEnv receiver typing. */ - -import type { SyntaxNode } from '../utils/ast-helpers.js'; - -export type ParsedJavaMethodReference = { - calledName: string; - callForm: 'member' | 'constructor'; - receiverName?: string; -}; - -/** Parse `expr::method`, `Type::new`, `this::m`, `super::m`. */ -export const parseJavaMethodReference = ( - callNode: SyntaxNode, -): ParsedJavaMethodReference | null => { - if (callNode.type !== 'method_reference') return null; - - const recv = callNode.namedChild(0); - if (!recv) return null; - - for (const c of callNode.children) { - if (c.type === 'new') { - if (recv.type !== 'identifier') return null; - return { calledName: recv.text, callForm: 'constructor' }; - } - } - - const rhs = callNode.child(callNode.childCount - 1); - if (!rhs || rhs.type !== 'identifier') return null; - const methodName = rhs.text; - - if (recv.type === 'identifier') { - return { calledName: methodName, callForm: 'member', receiverName: recv.text }; - } - if (recv.type === 'this') { - return { calledName: methodName, callForm: 'member', receiverName: 'this' }; - } - if (recv.type === 'super') { - return { calledName: methodName, callForm: 'member', receiverName: 'super' }; - } - return null; -}; diff --git a/gitnexus/src/core/ingestion/call-types.ts b/gitnexus/src/core/ingestion/call-types.ts new file mode 100644 index 000000000..b42ecbf16 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-types.ts @@ -0,0 +1,177 @@ +// gitnexus/src/core/ingestion/call-types.ts + +/** + * Types for the language-agnostic call extraction pipeline. + * + * Mirrors method-types.ts / field-types.ts: defines the domain interfaces + * consumed by createCallExtractor() and the per-language configs. + */ + +import type { SupportedLanguages } from 'gitnexus-shared'; +import type { SyntaxNode } from './utils/ast-helpers.js'; +import type { MixedChainStep } from './utils/call-analysis.js'; + +// --------------------------------------------------------------------------- +// Extracted result +// --------------------------------------------------------------------------- + +/** + * Per-node call extraction result. The parse worker enriches this with + * file-level context (filePath, sourceId, TypeEnv lookups, arg types) to + * produce the final `ExtractedCall` that enters the resolution pipeline. + */ +export interface ExtractedCallSite { + calledName: string; + callForm?: 'free' | 'member' | 'constructor'; + receiverName?: string; + argCount?: number; + /** Unified mixed chain for complex receivers (field + call chains). */ + receiverMixedChain?: MixedChainStep[]; + /** When true, the type-as-receiver heuristic applies: if receiverName + * starts with an uppercase letter and has no TypeEnv binding, treat it + * as a type name (e.g. Java `User::getName`). */ + typeAsReceiverHeuristic?: boolean; +} + +// --------------------------------------------------------------------------- +// Extractor interface (produced by createCallExtractor) +// --------------------------------------------------------------------------- + +export interface CallExtractor { + readonly language: SupportedLanguages; + /** + * Extract a call site from captured AST nodes. + * + * @param callNode The @call capture (call_expression, method_invocation, …) + * @param callNameNode The @call.name capture (identifier inside the call). + * May be undefined when the call shape has no name capture + * (e.g. Java method_reference via `::`). + * @returns Extracted call site, or null when no call can be derived. + */ + extract(callNode: SyntaxNode, callNameNode: SyntaxNode | undefined): ExtractedCallSite | null; +} + +// --------------------------------------------------------------------------- +// Config interface (one per language / language group) +// --------------------------------------------------------------------------- + +export interface CallExtractionConfig { + language: SupportedLanguages; + + /** + * Language-specific call site extraction. Called **before** the generic + * path. If it returns non-null, the generic `inferCallForm` / + * `extractReceiverName` path is skipped entirely. + * + * Use this for call shapes that don't follow the standard `@call` / + * `@call.name` pattern (e.g. Java `method_reference` via `::`). + */ + extractLanguageCallSite?: (callNode: SyntaxNode) => ExtractedCallSite | null; + + /** + * Whether the type-as-receiver heuristic applies for this language. + * When true and the receiver name starts with an uppercase letter, + * the receiver is treated as a type name when no TypeEnv binding exists. + * + * Applies to JVM and C# languages where `Type.method()` and `Type::method` + * are common patterns. + */ + typeAsReceiverHeuristic?: boolean; +} + +// --------------------------------------------------------------------------- +// Call-resolution DAG types +// --------------------------------------------------------------------------- +// +// The call-resolution pipeline is a typed DAG: +// +// extract-call ──▶ classify-form ──▶ infer-receiver ──▶ select-dispatch ──▶ resolve-target ──▶ emit-edge +// +// Provider hooks plug in at infer-receiver and select-dispatch; shared stages +// stay language-agnostic. Stages 1-2 run in the parse worker; stages 3-6 run +// on the main thread. DAG-internal types below are main-thread-only and never +// serialize to the graph. + +/** + * DAG stage 3 output: call record with receiver type and source discriminant. + * + * `receiverTypeName` is resolved via TypeEnv → constructor-map → class-as-receiver → + * mixed-chain, or synthesized by `inferImplicitReceiver`. `receiverSource` tags + * which path won and drives MRO strategy selection in stage 4. + * + * Invariants: + * - `receiverSource` MUST match how `receiverTypeName` was resolved; every + * discriminant must have a live reader and writer. + * - `hint` is opaque to shared stages; only the same provider's `selectDispatch` reads it. + * + * @see language-provider.ts § inferImplicitReceiver, selectDispatch + */ +export interface ReceiverEnriched { + readonly calledName: string; + readonly callForm: 'free' | 'member' | 'constructor' | undefined; + readonly receiverName: string | undefined; + readonly receiverTypeName: string | undefined; + readonly receiverSource: + | 'none' + | 'typed-binding' + | 'constructor-map' + | 'class-as-receiver' + | 'mixed-chain' + | 'implicit-self'; + /** Free-form hint from the provider hook; opaque to shared stages. */ + readonly hint?: string; +} + +/** + * Provider hook output for `LanguageProvider.inferImplicitReceiver` (DAG stage 3). + * + * Overlay applied to `ReceiverEnriched` when an implicit receiver is synthesized. + * Ruby example: bare `serialize` inside `Account#call_serialize` → + * `{ callForm: 'member', receiverName: 'self', receiverTypeName: 'Account', + * receiverSource: 'implicit-self', hint: 'instance' }` + * + * Invariants: + * - `receiverSource` is always `'implicit-self'` — the only variant this type produces. + * - `callForm` is always `'member'` — the rewrite converts bare-call to method invocation. + * - `hint` is opaque to shared stages; consumed by the same language's `selectDispatch`. + */ +export interface ImplicitReceiverOverride { + readonly callForm: 'free' | 'member' | 'constructor'; + readonly receiverName: string; + readonly receiverTypeName: string; + readonly receiverSource: Extract; + /** Free-form language tag (e.g. Ruby sets 'singleton' for `def self.foo` + * method bodies). Consumed by the same language's `selectDispatch` hook. */ + readonly hint?: string; +} + +/** + * DAG stage 4 output: dispatch strategy for resolving the target method. + * + * Encodes which resolver branch to try first and an optional fallback. + * Stage 5 delegates to `resolveMemberCall`, `resolveFreeCall`, or + * `resolveStaticCall` based on `primary`. + * + * - `primary`: `'owner-scoped'` = MRO walk, `'free'` = arity-tiered global lookup, + * `'constructor'` = type instantiation. + * - `fallback`: Only `'free-arity-narrowed'` exists; used by Ruby implicit-self + * to degrade to arity-tiered free lookup when the MRO walk misses. + * - `ancestryView`: Ruby `'ruby-mixin'` only. `'singleton'` walks extend providers + * only; a miss NEVER falls through to file-scoped lookup (enforced in + * resolveCallTarget). `'instance'` is the default. + * + * Common patterns: + * - `{primary: 'constructor'}` — constructor call + * - `{primary: 'owner-scoped'}` — member call with known type + * - `{primary: 'owner-scoped', fallback: 'free-arity-narrowed', ancestryView: 'instance'}` — Ruby implicit-self + * - `{primary: 'owner-scoped', ancestryView: 'singleton'}` — Ruby class-method call + * + * @see language-provider.ts § selectDispatch + * @see call-processor.ts § defaultDispatchDecision, resolveCallTarget + */ +export interface DispatchDecision { + readonly primary: 'owner-scoped' | 'free' | 'constructor'; + readonly fallback?: 'free-arity-narrowed'; + readonly ancestryView?: 'instance' | 'singleton'; + readonly hint?: string; +} diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts b/gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts new file mode 100644 index 000000000..fcc1a22bf --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts @@ -0,0 +1,15 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const cClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.C, + typeDeclarationNodes: ['struct_specifier', 'enum_specifier'], +}; + +export const cppClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.CPlusPlus, + typeDeclarationNodes: ['class_specifier', 'struct_specifier', 'enum_specifier'], + ancestorScopeNodeTypes: ['namespace_definition', 'class_specifier', 'struct_specifier'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts new file mode 100644 index 000000000..59b7be617 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts @@ -0,0 +1,24 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const csharpClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.CSharp, + typeDeclarationNodes: [ + 'class_declaration', + 'interface_declaration', + 'struct_declaration', + 'enum_declaration', + 'record_declaration', + ], + fileScopeNodeTypes: ['file_scoped_namespace_declaration'], + ancestorScopeNodeTypes: [ + 'namespace_declaration', + 'class_declaration', + 'interface_declaration', + 'struct_declaration', + 'enum_declaration', + 'record_declaration', + ], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/class-extractors/configs/dart.ts new file mode 100644 index 000000000..c46c05533 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/dart.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/dart.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const dartClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Dart, + typeDeclarationNodes: ['class_definition', 'extension_declaration', 'enum_declaration'], + ancestorScopeNodeTypes: ['class_definition', 'extension_declaration', 'enum_declaration'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/go.ts b/gitnexus/src/core/ingestion/class-extractors/configs/go.ts new file mode 100644 index 000000000..58dade9fa --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/go.ts @@ -0,0 +1,21 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/go.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const goClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Go, + typeDeclarationNodes: ['type_declaration'], + fileScopeNodeTypes: ['package_clause'], + extractName(node) { + const typeSpec = node.namedChildren.find((child) => child.type === 'type_spec'); + return typeSpec?.childForFieldName('name')?.text; + }, + extractType(node) { + const typeSpec = node.namedChildren.find((child) => child.type === 'type_spec'); + const typeNode = typeSpec?.childForFieldName('type'); + if (typeNode?.type === 'struct_type') return 'Struct'; + if (typeNode?.type === 'interface_type') return 'Interface'; + return undefined; + }, +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts new file mode 100644 index 000000000..fbd22f545 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts @@ -0,0 +1,40 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +// --------------------------------------------------------------------------- +// Java +// --------------------------------------------------------------------------- + +export const javaClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Java, + typeDeclarationNodes: [ + 'class_declaration', + 'interface_declaration', + 'enum_declaration', + 'record_declaration', + ], + fileScopeNodeTypes: ['package_declaration'], + ancestorScopeNodeTypes: [ + 'class_declaration', + 'interface_declaration', + 'enum_declaration', + 'record_declaration', + ], +}; + +// --------------------------------------------------------------------------- +// Kotlin +// --------------------------------------------------------------------------- + +export const kotlinClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Kotlin, + typeDeclarationNodes: ['class_declaration', 'object_declaration', 'companion_object'], + fileScopeNodeTypes: ['package_header'], + ancestorScopeNodeTypes: ['class_declaration', 'object_declaration', 'companion_object'], + extractType(node) { + if (node.type !== 'class_declaration') return undefined; + return node.children.some((child) => child?.text === 'interface') ? 'Interface' : 'Class'; + }, +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/php.ts b/gitnexus/src/core/ingestion/class-extractors/configs/php.ts new file mode 100644 index 000000000..850b415b4 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/php.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/php.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const phpClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.PHP, + typeDeclarationNodes: ['class_declaration', 'interface_declaration', 'enum_declaration'], + ancestorScopeNodeTypes: ['namespace_definition'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/python.ts b/gitnexus/src/core/ingestion/class-extractors/configs/python.ts new file mode 100644 index 000000000..42761468e --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/python.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/python.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const pythonClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Python, + typeDeclarationNodes: ['class_definition'], + ancestorScopeNodeTypes: ['class_definition'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts b/gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts new file mode 100644 index 000000000..2c4c711bd --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const rubyClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Ruby, + typeDeclarationNodes: ['class'], + ancestorScopeNodeTypes: ['module', 'class'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/rust.ts b/gitnexus/src/core/ingestion/class-extractors/configs/rust.ts new file mode 100644 index 000000000..7f3873802 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/rust.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/rust.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const rustClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Rust, + typeDeclarationNodes: ['struct_item', 'enum_item'], + ancestorScopeNodeTypes: ['mod_item', 'struct_item', 'enum_item'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/class-extractors/configs/swift.ts new file mode 100644 index 000000000..713a02496 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/swift.ts @@ -0,0 +1,17 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/swift.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const swiftClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Swift, + typeDeclarationNodes: ['class_declaration', 'protocol_declaration'], + ancestorScopeNodeTypes: ['class_declaration', 'protocol_declaration'], + extractType(node) { + if (node.type === 'protocol_declaration') return 'Interface'; + if (node.type !== 'class_declaration') return undefined; + if (node.children.some((child) => child?.text === 'struct')) return 'Struct'; + if (node.children.some((child) => child?.text === 'enum')) return 'Enum'; + return 'Class'; + }, +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts new file mode 100644 index 000000000..b2262432d --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts @@ -0,0 +1,34 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +const shared: Omit = { + typeDeclarationNodes: [ + 'class_declaration', + 'abstract_class_declaration', + 'interface_declaration', + 'enum_declaration', + ], + ancestorScopeNodeTypes: [ + 'class_declaration', + 'abstract_class_declaration', + 'interface_declaration', + 'enum_declaration', + ], +}; + +export const typescriptClassConfig: ClassExtractionConfig = { + ...shared, + language: SupportedLanguages.TypeScript, +}; + +export const javascriptClassConfig: ClassExtractionConfig = { + ...shared, + language: SupportedLanguages.JavaScript, +}; + +export const vueClassConfig: ClassExtractionConfig = { + ...shared, + language: SupportedLanguages.Vue, +}; diff --git a/gitnexus/src/core/ingestion/cobol-processor.ts b/gitnexus/src/core/ingestion/cobol-processor.ts index 1174b302d..2e0551770 100644 --- a/gitnexus/src/core/ingestion/cobol-processor.ts +++ b/gitnexus/src/core/ingestion/cobol-processor.ts @@ -92,7 +92,7 @@ function isCopybook(filePath: string): boolean { export const processCobol = ( graph: KnowledgeGraph, files: CobolFile[], - allPathSet: Set, + allPathSet: ReadonlySet, ): CobolProcessResult => { const result: CobolProcessResult = { programs: 0, diff --git a/gitnexus/src/core/ingestion/field-types.ts b/gitnexus/src/core/ingestion/field-types.ts index 5e89d34bb..7867ae8a6 100644 --- a/gitnexus/src/core/ingestion/field-types.ts +++ b/gitnexus/src/core/ingestion/field-types.ts @@ -1,7 +1,7 @@ // gitnexus/src/core/ingestion/field-types.ts import type { TypeEnvironment } from './type-env.js'; -import type { SymbolTable } from './symbol-table.js'; +import type { SymbolTableReader } from './model/index.js'; import { SupportedLanguages } from 'gitnexus-shared'; /** @@ -57,7 +57,7 @@ export interface FieldExtractorContext { /** Type environment for resolution */ typeEnv: TypeEnvironment; /** Symbol table for FQN lookups */ - symbolTable: SymbolTable; + symbolTable: SymbolTableReader; /** Current file path */ filePath: string; /** Language ID */ diff --git a/gitnexus/src/core/ingestion/heritage-extractors/configs/go.ts b/gitnexus/src/core/ingestion/heritage-extractors/configs/go.ts new file mode 100644 index 000000000..63768172d --- /dev/null +++ b/gitnexus/src/core/ingestion/heritage-extractors/configs/go.ts @@ -0,0 +1,24 @@ +// gitnexus/src/core/ingestion/heritage-extractors/configs/go.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { HeritageExtractionConfig } from '../../heritage-types.js'; + +/** + * Go heritage extraction config. + * + * Go struct embedding: the tree-sitter query matches ALL field_declarations + * with type_identifier, but only anonymous fields (no name) are embedded. + * Named fields like `Breed string` also match — skip them. + * + * The shouldSkipExtends hook checks if the extends node's parent is a + * field_declaration with a named field child, indicating a regular + * (non-embedded) field that should not produce a heritage record. + */ +export const goHeritageConfig: HeritageExtractionConfig = { + language: SupportedLanguages.Go, + + shouldSkipExtends(extendsNode) { + const fieldDecl = extendsNode.parent; + return fieldDecl?.type === 'field_declaration' && fieldDecl.childForFieldName?.('name') != null; + }, +}; diff --git a/gitnexus/src/core/ingestion/heritage-extractors/configs/ruby.ts b/gitnexus/src/core/ingestion/heritage-extractors/configs/ruby.ts new file mode 100644 index 000000000..41bca1ff2 --- /dev/null +++ b/gitnexus/src/core/ingestion/heritage-extractors/configs/ruby.ts @@ -0,0 +1,73 @@ +// gitnexus/src/core/ingestion/heritage-extractors/configs/ruby.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { HeritageExtractionConfig, HeritageInfo } from '../../heritage-types.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; + +/** + * Maximum parent depth for enclosing class/module walk. + * Prevents runaway walks on malformed/deeply-nested ASTs. + */ +const MAX_PARENT_DEPTH = 50; + +/** + * Walk up the AST from a call node to find the enclosing class or module name. + * Ruby include/extend/prepend calls must be inside a class or module body. + */ +function findEnclosingClassName(callNode: SyntaxNode): string | null { + let current = callNode.parent; + let depth = 0; + while (current && ++depth <= MAX_PARENT_DEPTH) { + if (current.type === 'class' || current.type === 'module') { + const nameNode = current.childForFieldName?.('name'); + if (nameNode) return nameNode.text; + } + current = current.parent; + } + return null; +} + +/** Ruby heritage call names that express mixin inclusion. */ +const RUBY_HERITAGE_CALL_NAMES: ReadonlySet = new Set(['include', 'extend', 'prepend']); + +/** + * Ruby heritage extraction config. + * + * Ruby expresses inheritance in two ways, and only one of them has + * dedicated tree-sitter heritage captures: + * + * 1. Class inheritance (`class A < B`) produces standard + * `@heritage.extends` captures and flows through the generic + * capture-based `extract` hook (not defined here — the factory + * handles it). + * 2. Mixin calls (`include`/`extend`/`prepend`) have no dedicated + * heritage captures; they surface as ordinary call sites. The + * `callBasedHeritage` hook below intercepts them before the call + * router, absorbing the mixin routing logic that previously lived + * in call-routing.ts (routeRubyCall). + */ +export const rubyHeritageConfig: HeritageExtractionConfig = { + language: SupportedLanguages.Ruby, + + callBasedHeritage: { + callNames: RUBY_HERITAGE_CALL_NAMES, + + extract(calledName, callNode, _filePath): HeritageInfo[] { + const enclosingClass = findEnclosingClassName(callNode); + if (!enclosingClass) return []; + + const results: HeritageInfo[] = []; + const argList = callNode.childForFieldName?.('arguments'); + for (const arg of argList?.children ?? []) { + if (arg.type === 'constant' || arg.type === 'scope_resolution') { + results.push({ + className: enclosingClass, + parentName: arg.text, + kind: calledName, // 'include' | 'extend' | 'prepend' + }); + } + } + return results; + }, + }, +}; diff --git a/gitnexus/src/core/ingestion/heritage-extractors/generic.ts b/gitnexus/src/core/ingestion/heritage-extractors/generic.ts new file mode 100644 index 000000000..39c3d34c8 --- /dev/null +++ b/gitnexus/src/core/ingestion/heritage-extractors/generic.ts @@ -0,0 +1,84 @@ +// gitnexus/src/core/ingestion/heritage-extractors/generic.ts + +/** + * Generic table-driven heritage extractor factory. + * + * Follows the same config+factory pattern as method-extractors/generic.ts, + * field-extractors/generic.ts, call-extractors/generic.ts, and + * variable-extractors/generic.ts. + * + * Languages with custom extraction hooks (Go: shouldSkipExtends, Ruby: + * callBasedHeritage) pass a full HeritageExtractionConfig. Languages + * that use the default capture-based extraction can pass just the + * SupportedLanguages enum value — no per-language config file needed. + */ + +import type { SupportedLanguages } from 'gitnexus-shared'; +import type { CaptureMap } from '../language-provider.js'; +import type { + HeritageExtractionConfig, + HeritageExtractor, + HeritageExtractorContext, + HeritageInfo, +} from '../heritage-types.js'; +import type { SyntaxNode } from '../utils/ast-helpers.js'; + +/** + * Create a HeritageExtractor from a declarative config or a language enum. + * + * When a full HeritageExtractionConfig is provided, custom hooks + * (shouldSkipExtends, callBasedHeritage) drive the extraction. + * When only a SupportedLanguages value is provided, the factory produces + * a default extractor that handles the standard @heritage.* captures. + */ +export function createHeritageExtractor( + config: HeritageExtractionConfig | SupportedLanguages, +): HeritageExtractor { + const actualConfig: HeritageExtractionConfig = + typeof config === 'string' ? { language: config } : config; + const callNameSet = actualConfig.callBasedHeritage?.callNames; + + return { + language: actualConfig.language, + + extract(captureMap: CaptureMap, context: HeritageExtractorContext): HeritageInfo[] { + const classNode = captureMap['heritage.class']; + if (!classNode) return []; + + const className = classNode.text; + const results: HeritageInfo[] = []; + + const extendsNode = captureMap['heritage.extends']; + if (extendsNode) { + if (!actualConfig.shouldSkipExtends?.(extendsNode)) { + results.push({ className, parentName: extendsNode.text, kind: 'extends' }); + } + } + + const implementsNode = captureMap['heritage.implements']; + if (implementsNode) { + results.push({ className, parentName: implementsNode.text, kind: 'implements' }); + } + + const traitNode = captureMap['heritage.trait']; + if (traitNode) { + results.push({ className, parentName: traitNode.text, kind: 'trait-impl' }); + } + + return results; + }, + + ...(callNameSet + ? { + extractFromCall( + calledName: string, + callNode: SyntaxNode, + context: HeritageExtractorContext, + ): HeritageInfo[] | null { + if (!callNameSet.has(calledName)) return null; + return actualConfig.callBasedHeritage!.extract(calledName, callNode, context.filePath); + }, + } + : {}), + }; +} diff --git a/gitnexus/src/core/ingestion/heritage-map.ts b/gitnexus/src/core/ingestion/heritage-map.ts deleted file mode 100644 index 46d0c2120..000000000 --- a/gitnexus/src/core/ingestion/heritage-map.ts +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Heritage Map - * - * Unified inheritance data structure built from accumulated - * {@link ExtractedHeritage} records **after all chunks complete** (between - * chunk processing and call resolution). Consumes `ExtractedHeritage[]` and - * resolves type names to nodeIds via `lookupClassByName`, NOT graph-edge - * queries. - * - * Combines two previously separate concerns: - * 1. **Parent/ancestor lookup** (MRO-aware method resolution) - * 2. **Implementor lookup** (interface dispatch — which files contain - * classes implementing a given interface) - */ - -import type { ExtractedHeritage } from './workers/parse-worker.js'; -import type { ResolutionContext } from './resolution-context.js'; -import { getLanguageFromFilename } from 'gitnexus-shared'; -import { resolveExtendsType } from './heritage-processor.js'; - -// --------------------------------------------------------------------------- -// Public types -// --------------------------------------------------------------------------- - -/** Maximum ancestor chain depth to prevent runaway traversal. */ -const MAX_ANCESTOR_DEPTH = 32; - -export interface HeritageMap { - /** Direct parents of `childNodeId` (extends + implements + trait-impl). */ - getParents(childNodeId: string): string[]; - /** Full ancestor chain (BFS, bounded depth, cycle-safe). */ - getAncestors(childNodeId: string): string[]; - /** - * File paths of classes that directly implement or extend-as-interface the - * given interface/abstract-class **name**. Replaces the standalone - * `ImplementorMap` — used by interface-dispatch in call resolution. - */ - getImplementorFiles(interfaceName: string): ReadonlySet; -} - -/** Shared empty set returned when no implementors are found. */ -const EMPTY_SET: ReadonlySet = new Set(); - -// --------------------------------------------------------------------------- -// Builder -// --------------------------------------------------------------------------- - -/** - * Build a HeritageMap from accumulated ExtractedHeritage records. - * - * Resolves class/interface/struct/trait names to nodeIds via - * `ctx.symbols.lookupClassByName`. When a name resolves to multiple - * candidates, all are recorded (partial-class / cross-file scenario). - * Unresolvable names are silently skipped — a missing parent is better - * than a wrong edge. - * - * Also builds the implementor index (interface name → implementing file - * paths) that was previously maintained by `buildImplementorMap` in - * call-processor.ts. - */ -export const buildHeritageMap = ( - heritage: readonly ExtractedHeritage[], - ctx: ResolutionContext, -): HeritageMap => { - // childNodeId → Set (Set to deduplicate cross-chunk duplicates) - const directParents = new Map>(); - - // interfaceName → Set (implementor lookup for interface dispatch) - const implementorFiles = new Map>(); - - for (const h of heritage) { - // ── Parent lookup (nodeId-based) ──────────────────────────────── - const childDefs = ctx.symbols.lookupClassByName(h.className); - const parentDefs = ctx.symbols.lookupClassByName(h.parentName); - - if (childDefs.length > 0 && parentDefs.length > 0) { - for (const child of childDefs) { - for (const parent of parentDefs) { - // Skip self-references - if (child.nodeId === parent.nodeId) continue; - - let parents = directParents.get(child.nodeId); - if (!parents) { - parents = new Set(); - directParents.set(child.nodeId, parents); - } - parents.add(parent.nodeId); - } - } - } - - // ── Implementor index (name-based) ────────────────────────────── - // - // Known limitation: Rust `kind: 'trait-impl'` entries are intentionally NOT - // added to the implementor index. Interface dispatch resolution currently - // does not traverse Rust trait objects, so recording them here would - // inflate the index without a consumer. Revisit if/when trait-object - // dispatch is added. - // - // Known limitation: `getImplementorFiles` is keyed by interface **name** - // (string), so two interfaces with the same unqualified name in different - // packages (e.g. `pkgA.IRepository` vs `pkgB.IRepository`) collide. This - // matches the behavior of the prior standalone `ImplementorMap` and is - // not a regression introduced by this consolidation. - let isImpl = false; - if (h.kind === 'implements') { - isImpl = true; - } else if (h.kind === 'extends') { - const lang = getLanguageFromFilename(h.filePath); - if (lang) { - const { type } = resolveExtendsType(h.parentName, h.filePath, ctx, lang); - isImpl = type === 'IMPLEMENTS'; - } - } - if (isImpl) { - let files = implementorFiles.get(h.parentName); - if (!files) { - files = new Set(); - implementorFiles.set(h.parentName, files); - } - files.add(h.filePath); - } - } - - // --- Public API --------------------------------------------------- - - const getParents = (childNodeId: string): string[] => { - const parents = directParents.get(childNodeId); - return parents ? [...parents] : []; - }; - - const getAncestors = (childNodeId: string): string[] => { - const result: string[] = []; - const visited = new Set(); - visited.add(childNodeId); // prevent cycles through the start node - - // BFS with bounded depth - let frontier = getParents(childNodeId); - let depth = 0; - - while (frontier.length > 0 && depth < MAX_ANCESTOR_DEPTH) { - const nextFrontier: string[] = []; - for (const parentId of frontier) { - if (visited.has(parentId)) continue; - visited.add(parentId); - result.push(parentId); - // Expand parent's own parents for next level - const grandparents = directParents.get(parentId); - if (grandparents) { - for (const gp of grandparents) { - if (!visited.has(gp)) nextFrontier.push(gp); - } - } - } - frontier = nextFrontier; - depth++; - } - - return result; - }; - - const getImplementorFiles = (interfaceName: string): ReadonlySet => { - return implementorFiles.get(interfaceName) ?? EMPTY_SET; - }; - - return { getParents, getAncestors, getImplementorFiles }; -}; diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts index 37c3653a8..6692d1c95 100644 --- a/gitnexus/src/core/ingestion/heritage-processor.ts +++ b/gitnexus/src/core/ingestion/heritage-processor.ts @@ -19,47 +19,35 @@ import { ASTCache } from './ast-cache.js'; import Parser from 'tree-sitter'; import { isLanguageAvailable, loadParser, loadLanguage } from '../tree-sitter/parser-loader.js'; import { generateId } from '../../lib/utils.js'; -import { getLanguageFromFilename } from 'gitnexus-shared'; +import { getLanguageFromFilename, type NodeLabel, type SupportedLanguages } from 'gitnexus-shared'; import { isVerboseIngestionEnabled } from './utils/verbose.js'; import { yieldToEventLoop } from './utils/event-loop.js'; -import { SupportedLanguages } from 'gitnexus-shared'; import { getProvider } from './languages/index.js'; import { getTreeSitterBufferSize } from './constants.js'; -import type { ExtractedHeritage } from './workers/parse-worker.js'; -import type { ResolutionContext } from './resolution-context.js'; -import { TIER_CONFIDENCE } from './resolution-context.js'; +import type { + ExtractedHeritage, + HeritageResolutionStrategy, + HeritageStrategyLookup, +} from './model/heritage-map.js'; +import { resolveExtendsType } from './model/heritage-map.js'; +import type { ResolutionContext } from './model/resolution-context.js'; +import { TIER_CONFIDENCE } from './model/resolution-context.js'; +import type { HeritageInfo } from './heritage-types.js'; /** - * Determine whether a heritage.extends capture is actually an IMPLEMENTS relationship. - * Uses the symbol table first (authoritative — Tier 1); falls back to provider-defined - * heuristics for external symbols not present in the graph: - * - interfaceNamePattern: matched against parent name (e.g., /^I[A-Z]/ for C#/Java) - * - heritageDefaultEdge: 'IMPLEMENTS' causes all unresolved parents to map to IMPLEMENTS - * - All others: default EXTENDS + * Derive the heritage-resolution strategy for a language from its + * `LanguageProvider`. This is the production wiring that `buildHeritageMap` + * and the standalone `resolveExtendsType` call site use — the model layer + * itself stays unaware of the provider registry. */ -/** Exported for implementor-map construction (C#/Java: `extends` rows in base_list may be interfaces). */ -export const resolveExtendsType = ( - parentName: string, - currentFilePath: string, - ctx: ResolutionContext, - language: SupportedLanguages, -): { type: 'EXTENDS' | 'IMPLEMENTS'; idPrefix: string } => { - const resolved = ctx.resolve(parentName, currentFilePath); - if (resolved && resolved.candidates.length > 0) { - const isInterface = resolved.candidates[0].type === 'Interface'; - return isInterface - ? { type: 'IMPLEMENTS', idPrefix: 'Interface' } - : { type: 'EXTENDS', idPrefix: 'Class' }; - } - // Unresolved symbol — fall back to provider-defined heuristics - const provider = getProvider(language); - if (provider.interfaceNamePattern?.test(parentName)) { - return { type: 'IMPLEMENTS', idPrefix: 'Interface' }; - } - if (provider.heritageDefaultEdge === 'IMPLEMENTS') { - return { type: 'IMPLEMENTS', idPrefix: 'Interface' }; - } - return { type: 'EXTENDS', idPrefix: 'Class' }; +export const getHeritageStrategyForLanguage: HeritageStrategyLookup = ( + lang: SupportedLanguages, +): HeritageResolutionStrategy => { + const provider = getProvider(lang); + return { + interfaceNamePattern: provider.interfaceNamePattern, + defaultEdge: provider.heritageDefaultEdge ?? 'EXTENDS', + }; }; /** @@ -96,6 +84,103 @@ const resolveHeritageId = ( }; }; +/** + * Resolve a single HeritageInfo to a graph edge, using the same resolution + * logic as processHeritageFromExtracted. This bridges the heritage extractor + * output format to the graph-resolution side. + */ +const resolveAndAddHeritageEdge = ( + graph: KnowledgeGraph, + item: HeritageInfo, + filePath: string, + language: SupportedLanguages, + ctx: ResolutionContext, +): void => { + if (item.kind === 'extends') { + const { type: relType, idPrefix } = resolveExtendsType( + item.parentName, + filePath, + ctx, + getHeritageStrategyForLanguage(language), + ); + + const child = resolveHeritageId( + item.className, + filePath, + ctx, + 'Class', + `${filePath}:${item.className}`, + ); + const parent = resolveHeritageId(item.parentName, filePath, ctx, idPrefix); + + if (child.id && parent.id && child.id !== parent.id) { + graph.addRelationship({ + id: generateId(relType, `${child.id}->${parent.id}`), + sourceId: child.id, + targetId: parent.id, + type: relType, + confidence: Math.sqrt(child.confidence * parent.confidence), + reason: '', + }); + } + } else if (item.kind === 'implements') { + const cls = resolveHeritageId( + item.className, + filePath, + ctx, + 'Class', + `${filePath}:${item.className}`, + ); + const iface = resolveHeritageId(item.parentName, filePath, ctx, 'Interface'); + + if (cls.id && iface.id) { + graph.addRelationship({ + id: generateId('IMPLEMENTS', `${cls.id}->${iface.id}`), + sourceId: cls.id, + targetId: iface.id, + type: 'IMPLEMENTS', + confidence: Math.sqrt(cls.confidence * iface.confidence), + reason: '', + }); + } + } else if ( + item.kind === 'trait-impl' || + item.kind === 'include' || + item.kind === 'extend' || + item.kind === 'prepend' + ) { + // Fallback label for an unresolved child name. Rust `trait-impl` children + // are structs; Ruby mixin children are classes or modules (Trait). For + // Ruby mixin kinds the common case resolves through the type registry + // post-plan-001, so the fallback only fires for true-unresolved references + // (e.g. mixin inside a singleton_class). `Class` is strictly better than + // `Struct` there because it matches the label the structure phase would + // emit for a Ruby `class` — the dominant shape. Ruby modules that fail + // to resolve still lose their `Trait` label in the synthesized id, but + // they fail to resolve rarely and the tradeoff is documented. + const childFallbackLabel: NodeLabel = item.kind === 'trait-impl' ? 'Struct' : 'Class'; + const strct = resolveHeritageId( + item.className, + filePath, + ctx, + childFallbackLabel, + `${filePath}:${item.className}`, + ); + const trait = resolveHeritageId(item.parentName, filePath, ctx, 'Trait'); + + if (strct.id && trait.id) { + graph.addRelationship({ + id: generateId('IMPLEMENTS', `${strct.id}->${trait.id}:${item.kind}`), + sourceId: strct.id, + targetId: trait.id, + type: 'IMPLEMENTS', + confidence: Math.sqrt(strct.confidence * trait.confidence), + reason: item.kind, + }); + } + } +}; + export const processHeritage = async ( graph: KnowledgeGraph, files: { path: string; content: string }[], @@ -148,112 +233,32 @@ export const processHeritage = async ( let query; let matches; try { - const language = parser.getLanguage(); - query = new Parser.Query(language, queryStr); + const treeSitterLang = parser.getLanguage(); + query = new Parser.Query(treeSitterLang, queryStr); matches = query.matches(tree.rootNode); } catch (queryError) { console.warn(`Heritage query error for ${file.path}:`, queryError); continue; } - // 4. Process heritage matches + // 4. Process heritage matches via provider heritage extractor + const heritageExtractor = provider.heritageExtractor; matches.forEach((match) => { const captureMap: Record = {}; match.captures.forEach((c) => { captureMap[c.name] = c.node; }); - // EXTENDS or IMPLEMENTS: resolve via symbol table for languages where - // the tree-sitter query can't distinguish classes from interfaces (C#, Java) - if (captureMap['heritage.class'] && captureMap['heritage.extends']) { - // Go struct embedding: skip named fields (only anonymous fields are embedded) - const extendsNode = captureMap['heritage.extends']; - const fieldDecl = extendsNode.parent; - if (fieldDecl?.type === 'field_declaration' && fieldDecl.childForFieldName('name')) { - return; // Named field, not struct embedding - } + if (!captureMap['heritage.class']) return; + if (!heritageExtractor) return; - const className = captureMap['heritage.class'].text; - const parentClassName = captureMap['heritage.extends'].text; + const heritageItems = heritageExtractor.extract(captureMap, { + filePath: file.path, + language, + }); - const { type: relType, idPrefix } = resolveExtendsType( - parentClassName, - file.path, - ctx, - language, - ); - - const child = resolveHeritageId( - className, - file.path, - ctx, - 'Class', - `${file.path}:${className}`, - ); - const parent = resolveHeritageId(parentClassName, file.path, ctx, idPrefix); - - if (child.id && parent.id && child.id !== parent.id) { - graph.addRelationship({ - id: generateId(relType, `${child.id}->${parent.id}`), - sourceId: child.id, - targetId: parent.id, - type: relType, - confidence: Math.sqrt(child.confidence * parent.confidence), - reason: '', - }); - } - } - - // IMPLEMENTS: Class implements Interface (TypeScript only) - if (captureMap['heritage.class'] && captureMap['heritage.implements']) { - const className = captureMap['heritage.class'].text; - const interfaceName = captureMap['heritage.implements'].text; - - const cls = resolveHeritageId( - className, - file.path, - ctx, - 'Class', - `${file.path}:${className}`, - ); - const iface = resolveHeritageId(interfaceName, file.path, ctx, 'Interface'); - - if (cls.id && iface.id) { - graph.addRelationship({ - id: generateId('IMPLEMENTS', `${cls.id}->${iface.id}`), - sourceId: cls.id, - targetId: iface.id, - type: 'IMPLEMENTS', - confidence: Math.sqrt(cls.confidence * iface.confidence), - reason: '', - }); - } - } - - // IMPLEMENTS (Rust): impl Trait for Struct - if (captureMap['heritage.trait'] && captureMap['heritage.class']) { - const structName = captureMap['heritage.class'].text; - const traitName = captureMap['heritage.trait'].text; - - const strct = resolveHeritageId( - structName, - file.path, - ctx, - 'Struct', - `${file.path}:${structName}`, - ); - const trait = resolveHeritageId(traitName, file.path, ctx, 'Trait'); - - if (strct.id && trait.id) { - graph.addRelationship({ - id: generateId('IMPLEMENTS', `${strct.id}->${trait.id}`), - sourceId: strct.id, - targetId: trait.id, - type: 'IMPLEMENTS', - confidence: Math.sqrt(strct.confidence * trait.confidence), - reason: 'trait-impl', - }); - } + for (const item of heritageItems) { + resolveAndAddHeritageEdge(graph, item, file.path, language, ctx); } }); @@ -296,7 +301,7 @@ export const processHeritageFromExtracted = async ( h.parentName, h.filePath, ctx, - fileLanguage, + getHeritageStrategyForLanguage(fileLanguage), ); const child = resolveHeritageId( @@ -344,11 +349,15 @@ export const processHeritageFromExtracted = async ( h.kind === 'extend' || h.kind === 'prepend' ) { + // See the per-item call above (processHeritageFromExtractedItem) for + // rationale: `Class` is the correct fallback for Ruby mixin kinds, + // `Struct` stays the Rust `trait-impl` default. + const childFallbackLabel: NodeLabel = h.kind === 'trait-impl' ? 'Struct' : 'Class'; const strct = resolveHeritageId( h.className, h.filePath, ctx, - 'Struct', + childFallbackLabel, `${h.filePath}:${h.className}`, ); const trait = resolveHeritageId(h.parentName, h.filePath, ctx, 'Trait'); @@ -374,6 +383,15 @@ export const processHeritageFromExtracted = async ( * {@link ExtractedHeritage} rows without mutating the graph. Used on the * sequential pipeline path so `buildHeritageMap(..., ctx)` can run before * `processCalls` (worker path defers calls until heritage from all chunks exists). + * + * This prepass extracts BOTH capture-based heritage (`@heritage.*` — extends / + * implements / trait-impl) AND call-based heritage (`@call.name` routed through + * `heritageExtractor.extractFromCall` — Ruby `include` / `extend` / `prepend`). + * Without the second pass, sequential-mode `sequentialHeritageMap` would not + * know about Ruby mixin ancestry before `processCalls` resolves calls against + * it, silently dropping mixed-in methods from the graph. This function stays + * read-only — `processCalls` still owns emission of heritage graph edges via + * its `rubyHeritage` return path. */ export async function extractExtractedHeritageFromFiles( files: { path: string; content: string }[], @@ -413,6 +431,8 @@ export async function extractExtractedHeritageFromFiles( continue; } + const callBasedEnabled = !!provider.heritageExtractor?.extractFromCall; + for (const match of matches) { const captureMap: Record = {}; match.captures.forEach((c) => { @@ -420,35 +440,44 @@ export async function extractExtractedHeritageFromFiles( }); if (captureMap['heritage.class']) { - if (captureMap['heritage.extends']) { - const extendsNode = captureMap['heritage.extends']; - const fieldDecl = extendsNode.parent; - const isNamedField = - fieldDecl?.type === 'field_declaration' && fieldDecl.childForFieldName('name'); - if (!isNamedField) { + if (provider.heritageExtractor) { + const heritageItems = provider.heritageExtractor.extract(captureMap, { + filePath: file.path, + language, + }); + for (const item of heritageItems) { out.push({ filePath: file.path, - className: captureMap['heritage.class'].text, - parentName: captureMap['heritage.extends'].text, - kind: 'extends', + className: item.className, + parentName: item.parentName, + kind: item.kind, }); } } - if (captureMap['heritage.implements']) { - out.push({ - filePath: file.path, - className: captureMap['heritage.class'].text, - parentName: captureMap['heritage.implements'].text, - kind: 'implements', - }); - } - if (captureMap['heritage.trait']) { - out.push({ - filePath: file.path, - className: captureMap['heritage.class'].text, - parentName: captureMap['heritage.trait'].text, - kind: 'trait-impl', - }); + continue; + } + + // Call-based heritage (e.g. Ruby include/extend/prepend). Matches the + // routing the worker path performs inline in parse-worker.ts — see the + // `provider.heritageExtractor?.extractFromCall` branch there. We only + // need call-based records here; other @call captures are consumed by + // processCalls later in the sequential loop. + if (callBasedEnabled && captureMap['call'] && captureMap['call.name']) { + const calledName: string = captureMap['call.name'].text; + const heritageItems = provider.heritageExtractor!.extractFromCall!( + calledName, + captureMap['call'], + { filePath: file.path, language }, + ); + if (heritageItems) { + for (const item of heritageItems) { + out.push({ + filePath: file.path, + className: item.className, + parentName: item.parentName, + kind: item.kind, + }); + } } } } diff --git a/gitnexus/src/core/ingestion/heritage-types.ts b/gitnexus/src/core/ingestion/heritage-types.ts new file mode 100644 index 000000000..82b6bbfad --- /dev/null +++ b/gitnexus/src/core/ingestion/heritage-types.ts @@ -0,0 +1,104 @@ +// gitnexus/src/core/ingestion/heritage-types.ts + +/** + * Types for the language-agnostic heritage extraction pipeline. + * + * Follows the same pattern as call-types.ts, variable-types.ts, and + * method-types.ts: defines the domain interfaces consumed by + * createHeritageExtractor() and the per-language configs. + * + * Heritage extraction handles extends/implements/trait-impl captures from + * tree-sitter queries, plus call-based heritage for languages like Ruby + * (include/extend/prepend expressed as method calls). + */ + +import type { SupportedLanguages } from 'gitnexus-shared'; +import type { SyntaxNode } from './utils/ast-helpers.js'; +import type { CaptureMap } from './language-provider.js'; + +// --------------------------------------------------------------------------- +// Extracted result +// --------------------------------------------------------------------------- + +/** + * Per-match heritage extraction result. The parse worker adds filePath to + * produce the final {@link ExtractedHeritage} that enters the resolution + * pipeline (heritage-processor.ts / heritage-map.ts). + */ +export interface HeritageInfo { + className: string; + parentName: string; + /** 'extends' | 'implements' | 'trait-impl' | 'include' | 'extend' | 'prepend' */ + kind: string; +} + +// --------------------------------------------------------------------------- +// Context +// --------------------------------------------------------------------------- + +export interface HeritageExtractorContext { + filePath: string; + language: SupportedLanguages; +} + +// --------------------------------------------------------------------------- +// Extractor interface (produced by createHeritageExtractor) +// --------------------------------------------------------------------------- + +export interface HeritageExtractor { + readonly language: SupportedLanguages; + + /** + * Extract heritage records from tree-sitter @heritage.* captures. + * + * @param captureMap The capture map from a single tree-sitter match + * @param context File path and language context + * @returns Array of heritage records (may be empty if captures don't match) + */ + extract(captureMap: CaptureMap, context: HeritageExtractorContext): HeritageInfo[]; + + /** + * Extract heritage from a call node (for languages where heritage is + * expressed as method calls, e.g., Ruby include/extend/prepend). + * + * @param calledName The method name (e.g. 'include', 'extend', 'prepend') + * @param callNode The tree-sitter call AST node + * @param context File path and language context + * @returns Heritage records if the call is heritage-related, or null to + * fall through to the call router / normal call handling. + */ + extractFromCall?( + calledName: string, + callNode: SyntaxNode, + context: HeritageExtractorContext, + ): HeritageInfo[] | null; +} + +// --------------------------------------------------------------------------- +// Config interface (one per language / language group) +// --------------------------------------------------------------------------- + +export interface HeritageExtractionConfig { + language: SupportedLanguages; + + /** + * Called for heritage.extends captures. Return true to skip this extends + * capture. Used by Go to skip named struct fields that match the + * field_declaration pattern but are not anonymous embeddings. + * + * Default: never skip (all extends captures are valid). + */ + shouldSkipExtends?: (extendsNode: SyntaxNode) => boolean; + + /** + * Call-based heritage extraction for languages where heritage is expressed + * as method calls (e.g., Ruby include/extend/prepend). + * + * callNames: set of method names that trigger heritage extraction. + * extract: extract heritage items from the call node + method name. + */ + callBasedHeritage?: { + readonly callNames: ReadonlySet; + extract(calledName: string, callNode: SyntaxNode, filePath: string): HeritageInfo[]; + }; +} diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts index 3c52fa7a6..b08482716 100644 --- a/gitnexus/src/core/ingestion/import-processor.ts +++ b/gitnexus/src/core/ingestion/import-processor.ts @@ -12,7 +12,11 @@ import type { ExtractedImport } from './workers/parse-worker.js'; import { getTreeSitterBufferSize } from './constants.js'; import { loadImportConfigs } from './language-config.js'; import { buildSuffixIndex } from './import-resolvers/utils.js'; -import type { ResolutionContext, ModuleAliasMap } from './resolution-context.js'; +import type { + ResolutionContext, + ModuleAliasMap, + NamedImportMap, +} from './model/resolution-context.js'; import type { ImportResult, ResolveCtx, @@ -20,8 +24,7 @@ import type { } from './import-resolvers/types.js'; import type { NamedBinding } from './named-bindings/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; - -const isDev = process.env.NODE_ENV === 'development'; +import { isDev } from './utils/env.js'; // Type: Map> // Stores all files that a given file imports from @@ -61,30 +64,6 @@ function wireImplicitImports( // Avoids expanding every Go package import into N individual ImportMap edges. export type PackageMap = Map>; -// Type: Map> -// Tracks which specific names a file imports from which sources (TS/Python only). -// Used to tighten Tier 2a resolution: `import { User } from './models'` -// means only `User` (not `Repo`) is visible from models.ts via this import. -// Stores both the resolved source path and the original exported name so that -// aliased imports (`import { User as U }`) can resolve U → User in the source file. -export interface NamedImportBinding { - sourcePath: string; - exportedName: string; -} -export type NamedImportMap = Map>; - -/** - * Check if a file path is directly inside a package directory identified by its suffix. - * Used by the symbol resolver for Go and C# directory-level import matching. - */ -export function isFileInPackageDir(filePath: string, dirSuffix: string): boolean { - // Prepend '/' so paths like "internal/auth/service.go" match suffix "/internal/auth/" - const normalized = '/' + filePath.replace(/\\/g, '/'); - if (!normalized.includes(dirSuffix)) return false; - const afterDir = normalized.substring(normalized.indexOf(dirSuffix) + dirSuffix.length); - return !afterDir.includes('/'); -} - // ImportResolutionContext is defined in ./import-resolvers/types.ts — re-exported here for consumers. export function buildImportResolutionContext(allPaths: string[]): ImportResolutionContext { diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/c-cpp.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/c-cpp.ts new file mode 100644 index 000000000..bcfddee0b --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/c-cpp.ts @@ -0,0 +1,18 @@ +/** + * C / C++ import resolution configs. + * Both use standard resolution for #include directives. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; + +export const cImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.C, + strategies: [createStandardStrategy(SupportedLanguages.C)], +}; + +export const cppImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.CPlusPlus, + strategies: [createStandardStrategy(SupportedLanguages.CPlusPlus)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts new file mode 100644 index 000000000..cb5f77145 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts @@ -0,0 +1,36 @@ +/** + * C# import resolution config. + * Namespace-based strategy via .csproj configs, then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveCSharpImportInternal, resolveCSharpNamespaceDir } from '../csharp.js'; + +/** C# namespace-based resolution strategy via .csproj configs. */ +export const csharpNamespaceStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const csharpConfigs = ctx.configs.csharpConfigs; + if (csharpConfigs.length > 0) { + const resolvedFiles = resolveCSharpImportInternal( + rawImportPath, + csharpConfigs, + ctx.normalizedFileList, + ctx.allFileList, + ctx.index, + ); + if (resolvedFiles.length > 1) { + const dirSuffix = resolveCSharpNamespaceDir(rawImportPath, csharpConfigs); + if (dirSuffix) { + return { kind: 'package', files: resolvedFiles, dirSuffix }; + } + } + if (resolvedFiles.length > 0) return { kind: 'files', files: resolvedFiles }; + } + return null; +}; + +export const csharpImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.CSharp, + strategies: [csharpNamespaceStrategy, createStandardStrategy(SupportedLanguages.CSharp)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/dart.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/dart.ts new file mode 100644 index 000000000..06dcea98a --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/dart.ts @@ -0,0 +1,58 @@ +/** + * Dart import resolution config. + * SDK/package strategy first, then relative import strategy (with ./ prepending). + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { resolveStandard } from '../standard.js'; + +/** + * Dart SDK and package: import strategy. + * Absorbs dart: SDK imports and external packages (returns empty result to stop chain). + * Returns null for relative imports to let the next strategy handle them. + */ +export const dartPackageStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + // Strip surrounding quotes from configurable_uri capture + const stripped = rawImportPath.replace(/^['"]|['"]$/g, ''); + + // Skip dart: SDK imports (dart:async, dart:io, etc.) + if (stripped.startsWith('dart:')) return { kind: 'files', files: [] }; + + // Local package: imports → resolve to lib/ + if (stripped.startsWith('package:')) { + const slashIdx = stripped.indexOf('/'); + if (slashIdx === -1) return { kind: 'files', files: [] }; + const relPath = stripped.slice(slashIdx + 1); + const candidates = [`lib/${relPath}`, relPath]; + const files: string[] = []; + for (const candidate of candidates) { + for (const fp of ctx.allFileList) { + if (fp.endsWith('/' + candidate) || fp === candidate) { + files.push(fp); + break; + } + } + if (files.length > 0) break; + } + if (files.length > 0) return { kind: 'files', files }; + return { kind: 'files', files: [] }; // external package + } + + return null; +}; + +/** + * Dart relative import strategy — prepends "./" for bare relative paths, + * then delegates to standard resolution. + */ +export const dartRelativeStrategy: ImportResolverStrategy = (rawImportPath, filePath, ctx) => { + const stripped = rawImportPath.replace(/^['"]|['"]$/g, ''); + const relPath = stripped.startsWith('.') ? stripped : './' + stripped; + return resolveStandard(relPath, filePath, ctx, SupportedLanguages.Dart); +}; + +export const dartImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Dart, + strategies: [dartPackageStrategy, dartRelativeStrategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/go.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/go.ts new file mode 100644 index 000000000..7eee17988 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/go.ts @@ -0,0 +1,35 @@ +/** + * Go import resolution config. + * Go-specific package strategy (go.mod), then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveGoPackageDir, resolveGoPackage } from '../go.js'; + +/** Go-specific package resolution strategy — resolves go.mod-based package imports. */ +export const goPackageStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const goModule = ctx.configs.goModule; + if (goModule && rawImportPath.startsWith(goModule.modulePath)) { + const pkgSuffix = resolveGoPackageDir(rawImportPath, goModule); + if (pkgSuffix) { + const pkgFiles = resolveGoPackage( + rawImportPath, + goModule, + ctx.normalizedFileList, + ctx.allFileList, + ); + if (pkgFiles.length > 0) { + return { kind: 'package', files: pkgFiles, dirSuffix: pkgSuffix }; + } + } + // Fall through if no files found (package might be external) + } + return null; +}; + +export const goImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Go, + strategies: [goPackageStrategy, createStandardStrategy(SupportedLanguages.Go)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/jvm.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/jvm.ts new file mode 100644 index 000000000..47e590fe8 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/jvm.ts @@ -0,0 +1,115 @@ +/** + * Java / Kotlin import resolution configs. + * JVM-specific wildcard/member strategy, then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveJvmWildcard, resolveJvmMemberImport, KOTLIN_EXTENSIONS } from '../jvm.js'; + +/** Java JVM resolution strategy — wildcard and member import resolution. */ +export const javaJvmStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + if (rawImportPath.endsWith('.*')) { + const matchedFiles = resolveJvmWildcard( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; + } else { + const memberResolved = resolveJvmMemberImport( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + if (memberResolved) return { kind: 'files', files: [memberResolved] }; + } + return null; +}; + +/** + * Kotlin JVM resolution strategy — wildcard/member with Java-interop + top-level function imports. + */ +export const kotlinJvmStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + if (rawImportPath.endsWith('.*')) { + const matchedFiles = resolveJvmWildcard( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + KOTLIN_EXTENSIONS, + ctx.index, + ); + if (matchedFiles.length === 0) { + const javaMatches = resolveJvmWildcard( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + if (javaMatches.length > 0) return { kind: 'files', files: javaMatches }; + } + if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; + } else { + let memberResolved = resolveJvmMemberImport( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + KOTLIN_EXTENSIONS, + ctx.index, + ); + if (!memberResolved) { + memberResolved = resolveJvmMemberImport( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + } + if (memberResolved) return { kind: 'files', files: [memberResolved] }; + + // Kotlin: top-level function imports (e.g. import models.getUser) have only 2 segments, + // which resolveJvmMemberImport skips (requires >=3). Fall back to package-directory scan + // for lowercase last segments (function/property imports). Uppercase last segments + // (class imports like models.User) fall through to standard suffix resolution. + const segments = rawImportPath.split('.'); + const lastSeg = segments[segments.length - 1]; + if (segments.length >= 2 && lastSeg[0] && lastSeg[0] === lastSeg[0].toLowerCase()) { + const pkgWildcard = segments.slice(0, -1).join('.') + '.*'; + let dirFiles = resolveJvmWildcard( + pkgWildcard, + ctx.normalizedFileList, + ctx.allFileList, + KOTLIN_EXTENSIONS, + ctx.index, + ); + if (dirFiles.length === 0) { + dirFiles = resolveJvmWildcard( + pkgWildcard, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + } + if (dirFiles.length > 0) return { kind: 'files', files: dirFiles }; + } + } + return null; +}; + +export const javaImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Java, + strategies: [javaJvmStrategy, createStandardStrategy(SupportedLanguages.Java)], +}; + +export const kotlinImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Kotlin, + strategies: [kotlinJvmStrategy, createStandardStrategy(SupportedLanguages.Kotlin)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/php.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/php.ts new file mode 100644 index 000000000..5a8446f25 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/php.ts @@ -0,0 +1,26 @@ +/** + * PHP import resolution config. + * PSR-4 strategy via composer.json — no standard fallback (PSR-4 includes its own suffix matching). + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { resolvePhpImportInternal } from '../php.js'; + +/** PHP PSR-4 resolution strategy via composer.json autoload mappings. */ +export const phpPsr4Strategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const resolved = resolvePhpImportInternal( + rawImportPath, + ctx.configs.composerConfig, + ctx.allFilePaths, + ctx.normalizedFileList, + ctx.allFileList, + ctx.index, + ); + return resolved ? { kind: 'files', files: [resolved] } : null; +}; + +export const phpImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.PHP, + strategies: [phpPsr4Strategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/python.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/python.ts new file mode 100644 index 000000000..d98304561 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/python.ts @@ -0,0 +1,47 @@ +/** + * Python import resolution config. + * PEP 328 relative + proximity-based strategy, then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolvePythonImportInternal } from '../python.js'; + +/** + * Python import resolution strategy — PEP 328 relative + proximity-based bare imports. + * Returns null to continue chain for non-relative imports. + * Absorbs unresolved relative imports (returns empty result to stop the chain). + */ +export const pythonImportStrategy: ImportResolverStrategy = (rawImportPath, filePath, ctx) => { + const resolved = resolvePythonImportInternal(filePath, rawImportPath, ctx.allFilePaths); + if (resolved) { + ctx.resolveCache.set(`${filePath}::${rawImportPath}`, resolved); + return { kind: 'files', files: [resolved] }; + } + // PEP 328: unresolved relative imports should not fall through to suffix matching + if (rawImportPath.startsWith('.')) return { kind: 'files', files: [] }; + + // External dotted imports like `django.apps` should not fall through to generic + // suffix matching when the repo has unrelated local files such as `accounts/apps.py`. + // Keep suffix fallback only when the leading segment appears somewhere in-repo, + // which preserves existing internal absolute-import behavior like `accounts.models`. + const pathLike = rawImportPath.replace(/\./g, '/'); + if (pathLike.includes('/')) { + const [leadingSegment] = pathLike.split('/').filter(Boolean); + const hasRepoCandidate = + !!leadingSegment && + (ctx.index.get(`${leadingSegment}.py`) !== undefined || + ctx.index.get(`${leadingSegment}/__init__.py`) !== undefined || + ctx.index.getFilesInDir(leadingSegment, '.py').length > 0); + + if (!hasRepoCandidate) return { kind: 'files', files: [] }; + } + + return null; +}; + +export const pythonImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Python, + strategies: [pythonImportStrategy, createStandardStrategy(SupportedLanguages.Python)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/ruby.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/ruby.ts new file mode 100644 index 000000000..bf2507f70 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/ruby.ts @@ -0,0 +1,20 @@ +/** + * Ruby import resolution config. + * Require/require_relative suffix matching — no standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { suffixResolve } from '../utils.js'; + +/** Ruby require/require_relative resolution strategy. */ +export const rubyRequireStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const pathParts = rawImportPath.replace(/^\.\//, '').split('/').filter(Boolean); + const resolved = suffixResolve(pathParts, ctx.normalizedFileList, ctx.allFileList, ctx.index); + return resolved ? { kind: 'files', files: [resolved] } : null; +}; + +export const rubyImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Ruby, + strategies: [rubyRequireStrategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/rust.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/rust.ts new file mode 100644 index 000000000..daa2173e7 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/rust.ts @@ -0,0 +1,56 @@ +/** + * Rust import resolution config. + * Rust module strategy (grouped imports, crate/super/self paths), then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveRustImportInternal } from '../rust.js'; + +/** Rust module resolution strategy — handles grouped imports and crate/super/self paths. */ +export const rustModuleStrategy: ImportResolverStrategy = (rawImportPath, filePath, ctx) => { + // Top-level grouped: use {crate::a, crate::b} + if (rawImportPath.startsWith('{') && rawImportPath.endsWith('}')) { + const inner = rawImportPath.slice(1, -1); + const parts = inner + .split(',') + .map((p) => p.trim()) + .filter(Boolean); + const resolved: string[] = []; + for (const part of parts) { + const r = resolveRustImportInternal(filePath, part, ctx.allFilePaths); + if (r) resolved.push(r); + } + return resolved.length > 0 ? { kind: 'files', files: resolved } : null; + } + + // Scoped grouped: use crate::models::{User, Repo} + const braceIdx = rawImportPath.indexOf('::{'); + if (braceIdx !== -1 && rawImportPath.endsWith('}')) { + const pathPrefix = rawImportPath.substring(0, braceIdx); + const braceContent = rawImportPath.substring(braceIdx + 3, rawImportPath.length - 1); + const items = braceContent + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + const resolved: string[] = []; + for (const item of items) { + // Handle `use crate::models::{User, Repo as R}` — strip alias for resolution + const itemName = item.includes(' as ') ? item.split(' as ')[0].trim() : item; + const r = resolveRustImportInternal(filePath, `${pathPrefix}::${itemName}`, ctx.allFilePaths); + if (r) resolved.push(r); + } + if (resolved.length > 0) return { kind: 'files', files: resolved }; + // Fallback: resolve the prefix path itself (e.g. crate::models -> models.rs) + const prefixResult = resolveRustImportInternal(filePath, pathPrefix, ctx.allFilePaths); + if (prefixResult) return { kind: 'files', files: [prefixResult] }; + } + + return null; +}; + +export const rustImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Rust, + strategies: [rustModuleStrategy, createStandardStrategy(SupportedLanguages.Rust)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/swift.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts similarity index 53% rename from gitnexus/src/core/ingestion/import-resolvers/swift.ts rename to gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts index 7ccc10458..f7d9ec195 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/swift.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts @@ -1,16 +1,13 @@ /** - * Swift module import resolution. - * Handles module imports via Package.swift target map. + * Swift import resolution config. + * Package.swift target map strategy — no standard fallback (unresolved = external framework). */ -import type { ImportResult, ResolveCtx } from './types.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; -/** Swift: module imports via Package.swift target map. */ -export function resolveSwiftImport( - rawImportPath: string, - _filePath: string, - ctx: ResolveCtx, -): ImportResult { +/** Swift Package.swift target map resolution strategy. */ +export const swiftPackageStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { const swiftPackageConfig = ctx.configs.swiftPackageConfig; if (swiftPackageConfig) { const targetDir = swiftPackageConfig.targets.get(rawImportPath); @@ -29,4 +26,9 @@ export function resolveSwiftImport( } } return null; // External framework (Foundation, UIKit, etc.) -} +}; + +export const swiftImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Swift, + strategies: [swiftPackageStrategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/typescript-javascript.ts new file mode 100644 index 000000000..b86b365fd --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/typescript-javascript.ts @@ -0,0 +1,28 @@ +/** + * TypeScript / JavaScript / Vue import resolution configs. + * All use standard resolution — TS/JS with tsconfig path aliases, + * Vue delegates to TypeScript's resolver. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; + +export const typescriptImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.TypeScript, + strategies: [createStandardStrategy(SupportedLanguages.TypeScript)], +}; + +export const javascriptImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.JavaScript, + strategies: [createStandardStrategy(SupportedLanguages.JavaScript)], +}; + +// Vue SFCs are preprocessed into TypeScript upstream of import resolution, +// so the resolver intentionally runs as TypeScript. `language: Vue` here is +// documentation-only metadata (see `ImportResolutionConfig.language` JSDoc +// and ARCHITECTURE.md §Vue); it is not consumed by `createImportResolver`. +export const vueImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Vue, + strategies: [createStandardStrategy(SupportedLanguages.TypeScript)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/csharp.ts b/gitnexus/src/core/ingestion/import-resolvers/csharp.ts index 33cc9c828..79548d7f6 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/csharp.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/csharp.ts @@ -1,13 +1,12 @@ /** - * C# namespace import resolution. - * Handles using-directive resolution via .csproj root namespace stripping. + * C# namespace import resolution — internal helpers. + * + * Strategy lives in configs/csharp.ts. + * This file contains shared helpers for namespace-based resolution. */ import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; import type { CSharpProjectConfig } from '../language-config.js'; /** @@ -126,29 +125,3 @@ export function resolveCSharpNamespaceDir( return null; } - -/** C#: namespace-based resolution via .csproj configs, with suffix-match fallback. */ -export function resolveCSharpImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - const csharpConfigs = ctx.configs.csharpConfigs; - if (csharpConfigs.length > 0) { - const resolvedFiles = resolveCSharpImportInternal( - rawImportPath, - csharpConfigs, - ctx.normalizedFileList, - ctx.allFileList, - ctx.index, - ); - if (resolvedFiles.length > 1) { - const dirSuffix = resolveCSharpNamespaceDir(rawImportPath, csharpConfigs); - if (dirSuffix) { - return { kind: 'package', files: resolvedFiles, dirSuffix }; - } - } - if (resolvedFiles.length > 0) return { kind: 'files', files: resolvedFiles }; - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.CSharp); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/dart.ts b/gitnexus/src/core/ingestion/import-resolvers/dart.ts deleted file mode 100644 index 2a3cf3f6b..000000000 --- a/gitnexus/src/core/ingestion/import-resolvers/dart.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Dart import resolution. - * Handles package: imports (local packages) and relative imports. - * SDK imports (dart:*) and external packages are skipped. - */ - -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; -import { SupportedLanguages } from 'gitnexus-shared'; - -export function resolveDartImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - // Strip surrounding quotes from configurable_uri capture - const stripped = rawImportPath.replace(/^['"]|['"]$/g, ''); - - // Skip dart: SDK imports (dart:async, dart:io, etc.) - if (stripped.startsWith('dart:')) return null; - - // Local package: imports → resolve to lib/ - if (stripped.startsWith('package:')) { - const slashIdx = stripped.indexOf('/'); - if (slashIdx === -1) return null; - const relPath = stripped.slice(slashIdx + 1); - const candidates = [`lib/${relPath}`, relPath]; - const files: string[] = []; - for (const candidate of candidates) { - for (const fp of ctx.allFileList) { - if (fp.endsWith('/' + candidate) || fp === candidate) { - files.push(fp); - break; - } - } - if (files.length > 0) break; - } - if (files.length > 0) return { kind: 'files', files }; - return null; - } - - // Relative imports — use standard resolution. - // Dart relative imports don't require a leading "./" (e.g. `import 'models.dart'`). - // The standard resolver only recognises paths starting with "." as relative, so - // prepend "./" when the path doesn't already start with "." to ensure correct - // same-directory resolution (without this, "models.dart" would be mangled by the - // generic dot-to-slash conversion intended for Java-style package imports). - const relPath = stripped.startsWith('.') ? stripped : './' + stripped; - return resolveStandard(relPath, filePath, ctx, SupportedLanguages.Dart); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/go.ts b/gitnexus/src/core/ingestion/import-resolvers/go.ts index 1b1eb3ebc..c33c46422 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/go.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/go.ts @@ -1,11 +1,10 @@ /** - * Go package import resolution. - * Handles Go module path-based package imports. + * Go package import resolution — internal helpers. + * + * Strategy lives in configs/go.ts. + * This file contains the shared helpers used by the strategy. */ -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; import type { GoModuleConfig } from '../language-config.js'; /** @@ -56,28 +55,3 @@ export function resolveGoPackage( return matches; } - -/** Go: package-level imports via go.mod module path. */ -export function resolveGoImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - const goModule = ctx.configs.goModule; - if (goModule && rawImportPath.startsWith(goModule.modulePath)) { - const pkgSuffix = resolveGoPackageDir(rawImportPath, goModule); - if (pkgSuffix) { - const pkgFiles = resolveGoPackage( - rawImportPath, - goModule, - ctx.normalizedFileList, - ctx.allFileList, - ); - if (pkgFiles.length > 0) { - return { kind: 'package', files: pkgFiles, dirSuffix: pkgSuffix }; - } - } - // Fall through if no files found (package might be external) - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Go); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/jvm.ts b/gitnexus/src/core/ingestion/import-resolvers/jvm.ts index 2dd7636ab..194cfdac8 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/jvm.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/jvm.ts @@ -1,13 +1,13 @@ /** - * JVM import resolution (Java + Kotlin). - * Handles wildcard imports, member/static imports, and Kotlin-specific patterns. + * JVM import resolution — internal helpers (Java + Kotlin). + * + * Strategies live in configs/jvm.ts. + * This file contains shared helpers for wildcard/member resolution + * and the Kotlin wildcard preprocessor. */ import type { SuffixIndex } from './utils.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; /** Kotlin file extensions for JVM resolver reuse */ export const KOTLIN_EXTENSIONS: readonly string[] = ['.kt', '.kts']; @@ -125,108 +125,3 @@ export function resolveJvmMemberImport( return null; } - -/** Java: JVM wildcard -> member import -> standard fallthrough */ -export function resolveJavaImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - if (rawImportPath.endsWith('.*')) { - const matchedFiles = resolveJvmWildcard( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; - } else { - const memberResolved = resolveJvmMemberImport( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - if (memberResolved) return { kind: 'files', files: [memberResolved] }; - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Java); -} - -/** - * Kotlin: JVM wildcard/member with Java-interop fallback -> top-level function imports -> standard. - * Kotlin can import from .kt/.kts files OR from .java files (Java interop). - */ -export function resolveKotlinImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - if (rawImportPath.endsWith('.*')) { - const matchedFiles = resolveJvmWildcard( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - KOTLIN_EXTENSIONS, - ctx.index, - ); - if (matchedFiles.length === 0) { - const javaMatches = resolveJvmWildcard( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - if (javaMatches.length > 0) return { kind: 'files', files: javaMatches }; - } - if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; - } else { - let memberResolved = resolveJvmMemberImport( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - KOTLIN_EXTENSIONS, - ctx.index, - ); - if (!memberResolved) { - memberResolved = resolveJvmMemberImport( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - } - if (memberResolved) return { kind: 'files', files: [memberResolved] }; - - // Kotlin: top-level function imports (e.g. import models.getUser) have only 2 segments, - // which resolveJvmMemberImport skips (requires >=3). Fall back to package-directory scan - // for lowercase last segments (function/property imports). Uppercase last segments - // (class imports like models.User) fall through to standard suffix resolution. - const segments = rawImportPath.split('.'); - const lastSeg = segments[segments.length - 1]; - if (segments.length >= 2 && lastSeg[0] && lastSeg[0] === lastSeg[0].toLowerCase()) { - const pkgWildcard = segments.slice(0, -1).join('.') + '.*'; - let dirFiles = resolveJvmWildcard( - pkgWildcard, - ctx.normalizedFileList, - ctx.allFileList, - KOTLIN_EXTENSIONS, - ctx.index, - ); - if (dirFiles.length === 0) { - dirFiles = resolveJvmWildcard( - pkgWildcard, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - } - if (dirFiles.length > 0) return { kind: 'files', files: dirFiles }; - } - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Kotlin); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/php.ts b/gitnexus/src/core/ingestion/import-resolvers/php.ts index 20517bbed..303bf5546 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/php.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/php.ts @@ -1,11 +1,12 @@ /** - * PHP PSR-4 import resolution. - * Handles use-statement resolution via composer.json autoload mappings. + * PHP PSR-4 import resolution — internal helpers. + * + * Strategy lives in configs/php.ts. + * This file contains the shared helper for PSR-4 resolution via composer.json. */ import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; -import type { ImportResult, ResolveCtx } from './types.js'; import type { ComposerConfig } from '../language-config.js'; /** Get or compute the sorted PSR-4 entries (cached after first call). */ @@ -91,20 +92,3 @@ export function resolvePhpImportInternal( const pathParts = normalized.split('/').filter(Boolean); return suffixResolve(pathParts, normalizedFileList, allFileList, index); } - -/** PHP: namespace-based resolution via composer.json PSR-4. */ -export function resolvePhpImport( - rawImportPath: string, - _filePath: string, - ctx: ResolveCtx, -): ImportResult { - const resolved = resolvePhpImportInternal( - rawImportPath, - ctx.configs.composerConfig, - ctx.allFilePaths, - ctx.normalizedFileList, - ctx.allFileList, - ctx.index, - ); - return resolved ? { kind: 'files', files: [resolved] } : null; -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/python.ts b/gitnexus/src/core/ingestion/import-resolvers/python.ts index 4e9c4bbb3..264103a09 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/python.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/python.ts @@ -1,12 +1,12 @@ /** * Python import resolution — PEP 328 relative imports and proximity-based bare imports. * Import system spec: PEP 302 (original), PEP 451 (current). + * + * Strategy lives in configs/python.ts. + * This file contains the shared internal helper used by the strategy and tests. */ import { tryResolveWithExtensions } from './utils.js'; -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; /** * Resolve a Python import to a file path (low-level helper). @@ -74,23 +74,3 @@ export function resolvePythonImportInternal( return null; } - -/** - * Python: relative imports (PEP 328) + proximity-based bare imports. - * Falls through to standard suffix resolution when proximity finds no match. - */ -export function resolvePythonImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - const resolved = resolvePythonImportInternal(filePath, rawImportPath, ctx.allFilePaths); - if (resolved) { - // Store in resolveCache so other files importing the same module skip the - // ancestor walk. The cache key matches resolveStandard's convention. - ctx.resolveCache.set(`${filePath}::${rawImportPath}`, resolved); - return { kind: 'files', files: [resolved] }; - } - if (rawImportPath.startsWith('.')) return null; // relative but unresolved -- don't suffix-match - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Python); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/resolver-factory.ts b/gitnexus/src/core/ingestion/import-resolvers/resolver-factory.ts new file mode 100644 index 000000000..4caf748a7 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/resolver-factory.ts @@ -0,0 +1,35 @@ +/** + * Import resolver factory — creates a composable import resolver from + * an ordered list of strategies. + * + * Mirrors the method-extractors/generic.ts and call-extractors/generic.ts + * pattern: declare a config per language, produce a runtime resolver via factory. + * + * Each strategy is tried in order. The first non-null result wins. + * A result with an empty `files` array is treated as "handled but unresolved" + * (stops the chain without producing import edges). + */ + +import type { ImportResolverFn, ImportResolutionConfig } from './types.js'; + +/** + * Create an ImportResolverFn from a declarative config. + * + * Chains strategies in declaration order — first non-null result wins. + * Returns null only if every strategy returns null. + * + * Error behaviour: if a strategy throws, the error propagates immediately + * and remaining strategies are not tried. Strategies are expected to be + * pure data transforms that never throw; any unexpected exception indicates + * a bug in the strategy implementation. + */ +export function createImportResolver(config: ImportResolutionConfig): ImportResolverFn { + const { strategies } = config; + return (rawImportPath, filePath, ctx) => { + for (const strategy of strategies) { + const result = strategy(rawImportPath, filePath, ctx); + if (result) return result; + } + return null; + }; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/ruby.ts b/gitnexus/src/core/ingestion/import-resolvers/ruby.ts index b8ada4cf4..4bf47d31f 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/ruby.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/ruby.ts @@ -1,11 +1,12 @@ /** - * Ruby require/require_relative import resolution. - * Handles path resolution for Ruby's require and require_relative calls. + * Ruby require/require_relative import resolution — internal helpers. + * + * Strategy lives in configs/ruby.ts. + * This file only contains the low-level helper used by the strategy. */ import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; -import type { ImportResult, ResolveCtx } from './types.js'; /** * Resolve a Ruby require/require_relative path to a matching .rb file (low-level helper). @@ -22,18 +23,3 @@ export function resolveRubyImportInternal( const pathParts = importPath.replace(/^\.\//, '').split('/').filter(Boolean); return suffixResolve(pathParts, normalizedFileList, allFileList, index); } - -/** Ruby: require / require_relative. */ -export function resolveRubyImport( - rawImportPath: string, - _filePath: string, - ctx: ResolveCtx, -): ImportResult { - const resolved = resolveRubyImportInternal( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ctx.index, - ); - return resolved ? { kind: 'files', files: [resolved] } : null; -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/rust.ts b/gitnexus/src/core/ingestion/import-resolvers/rust.ts index 632e19352..2d1dd864a 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/rust.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/rust.ts @@ -1,12 +1,10 @@ /** - * Rust module import resolution. - * Handles crate::, super::, self:: prefix paths and :: separators. + * Rust module import resolution — internal helpers. + * + * Strategy lives in configs/rust.ts. + * This file contains shared helpers used by the strategy and standard.ts. */ -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; - /** * Resolve Rust use-path to a file (low-level helper). * Handles crate::, super::, self:: prefixes and :: path separators. @@ -84,49 +82,3 @@ export function tryRustModulePath(modulePath: string, allFiles: Set): st return null; } - -/** Rust: expand grouped imports: use {crate::a, crate::b} and use crate::models::{User, Repo}. */ -export function resolveRustImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - // Top-level grouped: use {crate::a, crate::b} - if (rawImportPath.startsWith('{') && rawImportPath.endsWith('}')) { - const inner = rawImportPath.slice(1, -1); - const parts = inner - .split(',') - .map((p) => p.trim()) - .filter(Boolean); - const resolved: string[] = []; - for (const part of parts) { - const r = resolveRustImportInternal(filePath, part, ctx.allFilePaths); - if (r) resolved.push(r); - } - return resolved.length > 0 ? { kind: 'files', files: resolved } : null; - } - - // Scoped grouped: use crate::models::{User, Repo} - const braceIdx = rawImportPath.indexOf('::{'); - if (braceIdx !== -1 && rawImportPath.endsWith('}')) { - const pathPrefix = rawImportPath.substring(0, braceIdx); - const braceContent = rawImportPath.substring(braceIdx + 3, rawImportPath.length - 1); - const items = braceContent - .split(',') - .map((s) => s.trim()) - .filter(Boolean); - const resolved: string[] = []; - for (const item of items) { - // Handle `use crate::models::{User, Repo as R}` — strip alias for resolution - const itemName = item.includes(' as ') ? item.split(' as ')[0].trim() : item; - const r = resolveRustImportInternal(filePath, `${pathPrefix}::${itemName}`, ctx.allFilePaths); - if (r) resolved.push(r); - } - if (resolved.length > 0) return { kind: 'files', files: resolved }; - // Fallback: resolve the prefix path itself (e.g. crate::models -> models.rs) - const prefixResult = resolveRustImportInternal(filePath, pathPrefix, ctx.allFilePaths); - if (prefixResult) return { kind: 'files', files: [prefixResult] }; - } - - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Rust); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/standard.ts b/gitnexus/src/core/ingestion/import-resolvers/standard.ts index a29810a2f..f8aae9625 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/standard.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/standard.ts @@ -8,7 +8,7 @@ import type { SuffixIndex } from './utils.js'; import { tryResolveWithExtensions, suffixResolve } from './utils.js'; import { resolveRustImportInternal } from './rust.js'; import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ImportResolverFn, ResolveCtx } from './types.js'; +import type { ImportResult, ImportResolverStrategy, ResolveCtx } from './types.js'; import type { TsconfigPaths } from '../language-config.js'; /** Max entries in the resolve cache. Beyond this, entries are evicted. @@ -174,18 +174,11 @@ export function resolveStandard( return resolvedPath ? { kind: 'files', files: [resolvedPath] } : null; } -/** JavaScript: standard single-file resolution. */ -export const resolveJavascriptImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.JavaScript); +// ============================================================================ +// Strategy factory — composable hook for ImportResolutionConfig +// ============================================================================ -/** TypeScript: standard single-file resolution. */ -export const resolveTypescriptImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.TypeScript); - -/** C: standard single-file resolution for #include directives. */ -export const resolveCImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.C); - -/** C++: standard single-file resolution for #include directives. */ -export const resolveCppImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.CPlusPlus); +/** Create a reusable standard-resolution strategy for a given language. */ +export function createStandardStrategy(language: SupportedLanguages): ImportResolverStrategy { + return (raw, fp, ctx) => resolveStandard(raw, fp, ctx, language); +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/types.ts b/gitnexus/src/core/ingestion/import-resolvers/types.ts index 3e68c38b2..66e23a79b 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/types.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/types.ts @@ -12,6 +12,7 @@ import type { } from '../language-config.js'; import type { SwiftPackageConfig } from '../language-config.js'; import type { SuffixIndex } from './utils.js'; +import type { SupportedLanguages } from 'gitnexus-shared'; /** * Result of resolving an import via language-specific dispatch. @@ -53,3 +54,28 @@ export type ImportResolverFn = ( filePath: string, resolveCtx: ResolveCtx, ) => ImportResult; + +/** + * A single import resolution strategy — one step in a composable chain. + * Same signature as ImportResolverFn. Returns null to let the next strategy + * in the chain try; returns a result (even with empty files) to stop the chain. + */ +export type ImportResolverStrategy = ImportResolverFn; + +/** + * Declarative config for composable import resolution — mirrors the + * MethodExtractionConfig / CallExtractionConfig pattern. + * + * Each language declares an ordered list of strategies to try. + * The factory (`createImportResolver`) chains them: first non-null result wins. + */ +export interface ImportResolutionConfig { + /** + * Documentation-only metadata identifying which language this config serves. + * **Not used by `createImportResolver`** — the factory only iterates `strategies`. + * Useful for logging, debugging, and compile-time exhaustiveness checks when + * mapping `SupportedLanguages → ImportResolutionConfig` in language providers. + */ + readonly language: SupportedLanguages; + readonly strategies: readonly ImportResolverStrategy[]; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/vue.ts b/gitnexus/src/core/ingestion/import-resolvers/vue.ts deleted file mode 100644 index c46e3f725..000000000 --- a/gitnexus/src/core/ingestion/import-resolvers/vue.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Vue import resolver — delegates to TypeScript's standard resolver. - * - * Vue