diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 292cb435d..f3530e4d3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,3 +14,62 @@ updates: labels: - dependencies - ci + + # Gitnexus npm deps — tree-sitter grammars checked daily so we catch + # new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars + # are grouped so lockstep bumps produce a single PR. The tree-sitter + # RUNTIME is pinned — upgrade deliberately via the drift check workflow. + # See .github/scripts/check-tree-sitter-upgrade-readiness.py for + # the upgrade readiness tracker. + - package-ecosystem: npm + directory: /gitnexus + schedule: + interval: daily + open-pull-requests-limit: 10 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + groups: + tree-sitter-grammars: + patterns: + - tree-sitter-* + exclude-patterns: + - tree-sitter + - tree-sitter-cli + ignore: + # Pin the tree-sitter runtime at 0.21.x until the drift check + # reports all grammars are peer-dep compatible with 0.25. + - dependency-name: tree-sitter + update-types: + - version-update:semver-major + - version-update:semver-minor + # tree-sitter-cli follows the runtime's version cadence. Bump when + # regenerating vendor/tree-sitter-proto/src/parser.c, not on a schedule. + - dependency-name: tree-sitter-cli + + # gitnexus-web (thin frontend client). + - package-ecosystem: npm + directory: /gitnexus-web + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + - frontend + + # Shared types package. + - package-ecosystem: npm + directory: /gitnexus-shared + schedule: + interval: weekly + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies diff --git a/.github/scripts/check-tree-sitter-upgrade-readiness.py b/.github/scripts/check-tree-sitter-upgrade-readiness.py new file mode 100644 index 000000000..df21533a8 --- /dev/null +++ b/.github/scripts/check-tree-sitter-upgrade-readiness.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Monitor tree-sitter 0.25 upgrade readiness. + +Tracks two things Dependabot cannot see: + + 1. Peer-dep compatibility. Each tree-sitter-* grammar declares a peer + dependency on the tree-sitter runtime. We want to know when every + grammar's *latest npm release* satisfies tree-sitter@0.25.0 so we + can upgrade without --legacy-peer-deps. + + 2. Vendored upstream drift. vendor/tree-sitter-proto/ is a snapshot of + coder3101/tree-sitter-proto's parser.c. When upstream moves, we want + to know whether we can pick it up. + +Invoked from .github/workflows/tree-sitter-upgrade-readiness.yml daily. +Runs locally too: + + python3 .github/scripts/check-tree-sitter-upgrade-readiness.py + +Outputs Markdown to stdout. Exit 0 when every grammar is upgrade-ready +and the vendored proto is in sync. Exit 1 when blockers remain (the +workflow uses this to open or update a tracking issue). + +No external deps -- stdlib only, so it runs on any vanilla runner. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import sys +import urllib.error +import urllib.request + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +GITNEXUS_DIR = REPO_ROOT / "gitnexus" +VENDOR_PROTO_DIR = GITNEXUS_DIR / "vendor" / "tree-sitter-proto" + +# ── Upgrade target ────────────────────────────────────────────────────── +# The runtime version we want to upgrade TO. Update this when the goal +# changes (e.g. once 0.25 lands and we target 0.26). +TARGET_RUNTIME = "0.25.0" +TARGET_RUNTIME_MAJOR_MINOR = ".".join(TARGET_RUNTIME.split(".")[:2]) + +# Tree-sitter runtime -> (min_abi, max_abi) it can load. Only the current +# and target entries matter; extend when changing TARGET_RUNTIME. +RUNTIME_ABI_RANGES: dict[str, tuple[int, int]] = { + "0.21": (13, 14), + "0.25": (13, 15), +} + +assert TARGET_RUNTIME_MAJOR_MINOR in RUNTIME_ABI_RANGES, ( + f"RUNTIME_ABI_RANGES has no entry for {TARGET_RUNTIME_MAJOR_MINOR!r}. " + f"Add the ABI range after auditing the upstream release notes." +) + +# Grammars we use. Values are the upstream GitHub repos to check for +# unreleased ABI bumps (owner/repo, branch, parser.c path). +GRAMMARS: dict[str, tuple[str, str, str]] = { + "tree-sitter-c": ("tree-sitter/tree-sitter-c", "master", "src/parser.c"), + "tree-sitter-c-sharp": ("tree-sitter/tree-sitter-c-sharp", "master", "src/parser.c"), + "tree-sitter-cpp": ("tree-sitter/tree-sitter-cpp", "master", "src/parser.c"), + "tree-sitter-dart": ("UserNobody14/tree-sitter-dart", "master", "src/parser.c"), + "tree-sitter-go": ("tree-sitter/tree-sitter-go", "master", "src/parser.c"), + "tree-sitter-java": ("tree-sitter/tree-sitter-java", "master", "src/parser.c"), + "tree-sitter-javascript": ("tree-sitter/tree-sitter-javascript", "master", "src/parser.c"), + "tree-sitter-kotlin": ("fwcd/tree-sitter-kotlin", "main", "src/parser.c"), + "tree-sitter-php": ("tree-sitter/tree-sitter-php", "master", "php/src/parser.c"), + "tree-sitter-python": ("tree-sitter/tree-sitter-python", "master", "src/parser.c"), + "tree-sitter-ruby": ("tree-sitter/tree-sitter-ruby", "master", "src/parser.c"), + "tree-sitter-rust": ("tree-sitter/tree-sitter-rust", "master", "src/parser.c"), + "tree-sitter-swift": ("alex-pinkus/tree-sitter-swift", "main", "src/parser.c"), + "tree-sitter-typescript": ("tree-sitter/tree-sitter-typescript", "master", "typescript/src/parser.c"), +} + +UPSTREAM_PROTO_OWNER = "coder3101" +UPSTREAM_PROTO_REPO = "tree-sitter-proto" +UPSTREAM_PROTO_BRANCH = "main" + + +# ── Helpers ───────────────────────────────────────────────────────────── + +def read_current_runtime() -> str: + """Return the tree-sitter runtime version pinned in package.json (e.g. '0.21').""" + pkg = json.loads((GITNEXUS_DIR / "package.json").read_text()) + raw = pkg["dependencies"]["tree-sitter"] + match = re.search(r"(\d+)\.(\d+)", raw) + if not match: + raise SystemExit(f"could not parse tree-sitter version: {raw!r}") + return f"{match.group(1)}.{match.group(2)}" + + +def npm_view_json(pkg: str) -> dict | None: + """Fetch package metadata from the npm registry via HTTPS. + + Uses the registry API directly so we don't depend on the npm CLI + being available (it's a batch file on Windows which complicates + subprocess calls). + """ + url = f"https://registry.npmjs.org/{pkg}/latest" + try: + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=8) as resp: + return json.loads(resp.read().decode("utf-8")) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError): + return None + + +def satisfies_target(peer_range: str | None, target: str) -> bool: + """Check if a semver range like '^0.22.4' or '^0.25.0' satisfies the target. + + Simple heuristic: extract the minimum version from the range and check + if target >= min. For caret ranges (^X.Y.Z), the upper bound is the + next major (for X>0) or next minor (for X==0). We check both bounds. + """ + if peer_range is None: + # No peer dep declared = no constraint = compatible. + return True + match = re.search(r"(\d+)\.(\d+)\.(\d+)", peer_range) + if not match: + return False + min_major, min_minor, min_patch = int(match.group(1)), int(match.group(2)), int(match.group(3)) + + t_match = re.search(r"(\d+)\.(\d+)\.(\d+)", target) + if not t_match: + return False + t_major, t_minor, t_patch = int(t_match.group(1)), int(t_match.group(2)), int(t_match.group(3)) + + # Target must be >= minimum. + target_tuple = (t_major, t_minor, t_patch) + min_tuple = (min_major, min_minor, min_patch) + if target_tuple < min_tuple: + return False + + # For caret ranges with major 0: ^0.X.Y allows [0.X.Y, 0.(X+1).0). + if peer_range.startswith("^") and min_major == 0: + if t_major != 0 or t_minor >= min_minor + 1: + return False + # For caret ranges with major >0: ^X.Y.Z allows [X.Y.Z, (X+1).0.0). + elif peer_range.startswith("^") and min_major > 0: + if t_major >= min_major + 1: + return False + + return True + + +_GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + + +def fetch_text(url: str, timeout: int = 8) -> str | None: + """Fetch a URL and return its text, or None on failure. + + Adds an Authorization header for github.com URLs when GITHUB_TOKEN is + set (raises the rate limit from 60 to 5 000 requests/hour). + """ + headers: dict[str, str] = {} + if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url): + headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}" + try: + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return resp.read().decode("utf-8", errors="ignore") + except (urllib.error.URLError, urllib.error.HTTPError): + return None + + +def extract_abi_from_text(text: str) -> int | None: + """Extract LANGUAGE_VERSION from parser.c text.""" + match = re.search(r"#define\s+LANGUAGE_VERSION\s+(\d+)", text[:4096]) + return int(match.group(1)) if match else None + + +def extract_language_version(parser_c: pathlib.Path) -> int | None: + """Return the LANGUAGE_VERSION defined in a parser.c, or None if absent.""" + if not parser_c.is_file(): + return None + with parser_c.open("r", encoding="utf-8", errors="ignore") as fh: + head = fh.read(4096) + return extract_abi_from_text(head) + + +def md_h(text: str, level: int = 2) -> str: + return f"{'#' * level} {text}\n" + + +# ── Main ──────────────────────────────────────────────────────────────── + +def main() -> int: + blockers: dict[str, str] = {} + lines: list[str] = [] + lines.append(md_h("Tree-sitter 0.25 upgrade readiness", 1)) + lines.append("") + + current_runtime = read_current_runtime() + current_abi_range = RUNTIME_ABI_RANGES.get(current_runtime, (0, 0)) + target_abi_range = RUNTIME_ABI_RANGES.get(TARGET_RUNTIME_MAJOR_MINOR, (0, 0)) + + lines.append(f"- Current runtime: `tree-sitter@{current_runtime}.x` (ABI {current_abi_range[0]}..{current_abi_range[1]})") + lines.append(f"- Target runtime: `tree-sitter@{TARGET_RUNTIME}` (ABI {target_abi_range[0]}..{target_abi_range[1]})") + lines.append("") + + # ── Grammar peer-dep compatibility ─────────────────────────────── + lines.append(md_h("Grammar compatibility", 2)) + lines.append("| Grammar | npm latest | Peer dep | Satisfies 0.25? | ABI | Upstream ABI | Status |") + lines.append("|---|---|---|---|---|---|---|") + + ready_count = 0 + total_count = len(GRAMMARS) + + for name, (upstream_repo, upstream_branch, parser_path) in sorted(GRAMMARS.items()): + # Fetch latest npm metadata. + info = npm_view_json(name) + fetch_failed = info is None + npm_version = "?" + peer_range = None + peer_optional = True + if info: + npm_version = info.get("version", "?") + peers = info.get("peerDependencies") or {} + peer_range = peers.get("tree-sitter") + meta = info.get("peerDependenciesMeta") or {} + ts_meta = meta.get("tree-sitter") or {} + peer_optional = ts_meta.get("optional", False) if peer_range else True + + if fetch_failed: + peer_display = "? (fetch failed)" + compatible = False + else: + peer_display = peer_range or "none" + if peer_range and not peer_optional: + peer_display += " (required)" + compatible = satisfies_target(peer_range, TARGET_RUNTIME) + + # Check installed ABI using the same parser_path from GRAMMARS. + installed_parser = GITNEXUS_DIR / "node_modules" / name / parser_path + if not installed_parser.is_file(): + # Fallback to default location. + installed_parser = GITNEXUS_DIR / "node_modules" / name / "src" / "parser.c" + installed_abi = extract_language_version(installed_parser) + abi_display = str(installed_abi) if installed_abi else "?" + + # Check upstream (main/master branch) ABI for unreleased work. + upstream_url = ( + f"https://raw.githubusercontent.com/{upstream_repo}/" + f"{upstream_branch}/{parser_path}" + ) + upstream_text = fetch_text(upstream_url) + upstream_abi = extract_abi_from_text(upstream_text) if upstream_text else None + upstream_abi_display = str(upstream_abi) if upstream_abi else "?" + + # Determine status. + if fetch_failed: + status = "Unknown (fetch failed)" + blockers[name] = f"`{name}`: npm registry fetch failed — could not verify peer dep" + elif compatible: + status = "Ready" + ready_count += 1 + elif upstream_abi and upstream_abi >= 15: + status = "Unreleased (ABI 15 on main)" + blockers[name] = f"`{name}`: ABI 15 on `{upstream_repo}` main but not published to npm" + else: + status = "Blocking" + blockers[name] = f"`{name}@{npm_version}`: peer `{peer_display}` incompatible with 0.25" + + # Also check upstream package.json for relaxed peer dep. + if not compatible and not fetch_failed: + upstream_pkg_url = ( + f"https://raw.githubusercontent.com/{upstream_repo}/" + f"{upstream_branch}/package.json" + ) + upstream_pkg_text = fetch_text(upstream_pkg_url) + if upstream_pkg_text: + try: + upstream_pkg = json.loads(upstream_pkg_text) + upstream_peer = (upstream_pkg.get("peerDependencies") or {}).get("tree-sitter") + if upstream_peer and satisfies_target(upstream_peer, TARGET_RUNTIME): + status = "Unreleased (peer relaxed on main)" + blockers[name] = f"`{name}`: peer dep relaxed on `{upstream_repo}` main but not published to npm" + except json.JSONDecodeError: + pass + + compat_icon = "Yes" if compatible else "**No**" + lines.append( + f"| `{name}` | {npm_version} | {peer_display} | {compat_icon} | {abi_display} | {upstream_abi_display} | {status} |" + ) + + lines.append("") + lines.append(f"**{ready_count}/{total_count}** grammars ready for `tree-sitter@{TARGET_RUNTIME}`.") + lines.append("") + + # ── Vendored proto drift ───────────────────────────────────────── + lines.append(md_h("Vendored tree-sitter-proto", 2)) + vendored_abi = extract_language_version(VENDOR_PROTO_DIR / "src" / "parser.c") + + upstream_proto_url = ( + f"https://raw.githubusercontent.com/{UPSTREAM_PROTO_OWNER}/" + f"{UPSTREAM_PROTO_REPO}/{UPSTREAM_PROTO_BRANCH}/src/parser.c" + ) + upstream_proto_text = fetch_text(upstream_proto_url) + upstream_proto_abi = extract_abi_from_text(upstream_proto_text) if upstream_proto_text else None + + sha_url = ( + f"https://api.github.com/repos/{UPSTREAM_PROTO_OWNER}/" + f"{UPSTREAM_PROTO_REPO}/commits/{UPSTREAM_PROTO_BRANCH}" + ) + sha_text = fetch_text(sha_url) + upstream_sha = "?" + if sha_text: + try: + upstream_sha = json.loads(sha_text).get("sha", "?")[:12] + except json.JSONDecodeError: + pass + + local_proto_path = VENDOR_PROTO_DIR / "src" / "parser.c" + local_proto_text = local_proto_path.read_text(encoding="utf-8", errors="ignore") if local_proto_path.is_file() else "" + in_sync = bool( + upstream_proto_text + and local_proto_text.replace("\r\n", "\n") + == upstream_proto_text.replace("\r\n", "\n") + ) + + lines.append(f"- Upstream: `{UPSTREAM_PROTO_OWNER}/{UPSTREAM_PROTO_REPO}@{UPSTREAM_PROTO_BRANCH}` (HEAD `{upstream_sha}`)") + lines.append(f"- Upstream ABI: **{upstream_proto_abi}**") + lines.append(f"- Vendored ABI: **{vendored_abi}**") + lines.append(f"- In sync: {'yes' if in_sync else 'no — upstream has diverged'}") + + if upstream_proto_abi and vendored_abi and upstream_proto_abi > vendored_abi: + can_upgrade = upstream_proto_abi <= target_abi_range[1] + lines.append(f"- Upstream ABI {upstream_proto_abi} {'is' if can_upgrade else 'is NOT'} within target runtime range ({target_abi_range[0]}..{target_abi_range[1]})") + if can_upgrade: + lines.append(f"- **Action:** after upgrading to tree-sitter@{TARGET_RUNTIME}, regenerate vendored parser.c from upstream `{upstream_sha}`") + else: + lines.append(f"- **Action:** wait for runtime upgrade beyond {TARGET_RUNTIME} that supports ABI {upstream_proto_abi}") + blockers["vendored-proto-abi"] = f"vendored tree-sitter-proto: upstream ABI {upstream_proto_abi} outside target range" + elif not in_sync: + lines.append("- **Action:** review upstream changes; vendored copy may need updating") + blockers["vendored-proto-sync"] = "vendored tree-sitter-proto: out of sync with upstream" + + # ── Summary ────────────────────────────────────────────────────── + lines.append("") + lines.append(md_h("Summary", 2)) + if blockers: + lines.append(f"**{len(blockers)} blocker(s) remaining:**\n") + for b in blockers.values(): + lines.append(f"- {b}") + lines.append("") + lines.append("Upgrade to `tree-sitter@0.25` is **blocked**.") + else: + lines.append("All grammars are compatible. Upgrade to `tree-sitter@0.25` is **ready**.") + + print("\n".join(lines)) + return 1 if blockers else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/ci-global-upgrade.yml b/.github/workflows/ci-global-upgrade.yml deleted file mode 100644 index e181f46ad..000000000 --- a/.github/workflows/ci-global-upgrade.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Global Install Upgrade Smoke - -# Catches regressions where `npm install -g gitnexus@` fails to upgrade -# cleanly over a prior global install. Prior precedent: issue #836 and PR #843's -# incomplete fix slipped past CI because no global-upgrade test existed. -# -# Reusable workflow — only callable from ci.yml. Concurrency is governed by the -# caller (ci.yml), so no `concurrency:` block here. - -on: - workflow_call: - -jobs: - global-upgrade: - name: ${{ matrix.os }} / upgrade over ${{ matrix.prior }} - strategy: - fail-fast: false - matrix: - # macOS is the reporter's platform (issue #836) and the highest-risk - # surface for npm global-install rmdir behavior. Linux and Windows - # provide cross-platform regression coverage. - os: [macos-latest, ubuntu-latest, windows-latest] - # Prior version that must be upgraded OVER. Should be a published rc - # that preceded the fix. Bump when a known-bad version changes. - prior: ['1.6.2-rc.8'] - runs-on: ${{ matrix.os }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - uses: ./.github/actions/setup-gitnexus - with: - build: 'false' - - - name: Install prior published version globally - run: npm install -g gitnexus@${{ matrix.prior }} - - - name: Verify prior version installed - run: gitnexus --version - - - name: Pack current branch - working-directory: gitnexus - run: npm pack - shell: bash - - - name: Compute packed tarball path - id: tarball - working-directory: gitnexus - run: | - TARBALL=$(ls gitnexus-*.tgz | head -1) - echo "path=$(pwd)/$TARBALL" >> "$GITHUB_OUTPUT" - shell: bash - - - name: Upgrade over prior version (the actual regression test) - run: npm install -g "${{ steps.tarball.outputs.path }}" - shell: bash - - - name: Verify upgraded version runs - run: gitnexus --version - - - name: Verify vendor/ has no nested node_modules after install - shell: bash - run: | - # The original #836 bug was about vendor/tree-sitter-proto/node_modules/ - # blocking rmdir on upgrade. That is what the fix eliminates. A - # vendor/tree-sitter-proto/build/ directory can still appear because - # node-gyp-build compiles through the npm-created symlink; the - # contents are plain object files and .node binaries that rmdir - # handles fine, evidenced by this test getting past the upgrade step. - GLOBAL_PREFIX=$(npm root -g) - if [ -d "$GLOBAL_PREFIX/gitnexus/vendor/tree-sitter-proto" ]; then - echo "=== Contents of global vendor/tree-sitter-proto/ ===" - ls -la "$GLOBAL_PREFIX/gitnexus/vendor/tree-sitter-proto/" - if [ -d "$GLOBAL_PREFIX/gitnexus/vendor/tree-sitter-proto/node_modules" ]; then - echo "::error::vendor/tree-sitter-proto/node_modules/ was created — this is the #836 hazard" - exit 1 - fi - fi - - ignore-scripts: - name: ${{ matrix.os }} / --ignore-scripts degraded mode - strategy: - fail-fast: false - matrix: - os: [macos-latest, ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - timeout-minutes: 10 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - - uses: ./.github/actions/setup-gitnexus - with: - build: 'false' - - - name: Pack current branch - working-directory: gitnexus - run: npm pack - shell: bash - - - name: Compute packed tarball path - id: tarball - working-directory: gitnexus - run: | - TARBALL=$(ls gitnexus-*.tgz | head -1) - echo "path=$(pwd)/$TARBALL" >> "$GITHUB_OUTPUT" - shell: bash - - - name: Install globally with --ignore-scripts - run: npm install -g --ignore-scripts "${{ steps.tarball.outputs.path }}" - shell: bash - - - name: Verify CLI boots without postinstall (proto parsing may be unavailable) - run: gitnexus --version diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index a1fa33219..ab8d8362c 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -36,7 +36,7 @@ jobs: steps: # ── Download artifacts from the CI run ──────────────────────── - name: Download artifacts - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | const fs = require('fs'); @@ -132,7 +132,7 @@ jobs: - name: Fetch base branch coverage if: steps.meta.outputs.skip != 'true' id: base-coverage - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | const fs = require('fs'); @@ -416,7 +416,7 @@ jobs: - name: Comment on PR if: steps.meta.outputs.skip != 'true' - uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2 + uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v2 with: header: ci-report number: ${{ steps.meta.outputs.pr_number }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2eeeaab2..cf0c6d5c5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,11 +48,6 @@ jobs: permissions: contents: read - global-upgrade: - uses: ./.github/workflows/ci-global-upgrade.yml - permissions: - contents: read - # ── Save PR metadata for the reporting workflow ───────────────── # The ci-report.yml workflow (triggered by workflow_run) needs the # PR number and job results to post a comment. We save them as an @@ -61,7 +56,7 @@ jobs: save-pr-meta: name: Save PR Metadata if: always() && github.event_name == 'pull_request' - needs: [quality, tests, e2e, global-upgrade] + needs: [quality, tests, e2e] runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -72,7 +67,6 @@ jobs: QUALITY: ${{ needs.quality.result }} TESTS: ${{ needs.tests.result }} E2E: ${{ needs.e2e.result }} - GLOBAL_UPGRADE: ${{ needs.global-upgrade.result }} run: | mkdir -p pr-meta echo "$PR_NUMBER" > pr-meta/pr_number @@ -101,7 +95,7 @@ jobs: # Single required check for branch protection. ci-status: name: CI Gate - needs: [quality, tests, e2e, global-upgrade] + needs: [quality, tests, e2e] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -112,12 +106,10 @@ jobs: QUALITY: ${{ needs.quality.result }} TESTS: ${{ needs.tests.result }} E2E: ${{ needs.e2e.result }} - GLOBAL_UPGRADE: ${{ needs.global-upgrade.result }} run: | echo "Quality: $QUALITY" echo "Tests: $TESTS" echo "E2E: $E2E" - echo "Global upgrade: $GLOBAL_UPGRADE" if [[ "$QUALITY" != "success" ]] || [[ "$TESTS" != "success" ]]; then echo "::error::Quality or test jobs failed" @@ -127,7 +119,3 @@ jobs: echo "::error::E2E job failed" exit 1 fi - if [[ "$GLOBAL_UPGRADE" != "success" && "$GLOBAL_UPGRADE" != "skipped" ]]; then - echo "::error::Global upgrade smoke failed" - exit 1 - fi diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index d9d9decf6..e5642cb3e 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -57,7 +57,7 @@ jobs: # For issue_comment triggers, resolve the PR number, head SHA, and fork repo - name: Resolve PR context id: pr - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | let pr; diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index c4a2e9450..553d3ab0d 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -59,7 +59,7 @@ jobs: # For PR-related triggers, resolve the fork repo so we can checkout correctly. - name: Resolve PR context id: pr - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7 with: script: | // Determine if this event is PR-related diff --git a/.github/workflows/pr-description-check.yml b/.github/workflows/pr-description-check.yml index de562fd35..cec32de0f 100644 --- a/.github/workflows/pr-description-check.yml +++ b/.github/workflows/pr-description-check.yml @@ -19,7 +19,7 @@ jobs: timeout-minutes: 5 steps: - name: Check PR description quality - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const MIN_BODY_LENGTH = 50; diff --git a/.github/workflows/pr-labeler.yml b/.github/workflows/pr-labeler.yml index aab2fcaef..3c0c52725 100644 --- a/.github/workflows/pr-labeler.yml +++ b/.github/workflows/pr-labeler.yml @@ -12,7 +12,7 @@ name: PR Conventional Labeler # autolabel (on: pull_request_target) # Needs `pull-requests: write` to apply labels, so must be # pull_request_target. Uses `release-drafter/release-drafter` with -# `disable-releaser: true` to only run the autolabeler against the +# `dry-run: true` to only run the autolabeler against the # `.github/release-drafter.yml` config from the BASE ref (release- # drafter reads the config from the repository's default branch, NOT # the PR head — verify with `gh api repos/release-drafter/release-drafter/contents/...` @@ -57,9 +57,9 @@ jobs: permissions: pull-requests: read steps: - # Pinned to v5.5.3. Verify SHA via: - # gh api repos/amannn/action-semantic-pull-request/git/refs/tags/v5.5.3 - - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 + # Pinned to v6.1.1. Verify SHA via: + # gh api repos/amannn/action-semantic-pull-request/git/refs/tags/v6.1.1 + - uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -102,12 +102,12 @@ jobs: contents: read pull-requests: write steps: - # Pinned to v6.0.0. Verify SHA via: - # gh api repos/release-drafter/release-drafter/git/refs/tags/v6.0.0 - # Note: dependabot will likely propose a bump to v6.x on first run. - - uses: release-drafter/release-drafter@3f0f87098bd6b5c5b9a36d49c41d998ea58f9348 # v6.0.0 + # Pinned to v7.2.0. Verify SHA via: + # gh api repos/release-drafter/release-drafter/git/refs/tags/v7.2.0 + # v7 removed `disable-releaser`; use `dry-run: true` to only autolabel. + - uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0 with: config-name: release-drafter.yml - disable-releaser: true + dry-run: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 03898a267..3d883425a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -91,7 +91,7 @@ jobs: fi - name: Create GitHub Release - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 with: body_path: ${{ steps.changelog.outputs.fallback == 'false' && '/tmp/release-notes.md' || '' }} generate_release_notes: ${{ steps.changelog.outputs.fallback == 'true' }} diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index cc22f4749..d4db75db0 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -346,7 +346,7 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - name: Create GitHub prerelease - uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2 + uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 with: tag_name: ${{ steps.reltag.outputs.vtag }} name: Release Candidate ${{ steps.reltag.outputs.vtag }} diff --git a/.github/workflows/tree-sitter-upgrade-readiness.yml b/.github/workflows/tree-sitter-upgrade-readiness.yml new file mode 100644 index 000000000..74ce72a27 --- /dev/null +++ b/.github/workflows/tree-sitter-upgrade-readiness.yml @@ -0,0 +1,185 @@ +name: Tree-sitter Upgrade Readiness + +# Monitors readiness for upgrading tree-sitter to 0.25.x. Tracks: +# 1. Peer-dep compatibility — can each grammar install cleanly with +# tree-sitter@0.25.0 without --legacy-peer-deps? +# 2. Vendored proto drift — has coder3101/tree-sitter-proto moved +# ahead of our vendored snapshot? +# See .github/scripts/check-tree-sitter-upgrade-readiness.py for the logic. +# +# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". + +on: + schedule: + # Daily at 09:00 UTC. Matches Dependabot's daily cadence so drift + # and dep PRs surface together. + - cron: '0 9 * * *' + workflow_dispatch: + pull_request: + paths: + - '.github/scripts/check-tree-sitter-upgrade-readiness.py' + - '.github/workflows/tree-sitter-upgrade-readiness.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + readiness: + name: Check upgrade readiness + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + # Needed to open/update the tracking issue on scheduled runs. + issues: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - uses: ./.github/actions/setup-gitnexus + with: + build: 'false' + + - name: Run upgrade readiness check + id: readiness + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + python3 .github/scripts/check-tree-sitter-upgrade-readiness.py > drift-report.md + code=$? + set -e + echo "exit_code=$code" >> "$GITHUB_OUTPUT" + { + echo 'report<> "$GITHUB_OUTPUT" + echo "=== Report ===" + cat drift-report.md + + # On PR runs, the script validates that it runs correctly. Blockers + # are informational — the scheduled run opens a tracking issue. + - name: Annotate PR with readiness status + if: github.event_name == 'pull_request' && steps.readiness.outputs.exit_code != '0' + run: | + echo "::warning::Tree-sitter 0.25 upgrade has blockers. See job output for the full readiness report." + + - name: Upsert tracking issue on scheduled runs + if: > + github.event_name == 'schedule' && + steps.readiness.outputs.exit_code != '0' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + REPORT: ${{ steps.readiness.outputs.report }} + with: + script: | + const title = 'Tree-sitter 0.25 upgrade readiness'; + const report = process.env.REPORT; + const body = report + '\n\n' + + 'Generated daily by `.github/workflows/tree-sitter-upgrade-readiness.yml`. ' + + 'Closes automatically when all blockers are resolved.'; + const { data: open } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'tree-sitter-drift', + per_page: 10, + }); + const existing = open.find(i => i.title === title); + if (existing) { + // Extract ready/total count for the changelog comment. + const readyMatch = report.match(/\*\*(\d+)\/(\d+)\*\* grammars ready/); + const blockerMatch = report.match(/\*\*(\d+) blocker/); + const ready = readyMatch ? readyMatch[1] : '?'; + const total = readyMatch ? readyMatch[2] : '?'; + const blockers = blockerMatch ? blockerMatch[1] : '?'; + + // Find grammars whose status changed by diffing the old and + // new table rows. Each row looks like: + // | `tree-sitter-foo` | ... | Ready | + // | `tree-sitter-foo` | ... | Blocking | + const parseRows = (md) => { + const map = {}; + for (const m of md.matchAll(/\| `(tree-sitter-[^`]+)` \|.*?\| (\S+(?:\s\S+)*?) \|$/gm)) { + map[m[1]] = m[2].trim(); + } + return map; + }; + const oldRows = parseRows(existing.body || ''); + const newRows = parseRows(report); + const changes = []; + for (const [name, newStatus] of Object.entries(newRows)) { + const oldStatus = oldRows[name]; + if (oldStatus && oldStatus !== newStatus) { + changes.push(`\`${name}\`: ${oldStatus} → ${newStatus}`); + } + } + + const today = new Date().toISOString().slice(0, 10); + let comment = `**${today}:** ${ready}/${total} ready. ${blockers} blocker(s) remaining.`; + if (changes.length > 0) { + comment += '\n\nChanges:\n' + changes.map(c => `- ${c}`).join('\n'); + } else { + comment += ' No changes from previous run.'; + } + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: comment, + }); + + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body, + }); + core.info(`Updated existing issue #${existing.number}`); + } else { + const { data: created } = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title, + body, + labels: ['tree-sitter-drift', 'dependencies'], + }); + core.info(`Opened issue #${created.number}`); + } + + - name: Close tracking issue on clean scheduled runs + if: > + github.event_name == 'schedule' && + steps.readiness.outputs.exit_code == '0' + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const title = 'Tree-sitter 0.25 upgrade readiness'; + const { data: open } = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: 'tree-sitter-drift', + per_page: 10, + }); + const existing = open.find(i => i.title === title); + if (existing) { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + body: 'All grammars are now compatible with tree-sitter@0.25. Upgrade is ready! Closing automatically.', + }); + await github.rest.issues.update({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existing.number, + state: 'closed', + }); + core.info(`Closed issue #${existing.number}`); + } diff --git a/.gitignore b/.gitignore index c775f0ba5..95c9164e0 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,4 @@ gitnexus/vendor/**/node_modules/ .swarm/ local_docs/ + diff --git a/AGENTS.md b/AGENTS.md index 651657b02..f212fae0c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,117 +1,120 @@ - - + + -Last reviewed: 2026-04-13 +Last reviewed: 2026-04-16 **Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub) -This file uses a standard agent header (version, scope, model policy, reference docs, changelog), adapted for this **TypeScript/JavaScript monorepo**. - ## Scope -| | | -|--|--| -| **Reads** | Repository tree as needed for the task: `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/` when present, and docs. | -| **Writes** | Only paths required for the requested change; keep diffs minimal. Update lockfiles when dependencies change. | -| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/` when applicable; shell utilities for documented CI/dev workflows. | -| **Off-limits** | User secrets (e.g. real `.env`), production deployment credentials, unrelated repositories, destructive git history operations without explicit human confirmation. | +| Boundary | Rule | +|----------|------| +| **Reads** | `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/`, docs. | +| **Writes** | Only paths required for the change; keep diffs minimal. Update lockfiles when deps change. | +| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/`; documented CI/dev workflows. | +| **Off-limits** | Real `.env` / secrets, production credentials, unrelated repos, destructive git ops without confirmation. | ## Model Configuration -- **Primary:** Pin in **Cursor** (Settings → model). Use a **named** model (e.g. GPT-5.2, Claude Sonnet 4.x). Avoid relying on **Auto** when reproducibility or audit trail matters. -- **Fallback:** As configured in Cursor or your organization (do not encode `latest` or wildcards in automation configs). -- **Notes:** The open-source GitNexus CLI indexer does not call an LLM. Optional Nexus AI in the web UI uses end-user provider keys and models. +- **Primary:** Use a named model (e.g. Claude Sonnet 4.x). Avoid `Auto` or unversioned `latest` when reproducibility matters. +- **Notes:** The GitNexus CLI indexer does not call an LLM. ## Execution Sequence (complex tasks) -Long sessions dilute instructions. For **multi-step** work, state up front: - +For multi-step work, state up front: 1. Which rules in this file and **[GUARDRAILS.md](GUARDRAILS.md)** apply (and any relevant Signs). -2. Current **Scope** boundaries (Reads / Writes / Off-limits). -3. Which **validation commands** you will run (e.g. `cd gitnexus && npm test`, `npx tsc --noEmit`). +2. Current **Scope** boundaries. +3. Which **validation commands** you will run (`cd gitnexus && npm test`, `npx tsc --noEmit`). -On very long threads, the human may add *“Remember: apply all AGENTS.md rules”* to re-weight rule tokens against context dilution. +On long threads, *"Remember: apply all AGENTS.md rules"* re-weights these instructions against context dilution. ## Claude Code hooks -Hooks enforce gates that prompts cannot. In **Claude Code**, **PreToolUse** hooks can block tools such as `git_commit` until checks pass. Adapt to this repo: e.g. `cd gitnexus && npm test` before commit. +**PreToolUse** hooks can block tools (e.g. `git_commit`) until checks pass. Adapt to this repo: `cd gitnexus && npm test` before commit. -## Context budget (Cursor / standards) +## Context budget -Generic “core standards” playbooks are often long and stack-specific. For this monorepo, commands and gotchas live under **Cursor Cloud specific instructions** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split domain rules into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules live in **`.cursor/index.mdc`** (YAML frontmatter with `alwaysApply: true`). **Claude Code:** optionally load a **`STANDARDS.md`** only when needed (e.g. *“When writing new code, read STANDARDS.md”*) to save context. +Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules in `.cursor/index.mdc`. **Claude Code:** load `STANDARDS.md` only when needed. -## Reference Documentation +## Reference docs -- **This repository:** **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)**. -- **Cursor:** `.cursor/index.mdc` (always-on rules); optional `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` is deprecated — see `.cursor/index.mdc`. -- **Optional local files:** `NOTES.md` (short vendor-neutral project snapshot). For handoffs, keep notes local (e.g., a scratch file outside the repo) rather than committing `HANDOFF.md`. -- **GitNexus:** skills under `.claude/skills/gitnexus/`; machine-oriented rules in the `gitnexus:start` … `gitnexus:end` block below. +- **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)** +- **Cursor:** `.cursor/index.mdc` (always-on); `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` deprecated. +- **GitNexus:** skills in `.claude/skills/gitnexus/`; MCP rules in `gitnexus:start` block below. ## Changelog | Date | Version | Change | |------|---------|--------| +| 2026-04-16 | 1.4.0 | Fixed: web UI description, pre-commit behavior, MCP tools (7->16), added gitnexus-shared, removed stale vite-plugin-wasm gotcha. | | 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. | -| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication (was inlined in Reference Docs bullet). | -| 2026-03-23 | 1.1.0 | Updated agent instructions (sections, references, Cursor layout). | -| 2026-03-22 | 1.0.0 | Added structured agent header and changelog. | +| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication. | +| 2026-03-23 | 1.1.0 | Updated agent instructions, references, Cursor layout. | +| 2026-03-22 | 1.0.0 | Initial structured header and changelog. | --- # GitNexus — Code Intelligence -This project is indexed by GitNexus as **GitNexus** (4325 symbols, 10556 relationships, 300 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely. +Indexed as **GitNexus** (4325 symbols, 10556 relationships, 300 execution flows). Use MCP tools to understand code, assess impact, and navigate safely. -> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first. +> If any tool warns the index is stale, run `npx gitnexus analyze` first. ## Always Do -- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user. -- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows. -- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits. -- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance. -- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`. +- **MUST run impact analysis before editing any symbol.** `gitnexus_impact({target: "symbolName", direction: "upstream"})` — report blast radius to the user. +- **MUST run `gitnexus_detect_changes()` before committing** — verify only expected symbols and flows are affected. +- **MUST warn the user** if impact returns HIGH or CRITICAL risk. +- Explore unfamiliar code with `gitnexus_query({query: "concept"})` (process-grouped, ranked) instead of grepping. +- Full context on a symbol: `gitnexus_context({name: "symbolName"})`. ## When Debugging -1. `gitnexus_query({query: ""})` — find execution flows related to the issue -2. `gitnexus_context({name: ""})` — see all callers, callees, and process participation -3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step -4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed +1. `gitnexus_query({query: ""})` — find related execution flows +2. `gitnexus_context({name: ""})` — callers, callees, process participation +3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace flow step by step +4. Regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` ## When Refactoring -- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`. -- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code. -- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed. +- **Rename:** `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Graph edits are safe; text_search edits need manual review. +- **Extract/Split:** `gitnexus_context` (incoming/outgoing refs) then `gitnexus_impact` (upstream callers) before moving code. +- **After any refactor:** `gitnexus_detect_changes({scope: "all"})` to verify scope. ## Never Do -- NEVER edit a function, class, or method without first running `gitnexus_impact` on it. -- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis. -- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph. -- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope. +- Edit a symbol without running `gitnexus_impact` first. +- Ignore HIGH/CRITICAL risk warnings. +- Rename with find-and-replace — use `gitnexus_rename`. +- Commit without `gitnexus_detect_changes()`. ## Tools Quick Reference -| Tool | When to use | Command | +| Tool | When to use | Example | |------|-------------|---------| +| `list_repos` | Discover indexed repos | `gitnexus_list_repos({})` | | `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` | | `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` | | `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` | | `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` | | `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` | | `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` | +| `api_impact` | Pre-change API route impact | `gitnexus_api_impact({route: "/api/users", method: "GET"})` | +| `route_map` | Route → handler → consumer map | `gitnexus_route_map({})` | +| `tool_map` | MCP/RPC tool definitions | `gitnexus_tool_map({})` | +| `shape_check` | Response shape vs consumer access | `gitnexus_shape_check({route: "/api/users"})` | +| `group_list` | List repo groups | `gitnexus_group_list({})` | +| `group_query` | Cross-repo search in a group | `gitnexus_group_query({name: "myGroup", query: "auth"})` | +| `group_sync` | Rebuild group Contract Registry | `gitnexus_group_sync({name: "myGroup"})` | +| `group_contracts` | Inspect group contracts | `gitnexus_group_contracts({name: "myGroup"})` | +| `group_status` | Group staleness report | `gitnexus_group_status({name: "myGroup"})` | ## Impact Risk Levels | Depth | Meaning | Action | |-------|---------|--------| -| d=1 | WILL BREAK — direct callers/importers | MUST update these | +| d=1 | WILL BREAK — direct callers/importers | MUST update | | d=2 | LIKELY AFFECTED — indirect deps | Should test | | d=3 | MAY NEED TESTING — transitive | Test if critical path | @@ -119,87 +122,80 @@ This project is indexed by GitNexus as **GitNexus** (4325 symbols, 10556 relatio | Resource | Use for | |----------|---------| -| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness | +| `gitnexus://repo/GitNexus/context` | Codebase overview, index freshness | | `gitnexus://repo/GitNexus/clusters` | All functional areas | | `gitnexus://repo/GitNexus/processes` | All execution flows | | `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace | ## Self-Check Before Finishing -Before completing any code modification task, verify: 1. `gitnexus_impact` was run for all modified symbols -2. No HIGH/CRITICAL risk warnings were ignored -3. `gitnexus_detect_changes()` confirms changes match expected scope -4. All d=1 (WILL BREAK) dependents were updated +2. No HIGH/CRITICAL warnings were ignored +3. `gitnexus_detect_changes()` confirms expected scope +4. All d=1 dependents were updated ## Keeping the Index Fresh -After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it: - ```bash -npx gitnexus analyze +npx gitnexus analyze # basic refresh +npx gitnexus analyze --embeddings # preserve embeddings ``` -If the index previously included embeddings, preserve them by adding `--embeddings`: +Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). Running without `--embeddings` deletes existing vectors. -```bash -npx gitnexus analyze --embeddings -``` +> Claude Code: PostToolUse hook handles this after `git commit` and `git merge`. -To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.** +## CLI Skills -> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`. - -## CLI - -| Task | Read this skill file | -|------|---------------------| -| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | -| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | -| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | -| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | -| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | -| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | +| Task | Skill file | +|------|-----------| +| Architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` | +| Blast radius / "What breaks?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` | +| Debugging / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` | +| Refactoring | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` | +| Tools/resources/schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` | +| CLI commands (index, status, clean, wiki) | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` | -## Cursor Cloud specific instructions +## Repo reference -### Repository structure +### Packages -This is a monorepo with two main products and supporting config packages: - -| Component | Path | Purpose | -|-----------|------|---------| -| **GitNexus CLI/Core** | `gitnexus/` | Main product — TypeScript CLI, indexing pipeline, MCP server. Published to npm. | -| **GitNexus Web UI** | `gitnexus-web/` | React/Vite browser app — graph explorer + AI chat. Runs entirely in WASM. | -| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace (no build). | -| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor (no build). | -| SWE-bench Eval | `eval/` | Python evaluation harness (optional; needs Docker + LLM API keys). | +| Package | Path | Purpose | +|---------|------|---------| +| **CLI/Core** | `gitnexus/` | TypeScript CLI, indexing pipeline, MCP server. Published to npm. | +| **Web UI** | `gitnexus-web/` | React/Vite thin client. All queries via `gitnexus serve` HTTP API. | +| **Shared** | `gitnexus-shared/` | Shared TypeScript types and constants. | +| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace. | +| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor. | +| Eval | `eval/` | Python evaluation harness (Docker + LLM API keys). | ### Running services -- **CLI/Core**: `cd gitnexus && npm run dev` (tsx watch mode) or `npm run build && node dist/cli/index.js ` -- **Web UI**: `cd gitnexus-web && npm run dev` (Vite on port 5173) -- **Backend mode**: `cd && node /workspace/gitnexus/dist/cli/index.js serve` (HTTP API on port 3741 by default) +```bash +cd gitnexus && npm run dev # CLI: tsx watch mode +cd gitnexus-web && npm run dev # Web UI: Vite on port 5173 +npx gitnexus serve # HTTP API on port 4747 (from any indexed repo) +``` ### Testing **CLI / Core (`gitnexus/`)** -- **Unit tests**: `cd gitnexus && npm test` (vitest, ~2000 tests) -- **Integration tests**: `cd gitnexus && npm run test:integration` (vitest, ~1850 tests). Two LadybugDB file-locking tests (`lbug-core-adapter`, `search-core`) may fail in containerized environments due to `/tmp` locking limitations — this is a known environment issue, not a code bug. -- **TypeScript check**: `cd gitnexus && npx tsc --noEmit` +- `npm test` — full vitest suite (~2000 tests) +- `npm run test:unit` — unit tests only +- `npm run test:integration` — integration (~1850 tests). LadybugDB file-locking tests may fail in containers (known env issue). +- `npx tsc --noEmit` — typecheck **Web UI (`gitnexus-web/`)** -- **Unit tests**: `cd gitnexus-web && npm test` (vitest, ~200 tests) -- **E2E tests**: `cd gitnexus-web && E2E=1 npx playwright test` (Playwright, 5 tests — requires `gitnexus serve` + `npm run dev` running) -- **TypeScript check**: `cd gitnexus-web && npx tsc -b --noEmit` +- `npm test` — vitest (~200 tests) +- `npm run test:e2e` — Playwright (7 spec files; requires `gitnexus serve` + `npm run dev`) +- `npx tsc -b --noEmit` — typecheck -No separate lint command is configured; TypeScript strict checking serves as the primary static analysis. +**Pre-commit hook** (`.husky/pre-commit`): formatting (prettier via lint-staged) + typecheck for staged packages. Tests do **not** run in pre-commit — CI only. ### Gotchas -- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift). Native tree-sitter bindings require `python3`, `make`, and `g++` to be present. -- `tree-sitter-kotlin` and `tree-sitter-swift` are optional dependencies — install warnings for these are expected and non-blocking. -- The Web UI uses `vite-plugin-wasm` and requires `Cross-Origin-Opener-Policy`/`Cross-Origin-Embedder-Policy` headers for `SharedArrayBuffer` (handled automatically by Vite dev server). -- There is no ESLint/Prettier configuration in this repo. +- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift, builds tree-sitter-proto). Native bindings need `python3`, `make`, `g++`. +- `tree-sitter-kotlin` and `tree-sitter-swift` are optional — install warnings expected. +- ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ac4f46aef..76fd3bcb6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,99 +1,129 @@ # Architecture — GitNexus -This repository is a **monorepo** with two main products: the **CLI / MCP package** (`gitnexus/`) and the **browser UI** (`gitnexus-web/`). Supporting folders ship editor integrations and plugins without changing the core graph engine. +Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`). ## Repository layout | Path | Role | |------|------| -| `gitnexus/` | Published npm package `gitnexus`: CLI, MCP server (stdio), local HTTP API for bridge mode, ingestion pipeline, LadybugDB graph, embeddings (optional). | -| `gitnexus-web/` | Vite + React UI: in-browser indexing (WASM), graph visualization, optional connection to `gitnexus serve`. | -| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Packaged **skills** and plugin metadata so agents discover the same workflows as documented in `AGENTS.md`. | -| `eval/` | Evaluation harnesses and docs for benchmarking tool usage. | -| `.github/` | CI workflows (quality, unit, integration, E2E) and composite actions. | +| `gitnexus/` | npm package `gitnexus`: CLI, MCP server (stdio), HTTP API, ingestion pipeline, LadybugDB graph, embeddings. | +| `gitnexus-web/` | Vite + React thin client: graph explorer + AI chat. All queries via `gitnexus serve` HTTP API. | +| `gitnexus-shared/` | Shared TypeScript types and constants (consumed by CLI and Web). | +| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Agent skills and plugin metadata. | +| `eval/` | Evaluation harnesses for benchmarking tool usage. | +| `.github/` | CI workflows + composite actions (`setup-gitnexus/`, `setup-gitnexus-web/`). | ## End-to-end flow: index → graph → tools -1. **Ingestion** (`gitnexus analyze`) - - Entry: `gitnexus/src/cli/analyze.ts` → `runPipelineFromRepo` in `gitnexus/src/core/ingestion/pipeline.ts`. - - The pipeline is structured as a **DAG (Directed Acyclic Graph)** of named phases (see [Pipeline Phase DAG](#pipeline-phase-dag) below). - - Output is loaded into **LadybugDB** under **`.gitnexus/`** at the repo root (`lbug/`, `meta.json`, etc.). Optional **FTS** indexes and **embeddings** attach to the same store. - - The repo is registered in **`~/.gitnexus/registry.json`** so MCP can find it from any working directory. +1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 12 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery. -2. **Persistence & metadata** - - `gitnexus/src/storage/repo-manager.ts` — paths, registry, cleanup of legacy Kuzu artifacts. - - `gitnexus/src/core/lbug/lbug-adapter.ts` — graph load, queries, embedding restore batches. +2. **Persistence** — `repo-manager.ts` (paths, registry, KuzuDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches). -3. **Query & agents** - - **MCP (stdio):** `gitnexus/src/cli/mcp.ts` → `startMCPServer` → `LocalBackend` (`gitnexus/src/mcp/local/local-backend.ts`) opens registered repos and serves **tools** from `gitnexus/src/mcp/tools.ts` and **resources** from `gitnexus/src/mcp/resources.ts`. - - **Bridge HTTP:** `gitnexus/src/cli/serve.ts` → Express app in `gitnexus/src/server/api.ts` (CORS-limited) exposes REST + MCP-over-HTTP for the web UI. - - **CLI tools (no MCP):** `gitnexus query`, `context`, `impact`, `cypher` in `gitnexus/src/cli/tool.ts` call the same backend for scripts and CI. +3. **Query layer** — three interfaces to the same backend: + - **MCP (stdio):** `mcp.ts` → `LocalBackend` → tools (`tools.ts`) + resources (`resources.ts`) + - **HTTP bridge:** `serve.ts` → Express (`api.ts`, `mcp-http.ts`) for web UI + - **CLI direct:** `gitnexus query|context|impact|cypher` in `tool.ts` -4. **Staleness** - - `gitnexus/src/mcp/staleness.ts` compares indexed `lastCommit` to `HEAD` and surfaces hints when the graph is behind git. +4. **Staleness** — `staleness.ts` compares indexed `lastCommit` to `HEAD`, surfaces hints. -## MCP tools (summary) +## MCP tools | Tool | Purpose | |------|---------| -| `list_repos` | Discover indexed repositories when more than one is registered. | -| `query` | Natural-language / keyword search over the graph (hybrid BM25 + optional vectors). | -| `cypher` | Ad hoc **Cypher** against the schema (see resource `gitnexus://repo/{name}/schema`). | -| `context` | Callers, callees, processes for one symbol (with disambiguation). | -| `impact` | Blast radius (upstream/downstream) with depth and risk summary. | -| `detect_changes` | Map git diffs to affected symbols and processes. | -| `rename` | Graph-assisted rename with `dry_run` preview (`graph` vs `text_search` confidence). | +| `list_repos` | Discover indexed repos | +| `query` | Hybrid BM25 + vector search over the graph | +| `cypher` | Ad hoc Cypher against the schema | +| `context` | Callers, callees, processes for one symbol | +| `impact` | Blast radius (upstream/downstream) with risk summary | +| `detect_changes` | Map git diffs to affected symbols and processes | +| `rename` | Graph-assisted multi-file rename with `dry_run` preview | +| `api_impact` | Pre-change impact report for an API route handler | +| `route_map` | API route → handler → consumer mappings | +| `tool_map` | MCP/RPC tool definitions and handlers | +| `shape_check` | Response shape vs consumer property access mismatches | +| `group_list` | List repo groups or details for one group | +| `group_query` | Cross-repo search in a group (reciprocal rank fusion) | +| `group_sync` | Rebuild group Contract Registry (`contracts.json`) | +| `group_contracts` | Inspect group contracts and cross-links | +| `group_status` | Index and Contract Registry staleness per repo in a group | ## Where to change what -| If you are changing… | Start in… | -|----------------------|-----------| -| CLI commands / flags | `gitnexus/src/cli/` (`index.ts`, per-command modules). | -| Parsing or graph construction | `gitnexus/src/core/ingestion/pipeline-phases/` (individual phase files), `pipeline.ts` (orchestrator). | -| Graph schema / DB access | `gitnexus/src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`), `gitnexus/src/mcp/core/lbug-adapter.ts` if MCP-specific. | -| MCP protocol, tools, resources | `gitnexus/src/mcp/server.ts`, `tools.ts`, `resources.ts`. | -| Search ranking | `gitnexus/src/core/search/` (BM25, hybrid fusion). | -| Embeddings | `gitnexus/src/core/embeddings/`, phases in `analyze.ts`. | -| Wiki generation | `gitnexus/src/core/wiki/`. | -| Web UI behavior | `gitnexus-web/src/` (components, workers, graph client). | -| CI | `.github/workflows/*.yml`, `.github/actions/setup-gitnexus/`. | +| Concern | Start in | +|---------|----------| +| CLI commands/flags | `src/cli/` (`index.ts`, per-command modules) | +| Parsing/graph construction | `src/core/ingestion/pipeline-phases/` + `pipeline.ts` | +| Graph schema/DB | `src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`) | +| MCP tools/resources | `src/mcp/server.ts`, `tools.ts`, `resources.ts` | +| Search ranking | `src/core/search/` (BM25, hybrid fusion) | +| Embeddings | `src/core/embeddings/` + `src/core/run-analyze.ts` | +| Wiki generation | `src/core/wiki/` | +| Language support | `src/core/ingestion/languages/` + `tree-sitter-queries.ts` + `gitnexus-shared/src/languages.ts` | +| Import resolution | `src/core/ingestion/import-processor.ts` + `import-resolvers/configs/` + `model/resolution-context.ts` | +| Call resolution/MRO | `src/core/ingestion/call-processor.ts` + `model/resolve.ts` | +| Type extraction | `src/core/ingestion/type-extractors/` | +| Worker pool | `src/core/ingestion/workers/` | +| Web UI | `gitnexus-web/src/` | +| CI | `.github/workflows/*.yml`, `.github/actions/` | + +> Paths above are relative to `gitnexus/` unless they start with `gitnexus-web/` or `.github/`. + +--- ## Pipeline Phase DAG -The ingestion pipeline is a DAG of named phases. Each phase is defined in its own file under `gitnexus/src/core/ingestion/pipeline-phases/` with explicit dependencies, typed inputs, and typed outputs. +12 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output. ``` scan → structure → [markdown, cobol] → parse → [routes, tools, orm] → crossFile → mro → communities → processes ``` -### Phase files +| Phase | File | Deps | Output | +|-------|------|------|--------| +| `scan` | `scan.ts` | (root) | File paths + sizes | +| `structure` | `structure.ts` | `scan` | File/Folder nodes, CONTAINS edges, `allPathSet` | +| `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx | +| `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) | +| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries | +| `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators) | +| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges | +| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) | +| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order | +| `mro` | `mro.ts` | `crossFile`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges | +| `communities` | `communities.ts` | `mro`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) | +| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `structure` | Process nodes + STEP_IN_PROCESS edges | -| Phase | File | Dependencies | What it does | -|-------|------|-------------|--------------| -| `scan` | `scan.ts` | (root) | Walk repo filesystem, collect paths + sizes | -| `structure` | `structure.ts` | `scan` | Build File/Folder nodes + CONTAINS edges | -| `markdown` | `markdown.ts` | `structure` | Extract headings and cross-links from .md/.mdx | -| `cobol` | `cobol.ts` | `structure` | Regex-based COBOL/JCL extraction | -| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Chunked tree-sitter parse, import/call/heritage resolution | -| `routes` | `routes.ts` | `parse` | Route registry (Next.js, Expo, PHP, decorator-based) | -| `tools` | `tools.ts` | `parse` | MCP/RPC tool detection | -| `orm` | `orm.ts` | `parse` | Prisma/Supabase ORM query edges | -| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological order | -| `mro` | `mro.ts` | `crossFile` | Method Resolution Order, METHOD_OVERRIDES edges | -| `communities` | `communities.ts` | `mro` | Leiden community detection | -| `processes` | `processes.ts` | `communities`, `routes`, `tools` | Execution flow detection, Route/Tool → Process links | +**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `orm-extraction.ts` (sequential ORM fallback), `types.ts`, `runner.ts`, `index.ts`. + +### DAG runner + +`runner.ts` — static phase graph, no plugins, compile-time type safety. + +1. **Validation** — Kahn's topological sort. Rejects on: duplicate names, missing deps, cycles (DFS traces the concrete cycle path, e.g., `A -> B -> C -> A`, plus count of transitively blocked dependents). + +2. **Execution** — sequential in topological order. Each phase receives: + - `ctx: PipelineContext` — shared mutable `KnowledgeGraph`, `repoPath`, progress callback, options + - `deps: ReadonlyMap` — **declared deps only** (runner filters the results map to prevent hidden coupling) + +3. **Error handling** — wraps phase errors with the phase name, emits terminal `error` progress event, swallows progress handler errors to preserve the original cause. + +4. **Timing** — per-phase `durationMs` in `PhaseResult`, dev-mode console logging. + +**Design patterns:** +- **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output. +- **Typed phase access** — `getPhaseOutput(deps, 'name')` for type-safe upstream results. +- **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership. +- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests). `skipWorkers` forces sequential parsing. ### How to add a new phase -1. Create a new file in `pipeline-phases/` (e.g. `my-phase.ts`) -2. Define a `PipelinePhase` object with `name`, `deps`, and `execute(ctx, deps)` -3. Export it from `pipeline-phases/index.ts` -4. Add it to the `buildPhaseList()` function in `pipeline.ts` +1. Create `pipeline-phases/my-phase.ts` with a `PipelinePhase` (name, deps, execute) +2. Export from `pipeline-phases/index.ts` +3. Add to `buildPhaseList()` in `pipeline.ts` ```typescript -// pipeline-phases/my-phase.ts -import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js'; +import type { PipelinePhase, PhaseResult } from './types.js'; import { getPhaseOutput } from './types.js'; import type { ParseOutput } from './parse.js'; @@ -101,81 +131,170 @@ export interface MyPhaseOutput { /* ... */ } export const myPhase: PipelinePhase = { name: 'myPhase', - deps: ['parse'], // runs after parse completes + deps: ['parse'], async execute(ctx, deps) { const { allPaths } = getPhaseOutput(deps, 'parse'); - // ... do work, write to ctx.graph ... + // ... write to ctx.graph ... return { /* typed output */ }; }, }; ``` -### DAG runner +--- -The runner (`pipeline-phases/runner.ts`) validates the DAG at startup (detects cycles and missing deps via topological sort), then executes phases in dependency order. Each phase receives: -- `ctx: PipelineContext` — shared graph, repoPath, progress callback -- `deps: Map` — outputs from all upstream phases +## Language-agnostic graph feeding + +16 languages → single unified graph. Four abstraction layers: + +``` + Unified Graph Schema (44 node types, 21 relationship types) + ↑ + Unified Resolution (3-tier name lookup + MRO walk) + ↑ + Language Providers (import semantics, type config, export checker, MRO strategy) + ↑ + Tree-Sitter Queries (per-language S-expressions, unified capture tags) +``` + +### Language providers + +Each language implements `LanguageProvider` (`language-provider.ts`). Key fields: + +| Field | Purpose | +|-------|---------| +| `id`, `extensions` | Language identity and file matching | +| `treeSitterQueries` | S-expression queries for AST extraction | +| `importSemantics` | `named` / `wildcard-leaf` / `wildcard-transitive` / `namespace` | +| `importResolver` | Language-specific path → file resolution | +| `exportChecker` | Public/exported symbol detection | +| `typeConfig` | Type annotation extraction rules | +| `mroStrategy` | `first-wins` / `c3` / `none` | + +16 providers in `languages/index.ts` via `satisfies Record` — missing a language is a compile error. + +### Unified capture tags + +Per-language tree-sitter queries use different AST node names but produce the **same semantic capture tags**: `@definition.class`, `@definition.function`, `@call.name`, `@import.source`, `@heritage.extends`. Downstream extraction needs no language branching. Defined in `tree-sitter-queries.ts`. + +### Import resolution + +Per-language import resolution uses the **configs + factory** pattern (like call/method/class extractors). Each language declares an `ImportResolutionConfig` in `import-resolvers/configs/`, listing an ordered chain of `ImportResolverStrategy` functions. `createImportResolver()` (in `resolver-factory.ts`) composes them: first non-null result wins. Low-level helpers shared across strategies live alongside the configs in `import-resolvers/` (e.g. `go.ts`, `rust.ts`, `python.ts`). + +Unified 3-tier algorithm (`model/resolution-context.ts`), per-language `importSemantics` controls which tier activates: + +| Tier | Confidence | Mechanism | +|------|-----------|-----------| +| 1 — same-file | 0.95 | Symbol table for caller's file | +| 2 — import-scoped | 0.9 | `NamedImportMap` chains (named) or all files in `importMap` (wildcard) | +| 3 — global | 0.5 | O(1) index lookups: class, impl, callable. Fallback only | + +| Import strategy | Languages | Behavior | +|----------------|-----------|----------| +| `named` | TS, JS, Java, C#, Rust, PHP, Kotlin | Only explicitly imported names visible | +| `wildcard-leaf` | Go, Ruby, Swift, Dart | Whole-package import, no transitive re-exports | +| `wildcard-transitive` | C, C++ | `#include` closure chains through re-exports | +| `namespace` | Python | Module aliases resolved at call site | + +### Chunked parse-and-resolve + +`parse` processes files in ~20 MB byte-budget chunks to bound memory. Per chunk: +1. Worker pool dispatches files (or sequential fallback via `skipWorkers`) +2. Each worker: detect language → load grammar → run queries → return unified `ParseWorkerResult` +3. Synthesize wildcard bindings (`wildcard-synthesis.ts`) +4. Resolve imports and heritage +5. Collect `BindingAccumulator` entries for cross-file propagation + +Workers: `workers/worker-pool.ts`, `workers/parse-worker.ts`. + +### Heritage and MRO + +All languages emit unified `ExtractedHeritage` (child, parent, `EXTENDS`/`IMPLEMENTS`). MRO phase walks the heritage graph using per-language strategy: +- **`first-wins`** — Java, C#, C++, TS, Ruby, Go +- **`c3`** — Python (C3 linearization) +- **`none`** — single-inheritance languages + +Unified walk: `lookupMethodByOwnerWithMRO()` in `model/resolve.ts`. + +--- + +## Full analysis flow + +`runFullAnalysis` in `run-analyze.ts` orchestrates everything around the pipeline: + +``` +CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks) + 1. Early exit if lastCommit == HEAD (unless --force) [0%] + 2. Cache existing embeddings from prior index [0%] + 3. runPipelineFromRepo() → KnowledgeGraph [0-60%] + 4. Clean up legacy KuzuDB files [60%] + 5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%] + 6. Create FTS indexes (File, Function, Class, Method...) [85-90%] + 7. Restore cached embeddings (batch insert) [88%] + 8. Generate new embeddings if --embeddings [90-98%] + 9. Save metadata + register repo + update .gitignore [98-100%] + 10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%] +``` + +**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`. + +## Storage + +``` +/.gitnexus/ + ├── lbug # LadybugDB database + ├── lbug.wal # Write-ahead log + ├── lbug.lock # Single-writer lock + └── meta.json # lastCommit, indexedAt, stats + +~/.gitnexus/ + └── registry.json # Global repo registry (MCP discovery) +``` + +Managed by `repo-manager.ts`. + +## LadybugDB schema + +Defined in `lbug/schema.ts`. Separate node tables per type, single `CodeRelation` table. + +**Node tables:** File, Folder, Function, Class, Interface, Method, Constructor, CodeElement, Struct, Enum, Macro, Typedef, Union, Namespace, Trait, Impl, TypeAlias, Const, Static, Property, Record, Delegate, Annotation, Template, Module, Community, Process, Route, Tool, Section, Embedding. + +**Relation types** (`CodeRelation.type`): CONTAINS, DEFINES, CALLS, IMPORTS, EXTENDS, IMPLEMENTS, HAS_METHOD, HAS_PROPERTY, ACCESSES, METHOD_OVERRIDES, METHOD_IMPLEMENTS, MEMBER_OF, STEP_IN_PROCESS, HANDLES_ROUTE, FETCHES, HANDLES_TOOL, ENTRY_POINT_OF. + +## Embeddings and search + +**Embeddings** (`src/core/embeddings/`): Snowflake arctic-embed-xs (384D). Embeddable: File, Function, Class, Method, Interface. Incremental via SHA1 content hash. Separate `Embedding` table. + +**Search** (`src/core/search/`): Hybrid BM25 + semantic vector, merged via Reciprocal Rank Fusion (K=60). ## Known limitations ### Overloaded method resolution -Method and Constructor node IDs include an arity suffix (`#`) to -disambiguate overloaded methods. Two overloads with different parameter counts -produce distinct graph nodes: `Method:file:Class.method#1` vs -`Method:file:Class.method#2`. +Node IDs use arity suffix (`#`): `Method:file:Class.method#1` vs `#2`. -**Same-arity overload disambiguation:** When two overloads share the same -parameter count but differ in types (e.g. `save(int)` vs `save(String)`), a -type-hash suffix `~type1,type2` is appended to produce distinct node IDs: -`Method:file:Class.save#1~int` vs `Method:file:Class.save#1~String`. The suffix -is only added when a same-arity collision is detected within a class and all -parameters have non-null type annotations. Languages without type info (Python, -Ruby, JS) fall back to arity-only IDs. TypeScript/JavaScript overload signatures -are intentionally excluded from type-hashing because they are declaration-only -contracts that should collapse to the implementation body's node ID. See issue -\#651. +**Same-arity disambiguation:** type-hash suffix `~type1,type2` when collision detected and type annotations present. Languages without types (Python, Ruby, JS) use arity-only. TS/JS overload signatures excluded (collapse to implementation body). See #651. -**C++ const-qualified overload disambiguation:** Methods overloaded by const -qualification (e.g. `begin()` vs `begin() const`) are disambiguated via an -`isConst` property and a `$const` ID suffix appended to the const-qualified -variant when a non-const collision exists. The `$const` suffix appears after the -type-hash suffix: e.g. `Method:file:Container.begin#0$const`. +**C++ const-qualified:** `$const` suffix after type-hash when non-const collision exists: `Method:file:Container.begin#0$const`. -**Generic/template type preservation in type-hash:** The type-hash suffix uses -`rawType` (full AST text including generic/template args) rather than the -simplified `type` from `extractSimpleTypeName`. This means C++ template overloads -like `process(vector)` vs `process(vector)` produce distinct IDs: -`~vector` vs `~vector`. Java generic overloads like -`process(List)` vs `process(List)` are a compile error due to -type erasure, so this gap is theoretical for Java. +**Generic/template types:** type-hash uses `rawType` (full AST text including generics): `~vector` vs `~vector`. -**ID stability on first overload:** Type and const tags are collision-only. When -a class has `save(int)` as its only `save` method, the ID is `save#1` (no tag). -Adding `save(String)` changes the original to `save#1~int`. This is correct for -fresh analysis but means IDs are not stable across overload additions. Future -incremental re-analysis should account for this. +**ID stability:** collision-only tags mean IDs change when overloads are added. `save#1` becomes `save#1~int` when `save(String)` is added. -**Variadic method matching:** When one side is variadic (`parameterCount` -undefined) and the other has a fixed count, `METHOD_IMPLEMENTS` edges are -emitted with confidence 0.7 instead of 1.0. Variadic methods like -`foo(String... args)` may superficially match `foo(String s)` by type but -are not guaranteed to be interchangeable across all languages (Java/Kotlin -accept this via varargs sugar; TypeScript, C#, Rust do not). +**Variadic matching:** confidence 0.7 when one side is variadic and the other has fixed count. -**Confidence tiering** for `METHOD_IMPLEMENTS` edges: +**METHOD_IMPLEMENTS confidence tiering:** -| Match quality | Confidence | When | -|---|---|---| -| Exact parameter types match | 1.0 | Both sides have `parameterTypes` arrays and they match | -| Arity (count) matches | 1.0 | Both sides have `parameterCount`, types unavailable | -| Variadic vs fixed | 0.7 | One side is variadic, other has fixed count | -| Lenient (insufficient info) | 0.7 | One or both sides lack type and count data | +| Match quality | Confidence | +|---|---| +| Exact parameter types match | 1.0 | +| Arity match, types unavailable | 1.0 | +| Variadic vs fixed | 0.7 | +| Insufficient info | 0.7 | ## Related docs -- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance. -- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery. -- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents. -- [TESTING.md](TESTING.md) — how to run tests. -- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage expectations for **this** repo when indexed by GitNexus. +- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance +- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery +- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents +- [TESTING.md](TESTING.md) — how to run tests +- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d2d48f017..22104edb4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -62,7 +62,7 @@ Commits within a PR may use any style — only the **merged PR title** shows up - [ ] Typecheck passes: `npx tsc --noEmit` in `gitnexus/` and `npx tsc -b --noEmit` in `gitnexus-web/`. - [ ] No secrets, tokens, or machine-specific paths committed. - [ ] Documentation updated if behavior or public CLI/MCP contract changes. -- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — typecheck + unit tests for staged packages). +- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — formatting via lint-staged + typecheck for staged packages; tests run in CI only). ## Code review diff --git a/GUARDRAILS.md b/GUARDRAILS.md index 7401c79c5..ac48ab906 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -1,72 +1,69 @@ -# Guardrails — GitNexus (repo + agents) +# Guardrails — GitNexus -Rules for **human contributors** and **AI agents** working on this codebase or publishing artifacts. These complement `AGENTS.md` / `CLAUDE.md` (which focus on GitNexus-in-GitNexus workflows). +Rules for **human contributors** and **AI agents**. Complements `AGENTS.md` (workflows) and `CONTRIBUTING.md` (PR process). -## Scope (typical agent session) +## Scope (least privilege) -When automating changes in this repository, treat scope as **least privilege**: +- **Read:** Source, tests, docs, public config as needed. +- **Write:** Only files required for the fix or feature; no unrelated formatting or refactors. +- **Execute:** Tests, typecheck, documented CLI commands. No destructive commands on user data without approval. +- **Off-limits:** Other people's machines, production deployments you don't own, credentials you lack permission to use. -- **Read:** Source, tests, docs, public config as needed for the task. -- **Write:** Only files required for the requested fix or feature; avoid unrelated formatting or refactors. -- **Execute:** Tests, typecheck, and documented CLI commands; do not run destructive commands on user data outside the repo without explicit approval. -- **Off-limits:** Other people’s machines, production deployments you don’t own, and credentials you didn’t receive permission to use. - -Adjust explicitly if the maintainer defines a different scope for a task. +Maintainer may widen scope per task. --- ## Non-negotiables -1. **Never commit secrets** — API keys, tokens, `.env` with real values, private URLs, or session cookies. Use `.env.example` with placeholders only. -2. **Never rename symbols with blind find-and-replace** when working in a GitNexus-indexed project — use the **`rename` MCP tool** with **`dry_run: true` first**, then review `graph` vs `text_search` edits. (There is no separate `gitnexus rename` CLI; renaming goes through MCP or editor integration.) -3. **Run impact analysis before editing shared symbols** — use **`impact`** (upstream) for functions/classes/methods others call; do not ignore **HIGH** / **CRITICAL** risk without maintainer sign-off. -4. **Prefer `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. -5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, run `npx gitnexus analyze --embeddings` when refreshing the index; plain `analyze` can drop them. +1. **Never commit secrets** — API keys, tokens, real `.env` values, private URLs, session cookies. Use `.env.example` with placeholders. +2. **Never rename with find-and-replace** in GitNexus-indexed projects — use `rename` MCP tool with `dry_run: true` first, review `graph` vs `text_search` edits. No separate `gitnexus rename` CLI exists. +3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off. +4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available. +5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, use `npx gitnexus analyze --embeddings`; plain `analyze` drops them. --- ## Signs (recurring failure patterns) -Use this format: **Trigger → Instruction → Reason**. -Append new Signs here when the same mistake repeats (e.g. CI broken twice the same way). +Format: **Trigger → Instruction → Reason**. Append new Signs when the same mistake repeats. -### Sign: Stale graph after edits +### Stale graph after edits -- **Trigger:** MCP or resources warn the index is behind `HEAD`, or code search doesn’t match latest commit. -- **Instruction:** Run `npx gitnexus analyze` from the repo root (plus `--embeddings` if the project used them). -- **Reason:** Tools query LadybugDB built at last analyze; git changes are invisible until re-indexed. +- **Trigger:** MCP warns index is behind `HEAD`, or search doesn't match latest commit. +- **Do:** `npx gitnexus analyze` (plus `--embeddings` if used). +- **Why:** Tools query LadybugDB from last analyze; git changes are invisible until re-indexed. -### Sign: Embeddings vanished after analyze +### Embeddings vanished after analyze -- **Trigger:** Semantic search quality drops; `stats.embeddings` in `.gitnexus/meta.json` is 0 after a refresh. -- **Instruction:** Re-run `npx gitnexus analyze --embeddings` and confirm `meta.json` reflects stored embeddings. -- **Reason:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors. +- **Trigger:** Semantic search quality drops; `stats.embeddings` in `meta.json` is 0 after refresh. +- **Do:** `npx gitnexus analyze --embeddings`, confirm `meta.json` reflects stored embeddings. +- **Why:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors. -### Sign: MCP lists no repos +### MCP lists no repos -- **Trigger:** MCP stderr says no indexed repos. -- **Instruction:** Run `npx gitnexus analyze` in the target repository; verify `npx gitnexus list` shows it. -- **Reason:** The MCP server discovers repos via `~/.gitnexus/registry.json`, populated by analyze. +- **Trigger:** MCP stderr says no indexed repos. +- **Do:** `npx gitnexus analyze` in the target repo; verify `npx gitnexus list` shows it. +- **Why:** MCP discovers repos via `~/.gitnexus/registry.json`, populated by analyze. -### Sign: Wrong repo in multi-repo setups +### Wrong repo in multi-repo setups -- **Trigger:** Query/impact results clearly belong to another project. -- **Instruction:** Call `list_repos`, then pass **`repo`** on subsequent tools (or use per-workspace MCP config). -- **Reason:** Default target may be ambiguous when multiple repos are registered. +- **Trigger:** Query/impact results belong to another project. +- **Do:** Call `list_repos`, then pass `repo` on subsequent tools. +- **Why:** Default target is ambiguous when multiple repos are registered. -### Sign: LadybugDB lock / “database busy” +### LadybugDB lock / "database busy" -- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run. -- **Instruction:** Stop overlapping processes; one writer at a time. Retry analyze or restart MCP. -- **Reason:** Embedded DB expects single-process ownership of the store. +- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run. +- **Do:** Stop overlapping processes (one writer at a time). Retry analyze or restart MCP. +- **Why:** Embedded DB expects single-process ownership. --- ## Publishing & supply chain -- **npm:** Do not publish from unreviewed automation; follow maintainer release process. Bump version intentionally; tag releases to match `package.json`. -- **Dependencies:** Prefer minimal, auditable changes to `package.json`; run tests and CI after lockfile updates. -- **License:** This project ships under **PolyForm Noncommercial 1.0.0** — do not relicense or imply a different license in docs or metadata without maintainer approval. +- **npm:** Do not publish from unreviewed automation. Bump version intentionally; tag releases to match `package.json`. +- **Dependencies:** Minimal, auditable `package.json` changes; run tests and CI after lockfile updates. +- **License:** PolyForm Noncommercial 1.0.0 — do not relicense without maintainer approval. --- @@ -74,15 +71,15 @@ Append new Signs here when the same mistake repeats (e.g. CI broken twice the sa Stop and ask a **human maintainer** when: -- Impact analysis shows **HIGH** / **CRITICAL** risk and the task still requires the change. -- You need to alter **CI**, **release**, or **security-sensitive** config. -- Requirements conflict (e.g. “speed up analyze” vs “must keep all embeddings on huge repo”). +- Impact analysis shows HIGH/CRITICAL risk and the task still requires the change. +- You need to alter CI, release, or security-sensitive config. +- Requirements conflict (e.g. "speed up analyze" vs "must keep all embeddings on huge repo"). - You are unsure whether data loss is acceptable (`clean`, forced migrations, schema changes). --- ## Related docs -- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow. -- [RUNBOOK.md](RUNBOOK.md) — commands for recovery. -- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations. +- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow +- [RUNBOOK.md](RUNBOOK.md) — commands for recovery +- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations diff --git a/TESTING.md b/TESTING.md index 8d267983a..cf481d32b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -20,9 +20,9 @@ From repository root, unless noted: cd gitnexus npm install npm run build -npm test # unit: vitest run test/unit +npm test # full suite: vitest run +npm run test:unit # unit only: vitest run test/unit npm run test:integration # integration suite -npm run test:all npm run test:coverage npx tsc --noEmit # typecheck (matches CI) ``` @@ -42,8 +42,11 @@ npm run test:e2e # Playwright (requires gitnexus serve + npm run dev) A husky pre-commit hook (`.husky/pre-commit`) runs automatically on every `git commit`: -- **`gitnexus-web/` files staged** → `tsc -b --noEmit` + `vitest run` -- **`gitnexus/` files staged** → `tsc --noEmit` + `vitest run --project default` +1. **Formatting** — `lint-staged` runs prettier on staged files +2. **`gitnexus-web/` files staged** → `tsc -b --noEmit` +3. **`gitnexus/` files staged** → `tsc --noEmit` + +Tests do **not** run in the pre-commit hook — they run in CI (`ci-tests.yml`) only. Skip with `git commit --no-verify` (use sparingly). @@ -77,7 +80,7 @@ Re-run the full relevant suite when: GitHub Actions (`.github/workflows/ci.yml`) orchestrate: -- **`ci-quality.yml`** — `tsc --noEmit` for `gitnexus/` + `tsc -b --noEmit` for `gitnexus-web/` +- **`ci-quality.yml`** — prettier format check, eslint lint, `tsc --noEmit` for `gitnexus/`, `tsc -b --noEmit` for `gitnexus-web/` - **`ci-tests.yml`** — `vitest run` with coverage (ubuntu) + cross-platform (macOS, Windows) - **`ci-e2e.yml`** — Playwright E2E tests, gated on `gitnexus-web/**` changes diff --git a/gitnexus-shared/src/lbug/schema-constants.ts b/gitnexus-shared/src/lbug/schema-constants.ts index 0eca57286..656ffe552 100644 --- a/gitnexus-shared/src/lbug/schema-constants.ts +++ b/gitnexus-shared/src/lbug/schema-constants.ts @@ -30,6 +30,7 @@ export const NODE_TABLES = [ 'TypeAlias', 'Const', 'Static', + 'Variable', 'Property', 'Record', 'Delegate', diff --git a/gitnexus/src/cli/analyze.ts b/gitnexus/src/cli/analyze.ts index 26d1ae8c6..1e75ea675 100644 --- a/gitnexus/src/cli/analyze.ts +++ b/gitnexus/src/cli/analyze.ts @@ -147,9 +147,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption const origLog = console.log.bind(console); const origWarn = console.warn.bind(console); const origError = console.error.bind(console); + let barCurrentValue = 0; const barLog = (...args: any[]) => { process.stdout.write('\x1b[2K\r'); origLog(args.map((a) => (typeof a === 'string' ? a : String(a))).join(' ')); + bar.update(barCurrentValue); }; console.log = barLog; console.warn = barLog; @@ -160,6 +162,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption let phaseStart = Date.now(); const updateBar = (value: number, phaseLabel: string) => { + barCurrentValue = value; if (phaseLabel !== lastPhaseLabel) { lastPhaseLabel = phaseLabel; phaseStart = Date.now(); diff --git a/gitnexus/src/core/embeddings/ast-utils.ts b/gitnexus/src/core/embeddings/ast-utils.ts new file mode 100644 index 000000000..fb51ac052 --- /dev/null +++ b/gitnexus/src/core/embeddings/ast-utils.ts @@ -0,0 +1,112 @@ +/** + * Shared AST utilities for the embedding pipeline. + * Centralizes parser caching and tree-sitter node lookups + * used by both chunker.ts and structural-extractor.ts. + */ + +import { getLanguageFromFilename } from 'gitnexus-shared'; +import { + createParserForLanguage, + isLanguageAvailable, + resolveLanguageKey, +} from '../tree-sitter/parser-loader.js'; + +const parserCache = new Map(); + +/** + * Ensure parser is initialized and language is loaded, then parse content. + * Returns null if language is unavailable or parsing fails. + */ +export const ensureAndParse = async (content: string, filePath: string): Promise => { + const language = getLanguageFromFilename(filePath); + if (!language) return null; + if (!isLanguageAvailable(language)) return null; + + const parserKey = resolveLanguageKey(language, filePath); + let parserInstance = parserCache.get(parserKey); + if (!parserInstance) { + parserInstance = await createParserForLanguage(language, filePath); + parserCache.set(parserKey, parserInstance); + } + + return parserInstance.parse(content); +}; + +const FUNCTION_LIKE_TYPES = new Set([ + 'function_declaration', + 'function_definition', + 'method_declaration', + 'method_definition', + 'function_item', + 'function_signature_item', + 'arrow_function', + 'function_expression', + 'generator_function_declaration', + 'generator_function', + 'async_function_declaration', + 'async_arrow_function', + 'constructor_declaration', + 'constructor_definition', + 'compact_constructor_declaration', + 'short_function_declaration', + 'proc_declaration', + 'func_literal', + 'local_function_statement', + 'anonymous_function', + 'lambda_literal', + 'init_declaration', + 'deinit_declaration', +]); + +/** + * Find the first function/method-like declaration in a snippet AST. + * Used by the chunker when parsing node.content where absolute line + * numbers don't apply. + */ +export const findFunctionNode = (root: any): any | null => { + if (FUNCTION_LIKE_TYPES.has(root.type)) return root; + + for (let i = 0; i < root.namedChildCount; i++) { + const child = root.namedChild(i); + if (!child) continue; + if (FUNCTION_LIKE_TYPES.has(child.type)) return child; + const found = findFunctionNode(child); + if (found) return found; + } + + return null; +}; + +/** + * Find the first class/struct/interface/enum-like declaration in an AST. + * Used when parsing node.content (a snippet, not a full file) where + * absolute line numbers don't apply. + */ +export const findDeclarationNode = (root: any): any | null => { + const CLASS_LIKE_TYPES = new Set([ + 'class_declaration', + 'class_definition', + 'struct_declaration', + 'struct_item', + 'interface_declaration', + 'interface_definition', + 'enum_declaration', + 'enum_item', + 'type_declaration', // Go: type X struct + 'declaration', // Go: type X struct + 'object_declaration', // Kotlin: object + 'impl_item', // Rust: impl + ]); + + if (CLASS_LIKE_TYPES.has(root.type)) return root; + + for (let i = 0; i < root.namedChildCount; i++) { + const child = root.namedChild(i); + if (!child) continue; + if (CLASS_LIKE_TYPES.has(child.type)) return child; + const found = findDeclarationNode(child); + if (found) return found; + } + + return null; +}; diff --git a/gitnexus/src/core/embeddings/character-chunk.ts b/gitnexus/src/core/embeddings/character-chunk.ts new file mode 100644 index 000000000..e4f6f949c --- /dev/null +++ b/gitnexus/src/core/embeddings/character-chunk.ts @@ -0,0 +1,63 @@ +/** + * Character-based sliding window chunking (pure, no tree-sitter dependency) + */ + +import { buildLineIndex, resolveChunkLines } from './line-index.js'; + +export interface Chunk { + text: string; + chunkIndex: number; + startOffset: number; + endOffset: number; + startLine: number; + endLine: number; +} + +export const characterChunk = ( + content: string, + startLine: number, + endLine: number, + chunkSize: number = 1200, + overlap: number = 120, +): Chunk[] => { + if (content.length <= chunkSize) { + return [ + { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }, + ]; + } + + const chunks: Chunk[] = []; + let offset = 0; + const lineOffsets = buildLineIndex(content); + + while (offset < content.length) { + const end = Math.min(offset + chunkSize, content.length); + const chunkText = content.slice(offset, end); + const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine); + + chunks.push({ + text: chunkText, + chunkIndex: chunks.length, + startOffset: offset, + endOffset: end, + startLine: lineRange.startLine, + endLine: lineRange.endLine, + }); + + offset = end - overlap; + if (offset >= content.length) break; + if (end >= content.length) break; + if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) { + offset = end; + } + } + + return chunks; +}; diff --git a/gitnexus/src/core/embeddings/chunker.ts b/gitnexus/src/core/embeddings/chunker.ts new file mode 100644 index 000000000..114a5f68c --- /dev/null +++ b/gitnexus/src/core/embeddings/chunker.ts @@ -0,0 +1,363 @@ +/** + * Chunker Module + * + * Splits code nodes into chunks for embedding. + * - Function/Method: AST-aware chunking by statement boundaries + * - Other types: character-based sliding window fallback + * - Short content (≤ chunkSize): no chunking + */ + +export { type Chunk, characterChunk } from './character-chunk.js'; + +import { characterChunk } from './character-chunk.js'; +import type { Chunk } from './character-chunk.js'; +import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js'; +import { buildLineIndex, resolveChunkLines } from './line-index.js'; + +/** + * Main chunkNode function: dispatches by label + */ +export const chunkNode = async ( + label: string, + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number = 1200, + overlap: number = 120, +): Promise => { + // Content fits in one chunk — no splitting needed + if (content.length <= chunkSize) { + return [ + { + text: content, + chunkIndex: 0, + startOffset: 0, + endOffset: content.length, + startLine, + endLine, + }, + ]; + } + + // Only function-like labels get AST chunking + if (label === 'Function' || label === 'Method' || label === 'Constructor') { + try { + const astChunks = await astChunk(content, filePath, startLine, endLine, chunkSize, overlap); + if (astChunks.length > 0) return astChunks; + } catch { + // AST parsing failed — fall through to character fallback + } + } + + if (label === 'Class' || label === 'Interface') { + try { + const declarationChunks = await declarationChunk( + label, + content, + filePath, + startLine, + endLine, + chunkSize, + overlap, + ); + if (declarationChunks.length > 0) return declarationChunks; + } catch { + // AST parsing failed — fall through to character fallback + } + } + + // Character-based fallback for everything else + return characterChunk(content, startLine, endLine, chunkSize, overlap); +}; + +/** + * AST-based chunking for Function/Method + * Parse snippet content, locate the function declaration node, + * split body by statement boundaries. + */ +const astChunk = async ( + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number, + overlap: number, +): Promise => { + const tree = await ensureAndParse(content, filePath); + if (!tree) return []; + + const root = tree.rootNode; + const lineOffsets = buildLineIndex(content); + + // Find the function/method declaration in the snippet AST. + // tree-sitter parses node.content (a snippet), so rows are relative (0-based). + const targetNode = findFunctionNode(root); + if (!targetNode) return []; + + // Get the body (statements) via childForFieldName('body') + const bodyNode = targetNode.childForFieldName('body'); + if (!bodyNode) return []; + + // Extract individual statements + const statements: Array<{ startIndex: number; endIndex: number }> = []; + for (let i = 0; i < bodyNode.namedChildCount; i++) { + const child = bodyNode.namedChild(i); + if (!child) continue; + statements.push({ + startIndex: child.startIndex, + endIndex: child.endIndex, + }); + } + + if (statements.length === 0) return []; + + return chunkByUnits( + content, + lineOffsets, + startLine, + chunkSize, + overlap, + statements, + targetNode.startIndex, + targetNode.endIndex, + true, + true, + ); +}; + +const DECLARATION_BODY_NODE_TYPES = new Set([ + 'class_body', + 'object_type', + 'declaration_list', + 'interface_body', +]); + +const FIELD_LIKE_MEMBER_TYPES = new Set([ + 'field_definition', + 'public_field_definition', + 'property_definition', + 'property_signature', + 'variable_declarator', + 'lexical_declaration', + 'pair', + 'enum_assignment', +]); + +const declarationChunk = async ( + label: 'Class' | 'Interface', + content: string, + filePath: string, + startLine: number, + endLine: number, + chunkSize: number, + overlap: number, +): Promise => { + const tree = await ensureAndParse(content, filePath); + if (!tree) return []; + + const targetNode = findDeclarationNode(tree.rootNode); + if (!targetNode) return []; + + const bodyNode = getDeclarationBodyNode(targetNode); + if (!bodyNode) return []; + + const members = collectDeclarationUnits(bodyNode, label); + if (members.length === 0) return []; + + return chunkByUnits( + content, + buildLineIndex(content), + startLine, + chunkSize, + overlap, + members, + targetNode.startIndex, + targetNode.endIndex, + false, + false, + ); +}; + +const buildChunk = ( + content: string, + lineOffsets: Int32Array, + chunkIndex: number, + startOffset: number, + endOffset: number, + baseStartLine: number, +): Chunk => { + const lineRange = resolveChunkLines(lineOffsets, startOffset, endOffset, baseStartLine); + return { + text: content.slice(startOffset, endOffset), + chunkIndex, + startOffset, + endOffset, + startLine: lineRange.startLine, + endLine: lineRange.endLine, + }; +}; + +const chunkByUnits = ( + content: string, + lineOffsets: Int32Array, + baseStartLine: number, + chunkSize: number, + overlap: number, + units: Array<{ startIndex: number; endIndex: number }>, + containerStartOffset: number, + containerEndOffset: number, + includeContainerPrefixOnFirstChunk: boolean, + includeContainerSuffixOnLastChunk: boolean, +): Chunk[] => { + const chunks: Chunk[] = []; + let chunkStartUnitIdx = 0; + + while (chunkStartUnitIdx < units.length) { + const chunkStartOffset = + chunkStartUnitIdx === 0 && includeContainerPrefixOnFirstChunk + ? containerStartOffset + : units[chunkStartUnitIdx].startIndex; + + let chunkEndUnitIdx = chunkStartUnitIdx; + let candidateEndOffset = + chunkEndUnitIdx === units.length - 1 && includeContainerSuffixOnLastChunk + ? containerEndOffset + : units[chunkEndUnitIdx].endIndex; + + while (chunkEndUnitIdx + 1 < units.length) { + const nextEndOffset = + chunkEndUnitIdx + 1 === units.length - 1 && includeContainerSuffixOnLastChunk + ? containerEndOffset + : units[chunkEndUnitIdx + 1].endIndex; + if (nextEndOffset - chunkStartOffset > chunkSize) break; + chunkEndUnitIdx += 1; + candidateEndOffset = nextEndOffset; + } + + if (candidateEndOffset - chunkStartOffset > chunkSize) { + const oversizedUnit = units[chunkStartUnitIdx]; + const oversizedLineRange = resolveChunkLines( + lineOffsets, + oversizedUnit.startIndex, + oversizedUnit.endIndex, + baseStartLine, + ); + const oversizedChunks = characterChunk( + content.slice(oversizedUnit.startIndex, oversizedUnit.endIndex), + oversizedLineRange.startLine, + oversizedLineRange.endLine, + chunkSize, + overlap, + ).map((chunk, offsetIdx) => ({ + ...chunk, + chunkIndex: chunks.length + offsetIdx, + startOffset: chunk.startOffset + oversizedUnit.startIndex, + endOffset: chunk.endOffset + oversizedUnit.startIndex, + })); + chunks.push(...oversizedChunks); + chunkStartUnitIdx += 1; + continue; + } + + chunks.push( + buildChunk( + content, + lineOffsets, + chunks.length, + chunkStartOffset, + candidateEndOffset, + baseStartLine, + ), + ); + + if (chunkEndUnitIdx === units.length - 1) { + break; + } + + const nextChunkStartUnitIdx = findOverlapStartIndex( + units, + chunkStartUnitIdx, + chunkEndUnitIdx, + overlap, + ); + if (nextChunkStartUnitIdx <= chunkStartUnitIdx) { + chunkStartUnitIdx = chunkEndUnitIdx + 1; + } else { + chunkStartUnitIdx = nextChunkStartUnitIdx; + } + } + + return chunks; +}; + +const findOverlapStartIndex = ( + statements: Array<{ startIndex: number; endIndex: number }>, + chunkStartStmtIdx: number, + chunkEndStmtIdx: number, + overlapSize: number, +): number => { + if (overlapSize <= 0) return chunkEndStmtIdx + 1; + + let overlapStartIdx = chunkEndStmtIdx; + while (overlapStartIdx > chunkStartStmtIdx) { + const overlapLength = + statements[chunkEndStmtIdx].endIndex - statements[overlapStartIdx - 1].startIndex; + if (overlapLength > overlapSize) break; + overlapStartIdx -= 1; + } + + return overlapStartIdx; +}; + +const getDeclarationBodyNode = (node: any): any | null => { + const bodyNode = node.childForFieldName?.('body'); + if (bodyNode) return bodyNode; + + for (let i = 0; i < node.namedChildCount; i++) { + const child = node.namedChild(i); + if (!child) continue; + if (DECLARATION_BODY_NODE_TYPES.has(child.type)) return child; + } + + return null; +}; + +const collectDeclarationUnits = ( + bodyNode: any, + label: 'Class' | 'Interface', +): Array<{ startIndex: number; endIndex: number }> => { + const members: Array<{ startIndex: number; endIndex: number; groupable: boolean }> = []; + + for (let i = 0; i < bodyNode.namedChildCount; i++) { + const child = bodyNode.namedChild(i); + if (!child) continue; + members.push({ + startIndex: child.startIndex, + endIndex: child.endIndex, + groupable: label === 'Class' && FIELD_LIKE_MEMBER_TYPES.has(child.type), + }); + } + + if (members.length === 0) return []; + + const grouped: Array<{ startIndex: number; endIndex: number }> = []; + let current = members[0]; + + for (let i = 1; i < members.length; i++) { + const next = members[i]; + if (current.groupable && next.groupable) { + current = { + startIndex: current.startIndex, + endIndex: next.endIndex, + groupable: true, + }; + continue; + } + grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex }); + current = next; + } + + grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex }); + return grouped; +}; diff --git a/gitnexus/src/core/embeddings/embedding-pipeline.ts b/gitnexus/src/core/embeddings/embedding-pipeline.ts index cb28ca945..302903f8b 100644 --- a/gitnexus/src/core/embeddings/embedding-pipeline.ts +++ b/gitnexus/src/core/embeddings/embedding-pipeline.ts @@ -3,9 +3,9 @@ * * Orchestrates the background embedding process: * 1. Query embeddable nodes from LadybugDB - * 2. Generate text representations - * 3. Batch embed using transformers.js - * 4. Update LadybugDB with embeddings + * 2. Generate text representations with enriched metadata + * 3. Chunk long nodes, batch embed + * 4. Update LadybugDB with chunk-aware embeddings * 5. Create vector index for semantic search */ @@ -17,20 +17,28 @@ import { embeddingToArray, isEmbedderReady, } from './embedder.js'; -import { generateEmbeddingText, generateBatchEmbeddingTexts } from './text-generator.js'; +import { generateEmbeddingText } from './text-generator.js'; +import { chunkNode, characterChunk } from './chunker.js'; +import { extractStructuralNames } from './structural-extractor.js'; import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult, type ModelProgress, + type EmbeddingContext, DEFAULT_EMBEDDING_CONFIG, EMBEDDABLE_LABELS, + isShortLabel, + LABELS_WITH_EXPORTED, + STRUCTURAL_LABELS, + collectBestChunks, } from './types.js'; import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, CREATE_VECTOR_INDEX_QUERY, + STALE_HASH_SENTINEL, } from '../lbug/schema.js'; import { loadVectorExtension } from '../lbug/lbug-adapter.js'; @@ -46,7 +54,14 @@ export const contentHashForNode = ( node: EmbeddableNode, config: Partial = {}, ): string => { - const text = generateEmbeddingText(node, config); + // Hash must be deterministic across runs, so exclude methodNames/fieldNames + // which are populated during the batch loop via AST extraction. + // Using only node.content ensures the hash stays stable. + const text = generateEmbeddingText( + { ...node, methodNames: undefined, fieldNames: undefined }, + node.content, + config, + ); return createHash('sha1').update(text).digest('hex'); }; @@ -57,37 +72,50 @@ export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void; /** * Query all embeddable nodes from LadybugDB - * Uses table-specific queries (File has different schema than code elements) + * Uses table-specific queries for different label types */ const queryEmbeddableNodes = async ( executeQuery: (cypher: string) => Promise, ): Promise => { const allNodes: EmbeddableNode[] = []; - // Query each embeddable table with table-specific columns for (const label of EMBEDDABLE_LABELS) { try { let query: string; - if (label === 'File') { - // File nodes don't have startLine/endLine + if (label === 'Method') { + // Method has parameterCount and returnType query = ` - MATCH (n:File) - RETURN n.id AS id, n.name AS name, 'File' AS label, - n.filePath AS filePath, n.content AS content + MATCH (n:Method) + RETURN n.id AS id, n.name AS name, 'Method' AS label, + n.filePath AS filePath, n.content AS content, + n.startLine AS startLine, n.endLine AS endLine, + n.isExported AS isExported, n.description AS description, + n.parameterCount AS parameterCount, n.returnType AS returnType + `; + } else if (LABELS_WITH_EXPORTED.has(label)) { + // Function, Class, Interface have isExported and description + query = ` + MATCH (n:\`${label}\`) + RETURN n.id AS id, n.name AS name, '${label}' AS label, + n.filePath AS filePath, n.content AS content, + n.startLine AS startLine, n.endLine AS endLine, + n.isExported AS isExported, n.description AS description `; } else { - // Code elements have startLine/endLine + // Multi-language tables (Struct, Enum, etc.) — have description but no isExported query = ` - MATCH (n:${label}) - RETURN n.id AS id, n.name AS name, '${label}' AS label, + MATCH (n:\`${label}\`) + RETURN n.id AS id, n.name AS name, '${label}' AS label, n.filePath AS filePath, n.content AS content, - n.startLine AS startLine, n.endLine AS endLine + n.startLine AS startLine, n.endLine AS endLine, + n.description AS description `; } const rows = await executeQuery(query); for (const row of rows) { + const hasExportedColumn = label === 'Method' || LABELS_WITH_EXPORTED.has(label); allNodes.push({ id: row.id ?? row[0], name: row.name ?? row[1], @@ -96,10 +124,17 @@ const queryEmbeddableNodes = async ( content: row.content ?? row[4] ?? '', startLine: row.startLine ?? row[5], endLine: row.endLine ?? row[6], + isExported: hasExportedColumn ? (row.isExported ?? row[7]) : undefined, + description: row.description ?? (hasExportedColumn ? row[8] : row[7]), + ...(label === 'Method' + ? { + parameterCount: row.parameterCount ?? row[9], + returnType: row.returnType ?? row[10], + } + : {}), }); } } catch (error) { - // Table might not exist or be empty, continue if (isDev) { console.warn(`Query for ${label} nodes failed:`, error); } @@ -110,32 +145,42 @@ const queryEmbeddableNodes = async ( }; /** - * Batch INSERT embeddings into separate CodeEmbedding table - * Using a separate lightweight table avoids copy-on-write overhead - * that occurs when UPDATEing nodes with large content fields + * Batch INSERT chunk-aware embeddings into CodeEmbedding table */ -const batchInsertEmbeddings = async ( +export const batchInsertEmbeddings = async ( executeWithReusedStatement: ( cypher: string, paramsList: Array>, ) => Promise, - updates: Array<{ id: string; embedding: number[]; contentHash: string }>, + updates: Array<{ + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + embedding: number[]; + contentHash?: string; + }>, ): Promise => { - // MERGE instead of CREATE — idempotent, handles concurrent analyzes and partial prior runs - const cypher = `MERGE (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) SET e.embedding = $embedding, e.contentHash = $contentHash`; + const cypher = `CREATE (e:${EMBEDDING_TABLE_NAME} {id: $id, nodeId: $nodeId, chunkIndex: $chunkIndex, startLine: $startLine, endLine: $endLine, embedding: $embedding, contentHash: $contentHash})`; const paramsList = updates.map((u) => ({ - nodeId: u.id, + id: `${u.nodeId}:${u.chunkIndex}`, + nodeId: u.nodeId, + chunkIndex: u.chunkIndex, + startLine: u.startLine, + endLine: u.endLine, embedding: u.embedding, - contentHash: u.contentHash, + contentHash: u.contentHash ?? STALE_HASH_SENTINEL, })); await executeWithReusedStatement(cypher, paramsList); }; /** * Create the vector index for semantic search + * Now indexes the separate CodeEmbedding table. * Delegates extension loading to lbug-adapter's loadVectorExtension(), * which owns the VECTOR extension lifecycle and state tracking. + */ const createVectorIndex = async ( executeQuery: (cypher: string) => Promise, @@ -146,7 +191,6 @@ const createVectorIndex = async ( try { await executeQuery(CREATE_VECTOR_INDEX_QUERY); } catch (error) { - // Index might already exist if (isDev) { console.warn('Vector index creation warning:', error); } @@ -160,9 +204,12 @@ const createVectorIndex = async ( * @param executeWithReusedStatement - Function to execute with reused prepared statement * @param onProgress - Callback for progress updates * @param config - Optional configuration override + * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode) + * @param context - Optional repo/server context for metadata enrichment * @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode. * Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd * and re-embedded; nodes not in the map are embedded fresh. + */ export const runEmbeddingPipeline = async ( executeQuery: (cypher: string) => Promise, @@ -172,6 +219,8 @@ export const runEmbeddingPipeline = async ( ) => Promise, onProgress: EmbeddingProgressCallback, config: Partial = {}, + skipNodeIds?: Set, + context?: EmbeddingContext, existingEmbeddings?: Map, ): Promise => { const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config }; @@ -208,6 +257,14 @@ export const runEmbeddingPipeline = async ( // Phase 2: Query embeddable nodes let nodes = await queryEmbeddableNodes(executeQuery); + // Apply context metadata + if (context?.repoName) { + for (const node of nodes) { + node.repoName = context.repoName; + node.serverName = context.serverName; + } + } + // Incremental mode: compare content hashes, delete stale rows, skip fresh ones. // Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them // (avoids double computation). @@ -284,10 +341,12 @@ export const runEmbeddingPipeline = async ( return; } - // Phase 3: Batch embed nodes + // Phase 3: Chunk + embed nodes const batchSize = finalConfig.batchSize; - const totalBatches = Math.ceil(totalNodes / batchSize); + const chunkSize = finalConfig.chunkSize; + const overlap = finalConfig.overlap; let processedNodes = 0; + let totalChunks = 0; onProgress({ phase: 'embedding', @@ -295,40 +354,116 @@ export const runEmbeddingPipeline = async ( nodesProcessed: 0, totalNodes, currentBatch: 0, - totalBatches, + totalBatches: Math.ceil(totalNodes / batchSize), }); - for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) { - const start = batchIndex * batchSize; - const end = Math.min(start + batchSize, totalNodes); - const batch = nodes.slice(start, end); + // Process in batches of nodes + for (let batchIndex = 0; batchIndex < totalNodes; batchIndex += batchSize) { + const batch = nodes.slice(batchIndex, batchIndex + batchSize); - // Generate texts for this batch - const texts = generateBatchEmbeddingTexts(batch, finalConfig); + // Chunk each node and generate text + const allTexts: string[] = []; + const allUpdates: Array<{ + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + contentHash: string; + }> = []; - // Embed the batch - const embeddings = await embedBatch(texts); + for (const node of batch) { + const isShort = isShortLabel(node.label); + const startLine = node.startLine ?? 0; + const endLine = node.endLine ?? 0; - // Update LadybugDB with embeddings - const updates = batch.map((node, i) => ({ - id: node.id, - embedding: embeddingToArray(embeddings[i]), - contentHash: computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig), - })); + // Extract structural names for class-like nodes via AST extractors + if (!isShort && STRUCTURAL_LABELS.has(node.label)) { + try { + const names = await extractStructuralNames(node.content, node.filePath); + node.methodNames = names.methodNames; + node.fieldNames = names.fieldNames; + } catch { + // AST extraction failed — names stay undefined, text-generator handles gracefully + } + } - await batchInsertEmbeddings(executeWithReusedStatement, updates); + // Compute content hash once per node (re-use cached value for stale nodes) + const hash = computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig); + + let chunks: Array<{ text: string; chunkIndex: number; startLine: number; endLine: number }>; + if (isShort) { + chunks = [{ text: node.content, chunkIndex: 0, startLine, endLine }]; + } else { + try { + chunks = await chunkNode( + node.label, + node.content, + node.filePath, + startLine, + endLine, + chunkSize, + overlap, + ); + } catch (chunkErr) { + if (isDev) { + console.warn( + `⚠️ AST chunking failed for ${node.label} "${node.name}" (${node.filePath}), falling back to character-based chunking:`, + chunkErr, + ); + } + chunks = characterChunk(node.content, startLine, endLine, chunkSize, overlap); + } + } + + for (const chunk of chunks) { + const text = generateEmbeddingText(node, chunk.text, finalConfig); + allTexts.push(text); + allUpdates.push({ + nodeId: node.id, + chunkIndex: chunk.chunkIndex, + startLine: chunk.startLine, + endLine: chunk.endLine, + contentHash: hash, + }); + } + } + + // Embed chunk texts in sub-batches to control memory + const EMBED_SUB_BATCH = 8; + for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) { + const subTexts = allTexts.slice(si, si + EMBED_SUB_BATCH); + const subUpdates = allUpdates.slice(si, si + EMBED_SUB_BATCH); + + let embeddings: Float32Array[]; + try { + embeddings = await embedBatch(subTexts); + } catch (embedErr) { + console.error( + `❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`, + embedErr, + ); + throw embedErr; + } + + const dbUpdates = subUpdates.map((u, i) => ({ + ...u, + embedding: embeddingToArray(embeddings[i]), + })); + + await batchInsertEmbeddings(executeWithReusedStatement, dbUpdates); + } processedNodes += batch.length; + totalChunks += allUpdates.length; - // Report progress (20-90% for embedding phase) const embeddingProgress = 20 + (processedNodes / totalNodes) * 70; onProgress({ phase: 'embedding', percent: Math.round(embeddingProgress), nodesProcessed: processedNodes, totalNodes, - currentBatch: batchIndex + 1, - totalBatches, + currentBatch: Math.floor(batchIndex / batchSize) + 1, + totalBatches: Math.ceil(totalNodes / batchSize), }); } @@ -346,7 +481,6 @@ export const runEmbeddingPipeline = async ( await createVectorIndex(executeQuery); - // Complete onProgress({ phase: 'ready', percent: 100, @@ -355,7 +489,9 @@ export const runEmbeddingPipeline = async ( }); if (isDev) { - console.log('✅ Embedding pipeline complete!'); + console.log( + `✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`, + ); } } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; @@ -375,15 +511,7 @@ export const runEmbeddingPipeline = async ( }; /** - * Perform semantic search using the vector index - * - * Uses CodeEmbedding table and queries each node table to get metadata - * - * @param executeQuery - Function to execute Cypher queries - * @param query - Search query text - * @param k - Number of results to return (default: 10) - * @param maxDistance - Maximum distance threshold (default: 0.5) - * @returns Array of search results ordered by relevance + * Perform semantic search using the vector index with chunk deduplication */ export const semanticSearch = async ( executeQuery: (cypher: string) => Promise, @@ -395,37 +523,46 @@ export const semanticSearch = async ( throw new Error('Embedding model not initialized. Run embedding pipeline first.'); } - // Embed the query const queryEmbedding = await embedText(query); const queryVec = embeddingToArray(queryEmbedding); const queryVecStr = `[${queryVec.join(',')}]`; - // Query the vector index on CodeEmbedding to get nodeIds and distances - const vectorQuery = ` - CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', - CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${k}) - YIELD node AS emb, distance - WITH emb, distance - WHERE distance < ${maxDistance} - RETURN emb.nodeId AS nodeId, distance - ORDER BY distance - `; + const bestChunks = await collectBestChunks(k, async (fetchLimit) => { + const vectorQuery = ` + CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', + CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit}) + YIELD node AS emb, distance + WITH emb, distance + WHERE distance < ${maxDistance} + RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex, + emb.startLine AS startLine, emb.endLine AS endLine, distance + ORDER BY distance + `; - const embResults = await executeQuery(vectorQuery); + const embResults = await executeQuery(vectorQuery); + return embResults.map((row) => ({ + nodeId: row.nodeId ?? row[0], + chunkIndex: row.chunkIndex ?? row[1] ?? 0, + startLine: row.startLine ?? row[2] ?? 0, + endLine: row.endLine ?? row[3] ?? 0, + distance: row.distance ?? row[4], + })); + }); - if (embResults.length === 0) { + if (bestChunks.size === 0) { return []; } // Group results by label for batched metadata queries - const byLabel = new Map>(); - for (const embRow of embResults) { - const nodeId = embRow.nodeId ?? embRow[0]; - const distance = embRow.distance ?? embRow[1]; + const byLabel = new Map< + string, + Array<{ nodeId: string; distance: number } & Record> + >(); + for (const [nodeId, chunk] of Array.from(bestChunks.entries()).slice(0, k)) { const labelEndIdx = nodeId.indexOf(':'); const label = labelEndIdx > 0 ? nodeId.substring(0, labelEndIdx) : 'Unknown'; if (!byLabel.has(label)) byLabel.set(label, []); - byLabel.get(label)!.push({ nodeId, distance }); + byLabel.get(label)!.push({ nodeId, ...chunk }); } // Batch-fetch metadata per label @@ -434,19 +571,11 @@ export const semanticSearch = async ( for (const [label, items] of byLabel) { const idList = items.map((i) => `'${i.nodeId.replace(/'/g, "''")}'`).join(', '); try { - let nodeQuery: string; - if (label === 'File') { - nodeQuery = ` - MATCH (n:File) WHERE n.id IN [${idList}] - RETURN n.id AS id, n.name AS name, n.filePath AS filePath - `; - } else { - nodeQuery = ` - MATCH (n:${label}) WHERE n.id IN [${idList}] - RETURN n.id AS id, n.name AS name, n.filePath AS filePath, - n.startLine AS startLine, n.endLine AS endLine - `; - } + const nodeQuery = ` + MATCH (n:\`${label}\`) WHERE n.id IN [${idList}] + RETURN n.id AS id, n.name AS name, n.filePath AS filePath, + n.startLine AS startLine, n.endLine AS endLine + `; const nodeRows = await executeQuery(nodeQuery); const rowMap = new Map(); for (const row of nodeRows) { @@ -462,8 +591,8 @@ export const semanticSearch = async ( label, filePath: nodeRow.filePath ?? nodeRow[2] ?? '', distance: item.distance, - startLine: label !== 'File' ? (nodeRow.startLine ?? nodeRow[3]) : undefined, - endLine: label !== 'File' ? (nodeRow.endLine ?? nodeRow[4]) : undefined, + startLine: item.startLine, + endLine: item.endLine, }); } } @@ -472,7 +601,6 @@ export const semanticSearch = async ( } } - // Re-sort by distance since batch queries may have mixed order results.sort((a, b) => a.distance - b.distance); return results; @@ -480,16 +608,6 @@ export const semanticSearch = async ( /** * Semantic search with graph expansion (flattened results) - * - * Note: With multi-table schema, graph traversal is simplified. - * Returns semantic matches with their metadata. - * For full graph traversal, use execute_vector_cypher tool directly. - * - * @param executeQuery - Function to execute Cypher queries - * @param query - Search query text - * @param k - Number of initial semantic matches (default: 5) - * @param _hops - Unused (kept for API compatibility). - * @returns Semantic matches with metadata */ export const semanticSearchWithContext = async ( executeQuery: (cypher: string) => Promise, @@ -497,8 +615,6 @@ export const semanticSearchWithContext = async ( k: number = 5, _hops: number = 1, ): Promise => { - // For multi-table schema, just return semantic search results - // Graph traversal is complex with separate tables - use execute_vector_cypher instead const results = await semanticSearch(executeQuery, query, k, 0.5); return results.map((r) => ({ diff --git a/gitnexus/src/core/embeddings/line-index.ts b/gitnexus/src/core/embeddings/line-index.ts new file mode 100644 index 000000000..2dc12de3d --- /dev/null +++ b/gitnexus/src/core/embeddings/line-index.ts @@ -0,0 +1,50 @@ +export interface ResolvedLineRange { + startLine: number; + endLine: number; +} + +export const buildLineIndex = (content: string): Int32Array => { + const offsets: number[] = [0]; + for (let i = 0; i < content.length; i++) { + if (content.charCodeAt(i) === 10) offsets.push(i + 1); + } + return new Int32Array(offsets); +}; + +const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => { + if (lineOffsets.length === 0) return 0; + const maxOffset = lineOffsets[lineOffsets.length - 1]; + if (charOffset < 0) return 0; + if (charOffset > maxOffset) return maxOffset; + return charOffset; +}; + +export const lineFromOffset = (lineOffsets: Int32Array, charOffset: number): number => { + if (lineOffsets.length === 0) return 0; + + const clamped = clampOffset(lineOffsets, charOffset); + let lo = 0; + let hi = lineOffsets.length - 1; + while (lo < hi) { + const mid = (lo + hi + 1) >> 1; + if (lineOffsets[mid] <= clamped) lo = mid; + else hi = mid - 1; + } + return lo; +}; + +export const resolveChunkLines = ( + lineOffsets: Int32Array, + startOffset: number, + endOffset: number, + baseStartLine: number, +): ResolvedLineRange => { + const relativeStartLine = lineFromOffset(lineOffsets, startOffset); + const effectiveEndOffset = endOffset > startOffset ? endOffset - 1 : startOffset; + const relativeEndLine = lineFromOffset(lineOffsets, effectiveEndOffset); + + return { + startLine: baseStartLine + relativeStartLine, + endLine: baseStartLine + relativeEndLine, + }; +}; diff --git a/gitnexus/src/core/embeddings/server-mapping.ts b/gitnexus/src/core/embeddings/server-mapping.ts new file mode 100644 index 000000000..fa685ac8f --- /dev/null +++ b/gitnexus/src/core/embeddings/server-mapping.ts @@ -0,0 +1,37 @@ +/** + * Server Mapping Configuration + * + * Reads ~/.gitnexus/server-mapping.json to map repo names to service names. + * Used in embedding text to enrich metadata with microservice context. + */ + +import fs from 'fs/promises'; +import path from 'path'; +import os from 'os'; + +const MAPPING_FILE = path.join(os.homedir(), '.gitnexus', 'server-mapping.json'); + +let cachedMapping: Record | null = null; + +/** + * Read the server mapping file and return the serverName for a given repoName. + * Returns undefined if no mapping exists. + */ +export const readServerMapping = async (repoName: string): Promise => { + try { + if (!cachedMapping) { + const raw = await fs.readFile(MAPPING_FILE, 'utf-8'); + cachedMapping = JSON.parse(raw); + } + return cachedMapping[repoName]; + } catch { + return undefined; + } +}; + +/** + * Clear the cached mapping (useful for testing or after file changes) + */ +export const clearServerMappingCache = (): void => { + cachedMapping = null; +}; diff --git a/gitnexus/src/core/embeddings/structural-extractor.ts b/gitnexus/src/core/embeddings/structural-extractor.ts new file mode 100644 index 000000000..7035ee569 --- /dev/null +++ b/gitnexus/src/core/embeddings/structural-extractor.ts @@ -0,0 +1,89 @@ +/** + * Structural Extractor Module + * + * Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor + * to extract method and field names for embedding text generation. + */ + +import { getProviderForFile } from '../ingestion/languages/index.js'; +import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js'; +import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js'; +import type { LanguageProvider } from '../ingestion/language-provider.js'; +import { buildTypeEnv } from '../ingestion/type-env.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import { ensureAndParse, findDeclarationNode } from './ast-utils.js'; + +export interface StructuralNames { + methodNames: string[]; + fieldNames: string[]; +} + +const NOOP_SYMBOL_TABLE = { + lookupExactAll: () => [], + lookupExact: () => undefined, + lookupExactFull: () => undefined, +} as any; + +/** + * Extract method and field names from a class/struct/interface node + * using the ingestion pipeline's AST extractors. + */ +export const extractStructuralNames = async ( + content: string, + filePath: string, +): Promise => { + const provider = getProviderForFile(filePath); + if (!provider) return { methodNames: [], fieldNames: [] }; + + const tree = await ensureAndParse(content, filePath); + if (!tree) return { methodNames: [], fieldNames: [] }; + + // Parse node.content (a snippet) — find declaration directly, not by range + const classNode = findDeclarationNode(tree.rootNode); + if (!classNode) return { methodNames: [], fieldNames: [] }; + + const language = provider.id; + + const methodNames = extractMethodNames(classNode, provider, filePath, language); + const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language); + + return { methodNames, fieldNames }; +}; + +function extractMethodNames( + classNode: any, + provider: LanguageProvider, + filePath: string, + language: SupportedLanguages, +): string[] { + if (!provider.methodExtractor) return []; + + const context: MethodExtractorContext = { filePath, language }; + const result: ExtractedMethods | null = provider.methodExtractor.extract(classNode, context); + if (!result?.methods?.length) return []; + + return result.methods.map((m) => m.name); +} + +function extractFieldNames( + classNode: any, + provider: LanguageProvider, + tree: any, + filePath: string, + language: SupportedLanguages, +): string[] { + if (!provider.fieldExtractor) return []; + + const typeEnv = buildTypeEnv(tree, language); + + const context: FieldExtractorContext = { + typeEnv, + symbolTable: NOOP_SYMBOL_TABLE, + filePath, + language, + }; + const result: ExtractedFields | null = provider.fieldExtractor.extract(classNode, context); + if (!result?.fields?.length) return []; + + return result.fields.map((f) => f.name); +} diff --git a/gitnexus/src/core/embeddings/text-generator.ts b/gitnexus/src/core/embeddings/text-generator.ts index d74a9f894..5b96f6b5e 100644 --- a/gitnexus/src/core/embeddings/text-generator.ts +++ b/gitnexus/src/core/embeddings/text-generator.ts @@ -1,206 +1,253 @@ /** * Text Generator Module * - * Pure functions to generate embedding text from code nodes. - * Combines node metadata with code snippets for semantic matching. + * Generates enriched embedding text from code nodes with metadata. + * Supports chunkable labels (Function/Method with AST chunking), + * Class-specific structural text, and short-node direct embed. + * + * Method/field names for Class nodes are extracted by the ingestion + * pipeline's AST extractors and passed via node.methodNames/node.fieldNames. */ import type { EmbeddableNode, EmbeddingConfig } from './types.js'; -import { DEFAULT_EMBEDDING_CONFIG } from './types.js'; +import { DEFAULT_EMBEDDING_CONFIG, isShortLabel } from './types.js'; /** - * Extract the filename from a file path + * Truncate description to max length at sentence/word boundary */ -const getFileName = (filePath: string): string => { - const parts = filePath.split('/'); - return parts[parts.length - 1] || filePath; -}; +const truncateDescription = (text: string, maxLength: number): string => { + if (text.length <= maxLength) return text; -/** - * Extract the directory path from a file path - */ -const getDirectory = (filePath: string): string => { - const parts = filePath.split('/'); - parts.pop(); - return parts.join('/') || ''; -}; + const truncated = text.slice(0, maxLength); -/** - * Truncate content to max length, preserving word boundaries - */ -const truncateContent = (content: string, maxLength: number): string => { - if (content.length <= maxLength) { - return content; + // Try sentence boundary (. ! ?) + const sentenceEnd = Math.max( + truncated.lastIndexOf('. '), + truncated.lastIndexOf('! '), + truncated.lastIndexOf('? '), + ); + if (sentenceEnd > maxLength * 0.5) { + return truncated.slice(0, sentenceEnd + 1); } - // Find last space before maxLength to avoid cutting words - const truncated = content.slice(0, maxLength); + // Try word boundary const lastSpace = truncated.lastIndexOf(' '); - - if (lastSpace > maxLength * 0.8) { - return truncated.slice(0, lastSpace) + '...'; + if (lastSpace > maxLength * 0.5) { + return truncated.slice(0, lastSpace); } - return truncated + '...'; + return truncated; }; /** * Clean code content for embedding - * Removes excessive whitespace while preserving structure */ const cleanContent = (content: string): string => { - return ( - content - // Normalize line endings - .replace(/\r\n/g, '\n') - // Remove excessive blank lines (more than 2) - .replace(/\n{3,}/g, '\n\n') - // Trim each line - .split('\n') - .map((line) => line.trimEnd()) - .join('\n') - .trim() - ); + return content + .replace(/\r\n/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .split('\n') + .map((line) => line.trimEnd()) + .join('\n') + .trim(); }; /** - * Generate embedding text for a Function node + * Build metadata header for a node */ -const generateFunctionText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Function: ${node.name}`, `File: ${getFileName(node.filePath)}`]; +const buildMetadataHeader = (node: EmbeddableNode, config: Partial): string => { + const parts: string[] = []; - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); + // Label + name + parts.push(`${node.label}: ${node.name}`); + + // Repo name + if (node.repoName) { + parts.push(`Repo: ${node.repoName}`); } - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); + // Server name (optional) + if (node.serverName) { + parts.push(`Server: ${node.serverName}`); + } + + // Full file path + parts.push(`Path: ${node.filePath}`); + + // Export status + if (node.isExported !== undefined) { + parts.push(`Export: ${node.isExported}`); + } + + // Description (truncated) + if (node.description) { + const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength; + const truncated = truncateDescription(node.description, maxLen); + if (truncated) { + parts.push(truncated); + } } return parts.join('\n'); }; -/** - * Generate embedding text for a Class node - */ -const generateClassText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Class: ${node.name}`, `File: ${getFileName(node.filePath)}`]; +const generateCodeBodyText = ( + node: EmbeddableNode, + codeBody: string, + config: Partial, +): string => { + const header = buildMetadataHeader(node, config); + const cleaned = cleanContent(codeBody); + return `${header}\n\n${cleaned}`; +}; - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); +/** + * Generate embedding text for Class nodes + * Signature + properties + method name list only (no method bodies) + * Method/field names come from AST extractors via node.methodNames/node.fieldNames. + */ +const generateClassText = ( + node: EmbeddableNode, + codeBody: string, + config: Partial, +): string => { + return generateStructuralTypeText(node, codeBody, config); +}; + +const generateStructuralTypeText = ( + node: EmbeddableNode, + codeBody: string, + config: Partial, +): string => { + const header = buildMetadataHeader(node, config); + const parts: string[] = [header]; + + if (node.methodNames?.length) { + parts.push(`Methods: ${node.methodNames.join(', ')}`); + } + if (node.fieldNames?.length) { + parts.push(`Properties: ${node.fieldNames.join(', ')}`); } - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); + const declarationOnly = extractDeclarationOnly(cleanContent(node.content)); + if (declarationOnly) { + parts.push('', declarationOnly); + } + + const cleanedChunk = cleanContent(codeBody); + if (cleanedChunk && cleanedChunk !== cleanContent(node.content)) { + parts.push('', cleanedChunk); } return parts.join('\n'); }; -/** - * Generate embedding text for a Method node - */ -const generateMethodText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Method: ${node.name}`, `File: ${getFileName(node.filePath)}`]; - - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); - } - - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); - } - - return parts.join('\n'); -}; +const DECL_START_RE = + /^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/; /** - * Generate embedding text for an Interface node + * Extract class/interface/struct declaration lines, skipping method bodies. + * - Brace-based languages: detects method signatures (lines with `(` and `{`) + * and skips until depth returns to class body level. + * - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction). */ -const generateInterfaceText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`Interface: ${node.name}`, `File: ${getFileName(node.filePath)}`]; +export const extractDeclarationOnly = (content: string): string => { + const lines = content.split('\n'); + const declLines: string[] = []; + let depth = 0; + let started = false; + let classDepth = 0; + let skipDepth = 0; - const dir = getDirectory(node.filePath); - if (dir) { - parts.push(`Directory: ${dir}`); + for (const [idx, line] of lines.entries()) { + const trimmed = line.trim(); + + if (!started) { + if (DECL_START_RE.test(trimmed)) { + // Non-brace language check: current line or next 3 lines must have `{` + const nextLines = lines.slice(idx + 1, idx + 4); + if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) { + return ''; + } + started = true; + declLines.push(trimmed); + for (const ch of trimmed) { + if (ch === '{') depth++; + else if (ch === '}') depth--; + } + if (depth > 0) classDepth = depth; + } + continue; + } + + // Always update depth (even when skipping) + const opens = (trimmed.match(/{/g) || []).length; + const closes = (trimmed.match(/}/g) || []).length; + const prevDepth = depth; + depth += opens - closes; + + if (skipDepth > 0) { + if (depth <= classDepth) { + skipDepth = 0; + // Closing brace of class + if (depth <= 0) { + declLines.push(trimmed); + break; + } + } + continue; + } + + // Detect method signature: line has both `(` and `{` and goes deeper than class body + const hasParens = trimmed.includes('('); + const hasOpenBrace = opens > 0; + if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) { + if (opens === closes && trimmed.endsWith(';')) { + // Property with function/object initializer like `config = { timeout: 5000 };` — keep + declLines.push(trimmed); + } + // else: single-line or multi-line method — skip entirely + if (opens !== closes) { + skipDepth = classDepth; + } + continue; + } + + declLines.push(trimmed); + + if (depth <= 0 && declLines.length > 1) break; } - if (node.content) { - const cleanedContent = cleanContent(node.content); - const snippet = truncateContent(cleanedContent, maxSnippetLength); - parts.push('', snippet); - } - - return parts.join('\n'); -}; - -/** - * Generate embedding text for a File node - * Uses file name and first N characters of content - */ -const generateFileText = (node: EmbeddableNode, maxSnippetLength: number): string => { - const parts: string[] = [`File: ${node.name}`, `Path: ${node.filePath}`]; - - if (node.content) { - const cleanedContent = cleanContent(node.content); - // For files, use a shorter snippet since they can be very long - const snippet = truncateContent(cleanedContent, Math.min(maxSnippetLength, 300)); - parts.push('', snippet); - } - - return parts.join('\n'); + return declLines.join('\n').trim(); }; /** * Generate embedding text for any embeddable node * Dispatches to the appropriate generator based on node label - * - * @param node - The node to generate text for - * @param config - Optional configuration for max snippet length - * @returns Text suitable for embedding */ export const generateEmbeddingText = ( node: EmbeddableNode, + codeBody: string, config: Partial = {}, ): string => { - const maxSnippetLength = config.maxSnippetLength ?? DEFAULT_EMBEDDING_CONFIG.maxSnippetLength; - - switch (node.label) { - case 'Function': - return generateFunctionText(node, maxSnippetLength); - case 'Class': - return generateClassText(node, maxSnippetLength); - case 'Method': - return generateMethodText(node, maxSnippetLength); - case 'Interface': - return generateInterfaceText(node, maxSnippetLength); - case 'File': - return generateFileText(node, maxSnippetLength); - default: - // Fallback for any other embeddable type - return `${node.label}: ${node.name}\nPath: ${node.filePath}`; + if (isShortLabel(node.label)) { + const header = buildMetadataHeader(node, config); + const cleaned = cleanContent(node.content); + return `${header}\n\n${cleaned}`; } + + if (node.label === 'Class') { + return generateClassText(node, codeBody, config); + } + + if (node.label === 'Interface') { + return generateStructuralTypeText(node, codeBody, config); + } + + return generateCodeBodyText(node, codeBody, config); }; /** - * Generate embedding texts for a batch of nodes - * - * @param nodes - Array of nodes to generate text for - * @param config - Optional configuration - * @returns Array of texts in the same order as input nodes + * Export truncation helper for testing */ -export const generateBatchEmbeddingTexts = ( - nodes: EmbeddableNode[], - config: Partial = {}, -): string[] => { - return nodes.map((node) => generateEmbeddingText(node, config)); -}; +export { truncateDescription }; diff --git a/gitnexus/src/core/embeddings/types.ts b/gitnexus/src/core/embeddings/types.ts index 599faa3a9..c24dcdf40 100644 --- a/gitnexus/src/core/embeddings/types.ts +++ b/gitnexus/src/core/embeddings/types.ts @@ -5,10 +5,40 @@ */ /** - * Node labels that should be embedded for semantic search - * These are code elements that benefit from semantic matching + * Node labels that need chunking (have code body, potentially long) */ -export const EMBEDDABLE_LABELS = ['Function', 'Class', 'Method', 'Interface', 'File'] as const; +export const CHUNKABLE_LABELS = [ + 'Function', + 'Method', + 'Constructor', + 'Class', + 'Interface', + 'Struct', + 'Enum', + 'Trait', + 'Impl', + 'Macro', + 'Namespace', +] as const; + +/** + * Node labels that are short (no chunking needed, embed directly) + */ +export const SHORT_LABELS = [ + 'TypeAlias', + 'Typedef', + 'Const', + 'Property', + 'Record', + 'Union', + 'Static', + 'Variable', +] as const; + +/** + * All embeddable labels (union of CHUNKABLE + SHORT) + */ +export const EMBEDDABLE_LABELS = [...CHUNKABLE_LABELS, ...SHORT_LABELS] as const; export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number]; @@ -18,6 +48,39 @@ export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number]; export const isEmbeddableLabel = (label: string): label is EmbeddableLabel => EMBEDDABLE_LABELS.includes(label as EmbeddableLabel); +/** + * Check if a label needs chunking + */ +export const isChunkableLabel = (label: string): boolean => + (CHUNKABLE_LABELS as readonly string[]).includes(label); + +/** + * Check if a label is a short type (no chunking) + */ +export const isShortLabel = (label: string): boolean => + (SHORT_LABELS as readonly string[]).includes(label); + +/** + * Node labels that have structural names (methods/fields) extractable via AST + */ +export const STRUCTURAL_LABELS: ReadonlySet = new Set([ + 'Class', + 'Struct', + 'Interface', + 'Enum', +]); + +/** + * Node labels that have isExported column in their schema + */ +export const LABELS_WITH_EXPORTED = new Set([ + 'Function', + 'Class', + 'Interface', + 'Method', + 'CodeElement', +]) as ReadonlySet; + /** * Embedding pipeline phases */ @@ -57,6 +120,12 @@ export interface EmbeddingConfig { device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm'; /** Maximum characters of code snippet to include */ maxSnippetLength: number; + /** Maximum code chunk size in characters (for chunking long code) */ + chunkSize: number; + /** Overlap between chunks in characters */ + overlap: number; + /** Maximum description length in characters */ + maxDescriptionLength: number; } /** @@ -70,6 +139,9 @@ export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = { dimensions: 384, device: 'auto', maxSnippetLength: 500, + chunkSize: 1200, + overlap: 120, + maxDescriptionLength: 150, }; /** @@ -96,6 +168,34 @@ export interface EmbeddableNode { content: string; startLine?: number; endLine?: number; + isExported?: boolean; + description?: string; + parameterCount?: number; + returnType?: string; + repoName?: string; + serverName?: string; + methodNames?: string[]; + fieldNames?: string[]; +} + +/** + * Cached embedding entry restored from LadybugDB before a graph rebuild + */ +export interface CachedEmbedding { + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + embedding: number[]; + contentHash?: string; +} + +/** + * Context info for embedding pipeline (repo/server metadata enrichment) + */ +export interface EmbeddingContext { + repoName?: string; + serverName?: string; } /** @@ -108,3 +208,75 @@ export interface ModelProgress { loaded?: number; total?: number; } + +export interface ChunkSearchRow { + nodeId: string; + chunkIndex: number; + startLine: number; + endLine: number; + distance: number; +} + +export interface BestChunkMatch { + chunkIndex: number; + startLine: number; + endLine: number; + distance: number; +} + +/** + * Deduplicate vector search chunk results by nodeId, + * keeping the chunk with smallest distance for each node. + */ +export const dedupBestChunks = ( + rows: ChunkSearchRow[], + limit?: number, +): Map => { + const best = new Map(); + for (const row of rows) { + const existing = best.get(row.nodeId); + if (!existing || row.distance < existing.distance) { + best.set(row.nodeId, { + chunkIndex: row.chunkIndex, + startLine: row.startLine, + endLine: row.endLine, + distance: row.distance, + }); + } + if (limit !== undefined && best.size >= limit) break; + } + return best; +}; + +const DEFAULT_FETCH_MULTIPLIER = 4; +const DEFAULT_FETCH_BUFFER = 8; +const DEFAULT_MAX_FETCH = 200; + +/** + * Fetch vector-search chunks until we have enough unique nodeIds + * or can tell the result set is exhausted. + */ +export const collectBestChunks = async ( + limit: number, + fetchRows: (fetchLimit: number) => Promise, + maxFetch: number = DEFAULT_MAX_FETCH, +): Promise> => { + if (limit <= 0) return new Map(); + + let fetchLimit = Math.max(limit * DEFAULT_FETCH_MULTIPLIER, limit + DEFAULT_FETCH_BUFFER); + let previousFetchLimit = 0; + + while (fetchLimit > previousFetchLimit) { + const rows = await fetchRows(fetchLimit); + const bestChunks = dedupBestChunks(rows, limit); + + if (bestChunks.size >= limit || rows.length < fetchLimit) { + return bestChunks; + } + + previousFetchLimit = fetchLimit; + fetchLimit = fetchLimit >= maxFetch ? fetchLimit * 2 : Math.min(maxFetch, fetchLimit * 2); + } + + return new Map(); +}; diff --git a/gitnexus/src/core/group/extractors/http-patterns/node.ts b/gitnexus/src/core/group/extractors/http-patterns/node.ts index 587f48e8c..fbf988665 100644 --- a/gitnexus/src/core/group/extractors/http-patterns/node.ts +++ b/gitnexus/src/core/group/extractors/http-patterns/node.ts @@ -17,6 +17,9 @@ import type { HttpDetection, HttpLanguagePlugin } from './types.js'; * - Express `router.get(...)` / `app.post(...)` providers * - `fetch(url)` / `fetch(url, { method: 'POST' })` consumers * - `axios.get(url)` / `axios.delete(url)` consumers + * - `axios({ method, url })` object-form consumers + * - jQuery `$.get(url)` / `$.post(url, ...)` shorthand consumers + * - jQuery `$.ajax({ url, method | type })` consumers * * Because the JavaScript and TypeScript tree-sitter grammars share * node type names for every construct we query, pattern sources are @@ -103,6 +106,48 @@ const AXIOS_SPEC: PatternSpec> = { `, }; +// ─── Consumer: jQuery shorthand $.get(url) / $.post(url, ...) ──────── +// `$` is a valid JS identifier, so tree-sitter parses `$.get(...)` as a +// call_expression whose function is a member_expression on identifier `$`. +const JQUERY_SHORTHAND_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "$") + property: (property_identifier) @http_method (#match? @http_method "^(get|post)$")) + arguments: (arguments . [(string) (template_string)] @path)) + `, +}; + +// ─── Consumer: jQuery $.ajax({ url, method|type }) ─────────────────── +// The query captures the options object only; key/value pairs are read +// programmatically via `readStringProp` below, which tolerates any key +// order and accepts either `method:` or `type:` (jQuery supports both). +const JQUERY_AJAX_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (member_expression + object: (identifier) @obj (#eq? @obj "$") + property: (property_identifier) @fn (#eq? @fn "ajax")) + arguments: (arguments (object) @options)) + `, +}; + +// ─── Consumer: axios({ method, url }) object form ──────────────────── +// Distinct from AXIOS_SPEC above because the call target is an identifier +// (`axios`) rather than a member expression (`axios.get`). As with the +// jQuery ajax form, option keys are resolved programmatically. +const AXIOS_OBJECT_SPEC: PatternSpec> = { + meta: {}, + query: ` + (call_expression + function: (identifier) @fn (#eq? @fn "axios") + arguments: (arguments (object) @options)) + `, +}; + interface NodePatternBundle { controller: CompiledPatterns>; methodDecorator: CompiledPatterns>; @@ -110,6 +155,9 @@ interface NodePatternBundle { fetchNoOptions: CompiledPatterns>; fetchWithOptions: CompiledPatterns>; axios: CompiledPatterns>; + jqueryShorthand: CompiledPatterns>; + jqueryAjax: CompiledPatterns>; + axiosObject: CompiledPatterns>; } function compileBundle(language: unknown, name: string): NodePatternBundle { @@ -126,6 +174,9 @@ function compileBundle(language: unknown, name: string): NodePatternBundle { fetchNoOptions: mk(FETCH_NO_OPTIONS_SPEC, 'fetch-no-options'), fetchWithOptions: mk(FETCH_WITH_OPTIONS_SPEC, 'fetch-with-options'), axios: mk(AXIOS_SPEC, 'axios'), + jqueryShorthand: mk(JQUERY_SHORTHAND_SPEC, 'jquery-shorthand'), + jqueryAjax: mk(JQUERY_AJAX_SPEC, 'jquery-ajax'), + axiosObject: mk(AXIOS_OBJECT_SPEC, 'axios-object'), }; } @@ -160,6 +211,28 @@ function joinPath(prefix: string, sub: string): string { return `/${cleanPrefix}/${cleanSub}`; } +/** + * Walk `pair` children of an `object` literal and return the unquoted + * string/template_string value for the first pair whose key matches one + * of `keyNames`. Returns null when no matching pair is present or the + * value is not a string literal. Used by the jQuery ajax / axios object + * consumers to resolve `url` / `method` / `type` keys in any order. + */ +function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string[]): string | null { + for (let i = 0; i < objectNode.namedChildCount; i++) { + const pair = objectNode.namedChild(i); + if (!pair || pair.type !== 'pair') continue; + const keyNode = pair.childForFieldName('key'); + const valueNode = pair.childForFieldName('value'); + if (!keyNode || !valueNode) continue; + if (!keyNames.includes(keyNode.text)) continue; + if (valueNode.type !== 'string' && valueNode.type !== 'template_string') continue; + const lit = unquoteLiteral(valueNode.text); + if (lit !== null) return lit; + } + return null; +} + /** * For a standalone `decorator` node (child of class_body / program), * find the related `class_declaration` node that it decorates. In @@ -351,6 +424,62 @@ function scanBundle(bundle: NodePatternBundle, tree: Parser.Tree): HttpDetection }); } + // Consumer: jQuery shorthand $.get(url) / $.post(url, ...) + for (const match of runCompiledPatterns(bundle.jqueryShorthand, tree)) { + const methodNode = match.captures.http_method; + const pathNode = match.captures.path; + if (!methodNode || !pathNode) continue; + const path = unquoteLiteral(pathNode.text); + if (path === null) continue; + out.push({ + role: 'consumer', + framework: 'jquery', + method: methodNode.text.toUpperCase(), + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: jQuery $.ajax({ url, method|type }). jQuery accepts either + // `method:` or `type:`; both default to GET when absent. + for (const match of runCompiledPatterns(bundle.jqueryAjax, tree)) { + const optionsNode = match.captures.options; + if (!optionsNode) continue; + const path = readStringProp(optionsNode, ['url']); + if (path === null) continue; + const rawMethod = readStringProp(optionsNode, ['method', 'type']); + const method = (rawMethod ?? 'GET').toUpperCase(); + out.push({ + role: 'consumer', + framework: 'jquery', + method, + path, + name: null, + confidence: 0.7, + }); + } + + // Consumer: axios({ method, url }) object form. Structurally distinct + // from axios.(url) (identifier vs member_expression call), so no + // dedup against the member-form loop above is required. + for (const match of runCompiledPatterns(bundle.axiosObject, tree)) { + const optionsNode = match.captures.options; + if (!optionsNode) continue; + const path = readStringProp(optionsNode, ['url']); + if (path === null) continue; + const rawMethod = readStringProp(optionsNode, ['method']); + const method = (rawMethod ?? 'GET').toUpperCase(); + out.push({ + role: 'consumer', + framework: 'axios', + method, + path, + name: null, + confidence: 0.7, + }); + } + return out; } diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts b/gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts new file mode 100644 index 000000000..02a6ed60f --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts @@ -0,0 +1,12 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const cCallConfig: CallExtractionConfig = { + language: SupportedLanguages.C, +}; + +export const cppCallConfig: CallExtractionConfig = { + language: SupportedLanguages.CPlusPlus, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts new file mode 100644 index 000000000..e2c0415c2 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts @@ -0,0 +1,9 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const csharpCallConfig: CallExtractionConfig = { + language: SupportedLanguages.CSharp, + typeAsReceiverHeuristic: true, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/call-extractors/configs/dart.ts new file mode 100644 index 000000000..9d3c08def --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/dart.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/dart.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const dartCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Dart, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/go.ts b/gitnexus/src/core/ingestion/call-extractors/configs/go.ts new file mode 100644 index 000000000..870fc88e0 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/go.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/go.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const goCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Go, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts new file mode 100644 index 000000000..51de04228 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts @@ -0,0 +1,59 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig, ExtractedCallSite } from '../../call-types.js'; +import type { SyntaxNode } from '../../utils/ast-helpers.js'; + +// --------------------------------------------------------------------------- +// Java method_reference (::) parsing — absorbs call-sites/java.ts +// --------------------------------------------------------------------------- + +/** + * Parse Java `method_reference` nodes (`expr::method`, `Type::new`, + * `this::m`, `super::m`). + */ +function parseJavaMethodReference(callNode: SyntaxNode): ExtractedCallSite | null { + if (callNode.type !== 'method_reference') return null; + + const recv = callNode.namedChild(0); + if (!recv) return null; + + // Type::new → constructor call + for (const c of callNode.children) { + if (c.type === 'new') { + if (recv.type !== 'identifier') return null; + return { calledName: recv.text, callForm: 'constructor' }; + } + } + + // expr::method → member call with receiver + const rhs = callNode.child(callNode.childCount - 1); + if (!rhs || rhs.type !== 'identifier') return null; + const methodName = rhs.text; + + if (recv.type === 'identifier') { + return { calledName: methodName, callForm: 'member', receiverName: recv.text }; + } + if (recv.type === 'this') { + return { calledName: methodName, callForm: 'member', receiverName: 'this' }; + } + if (recv.type === 'super') { + return { calledName: methodName, callForm: 'member', receiverName: 'super' }; + } + return null; +} + +// --------------------------------------------------------------------------- +// Configs +// --------------------------------------------------------------------------- + +export const javaCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Java, + extractLanguageCallSite: parseJavaMethodReference, + typeAsReceiverHeuristic: true, +}; + +export const kotlinCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Kotlin, + typeAsReceiverHeuristic: true, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/php.ts b/gitnexus/src/core/ingestion/call-extractors/configs/php.ts new file mode 100644 index 000000000..25ed0b9ab --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/php.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/php.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const phpCallConfig: CallExtractionConfig = { + language: SupportedLanguages.PHP, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/python.ts b/gitnexus/src/core/ingestion/call-extractors/configs/python.ts new file mode 100644 index 000000000..35ab87305 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/python.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/python.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const pythonCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Python, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts b/gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts new file mode 100644 index 000000000..d829c5c89 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/ruby.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const rubyCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Ruby, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/rust.ts b/gitnexus/src/core/ingestion/call-extractors/configs/rust.ts new file mode 100644 index 000000000..03c48f781 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/rust.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/rust.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const rustCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Rust, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/call-extractors/configs/swift.ts new file mode 100644 index 000000000..28f2c180c --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/swift.ts @@ -0,0 +1,8 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/swift.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const swiftCallConfig: CallExtractionConfig = { + language: SupportedLanguages.Swift, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts new file mode 100644 index 000000000..20a63cda9 --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts @@ -0,0 +1,12 @@ +// gitnexus/src/core/ingestion/call-extractors/configs/typescript-javascript.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { CallExtractionConfig } from '../../call-types.js'; + +export const typescriptCallConfig: CallExtractionConfig = { + language: SupportedLanguages.TypeScript, +}; + +export const javascriptCallConfig: CallExtractionConfig = { + language: SupportedLanguages.JavaScript, +}; diff --git a/gitnexus/src/core/ingestion/call-extractors/generic.ts b/gitnexus/src/core/ingestion/call-extractors/generic.ts new file mode 100644 index 000000000..82e38457b --- /dev/null +++ b/gitnexus/src/core/ingestion/call-extractors/generic.ts @@ -0,0 +1,86 @@ +// gitnexus/src/core/ingestion/call-extractors/generic.ts + +/** + * Generic table-driven call extractor factory. + * + * Mirrors method-extractors/generic.ts and field-extractors/generic.ts — + * define a config per language and generate extractors from configs. + * + * The factory converts a declarative {@link CallExtractionConfig} into a + * runtime {@link CallExtractor} whose `extract()` method: + * 1. Tries `config.extractLanguageCallSite(callNode)` for non-standard shapes. + * 2. Falls through to the generic path using shared utilities from + * `utils/call-analysis.ts` (`inferCallForm`, `extractReceiverName`, etc.). + */ + +import type { SyntaxNode } from '../utils/ast-helpers.js'; +import { + inferCallForm, + extractReceiverName, + extractReceiverNode, + extractMixedChain, + countCallArguments, +} from '../utils/call-analysis.js'; +import type { CallExtractor, CallExtractionConfig, ExtractedCallSite } from '../call-types.js'; + +/** + * Create a CallExtractor from a declarative config. + */ +export function createCallExtractor(config: CallExtractionConfig): CallExtractor { + return { + language: config.language, + + extract(callNode: SyntaxNode, callNameNode: SyntaxNode | undefined): ExtractedCallSite | null { + // ── Path 1: Language-specific call site ────────────────────────── + // Non-standard call shapes (e.g. Java `::` method references) are + // handled entirely by the config hook. When it returns a result, + // the generic path is skipped — no argCount, no mixed chain. + // + // Note: `extractLanguageCallSite` is called on every `extract()` + // invocation — both `extract(callNode, undefined)` (parse-worker + // Path 1) and `extract(callNode, callNameNode)` (Path 2). + // Language hooks must therefore be idempotent and cheap (e.g. a + // single node-type check). + if (config.extractLanguageCallSite) { + const seed = config.extractLanguageCallSite(callNode); + if (seed) { + return { + ...seed, + ...(config.typeAsReceiverHeuristic ? { typeAsReceiverHeuristic: true } : {}), + }; + } + } + + // ── Path 2: Generic extraction via @call.name ──────────────────── + if (!callNameNode) return null; + + const calledName = callNameNode.text; + const callForm = inferCallForm(callNode, callNameNode); + let receiverName = callForm === 'member' ? extractReceiverName(callNameNode) : undefined; + let receiverMixedChain: ExtractedCallSite['receiverMixedChain']; + + // When the receiver is a complex expression (call chain, field chain, + // or mixed), extractReceiverName returns undefined. Walk the receiver + // node to build a unified mixed chain for deferred resolution. + if (callForm === 'member' && receiverName === undefined) { + const receiverNode = extractReceiverNode(callNameNode); + if (receiverNode) { + const extracted = extractMixedChain(receiverNode); + if (extracted && extracted.chain.length > 0) { + receiverMixedChain = extracted.chain; + receiverName = extracted.baseReceiverName; + } + } + } + + return { + calledName, + ...(callForm !== undefined ? { callForm } : {}), + ...(receiverName !== undefined ? { receiverName } : {}), + argCount: countCallArguments(callNode), + ...(receiverMixedChain !== undefined ? { receiverMixedChain } : {}), + ...(config.typeAsReceiverHeuristic ? { typeAsReceiverHeuristic: true } : {}), + }; + }, + }; +} diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index ce0364a4d..6c9191970 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -1,7 +1,12 @@ import { KnowledgeGraph } from '../graph/types.js'; import { ASTCache } from './ast-cache.js'; -import type { SymbolDefinition, SymbolTableReader } from './model/symbol-table.js'; -import { CLASS_TYPES, CALL_TARGET_TYPES } from './model/symbol-table.js'; +import type { + SymbolDefinition, + SymbolTableReader, + HeritageMap, + ExtractedHeritage, +} from './model/index.js'; +import { CLASS_TYPES, CALL_TARGET_TYPES, lookupMethodByOwnerWithMRO } from './model/index.js'; import Parser from 'tree-sitter'; import type { ResolutionContext } from './model/resolution-context.js'; import { TIER_CONFIDENCE, type ResolutionTier } from './model/resolution-context.js'; @@ -32,7 +37,6 @@ import { } from './utils/call-analysis.js'; import { buildTypeEnv, isSubclassOf } from './type-env.js'; import type { ConstructorBinding, TypeEnvironment } from './type-env.js'; -import type { HeritageMap } from './model/heritage-map.js'; import type { BindingAccumulator } from './binding-accumulator.js'; import { getTreeSitterBufferSize } from './constants.js'; import type { @@ -42,14 +46,11 @@ import type { ExtractedFetchCall, FileConstructorBindings, } from './workers/parse-worker.js'; -import type { ExtractedHeritage } from './model/heritage-map.js'; import { normalizeFetchURL, routeMatches } from './route-extractors/nextjs.js'; import { extractTemplateComponents } from './vue-sfc-extractor.js'; import { extractReturnTypeName, stripNullable } from './type-extractors/shared.js'; import type { LiteralTypeInferrer } from './type-extractors/types.js'; import type { SyntaxNode } from './utils/ast-helpers.js'; -import { extractParsedCallSite } from './call-sites/extract-language-call-site.js'; -import { lookupMethodByOwnerWithMRO } from './model/resolve.js'; /** Per-file resolved type bindings for exported symbols. * Populated during call processing, consumed by Phase 14 re-resolution pass. */ @@ -910,74 +911,79 @@ export const processCalls = async ( if (!captureMap['call']) return; const callNode = captureMap['call']; - const languageSeed = extractParsedCallSite(language, callNode); - if (languageSeed) { - if (provider.isBuiltInName(languageSeed.calledName)) return; + const callExtractor = provider.callExtractor; - const sourceId = - findEnclosingFunction(callNode, file.path, ctx, provider) || - generateId('File', file.path); - const receiverName = - languageSeed.callForm === 'member' ? languageSeed.receiverName : undefined; - let receiverTypeName = - receiverName && typeEnv ? typeEnv.lookup(receiverName, callNode) : undefined; + // ── Language-specific call site (e.g. Java :: method references) ── + if (callExtractor) { + const langCallSite = callExtractor.extract(callNode, undefined); + if (langCallSite) { + if (provider.isBuiltInName(langCallSite.calledName)) return; - if ( - receiverName !== undefined && - receiverTypeName === undefined && - languageSeed.callForm === 'member' && - (language === 'java' || language === 'csharp' || language === 'kotlin') - ) { - const c0 = receiverName.charCodeAt(0); - if (c0 >= 65 && c0 <= 90) receiverTypeName = receiverName; - } + const sourceId = + findEnclosingFunction(callNode, file.path, ctx, provider) || + generateId('File', file.path); + const receiverName = + langCallSite.callForm === 'member' ? langCallSite.receiverName : undefined; + let receiverTypeName = + receiverName && typeEnv ? typeEnv.lookup(receiverName, callNode) : undefined; - const resolved = resolveCallTarget( - { - calledName: languageSeed.calledName, - callForm: languageSeed.callForm, - ...(receiverTypeName !== undefined ? { receiverTypeName } : {}), - ...(receiverName !== undefined ? { receiverName } : {}), - }, - file.path, - ctx, - undefined, - widenCache, - undefined, - heritageMap, - ); + if ( + langCallSite.typeAsReceiverHeuristic && + receiverName !== undefined && + receiverTypeName === undefined && + langCallSite.callForm === 'member' + ) { + const c0 = receiverName.charCodeAt(0); + if (c0 >= 65 && c0 <= 90) receiverTypeName = receiverName; + } - if (!resolved) return; - graph.addRelationship({ - id: generateId('CALLS', `${sourceId}:${languageSeed.calledName}->${resolved.nodeId}`), - sourceId, - targetId: resolved.nodeId, - type: 'CALLS', - confidence: resolved.confidence, - reason: resolved.reason, - }); - - if (heritageMap && languageSeed.callForm === 'member' && receiverTypeName) { - const implTargets = findInterfaceDispatchTargets( - languageSeed.calledName, - receiverTypeName, + const resolved = resolveCallTarget( + { + calledName: langCallSite.calledName, + callForm: langCallSite.callForm, + ...(receiverTypeName !== undefined ? { receiverTypeName } : {}), + ...(receiverName !== undefined ? { receiverName } : {}), + }, file.path, ctx, + undefined, + widenCache, + undefined, heritageMap, - resolved.nodeId, ); - for (const impl of implTargets) { - graph.addRelationship({ - id: generateId('CALLS', `${sourceId}:${languageSeed.calledName}->${impl.nodeId}`), - sourceId, - targetId: impl.nodeId, - type: 'CALLS', - confidence: impl.confidence, - reason: impl.reason, - }); + + if (!resolved) return; + graph.addRelationship({ + id: generateId('CALLS', `${sourceId}:${langCallSite.calledName}->${resolved.nodeId}`), + sourceId, + targetId: resolved.nodeId, + type: 'CALLS', + confidence: resolved.confidence, + reason: resolved.reason, + }); + + if (heritageMap && langCallSite.callForm === 'member' && receiverTypeName) { + const implTargets = findInterfaceDispatchTargets( + langCallSite.calledName, + receiverTypeName, + file.path, + ctx, + heritageMap, + resolved.nodeId, + ); + for (const impl of implTargets) { + graph.addRelationship({ + id: generateId('CALLS', `${sourceId}:${langCallSite.calledName}->${impl.nodeId}`), + sourceId, + targetId: impl.nodeId, + type: 'CALLS', + confidence: impl.confidence, + reason: impl.reason, + }); + } } + return; } - return; } const nameNode = captureMap['call.name']; diff --git a/gitnexus/src/core/ingestion/call-sites/extract-language-call-site.ts b/gitnexus/src/core/ingestion/call-sites/extract-language-call-site.ts deleted file mode 100644 index feed2cd70..000000000 --- a/gitnexus/src/core/ingestion/call-sites/extract-language-call-site.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** Non-generic @call shapes → { calledName, callForm, receiverName? } (used from call-processor / parse-worker). */ - -import { SupportedLanguages } from '../../../config/supported-languages.js'; -import type { SyntaxNode } from '../utils/ast-helpers.js'; -import { parseJavaMethodReference } from './java.js'; - -export type ParsedCallSite = { - calledName: string; - callForm: 'free' | 'member' | 'constructor'; - receiverName?: string; -}; - -/** Non-null → seed replaces @call.name; null → use @call.name + inferCallForm / extractReceiverName. */ -export function extractParsedCallSite( - language: SupportedLanguages, - callNode: SyntaxNode, -): ParsedCallSite | null { - switch (language) { - case SupportedLanguages.Java: - if (callNode.type === 'method_reference') { - const parsed = parseJavaMethodReference(callNode); - if (!parsed) return null; - return { - calledName: parsed.calledName, - callForm: parsed.callForm, - ...(parsed.receiverName !== undefined ? { receiverName: parsed.receiverName } : {}), - }; - } - return null; - default: - return null; - } -} diff --git a/gitnexus/src/core/ingestion/call-sites/java.ts b/gitnexus/src/core/ingestion/call-sites/java.ts deleted file mode 100644 index e22c71cca..000000000 --- a/gitnexus/src/core/ingestion/call-sites/java.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** Java `method_reference` (`::`) nodes (tree-sitter-java). `super::` still lacks TypeEnv receiver typing. */ - -import type { SyntaxNode } from '../utils/ast-helpers.js'; - -export type ParsedJavaMethodReference = { - calledName: string; - callForm: 'member' | 'constructor'; - receiverName?: string; -}; - -/** Parse `expr::method`, `Type::new`, `this::m`, `super::m`. */ -export const parseJavaMethodReference = ( - callNode: SyntaxNode, -): ParsedJavaMethodReference | null => { - if (callNode.type !== 'method_reference') return null; - - const recv = callNode.namedChild(0); - if (!recv) return null; - - for (const c of callNode.children) { - if (c.type === 'new') { - if (recv.type !== 'identifier') return null; - return { calledName: recv.text, callForm: 'constructor' }; - } - } - - const rhs = callNode.child(callNode.childCount - 1); - if (!rhs || rhs.type !== 'identifier') return null; - const methodName = rhs.text; - - if (recv.type === 'identifier') { - return { calledName: methodName, callForm: 'member', receiverName: recv.text }; - } - if (recv.type === 'this') { - return { calledName: methodName, callForm: 'member', receiverName: 'this' }; - } - if (recv.type === 'super') { - return { calledName: methodName, callForm: 'member', receiverName: 'super' }; - } - return null; -}; diff --git a/gitnexus/src/core/ingestion/call-types.ts b/gitnexus/src/core/ingestion/call-types.ts new file mode 100644 index 000000000..da175720d --- /dev/null +++ b/gitnexus/src/core/ingestion/call-types.ts @@ -0,0 +1,80 @@ +// gitnexus/src/core/ingestion/call-types.ts + +/** + * Types for the language-agnostic call extraction pipeline. + * + * Mirrors method-types.ts / field-types.ts: defines the domain interfaces + * consumed by createCallExtractor() and the per-language configs. + */ + +import type { SupportedLanguages } from 'gitnexus-shared'; +import type { SyntaxNode } from './utils/ast-helpers.js'; +import type { MixedChainStep } from './utils/call-analysis.js'; + +// --------------------------------------------------------------------------- +// Extracted result +// --------------------------------------------------------------------------- + +/** + * Per-node call extraction result. The parse worker enriches this with + * file-level context (filePath, sourceId, TypeEnv lookups, arg types) to + * produce the final `ExtractedCall` that enters the resolution pipeline. + */ +export interface ExtractedCallSite { + calledName: string; + callForm?: 'free' | 'member' | 'constructor'; + receiverName?: string; + argCount?: number; + /** Unified mixed chain for complex receivers (field + call chains). */ + receiverMixedChain?: MixedChainStep[]; + /** When true, the type-as-receiver heuristic applies: if receiverName + * starts with an uppercase letter and has no TypeEnv binding, treat it + * as a type name (e.g. Java `User::getName`). */ + typeAsReceiverHeuristic?: boolean; +} + +// --------------------------------------------------------------------------- +// Extractor interface (produced by createCallExtractor) +// --------------------------------------------------------------------------- + +export interface CallExtractor { + readonly language: SupportedLanguages; + /** + * Extract a call site from captured AST nodes. + * + * @param callNode The @call capture (call_expression, method_invocation, …) + * @param callNameNode The @call.name capture (identifier inside the call). + * May be undefined when the call shape has no name capture + * (e.g. Java method_reference via `::`). + * @returns Extracted call site, or null when no call can be derived. + */ + extract(callNode: SyntaxNode, callNameNode: SyntaxNode | undefined): ExtractedCallSite | null; +} + +// --------------------------------------------------------------------------- +// Config interface (one per language / language group) +// --------------------------------------------------------------------------- + +export interface CallExtractionConfig { + language: SupportedLanguages; + + /** + * Language-specific call site extraction. Called **before** the generic + * path. If it returns non-null, the generic `inferCallForm` / + * `extractReceiverName` path is skipped entirely. + * + * Use this for call shapes that don't follow the standard `@call` / + * `@call.name` pattern (e.g. Java `method_reference` via `::`). + */ + extractLanguageCallSite?: (callNode: SyntaxNode) => ExtractedCallSite | null; + + /** + * Whether the type-as-receiver heuristic applies for this language. + * When true and the receiver name starts with an uppercase letter, + * the receiver is treated as a type name when no TypeEnv binding exists. + * + * Applies to JVM and C# languages where `Type.method()` and `Type::method` + * are common patterns. + */ + typeAsReceiverHeuristic?: boolean; +} diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts b/gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts new file mode 100644 index 000000000..fcc1a22bf --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts @@ -0,0 +1,15 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/c-cpp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const cClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.C, + typeDeclarationNodes: ['struct_specifier', 'enum_specifier'], +}; + +export const cppClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.CPlusPlus, + typeDeclarationNodes: ['class_specifier', 'struct_specifier', 'enum_specifier'], + ancestorScopeNodeTypes: ['namespace_definition', 'class_specifier', 'struct_specifier'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts new file mode 100644 index 000000000..59b7be617 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts @@ -0,0 +1,24 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/csharp.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const csharpClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.CSharp, + typeDeclarationNodes: [ + 'class_declaration', + 'interface_declaration', + 'struct_declaration', + 'enum_declaration', + 'record_declaration', + ], + fileScopeNodeTypes: ['file_scoped_namespace_declaration'], + ancestorScopeNodeTypes: [ + 'namespace_declaration', + 'class_declaration', + 'interface_declaration', + 'struct_declaration', + 'enum_declaration', + 'record_declaration', + ], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/dart.ts b/gitnexus/src/core/ingestion/class-extractors/configs/dart.ts new file mode 100644 index 000000000..c46c05533 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/dart.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/dart.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const dartClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Dart, + typeDeclarationNodes: ['class_definition', 'extension_declaration', 'enum_declaration'], + ancestorScopeNodeTypes: ['class_definition', 'extension_declaration', 'enum_declaration'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/go.ts b/gitnexus/src/core/ingestion/class-extractors/configs/go.ts new file mode 100644 index 000000000..58dade9fa --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/go.ts @@ -0,0 +1,21 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/go.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const goClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Go, + typeDeclarationNodes: ['type_declaration'], + fileScopeNodeTypes: ['package_clause'], + extractName(node) { + const typeSpec = node.namedChildren.find((child) => child.type === 'type_spec'); + return typeSpec?.childForFieldName('name')?.text; + }, + extractType(node) { + const typeSpec = node.namedChildren.find((child) => child.type === 'type_spec'); + const typeNode = typeSpec?.childForFieldName('type'); + if (typeNode?.type === 'struct_type') return 'Struct'; + if (typeNode?.type === 'interface_type') return 'Interface'; + return undefined; + }, +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts b/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts new file mode 100644 index 000000000..fbd22f545 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts @@ -0,0 +1,40 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/jvm.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +// --------------------------------------------------------------------------- +// Java +// --------------------------------------------------------------------------- + +export const javaClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Java, + typeDeclarationNodes: [ + 'class_declaration', + 'interface_declaration', + 'enum_declaration', + 'record_declaration', + ], + fileScopeNodeTypes: ['package_declaration'], + ancestorScopeNodeTypes: [ + 'class_declaration', + 'interface_declaration', + 'enum_declaration', + 'record_declaration', + ], +}; + +// --------------------------------------------------------------------------- +// Kotlin +// --------------------------------------------------------------------------- + +export const kotlinClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Kotlin, + typeDeclarationNodes: ['class_declaration', 'object_declaration', 'companion_object'], + fileScopeNodeTypes: ['package_header'], + ancestorScopeNodeTypes: ['class_declaration', 'object_declaration', 'companion_object'], + extractType(node) { + if (node.type !== 'class_declaration') return undefined; + return node.children.some((child) => child?.text === 'interface') ? 'Interface' : 'Class'; + }, +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/php.ts b/gitnexus/src/core/ingestion/class-extractors/configs/php.ts new file mode 100644 index 000000000..850b415b4 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/php.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/php.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const phpClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.PHP, + typeDeclarationNodes: ['class_declaration', 'interface_declaration', 'enum_declaration'], + ancestorScopeNodeTypes: ['namespace_definition'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/python.ts b/gitnexus/src/core/ingestion/class-extractors/configs/python.ts new file mode 100644 index 000000000..42761468e --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/python.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/python.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const pythonClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Python, + typeDeclarationNodes: ['class_definition'], + ancestorScopeNodeTypes: ['class_definition'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts b/gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts new file mode 100644 index 000000000..2c4c711bd --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/ruby.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const rubyClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Ruby, + typeDeclarationNodes: ['class'], + ancestorScopeNodeTypes: ['module', 'class'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/rust.ts b/gitnexus/src/core/ingestion/class-extractors/configs/rust.ts new file mode 100644 index 000000000..7f3873802 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/rust.ts @@ -0,0 +1,10 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/rust.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const rustClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Rust, + typeDeclarationNodes: ['struct_item', 'enum_item'], + ancestorScopeNodeTypes: ['mod_item', 'struct_item', 'enum_item'], +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/swift.ts b/gitnexus/src/core/ingestion/class-extractors/configs/swift.ts new file mode 100644 index 000000000..713a02496 --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/swift.ts @@ -0,0 +1,17 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/swift.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +export const swiftClassConfig: ClassExtractionConfig = { + language: SupportedLanguages.Swift, + typeDeclarationNodes: ['class_declaration', 'protocol_declaration'], + ancestorScopeNodeTypes: ['class_declaration', 'protocol_declaration'], + extractType(node) { + if (node.type === 'protocol_declaration') return 'Interface'; + if (node.type !== 'class_declaration') return undefined; + if (node.children.some((child) => child?.text === 'struct')) return 'Struct'; + if (node.children.some((child) => child?.text === 'enum')) return 'Enum'; + return 'Class'; + }, +}; diff --git a/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts new file mode 100644 index 000000000..b2262432d --- /dev/null +++ b/gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts @@ -0,0 +1,34 @@ +// gitnexus/src/core/ingestion/class-extractors/configs/typescript-javascript.ts + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ClassExtractionConfig } from '../../class-types.js'; + +const shared: Omit = { + typeDeclarationNodes: [ + 'class_declaration', + 'abstract_class_declaration', + 'interface_declaration', + 'enum_declaration', + ], + ancestorScopeNodeTypes: [ + 'class_declaration', + 'abstract_class_declaration', + 'interface_declaration', + 'enum_declaration', + ], +}; + +export const typescriptClassConfig: ClassExtractionConfig = { + ...shared, + language: SupportedLanguages.TypeScript, +}; + +export const javascriptClassConfig: ClassExtractionConfig = { + ...shared, + language: SupportedLanguages.JavaScript, +}; + +export const vueClassConfig: ClassExtractionConfig = { + ...shared, + language: SupportedLanguages.Vue, +}; diff --git a/gitnexus/src/core/ingestion/field-types.ts b/gitnexus/src/core/ingestion/field-types.ts index 8dd1a9f6c..7867ae8a6 100644 --- a/gitnexus/src/core/ingestion/field-types.ts +++ b/gitnexus/src/core/ingestion/field-types.ts @@ -1,7 +1,7 @@ // gitnexus/src/core/ingestion/field-types.ts import type { TypeEnvironment } from './type-env.js'; -import type { SymbolTableReader } from './model/symbol-table.js'; +import type { SymbolTableReader } from './model/index.js'; import { SupportedLanguages } from 'gitnexus-shared'; /** diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/c-cpp.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/c-cpp.ts new file mode 100644 index 000000000..bcfddee0b --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/c-cpp.ts @@ -0,0 +1,18 @@ +/** + * C / C++ import resolution configs. + * Both use standard resolution for #include directives. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; + +export const cImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.C, + strategies: [createStandardStrategy(SupportedLanguages.C)], +}; + +export const cppImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.CPlusPlus, + strategies: [createStandardStrategy(SupportedLanguages.CPlusPlus)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts new file mode 100644 index 000000000..cb5f77145 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/csharp.ts @@ -0,0 +1,36 @@ +/** + * C# import resolution config. + * Namespace-based strategy via .csproj configs, then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveCSharpImportInternal, resolveCSharpNamespaceDir } from '../csharp.js'; + +/** C# namespace-based resolution strategy via .csproj configs. */ +export const csharpNamespaceStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const csharpConfigs = ctx.configs.csharpConfigs; + if (csharpConfigs.length > 0) { + const resolvedFiles = resolveCSharpImportInternal( + rawImportPath, + csharpConfigs, + ctx.normalizedFileList, + ctx.allFileList, + ctx.index, + ); + if (resolvedFiles.length > 1) { + const dirSuffix = resolveCSharpNamespaceDir(rawImportPath, csharpConfigs); + if (dirSuffix) { + return { kind: 'package', files: resolvedFiles, dirSuffix }; + } + } + if (resolvedFiles.length > 0) return { kind: 'files', files: resolvedFiles }; + } + return null; +}; + +export const csharpImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.CSharp, + strategies: [csharpNamespaceStrategy, createStandardStrategy(SupportedLanguages.CSharp)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/dart.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/dart.ts new file mode 100644 index 000000000..06dcea98a --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/dart.ts @@ -0,0 +1,58 @@ +/** + * Dart import resolution config. + * SDK/package strategy first, then relative import strategy (with ./ prepending). + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { resolveStandard } from '../standard.js'; + +/** + * Dart SDK and package: import strategy. + * Absorbs dart: SDK imports and external packages (returns empty result to stop chain). + * Returns null for relative imports to let the next strategy handle them. + */ +export const dartPackageStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + // Strip surrounding quotes from configurable_uri capture + const stripped = rawImportPath.replace(/^['"]|['"]$/g, ''); + + // Skip dart: SDK imports (dart:async, dart:io, etc.) + if (stripped.startsWith('dart:')) return { kind: 'files', files: [] }; + + // Local package: imports → resolve to lib/ + if (stripped.startsWith('package:')) { + const slashIdx = stripped.indexOf('/'); + if (slashIdx === -1) return { kind: 'files', files: [] }; + const relPath = stripped.slice(slashIdx + 1); + const candidates = [`lib/${relPath}`, relPath]; + const files: string[] = []; + for (const candidate of candidates) { + for (const fp of ctx.allFileList) { + if (fp.endsWith('/' + candidate) || fp === candidate) { + files.push(fp); + break; + } + } + if (files.length > 0) break; + } + if (files.length > 0) return { kind: 'files', files }; + return { kind: 'files', files: [] }; // external package + } + + return null; +}; + +/** + * Dart relative import strategy — prepends "./" for bare relative paths, + * then delegates to standard resolution. + */ +export const dartRelativeStrategy: ImportResolverStrategy = (rawImportPath, filePath, ctx) => { + const stripped = rawImportPath.replace(/^['"]|['"]$/g, ''); + const relPath = stripped.startsWith('.') ? stripped : './' + stripped; + return resolveStandard(relPath, filePath, ctx, SupportedLanguages.Dart); +}; + +export const dartImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Dart, + strategies: [dartPackageStrategy, dartRelativeStrategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/go.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/go.ts new file mode 100644 index 000000000..7eee17988 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/go.ts @@ -0,0 +1,35 @@ +/** + * Go import resolution config. + * Go-specific package strategy (go.mod), then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveGoPackageDir, resolveGoPackage } from '../go.js'; + +/** Go-specific package resolution strategy — resolves go.mod-based package imports. */ +export const goPackageStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const goModule = ctx.configs.goModule; + if (goModule && rawImportPath.startsWith(goModule.modulePath)) { + const pkgSuffix = resolveGoPackageDir(rawImportPath, goModule); + if (pkgSuffix) { + const pkgFiles = resolveGoPackage( + rawImportPath, + goModule, + ctx.normalizedFileList, + ctx.allFileList, + ); + if (pkgFiles.length > 0) { + return { kind: 'package', files: pkgFiles, dirSuffix: pkgSuffix }; + } + } + // Fall through if no files found (package might be external) + } + return null; +}; + +export const goImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Go, + strategies: [goPackageStrategy, createStandardStrategy(SupportedLanguages.Go)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/jvm.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/jvm.ts new file mode 100644 index 000000000..47e590fe8 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/jvm.ts @@ -0,0 +1,115 @@ +/** + * Java / Kotlin import resolution configs. + * JVM-specific wildcard/member strategy, then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveJvmWildcard, resolveJvmMemberImport, KOTLIN_EXTENSIONS } from '../jvm.js'; + +/** Java JVM resolution strategy — wildcard and member import resolution. */ +export const javaJvmStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + if (rawImportPath.endsWith('.*')) { + const matchedFiles = resolveJvmWildcard( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; + } else { + const memberResolved = resolveJvmMemberImport( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + if (memberResolved) return { kind: 'files', files: [memberResolved] }; + } + return null; +}; + +/** + * Kotlin JVM resolution strategy — wildcard/member with Java-interop + top-level function imports. + */ +export const kotlinJvmStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + if (rawImportPath.endsWith('.*')) { + const matchedFiles = resolveJvmWildcard( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + KOTLIN_EXTENSIONS, + ctx.index, + ); + if (matchedFiles.length === 0) { + const javaMatches = resolveJvmWildcard( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + if (javaMatches.length > 0) return { kind: 'files', files: javaMatches }; + } + if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; + } else { + let memberResolved = resolveJvmMemberImport( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + KOTLIN_EXTENSIONS, + ctx.index, + ); + if (!memberResolved) { + memberResolved = resolveJvmMemberImport( + rawImportPath, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + } + if (memberResolved) return { kind: 'files', files: [memberResolved] }; + + // Kotlin: top-level function imports (e.g. import models.getUser) have only 2 segments, + // which resolveJvmMemberImport skips (requires >=3). Fall back to package-directory scan + // for lowercase last segments (function/property imports). Uppercase last segments + // (class imports like models.User) fall through to standard suffix resolution. + const segments = rawImportPath.split('.'); + const lastSeg = segments[segments.length - 1]; + if (segments.length >= 2 && lastSeg[0] && lastSeg[0] === lastSeg[0].toLowerCase()) { + const pkgWildcard = segments.slice(0, -1).join('.') + '.*'; + let dirFiles = resolveJvmWildcard( + pkgWildcard, + ctx.normalizedFileList, + ctx.allFileList, + KOTLIN_EXTENSIONS, + ctx.index, + ); + if (dirFiles.length === 0) { + dirFiles = resolveJvmWildcard( + pkgWildcard, + ctx.normalizedFileList, + ctx.allFileList, + ['.java'], + ctx.index, + ); + } + if (dirFiles.length > 0) return { kind: 'files', files: dirFiles }; + } + } + return null; +}; + +export const javaImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Java, + strategies: [javaJvmStrategy, createStandardStrategy(SupportedLanguages.Java)], +}; + +export const kotlinImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Kotlin, + strategies: [kotlinJvmStrategy, createStandardStrategy(SupportedLanguages.Kotlin)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/php.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/php.ts new file mode 100644 index 000000000..5a8446f25 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/php.ts @@ -0,0 +1,26 @@ +/** + * PHP import resolution config. + * PSR-4 strategy via composer.json — no standard fallback (PSR-4 includes its own suffix matching). + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { resolvePhpImportInternal } from '../php.js'; + +/** PHP PSR-4 resolution strategy via composer.json autoload mappings. */ +export const phpPsr4Strategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const resolved = resolvePhpImportInternal( + rawImportPath, + ctx.configs.composerConfig, + ctx.allFilePaths, + ctx.normalizedFileList, + ctx.allFileList, + ctx.index, + ); + return resolved ? { kind: 'files', files: [resolved] } : null; +}; + +export const phpImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.PHP, + strategies: [phpPsr4Strategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/python.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/python.ts new file mode 100644 index 000000000..d98304561 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/python.ts @@ -0,0 +1,47 @@ +/** + * Python import resolution config. + * PEP 328 relative + proximity-based strategy, then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolvePythonImportInternal } from '../python.js'; + +/** + * Python import resolution strategy — PEP 328 relative + proximity-based bare imports. + * Returns null to continue chain for non-relative imports. + * Absorbs unresolved relative imports (returns empty result to stop the chain). + */ +export const pythonImportStrategy: ImportResolverStrategy = (rawImportPath, filePath, ctx) => { + const resolved = resolvePythonImportInternal(filePath, rawImportPath, ctx.allFilePaths); + if (resolved) { + ctx.resolveCache.set(`${filePath}::${rawImportPath}`, resolved); + return { kind: 'files', files: [resolved] }; + } + // PEP 328: unresolved relative imports should not fall through to suffix matching + if (rawImportPath.startsWith('.')) return { kind: 'files', files: [] }; + + // External dotted imports like `django.apps` should not fall through to generic + // suffix matching when the repo has unrelated local files such as `accounts/apps.py`. + // Keep suffix fallback only when the leading segment appears somewhere in-repo, + // which preserves existing internal absolute-import behavior like `accounts.models`. + const pathLike = rawImportPath.replace(/\./g, '/'); + if (pathLike.includes('/')) { + const [leadingSegment] = pathLike.split('/').filter(Boolean); + const hasRepoCandidate = + !!leadingSegment && + (ctx.index.get(`${leadingSegment}.py`) !== undefined || + ctx.index.get(`${leadingSegment}/__init__.py`) !== undefined || + ctx.index.getFilesInDir(leadingSegment, '.py').length > 0); + + if (!hasRepoCandidate) return { kind: 'files', files: [] }; + } + + return null; +}; + +export const pythonImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Python, + strategies: [pythonImportStrategy, createStandardStrategy(SupportedLanguages.Python)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/ruby.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/ruby.ts new file mode 100644 index 000000000..bf2507f70 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/ruby.ts @@ -0,0 +1,20 @@ +/** + * Ruby import resolution config. + * Require/require_relative suffix matching — no standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { suffixResolve } from '../utils.js'; + +/** Ruby require/require_relative resolution strategy. */ +export const rubyRequireStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { + const pathParts = rawImportPath.replace(/^\.\//, '').split('/').filter(Boolean); + const resolved = suffixResolve(pathParts, ctx.normalizedFileList, ctx.allFileList, ctx.index); + return resolved ? { kind: 'files', files: [resolved] } : null; +}; + +export const rubyImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Ruby, + strategies: [rubyRequireStrategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/rust.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/rust.ts new file mode 100644 index 000000000..daa2173e7 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/rust.ts @@ -0,0 +1,56 @@ +/** + * Rust import resolution config. + * Rust module strategy (grouped imports, crate/super/self paths), then standard fallback. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; +import { resolveRustImportInternal } from '../rust.js'; + +/** Rust module resolution strategy — handles grouped imports and crate/super/self paths. */ +export const rustModuleStrategy: ImportResolverStrategy = (rawImportPath, filePath, ctx) => { + // Top-level grouped: use {crate::a, crate::b} + if (rawImportPath.startsWith('{') && rawImportPath.endsWith('}')) { + const inner = rawImportPath.slice(1, -1); + const parts = inner + .split(',') + .map((p) => p.trim()) + .filter(Boolean); + const resolved: string[] = []; + for (const part of parts) { + const r = resolveRustImportInternal(filePath, part, ctx.allFilePaths); + if (r) resolved.push(r); + } + return resolved.length > 0 ? { kind: 'files', files: resolved } : null; + } + + // Scoped grouped: use crate::models::{User, Repo} + const braceIdx = rawImportPath.indexOf('::{'); + if (braceIdx !== -1 && rawImportPath.endsWith('}')) { + const pathPrefix = rawImportPath.substring(0, braceIdx); + const braceContent = rawImportPath.substring(braceIdx + 3, rawImportPath.length - 1); + const items = braceContent + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + const resolved: string[] = []; + for (const item of items) { + // Handle `use crate::models::{User, Repo as R}` — strip alias for resolution + const itemName = item.includes(' as ') ? item.split(' as ')[0].trim() : item; + const r = resolveRustImportInternal(filePath, `${pathPrefix}::${itemName}`, ctx.allFilePaths); + if (r) resolved.push(r); + } + if (resolved.length > 0) return { kind: 'files', files: resolved }; + // Fallback: resolve the prefix path itself (e.g. crate::models -> models.rs) + const prefixResult = resolveRustImportInternal(filePath, pathPrefix, ctx.allFilePaths); + if (prefixResult) return { kind: 'files', files: [prefixResult] }; + } + + return null; +}; + +export const rustImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Rust, + strategies: [rustModuleStrategy, createStandardStrategy(SupportedLanguages.Rust)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/swift.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts similarity index 53% rename from gitnexus/src/core/ingestion/import-resolvers/swift.ts rename to gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts index 7ccc10458..f7d9ec195 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/swift.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/swift.ts @@ -1,16 +1,13 @@ /** - * Swift module import resolution. - * Handles module imports via Package.swift target map. + * Swift import resolution config. + * Package.swift target map strategy — no standard fallback (unresolved = external framework). */ -import type { ImportResult, ResolveCtx } from './types.js'; +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig, ImportResolverStrategy } from '../types.js'; -/** Swift: module imports via Package.swift target map. */ -export function resolveSwiftImport( - rawImportPath: string, - _filePath: string, - ctx: ResolveCtx, -): ImportResult { +/** Swift Package.swift target map resolution strategy. */ +export const swiftPackageStrategy: ImportResolverStrategy = (rawImportPath, _filePath, ctx) => { const swiftPackageConfig = ctx.configs.swiftPackageConfig; if (swiftPackageConfig) { const targetDir = swiftPackageConfig.targets.get(rawImportPath); @@ -29,4 +26,9 @@ export function resolveSwiftImport( } } return null; // External framework (Foundation, UIKit, etc.) -} +}; + +export const swiftImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Swift, + strategies: [swiftPackageStrategy], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/configs/typescript-javascript.ts b/gitnexus/src/core/ingestion/import-resolvers/configs/typescript-javascript.ts new file mode 100644 index 000000000..b86b365fd --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/configs/typescript-javascript.ts @@ -0,0 +1,28 @@ +/** + * TypeScript / JavaScript / Vue import resolution configs. + * All use standard resolution — TS/JS with tsconfig path aliases, + * Vue delegates to TypeScript's resolver. + */ + +import { SupportedLanguages } from 'gitnexus-shared'; +import type { ImportResolutionConfig } from '../types.js'; +import { createStandardStrategy } from '../standard.js'; + +export const typescriptImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.TypeScript, + strategies: [createStandardStrategy(SupportedLanguages.TypeScript)], +}; + +export const javascriptImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.JavaScript, + strategies: [createStandardStrategy(SupportedLanguages.JavaScript)], +}; + +// Vue SFCs are preprocessed into TypeScript upstream of import resolution, +// so the resolver intentionally runs as TypeScript. `language: Vue` here is +// documentation-only metadata (see `ImportResolutionConfig.language` JSDoc +// and ARCHITECTURE.md §Vue); it is not consumed by `createImportResolver`. +export const vueImportConfig: ImportResolutionConfig = { + language: SupportedLanguages.Vue, + strategies: [createStandardStrategy(SupportedLanguages.TypeScript)], +}; diff --git a/gitnexus/src/core/ingestion/import-resolvers/csharp.ts b/gitnexus/src/core/ingestion/import-resolvers/csharp.ts index 33cc9c828..79548d7f6 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/csharp.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/csharp.ts @@ -1,13 +1,12 @@ /** - * C# namespace import resolution. - * Handles using-directive resolution via .csproj root namespace stripping. + * C# namespace import resolution — internal helpers. + * + * Strategy lives in configs/csharp.ts. + * This file contains shared helpers for namespace-based resolution. */ import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; import type { CSharpProjectConfig } from '../language-config.js'; /** @@ -126,29 +125,3 @@ export function resolveCSharpNamespaceDir( return null; } - -/** C#: namespace-based resolution via .csproj configs, with suffix-match fallback. */ -export function resolveCSharpImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - const csharpConfigs = ctx.configs.csharpConfigs; - if (csharpConfigs.length > 0) { - const resolvedFiles = resolveCSharpImportInternal( - rawImportPath, - csharpConfigs, - ctx.normalizedFileList, - ctx.allFileList, - ctx.index, - ); - if (resolvedFiles.length > 1) { - const dirSuffix = resolveCSharpNamespaceDir(rawImportPath, csharpConfigs); - if (dirSuffix) { - return { kind: 'package', files: resolvedFiles, dirSuffix }; - } - } - if (resolvedFiles.length > 0) return { kind: 'files', files: resolvedFiles }; - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.CSharp); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/dart.ts b/gitnexus/src/core/ingestion/import-resolvers/dart.ts deleted file mode 100644 index 2a3cf3f6b..000000000 --- a/gitnexus/src/core/ingestion/import-resolvers/dart.ts +++ /dev/null @@ -1,50 +0,0 @@ -/** - * Dart import resolution. - * Handles package: imports (local packages) and relative imports. - * SDK imports (dart:*) and external packages are skipped. - */ - -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; -import { SupportedLanguages } from 'gitnexus-shared'; - -export function resolveDartImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - // Strip surrounding quotes from configurable_uri capture - const stripped = rawImportPath.replace(/^['"]|['"]$/g, ''); - - // Skip dart: SDK imports (dart:async, dart:io, etc.) - if (stripped.startsWith('dart:')) return null; - - // Local package: imports → resolve to lib/ - if (stripped.startsWith('package:')) { - const slashIdx = stripped.indexOf('/'); - if (slashIdx === -1) return null; - const relPath = stripped.slice(slashIdx + 1); - const candidates = [`lib/${relPath}`, relPath]; - const files: string[] = []; - for (const candidate of candidates) { - for (const fp of ctx.allFileList) { - if (fp.endsWith('/' + candidate) || fp === candidate) { - files.push(fp); - break; - } - } - if (files.length > 0) break; - } - if (files.length > 0) return { kind: 'files', files }; - return null; - } - - // Relative imports — use standard resolution. - // Dart relative imports don't require a leading "./" (e.g. `import 'models.dart'`). - // The standard resolver only recognises paths starting with "." as relative, so - // prepend "./" when the path doesn't already start with "." to ensure correct - // same-directory resolution (without this, "models.dart" would be mangled by the - // generic dot-to-slash conversion intended for Java-style package imports). - const relPath = stripped.startsWith('.') ? stripped : './' + stripped; - return resolveStandard(relPath, filePath, ctx, SupportedLanguages.Dart); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/go.ts b/gitnexus/src/core/ingestion/import-resolvers/go.ts index 1b1eb3ebc..c33c46422 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/go.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/go.ts @@ -1,11 +1,10 @@ /** - * Go package import resolution. - * Handles Go module path-based package imports. + * Go package import resolution — internal helpers. + * + * Strategy lives in configs/go.ts. + * This file contains the shared helpers used by the strategy. */ -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; import type { GoModuleConfig } from '../language-config.js'; /** @@ -56,28 +55,3 @@ export function resolveGoPackage( return matches; } - -/** Go: package-level imports via go.mod module path. */ -export function resolveGoImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - const goModule = ctx.configs.goModule; - if (goModule && rawImportPath.startsWith(goModule.modulePath)) { - const pkgSuffix = resolveGoPackageDir(rawImportPath, goModule); - if (pkgSuffix) { - const pkgFiles = resolveGoPackage( - rawImportPath, - goModule, - ctx.normalizedFileList, - ctx.allFileList, - ); - if (pkgFiles.length > 0) { - return { kind: 'package', files: pkgFiles, dirSuffix: pkgSuffix }; - } - } - // Fall through if no files found (package might be external) - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Go); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/jvm.ts b/gitnexus/src/core/ingestion/import-resolvers/jvm.ts index 2dd7636ab..194cfdac8 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/jvm.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/jvm.ts @@ -1,13 +1,13 @@ /** - * JVM import resolution (Java + Kotlin). - * Handles wildcard imports, member/static imports, and Kotlin-specific patterns. + * JVM import resolution — internal helpers (Java + Kotlin). + * + * Strategies live in configs/jvm.ts. + * This file contains shared helpers for wildcard/member resolution + * and the Kotlin wildcard preprocessor. */ import type { SuffixIndex } from './utils.js'; import type { SyntaxNode } from '../utils/ast-helpers.js'; -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; /** Kotlin file extensions for JVM resolver reuse */ export const KOTLIN_EXTENSIONS: readonly string[] = ['.kt', '.kts']; @@ -125,108 +125,3 @@ export function resolveJvmMemberImport( return null; } - -/** Java: JVM wildcard -> member import -> standard fallthrough */ -export function resolveJavaImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - if (rawImportPath.endsWith('.*')) { - const matchedFiles = resolveJvmWildcard( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; - } else { - const memberResolved = resolveJvmMemberImport( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - if (memberResolved) return { kind: 'files', files: [memberResolved] }; - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Java); -} - -/** - * Kotlin: JVM wildcard/member with Java-interop fallback -> top-level function imports -> standard. - * Kotlin can import from .kt/.kts files OR from .java files (Java interop). - */ -export function resolveKotlinImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - if (rawImportPath.endsWith('.*')) { - const matchedFiles = resolveJvmWildcard( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - KOTLIN_EXTENSIONS, - ctx.index, - ); - if (matchedFiles.length === 0) { - const javaMatches = resolveJvmWildcard( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - if (javaMatches.length > 0) return { kind: 'files', files: javaMatches }; - } - if (matchedFiles.length > 0) return { kind: 'files', files: matchedFiles }; - } else { - let memberResolved = resolveJvmMemberImport( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - KOTLIN_EXTENSIONS, - ctx.index, - ); - if (!memberResolved) { - memberResolved = resolveJvmMemberImport( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - } - if (memberResolved) return { kind: 'files', files: [memberResolved] }; - - // Kotlin: top-level function imports (e.g. import models.getUser) have only 2 segments, - // which resolveJvmMemberImport skips (requires >=3). Fall back to package-directory scan - // for lowercase last segments (function/property imports). Uppercase last segments - // (class imports like models.User) fall through to standard suffix resolution. - const segments = rawImportPath.split('.'); - const lastSeg = segments[segments.length - 1]; - if (segments.length >= 2 && lastSeg[0] && lastSeg[0] === lastSeg[0].toLowerCase()) { - const pkgWildcard = segments.slice(0, -1).join('.') + '.*'; - let dirFiles = resolveJvmWildcard( - pkgWildcard, - ctx.normalizedFileList, - ctx.allFileList, - KOTLIN_EXTENSIONS, - ctx.index, - ); - if (dirFiles.length === 0) { - dirFiles = resolveJvmWildcard( - pkgWildcard, - ctx.normalizedFileList, - ctx.allFileList, - ['.java'], - ctx.index, - ); - } - if (dirFiles.length > 0) return { kind: 'files', files: dirFiles }; - } - } - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Kotlin); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/php.ts b/gitnexus/src/core/ingestion/import-resolvers/php.ts index 20517bbed..303bf5546 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/php.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/php.ts @@ -1,11 +1,12 @@ /** - * PHP PSR-4 import resolution. - * Handles use-statement resolution via composer.json autoload mappings. + * PHP PSR-4 import resolution — internal helpers. + * + * Strategy lives in configs/php.ts. + * This file contains the shared helper for PSR-4 resolution via composer.json. */ import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; -import type { ImportResult, ResolveCtx } from './types.js'; import type { ComposerConfig } from '../language-config.js'; /** Get or compute the sorted PSR-4 entries (cached after first call). */ @@ -91,20 +92,3 @@ export function resolvePhpImportInternal( const pathParts = normalized.split('/').filter(Boolean); return suffixResolve(pathParts, normalizedFileList, allFileList, index); } - -/** PHP: namespace-based resolution via composer.json PSR-4. */ -export function resolvePhpImport( - rawImportPath: string, - _filePath: string, - ctx: ResolveCtx, -): ImportResult { - const resolved = resolvePhpImportInternal( - rawImportPath, - ctx.configs.composerConfig, - ctx.allFilePaths, - ctx.normalizedFileList, - ctx.allFileList, - ctx.index, - ); - return resolved ? { kind: 'files', files: [resolved] } : null; -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/python.ts b/gitnexus/src/core/ingestion/import-resolvers/python.ts index 4e9c4bbb3..264103a09 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/python.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/python.ts @@ -1,12 +1,12 @@ /** * Python import resolution — PEP 328 relative imports and proximity-based bare imports. * Import system spec: PEP 302 (original), PEP 451 (current). + * + * Strategy lives in configs/python.ts. + * This file contains the shared internal helper used by the strategy and tests. */ import { tryResolveWithExtensions } from './utils.js'; -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; /** * Resolve a Python import to a file path (low-level helper). @@ -74,23 +74,3 @@ export function resolvePythonImportInternal( return null; } - -/** - * Python: relative imports (PEP 328) + proximity-based bare imports. - * Falls through to standard suffix resolution when proximity finds no match. - */ -export function resolvePythonImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - const resolved = resolvePythonImportInternal(filePath, rawImportPath, ctx.allFilePaths); - if (resolved) { - // Store in resolveCache so other files importing the same module skip the - // ancestor walk. The cache key matches resolveStandard's convention. - ctx.resolveCache.set(`${filePath}::${rawImportPath}`, resolved); - return { kind: 'files', files: [resolved] }; - } - if (rawImportPath.startsWith('.')) return null; // relative but unresolved -- don't suffix-match - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Python); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/resolver-factory.ts b/gitnexus/src/core/ingestion/import-resolvers/resolver-factory.ts new file mode 100644 index 000000000..4caf748a7 --- /dev/null +++ b/gitnexus/src/core/ingestion/import-resolvers/resolver-factory.ts @@ -0,0 +1,35 @@ +/** + * Import resolver factory — creates a composable import resolver from + * an ordered list of strategies. + * + * Mirrors the method-extractors/generic.ts and call-extractors/generic.ts + * pattern: declare a config per language, produce a runtime resolver via factory. + * + * Each strategy is tried in order. The first non-null result wins. + * A result with an empty `files` array is treated as "handled but unresolved" + * (stops the chain without producing import edges). + */ + +import type { ImportResolverFn, ImportResolutionConfig } from './types.js'; + +/** + * Create an ImportResolverFn from a declarative config. + * + * Chains strategies in declaration order — first non-null result wins. + * Returns null only if every strategy returns null. + * + * Error behaviour: if a strategy throws, the error propagates immediately + * and remaining strategies are not tried. Strategies are expected to be + * pure data transforms that never throw; any unexpected exception indicates + * a bug in the strategy implementation. + */ +export function createImportResolver(config: ImportResolutionConfig): ImportResolverFn { + const { strategies } = config; + return (rawImportPath, filePath, ctx) => { + for (const strategy of strategies) { + const result = strategy(rawImportPath, filePath, ctx); + if (result) return result; + } + return null; + }; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/ruby.ts b/gitnexus/src/core/ingestion/import-resolvers/ruby.ts index b8ada4cf4..4bf47d31f 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/ruby.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/ruby.ts @@ -1,11 +1,12 @@ /** - * Ruby require/require_relative import resolution. - * Handles path resolution for Ruby's require and require_relative calls. + * Ruby require/require_relative import resolution — internal helpers. + * + * Strategy lives in configs/ruby.ts. + * This file only contains the low-level helper used by the strategy. */ import type { SuffixIndex } from './utils.js'; import { suffixResolve } from './utils.js'; -import type { ImportResult, ResolveCtx } from './types.js'; /** * Resolve a Ruby require/require_relative path to a matching .rb file (low-level helper). @@ -22,18 +23,3 @@ export function resolveRubyImportInternal( const pathParts = importPath.replace(/^\.\//, '').split('/').filter(Boolean); return suffixResolve(pathParts, normalizedFileList, allFileList, index); } - -/** Ruby: require / require_relative. */ -export function resolveRubyImport( - rawImportPath: string, - _filePath: string, - ctx: ResolveCtx, -): ImportResult { - const resolved = resolveRubyImportInternal( - rawImportPath, - ctx.normalizedFileList, - ctx.allFileList, - ctx.index, - ); - return resolved ? { kind: 'files', files: [resolved] } : null; -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/rust.ts b/gitnexus/src/core/ingestion/import-resolvers/rust.ts index 632e19352..2d1dd864a 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/rust.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/rust.ts @@ -1,12 +1,10 @@ /** - * Rust module import resolution. - * Handles crate::, super::, self:: prefix paths and :: separators. + * Rust module import resolution — internal helpers. + * + * Strategy lives in configs/rust.ts. + * This file contains shared helpers used by the strategy and standard.ts. */ -import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ResolveCtx } from './types.js'; -import { resolveStandard } from './standard.js'; - /** * Resolve Rust use-path to a file (low-level helper). * Handles crate::, super::, self:: prefixes and :: path separators. @@ -84,49 +82,3 @@ export function tryRustModulePath(modulePath: string, allFiles: Set): st return null; } - -/** Rust: expand grouped imports: use {crate::a, crate::b} and use crate::models::{User, Repo}. */ -export function resolveRustImport( - rawImportPath: string, - filePath: string, - ctx: ResolveCtx, -): ImportResult { - // Top-level grouped: use {crate::a, crate::b} - if (rawImportPath.startsWith('{') && rawImportPath.endsWith('}')) { - const inner = rawImportPath.slice(1, -1); - const parts = inner - .split(',') - .map((p) => p.trim()) - .filter(Boolean); - const resolved: string[] = []; - for (const part of parts) { - const r = resolveRustImportInternal(filePath, part, ctx.allFilePaths); - if (r) resolved.push(r); - } - return resolved.length > 0 ? { kind: 'files', files: resolved } : null; - } - - // Scoped grouped: use crate::models::{User, Repo} - const braceIdx = rawImportPath.indexOf('::{'); - if (braceIdx !== -1 && rawImportPath.endsWith('}')) { - const pathPrefix = rawImportPath.substring(0, braceIdx); - const braceContent = rawImportPath.substring(braceIdx + 3, rawImportPath.length - 1); - const items = braceContent - .split(',') - .map((s) => s.trim()) - .filter(Boolean); - const resolved: string[] = []; - for (const item of items) { - // Handle `use crate::models::{User, Repo as R}` — strip alias for resolution - const itemName = item.includes(' as ') ? item.split(' as ')[0].trim() : item; - const r = resolveRustImportInternal(filePath, `${pathPrefix}::${itemName}`, ctx.allFilePaths); - if (r) resolved.push(r); - } - if (resolved.length > 0) return { kind: 'files', files: resolved }; - // Fallback: resolve the prefix path itself (e.g. crate::models -> models.rs) - const prefixResult = resolveRustImportInternal(filePath, pathPrefix, ctx.allFilePaths); - if (prefixResult) return { kind: 'files', files: [prefixResult] }; - } - - return resolveStandard(rawImportPath, filePath, ctx, SupportedLanguages.Rust); -} diff --git a/gitnexus/src/core/ingestion/import-resolvers/standard.ts b/gitnexus/src/core/ingestion/import-resolvers/standard.ts index a29810a2f..f8aae9625 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/standard.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/standard.ts @@ -8,7 +8,7 @@ import type { SuffixIndex } from './utils.js'; import { tryResolveWithExtensions, suffixResolve } from './utils.js'; import { resolveRustImportInternal } from './rust.js'; import { SupportedLanguages } from 'gitnexus-shared'; -import type { ImportResult, ImportResolverFn, ResolveCtx } from './types.js'; +import type { ImportResult, ImportResolverStrategy, ResolveCtx } from './types.js'; import type { TsconfigPaths } from '../language-config.js'; /** Max entries in the resolve cache. Beyond this, entries are evicted. @@ -174,18 +174,11 @@ export function resolveStandard( return resolvedPath ? { kind: 'files', files: [resolvedPath] } : null; } -/** JavaScript: standard single-file resolution. */ -export const resolveJavascriptImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.JavaScript); +// ============================================================================ +// Strategy factory — composable hook for ImportResolutionConfig +// ============================================================================ -/** TypeScript: standard single-file resolution. */ -export const resolveTypescriptImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.TypeScript); - -/** C: standard single-file resolution for #include directives. */ -export const resolveCImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.C); - -/** C++: standard single-file resolution for #include directives. */ -export const resolveCppImport: ImportResolverFn = (raw, fp, ctx) => - resolveStandard(raw, fp, ctx, SupportedLanguages.CPlusPlus); +/** Create a reusable standard-resolution strategy for a given language. */ +export function createStandardStrategy(language: SupportedLanguages): ImportResolverStrategy { + return (raw, fp, ctx) => resolveStandard(raw, fp, ctx, language); +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/types.ts b/gitnexus/src/core/ingestion/import-resolvers/types.ts index 3e68c38b2..66e23a79b 100644 --- a/gitnexus/src/core/ingestion/import-resolvers/types.ts +++ b/gitnexus/src/core/ingestion/import-resolvers/types.ts @@ -12,6 +12,7 @@ import type { } from '../language-config.js'; import type { SwiftPackageConfig } from '../language-config.js'; import type { SuffixIndex } from './utils.js'; +import type { SupportedLanguages } from 'gitnexus-shared'; /** * Result of resolving an import via language-specific dispatch. @@ -53,3 +54,28 @@ export type ImportResolverFn = ( filePath: string, resolveCtx: ResolveCtx, ) => ImportResult; + +/** + * A single import resolution strategy — one step in a composable chain. + * Same signature as ImportResolverFn. Returns null to let the next strategy + * in the chain try; returns a result (even with empty files) to stop the chain. + */ +export type ImportResolverStrategy = ImportResolverFn; + +/** + * Declarative config for composable import resolution — mirrors the + * MethodExtractionConfig / CallExtractionConfig pattern. + * + * Each language declares an ordered list of strategies to try. + * The factory (`createImportResolver`) chains them: first non-null result wins. + */ +export interface ImportResolutionConfig { + /** + * Documentation-only metadata identifying which language this config serves. + * **Not used by `createImportResolver`** — the factory only iterates `strategies`. + * Useful for logging, debugging, and compile-time exhaustiveness checks when + * mapping `SupportedLanguages → ImportResolutionConfig` in language providers. + */ + readonly language: SupportedLanguages; + readonly strategies: readonly ImportResolverStrategy[]; +} diff --git a/gitnexus/src/core/ingestion/import-resolvers/vue.ts b/gitnexus/src/core/ingestion/import-resolvers/vue.ts deleted file mode 100644 index c46e3f725..000000000 --- a/gitnexus/src/core/ingestion/import-resolvers/vue.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Vue import resolver — delegates to TypeScript's standard resolver. - * - * Vue