mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-12 23:02:45 +00:00
Merge remote-tracking branch 'origin/main' into fix/stream-graph-response
# Conflicts: # gitnexus-web/src/services/backend-client.ts # gitnexus/src/server/api.ts
This commit is contained in:
commit
fbd9040647
406 changed files with 52684 additions and 8014 deletions
75
.github/dependabot.yml
vendored
Normal file
75
.github/dependabot.yml
vendored
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
version: 2
|
||||
updates:
|
||||
# Keep third-party Actions SHA pins current. See CONTRIBUTING.md — when
|
||||
# reviewing these bumps, verify the SHA corresponds to the claimed tag by
|
||||
# running `gh api repos/<owner>/<action>/git/refs/tags/<tag>` before merge.
|
||||
- package-ecosystem: github-actions
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore
|
||||
include: scope
|
||||
labels:
|
||||
- dependencies
|
||||
- ci
|
||||
|
||||
# Gitnexus npm deps — tree-sitter grammars checked daily so we catch
|
||||
# new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars
|
||||
# are grouped so lockstep bumps produce a single PR. The tree-sitter
|
||||
# RUNTIME is pinned — upgrade deliberately via the drift check workflow.
|
||||
# See .github/scripts/check-tree-sitter-upgrade-readiness.py for
|
||||
# the upgrade readiness tracker.
|
||||
- package-ecosystem: npm
|
||||
directory: /gitnexus
|
||||
schedule:
|
||||
interval: daily
|
||||
open-pull-requests-limit: 10
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
labels:
|
||||
- dependencies
|
||||
groups:
|
||||
tree-sitter-grammars:
|
||||
patterns:
|
||||
- tree-sitter-*
|
||||
exclude-patterns:
|
||||
- tree-sitter
|
||||
- tree-sitter-cli
|
||||
ignore:
|
||||
# Pin the tree-sitter runtime at 0.21.x until the drift check
|
||||
# reports all grammars are peer-dep compatible with 0.25.
|
||||
- dependency-name: tree-sitter
|
||||
update-types:
|
||||
- version-update:semver-major
|
||||
- version-update:semver-minor
|
||||
# tree-sitter-cli follows the runtime's version cadence. Bump when
|
||||
# regenerating vendor/tree-sitter-proto/src/parser.c, not on a schedule.
|
||||
- dependency-name: tree-sitter-cli
|
||||
|
||||
# gitnexus-web (thin frontend client).
|
||||
- package-ecosystem: npm
|
||||
directory: /gitnexus-web
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
labels:
|
||||
- dependencies
|
||||
- frontend
|
||||
|
||||
# Shared types package.
|
||||
- package-ecosystem: npm
|
||||
directory: /gitnexus-shared
|
||||
schedule:
|
||||
interval: weekly
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
labels:
|
||||
- dependencies
|
||||
53
.github/release-drafter.yml
vendored
Normal file
53
.github/release-drafter.yml
vendored
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
# release-drafter config — used only for PR autolabeling by
|
||||
# `.github/workflows/pr-labeler.yml` (the workflow passes `disable-releaser: true`,
|
||||
# so the draft-release side of release-drafter never runs).
|
||||
#
|
||||
# The labels applied here are the same ones `.github/release.yml` maps to
|
||||
# categorized release-notes sections.
|
||||
#
|
||||
# `sync-labels: true` removes managed autolabels that no longer match the PR —
|
||||
# critical for the breaking-change case: if a PR title drops the `!` or the body
|
||||
# drops `BREAKING CHANGE:`, the `breaking` label is pulled off automatically.
|
||||
|
||||
# Required by release-drafter; not used because releaser is disabled.
|
||||
name-template: 'unused'
|
||||
tag-template: 'unused'
|
||||
template: |
|
||||
$CHANGES
|
||||
|
||||
sync-labels: true
|
||||
|
||||
autolabeler:
|
||||
- label: enhancement
|
||||
title:
|
||||
- '/^feat(\([^)]+\))?!?:/i'
|
||||
- label: bug
|
||||
title:
|
||||
- '/^fix(\([^)]+\))?!?:/i'
|
||||
- label: performance
|
||||
title:
|
||||
- '/^perf(\([^)]+\))?!?:/i'
|
||||
- label: refactor
|
||||
title:
|
||||
- '/^refactor(\([^)]+\))?!?:/i'
|
||||
- label: documentation
|
||||
title:
|
||||
- '/^docs(\([^)]+\))?!?:/i'
|
||||
- label: test
|
||||
title:
|
||||
- '/^test(\([^)]+\))?!?:/i'
|
||||
- label: ci
|
||||
title:
|
||||
- '/^ci(\([^)]+\))?!?:/i'
|
||||
- label: dependencies
|
||||
title:
|
||||
- '/^(build|deps)(\([^)]+\))?!?:/i'
|
||||
- label: chore
|
||||
title:
|
||||
- '/^(chore|revert)(\([^)]+\))?!?:/i'
|
||||
# Breaking-change marker: either `!` in the type prefix or `BREAKING CHANGE:` in body.
|
||||
- label: breaking
|
||||
title:
|
||||
- '/^[a-z]+(\([^)]+\))?!:/i'
|
||||
body:
|
||||
- '/BREAKING[ -]CHANGE:/i'
|
||||
358
.github/scripts/check-tree-sitter-upgrade-readiness.py
vendored
Normal file
358
.github/scripts/check-tree-sitter-upgrade-readiness.py
vendored
Normal file
|
|
@ -0,0 +1,358 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Monitor tree-sitter 0.25 upgrade readiness.
|
||||
|
||||
Tracks two things Dependabot cannot see:
|
||||
|
||||
1. Peer-dep compatibility. Each tree-sitter-* grammar declares a peer
|
||||
dependency on the tree-sitter runtime. We want to know when every
|
||||
grammar's *latest npm release* satisfies tree-sitter@0.25.0 so we
|
||||
can upgrade without --legacy-peer-deps.
|
||||
|
||||
2. Vendored upstream drift. vendor/tree-sitter-proto/ is a snapshot of
|
||||
coder3101/tree-sitter-proto's parser.c. When upstream moves, we want
|
||||
to know whether we can pick it up.
|
||||
|
||||
Invoked from .github/workflows/tree-sitter-upgrade-readiness.yml daily.
|
||||
Runs locally too:
|
||||
|
||||
python3 .github/scripts/check-tree-sitter-upgrade-readiness.py
|
||||
|
||||
Outputs Markdown to stdout. Exit 0 when every grammar is upgrade-ready
|
||||
and the vendored proto is in sync. Exit 1 when blockers remain (the
|
||||
workflow uses this to open or update a tracking issue).
|
||||
|
||||
No external deps -- stdlib only, so it runs on any vanilla runner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
GITNEXUS_DIR = REPO_ROOT / "gitnexus"
|
||||
VENDOR_PROTO_DIR = GITNEXUS_DIR / "vendor" / "tree-sitter-proto"
|
||||
|
||||
# ── Upgrade target ──────────────────────────────────────────────────────
|
||||
# The runtime version we want to upgrade TO. Update this when the goal
|
||||
# changes (e.g. once 0.25 lands and we target 0.26).
|
||||
TARGET_RUNTIME = "0.25.0"
|
||||
TARGET_RUNTIME_MAJOR_MINOR = ".".join(TARGET_RUNTIME.split(".")[:2])
|
||||
|
||||
# Tree-sitter runtime -> (min_abi, max_abi) it can load. Only the current
|
||||
# and target entries matter; extend when changing TARGET_RUNTIME.
|
||||
RUNTIME_ABI_RANGES: dict[str, tuple[int, int]] = {
|
||||
"0.21": (13, 14),
|
||||
"0.25": (13, 15),
|
||||
}
|
||||
|
||||
assert TARGET_RUNTIME_MAJOR_MINOR in RUNTIME_ABI_RANGES, (
|
||||
f"RUNTIME_ABI_RANGES has no entry for {TARGET_RUNTIME_MAJOR_MINOR!r}. "
|
||||
f"Add the ABI range after auditing the upstream release notes."
|
||||
)
|
||||
|
||||
# Grammars we use. Values are the upstream GitHub repos to check for
|
||||
# unreleased ABI bumps (owner/repo, branch, parser.c path).
|
||||
GRAMMARS: dict[str, tuple[str, str, str]] = {
|
||||
"tree-sitter-c": ("tree-sitter/tree-sitter-c", "master", "src/parser.c"),
|
||||
"tree-sitter-c-sharp": ("tree-sitter/tree-sitter-c-sharp", "master", "src/parser.c"),
|
||||
"tree-sitter-cpp": ("tree-sitter/tree-sitter-cpp", "master", "src/parser.c"),
|
||||
"tree-sitter-dart": ("UserNobody14/tree-sitter-dart", "master", "src/parser.c"),
|
||||
"tree-sitter-go": ("tree-sitter/tree-sitter-go", "master", "src/parser.c"),
|
||||
"tree-sitter-java": ("tree-sitter/tree-sitter-java", "master", "src/parser.c"),
|
||||
"tree-sitter-javascript": ("tree-sitter/tree-sitter-javascript", "master", "src/parser.c"),
|
||||
"tree-sitter-kotlin": ("fwcd/tree-sitter-kotlin", "main", "src/parser.c"),
|
||||
"tree-sitter-php": ("tree-sitter/tree-sitter-php", "master", "php/src/parser.c"),
|
||||
"tree-sitter-python": ("tree-sitter/tree-sitter-python", "master", "src/parser.c"),
|
||||
"tree-sitter-ruby": ("tree-sitter/tree-sitter-ruby", "master", "src/parser.c"),
|
||||
"tree-sitter-rust": ("tree-sitter/tree-sitter-rust", "master", "src/parser.c"),
|
||||
"tree-sitter-swift": ("alex-pinkus/tree-sitter-swift", "main", "src/parser.c"),
|
||||
"tree-sitter-typescript": ("tree-sitter/tree-sitter-typescript", "master", "typescript/src/parser.c"),
|
||||
}
|
||||
|
||||
UPSTREAM_PROTO_OWNER = "coder3101"
|
||||
UPSTREAM_PROTO_REPO = "tree-sitter-proto"
|
||||
UPSTREAM_PROTO_BRANCH = "main"
|
||||
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def read_current_runtime() -> str:
|
||||
"""Return the tree-sitter runtime version pinned in package.json (e.g. '0.21')."""
|
||||
pkg = json.loads((GITNEXUS_DIR / "package.json").read_text())
|
||||
raw = pkg["dependencies"]["tree-sitter"]
|
||||
match = re.search(r"(\d+)\.(\d+)", raw)
|
||||
if not match:
|
||||
raise SystemExit(f"could not parse tree-sitter version: {raw!r}")
|
||||
return f"{match.group(1)}.{match.group(2)}"
|
||||
|
||||
|
||||
def npm_view_json(pkg: str) -> dict | None:
|
||||
"""Fetch package metadata from the npm registry via HTTPS.
|
||||
|
||||
Uses the registry API directly so we don't depend on the npm CLI
|
||||
being available (it's a batch file on Windows which complicates
|
||||
subprocess calls).
|
||||
"""
|
||||
url = f"https://registry.npmjs.org/{pkg}/latest"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"Accept": "application/json"})
|
||||
with urllib.request.urlopen(req, timeout=8) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def satisfies_target(peer_range: str | None, target: str) -> bool:
|
||||
"""Check if a semver range like '^0.22.4' or '^0.25.0' satisfies the target.
|
||||
|
||||
Simple heuristic: extract the minimum version from the range and check
|
||||
if target >= min. For caret ranges (^X.Y.Z), the upper bound is the
|
||||
next major (for X>0) or next minor (for X==0). We check both bounds.
|
||||
"""
|
||||
if peer_range is None:
|
||||
# No peer dep declared = no constraint = compatible.
|
||||
return True
|
||||
match = re.search(r"(\d+)\.(\d+)\.(\d+)", peer_range)
|
||||
if not match:
|
||||
return False
|
||||
min_major, min_minor, min_patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
|
||||
|
||||
t_match = re.search(r"(\d+)\.(\d+)\.(\d+)", target)
|
||||
if not t_match:
|
||||
return False
|
||||
t_major, t_minor, t_patch = int(t_match.group(1)), int(t_match.group(2)), int(t_match.group(3))
|
||||
|
||||
# Target must be >= minimum.
|
||||
target_tuple = (t_major, t_minor, t_patch)
|
||||
min_tuple = (min_major, min_minor, min_patch)
|
||||
if target_tuple < min_tuple:
|
||||
return False
|
||||
|
||||
# For caret ranges with major 0: ^0.X.Y allows [0.X.Y, 0.(X+1).0).
|
||||
if peer_range.startswith("^") and min_major == 0:
|
||||
if t_major != 0 or t_minor >= min_minor + 1:
|
||||
return False
|
||||
# For caret ranges with major >0: ^X.Y.Z allows [X.Y.Z, (X+1).0.0).
|
||||
elif peer_range.startswith("^") and min_major > 0:
|
||||
if t_major >= min_major + 1:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
_GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
|
||||
|
||||
|
||||
def fetch_text(url: str, timeout: int = 8) -> str | None:
|
||||
"""Fetch a URL and return its text, or None on failure.
|
||||
|
||||
Adds an Authorization header for github.com URLs when GITHUB_TOKEN is
|
||||
set (raises the rate limit from 60 to 5 000 requests/hour).
|
||||
"""
|
||||
headers: dict[str, str] = {}
|
||||
if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url):
|
||||
headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.read().decode("utf-8", errors="ignore")
|
||||
except (urllib.error.URLError, urllib.error.HTTPError):
|
||||
return None
|
||||
|
||||
|
||||
def extract_abi_from_text(text: str) -> int | None:
|
||||
"""Extract LANGUAGE_VERSION from parser.c text."""
|
||||
match = re.search(r"#define\s+LANGUAGE_VERSION\s+(\d+)", text[:4096])
|
||||
return int(match.group(1)) if match else None
|
||||
|
||||
|
||||
def extract_language_version(parser_c: pathlib.Path) -> int | None:
|
||||
"""Return the LANGUAGE_VERSION defined in a parser.c, or None if absent."""
|
||||
if not parser_c.is_file():
|
||||
return None
|
||||
with parser_c.open("r", encoding="utf-8", errors="ignore") as fh:
|
||||
head = fh.read(4096)
|
||||
return extract_abi_from_text(head)
|
||||
|
||||
|
||||
def md_h(text: str, level: int = 2) -> str:
|
||||
return f"{'#' * level} {text}\n"
|
||||
|
||||
|
||||
# ── Main ────────────────────────────────────────────────────────────────
|
||||
|
||||
def main() -> int:
|
||||
blockers: dict[str, str] = {}
|
||||
lines: list[str] = []
|
||||
lines.append(md_h("Tree-sitter 0.25 upgrade readiness", 1))
|
||||
lines.append("")
|
||||
|
||||
current_runtime = read_current_runtime()
|
||||
current_abi_range = RUNTIME_ABI_RANGES.get(current_runtime, (0, 0))
|
||||
target_abi_range = RUNTIME_ABI_RANGES.get(TARGET_RUNTIME_MAJOR_MINOR, (0, 0))
|
||||
|
||||
lines.append(f"- Current runtime: `tree-sitter@{current_runtime}.x` (ABI {current_abi_range[0]}..{current_abi_range[1]})")
|
||||
lines.append(f"- Target runtime: `tree-sitter@{TARGET_RUNTIME}` (ABI {target_abi_range[0]}..{target_abi_range[1]})")
|
||||
lines.append("")
|
||||
|
||||
# ── Grammar peer-dep compatibility ───────────────────────────────
|
||||
lines.append(md_h("Grammar compatibility", 2))
|
||||
lines.append("| Grammar | npm latest | Peer dep | Satisfies 0.25? | ABI | Upstream ABI | Status |")
|
||||
lines.append("|---|---|---|---|---|---|---|")
|
||||
|
||||
ready_count = 0
|
||||
total_count = len(GRAMMARS)
|
||||
|
||||
for name, (upstream_repo, upstream_branch, parser_path) in sorted(GRAMMARS.items()):
|
||||
# Fetch latest npm metadata.
|
||||
info = npm_view_json(name)
|
||||
fetch_failed = info is None
|
||||
npm_version = "?"
|
||||
peer_range = None
|
||||
peer_optional = True
|
||||
if info:
|
||||
npm_version = info.get("version", "?")
|
||||
peers = info.get("peerDependencies") or {}
|
||||
peer_range = peers.get("tree-sitter")
|
||||
meta = info.get("peerDependenciesMeta") or {}
|
||||
ts_meta = meta.get("tree-sitter") or {}
|
||||
peer_optional = ts_meta.get("optional", False) if peer_range else True
|
||||
|
||||
if fetch_failed:
|
||||
peer_display = "? (fetch failed)"
|
||||
compatible = False
|
||||
else:
|
||||
peer_display = peer_range or "none"
|
||||
if peer_range and not peer_optional:
|
||||
peer_display += " (required)"
|
||||
compatible = satisfies_target(peer_range, TARGET_RUNTIME)
|
||||
|
||||
# Check installed ABI using the same parser_path from GRAMMARS.
|
||||
installed_parser = GITNEXUS_DIR / "node_modules" / name / parser_path
|
||||
if not installed_parser.is_file():
|
||||
# Fallback to default location.
|
||||
installed_parser = GITNEXUS_DIR / "node_modules" / name / "src" / "parser.c"
|
||||
installed_abi = extract_language_version(installed_parser)
|
||||
abi_display = str(installed_abi) if installed_abi else "?"
|
||||
|
||||
# Check upstream (main/master branch) ABI for unreleased work.
|
||||
upstream_url = (
|
||||
f"https://raw.githubusercontent.com/{upstream_repo}/"
|
||||
f"{upstream_branch}/{parser_path}"
|
||||
)
|
||||
upstream_text = fetch_text(upstream_url)
|
||||
upstream_abi = extract_abi_from_text(upstream_text) if upstream_text else None
|
||||
upstream_abi_display = str(upstream_abi) if upstream_abi else "?"
|
||||
|
||||
# Determine status.
|
||||
if fetch_failed:
|
||||
status = "Unknown (fetch failed)"
|
||||
blockers[name] = f"`{name}`: npm registry fetch failed — could not verify peer dep"
|
||||
elif compatible:
|
||||
status = "Ready"
|
||||
ready_count += 1
|
||||
elif upstream_abi and upstream_abi >= 15:
|
||||
status = "Unreleased (ABI 15 on main)"
|
||||
blockers[name] = f"`{name}`: ABI 15 on `{upstream_repo}` main but not published to npm"
|
||||
else:
|
||||
status = "Blocking"
|
||||
blockers[name] = f"`{name}@{npm_version}`: peer `{peer_display}` incompatible with 0.25"
|
||||
|
||||
# Also check upstream package.json for relaxed peer dep.
|
||||
if not compatible and not fetch_failed:
|
||||
upstream_pkg_url = (
|
||||
f"https://raw.githubusercontent.com/{upstream_repo}/"
|
||||
f"{upstream_branch}/package.json"
|
||||
)
|
||||
upstream_pkg_text = fetch_text(upstream_pkg_url)
|
||||
if upstream_pkg_text:
|
||||
try:
|
||||
upstream_pkg = json.loads(upstream_pkg_text)
|
||||
upstream_peer = (upstream_pkg.get("peerDependencies") or {}).get("tree-sitter")
|
||||
if upstream_peer and satisfies_target(upstream_peer, TARGET_RUNTIME):
|
||||
status = "Unreleased (peer relaxed on main)"
|
||||
blockers[name] = f"`{name}`: peer dep relaxed on `{upstream_repo}` main but not published to npm"
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
compat_icon = "Yes" if compatible else "**No**"
|
||||
lines.append(
|
||||
f"| `{name}` | {npm_version} | {peer_display} | {compat_icon} | {abi_display} | {upstream_abi_display} | {status} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
lines.append(f"**{ready_count}/{total_count}** grammars ready for `tree-sitter@{TARGET_RUNTIME}`.")
|
||||
lines.append("")
|
||||
|
||||
# ── Vendored proto drift ─────────────────────────────────────────
|
||||
lines.append(md_h("Vendored tree-sitter-proto", 2))
|
||||
vendored_abi = extract_language_version(VENDOR_PROTO_DIR / "src" / "parser.c")
|
||||
|
||||
upstream_proto_url = (
|
||||
f"https://raw.githubusercontent.com/{UPSTREAM_PROTO_OWNER}/"
|
||||
f"{UPSTREAM_PROTO_REPO}/{UPSTREAM_PROTO_BRANCH}/src/parser.c"
|
||||
)
|
||||
upstream_proto_text = fetch_text(upstream_proto_url)
|
||||
upstream_proto_abi = extract_abi_from_text(upstream_proto_text) if upstream_proto_text else None
|
||||
|
||||
sha_url = (
|
||||
f"https://api.github.com/repos/{UPSTREAM_PROTO_OWNER}/"
|
||||
f"{UPSTREAM_PROTO_REPO}/commits/{UPSTREAM_PROTO_BRANCH}"
|
||||
)
|
||||
sha_text = fetch_text(sha_url)
|
||||
upstream_sha = "?"
|
||||
if sha_text:
|
||||
try:
|
||||
upstream_sha = json.loads(sha_text).get("sha", "?")[:12]
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
local_proto_path = VENDOR_PROTO_DIR / "src" / "parser.c"
|
||||
local_proto_text = local_proto_path.read_text(encoding="utf-8", errors="ignore") if local_proto_path.is_file() else ""
|
||||
in_sync = bool(
|
||||
upstream_proto_text
|
||||
and local_proto_text.replace("\r\n", "\n")
|
||||
== upstream_proto_text.replace("\r\n", "\n")
|
||||
)
|
||||
|
||||
lines.append(f"- Upstream: `{UPSTREAM_PROTO_OWNER}/{UPSTREAM_PROTO_REPO}@{UPSTREAM_PROTO_BRANCH}` (HEAD `{upstream_sha}`)")
|
||||
lines.append(f"- Upstream ABI: **{upstream_proto_abi}**")
|
||||
lines.append(f"- Vendored ABI: **{vendored_abi}**")
|
||||
lines.append(f"- In sync: {'yes' if in_sync else 'no — upstream has diverged'}")
|
||||
|
||||
if upstream_proto_abi and vendored_abi and upstream_proto_abi > vendored_abi:
|
||||
can_upgrade = upstream_proto_abi <= target_abi_range[1]
|
||||
lines.append(f"- Upstream ABI {upstream_proto_abi} {'is' if can_upgrade else 'is NOT'} within target runtime range ({target_abi_range[0]}..{target_abi_range[1]})")
|
||||
if can_upgrade:
|
||||
lines.append(f"- **Action:** after upgrading to tree-sitter@{TARGET_RUNTIME}, regenerate vendored parser.c from upstream `{upstream_sha}`")
|
||||
else:
|
||||
lines.append(f"- **Action:** wait for runtime upgrade beyond {TARGET_RUNTIME} that supports ABI {upstream_proto_abi}")
|
||||
blockers["vendored-proto-abi"] = f"vendored tree-sitter-proto: upstream ABI {upstream_proto_abi} outside target range"
|
||||
elif not in_sync:
|
||||
lines.append("- **Action:** review upstream changes; vendored copy may need updating")
|
||||
blockers["vendored-proto-sync"] = "vendored tree-sitter-proto: out of sync with upstream"
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────
|
||||
lines.append("")
|
||||
lines.append(md_h("Summary", 2))
|
||||
if blockers:
|
||||
lines.append(f"**{len(blockers)} blocker(s) remaining:**\n")
|
||||
for b in blockers.values():
|
||||
lines.append(f"- {b}")
|
||||
lines.append("")
|
||||
lines.append("Upgrade to `tree-sitter@0.25` is **blocked**.")
|
||||
else:
|
||||
lines.append("All grammars are compatible. Upgrade to `tree-sitter@0.25` is **ready**.")
|
||||
|
||||
print("\n".join(lines))
|
||||
return 1 if blockers else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
173
.github/scripts/check-workflow-concurrency.py
vendored
Normal file
173
.github/scripts/check-workflow-concurrency.py
vendored
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Enforce the GitHub Actions concurrency convention.
|
||||
|
||||
See CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention" for the rules.
|
||||
|
||||
Invoked from .github/workflows/ci-quality.yml. Runs locally too:
|
||||
python3 .github/scripts/check-workflow-concurrency.py .github/workflows
|
||||
|
||||
Rules:
|
||||
1. Every entry-point (non-reusable) workflow declares a top-level
|
||||
`concurrency:` block.
|
||||
2. Reusable workflows (on: workflow_call ONLY) do NOT declare one.
|
||||
3. The `concurrency.group` expression MUST reference either
|
||||
`${{ github.workflow }}` or a literal `CI-` prefix (the documented
|
||||
ci.yml reusable-workflow-safe exception). This is checked by substring
|
||||
containment rather than prefix match because ci.yml's group is a
|
||||
conditional expression that resolves to a `CI-…` literal at runtime.
|
||||
|
||||
We deliberately do not use a YAML library — keeps the script dependency-free
|
||||
on any vanilla runner. `on:` block parsing is line-based and handles both the
|
||||
flat (`on: workflow_call`) and mapping (`on:\n workflow_call:`) forms.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
REQUIRED_TOKENS = ("${{ github.workflow }}", "CI-")
|
||||
|
||||
|
||||
def is_reusable(lines: list[str]) -> bool:
|
||||
"""Return True iff the workflow's `on:` block names only `workflow_call`."""
|
||||
in_on = False
|
||||
on_indent: int | None = None
|
||||
keys: list[str] = []
|
||||
|
||||
for raw in lines:
|
||||
# Skip blank lines and comments
|
||||
stripped = raw.strip()
|
||||
if not stripped or stripped.startswith("#"):
|
||||
continue
|
||||
|
||||
indent = len(raw) - len(raw.lstrip(" "))
|
||||
|
||||
if not in_on:
|
||||
if raw.startswith("on:"):
|
||||
remainder = raw[len("on:"):].strip()
|
||||
if not remainder:
|
||||
# `on:` followed by indented mapping on next lines
|
||||
in_on = True
|
||||
on_indent = indent
|
||||
continue
|
||||
if remainder.startswith("[") and remainder.endswith("]"):
|
||||
# Flow-style list: on: [workflow_call]
|
||||
items = [
|
||||
item.strip() for item in remainder.strip("[]").split(",")
|
||||
]
|
||||
return items == ["workflow_call"]
|
||||
# Scalar form: on: workflow_call (or a single other event)
|
||||
return remainder == "workflow_call"
|
||||
continue
|
||||
|
||||
# Inside the `on:` block; stop when indentation returns to <= on_indent
|
||||
if on_indent is not None and indent <= on_indent:
|
||||
break
|
||||
|
||||
# Only consider keys at on_indent + indentation step (anything deeper
|
||||
# is nested config like `types:`)
|
||||
if ":" not in stripped:
|
||||
continue
|
||||
# Heuristic: first-level event keys are those with indent == on_indent + 2
|
||||
# (the canonical step for a 2-space YAML doc). We collect all first-level
|
||||
# keys by tracking the smallest indent seen inside the block.
|
||||
keys.append((indent, stripped.split(":", 1)[0].strip()))
|
||||
|
||||
if not keys:
|
||||
return False
|
||||
|
||||
# Take only the outermost-indented keys as the event list
|
||||
min_indent = min(i for i, _ in keys)
|
||||
events = [name for i, name in keys if i == min_indent]
|
||||
return events == ["workflow_call"]
|
||||
|
||||
|
||||
CONCURRENCY_RE = re.compile(r"^concurrency:\s*$")
|
||||
GROUP_RE = re.compile(r"^\s+group:\s*(.+?)\s*$")
|
||||
|
||||
|
||||
def extract_group_key(lines: list[str]) -> str | None:
|
||||
"""Return the `group:` value of the top-level `concurrency:` block, or None."""
|
||||
for idx, raw in enumerate(lines):
|
||||
if CONCURRENCY_RE.match(raw):
|
||||
# Scan forward until we leave the concurrency block (next top-level key
|
||||
# is at column 0 and ends with `:`).
|
||||
for follow in lines[idx + 1:]:
|
||||
if follow and not follow.startswith(" ") and follow.rstrip().endswith(":"):
|
||||
break
|
||||
m = GROUP_RE.match(follow)
|
||||
if m:
|
||||
return m.group(1).strip().strip("'").strip('"')
|
||||
break
|
||||
return None
|
||||
|
||||
|
||||
def has_top_level_concurrency(lines: list[str]) -> bool:
|
||||
return any(CONCURRENCY_RE.match(raw) for raw in lines)
|
||||
|
||||
|
||||
def check(workflows_dir: pathlib.Path) -> int:
|
||||
fail = 0
|
||||
files = sorted(
|
||||
list(workflows_dir.glob("*.yml")) + list(workflows_dir.glob("*.yaml"))
|
||||
)
|
||||
for path in files:
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
reusable = is_reusable(lines)
|
||||
has_conc = has_top_level_concurrency(lines)
|
||||
|
||||
if reusable:
|
||||
if has_conc:
|
||||
print(
|
||||
f"::error file={path}::Reusable workflow (on: workflow_call) "
|
||||
"must NOT declare its own concurrency block — it inherits "
|
||||
"from the caller. See CONTRIBUTING.md -> GitHub Actions — "
|
||||
"Concurrency Convention."
|
||||
)
|
||||
fail = 1
|
||||
continue
|
||||
|
||||
if not has_conc:
|
||||
print(
|
||||
f"::error file={path}::Missing top-level concurrency block. "
|
||||
"See CONTRIBUTING.md -> GitHub Actions — Concurrency Convention."
|
||||
)
|
||||
fail = 1
|
||||
continue
|
||||
|
||||
group = extract_group_key(lines)
|
||||
if group is None:
|
||||
print(
|
||||
f"::error file={path}::concurrency block is missing a "
|
||||
"`group:` key."
|
||||
)
|
||||
fail = 1
|
||||
continue
|
||||
|
||||
if not any(token in group for token in REQUIRED_TOKENS):
|
||||
print(
|
||||
f"::error file={path}::concurrency.group `{group}` must "
|
||||
f"reference one of {REQUIRED_TOKENS}. See CONTRIBUTING.md -> "
|
||||
"GitHub Actions — Concurrency Convention."
|
||||
)
|
||||
fail = 1
|
||||
|
||||
return fail
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) != 2:
|
||||
print(f"usage: {argv[0]} <workflows-dir>", file=sys.stderr)
|
||||
return 2
|
||||
workflows_dir = pathlib.Path(argv[1])
|
||||
if not workflows_dir.is_dir():
|
||||
print(f"not a directory: {workflows_dir}", file=sys.stderr)
|
||||
return 2
|
||||
return check(workflows_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
8
.github/workflows/ci-e2e.yml
vendored
8
.github/workflows/ci-e2e.yml
vendored
|
|
@ -11,8 +11,8 @@ jobs:
|
|||
outputs:
|
||||
web_changed: ${{ steps.filter.outputs.web }}
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v3
|
||||
id: filter
|
||||
with:
|
||||
filters: |
|
||||
|
|
@ -26,7 +26,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: ./.github/actions/setup-gitnexus-web
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ jobs:
|
|||
|
||||
- name: Upload test results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: e2e-results
|
||||
path: |
|
||||
|
|
|
|||
35
.github/workflows/ci-quality.yml
vendored
35
.github/workflows/ci-quality.yml
vendored
|
|
@ -8,8 +8,8 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
|
|
@ -21,8 +21,8 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: 20
|
||||
cache: npm
|
||||
|
|
@ -34,7 +34,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
- run: npx tsc --noEmit
|
||||
working-directory: gitnexus
|
||||
|
|
@ -43,7 +43,30 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: ./.github/actions/setup-gitnexus-web
|
||||
- run: npx tsc -b --noEmit
|
||||
working-directory: gitnexus-web
|
||||
|
||||
# Enforces the convention documented in CONTRIBUTING.md → "GitHub Actions —
|
||||
# Concurrency Convention":
|
||||
# 1. Every entry-point (non-reusable) workflow declares a top-level
|
||||
# `concurrency:` block.
|
||||
# 2. Reusable workflows (`on: workflow_call` only) do NOT declare one —
|
||||
# they inherit concurrency from the caller.
|
||||
# 3. The concurrency group key starts with `${{ github.workflow }}` or
|
||||
# the literal `CI-` prefix (the documented ci.yml exception for
|
||||
# reusable-workflow-safe grouping).
|
||||
# Reusability is detected by parsing each workflow's `on:` block, not an
|
||||
# allowlist, so new reusable workflows never produce false positives.
|
||||
workflow-convention:
|
||||
name: Workflow concurrency convention
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Validate workflow concurrency convention
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 .github/scripts/check-workflow-concurrency.py .github/workflows
|
||||
|
|
|
|||
18
.github/workflows/ci-report.yml
vendored
18
.github/workflows/ci-report.yml
vendored
|
|
@ -14,6 +14,16 @@ permissions:
|
|||
contents: read # needed for sparse checkout of vitest.config.ts
|
||||
pull-requests: write # needed to post sticky PR comment
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Serialize sticky-comment writes per PR so two rapid CI completions don't race.
|
||||
# Internal PRs surface in `pull_requests[0].number`. Fork PRs leave that array empty,
|
||||
# so we fall back to `<head-repo-full-name>/<head-branch>`, which is stable across
|
||||
# reruns and subsequent pushes for the same fork PR (unlike `workflow_run.id` which
|
||||
# is unique per run and therefore does not serialize anything).
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
pr-report:
|
||||
name: PR Report
|
||||
|
|
@ -26,7 +36,7 @@ jobs:
|
|||
steps:
|
||||
# ── Download artifacts from the CI run ────────────────────────
|
||||
- name: Download artifacts
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
|
|
@ -113,7 +123,7 @@ jobs:
|
|||
|
||||
- name: Checkout (for vitest config)
|
||||
if: steps.meta.outputs.skip != 'true'
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
sparse-checkout: gitnexus/vitest.config.ts
|
||||
sparse-checkout-cone-mode: false
|
||||
|
|
@ -122,7 +132,7 @@ jobs:
|
|||
- name: Fetch base branch coverage
|
||||
if: steps.meta.outputs.skip != 'true'
|
||||
id: base-coverage
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
|
|
@ -406,7 +416,7 @@ jobs:
|
|||
|
||||
- name: Comment on PR
|
||||
if: steps.meta.outputs.skip != 'true'
|
||||
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2
|
||||
uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v2
|
||||
with:
|
||||
header: ci-report
|
||||
number: ${{ steps.meta.outputs.pr_number }}
|
||||
|
|
|
|||
6
.github/workflows/ci-tests.yml
vendored
6
.github/workflows/ci-tests.yml
vendored
|
|
@ -9,7 +9,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
with:
|
||||
build: 'true'
|
||||
|
|
@ -43,7 +43,7 @@ jobs:
|
|||
|
||||
- name: Upload test reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: test-reports
|
||||
path: |
|
||||
|
|
@ -63,7 +63,7 @@ jobs:
|
|||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
with:
|
||||
build: 'true'
|
||||
|
|
|
|||
23
.github/workflows/ci.yml
vendored
23
.github/workflows/ci.yml
vendored
|
|
@ -9,9 +9,20 @@ on:
|
|||
paths-ignore: ['**.md', 'docs/**', 'LICENSE']
|
||||
workflow_call:
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Hardcoded `CI-` prefix (not `${{ github.workflow }}`) because this workflow is
|
||||
# invoked as a reusable workflow from publish.yml and release-candidate.yml. In
|
||||
# called-workflow context `github.workflow` evaluation is ambiguous across GitHub
|
||||
# Actions versions, and a prefix that could resolve to the caller's name would
|
||||
# share a concurrency group with the caller → deadlock. A literal prefix is
|
||||
# immune. Direct `push`/`pull_request` invocations use `CI-<ref>`; invocations
|
||||
# from a reusable-workflow caller fall into a per-run-unique group that never
|
||||
# serializes with the caller.
|
||||
# cancel-in-progress is event-aware: cancel superseded PR runs, queue every other
|
||||
# event (push to main, workflow_call from publish.yml, etc.).
|
||||
concurrency:
|
||||
group: ci-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
group: ${{ (github.event_name == 'pull_request' || github.event_name == 'push') && format('CI-{0}', github.ref) || format('CI-nested-{0}', github.run_id) }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
# ── Reusable workflow orchestration ─────────────────────────────────
|
||||
# Each concern lives in its own workflow file for maintainability:
|
||||
|
|
@ -74,7 +85,7 @@ jobs:
|
|||
cp pr-meta/e2e_result pr-meta/e2e-result
|
||||
|
||||
- name: Upload PR metadata
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: pr-meta
|
||||
path: pr-meta/
|
||||
|
|
@ -96,9 +107,9 @@ jobs:
|
|||
TESTS: ${{ needs.tests.result }}
|
||||
E2E: ${{ needs.e2e.result }}
|
||||
run: |
|
||||
echo "Quality: $QUALITY"
|
||||
echo "Tests: $TESTS"
|
||||
echo "E2E: $E2E"
|
||||
echo "Quality: $QUALITY"
|
||||
echo "Tests: $TESTS"
|
||||
echo "E2E: $E2E"
|
||||
if [[ "$QUALITY" != "success" ]] ||
|
||||
[[ "$TESTS" != "success" ]]; then
|
||||
echo "::error::Quality or test jobs failed"
|
||||
|
|
|
|||
7
.github/workflows/claude-code-review.yml
vendored
7
.github/workflows/claude-code-review.yml
vendored
|
|
@ -16,9 +16,10 @@ on:
|
|||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Serialize per-PR to avoid racing review comments.
|
||||
concurrency:
|
||||
group: claude-review-${{ github.event.issue.number || github.event.pull_request.number }}
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
|
@ -56,7 +57,7 @@ jobs:
|
|||
# For issue_comment triggers, resolve the PR number, head SHA, and fork repo
|
||||
- name: Resolve PR context
|
||||
id: pr
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
|
||||
with:
|
||||
script: |
|
||||
let pr;
|
||||
|
|
@ -76,7 +77,7 @@ jobs:
|
|||
core.setOutput('branch', pr.head.ref);
|
||||
|
||||
- name: Checkout PR head
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ${{ steps.pr.outputs.repo }}
|
||||
ref: ${{ steps.pr.outputs.sha }}
|
||||
|
|
|
|||
7
.github/workflows/claude.yml
vendored
7
.github/workflows/claude.yml
vendored
|
|
@ -10,9 +10,10 @@ on:
|
|||
pull_request_review:
|
||||
types: [submitted]
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Serialize per-PR/issue to avoid racing comments.
|
||||
concurrency:
|
||||
group: claude-code-${{ github.event.issue.number || github.event.pull_request.number || github.event.issue.id }}
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number || github.event.issue.id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
|
|
@ -58,7 +59,7 @@ jobs:
|
|||
# For PR-related triggers, resolve the fork repo so we can checkout correctly.
|
||||
- name: Resolve PR context
|
||||
id: pr
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
|
||||
with:
|
||||
script: |
|
||||
// Determine if this event is PR-related
|
||||
|
|
@ -90,7 +91,7 @@ jobs:
|
|||
core.setOutput('branch', pr.head.ref);
|
||||
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.repo || github.repository }}
|
||||
ref: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.sha || '' }}
|
||||
|
|
|
|||
5
.github/workflows/pr-description-check.yml
vendored
5
.github/workflows/pr-description-check.yml
vendored
|
|
@ -8,8 +8,9 @@ on:
|
|||
permissions:
|
||||
pull-requests: write
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
concurrency:
|
||||
group: pr-desc-${{ github.event.pull_request.number }}
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
|
@ -18,7 +19,7 @@ jobs:
|
|||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Check PR description quality
|
||||
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const MIN_BODY_LENGTH = 50;
|
||||
|
|
|
|||
113
.github/workflows/pr-labeler.yml
vendored
Normal file
113
.github/workflows/pr-labeler.yml
vendored
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
name: PR Conventional Labeler
|
||||
|
||||
# Two workflows in one file with different triggers, matched to the minimum
|
||||
# privilege each needs:
|
||||
#
|
||||
# validate-title (on: pull_request)
|
||||
# Fork-safe. Runs with the PR-head's read-only GITHUB_TOKEN. Uses
|
||||
# `amannn/action-semantic-pull-request` to fail the check when the PR
|
||||
# title doesn't follow the conventional-commit format. Because the
|
||||
# action only reads the event payload, no fork-controlled code runs.
|
||||
#
|
||||
# autolabel (on: pull_request_target)
|
||||
# Needs `pull-requests: write` to apply labels, so must be
|
||||
# pull_request_target. Uses `release-drafter/release-drafter` with
|
||||
# `dry-run: true` to only run the autolabeler against the
|
||||
# `.github/release-drafter.yml` config from the BASE ref (release-
|
||||
# drafter reads the config from the repository's default branch, NOT
|
||||
# the PR head — verify with `gh api repos/release-drafter/release-drafter/contents/...`
|
||||
# or a fork-test PR before merging if the repo is high-value).
|
||||
# `sync-labels: true` in the config removes managed autolabels that no
|
||||
# longer match (e.g. when `!` or `BREAKING CHANGE:` is dropped).
|
||||
#
|
||||
# Title format: <type>[(scope)][!]: <subject>
|
||||
# Allowed types: feat, fix, perf, refactor, docs, test, ci, build, chore, revert, deps
|
||||
# Trailing `!` on the type marks a breaking change.
|
||||
# See CONTRIBUTING.md → "Pull request titles".
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
# Title-only changes fire `edited`. `opened` and `reopened` cover creation.
|
||||
# `synchronize` (push to the PR branch) is intentionally excluded — titles
|
||||
# don't change on push, so it only wastes CI minutes and broadens the
|
||||
# privileged-token exposure window on the autolabel job.
|
||||
types: [opened, edited, reopened]
|
||||
pull_request_target:
|
||||
types: [opened, edited, reopened]
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Include `github.event_name` so `pull_request` (validate-title) and
|
||||
# `pull_request_target` (autolabel) runs for the same PR do NOT share a slot
|
||||
# and therefore cannot cancel each other — a cancelled required-check would
|
||||
# permanently block merge until the next title edit.
|
||||
# Within each trigger the latest title edit still supersedes the prior run.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate-title:
|
||||
# Fork-safe job — only runs on `pull_request` (not `pull_request_target`).
|
||||
# Token is read-only; writes a commit status that branch protection can
|
||||
# require before merge.
|
||||
name: Validate PR title
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
pull-requests: read
|
||||
steps:
|
||||
# Pinned to v6.1.1. Verify SHA via:
|
||||
# gh api repos/amannn/action-semantic-pull-request/git/refs/tags/v6.1.1
|
||||
- uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
types: |
|
||||
feat
|
||||
fix
|
||||
perf
|
||||
refactor
|
||||
docs
|
||||
test
|
||||
ci
|
||||
build
|
||||
chore
|
||||
revert
|
||||
deps
|
||||
requireScope: false
|
||||
# Subject must be non-empty. We DO allow capitalized proper nouns
|
||||
# (MCP, GitHub, API, etc.) — the old `^(?![A-Z]).+$` pattern
|
||||
# rejected legitimate titles like `fix: MCP tool schema`.
|
||||
subjectPattern: ^\S.{2,}$
|
||||
subjectPatternError: |
|
||||
The subject "{subject}" in PR title "{title}" is invalid.
|
||||
Subjects must be at least 3 characters and must not start with whitespace.
|
||||
wip: false
|
||||
|
||||
autolabel:
|
||||
# Privileged job — runs only on `pull_request_target` so it can write labels.
|
||||
# Never checks out fork code, never executes fork-controlled input; only
|
||||
# reads the PR metadata (title, body, labels) and calls the GitHub API.
|
||||
name: Apply conventional label
|
||||
if: github.event_name == 'pull_request_target'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
# `contents: read` is required — release-drafter's context.config() reads
|
||||
# `.github/release-drafter.yml` from the repo's default branch via the
|
||||
# repo-contents API. Without it the job silently 403s and no labels are
|
||||
# applied. Job-level permissions nullify all unlisted scopes, so an
|
||||
# explicit grant is necessary here.
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
# Pinned to v7.2.0. Verify SHA via:
|
||||
# gh api repos/release-drafter/release-drafter/git/refs/tags/v7.2.0
|
||||
# v7 removed `disable-releaser`; use `dry-run: true` to only autolabel.
|
||||
- uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0
|
||||
with:
|
||||
config-name: release-drafter.yml
|
||||
dry-run: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
17
.github/workflows/publish.yml
vendored
17
.github/workflows/publish.yml
vendored
|
|
@ -7,13 +7,22 @@ on:
|
|||
|
||||
# No workflow-level permissions — scoped per job below.
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Tag refs are unique per release, so distinct tags run in parallel. Re-pushes of the
|
||||
# same tag serialize. cancel-in-progress: false — never cancel a publish mid-flight.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
ci:
|
||||
uses: ./.github/workflows/ci.yml
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: write
|
||||
# No pull-requests:write — `ci.yml`'s save-pr-meta job is gated on
|
||||
# `github.event_name == 'pull_request'`, so it never runs during a
|
||||
# tag-triggered publish. Least-privilege for release-critical paths.
|
||||
|
||||
publish:
|
||||
needs: ci
|
||||
|
|
@ -23,8 +32,8 @@ jobs:
|
|||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: 20
|
||||
registry-url: https://registry.npmjs.org
|
||||
|
|
@ -82,7 +91,7 @@ jobs:
|
|||
fi
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2
|
||||
with:
|
||||
body_path: ${{ steps.changelog.outputs.fallback == 'false' && '/tmp/release-notes.md' || '' }}
|
||||
generate_release_notes: ${{ steps.changelog.outputs.fallback == 'true' }}
|
||||
|
|
|
|||
366
.github/workflows/release-candidate.yml
vendored
Normal file
366
.github/workflows/release-candidate.yml
vendored
Normal file
|
|
@ -0,0 +1,366 @@
|
|||
name: Release Candidate
|
||||
|
||||
on:
|
||||
# Publish a release-candidate build whenever a merge/commit lands on main.
|
||||
# Docs/README-only changes are filtered out so prose updates don't
|
||||
# cut a release.
|
||||
push:
|
||||
branches: [main]
|
||||
paths-ignore:
|
||||
- '**.md'
|
||||
- 'docs/**'
|
||||
- 'LICENSE'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
bump:
|
||||
description: >-
|
||||
Cycle policy. 'auto' (default) continues the active rc cycle on
|
||||
this branch if there is one, otherwise bumps patch from latest.
|
||||
Choose 'patch' / 'minor' / 'major' to explicitly start or reset
|
||||
an rc cycle.
|
||||
required: false
|
||||
default: 'auto'
|
||||
type: choice
|
||||
options:
|
||||
- auto
|
||||
- patch
|
||||
- minor
|
||||
- major
|
||||
force:
|
||||
description: 'Publish even when HEAD already has an rc marker'
|
||||
required: false
|
||||
default: 'false'
|
||||
type: choice
|
||||
options:
|
||||
- 'false'
|
||||
- 'true'
|
||||
|
||||
# No workflow-level permissions — scoped per job below.
|
||||
permissions: {}
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Serialize all runs on the same ref (push + workflow_dispatch) to prevent two publishes
|
||||
# racing on the rc counter. cancel-in-progress: false — the earlier merge publishes first.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
# ── Skip when HEAD already has an rc marker (retry / duplicate dispatch) ──
|
||||
# The marker is a lightweight tag `rc/<HEAD_SHA>` pushed *before* `npm
|
||||
# publish`, so a failed publish leaves the marker in place and the guard
|
||||
# refuses to re-publish. Recovery path after a partial failure:
|
||||
# git push --delete origin rc/<HEAD_SHA> v<RC_VERSION>
|
||||
# then redispatch with force=true.
|
||||
guard:
|
||||
name: Check if release candidate should run
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
should_run: ${{ steps.decide.outputs.should_run }}
|
||||
head_sha: ${{ steps.decide.outputs.head_sha }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Decide
|
||||
id: decide
|
||||
shell: bash
|
||||
env:
|
||||
FORCE: ${{ inputs.force }}
|
||||
BUMP_INPUT: ${{ inputs.bump }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
HEAD_SHA=$(git rev-parse HEAD)
|
||||
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
|
||||
|
||||
if [ "$FORCE" = "true" ]; then
|
||||
echo "Force flag set — running regardless of marker tag."
|
||||
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# An explicit cycle reset on dispatch (bump != auto) also bypasses
|
||||
# the dedup guard — the maintainer is deliberately asking for a
|
||||
# new rc from the same commit.
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ] \
|
||||
&& [ -n "${BUMP_INPUT:-}" ] \
|
||||
&& [ "${BUMP_INPUT:-auto}" != "auto" ]; then
|
||||
echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup."
|
||||
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Dedup: is there already an rc/<HEAD_SHA> marker pointing at HEAD?
|
||||
MARKER="rc/${HEAD_SHA}"
|
||||
if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then
|
||||
echo "HEAD already has marker $MARKER — skipping."
|
||||
echo "should_run=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "No marker on HEAD — proceeding."
|
||||
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# ── Reuse the stable CI workflow ─────────────────────────────────────
|
||||
ci:
|
||||
needs: guard
|
||||
if: needs.guard.outputs.should_run == 'true'
|
||||
uses: ./.github/workflows/ci.yml
|
||||
permissions:
|
||||
contents: read
|
||||
secrets: inherit
|
||||
|
||||
# ── Publish the rc build to npm + create GitHub prerelease ───────────
|
||||
publish:
|
||||
name: Publish release candidate to npm
|
||||
needs: [guard, ci]
|
||||
if: needs.guard.outputs.should_run == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: write # push rc tag + marker
|
||||
id-token: write # npm provenance
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: 20
|
||||
registry-url: https://registry.npmjs.org
|
||||
cache: npm
|
||||
cache-dependency-path: gitnexus/package-lock.json
|
||||
|
||||
- name: Build gitnexus-shared
|
||||
run: npm install && npm run build
|
||||
working-directory: gitnexus-shared
|
||||
|
||||
- name: Install gitnexus dependencies
|
||||
run: npm ci
|
||||
working-directory: gitnexus
|
||||
|
||||
- name: Resolve rc version
|
||||
id: version
|
||||
shell: bash
|
||||
working-directory: gitnexus
|
||||
env:
|
||||
BUMP_INPUT: ${{ inputs.bump }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
PKG_NAME: gitnexus
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# 1. Current published `latest` — the floor for any new rc base.
|
||||
# Only E404 ("never published") falls back to package.json; any
|
||||
# other error (network, auth, malformed response) fails fast.
|
||||
NPM_STDERR_LATEST="$(mktemp)"
|
||||
if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then
|
||||
:
|
||||
else
|
||||
if grep -q 'E404' "$NPM_STDERR_LATEST"; then
|
||||
CURRENT_LATEST="$(node -p "require('./package.json').version")"
|
||||
echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST"
|
||||
else
|
||||
echo "::error::npm registry unreachable for 'view version':" >&2
|
||||
cat "$NPM_STDERR_LATEST" >&2
|
||||
rm -f "$NPM_STDERR_LATEST"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
rm -f "$NPM_STDERR_LATEST"
|
||||
CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}"
|
||||
|
||||
# 2. Full version list — needed for the counter and for active-cycle
|
||||
# inference. Same E404-only fallback.
|
||||
NPM_STDERR_VERSIONS="$(mktemp)"
|
||||
if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then
|
||||
:
|
||||
else
|
||||
if grep -q 'E404' "$NPM_STDERR_VERSIONS"; then
|
||||
VERSIONS_JSON='[]'
|
||||
echo "No published versions for $PKG_NAME yet (E404)."
|
||||
else
|
||||
echo "::error::npm registry unreachable for 'view versions':" >&2
|
||||
cat "$NPM_STDERR_VERSIONS" >&2
|
||||
rm -f "$NPM_STDERR_VERSIONS"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
rm -f "$NPM_STDERR_VERSIONS"
|
||||
|
||||
# 3. Base selection.
|
||||
# - workflow_dispatch + bump ∈ {patch,minor,major} → explicit cycle
|
||||
# reset from latest.
|
||||
# - Everything else (push, or dispatch with bump=auto) → continue
|
||||
# the highest active rc base > latest if one exists; else
|
||||
# default to patch from latest.
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ] \
|
||||
&& [ -n "${BUMP_INPUT:-}" ] \
|
||||
&& [ "${BUMP_INPUT:-auto}" != "auto" ]; then
|
||||
BASE="$(npx --yes -p semver@7 semver -i "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")"
|
||||
echo "Explicit bump=$BUMP_INPUT → BASE=$BASE"
|
||||
else
|
||||
cat > /tmp/active_base.mjs <<'NODESCRIPT'
|
||||
const latest = process.env.LATEST;
|
||||
let v;
|
||||
try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; }
|
||||
if (!Array.isArray(v)) v = [v];
|
||||
const parse = s => s.split(".").map(n => parseInt(n, 10));
|
||||
const gt = (a, b) => {
|
||||
const [A, B] = [parse(a), parse(b)];
|
||||
for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i];
|
||||
return false;
|
||||
};
|
||||
const bases = new Set();
|
||||
for (const s of v) {
|
||||
const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s);
|
||||
if (m && gt(m[1], latest)) bases.add(m[1]);
|
||||
}
|
||||
if (!bases.size) { process.stdout.write(""); process.exit(0); }
|
||||
const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1);
|
||||
process.stdout.write(sorted[sorted.length - 1]);
|
||||
NODESCRIPT
|
||||
ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)"
|
||||
if [ -n "$ACTIVE_BASE" ]; then
|
||||
BASE="$ACTIVE_BASE"
|
||||
echo "Continuing active rc cycle → BASE=$BASE"
|
||||
else
|
||||
BASE="$(npx --yes -p semver@7 semver -i patch "$CURRENT_LATEST_CLEAN")"
|
||||
echo "No active rc cycle → patch bump from latest → BASE=$BASE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1.
|
||||
cat > /tmp/next_rc.mjs <<'NODESCRIPT'
|
||||
const base = process.env.BASE;
|
||||
const prefix = base + "-rc.";
|
||||
let v;
|
||||
try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; }
|
||||
if (!Array.isArray(v)) v = [v];
|
||||
const ns = v
|
||||
.filter(s => typeof s === "string" && s.startsWith(prefix))
|
||||
.map(s => parseInt(s.slice(prefix.length), 10))
|
||||
.filter(n => Number.isInteger(n) && n >= 0);
|
||||
process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1));
|
||||
NODESCRIPT
|
||||
NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)"
|
||||
RC_VERSION="${BASE}-rc.${NEXT_N}"
|
||||
echo "Computed rc: $RC_VERSION"
|
||||
|
||||
# 5. Defensive: if the exact version already exists on the registry
|
||||
# (e.g., race with another run), abort before re-publishing.
|
||||
# Same E404-only pattern used above — a transient network
|
||||
# failure must fail loudly, not pretend the version is missing.
|
||||
NPM_STDERR_EXISTS="$(mktemp)"
|
||||
if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then
|
||||
rm -f "$NPM_STDERR_EXISTS"
|
||||
echo "::error::Version $RC_VERSION already exists on npm — aborting."
|
||||
exit 1
|
||||
else
|
||||
if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then
|
||||
rm -f "$NPM_STDERR_EXISTS"
|
||||
# Version doesn't exist — safe to proceed.
|
||||
else
|
||||
echo "::error::npm registry unreachable for existence check:" >&2
|
||||
cat "$NPM_STDERR_EXISTS" >&2
|
||||
rm -f "$NPM_STDERR_EXISTS"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "base=$BASE" >> "$GITHUB_OUTPUT"
|
||||
echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT"
|
||||
echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply rc version in-CI
|
||||
shell: bash
|
||||
working-directory: gitnexus
|
||||
run: |
|
||||
set -euo pipefail
|
||||
npm version "${{ steps.version.outputs.rc_version }}" \
|
||||
--no-git-tag-version --allow-same-version
|
||||
|
||||
- name: Build gitnexus
|
||||
run: npm run build
|
||||
working-directory: gitnexus
|
||||
|
||||
- name: Dry-run publish
|
||||
run: npm publish --dry-run --tag rc
|
||||
working-directory: gitnexus
|
||||
|
||||
# ── Acquire the "rc lock" BEFORE publishing (fixes idempotency) ─────
|
||||
# We create two tags and push them atomically:
|
||||
# v<RC_VERSION> → annotated tag on a detached release commit
|
||||
# whose tree contains the rewritten package.json
|
||||
# (so the tag's source matches the npm tarball)
|
||||
# rc/<HEAD_SHA> → lightweight tag on HEAD; the guard's dedup key
|
||||
# If this push fails, nothing is published — safe.
|
||||
# If this push succeeds but npm publish fails, the marker stays on
|
||||
# the remote and blocks retries until an operator manually cleans up.
|
||||
- name: Create and push rc tags
|
||||
id: reltag
|
||||
shell: bash
|
||||
working-directory: gitnexus
|
||||
env:
|
||||
RC_VERSION: ${{ steps.version.outputs.rc_version }}
|
||||
HEAD_SHA: ${{ needs.guard.outputs.head_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
VTAG="v${RC_VERSION}"
|
||||
MARKER="rc/${HEAD_SHA}"
|
||||
git config user.name 'github-actions[bot]'
|
||||
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
|
||||
|
||||
# Detached release commit with the version bump — keeps `main`
|
||||
# pristine but gives the v-tag a tree that matches the published
|
||||
# package contents exactly (fixes release-integrity gap).
|
||||
git add package.json package-lock.json 2>/dev/null || git add package.json
|
||||
git commit -m "release: ${VTAG}" --allow-empty
|
||||
RELEASE_SHA="$(git rev-parse HEAD)"
|
||||
echo "Detached release commit: $RELEASE_SHA"
|
||||
|
||||
# Annotated release tag on the release commit.
|
||||
git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG"
|
||||
# Lightweight marker on the user-visible HEAD for the guard.
|
||||
git tag "$MARKER" "$HEAD_SHA"
|
||||
|
||||
# Atomic push of both refs. If either would clobber an existing
|
||||
# remote ref, the push fails and we stop before npm publish.
|
||||
git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER"
|
||||
|
||||
echo "vtag=$VTAG" >> "$GITHUB_OUTPUT"
|
||||
echo "marker=$MARKER" >> "$GITHUB_OUTPUT"
|
||||
echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Publish to npm (rc dist-tag)
|
||||
run: npm publish --provenance --access public --tag rc
|
||||
working-directory: gitnexus
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
|
||||
- name: Create GitHub prerelease
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2
|
||||
with:
|
||||
tag_name: ${{ steps.reltag.outputs.vtag }}
|
||||
name: Release Candidate ${{ steps.reltag.outputs.vtag }}
|
||||
prerelease: true
|
||||
make_latest: 'false'
|
||||
generate_release_notes: true
|
||||
body: |
|
||||
Automated release candidate build from `main`.
|
||||
|
||||
**npm:** `npm install gitnexus@rc`
|
||||
**Version:** `${{ steps.version.outputs.rc_version }}`
|
||||
**Target base:** `${{ steps.version.outputs.base }}` (rc #${{ steps.version.outputs.rc_n }})
|
||||
**Source commit (main):** ${{ needs.guard.outputs.head_sha }}
|
||||
**Release commit (versioned tree):** ${{ steps.reltag.outputs.release_sha }}
|
||||
|
||||
Release candidates are pre-stable builds intended for early testing.
|
||||
Stable releases remain on the `latest` dist-tag.
|
||||
185
.github/workflows/tree-sitter-upgrade-readiness.yml
vendored
Normal file
185
.github/workflows/tree-sitter-upgrade-readiness.yml
vendored
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
name: Tree-sitter Upgrade Readiness
|
||||
|
||||
# Monitors readiness for upgrading tree-sitter to 0.25.x. Tracks:
|
||||
# 1. Peer-dep compatibility — can each grammar install cleanly with
|
||||
# tree-sitter@0.25.0 without --legacy-peer-deps?
|
||||
# 2. Vendored proto drift — has coder3101/tree-sitter-proto moved
|
||||
# ahead of our vendored snapshot?
|
||||
# See .github/scripts/check-tree-sitter-upgrade-readiness.py for the logic.
|
||||
#
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Daily at 09:00 UTC. Matches Dependabot's daily cadence so drift
|
||||
# and dep PRs surface together.
|
||||
- cron: '0 9 * * *'
|
||||
workflow_dispatch:
|
||||
pull_request:
|
||||
paths:
|
||||
- '.github/scripts/check-tree-sitter-upgrade-readiness.py'
|
||||
- '.github/workflows/tree-sitter-upgrade-readiness.yml'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
readiness:
|
||||
name: Check upgrade readiness
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
# Needed to open/update the tracking issue on scheduled runs.
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- uses: ./.github/actions/setup-gitnexus
|
||||
with:
|
||||
build: 'false'
|
||||
|
||||
- name: Run upgrade readiness check
|
||||
id: readiness
|
||||
shell: bash
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set +e
|
||||
python3 .github/scripts/check-tree-sitter-upgrade-readiness.py > drift-report.md
|
||||
code=$?
|
||||
set -e
|
||||
echo "exit_code=$code" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo 'report<<DRIFT_EOF'
|
||||
cat drift-report.md
|
||||
echo 'DRIFT_EOF'
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "=== Report ==="
|
||||
cat drift-report.md
|
||||
|
||||
# On PR runs, the script validates that it runs correctly. Blockers
|
||||
# are informational — the scheduled run opens a tracking issue.
|
||||
- name: Annotate PR with readiness status
|
||||
if: github.event_name == 'pull_request' && steps.readiness.outputs.exit_code != '0'
|
||||
run: |
|
||||
echo "::warning::Tree-sitter 0.25 upgrade has blockers. See job output for the full readiness report."
|
||||
|
||||
- name: Upsert tracking issue on scheduled runs
|
||||
if: >
|
||||
github.event_name == 'schedule' &&
|
||||
steps.readiness.outputs.exit_code != '0'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
REPORT: ${{ steps.readiness.outputs.report }}
|
||||
with:
|
||||
script: |
|
||||
const title = 'Tree-sitter 0.25 upgrade readiness';
|
||||
const report = process.env.REPORT;
|
||||
const body = report + '\n\n' +
|
||||
'<sub>Generated daily by `.github/workflows/tree-sitter-upgrade-readiness.yml`. ' +
|
||||
'Closes automatically when all blockers are resolved.</sub>';
|
||||
const { data: open } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
state: 'open',
|
||||
labels: 'tree-sitter-drift',
|
||||
per_page: 10,
|
||||
});
|
||||
const existing = open.find(i => i.title === title);
|
||||
if (existing) {
|
||||
// Extract ready/total count for the changelog comment.
|
||||
const readyMatch = report.match(/\*\*(\d+)\/(\d+)\*\* grammars ready/);
|
||||
const blockerMatch = report.match(/\*\*(\d+) blocker/);
|
||||
const ready = readyMatch ? readyMatch[1] : '?';
|
||||
const total = readyMatch ? readyMatch[2] : '?';
|
||||
const blockers = blockerMatch ? blockerMatch[1] : '?';
|
||||
|
||||
// Find grammars whose status changed by diffing the old and
|
||||
// new table rows. Each row looks like:
|
||||
// | `tree-sitter-foo` | ... | Ready |
|
||||
// | `tree-sitter-foo` | ... | Blocking |
|
||||
const parseRows = (md) => {
|
||||
const map = {};
|
||||
for (const m of md.matchAll(/\| `(tree-sitter-[^`]+)` \|.*?\| (\S+(?:\s\S+)*?) \|$/gm)) {
|
||||
map[m[1]] = m[2].trim();
|
||||
}
|
||||
return map;
|
||||
};
|
||||
const oldRows = parseRows(existing.body || '');
|
||||
const newRows = parseRows(report);
|
||||
const changes = [];
|
||||
for (const [name, newStatus] of Object.entries(newRows)) {
|
||||
const oldStatus = oldRows[name];
|
||||
if (oldStatus && oldStatus !== newStatus) {
|
||||
changes.push(`\`${name}\`: ${oldStatus} → ${newStatus}`);
|
||||
}
|
||||
}
|
||||
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
let comment = `**${today}:** ${ready}/${total} ready. ${blockers} blocker(s) remaining.`;
|
||||
if (changes.length > 0) {
|
||||
comment += '\n\nChanges:\n' + changes.map(c => `- ${c}`).join('\n');
|
||||
} else {
|
||||
comment += ' No changes from previous run.';
|
||||
}
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: existing.number,
|
||||
body: comment,
|
||||
});
|
||||
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: existing.number,
|
||||
body,
|
||||
});
|
||||
core.info(`Updated existing issue #${existing.number}`);
|
||||
} else {
|
||||
const { data: created } = await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title,
|
||||
body,
|
||||
labels: ['tree-sitter-drift', 'dependencies'],
|
||||
});
|
||||
core.info(`Opened issue #${created.number}`);
|
||||
}
|
||||
|
||||
- name: Close tracking issue on clean scheduled runs
|
||||
if: >
|
||||
github.event_name == 'schedule' &&
|
||||
steps.readiness.outputs.exit_code == '0'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const title = 'Tree-sitter 0.25 upgrade readiness';
|
||||
const { data: open } = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
state: 'open',
|
||||
labels: 'tree-sitter-drift',
|
||||
per_page: 10,
|
||||
});
|
||||
const existing = open.find(i => i.title === title);
|
||||
if (existing) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: existing.number,
|
||||
body: 'All grammars are now compatible with tree-sitter@0.25. Upgrade is ready! Closing automatically.',
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: existing.number,
|
||||
state: 'closed',
|
||||
});
|
||||
core.info(`Closed issue #${existing.number}`);
|
||||
}
|
||||
6
.github/workflows/triage-sweep.yml
vendored
6
.github/workflows/triage-sweep.yml
vendored
|
|
@ -47,8 +47,10 @@ permissions:
|
|||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
|
||||
# Single global slot — newest manual dispatch supersedes any in-flight run.
|
||||
concurrency:
|
||||
group: triage-sweep
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
|
|
@ -74,7 +76,7 @@ jobs:
|
|||
run: pip install -r .github/scripts/triage/requirements.txt
|
||||
|
||||
- name: Cache FastEmbed model weights
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
|
||||
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
|
||||
with:
|
||||
path: ${{ github.workspace }}/.fastembed_cache
|
||||
key: fastembed-bge-small-en-v1.5
|
||||
|
|
|
|||
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -81,6 +81,11 @@ GitNexus.sln
|
|||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Vendored tree-sitter grammar build artifacts (created at install time,
|
||||
# never committed). See docs/plans/2026-04-15-002-fix-tree-sitter-proto-vendor-deps-plan.md
|
||||
gitnexus/vendor/**/build/
|
||||
gitnexus/vendor/**/node_modules/
|
||||
|
||||
/github/scripts/triage/__pycache__/
|
||||
|
||||
.claude-flow/
|
||||
|
|
@ -95,4 +100,5 @@ GitNexus.sln
|
|||
|
||||
.swarm/
|
||||
|
||||
local_docs/
|
||||
local_docs/
|
||||
|
||||
|
|
|
|||
205
AGENTS.md
205
AGENTS.md
|
|
@ -1,116 +1,122 @@
|
|||
<!-- version: 1.2.0 -->
|
||||
<!--
|
||||
Metadata: version, last reviewed, scope, model policy, reference docs, changelog.
|
||||
Last updated: 2026-03-22
|
||||
-->
|
||||
<!-- version: 1.4.0 -->
|
||||
<!-- Last updated: 2026-04-16 -->
|
||||
|
||||
Last reviewed: 2026-03-24
|
||||
Last reviewed: 2026-04-16
|
||||
|
||||
**Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub)
|
||||
|
||||
This file uses a standard agent header (version, scope, model policy, reference docs, changelog), adapted for this **TypeScript/JavaScript monorepo**.
|
||||
|
||||
## Scope
|
||||
|
||||
| | |
|
||||
|--|--|
|
||||
| **Reads** | Repository tree as needed for the task: `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/` when present, and docs. |
|
||||
| **Writes** | Only paths required for the requested change; keep diffs minimal. Update lockfiles when dependencies change. |
|
||||
| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/` when applicable; shell utilities for documented CI/dev workflows. |
|
||||
| **Off-limits** | User secrets (e.g. real `.env`), production deployment credentials, unrelated repositories, destructive git history operations without explicit human confirmation. |
|
||||
| Boundary | Rule |
|
||||
|----------|------|
|
||||
| **Reads** | `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/`, docs. |
|
||||
| **Writes** | Only paths required for the change; keep diffs minimal. Update lockfiles when deps change. |
|
||||
| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/`; documented CI/dev workflows. |
|
||||
| **Off-limits** | Real `.env` / secrets, production credentials, unrelated repos, destructive git ops without confirmation. |
|
||||
|
||||
## Model Configuration
|
||||
|
||||
- **Primary:** Pin in **Cursor** (Settings → model). Use a **named** model (e.g. GPT-5.2, Claude Sonnet 4.x). Avoid relying on **Auto** when reproducibility or audit trail matters.
|
||||
- **Fallback:** As configured in Cursor or your organization (do not encode `latest` or wildcards in automation configs).
|
||||
- **Notes:** The open-source GitNexus CLI indexer does not call an LLM. Optional Nexus AI in the web UI uses end-user provider keys and models.
|
||||
- **Primary:** Use a named model (e.g. Claude Sonnet 4.x). Avoid `Auto` or unversioned `latest` when reproducibility matters.
|
||||
- **Notes:** The GitNexus CLI indexer does not call an LLM.
|
||||
|
||||
## Execution Sequence (complex tasks)
|
||||
|
||||
Long sessions dilute instructions. For **multi-step** work, state up front:
|
||||
|
||||
For multi-step work, state up front:
|
||||
1. Which rules in this file and **[GUARDRAILS.md](GUARDRAILS.md)** apply (and any relevant Signs).
|
||||
2. Current **Scope** boundaries (Reads / Writes / Off-limits).
|
||||
3. Which **validation commands** you will run (e.g. `cd gitnexus && npm test`, `npx tsc --noEmit`).
|
||||
2. Current **Scope** boundaries.
|
||||
3. Which **validation commands** you will run (`cd gitnexus && npm test`, `npx tsc --noEmit`).
|
||||
|
||||
On very long threads, the human may add *“Remember: apply all AGENTS.md rules”* to re-weight rule tokens against context dilution.
|
||||
On long threads, *"Remember: apply all AGENTS.md rules"* re-weights these instructions against context dilution.
|
||||
|
||||
## Claude Code hooks
|
||||
|
||||
Hooks enforce gates that prompts cannot. In **Claude Code**, **PreToolUse** hooks can block tools such as `git_commit` until checks pass. Adapt to this repo: e.g. `cd gitnexus && npm test` before commit.
|
||||
**PreToolUse** hooks can block tools (e.g. `git_commit`) until checks pass. Adapt to this repo: `cd gitnexus && npm test` before commit.
|
||||
|
||||
## Context budget (Cursor / standards)
|
||||
## Context budget
|
||||
|
||||
Generic “core standards” playbooks are often long and stack-specific. For this monorepo, commands and gotchas live under **Cursor Cloud specific instructions** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split domain rules into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules live in **`.cursor/index.mdc`** (YAML frontmatter with `alwaysApply: true`). **Claude Code:** optionally load a **`STANDARDS.md`** only when needed (e.g. *“When writing new code, read STANDARDS.md”*) to save context.
|
||||
Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules in `.cursor/index.mdc`. **Claude Code:** load `STANDARDS.md` only when needed.
|
||||
|
||||
## Reference Documentation
|
||||
## Reference docs
|
||||
|
||||
- **This repository:** **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)**.
|
||||
- **Cursor:** `.cursor/index.mdc` (always-on rules); optional `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` is deprecated — see `.cursor/index.mdc`.
|
||||
- **Optional local files:** `NOTES.md` (short vendor-neutral project snapshot). For handoffs, keep notes local (e.g., a scratch file outside the repo) rather than committing `HANDOFF.md`.
|
||||
- **GitNexus:** skills under `.claude/skills/gitnexus/`; machine-oriented rules in the `gitnexus:start` … `gitnexus:end` block below.
|
||||
- **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)**
|
||||
- **Call-resolution DAG:** See ARCHITECTURE.md § Call-Resolution DAG. Typed 6-stage DAG inside the `parse` phase; language-specific behavior behind `inferImplicitReceiver` / `selectDispatch` hooks on `LanguageProvider`. Shared code in `gitnexus/src/core/ingestion/` must not name languages. Types: `gitnexus/src/core/ingestion/call-types.ts`.
|
||||
- **Cursor:** `.cursor/index.mdc` (always-on); `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` deprecated.
|
||||
- **GitNexus:** skills in `.claude/skills/gitnexus/`; MCP rules in `gitnexus:start` block below.
|
||||
|
||||
## Changelog
|
||||
|
||||
| Date | Version | Change |
|
||||
|------|---------|--------|
|
||||
| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication (was inlined in Reference Docs bullet). |
|
||||
| 2026-03-23 | 1.1.0 | Updated agent instructions (sections, references, Cursor layout). |
|
||||
| 2026-03-22 | 1.0.0 | Added structured agent header and changelog. |
|
||||
| 2026-04-16 | 1.4.0 | Fixed: web UI description, pre-commit behavior, MCP tools (7->16), added gitnexus-shared, removed stale vite-plugin-wasm gotcha. |
|
||||
| 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. |
|
||||
| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication. |
|
||||
| 2026-03-23 | 1.1.0 | Updated agent instructions, references, Cursor layout. |
|
||||
| 2026-03-22 | 1.0.0 | Initial structured header and changelog. |
|
||||
|
||||
---
|
||||
|
||||
<!-- gitnexus:start -->
|
||||
# GitNexus — Code Intelligence
|
||||
|
||||
This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relationships, 185 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
|
||||
Indexed as **GitNexus** (4325 symbols, 10556 relationships, 300 execution flows). Use MCP tools to understand code, assess impact, and navigate safely.
|
||||
|
||||
> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first.
|
||||
> If any tool warns the index is stale, run `npx gitnexus analyze` first.
|
||||
|
||||
## Always Do
|
||||
|
||||
- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user.
|
||||
- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows.
|
||||
- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits.
|
||||
- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance.
|
||||
- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`.
|
||||
- **MUST run impact analysis before editing any symbol.** `gitnexus_impact({target: "symbolName", direction: "upstream"})` — report blast radius to the user.
|
||||
- **MUST run `gitnexus_detect_changes()` before committing** — verify only expected symbols and flows are affected.
|
||||
- **MUST warn the user** if impact returns HIGH or CRITICAL risk.
|
||||
- Explore unfamiliar code with `gitnexus_query({query: "concept"})` (process-grouped, ranked) instead of grepping.
|
||||
- Full context on a symbol: `gitnexus_context({name: "symbolName"})`.
|
||||
|
||||
## When Debugging
|
||||
|
||||
1. `gitnexus_query({query: "<error or symptom>"})` — find execution flows related to the issue
|
||||
2. `gitnexus_context({name: "<suspect function>"})` — see all callers, callees, and process participation
|
||||
3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step
|
||||
4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed
|
||||
1. `gitnexus_query({query: "<error or symptom>"})` — find related execution flows
|
||||
2. `gitnexus_context({name: "<suspect function>"})` — callers, callees, process participation
|
||||
3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace flow step by step
|
||||
4. Regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})`
|
||||
|
||||
## When Refactoring
|
||||
|
||||
- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`.
|
||||
- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code.
|
||||
- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed.
|
||||
- **Rename:** `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Graph edits are safe; text_search edits need manual review.
|
||||
- **Extract/Split:** `gitnexus_context` (incoming/outgoing refs) then `gitnexus_impact` (upstream callers) before moving code.
|
||||
- **After any refactor:** `gitnexus_detect_changes({scope: "all"})` to verify scope.
|
||||
|
||||
## Never Do
|
||||
|
||||
- NEVER edit a function, class, or method without first running `gitnexus_impact` on it.
|
||||
- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis.
|
||||
- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph.
|
||||
- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope.
|
||||
- Edit a symbol without running `gitnexus_impact` first.
|
||||
- Ignore HIGH/CRITICAL risk warnings.
|
||||
- Rename with find-and-replace — use `gitnexus_rename`.
|
||||
- Commit without `gitnexus_detect_changes()`.
|
||||
- Add language-specific behavior to shared ingestion code (`gitnexus/src/core/ingestion/`) — use a `LanguageProvider` hook. Seeing `provider.mroStrategy === 'xxx'` or an import from `languages/xxx.ts` in shared code means stop and add a hook.
|
||||
|
||||
## Tools Quick Reference
|
||||
|
||||
| Tool | When to use | Command |
|
||||
| Tool | When to use | Example |
|
||||
|------|-------------|---------|
|
||||
| `list_repos` | Discover indexed repos | `gitnexus_list_repos({})` |
|
||||
| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` |
|
||||
| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` |
|
||||
| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` |
|
||||
| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` |
|
||||
| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` |
|
||||
| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` |
|
||||
| `api_impact` | Pre-change API route impact | `gitnexus_api_impact({route: "/api/users", method: "GET"})` |
|
||||
| `route_map` | Route → handler → consumer map | `gitnexus_route_map({})` |
|
||||
| `tool_map` | MCP/RPC tool definitions | `gitnexus_tool_map({})` |
|
||||
| `shape_check` | Response shape vs consumer access | `gitnexus_shape_check({route: "/api/users"})` |
|
||||
| `group_list` | List repo groups | `gitnexus_group_list({})` |
|
||||
| `group_query` | Cross-repo search in a group | `gitnexus_group_query({name: "myGroup", query: "auth"})` |
|
||||
| `group_sync` | Rebuild group Contract Registry | `gitnexus_group_sync({name: "myGroup"})` |
|
||||
| `group_contracts` | Inspect group contracts | `gitnexus_group_contracts({name: "myGroup"})` |
|
||||
| `group_status` | Group staleness report | `gitnexus_group_status({name: "myGroup"})` |
|
||||
|
||||
## Impact Risk Levels
|
||||
|
||||
| Depth | Meaning | Action |
|
||||
|-------|---------|--------|
|
||||
| d=1 | WILL BREAK — direct callers/importers | MUST update these |
|
||||
| d=1 | WILL BREAK — direct callers/importers | MUST update |
|
||||
| d=2 | LIKELY AFFECTED — indirect deps | Should test |
|
||||
| d=3 | MAY NEED TESTING — transitive | Test if critical path |
|
||||
|
||||
|
|
@ -118,87 +124,80 @@ This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relation
|
|||
|
||||
| Resource | Use for |
|
||||
|----------|---------|
|
||||
| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness |
|
||||
| `gitnexus://repo/GitNexus/context` | Codebase overview, index freshness |
|
||||
| `gitnexus://repo/GitNexus/clusters` | All functional areas |
|
||||
| `gitnexus://repo/GitNexus/processes` | All execution flows |
|
||||
| `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace |
|
||||
|
||||
## Self-Check Before Finishing
|
||||
|
||||
Before completing any code modification task, verify:
|
||||
1. `gitnexus_impact` was run for all modified symbols
|
||||
2. No HIGH/CRITICAL risk warnings were ignored
|
||||
3. `gitnexus_detect_changes()` confirms changes match expected scope
|
||||
4. All d=1 (WILL BREAK) dependents were updated
|
||||
2. No HIGH/CRITICAL warnings were ignored
|
||||
3. `gitnexus_detect_changes()` confirms expected scope
|
||||
4. All d=1 dependents were updated
|
||||
|
||||
## Keeping the Index Fresh
|
||||
|
||||
After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it:
|
||||
|
||||
```bash
|
||||
npx gitnexus analyze
|
||||
npx gitnexus analyze # basic refresh
|
||||
npx gitnexus analyze --embeddings # preserve embeddings
|
||||
```
|
||||
|
||||
If the index previously included embeddings, preserve them by adding `--embeddings`:
|
||||
Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). Running without `--embeddings` deletes existing vectors.
|
||||
|
||||
```bash
|
||||
npx gitnexus analyze --embeddings
|
||||
```
|
||||
> Claude Code: PostToolUse hook handles this after `git commit` and `git merge`.
|
||||
|
||||
To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.**
|
||||
## CLI Skills
|
||||
|
||||
> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`.
|
||||
|
||||
## CLI
|
||||
|
||||
| Task | Read this skill file |
|
||||
|------|---------------------|
|
||||
| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
|
||||
| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
|
||||
| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
|
||||
| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
|
||||
| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
|
||||
| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
|
||||
| Task | Skill file |
|
||||
|------|-----------|
|
||||
| Architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
|
||||
| Blast radius / "What breaks?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
|
||||
| Debugging / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
|
||||
| Refactoring | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
|
||||
| Tools/resources/schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
|
||||
| CLI commands (index, status, clean, wiki) | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
|
||||
|
||||
<!-- gitnexus:end -->
|
||||
|
||||
## Cursor Cloud specific instructions
|
||||
## Repo reference
|
||||
|
||||
### Repository structure
|
||||
### Packages
|
||||
|
||||
This is a monorepo with two main products and supporting config packages:
|
||||
|
||||
| Component | Path | Purpose |
|
||||
|-----------|------|---------|
|
||||
| **GitNexus CLI/Core** | `gitnexus/` | Main product — TypeScript CLI, indexing pipeline, MCP server. Published to npm. |
|
||||
| **GitNexus Web UI** | `gitnexus-web/` | React/Vite browser app — graph explorer + AI chat. Runs entirely in WASM. |
|
||||
| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace (no build). |
|
||||
| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor (no build). |
|
||||
| SWE-bench Eval | `eval/` | Python evaluation harness (optional; needs Docker + LLM API keys). |
|
||||
| Package | Path | Purpose |
|
||||
|---------|------|---------|
|
||||
| **CLI/Core** | `gitnexus/` | TypeScript CLI, indexing pipeline, MCP server. Published to npm. |
|
||||
| **Web UI** | `gitnexus-web/` | React/Vite thin client. All queries via `gitnexus serve` HTTP API. |
|
||||
| **Shared** | `gitnexus-shared/` | Shared TypeScript types and constants. |
|
||||
| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace. |
|
||||
| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor. |
|
||||
| Eval | `eval/` | Python evaluation harness (Docker + LLM API keys). |
|
||||
|
||||
### Running services
|
||||
|
||||
- **CLI/Core**: `cd gitnexus && npm run dev` (tsx watch mode) or `npm run build && node dist/cli/index.js <command>`
|
||||
- **Web UI**: `cd gitnexus-web && npm run dev` (Vite on port 5173)
|
||||
- **Backend mode**: `cd <indexed-repo> && node /workspace/gitnexus/dist/cli/index.js serve` (HTTP API on port 3741 by default)
|
||||
```bash
|
||||
cd gitnexus && npm run dev # CLI: tsx watch mode
|
||||
cd gitnexus-web && npm run dev # Web UI: Vite on port 5173
|
||||
npx gitnexus serve # HTTP API on port 4747 (from any indexed repo)
|
||||
```
|
||||
|
||||
### Testing
|
||||
|
||||
**CLI / Core (`gitnexus/`)**
|
||||
- **Unit tests**: `cd gitnexus && npm test` (vitest, ~2000 tests)
|
||||
- **Integration tests**: `cd gitnexus && npm run test:integration` (vitest, ~1850 tests). Two LadybugDB file-locking tests (`lbug-core-adapter`, `search-core`) may fail in containerized environments due to `/tmp` locking limitations — this is a known environment issue, not a code bug.
|
||||
- **TypeScript check**: `cd gitnexus && npx tsc --noEmit`
|
||||
- `npm test` — full vitest suite (~2000 tests)
|
||||
- `npm run test:unit` — unit tests only
|
||||
- `npm run test:integration` — integration (~1850 tests). LadybugDB file-locking tests may fail in containers (known env issue).
|
||||
- `npx tsc --noEmit` — typecheck
|
||||
|
||||
**Web UI (`gitnexus-web/`)**
|
||||
- **Unit tests**: `cd gitnexus-web && npm test` (vitest, ~200 tests)
|
||||
- **E2E tests**: `cd gitnexus-web && E2E=1 npx playwright test` (Playwright, 5 tests — requires `gitnexus serve` + `npm run dev` running)
|
||||
- **TypeScript check**: `cd gitnexus-web && npx tsc -b --noEmit`
|
||||
- `npm test` — vitest (~200 tests)
|
||||
- `npm run test:e2e` — Playwright (7 spec files; requires `gitnexus serve` + `npm run dev`)
|
||||
- `npx tsc -b --noEmit` — typecheck
|
||||
|
||||
No separate lint command is configured; TypeScript strict checking serves as the primary static analysis.
|
||||
**Pre-commit hook** (`.husky/pre-commit`): formatting (prettier via lint-staged) + typecheck for staged packages. Tests do **not** run in pre-commit — CI only.
|
||||
|
||||
### Gotchas
|
||||
|
||||
- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift). Native tree-sitter bindings require `python3`, `make`, and `g++` to be present.
|
||||
- `tree-sitter-kotlin` and `tree-sitter-swift` are optional dependencies — install warnings for these are expected and non-blocking.
|
||||
- The Web UI uses `vite-plugin-wasm` and requires `Cross-Origin-Opener-Policy`/`Cross-Origin-Embedder-Policy` headers for `SharedArrayBuffer` (handled automatically by Vite dev server).
|
||||
- There is no ESLint/Prettier configuration in this repo.
|
||||
- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift, builds tree-sitter-proto). Native bindings need `python3`, `make`, `g++`.
|
||||
- `tree-sitter-kotlin` and `tree-sitter-swift` are optional — install warnings expected.
|
||||
- ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`.
|
||||
|
|
|
|||
418
ARCHITECTURE.md
418
ARCHITECTURE.md
|
|
@ -1,123 +1,365 @@
|
|||
# Architecture — GitNexus
|
||||
|
||||
This repository is a **monorepo** with two main products: the **CLI / MCP package** (`gitnexus/`) and the **browser UI** (`gitnexus-web/`). Supporting folders ship editor integrations and plugins without changing the core graph engine.
|
||||
Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`).
|
||||
|
||||
## Repository layout
|
||||
|
||||
| Path | Role |
|
||||
|------|------|
|
||||
| `gitnexus/` | Published npm package `gitnexus`: CLI, MCP server (stdio), local HTTP API for bridge mode, ingestion pipeline, LadybugDB graph, embeddings (optional). |
|
||||
| `gitnexus-web/` | Vite + React UI: in-browser indexing (WASM), graph visualization, optional connection to `gitnexus serve`. |
|
||||
| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Packaged **skills** and plugin metadata so agents discover the same workflows as documented in `AGENTS.md`. |
|
||||
| `eval/` | Evaluation harnesses and docs for benchmarking tool usage. |
|
||||
| `.github/` | CI workflows (quality, unit, integration, E2E) and composite actions. |
|
||||
| `gitnexus/` | npm package `gitnexus`: CLI, MCP server (stdio), HTTP API, ingestion pipeline, LadybugDB graph, embeddings. |
|
||||
| `gitnexus-web/` | Vite + React thin client: graph explorer + AI chat. All queries via `gitnexus serve` HTTP API. |
|
||||
| `gitnexus-shared/` | Shared TypeScript types and constants (consumed by CLI and Web). |
|
||||
| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Agent skills and plugin metadata. |
|
||||
| `eval/` | Evaluation harnesses for benchmarking tool usage. |
|
||||
| `.github/` | CI workflows + composite actions (`setup-gitnexus/`, `setup-gitnexus-web/`). |
|
||||
|
||||
## End-to-end flow: index → graph → tools
|
||||
|
||||
1. **Ingestion** (`gitnexus analyze`)
|
||||
- Entry: `gitnexus/src/cli/analyze.ts` → `runPipelineFromRepo` in `gitnexus/src/core/ingestion/pipeline.ts`.
|
||||
- Walks the git working tree, parses supported languages via **Tree-sitter**, resolves imports/calls/inheritance, detects **communities** and **processes** (execution flows), and builds an in-memory **knowledge graph** (`gitnexus/src/core/graph/`).
|
||||
- Output is loaded into **LadybugDB** under **`.gitnexus/`** at the repo root (`lbug/`, `meta.json`, etc.). Optional **FTS** indexes and **embeddings** attach to the same store.
|
||||
- The repo is registered in **`~/.gitnexus/registry.json`** so MCP can find it from any working directory.
|
||||
1. **Ingestion** — `analyze.ts` → `runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 12 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery.
|
||||
|
||||
2. **Persistence & metadata**
|
||||
- `gitnexus/src/storage/repo-manager.ts` — paths, registry, cleanup of legacy Kuzu artifacts.
|
||||
- `gitnexus/src/core/lbug/lbug-adapter.ts` — graph load, queries, embedding restore batches.
|
||||
2. **Persistence** — `repo-manager.ts` (paths, registry, KuzuDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches).
|
||||
|
||||
3. **Query & agents**
|
||||
- **MCP (stdio):** `gitnexus/src/cli/mcp.ts` → `startMCPServer` → `LocalBackend` (`gitnexus/src/mcp/local/local-backend.ts`) opens registered repos and serves **tools** from `gitnexus/src/mcp/tools.ts` and **resources** from `gitnexus/src/mcp/resources.ts`.
|
||||
- **Bridge HTTP:** `gitnexus/src/cli/serve.ts` → Express app in `gitnexus/src/server/api.ts` (CORS-limited) exposes REST + MCP-over-HTTP for the web UI.
|
||||
- **CLI tools (no MCP):** `gitnexus query`, `context`, `impact`, `cypher` in `gitnexus/src/cli/tool.ts` call the same backend for scripts and CI.
|
||||
3. **Query layer** — three interfaces to the same backend:
|
||||
- **MCP (stdio):** `mcp.ts` → `LocalBackend` → tools (`tools.ts`) + resources (`resources.ts`)
|
||||
- **HTTP bridge:** `serve.ts` → Express (`api.ts`, `mcp-http.ts`) for web UI
|
||||
- **CLI direct:** `gitnexus query|context|impact|cypher` in `tool.ts`
|
||||
|
||||
4. **Staleness**
|
||||
- `gitnexus/src/mcp/staleness.ts` compares indexed `lastCommit` to `HEAD` and surfaces hints when the graph is behind git.
|
||||
4. **Staleness** — `staleness.ts` compares indexed `lastCommit` to `HEAD`, surfaces hints.
|
||||
|
||||
## MCP tools (summary)
|
||||
## MCP tools
|
||||
|
||||
| Tool | Purpose |
|
||||
|------|---------|
|
||||
| `list_repos` | Discover indexed repositories when more than one is registered. |
|
||||
| `query` | Natural-language / keyword search over the graph (hybrid BM25 + optional vectors). |
|
||||
| `cypher` | Ad hoc **Cypher** against the schema (see resource `gitnexus://repo/{name}/schema`). |
|
||||
| `context` | Callers, callees, processes for one symbol (with disambiguation). |
|
||||
| `impact` | Blast radius (upstream/downstream) with depth and risk summary. |
|
||||
| `detect_changes` | Map git diffs to affected symbols and processes. |
|
||||
| `rename` | Graph-assisted rename with `dry_run` preview (`graph` vs `text_search` confidence). |
|
||||
| `list_repos` | Discover indexed repos |
|
||||
| `query` | Hybrid BM25 + vector search over the graph |
|
||||
| `cypher` | Ad hoc Cypher against the schema |
|
||||
| `context` | Callers, callees, processes for one symbol |
|
||||
| `impact` | Blast radius (upstream/downstream) with risk summary |
|
||||
| `detect_changes` | Map git diffs to affected symbols and processes |
|
||||
| `rename` | Graph-assisted multi-file rename with `dry_run` preview |
|
||||
| `api_impact` | Pre-change impact report for an API route handler |
|
||||
| `route_map` | API route → handler → consumer mappings |
|
||||
| `tool_map` | MCP/RPC tool definitions and handlers |
|
||||
| `shape_check` | Response shape vs consumer property access mismatches |
|
||||
| `group_list` | List repo groups or details for one group |
|
||||
| `group_query` | Cross-repo search in a group (reciprocal rank fusion) |
|
||||
| `group_sync` | Rebuild group Contract Registry (`contracts.json`) |
|
||||
| `group_contracts` | Inspect group contracts and cross-links |
|
||||
| `group_status` | Index and Contract Registry staleness per repo in a group |
|
||||
|
||||
## Where to change what
|
||||
|
||||
| If you are changing… | Start in… |
|
||||
|----------------------|-----------|
|
||||
| CLI commands / flags | `gitnexus/src/cli/` (`index.ts`, per-command modules). |
|
||||
| Parsing or graph construction | `gitnexus/src/core/ingestion/` (pipeline, processors, resolvers, type-extractors). |
|
||||
| Graph schema / DB access | `gitnexus/src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`), `gitnexus/src/mcp/core/lbug-adapter.ts` if MCP-specific. |
|
||||
| MCP protocol, tools, resources | `gitnexus/src/mcp/server.ts`, `tools.ts`, `resources.ts`. |
|
||||
| Search ranking | `gitnexus/src/core/search/` (BM25, hybrid fusion). |
|
||||
| Embeddings | `gitnexus/src/core/embeddings/`, phases in `analyze.ts`. |
|
||||
| Wiki generation | `gitnexus/src/core/wiki/`. |
|
||||
| Web UI behavior | `gitnexus-web/src/` (components, workers, graph client). |
|
||||
| CI | `.github/workflows/*.yml`, `.github/actions/setup-gitnexus/`. |
|
||||
| Concern | Start in |
|
||||
|---------|----------|
|
||||
| CLI commands/flags | `src/cli/` (`index.ts`, per-command modules) |
|
||||
| Parsing/graph construction | `src/core/ingestion/pipeline-phases/` + `pipeline.ts` |
|
||||
| Graph schema/DB | `src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`) |
|
||||
| MCP tools/resources | `src/mcp/server.ts`, `tools.ts`, `resources.ts` |
|
||||
| Search ranking | `src/core/search/` (BM25, hybrid fusion) |
|
||||
| Embeddings | `src/core/embeddings/` + `src/core/run-analyze.ts` |
|
||||
| Wiki generation | `src/core/wiki/` |
|
||||
| Language support | `src/core/ingestion/languages/` + `tree-sitter-queries.ts` + `gitnexus-shared/src/languages.ts` |
|
||||
| Import resolution | `src/core/ingestion/import-processor.ts` + `import-resolvers/configs/` + `model/resolution-context.ts` |
|
||||
| Call resolution/MRO | `src/core/ingestion/call-processor.ts` + `model/resolve.ts` |
|
||||
| Type extraction | `src/core/ingestion/type-extractors/` |
|
||||
| Worker pool | `src/core/ingestion/workers/` |
|
||||
| Web UI | `gitnexus-web/src/` |
|
||||
| CI | `.github/workflows/*.yml`, `.github/actions/` |
|
||||
|
||||
> Paths above are relative to `gitnexus/` unless they start with `gitnexus-web/` or `.github/`.
|
||||
|
||||
---
|
||||
|
||||
## Pipeline Phase DAG
|
||||
|
||||
12 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output.
|
||||
|
||||
```
|
||||
scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
|
||||
→ crossFile → mro → communities → processes
|
||||
```
|
||||
|
||||
| Phase | File | Deps | Output |
|
||||
|-------|------|------|--------|
|
||||
| `scan` | `scan.ts` | (root) | File paths + sizes |
|
||||
| `structure` | `structure.ts` | `scan` | File/Folder nodes, CONTAINS edges, `allPathSet` |
|
||||
| `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx |
|
||||
| `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) |
|
||||
| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries |
|
||||
| `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators) |
|
||||
| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges |
|
||||
| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) |
|
||||
| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order |
|
||||
| `mro` | `mro.ts` | `crossFile`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges |
|
||||
| `communities` | `communities.ts` | `mro`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) |
|
||||
| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `structure` | Process nodes + STEP_IN_PROCESS edges |
|
||||
|
||||
**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `orm-extraction.ts` (sequential ORM fallback), `types.ts`, `runner.ts`, `index.ts`.
|
||||
|
||||
### DAG runner
|
||||
|
||||
`runner.ts` — static phase graph, no plugins, compile-time type safety.
|
||||
|
||||
1. **Validation** — Kahn's topological sort. Rejects on: duplicate names, missing deps, cycles (DFS traces the concrete cycle path, e.g., `A -> B -> C -> A`, plus count of transitively blocked dependents).
|
||||
|
||||
2. **Execution** — sequential in topological order. Each phase receives:
|
||||
- `ctx: PipelineContext` — shared mutable `KnowledgeGraph`, `repoPath`, progress callback, options
|
||||
- `deps: ReadonlyMap<string, PhaseResult>` — **declared deps only** (runner filters the results map to prevent hidden coupling)
|
||||
|
||||
3. **Error handling** — wraps phase errors with the phase name, emits terminal `error` progress event, swallows progress handler errors to preserve the original cause.
|
||||
|
||||
4. **Timing** — per-phase `durationMs` in `PhaseResult`, dev-mode console logging.
|
||||
|
||||
**Design patterns:**
|
||||
- **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output.
|
||||
- **Typed phase access** — `getPhaseOutput<T>(deps, 'name')` for type-safe upstream results.
|
||||
- **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership.
|
||||
- **Skippable phases** — `skipGraphPhases` omits MRO/communities/processes (faster tests). `skipWorkers` forces sequential parsing.
|
||||
|
||||
### How to add a new phase
|
||||
|
||||
1. Create `pipeline-phases/my-phase.ts` with a `PipelinePhase<MyOutput>` (name, deps, execute)
|
||||
2. Export from `pipeline-phases/index.ts`
|
||||
3. Add to `buildPhaseList()` in `pipeline.ts`
|
||||
|
||||
```typescript
|
||||
import type { PipelinePhase, PhaseResult } from './types.js';
|
||||
import { getPhaseOutput } from './types.js';
|
||||
import type { ParseOutput } from './parse.js';
|
||||
|
||||
export interface MyPhaseOutput { /* ... */ }
|
||||
|
||||
export const myPhase: PipelinePhase<MyPhaseOutput> = {
|
||||
name: 'myPhase',
|
||||
deps: ['parse'],
|
||||
async execute(ctx, deps) {
|
||||
const { allPaths } = getPhaseOutput<ParseOutput>(deps, 'parse');
|
||||
// ... write to ctx.graph ...
|
||||
return { /* typed output */ };
|
||||
},
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Call-Resolution DAG
|
||||
|
||||
Typed 6-stage pipeline in `call-processor.ts` (inside the `parse` phase) that resolves method/function calls and emits CALLS edges. Language behavior plugs in at two `LanguageProvider` hook points (stages 3–4); shared code names no languages. Scope: call resolution only — import resolution, type extraction, heritage, and symbol-table population live in other phases.
|
||||
|
||||
### Stages
|
||||
|
||||
```
|
||||
extract-call ──▶ classify-form ──▶ infer-receiver ──▶ select-dispatch ──▶ resolve-target ──▶ emit-edge
|
||||
(1) (2) (3) [hook] (4) [hook] (5) (6)
|
||||
```
|
||||
|
||||
| Stage | Produces | Location |
|
||||
|-------|----------|----------|
|
||||
| **extract-call** | `ExtractedCallSite` (name, form, receiver, argCount) | `call-extractors/` (per-language); runs in worker |
|
||||
| **classify-form** | callForm (`free`/`member`/`constructor`) + arity | `call-analysis.ts` → `inferCallForm`; shared, runs in worker |
|
||||
| **infer-receiver** | `ReceiverEnriched` (receiver type finalized) | `call-processor.ts`; shared default chain, then `inferImplicitReceiver` hook |
|
||||
| **select-dispatch** | `DispatchDecision` (primary, fallback, ancestryView) | `selectDispatch` hook, falls back to shared default |
|
||||
| **resolve-target** | `TieredCandidates` | `model/resolve.ts` → `lookupMethodByOwnerWithMRO` (MRO walk) |
|
||||
| **emit-edge** | CALLS edge in graph | `call-processor.ts`; writes edge with confidence tier |
|
||||
|
||||
### Provider hooks
|
||||
|
||||
Both hooks are optional on `LanguageProvider`. Ruby is the only current implementer.
|
||||
|
||||
**`inferImplicitReceiver`** — called after shared infer-receiver defaults. Returns `ImplicitReceiverOverride | null`.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Inputs | `calledName`, `callForm`, `receiverName`, `receiverTypeName`, `callNode` (AST), `filePath` |
|
||||
| Non-null fields | `callForm`, `receiverName`, `receiverTypeName` (required); `receiverSource: 'implicit-self'` (fixed); `hint?` (opaque, passed to `selectDispatch`) |
|
||||
| Null | Keep existing `ReceiverEnriched` state |
|
||||
|
||||
**`selectDispatch`** — called after infer-receiver (including hook). Returns `DispatchDecision | null`; null uses shared default (constructor → `primary:'constructor'`; typed receiver → `primary:'owner-scoped'`; else → `primary:'free'`).
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Inputs | `calledName`, `callForm`, `receiverName`, `receiverTypeName`, `receiverSource`, `hint` |
|
||||
| Non-null fields | `primary: 'owner-scoped' \| 'free' \| 'constructor'`; `fallback?: 'free-arity-narrowed'`; `ancestryView?: 'instance' \| 'singleton'`; `hint?` |
|
||||
|
||||
**`DispatchDecision` field semantics:**
|
||||
- `primary: 'owner-scoped'` — MRO walk from receiver's type; used when receiver type is known.
|
||||
- `fallback: 'free-arity-narrowed'` — after owner-scoped miss, search free-call candidates by arity only (Ruby uses this for implicit-self calls that miss their owner's MRO).
|
||||
- `ancestryView: 'singleton'` — walk singleton/class ancestry instead of instance ancestry (Ruby `def self.foo` bodies, so `extend`-ed methods are found).
|
||||
|
||||
### Adding language behavior
|
||||
|
||||
1. **Implicit receivers** — implement `inferImplicitReceiver`: return null if call already has a receiver; otherwise use `findEnclosingClassInfo` (`ast-helpers.ts`) to find the enclosing context, return `ImplicitReceiverOverride` with `receiverSource: 'implicit-self'`, and optionally set `hint` for `selectDispatch`.
|
||||
2. **Custom dispatch** — implement `selectDispatch`: inspect `receiverSource` and `hint`, return `DispatchDecision` with `primary`, optional `fallback`, optional `ancestryView`; return null to keep shared defaults.
|
||||
3. **MRO strategy** — confirm `mroStrategy` is `'first-wins'`, `'c3'`, `'ruby-mixin'`, or `'none'`; consumed by `lookupMethodByOwnerWithMRO`.
|
||||
|
||||
**Ruby example** (`languages/ruby.ts` + `utils/ruby-self-call.ts`): `inferImplicitReceiver` rewrites bare-identifier calls to `self.method` and sets `hint` to `'instance'`/`'singleton'`; `selectDispatch` uses hint for `ancestryView` and adds `fallback: 'free-arity-narrowed'` for implicit-self calls.
|
||||
|
||||
### Code references
|
||||
|
||||
| Module | Purpose |
|
||||
|--------|---------|
|
||||
| `core/ingestion/call-types.ts` | DAG types: `ReceiverEnriched`, `DispatchDecision`, `ImplicitReceiverOverride` |
|
||||
| `core/ingestion/language-provider.ts` | Hook signatures: `inferImplicitReceiver`, `selectDispatch` |
|
||||
| `core/ingestion/call-processor.ts` | `processCalls`: stages 3–6 |
|
||||
| `core/ingestion/model/resolve.ts` | `lookupMethodByOwnerWithMRO`: stage 5 MRO walk |
|
||||
| `core/ingestion/languages/ruby.ts` | Both hooks + `mroStrategy: 'ruby-mixin'` |
|
||||
| `core/ingestion/utils/ruby-self-call.ts` | Bare-call rewrite for `inferImplicitReceiver` |
|
||||
|
||||
---
|
||||
|
||||
## Language-agnostic graph feeding
|
||||
|
||||
16 languages → single unified graph. Four abstraction layers:
|
||||
|
||||
```
|
||||
Unified Graph Schema (44 node types, 21 relationship types)
|
||||
↑
|
||||
Unified Resolution (3-tier name lookup + MRO walk)
|
||||
↑
|
||||
Language Providers (import semantics, type config, export checker, MRO strategy)
|
||||
↑
|
||||
Tree-Sitter Queries (per-language S-expressions, unified capture tags)
|
||||
```
|
||||
|
||||
### Language providers
|
||||
|
||||
Each language implements `LanguageProvider` (`language-provider.ts`). Key fields:
|
||||
|
||||
| Field | Purpose |
|
||||
|-------|---------|
|
||||
| `id`, `extensions` | Language identity and file matching |
|
||||
| `treeSitterQueries` | S-expression queries for AST extraction |
|
||||
| `importSemantics` | `named` / `wildcard-leaf` / `wildcard-transitive` / `namespace` |
|
||||
| `importResolver` | Language-specific path → file resolution |
|
||||
| `exportChecker` | Public/exported symbol detection |
|
||||
| `typeConfig` | Type annotation extraction rules |
|
||||
| `mroStrategy` | `first-wins` / `c3` / `none` |
|
||||
|
||||
16 providers in `languages/index.ts` via `satisfies Record<SupportedLanguages, LanguageProvider>` — missing a language is a compile error.
|
||||
|
||||
### Unified capture tags
|
||||
|
||||
Per-language tree-sitter queries use different AST node names but produce the **same semantic capture tags**: `@definition.class`, `@definition.function`, `@call.name`, `@import.source`, `@heritage.extends`. Downstream extraction needs no language branching. Defined in `tree-sitter-queries.ts`.
|
||||
|
||||
### Import resolution
|
||||
|
||||
Per-language import resolution uses the **configs + factory** pattern (like call/method/class extractors). Each language declares an `ImportResolutionConfig` in `import-resolvers/configs/`, listing an ordered chain of `ImportResolverStrategy` functions. `createImportResolver()` (in `resolver-factory.ts`) composes them: first non-null result wins. Low-level helpers shared across strategies live alongside the configs in `import-resolvers/` (e.g. `go.ts`, `rust.ts`, `python.ts`).
|
||||
|
||||
Unified 3-tier algorithm (`model/resolution-context.ts`), per-language `importSemantics` controls which tier activates:
|
||||
|
||||
| Tier | Confidence | Mechanism |
|
||||
|------|-----------|-----------|
|
||||
| 1 — same-file | 0.95 | Symbol table for caller's file |
|
||||
| 2 — import-scoped | 0.9 | `NamedImportMap` chains (named) or all files in `importMap` (wildcard) |
|
||||
| 3 — global | 0.5 | O(1) index lookups: class, impl, callable. Fallback only |
|
||||
|
||||
| Import strategy | Languages | Behavior |
|
||||
|----------------|-----------|----------|
|
||||
| `named` | TS, JS, Java, C#, Rust, PHP, Kotlin | Only explicitly imported names visible |
|
||||
| `wildcard-leaf` | Go, Ruby, Swift, Dart | Whole-package import, no transitive re-exports |
|
||||
| `wildcard-transitive` | C, C++ | `#include` closure chains through re-exports |
|
||||
| `namespace` | Python | Module aliases resolved at call site |
|
||||
|
||||
### Chunked parse-and-resolve
|
||||
|
||||
`parse` processes files in ~20 MB byte-budget chunks to bound memory. Per chunk:
|
||||
1. Worker pool dispatches files (or sequential fallback via `skipWorkers`)
|
||||
2. Each worker: detect language → load grammar → run queries → return unified `ParseWorkerResult`
|
||||
3. Synthesize wildcard bindings (`wildcard-synthesis.ts`)
|
||||
4. Resolve imports and heritage
|
||||
5. Collect `BindingAccumulator` entries for cross-file propagation
|
||||
|
||||
Workers: `workers/worker-pool.ts`, `workers/parse-worker.ts`.
|
||||
|
||||
### Heritage and MRO
|
||||
|
||||
All languages emit unified `ExtractedHeritage` (child, parent, `EXTENDS`/`IMPLEMENTS`). MRO phase walks the heritage graph using per-language strategy:
|
||||
- **`first-wins`** — Java, C#, C++, TS, Ruby, Go
|
||||
- **`c3`** — Python (C3 linearization)
|
||||
- **`none`** — single-inheritance languages
|
||||
|
||||
Unified walk: `lookupMethodByOwnerWithMRO()` in `model/resolve.ts`.
|
||||
|
||||
---
|
||||
|
||||
## Full analysis flow
|
||||
|
||||
`runFullAnalysis` in `run-analyze.ts` orchestrates everything around the pipeline:
|
||||
|
||||
```
|
||||
CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks)
|
||||
1. Early exit if lastCommit == HEAD (unless --force) [0%]
|
||||
2. Cache existing embeddings from prior index [0%]
|
||||
3. runPipelineFromRepo() → KnowledgeGraph [0-60%]
|
||||
4. Clean up legacy KuzuDB files [60%]
|
||||
5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%]
|
||||
6. Create FTS indexes (File, Function, Class, Method...) [85-90%]
|
||||
7. Restore cached embeddings (batch insert) [88%]
|
||||
8. Generate new embeddings if --embeddings [90-98%]
|
||||
9. Save metadata + register repo + update .gitignore [98-100%]
|
||||
10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%]
|
||||
```
|
||||
|
||||
**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`.
|
||||
|
||||
## Storage
|
||||
|
||||
```
|
||||
<repo>/.gitnexus/
|
||||
├── lbug # LadybugDB database
|
||||
├── lbug.wal # Write-ahead log
|
||||
├── lbug.lock # Single-writer lock
|
||||
└── meta.json # lastCommit, indexedAt, stats
|
||||
|
||||
~/.gitnexus/
|
||||
└── registry.json # Global repo registry (MCP discovery)
|
||||
```
|
||||
|
||||
Managed by `repo-manager.ts`.
|
||||
|
||||
## LadybugDB schema
|
||||
|
||||
Defined in `lbug/schema.ts`. Separate node tables per type, single `CodeRelation` table.
|
||||
|
||||
**Node tables:** File, Folder, Function, Class, Interface, Method, Constructor, CodeElement, Struct, Enum, Macro, Typedef, Union, Namespace, Trait, Impl, TypeAlias, Const, Static, Property, Record, Delegate, Annotation, Template, Module, Community, Process, Route, Tool, Section, Embedding.
|
||||
|
||||
**Relation types** (`CodeRelation.type`): CONTAINS, DEFINES, CALLS, IMPORTS, EXTENDS, IMPLEMENTS, HAS_METHOD, HAS_PROPERTY, ACCESSES, METHOD_OVERRIDES, METHOD_IMPLEMENTS, MEMBER_OF, STEP_IN_PROCESS, HANDLES_ROUTE, FETCHES, HANDLES_TOOL, ENTRY_POINT_OF.
|
||||
|
||||
## Embeddings and search
|
||||
|
||||
**Embeddings** (`src/core/embeddings/`): Snowflake arctic-embed-xs (384D). Embeddable: File, Function, Class, Method, Interface. Incremental via SHA1 content hash. Separate `Embedding` table.
|
||||
|
||||
**Search** (`src/core/search/`): Hybrid BM25 + semantic vector, merged via Reciprocal Rank Fusion (K=60).
|
||||
|
||||
## Known limitations
|
||||
|
||||
### Overloaded method resolution
|
||||
|
||||
Method and Constructor node IDs include an arity suffix (`#<paramCount>`) to
|
||||
disambiguate overloaded methods. Two overloads with different parameter counts
|
||||
produce distinct graph nodes: `Method:file:Class.method#1` vs
|
||||
`Method:file:Class.method#2`.
|
||||
Node IDs use arity suffix (`#<paramCount>`): `Method:file:Class.method#1` vs `#2`.
|
||||
|
||||
**Same-arity overload disambiguation:** When two overloads share the same
|
||||
parameter count but differ in types (e.g. `save(int)` vs `save(String)`), a
|
||||
type-hash suffix `~type1,type2` is appended to produce distinct node IDs:
|
||||
`Method:file:Class.save#1~int` vs `Method:file:Class.save#1~String`. The suffix
|
||||
is only added when a same-arity collision is detected within a class and all
|
||||
parameters have non-null type annotations. Languages without type info (Python,
|
||||
Ruby, JS) fall back to arity-only IDs. TypeScript/JavaScript overload signatures
|
||||
are intentionally excluded from type-hashing because they are declaration-only
|
||||
contracts that should collapse to the implementation body's node ID. See issue
|
||||
\#651.
|
||||
**Same-arity disambiguation:** type-hash suffix `~type1,type2` when collision detected and type annotations present. Languages without types (Python, Ruby, JS) use arity-only. TS/JS overload signatures excluded (collapse to implementation body). See #651.
|
||||
|
||||
**C++ const-qualified overload disambiguation:** Methods overloaded by const
|
||||
qualification (e.g. `begin()` vs `begin() const`) are disambiguated via an
|
||||
`isConst` property and a `$const` ID suffix appended to the const-qualified
|
||||
variant when a non-const collision exists. The `$const` suffix appears after the
|
||||
type-hash suffix: e.g. `Method:file:Container.begin#0$const`.
|
||||
**C++ const-qualified:** `$const` suffix after type-hash when non-const collision exists: `Method:file:Container.begin#0$const`.
|
||||
|
||||
**Generic/template type preservation in type-hash:** The type-hash suffix uses
|
||||
`rawType` (full AST text including generic/template args) rather than the
|
||||
simplified `type` from `extractSimpleTypeName`. This means C++ template overloads
|
||||
like `process(vector<int>)` vs `process(vector<string>)` produce distinct IDs:
|
||||
`~vector<int>` vs `~vector<std::string>`. Java generic overloads like
|
||||
`process(List<String>)` vs `process(List<Integer>)` are a compile error due to
|
||||
type erasure, so this gap is theoretical for Java.
|
||||
**Generic/template types:** type-hash uses `rawType` (full AST text including generics): `~vector<int>` vs `~vector<std::string>`.
|
||||
|
||||
**ID stability on first overload:** Type and const tags are collision-only. When
|
||||
a class has `save(int)` as its only `save` method, the ID is `save#1` (no tag).
|
||||
Adding `save(String)` changes the original to `save#1~int`. This is correct for
|
||||
fresh analysis but means IDs are not stable across overload additions. Future
|
||||
incremental re-analysis should account for this.
|
||||
**ID stability:** collision-only tags mean IDs change when overloads are added. `save#1` becomes `save#1~int` when `save(String)` is added.
|
||||
|
||||
**Variadic method matching:** When one side is variadic (`parameterCount`
|
||||
undefined) and the other has a fixed count, `METHOD_IMPLEMENTS` edges are
|
||||
emitted with confidence 0.7 instead of 1.0. Variadic methods like
|
||||
`foo(String... args)` may superficially match `foo(String s)` by type but
|
||||
are not guaranteed to be interchangeable across all languages (Java/Kotlin
|
||||
accept this via varargs sugar; TypeScript, C#, Rust do not).
|
||||
**Variadic matching:** confidence 0.7 when one side is variadic and the other has fixed count.
|
||||
|
||||
**Confidence tiering** for `METHOD_IMPLEMENTS` edges:
|
||||
**METHOD_IMPLEMENTS confidence tiering:**
|
||||
|
||||
| Match quality | Confidence | When |
|
||||
|---|---|---|
|
||||
| Exact parameter types match | 1.0 | Both sides have `parameterTypes` arrays and they match |
|
||||
| Arity (count) matches | 1.0 | Both sides have `parameterCount`, types unavailable |
|
||||
| Variadic vs fixed | 0.7 | One side is variadic, other has fixed count |
|
||||
| Lenient (insufficient info) | 0.7 | One or both sides lack type and count data |
|
||||
| Match quality | Confidence |
|
||||
|---|---|
|
||||
| Exact parameter types match | 1.0 |
|
||||
| Arity match, types unavailable | 1.0 |
|
||||
| Variadic vs fixed | 0.7 |
|
||||
| Insufficient info | 0.7 |
|
||||
|
||||
## Related docs
|
||||
|
||||
- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance.
|
||||
- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery.
|
||||
- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents.
|
||||
- [TESTING.md](TESTING.md) — how to run tests.
|
||||
- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage expectations for **this** repo when indexed by GitNexus.
|
||||
- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance
|
||||
- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery
|
||||
- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents
|
||||
- [TESTING.md](TESTING.md) — how to run tests
|
||||
- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage
|
||||
|
|
|
|||
110
CLAUDE.md
110
CLAUDE.md
|
|
@ -1,10 +1,10 @@
|
|||
<!-- version: 1.2.0 -->
|
||||
<!-- version: 1.3.0 -->
|
||||
<!--
|
||||
Metadata: version, last reviewed, scope, model policy, reference docs, changelog.
|
||||
Last updated: 2026-03-22
|
||||
-->
|
||||
|
||||
Last reviewed: 2026-03-24
|
||||
Last reviewed: 2026-04-13
|
||||
|
||||
**Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub)
|
||||
|
||||
|
|
@ -35,12 +35,14 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g
|
|||
## Reference Documentation
|
||||
|
||||
- **This repository:** [AGENTS.md](AGENTS.md) (Cursor + monorepo notes), [ARCHITECTURE.md](ARCHITECTURE.md), [CONTRIBUTING.md](CONTRIBUTING.md), [GUARDRAILS.md](GUARDRAILS.md).
|
||||
- **Call-resolution DAG:** See ARCHITECTURE.md § Call-Resolution DAG. Shared pipeline code in `gitnexus/src/core/ingestion/` must not name languages — use `LanguageProvider` hooks instead (see AGENTS.md).
|
||||
- **GitNexus:** `.claude/skills/gitnexus/`; MCP and indexed-repo rules live only in [AGENTS.md](AGENTS.md) (`gitnexus:start` … `gitnexus:end`). See **GitNexus rules** below.
|
||||
|
||||
## Changelog
|
||||
|
||||
| Date | Version | Change |
|
||||
|------|---------|--------|
|
||||
| 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. |
|
||||
| 2026-03-24 | 1.2.0 | Removed duplicated gitnexus:start block and scope table; replaced with pointers to AGENTS.md. |
|
||||
| 2026-03-23 | 1.1.0 | Updated agent instructions to match AGENTS.md. |
|
||||
| 2026-03-22 | 1.0.0 | Added structured header and changelog. |
|
||||
|
|
@ -49,106 +51,4 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g
|
|||
|
||||
## GitNexus rules
|
||||
|
||||
GitNexus MCP rules are in the `<!-- gitnexus:start -->` … `<!-- gitnexus:end -->` block in **[AGENTS.md](AGENTS.md)** — load that section when working with MCP tools or the graph index.
|
||||
|
||||
<!-- gitnexus:start -->
|
||||
# GitNexus — Code Intelligence
|
||||
|
||||
This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relationships, 185 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
|
||||
|
||||
> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first.
|
||||
|
||||
## Always Do
|
||||
|
||||
- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user.
|
||||
- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows.
|
||||
- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits.
|
||||
- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance.
|
||||
- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`.
|
||||
|
||||
## When Debugging
|
||||
|
||||
1. `gitnexus_query({query: "<error or symptom>"})` — find execution flows related to the issue
|
||||
2. `gitnexus_context({name: "<suspect function>"})` — see all callers, callees, and process participation
|
||||
3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step
|
||||
4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed
|
||||
|
||||
## When Refactoring
|
||||
|
||||
- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`.
|
||||
- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code.
|
||||
- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed.
|
||||
|
||||
## Never Do
|
||||
|
||||
- NEVER edit a function, class, or method without first running `gitnexus_impact` on it.
|
||||
- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis.
|
||||
- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph.
|
||||
- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope.
|
||||
|
||||
## Tools Quick Reference
|
||||
|
||||
| Tool | When to use | Command |
|
||||
|------|-------------|---------|
|
||||
| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` |
|
||||
| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` |
|
||||
| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` |
|
||||
| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` |
|
||||
| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` |
|
||||
| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` |
|
||||
|
||||
## Impact Risk Levels
|
||||
|
||||
| Depth | Meaning | Action |
|
||||
|-------|---------|--------|
|
||||
| d=1 | WILL BREAK — direct callers/importers | MUST update these |
|
||||
| d=2 | LIKELY AFFECTED — indirect deps | Should test |
|
||||
| d=3 | MAY NEED TESTING — transitive | Test if critical path |
|
||||
|
||||
## Resources
|
||||
|
||||
| Resource | Use for |
|
||||
|----------|---------|
|
||||
| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness |
|
||||
| `gitnexus://repo/GitNexus/clusters` | All functional areas |
|
||||
| `gitnexus://repo/GitNexus/processes` | All execution flows |
|
||||
| `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace |
|
||||
|
||||
## Self-Check Before Finishing
|
||||
|
||||
Before completing any code modification task, verify:
|
||||
1. `gitnexus_impact` was run for all modified symbols
|
||||
2. No HIGH/CRITICAL risk warnings were ignored
|
||||
3. `gitnexus_detect_changes()` confirms changes match expected scope
|
||||
4. All d=1 (WILL BREAK) dependents were updated
|
||||
|
||||
## Keeping the Index Fresh
|
||||
|
||||
After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it:
|
||||
|
||||
```bash
|
||||
npx gitnexus analyze
|
||||
```
|
||||
|
||||
If the index previously included embeddings, preserve them by adding `--embeddings`:
|
||||
|
||||
```bash
|
||||
npx gitnexus analyze --embeddings
|
||||
```
|
||||
|
||||
To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.**
|
||||
|
||||
> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`.
|
||||
|
||||
## CLI
|
||||
|
||||
| Task | Read this skill file |
|
||||
|------|---------------------|
|
||||
| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
|
||||
| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
|
||||
| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
|
||||
| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
|
||||
| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
|
||||
| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
|
||||
|
||||
<!-- gitnexus:end -->
|
||||
See the `<!-- gitnexus:start --> … <!-- gitnexus:end -->` block in **[AGENTS.md](AGENTS.md)** for the canonical MCP tools, impact analysis rules, and index instructions.
|
||||
|
|
|
|||
119
CONTRIBUTING.md
119
CONTRIBUTING.md
|
|
@ -21,30 +21,127 @@ This project uses the [PolyForm Noncommercial License 1.0.0](https://polyformpro
|
|||
## Branch and pull requests
|
||||
|
||||
- Use short-lived branches off the default branch of the repo you are targeting.
|
||||
- Prefer **conventional commits** (short prefix + description), for example:
|
||||
|
||||
```text
|
||||
feat: add graph export option
|
||||
fix: correct MCP tool schema for query
|
||||
test: cover cluster merge edge case
|
||||
docs: clarify analyze flags
|
||||
```
|
||||
|
||||
- **PR title:** `[area] Short description` (e.g. `[cli] Fix index refresh race`).
|
||||
- **PR titles MUST follow the conventional-commit format** — `pr-labeler.yml` enforces this on every PR and auto-applies the matching label so release notes group the change correctly.
|
||||
- **PR description:** what changed, why, how to verify (commands), and any risk or rollback notes.
|
||||
|
||||
### Pull request titles
|
||||
|
||||
Format: `<type>[(scope)][!]: <subject>`
|
||||
|
||||
Allowed types and the release-notes section each one lands in (defined in `.github/release.yml`):
|
||||
|
||||
| Type | Label applied | Release-notes section |
|
||||
|------|---------------|-----------------------|
|
||||
| `feat` | `enhancement` | 🚀 Features |
|
||||
| `fix` | `bug` | 🐛 Bug Fixes |
|
||||
| `perf` | `performance` | 🏎️ Performance |
|
||||
| `refactor` | `refactor` | 🔄 Refactoring |
|
||||
| `test` | `test` | 🧪 Tests |
|
||||
| `ci` | `ci` | 👷 CI/CD |
|
||||
| `build` / `deps` | `dependencies` | 📦 Dependencies |
|
||||
| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) |
|
||||
| `chore` / `revert` | `chore` | (excluded from release notes) |
|
||||
|
||||
Append `!` to the type (e.g. `feat(api)!: drop /v1 endpoint`) or include `BREAKING CHANGE:` in the PR body to flag a breaking change — the labeler then adds the `breaking` label and the 💥 Breaking Changes section is rendered first.
|
||||
|
||||
Examples:
|
||||
|
||||
```text
|
||||
feat(web): add smart chat scroll
|
||||
fix(extractors): resolve silent contract mis-resolution
|
||||
perf: avoid O(n²) traversal in heritage walker
|
||||
chore(deps): bump vitest to 3.0.0
|
||||
ci: standardize workflow concurrency
|
||||
```
|
||||
|
||||
Commits within a PR may use any style — only the **merged PR title** shows up in release notes, so that's the one the convention applies to.
|
||||
|
||||
## Before you open a PR
|
||||
|
||||
- [ ] Tests pass for the packages you touched (`gitnexus` and/or `gitnexus-web`).
|
||||
- [ ] Typecheck passes: `npx tsc --noEmit` in `gitnexus/` and `npx tsc -b --noEmit` in `gitnexus-web/`.
|
||||
- [ ] No secrets, tokens, or machine-specific paths committed.
|
||||
- [ ] Documentation updated if behavior or public CLI/MCP contract changes.
|
||||
- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — typecheck + unit tests for staged packages).
|
||||
- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — formatting via lint-staged + typecheck for staged packages; tests run in CI only).
|
||||
|
||||
## Code review
|
||||
|
||||
Maintainers may request changes for correctness, tests, performance, or consistency with existing patterns. Keeping diffs focused makes review faster.
|
||||
|
||||
## GitHub Actions — Concurrency Convention
|
||||
|
||||
Every workflow under `.github/workflows/` MUST declare a top-level `concurrency:` block using this convention:
|
||||
|
||||
- **Group key** starts with `${{ github.workflow }}` so no two workflows can collide on the same group name. The discriminator that follows is chosen per event shape:
|
||||
- Branch/tag scope: `${{ github.workflow }}-${{ github.ref }}`
|
||||
- Per-PR scope (for `issue_comment`, `pull_request_review*`, `pull_request` meta events): `${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}`
|
||||
- `workflow_run` scope (e.g. `ci-report.yml`): `${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}` — the fork fallback must be stable across reruns (never `workflow_run.id`, which is per-run-unique and defeats serialization).
|
||||
- Global single-slot (manual dispatch utilities): `${{ github.workflow }}`
|
||||
- **Reusable workflows invoked via `workflow_call`:** do NOT use `${{ github.workflow }}` in the group key — in called-workflow context its evaluation is ambiguous and can resolve to the caller's name, which would deadlock against the caller's own group. Use a hardcoded literal prefix and a `github.event_name`-aware expression that falls through to `github.run_id` for reusable invocations (see `ci.yml` for the canonical form).
|
||||
- **Merge queue (`merge_group`)**: when this event is added, use `${{ github.workflow }}-${{ github.event.merge_group.head_ref }}` with `cancel-in-progress: false` (every queue entry is a distinct ref; never cancel).
|
||||
- **`cancel-in-progress` policy:**
|
||||
|
||||
| Event | `cancel-in-progress` | Why |
|
||||
|-------|----------------------|-----|
|
||||
| `pull_request` CI run | `true` | New push supersedes old run |
|
||||
| `push` to `main` | `false` | Every main commit gets validated |
|
||||
| Tag push (`v*` publish) | `false` | Never cancel mid-publish |
|
||||
| `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish |
|
||||
| `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional |
|
||||
| `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race |
|
||||
| Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR |
|
||||
| PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins |
|
||||
| Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes |
|
||||
|
||||
- For workflows that serve multiple events at once (e.g. `ci.yml` handles `pull_request`, `push`, and `workflow_call`), make `cancel-in-progress` event-aware:
|
||||
|
||||
```yaml
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
```
|
||||
|
||||
- When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape.
|
||||
|
||||
## AI-assisted contributions
|
||||
|
||||
If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes.
|
||||
|
||||
## Releases
|
||||
|
||||
Two publish workflows ship `gitnexus` to npm:
|
||||
|
||||
- **Stable** (`.github/workflows/publish.yml`) — triggered by pushing any `v*`
|
||||
tag. Publishes to the `latest` dist-tag with a changelog-backed GitHub
|
||||
release. Maintainers are expected to tag from `main` as a convention; the
|
||||
workflow itself does not enforce branch reachability.
|
||||
- **Release Candidate** (`.github/workflows/release-candidate.yml`) — runs on
|
||||
every push to `main` (typically a merged PR) plus manual dispatch. Docs-only
|
||||
changes are skipped via `paths-ignore`. Publishes to the `rc` dist-tag with
|
||||
version `X.Y.Z-rc.N` and a GitHub prerelease, where:
|
||||
- `X.Y.Z` is selected automatically. On push (and on dispatch with
|
||||
`bump: auto`, the default) the workflow **continues the active rc cycle**:
|
||||
if the registry already has `X.Y.Z-rc.*` versions with `X.Y.Z` > current
|
||||
`latest`, it reuses the highest such base; otherwise it patch-bumps
|
||||
from `latest`. Dispatching with `bump: patch|minor|major` **resets**
|
||||
the cycle from `latest`.
|
||||
- `N` is auto-incremented against existing `X.Y.Z-rc.*` entries on the
|
||||
registry. First rc for a given base is `rc.1`.
|
||||
|
||||
Idempotency: the workflow pushes an `rc/<HEAD_SHA>` marker tag and a
|
||||
`v<RC>` release tag **atomically, before** calling `npm publish`. The guard
|
||||
refuses to re-run once the marker exists, so a post-publish failure will
|
||||
not mint a duplicate rc for the same commit. The `v<RC>` tag points at a
|
||||
detached release commit whose `package.json` matches the npm tarball
|
||||
exactly (traceable releases). Recovery after a partial failure:
|
||||
|
||||
```bash
|
||||
git push --delete origin rc/<HEAD_SHA> v<RC>
|
||||
# then redispatch the workflow with force: true
|
||||
```
|
||||
|
||||
The rc workflow never moves `latest`. To verify after a change, inspect dist-tags:
|
||||
|
||||
```bash
|
||||
npm view gitnexus dist-tags
|
||||
```
|
||||
|
|
|
|||
|
|
@ -1,72 +1,69 @@
|
|||
# Guardrails — GitNexus (repo + agents)
|
||||
# Guardrails — GitNexus
|
||||
|
||||
Rules for **human contributors** and **AI agents** working on this codebase or publishing artifacts. These complement `AGENTS.md` / `CLAUDE.md` (which focus on GitNexus-in-GitNexus workflows).
|
||||
Rules for **human contributors** and **AI agents**. Complements `AGENTS.md` (workflows) and `CONTRIBUTING.md` (PR process).
|
||||
|
||||
## Scope (typical agent session)
|
||||
## Scope (least privilege)
|
||||
|
||||
When automating changes in this repository, treat scope as **least privilege**:
|
||||
- **Read:** Source, tests, docs, public config as needed.
|
||||
- **Write:** Only files required for the fix or feature; no unrelated formatting or refactors.
|
||||
- **Execute:** Tests, typecheck, documented CLI commands. No destructive commands on user data without approval.
|
||||
- **Off-limits:** Other people's machines, production deployments you don't own, credentials you lack permission to use.
|
||||
|
||||
- **Read:** Source, tests, docs, public config as needed for the task.
|
||||
- **Write:** Only files required for the requested fix or feature; avoid unrelated formatting or refactors.
|
||||
- **Execute:** Tests, typecheck, and documented CLI commands; do not run destructive commands on user data outside the repo without explicit approval.
|
||||
- **Off-limits:** Other people’s machines, production deployments you don’t own, and credentials you didn’t receive permission to use.
|
||||
|
||||
Adjust explicitly if the maintainer defines a different scope for a task.
|
||||
Maintainer may widen scope per task.
|
||||
|
||||
---
|
||||
|
||||
## Non-negotiables
|
||||
|
||||
1. **Never commit secrets** — API keys, tokens, `.env` with real values, private URLs, or session cookies. Use `.env.example` with placeholders only.
|
||||
2. **Never rename symbols with blind find-and-replace** when working in a GitNexus-indexed project — use the **`rename` MCP tool** with **`dry_run: true` first**, then review `graph` vs `text_search` edits. (There is no separate `gitnexus rename` CLI; renaming goes through MCP or editor integration.)
|
||||
3. **Run impact analysis before editing shared symbols** — use **`impact`** (upstream) for functions/classes/methods others call; do not ignore **HIGH** / **CRITICAL** risk without maintainer sign-off.
|
||||
4. **Prefer `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available.
|
||||
5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, run `npx gitnexus analyze --embeddings` when refreshing the index; plain `analyze` can drop them.
|
||||
1. **Never commit secrets** — API keys, tokens, real `.env` values, private URLs, session cookies. Use `.env.example` with placeholders.
|
||||
2. **Never rename with find-and-replace** in GitNexus-indexed projects — use `rename` MCP tool with `dry_run: true` first, review `graph` vs `text_search` edits. No separate `gitnexus rename` CLI exists.
|
||||
3. **Run impact analysis before editing shared symbols** — `impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off.
|
||||
4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available.
|
||||
5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, use `npx gitnexus analyze --embeddings`; plain `analyze` drops them.
|
||||
|
||||
---
|
||||
|
||||
## Signs (recurring failure patterns)
|
||||
|
||||
Use this format: **Trigger → Instruction → Reason**.
|
||||
Append new Signs here when the same mistake repeats (e.g. CI broken twice the same way).
|
||||
Format: **Trigger → Instruction → Reason**. Append new Signs when the same mistake repeats.
|
||||
|
||||
### Sign: Stale graph after edits
|
||||
### Stale graph after edits
|
||||
|
||||
- **Trigger:** MCP or resources warn the index is behind `HEAD`, or code search doesn’t match latest commit.
|
||||
- **Instruction:** Run `npx gitnexus analyze` from the repo root (plus `--embeddings` if the project used them).
|
||||
- **Reason:** Tools query LadybugDB built at last analyze; git changes are invisible until re-indexed.
|
||||
- **Trigger:** MCP warns index is behind `HEAD`, or search doesn't match latest commit.
|
||||
- **Do:** `npx gitnexus analyze` (plus `--embeddings` if used).
|
||||
- **Why:** Tools query LadybugDB from last analyze; git changes are invisible until re-indexed.
|
||||
|
||||
### Sign: Embeddings vanished after analyze
|
||||
### Embeddings vanished after analyze
|
||||
|
||||
- **Trigger:** Semantic search quality drops; `stats.embeddings` in `.gitnexus/meta.json` is 0 after a refresh.
|
||||
- **Instruction:** Re-run `npx gitnexus analyze --embeddings` and confirm `meta.json` reflects stored embeddings.
|
||||
- **Reason:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors.
|
||||
- **Trigger:** Semantic search quality drops; `stats.embeddings` in `meta.json` is 0 after refresh.
|
||||
- **Do:** `npx gitnexus analyze --embeddings`, confirm `meta.json` reflects stored embeddings.
|
||||
- **Why:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors.
|
||||
|
||||
### Sign: MCP lists no repos
|
||||
### MCP lists no repos
|
||||
|
||||
- **Trigger:** MCP stderr says no indexed repos.
|
||||
- **Instruction:** Run `npx gitnexus analyze` in the target repository; verify `npx gitnexus list` shows it.
|
||||
- **Reason:** The MCP server discovers repos via `~/.gitnexus/registry.json`, populated by analyze.
|
||||
- **Trigger:** MCP stderr says no indexed repos.
|
||||
- **Do:** `npx gitnexus analyze` in the target repo; verify `npx gitnexus list` shows it.
|
||||
- **Why:** MCP discovers repos via `~/.gitnexus/registry.json`, populated by analyze.
|
||||
|
||||
### Sign: Wrong repo in multi-repo setups
|
||||
### Wrong repo in multi-repo setups
|
||||
|
||||
- **Trigger:** Query/impact results clearly belong to another project.
|
||||
- **Instruction:** Call `list_repos`, then pass **`repo`** on subsequent tools (or use per-workspace MCP config).
|
||||
- **Reason:** Default target may be ambiguous when multiple repos are registered.
|
||||
- **Trigger:** Query/impact results belong to another project.
|
||||
- **Do:** Call `list_repos`, then pass `repo` on subsequent tools.
|
||||
- **Why:** Default target is ambiguous when multiple repos are registered.
|
||||
|
||||
### Sign: LadybugDB lock / “database busy”
|
||||
### LadybugDB lock / "database busy"
|
||||
|
||||
- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run.
|
||||
- **Instruction:** Stop overlapping processes; one writer at a time. Retry analyze or restart MCP.
|
||||
- **Reason:** Embedded DB expects single-process ownership of the store.
|
||||
- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run.
|
||||
- **Do:** Stop overlapping processes (one writer at a time). Retry analyze or restart MCP.
|
||||
- **Why:** Embedded DB expects single-process ownership.
|
||||
|
||||
---
|
||||
|
||||
## Publishing & supply chain
|
||||
|
||||
- **npm:** Do not publish from unreviewed automation; follow maintainer release process. Bump version intentionally; tag releases to match `package.json`.
|
||||
- **Dependencies:** Prefer minimal, auditable changes to `package.json`; run tests and CI after lockfile updates.
|
||||
- **License:** This project ships under **PolyForm Noncommercial 1.0.0** — do not relicense or imply a different license in docs or metadata without maintainer approval.
|
||||
- **npm:** Do not publish from unreviewed automation. Bump version intentionally; tag releases to match `package.json`.
|
||||
- **Dependencies:** Minimal, auditable `package.json` changes; run tests and CI after lockfile updates.
|
||||
- **License:** PolyForm Noncommercial 1.0.0 — do not relicense without maintainer approval.
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -74,15 +71,15 @@ Append new Signs here when the same mistake repeats (e.g. CI broken twice the sa
|
|||
|
||||
Stop and ask a **human maintainer** when:
|
||||
|
||||
- Impact analysis shows **HIGH** / **CRITICAL** risk and the task still requires the change.
|
||||
- You need to alter **CI**, **release**, or **security-sensitive** config.
|
||||
- Requirements conflict (e.g. “speed up analyze” vs “must keep all embeddings on huge repo”).
|
||||
- Impact analysis shows HIGH/CRITICAL risk and the task still requires the change.
|
||||
- You need to alter CI, release, or security-sensitive config.
|
||||
- Requirements conflict (e.g. "speed up analyze" vs "must keep all embeddings on huge repo").
|
||||
- You are unsure whether data loss is acceptable (`clean`, forced migrations, schema changes).
|
||||
|
||||
---
|
||||
|
||||
## Related docs
|
||||
|
||||
- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow.
|
||||
- [RUNBOOK.md](RUNBOOK.md) — commands for recovery.
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations.
|
||||
- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow
|
||||
- [RUNBOOK.md](RUNBOOK.md) — commands for recovery
|
||||
- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations
|
||||
|
|
|
|||
13
TESTING.md
13
TESTING.md
|
|
@ -20,9 +20,9 @@ From repository root, unless noted:
|
|||
cd gitnexus
|
||||
npm install
|
||||
npm run build
|
||||
npm test # unit: vitest run test/unit
|
||||
npm test # full suite: vitest run
|
||||
npm run test:unit # unit only: vitest run test/unit
|
||||
npm run test:integration # integration suite
|
||||
npm run test:all
|
||||
npm run test:coverage
|
||||
npx tsc --noEmit # typecheck (matches CI)
|
||||
```
|
||||
|
|
@ -42,8 +42,11 @@ npm run test:e2e # Playwright (requires gitnexus serve + npm run dev)
|
|||
|
||||
A husky pre-commit hook (`.husky/pre-commit`) runs automatically on every `git commit`:
|
||||
|
||||
- **`gitnexus-web/` files staged** → `tsc -b --noEmit` + `vitest run`
|
||||
- **`gitnexus/` files staged** → `tsc --noEmit` + `vitest run --project default`
|
||||
1. **Formatting** — `lint-staged` runs prettier on staged files
|
||||
2. **`gitnexus-web/` files staged** → `tsc -b --noEmit`
|
||||
3. **`gitnexus/` files staged** → `tsc --noEmit`
|
||||
|
||||
Tests do **not** run in the pre-commit hook — they run in CI (`ci-tests.yml`) only.
|
||||
|
||||
Skip with `git commit --no-verify` (use sparingly).
|
||||
|
||||
|
|
@ -77,7 +80,7 @@ Re-run the full relevant suite when:
|
|||
|
||||
GitHub Actions (`.github/workflows/ci.yml`) orchestrate:
|
||||
|
||||
- **`ci-quality.yml`** — `tsc --noEmit` for `gitnexus/` + `tsc -b --noEmit` for `gitnexus-web/`
|
||||
- **`ci-quality.yml`** — prettier format check, eslint lint, `tsc --noEmit` for `gitnexus/`, `tsc -b --noEmit` for `gitnexus-web/`
|
||||
- **`ci-tests.yml`** — `vitest run` with coverage (ubuntu) + cross-platform (macOS, Windows)
|
||||
- **`ci-e2e.yml`** — Playwright E2E tests, gated on `gitnexus-web/**` changes
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ export type { NodeTableName, RelType } from './lbug/schema-constants.js';
|
|||
// Language support
|
||||
export { SupportedLanguages } from './languages.js';
|
||||
export { getLanguageFromFilename, getSyntaxLanguageFromFilename } from './language-detection.js';
|
||||
export type { MroStrategy } from './mro-strategy.js';
|
||||
|
||||
// Pipeline progress
|
||||
export type { PipelinePhase, PipelineProgress } from './pipeline.js';
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ export const NODE_TABLES = [
|
|||
'TypeAlias',
|
||||
'Const',
|
||||
'Static',
|
||||
'Variable',
|
||||
'Property',
|
||||
'Record',
|
||||
'Delegate',
|
||||
|
|
|
|||
46
gitnexus-shared/src/mro-strategy.ts
Normal file
46
gitnexus-shared/src/mro-strategy.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
/**
|
||||
* MRO (Method Resolution Order) strategy — shared canonical definition.
|
||||
*
|
||||
* Lives in `gitnexus-shared` so `model/resolve.ts` and `mro-processor.ts` share
|
||||
* the type without importing the language registry (avoids circular coupling).
|
||||
*
|
||||
* `first-wins` (default, Java/C#/Kotlin/Go/Swift/Dart):
|
||||
* BFS ancestor walk in declaration order; first match wins.
|
||||
*
|
||||
* `leftmost-base` (C++):
|
||||
* BFS walk; HeritageMap preserves source insertion order, so BFS naturally
|
||||
* picks the leftmost base in diamond inheritance.
|
||||
*
|
||||
* `c3` (Python):
|
||||
* C3-linearization; falls back to BFS on cyclic/inconsistent hierarchy.
|
||||
* See model/resolve.ts § c3Linearize.
|
||||
*
|
||||
* `implements-split` (Java/C#/Kotlin):
|
||||
* Low-level lookup is BFS; graph-level mro-processor detects and warns on
|
||||
* interface-default method ambiguity.
|
||||
*
|
||||
* `qualified-syntax` (Rust):
|
||||
* No auto-resolution — `lookupMethodByOwnerWithMRO` returns undefined immediately.
|
||||
* Rust requires explicit `<Type as Trait>::method` syntax.
|
||||
*
|
||||
* `ruby-mixin` (Ruby):
|
||||
* Kind-aware walk that does NOT short-circuit on direct owner first (`prepend`
|
||||
* must beat the class's own method). Walk order:
|
||||
* 1. Prepend providers (reverse declaration — last-prepended wins)
|
||||
* 2. Direct owner's own methods
|
||||
* 3. Include providers (reverse declaration)
|
||||
* 4. Transitive ancestors (BFS fallback)
|
||||
* Singleton dispatch: caller passes `ancestryOverride` (extend providers only);
|
||||
* becomes a simple left-to-right scan. Miss NEVER falls through to file-scoped
|
||||
* lookup — null-routes or honors `fallback`.
|
||||
*
|
||||
* @see model/resolve.ts § lookupMethodByOwnerWithMRO
|
||||
* @see languages/ruby.ts § selectDispatch
|
||||
*/
|
||||
export type MroStrategy =
|
||||
| 'first-wins'
|
||||
| 'c3'
|
||||
| 'leftmost-base'
|
||||
| 'implements-split'
|
||||
| 'qualified-syntax'
|
||||
| 'ruby-mixin';
|
||||
168
gitnexus-web/e2e/repo-switching.spec.ts
Normal file
168
gitnexus-web/e2e/repo-switching.spec.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
import { test, expect } from '@playwright/test';
|
||||
|
||||
/**
|
||||
* E2E tests for the repo-switching and false-404 fixes.
|
||||
*
|
||||
* Most tests use the live backend (same pattern as multi-repo-scoping.spec.ts).
|
||||
* The 503 hold-queue test uses route interception to simulate a slow analysis.
|
||||
*/
|
||||
|
||||
const BACKEND_URL = process.env.BACKEND_URL ?? 'http://localhost:4747';
|
||||
const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:5173';
|
||||
|
||||
let firstRepoName: string;
|
||||
|
||||
test.beforeAll(async () => {
|
||||
if (process.env.E2E) {
|
||||
try {
|
||||
const res = await fetch(`${BACKEND_URL}/api/repos`);
|
||||
const repos = await res.json();
|
||||
firstRepoName = repos[0]?.name ?? '';
|
||||
} catch {
|
||||
firstRepoName = '';
|
||||
}
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const [backendRes, frontendRes] = await Promise.allSettled([
|
||||
fetch(`${BACKEND_URL}/api/repos`),
|
||||
fetch(FRONTEND_URL),
|
||||
]);
|
||||
if (
|
||||
backendRes.status === 'rejected' ||
|
||||
(backendRes.status === 'fulfilled' && !backendRes.value.ok)
|
||||
) {
|
||||
test.skip(true, 'gitnexus serve not available');
|
||||
return;
|
||||
}
|
||||
if (
|
||||
frontendRes.status === 'rejected' ||
|
||||
(frontendRes.status === 'fulfilled' && !frontendRes.value.ok)
|
||||
) {
|
||||
test.skip(true, 'Vite dev server not available');
|
||||
return;
|
||||
}
|
||||
if (backendRes.status === 'fulfilled') {
|
||||
const repos = await backendRes.value.json();
|
||||
if (!repos.length) {
|
||||
test.skip(true, 'No indexed repos');
|
||||
return;
|
||||
}
|
||||
firstRepoName = repos[0].name;
|
||||
}
|
||||
} catch {
|
||||
test.skip(true, 'servers not available');
|
||||
}
|
||||
});
|
||||
|
||||
// ── 1. Hold-queue: 503 → descriptive user message ────────────────────────────
|
||||
|
||||
test.describe('Hold-queue timeout error', () => {
|
||||
test('shows descriptive message when /api/repo returns 503', async ({ page }, testInfo) => {
|
||||
// Intercept only /api/repo (singular) — not /api/repos — to return a 503
|
||||
// regex: /api/repo followed by end, ?, or # — NOT /api/repos
|
||||
await page.route(/\/api\/repo(?!s)(\?.*)?$/, (route) =>
|
||||
route.fulfill({
|
||||
status: 503,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
error: `Repository analysis for "${firstRepoName}" is taking longer than expected. Please try again in a moment.`,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
|
||||
|
||||
// UI should show the 503 error message
|
||||
await expect(page.getByText(/taking longer than expected/i)).toBeVisible({
|
||||
timeout: 20_000,
|
||||
});
|
||||
|
||||
await page.screenshot({ path: testInfo.outputPath('hold-queue-503.png') });
|
||||
});
|
||||
});
|
||||
|
||||
// ── 2. ?project= URL persistence ─────────────────────────────────────────────
|
||||
|
||||
test.describe('?project= URL persistence', () => {
|
||||
test('?project= is set in URL after connecting via ?server=', async ({ page }) => {
|
||||
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
|
||||
|
||||
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
const url = new URL(page.url());
|
||||
const project = url.searchParams.get('project');
|
||||
expect(project).toBeTruthy();
|
||||
// first repo returned by the live backend
|
||||
if (firstRepoName) expect(project).toBe(firstRepoName);
|
||||
});
|
||||
|
||||
test('?project= is still present after F5 reload', async ({ page }) => {
|
||||
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
|
||||
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
// After connect, URL has ?server=&project= — F5 re-uses both params
|
||||
await page.reload();
|
||||
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
const url = new URL(page.url());
|
||||
expect(url.searchParams.get('project')).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
// ── 3. ?project= + ?server= combined auto-connect ────────────────────────────
|
||||
|
||||
test.describe('?project= auto-connect', () => {
|
||||
test('navigating with ?server=&project= connects to the correct repo', async ({
|
||||
page,
|
||||
}, testInfo) => {
|
||||
if (!firstRepoName) test.skip(true, 'no repo name available');
|
||||
|
||||
await page.goto(
|
||||
`/?server=${encodeURIComponent(BACKEND_URL)}&project=${encodeURIComponent(firstRepoName)}`,
|
||||
);
|
||||
|
||||
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
// ?project= in URL should match what we passed in
|
||||
const url = new URL(page.url());
|
||||
expect(url.searchParams.get('project')).toBe(firstRepoName);
|
||||
|
||||
await page.screenshot({ path: testInfo.outputPath('project-param-connect.png') });
|
||||
});
|
||||
});
|
||||
|
||||
// ── 4. Windows path normalization ─────────────────────────────────────────────
|
||||
|
||||
test.describe('Windows path normalization', () => {
|
||||
test('project name uses basename when /api/repo returns a Windows-style repoPath', async ({
|
||||
page,
|
||||
}) => {
|
||||
const repoName = firstRepoName || 'test-repo';
|
||||
const windowsPath = `C:\\Users\\LENOVO\\.gitnexus\\repos\\${repoName}`;
|
||||
|
||||
// Mock /api/repo to return a Windows backslash path while keeping name correct
|
||||
await page.route(/\/api\/repo(?!s)(\?.*)?$/, (route) =>
|
||||
route.fulfill({
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
// intentionally omit `name` to force path-based extraction
|
||||
path: windowsPath,
|
||||
repoPath: windowsPath,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
|
||||
|
||||
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
// URL ?project= must be the short basename, NOT the full Windows path
|
||||
const url = new URL(page.url());
|
||||
const project = url.searchParams.get('project');
|
||||
expect(project).toBeTruthy();
|
||||
expect(project).not.toContain('\\');
|
||||
expect(project).not.toContain('LENOVO');
|
||||
expect(project).toBe(repoName);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { test, expect, type TestInfo } from '@playwright/test';
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
/**
|
||||
* E2E tests for the GitNexus web UI — exploring view features.
|
||||
|
|
@ -58,36 +58,41 @@ test.beforeAll(async () => {
|
|||
* For these tests we require at least one indexed repo, so pick the first
|
||||
* landing card when present and then wait for the exploring view.
|
||||
*/
|
||||
async function waitForGraphLoaded(page: import('@playwright/test').Page, testInfo: TestInfo) {
|
||||
async function waitForGraphLoaded(page: import('@playwright/test').Page) {
|
||||
await page.goto('/');
|
||||
|
||||
const landingCard = page.locator('[data-testid="landing-repo-card"]').first();
|
||||
const landingCards = page.locator('[data-testid="landing-repo-card"]');
|
||||
const preferredLandingCard = landingCards
|
||||
.filter({ hasText: /GitNexus|local-integration/ })
|
||||
.first();
|
||||
try {
|
||||
await landingCard.waitFor({ state: 'visible', timeout: 15_000 });
|
||||
await landingCards.first().waitFor({ state: 'visible', timeout: 15_000 });
|
||||
const landingCard =
|
||||
(await preferredLandingCard.count()) > 0 ? preferredLandingCard : landingCards.first();
|
||||
await landingCard.click();
|
||||
} catch {
|
||||
// Landing screen may not appear (e.g. ?server auto-connect)
|
||||
}
|
||||
|
||||
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
|
||||
await expect(page.getByText(/\d+ nodes/).first()).toBeVisible();
|
||||
await page.screenshot({ path: testInfo.outputPath('graph-loaded.png') });
|
||||
const statusBar = page.getByRole('contentinfo');
|
||||
await expect(statusBar.getByText('Ready', { exact: true })).toBeVisible({ timeout: 45_000 });
|
||||
await expect(statusBar).toContainText(/nodes/, {
|
||||
timeout: 20_000,
|
||||
});
|
||||
}
|
||||
|
||||
test.describe('Server Connection & Graph Loading', () => {
|
||||
test('selects a repo from landing and loads graph', async ({ page }, testInfo) => {
|
||||
await waitForGraphLoaded(page, testInfo);
|
||||
await page.screenshot({ path: testInfo.outputPath('graph-loaded-full.png'), fullPage: true });
|
||||
test('selects a repo from landing and loads graph', async ({ page }) => {
|
||||
await waitForGraphLoaded(page);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Nexus AI', () => {
|
||||
test('panel opens and agent initializes without error', async ({ page }, testInfo) => {
|
||||
await waitForGraphLoaded(page, testInfo);
|
||||
test('panel opens and agent initializes without error', async ({ page }) => {
|
||||
await waitForGraphLoaded(page);
|
||||
|
||||
await page.getByRole('button', { name: 'Nexus AI' }).click();
|
||||
await expect(page.getByText('Ask me anything')).toBeVisible({ timeout: 15_000 });
|
||||
await page.screenshot({ path: testInfo.outputPath('nexus-ai-panel.png'), fullPage: true });
|
||||
|
||||
const errorBanner = page.getByText('Database not ready');
|
||||
expect(await errorBanner.isVisible().catch(() => false)).toBe(false);
|
||||
|
|
@ -95,8 +100,8 @@ test.describe('Nexus AI', () => {
|
|||
});
|
||||
|
||||
test.describe('Processes Panel', () => {
|
||||
test('shows process list and View button works', async ({ page }, testInfo) => {
|
||||
await waitForGraphLoaded(page, testInfo);
|
||||
test('shows process list and View button works', async ({ page }) => {
|
||||
await waitForGraphLoaded(page);
|
||||
|
||||
await page.getByRole('button', { name: 'Nexus AI' }).click();
|
||||
await page.getByText('Processes').click();
|
||||
|
|
@ -104,7 +109,6 @@ test.describe('Processes Panel', () => {
|
|||
await expect(page.locator('[data-testid="process-list-loaded"]')).toBeVisible({
|
||||
timeout: 15_000,
|
||||
});
|
||||
await page.screenshot({ path: testInfo.outputPath('processes-panel.png'), fullPage: true });
|
||||
|
||||
const processRow = page.locator('[data-testid="process-row"]').first();
|
||||
await expect(processRow).toBeVisible({ timeout: 10_000 });
|
||||
|
|
@ -114,14 +118,10 @@ test.describe('Processes Panel', () => {
|
|||
await viewBtn.waitFor({ state: 'visible', timeout: 5_000 });
|
||||
await viewBtn.click();
|
||||
await expect(page.locator('[data-testid="process-modal"]')).toBeVisible({ timeout: 5_000 });
|
||||
await page.screenshot({
|
||||
path: testInfo.outputPath('process-view-clicked.png'),
|
||||
fullPage: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('lightbulb highlights nodes in graph', async ({ page }, testInfo) => {
|
||||
await waitForGraphLoaded(page, testInfo);
|
||||
test('lightbulb highlights nodes in graph', async ({ page }) => {
|
||||
await waitForGraphLoaded(page);
|
||||
|
||||
await page.getByRole('button', { name: 'Nexus AI' }).click();
|
||||
await page.getByText('Processes').click();
|
||||
|
|
@ -137,13 +137,12 @@ test.describe('Processes Panel', () => {
|
|||
await lightbulb.waitFor({ state: 'visible', timeout: 5_000 });
|
||||
await lightbulb.click();
|
||||
await expect(processRow).toHaveClass(/bg-amber-950/, { timeout: 5_000 });
|
||||
await page.screenshot({ path: testInfo.outputPath('after-highlight.png'), fullPage: true });
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('Turn Off All Highlights', () => {
|
||||
test('selecting a node dims others, button clears it', async ({ page }, testInfo) => {
|
||||
await waitForGraphLoaded(page, testInfo);
|
||||
test('selecting a node dims others, button clears it', async ({ page }) => {
|
||||
await waitForGraphLoaded(page);
|
||||
|
||||
await expect(page.locator('canvas').first()).toBeVisible({ timeout: 10_000 });
|
||||
|
||||
|
|
@ -160,6 +159,5 @@ test.describe('Turn Off All Highlights', () => {
|
|||
await expect(highlightToggle).toHaveAttribute('title', 'Turn on AI highlights', {
|
||||
timeout: 5_000,
|
||||
});
|
||||
await page.screenshot({ path: testInfo.outputPath('highlights-cleared.png'), fullPage: true });
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -56,16 +56,14 @@ const AppContent = () => {
|
|||
// backend calls (queries, search, grep, readFile) scope to this repo.
|
||||
const repoName = result.repoInfo.name;
|
||||
const repoPath = result.repoInfo.repoPath ?? result.repoInfo.path;
|
||||
// Normalize both Windows (\) and Unix (/) path separators before splitting
|
||||
const projectName =
|
||||
repoName || repoPath?.split('/').filter(Boolean).pop() || 'server-project';
|
||||
result.repoInfo.name ||
|
||||
(repoPath || '').replace(/\\/g, '/').split('/').filter(Boolean).pop() ||
|
||||
'server-project';
|
||||
setProjectName(projectName);
|
||||
setCurrentRepo(projectName);
|
||||
|
||||
// Update URL so F5 / bookmarks preserve which repo is open
|
||||
const url = new URL(window.location.href);
|
||||
url.searchParams.set('project', projectName);
|
||||
window.history.replaceState(null, '', url.toString());
|
||||
|
||||
// Build KnowledgeGraph from server data for visualization
|
||||
const graph = createKnowledgeGraph();
|
||||
for (const node of result.nodes) {
|
||||
|
|
@ -76,6 +74,11 @@ const AppContent = () => {
|
|||
}
|
||||
setGraph(graph);
|
||||
|
||||
// Persist the active project in the URL for bookmarkability and F5 refresh resilience
|
||||
const urlObj = new URL(window.location.href);
|
||||
urlObj.searchParams.set('project', projectName);
|
||||
window.history.replaceState(null, '', urlObj.toString());
|
||||
|
||||
// Transition directly to exploring view
|
||||
setViewMode('exploring');
|
||||
|
||||
|
|
@ -99,22 +102,17 @@ const AppContent = () => {
|
|||
],
|
||||
);
|
||||
|
||||
// Auto-connect when ?server query param is present (bookmarkable shortcut).
|
||||
// Also reads ?project= to connect to a specific repo.
|
||||
// Auto-connect when ?server or ?project query param is present (bookmarkable shortcut)
|
||||
const autoConnectRan = useRef(false);
|
||||
useEffect(() => {
|
||||
if (autoConnectRan.current) return;
|
||||
const params = new URLSearchParams(window.location.search);
|
||||
if (!params.has('server')) return;
|
||||
const serverUrlParam = params.get('server');
|
||||
const projectParam = params.get('project');
|
||||
|
||||
if (!serverUrlParam && !projectParam) return;
|
||||
autoConnectRan.current = true;
|
||||
|
||||
const serverUrl = params.get('server') || window.location.origin;
|
||||
const projectParam = params.get('project') || undefined;
|
||||
|
||||
// Keep ?server= in the URL so F5 reconnects to the same server.
|
||||
// autoConnectRan.current prevents re-trigger within the same session.
|
||||
// handleServerConnect() will add/update ?project= after connecting.
|
||||
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: 0,
|
||||
|
|
@ -123,39 +121,45 @@ const AppContent = () => {
|
|||
});
|
||||
setViewMode('loading');
|
||||
|
||||
const serverUrl = serverUrlParam || window.location.origin;
|
||||
const baseUrl = normalizeServerUrl(serverUrl);
|
||||
|
||||
connectToServer(
|
||||
serverUrl,
|
||||
(phase, downloaded, total) => {
|
||||
if (phase === 'validating') {
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: 5,
|
||||
message: 'Connecting to server...',
|
||||
detail: 'Validating server',
|
||||
});
|
||||
} else if (phase === 'downloading') {
|
||||
const pct = total ? Math.round((downloaded / total) * 90) + 5 : 50;
|
||||
const mb = (downloaded / (1024 * 1024)).toFixed(1);
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: pct,
|
||||
message: 'Downloading graph...',
|
||||
detail: `${mb} MB downloaded`,
|
||||
});
|
||||
} else if (phase === 'extracting') {
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: 97,
|
||||
message: 'Processing...',
|
||||
detail: 'Extracting file contents',
|
||||
});
|
||||
}
|
||||
},
|
||||
undefined,
|
||||
projectParam,
|
||||
)
|
||||
const tryConnect = async () => {
|
||||
return await connectToServer(
|
||||
serverUrl,
|
||||
(phase, downloaded, total) => {
|
||||
if (phase === 'validating') {
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: 5,
|
||||
message: 'Connecting to server...',
|
||||
detail: 'Validating server',
|
||||
});
|
||||
} else if (phase === 'downloading') {
|
||||
const pct = total ? Math.round((downloaded / total) * 90) + 5 : 50;
|
||||
const mb = (downloaded / (1024 * 1024)).toFixed(1);
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: pct,
|
||||
message: 'Downloading graph...',
|
||||
detail: `${mb} MB downloaded`,
|
||||
});
|
||||
} else if (phase === 'extracting') {
|
||||
setProgress({
|
||||
phase: 'extracting',
|
||||
percent: 97,
|
||||
message: 'Processing...',
|
||||
detail: 'Extracting file contents',
|
||||
});
|
||||
}
|
||||
},
|
||||
undefined,
|
||||
projectParam || undefined,
|
||||
{ awaitAnalysis: true }, // enable backend hold-queue for repos still being analyzed
|
||||
);
|
||||
};
|
||||
|
||||
tryConnect()
|
||||
.then(async (result) => {
|
||||
await handleServerConnect(result);
|
||||
setProgress(null);
|
||||
|
|
|
|||
|
|
@ -231,7 +231,7 @@ export const CodeReferencesPanel = ({ onFocusNode }: CodeReferencesPanelProps) =
|
|||
repo: projectName,
|
||||
};
|
||||
|
||||
readFile(selectedFilePath, options)
|
||||
readFile(selectedFilePath, { ...options, repo: projectName || undefined })
|
||||
.then((result) => {
|
||||
if (!cancelled) {
|
||||
setFileResult(result);
|
||||
|
|
|
|||
|
|
@ -8,8 +8,10 @@ import {
|
|||
Loader2,
|
||||
AlertTriangle,
|
||||
GitBranch,
|
||||
ArrowDown,
|
||||
} from '@/lib/lucide-icons';
|
||||
import { useAppState } from '../hooks/useAppState';
|
||||
import { useAutoScroll } from '../hooks/useAutoScroll';
|
||||
import { ToolCallCard } from './ToolCallCard';
|
||||
import { isProviderConfigured } from '../core/llm/settings-service';
|
||||
import { MarkdownRenderer } from './MarkdownRenderer';
|
||||
|
|
@ -35,14 +37,11 @@ export const RightPanel = () => {
|
|||
const [chatInput, setChatInput] = useState('');
|
||||
const [activeTab, setActiveTab] = useState<'chat' | 'processes'>('chat');
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const messagesEndRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Auto-scroll to bottom when messages update or while streaming
|
||||
useEffect(() => {
|
||||
if (messagesEndRef.current) {
|
||||
messagesEndRef.current.scrollIntoView({ behavior: 'smooth' });
|
||||
}
|
||||
}, [chatMessages, isChatLoading]);
|
||||
// Keep streamed replies pinned unless the user intentionally scrolls away from the bottom.
|
||||
const { scrollContainerRef, messagesContainerRef, isAtBottom, scrollToBottom } = useAutoScroll(
|
||||
chatMessages,
|
||||
isChatLoading,
|
||||
);
|
||||
|
||||
const resolveFilePathForUI = useCallback((_requestedPath: string): string | null => {
|
||||
return null;
|
||||
|
|
@ -265,7 +264,7 @@ export const RightPanel = () => {
|
|||
|
||||
{/* Chat Content - only show when chat tab is active */}
|
||||
{activeTab === 'chat' && (
|
||||
<div className="flex flex-1 flex-col overflow-hidden">
|
||||
<div className="relative flex flex-1 flex-col overflow-hidden">
|
||||
{/* Status bar */}
|
||||
<div className="flex items-center gap-2.5 border-b border-border-subtle bg-elevated/50 px-4 py-3">
|
||||
<div className="ml-auto flex items-center gap-2">
|
||||
|
|
@ -291,7 +290,7 @@ export const RightPanel = () => {
|
|||
)}
|
||||
|
||||
{/* Messages */}
|
||||
<div className="scrollbar-thin flex-1 overflow-y-auto p-4">
|
||||
<div ref={scrollContainerRef} className="scrollbar-thin flex-1 overflow-y-auto p-4">
|
||||
{chatMessages.length === 0 ? (
|
||||
<div className="flex h-full flex-col items-center justify-center px-4 text-center">
|
||||
<div className="mb-4 flex h-14 w-14 items-center justify-center rounded-xl bg-gradient-to-br from-accent to-node-interface text-2xl shadow-glow">
|
||||
|
|
@ -315,7 +314,7 @@ export const RightPanel = () => {
|
|||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex flex-col gap-6">
|
||||
<div ref={messagesContainerRef} className="flex flex-col gap-6">
|
||||
{chatMessages.map((message) => (
|
||||
<div key={message.id} className="animate-fade-in">
|
||||
{/* User message - compact label style */}
|
||||
|
|
@ -391,10 +390,22 @@ export const RightPanel = () => {
|
|||
))}
|
||||
</div>
|
||||
)}
|
||||
{/* Scroll anchor for auto-scroll */}
|
||||
<div ref={messagesEndRef} />
|
||||
</div>
|
||||
|
||||
{/* Scroll to bottom */}
|
||||
<button
|
||||
aria-label="Scroll to bottom"
|
||||
onClick={() => scrollToBottom()}
|
||||
className={`absolute bottom-20 left-1/2 z-10 -translate-x-1/2 rounded-full border border-border-subtle bg-elevated px-3 py-1.5 text-xs text-text-secondary shadow-lg transition-all duration-200 hover:border-accent hover:text-accent ${
|
||||
!isAtBottom && chatMessages.length > 0
|
||||
? 'translate-y-0 opacity-100'
|
||||
: 'pointer-events-none translate-y-2 opacity-0'
|
||||
}`}
|
||||
>
|
||||
<ArrowDown className="mr-1 inline h-3.5 w-3.5" />
|
||||
Scroll to bottom
|
||||
</button>
|
||||
|
||||
{/* Input */}
|
||||
<div className="border-t border-border-subtle bg-surface p-3">
|
||||
<div className="flex items-end gap-2 rounded-xl border border-border-subtle bg-elevated px-3 py-2 transition-all focus-within:border-accent focus-within:ring-2 focus-within:ring-accent/20">
|
||||
|
|
|
|||
|
|
@ -64,7 +64,7 @@ export const StatusBar = () => {
|
|||
</a>
|
||||
|
||||
{/* Right - Stats */}
|
||||
<div className="flex items-center gap-3">
|
||||
<div className="flex items-center gap-3" data-testid="graph-stats">
|
||||
{graph && (
|
||||
<>
|
||||
<span>{nodeCount} nodes</span>
|
||||
|
|
|
|||
|
|
@ -579,6 +579,13 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
|
|||
|
||||
try {
|
||||
const effectiveProjectName = overrideProjectName || projectName || 'project';
|
||||
|
||||
// Sync repoRef so all agent backend calls target the correct repo.
|
||||
// initializeAgent can be called from App.tsx (handleServerConnect) which
|
||||
// never sets repoRef.current directly — without this, queries default to repo[0].
|
||||
if (overrideProjectName) {
|
||||
repoRef.current = overrideProjectName;
|
||||
}
|
||||
const repo = repoRef.current;
|
||||
|
||||
// Build backend interface for Graph RAG tools
|
||||
|
|
@ -610,7 +617,8 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
|
|||
setIsAgentInitializing(false);
|
||||
}
|
||||
},
|
||||
[projectName],
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
[], // repoRef is a stable ref — we sync it explicitly on entry; no state deps needed
|
||||
);
|
||||
|
||||
const sendChatMessage = useCallback(
|
||||
|
|
@ -1042,6 +1050,9 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
|
|||
setCodePanelOpen(false);
|
||||
setCodeReferenceFocus(null);
|
||||
|
||||
let connectedRepo: BackendRepo | undefined;
|
||||
let pNameStr = repoName || 'server-project';
|
||||
|
||||
try {
|
||||
const result: ConnectResult = await connectToServer(
|
||||
serverBaseUrl,
|
||||
|
|
@ -1073,44 +1084,28 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
|
|||
},
|
||||
undefined,
|
||||
repoName,
|
||||
{ awaitAnalysis: true }, // enable backend hold-queue for repos still being analyzed
|
||||
);
|
||||
|
||||
// Build graph for visualization
|
||||
const repoPath = result.repoInfo.repoPath ?? result.repoInfo.path;
|
||||
// Prefer the registry name, then normalize Windows \ and Unix / paths
|
||||
const pName =
|
||||
repoName || result.repoInfo.name || repoPath?.split('/').pop() || 'server-project';
|
||||
repoName ||
|
||||
result.repoInfo.name ||
|
||||
(repoPath || '').replace(/\\/g, '/').split('/').filter(Boolean).pop() ||
|
||||
'server-project';
|
||||
setProjectName(pName);
|
||||
repoRef.current = pName;
|
||||
|
||||
// Update URL so F5 / bookmarks open the correct repo
|
||||
const url = new URL(window.location.href);
|
||||
url.searchParams.set('project', pName);
|
||||
window.history.replaceState(null, '', url.toString());
|
||||
connectedRepo = result.repoInfo;
|
||||
pNameStr = pName;
|
||||
|
||||
const newGraph = createKnowledgeGraph();
|
||||
for (const node of result.nodes) newGraph.addNode(node);
|
||||
for (const rel of result.relationships) newGraph.addRelationship(rel);
|
||||
setGraph(newGraph);
|
||||
|
||||
// No fileContents needed — grep/read tools use backend HTTP
|
||||
|
||||
// Initialize agent with backend queries, then start embeddings
|
||||
try {
|
||||
if (getActiveProviderConfig()) {
|
||||
await initializeAgent(pName);
|
||||
}
|
||||
setViewMode('exploring');
|
||||
startEmbeddingsWithFallback();
|
||||
setProgress(null);
|
||||
} catch (err) {
|
||||
console.warn('Failed to initialize agent:', err);
|
||||
setIsAgentReady(false);
|
||||
agentRef.current = null;
|
||||
setAgentError('Failed to initialize agent');
|
||||
setViewMode('exploring');
|
||||
setProgress(null);
|
||||
}
|
||||
} catch (err) {
|
||||
} catch (err: unknown) {
|
||||
console.error('Repo switch failed:', err);
|
||||
setProgress({
|
||||
phase: 'error',
|
||||
|
|
@ -1124,6 +1119,36 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
|
|||
setViewMode('exploring');
|
||||
setProgress(null);
|
||||
}, ERROR_RESET_DELAY_MS);
|
||||
return; // Abort the whole switchRepo process
|
||||
}
|
||||
|
||||
if (pNameStr) {
|
||||
// Persist the selected project in the URL so a refresh re-opens it
|
||||
const urlObj = new URL(window.location.href);
|
||||
urlObj.searchParams.set('project', pNameStr);
|
||||
window.history.replaceState(null, '', urlObj.toString());
|
||||
}
|
||||
|
||||
// Reset the agent and clear chat history so the AI starts fresh for the new repo
|
||||
agentRef.current = null;
|
||||
setIsAgentReady(false);
|
||||
setChatMessages([]);
|
||||
|
||||
// Re-initialize agent with the new repo's graph context
|
||||
try {
|
||||
if (getActiveProviderConfig()) {
|
||||
await initializeAgent(pNameStr);
|
||||
}
|
||||
setViewMode('exploring');
|
||||
startEmbeddingsWithFallback();
|
||||
setProgress(null);
|
||||
} catch (err) {
|
||||
console.warn('Failed to initialize agent:', err);
|
||||
setIsAgentReady(false);
|
||||
agentRef.current = null;
|
||||
setAgentError('Failed to initialize agent');
|
||||
setViewMode('exploring');
|
||||
setProgress(null);
|
||||
}
|
||||
},
|
||||
[
|
||||
|
|
@ -1143,6 +1168,7 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
|
|||
setCodeReferences,
|
||||
setCodePanelOpen,
|
||||
setCodeReferenceFocus,
|
||||
setChatMessages,
|
||||
],
|
||||
);
|
||||
|
||||
|
|
|
|||
145
gitnexus-web/src/hooks/useAutoScroll.ts
Normal file
145
gitnexus-web/src/hooks/useAutoScroll.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react';
|
||||
|
||||
const DEFAULT_BOTTOM_THRESHOLD = 100;
|
||||
const USER_SCROLL_EPSILON = 5;
|
||||
|
||||
export interface UseAutoScrollResult {
|
||||
scrollContainerRef: React.RefObject<HTMLDivElement>;
|
||||
messagesContainerRef: React.RefObject<HTMLDivElement>;
|
||||
isAtBottom: boolean;
|
||||
scrollToBottom: (behavior?: ScrollBehavior) => void;
|
||||
}
|
||||
|
||||
function isNearBottom(element: HTMLElement, threshold: number): boolean {
|
||||
return element.scrollHeight - element.scrollTop - element.clientHeight <= threshold;
|
||||
}
|
||||
|
||||
export function useAutoScroll<T>(
|
||||
chatMessages: T[],
|
||||
isChatLoading: boolean,
|
||||
bottomThreshold = DEFAULT_BOTTOM_THRESHOLD,
|
||||
): UseAutoScrollResult {
|
||||
const scrollContainerRef = useRef<HTMLDivElement>(null);
|
||||
const messagesContainerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const [isAtBottom, setIsAtBottom] = useState(true);
|
||||
|
||||
const shouldStickToBottomRef = useRef(true);
|
||||
const lastScrollTopRef = useRef(0);
|
||||
const scrollFrameIdRef = useRef<number | null>(null);
|
||||
|
||||
const syncScrollState = useCallback(() => {
|
||||
const element = scrollContainerRef.current;
|
||||
if (!element) return;
|
||||
|
||||
const currentScrollTop = element.scrollTop;
|
||||
const nearBottom = isNearBottom(element, bottomThreshold);
|
||||
|
||||
if (nearBottom) {
|
||||
shouldStickToBottomRef.current = true;
|
||||
} else if (currentScrollTop < lastScrollTopRef.current - USER_SCROLL_EPSILON) {
|
||||
shouldStickToBottomRef.current = false;
|
||||
}
|
||||
|
||||
lastScrollTopRef.current = currentScrollTop;
|
||||
setIsAtBottom(nearBottom);
|
||||
}, [bottomThreshold]);
|
||||
|
||||
const scrollToBottom = useCallback(
|
||||
(behavior: ScrollBehavior = 'smooth') => {
|
||||
const element = scrollContainerRef.current;
|
||||
if (!element) return;
|
||||
|
||||
shouldStickToBottomRef.current = true;
|
||||
|
||||
if (behavior === 'auto') {
|
||||
element.scrollTop = element.scrollHeight;
|
||||
lastScrollTopRef.current = element.scrollTop;
|
||||
setIsAtBottom(isNearBottom(element, bottomThreshold));
|
||||
return;
|
||||
}
|
||||
|
||||
element.scrollTo({
|
||||
top: element.scrollHeight,
|
||||
behavior,
|
||||
});
|
||||
},
|
||||
[bottomThreshold],
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const element = scrollContainerRef.current;
|
||||
if (!element) return;
|
||||
|
||||
lastScrollTopRef.current = element.scrollTop;
|
||||
|
||||
const handleScroll = () => {
|
||||
if (scrollFrameIdRef.current !== null) {
|
||||
cancelAnimationFrame(scrollFrameIdRef.current);
|
||||
}
|
||||
|
||||
scrollFrameIdRef.current = requestAnimationFrame(() => {
|
||||
scrollFrameIdRef.current = null;
|
||||
syncScrollState();
|
||||
});
|
||||
};
|
||||
|
||||
element.addEventListener('scroll', handleScroll, { passive: true });
|
||||
syncScrollState();
|
||||
|
||||
return () => {
|
||||
element.removeEventListener('scroll', handleScroll);
|
||||
|
||||
if (scrollFrameIdRef.current !== null) {
|
||||
cancelAnimationFrame(scrollFrameIdRef.current);
|
||||
scrollFrameIdRef.current = null;
|
||||
}
|
||||
};
|
||||
}, [syncScrollState]);
|
||||
|
||||
useEffect(() => {
|
||||
const content = messagesContainerRef.current;
|
||||
const scrollEl = scrollContainerRef.current;
|
||||
if (!content || !scrollEl || typeof ResizeObserver === 'undefined') return;
|
||||
|
||||
let resizeFrameId: number | null = null;
|
||||
|
||||
const observer = new ResizeObserver(() => {
|
||||
if (shouldStickToBottomRef.current) {
|
||||
if (resizeFrameId !== null) {
|
||||
cancelAnimationFrame(resizeFrameId);
|
||||
}
|
||||
|
||||
resizeFrameId = requestAnimationFrame(() => {
|
||||
resizeFrameId = null;
|
||||
scrollToBottom('auto');
|
||||
});
|
||||
} else {
|
||||
syncScrollState();
|
||||
}
|
||||
});
|
||||
|
||||
observer.observe(content);
|
||||
|
||||
return () => {
|
||||
observer.disconnect();
|
||||
|
||||
if (resizeFrameId !== null) {
|
||||
cancelAnimationFrame(resizeFrameId);
|
||||
resizeFrameId = null;
|
||||
}
|
||||
};
|
||||
}, [chatMessages.length, scrollToBottom, syncScrollState]);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
if (!shouldStickToBottomRef.current) return;
|
||||
scrollToBottom('auto');
|
||||
}, [chatMessages.length, isChatLoading, scrollToBottom]);
|
||||
|
||||
return {
|
||||
scrollContainerRef,
|
||||
messagesContainerRef,
|
||||
isAtBottom,
|
||||
scrollToBottom,
|
||||
};
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
export {
|
||||
AlertCircle,
|
||||
AlertTriangle,
|
||||
ArrowDown,
|
||||
ArrowRight,
|
||||
AtSign,
|
||||
Brain,
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ export function normalizeServerUrl(input: string): string {
|
|||
|
||||
// ── Internal Helpers ───────────────────────────────────────────────────────
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 10_000;
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const PROBE_TIMEOUT_MS = 2_000;
|
||||
|
||||
const fetchWithTimeout = async (
|
||||
|
|
@ -264,11 +264,13 @@ const fetchWithTimeout = async (
|
|||
const assertOk = async (response: Response): Promise<void> => {
|
||||
if (response.ok) return;
|
||||
|
||||
let message = `Backend returned ${response.status} ${response.statusText}`;
|
||||
let message = response.statusText;
|
||||
try {
|
||||
const body = await response.json();
|
||||
if (body && typeof body.error === 'string') {
|
||||
message = body.error;
|
||||
} else if (body && typeof body.message === 'string') {
|
||||
message = body.message;
|
||||
}
|
||||
} catch {
|
||||
// Response body was not JSON
|
||||
|
|
@ -386,10 +388,22 @@ export const fetchRepos = async (): Promise<BackendRepo[]> => {
|
|||
return response.json() as Promise<BackendRepo[]>;
|
||||
};
|
||||
|
||||
/** Fetch repo metadata. */
|
||||
export const fetchRepoInfo = async (repo?: string): Promise<BackendRepo> => {
|
||||
/** Fetch repo metadata.
|
||||
* Pass `awaitAnalysis: true` when connecting to a repo that may still be cloning/analyzing —
|
||||
* this enables the backend's hold-queue and uses a 5-minute timeout to match.
|
||||
* Normal calls (e.g. repo switching between already-indexed repos) use the default 10s timeout.
|
||||
*
|
||||
* Must stay in sync with HOLD_QUEUE_TIMEOUT_SECS in gitnexus/src/server/api.ts.
|
||||
*/
|
||||
const HOLD_QUEUE_TIMEOUT_MS = 300_000; // 5 minutes — matches backend HOLD_QUEUE_TIMEOUT_SECS
|
||||
|
||||
export const fetchRepoInfo = async (
|
||||
repo?: string,
|
||||
opts?: { awaitAnalysis?: boolean },
|
||||
): Promise<BackendRepo> => {
|
||||
const url = `${_backendUrl}/api/repo${repo ? `?${repoParam(repo)}` : ''}`;
|
||||
const response = await fetchWithTimeout(url);
|
||||
const timeout = opts?.awaitAnalysis ? HOLD_QUEUE_TIMEOUT_MS : undefined;
|
||||
const response = await fetchWithTimeout(url, {}, timeout);
|
||||
await assertOk(response);
|
||||
const data = await response.json();
|
||||
return { ...data, repoPath: data.repoPath ?? data.path };
|
||||
|
|
@ -404,13 +418,19 @@ export const fetchGraph = async (
|
|||
onProgress?: (downloaded: number, total: number | null) => void;
|
||||
},
|
||||
): Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }> => {
|
||||
const params = [repoParam(repo), opts?.includeContent ? 'includeContent=true' : '']
|
||||
const params = [repoParam(repo), opts?.includeContent ? 'includeContent=true' : '', 'stream=true']
|
||||
.filter(Boolean)
|
||||
.join('&');
|
||||
const url = `${_backendUrl}/api/graph${params ? `?${params}` : ''}`;
|
||||
// Large repos can take a while to serialize the graph — use an elevated timeout
|
||||
const response = await fetchWithTimeout(url, { signal: opts?.signal }, 300_000);
|
||||
await assertOk(response);
|
||||
|
||||
const contentType = response.headers.get('Content-Type') || '';
|
||||
if (contentType.includes('application/x-ndjson')) {
|
||||
return parseNdjsonGraphResponse(response, opts?.onProgress);
|
||||
}
|
||||
|
||||
if (!opts?.onProgress || !response.body) {
|
||||
return response.json() as Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }>;
|
||||
}
|
||||
|
|
@ -439,6 +459,66 @@ export const fetchGraph = async (
|
|||
return JSON.parse(new TextDecoder().decode(combined));
|
||||
};
|
||||
|
||||
const parseNdjsonGraphResponse = async (
|
||||
response: Response,
|
||||
onProgress?: (downloaded: number, total: number | null) => void,
|
||||
): Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }> => {
|
||||
if (!response.body) {
|
||||
throw new BackendError('No response body', response.status, 'server');
|
||||
}
|
||||
|
||||
const contentLength = response.headers.get('Content-Length');
|
||||
const total = contentLength ? parseInt(contentLength, 10) : null;
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
const nodes: GraphNode[] = [];
|
||||
const relationships: GraphRelationship[] = [];
|
||||
let buffer = '';
|
||||
let downloaded = 0;
|
||||
|
||||
const parseLine = (line: string) => {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) return;
|
||||
|
||||
const record = JSON.parse(trimmed) as
|
||||
| { type: 'node'; data: GraphNode }
|
||||
| { type: 'relationship'; data: GraphRelationship }
|
||||
| { type: 'error'; error: string };
|
||||
|
||||
if (record.type === 'node') {
|
||||
nodes.push(record.data);
|
||||
return;
|
||||
}
|
||||
if (record.type === 'relationship') {
|
||||
relationships.push(record.data);
|
||||
return;
|
||||
}
|
||||
if (record.type === 'error') {
|
||||
throw new BackendError(record.error, response.status || 500, 'server');
|
||||
}
|
||||
};
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
downloaded += value.length;
|
||||
onProgress?.(downloaded, total);
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() || '';
|
||||
for (const line of lines) {
|
||||
parseLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
buffer += decoder.decode();
|
||||
parseLine(buffer);
|
||||
|
||||
return { nodes, relationships };
|
||||
};
|
||||
|
||||
/** Execute a Cypher query. Returns rows. */
|
||||
export const runQuery = async (
|
||||
cypher: string,
|
||||
|
|
@ -670,18 +750,21 @@ export interface ConnectResult {
|
|||
/**
|
||||
* Connect to a server: validate, fetch repo info, download graph.
|
||||
* Content is NOT included (use readFile/grep for file access).
|
||||
* Pass `awaitAnalysis: true` when the repo may still be cloning/analyzing —
|
||||
* this enables the backend hold-queue and a 5-minute fetch timeout.
|
||||
*/
|
||||
export async function connectToServer(
|
||||
url: string,
|
||||
onProgress?: (phase: string, downloaded: number, total: number | null) => void,
|
||||
signal?: AbortSignal,
|
||||
repoName?: string,
|
||||
opts?: { awaitAnalysis?: boolean },
|
||||
): Promise<ConnectResult> {
|
||||
const baseUrl = normalizeServerUrl(url);
|
||||
setBackendUrl(baseUrl);
|
||||
|
||||
onProgress?.('validating', 0, null);
|
||||
const repoInfo = await fetchRepoInfo(repoName);
|
||||
const repoInfo = await fetchRepoInfo(repoName, { awaitAnalysis: opts?.awaitAnalysis });
|
||||
|
||||
onProgress?.('downloading', 0, null);
|
||||
const { nodes, relationships } = await fetchGraph(repoName, {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { normalizeServerUrl } from '../../src/services/backend-client';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { fetchGraph, normalizeServerUrl, setBackendUrl } from '../../src/services/backend-client';
|
||||
|
||||
describe('normalizeServerUrl', () => {
|
||||
it('adds http:// to localhost', () => {
|
||||
|
|
@ -31,3 +31,137 @@ describe('normalizeServerUrl', () => {
|
|||
expect(normalizeServerUrl('https://gitnexus.example.com')).toBe('https://gitnexus.example.com');
|
||||
});
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('fetchGraph', () => {
|
||||
it('requests streamed graph responses from the backend', async () => {
|
||||
setBackendUrl('http://localhost:4747');
|
||||
|
||||
const fetchMock = vi.fn().mockResolvedValue(
|
||||
new Response('{"nodes":[],"relationships":[]}', {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await fetchGraph('big-repo');
|
||||
|
||||
expect(fetchMock).toHaveBeenCalledWith(
|
||||
expect.stringContaining('/api/graph?repo=big-repo&stream=true'),
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it('parses NDJSON graph streams incrementally', async () => {
|
||||
setBackendUrl('http://localhost:4747');
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
[
|
||||
'{"type":"node","data":{"id":"File:src/app.ts","label":"File","properties":{"name":"app.ts","filePath":"src/app.ts"}}}\n',
|
||||
'{"type":"relationship","data":{"id":"File:src/app.ts_CONTAINS_Function:src/app.ts:main","type":"CONTAINS","sourceId":"File:src/app.ts","targetId":"Function:src/app.ts:main"}}\n',
|
||||
].join(''),
|
||||
),
|
||||
);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn().mockResolvedValue(
|
||||
new Response(stream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/x-ndjson',
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const progress = vi.fn();
|
||||
const result = await fetchGraph('big-repo', { onProgress: progress });
|
||||
|
||||
expect(result.nodes).toHaveLength(1);
|
||||
expect(result.relationships).toHaveLength(1);
|
||||
expect(result.nodes[0].id).toBe('File:src/app.ts');
|
||||
expect(result.relationships[0].type).toBe('CONTAINS');
|
||||
expect(progress).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('parses NDJSON graph lines split across chunks', async () => {
|
||||
setBackendUrl('http://localhost:4747');
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
'{"type":"node","data":{"id":"File:src/app.ts","label":"File","properties":{"name":"app.ts"',
|
||||
),
|
||||
);
|
||||
controller.enqueue(
|
||||
encoder.encode(
|
||||
',"filePath":"src/app.ts"}}}\n{"type":"relationship","data":{"id":"File:src/app.ts_CONTAINS_Function:src/app.ts:main","type":"CONTAINS","sourceId":"File:src/app.ts","targetId":"Function:src/app.ts:main"}}\n',
|
||||
),
|
||||
);
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn().mockResolvedValue(
|
||||
new Response(stream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/x-ndjson',
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const result = await fetchGraph('big-repo');
|
||||
|
||||
expect(result.nodes).toHaveLength(1);
|
||||
expect(result.relationships).toHaveLength(1);
|
||||
expect(result.nodes[0].properties.filePath).toBe('src/app.ts');
|
||||
});
|
||||
|
||||
it('throws backend errors emitted in the NDJSON stream', async () => {
|
||||
setBackendUrl('http://localhost:4747');
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
controller.enqueue(encoder.encode('{"type":"error","error":"stream failed"}\n'));
|
||||
controller.close();
|
||||
},
|
||||
});
|
||||
|
||||
vi.stubGlobal(
|
||||
'fetch',
|
||||
vi.fn().mockResolvedValue(
|
||||
new Response(stream, {
|
||||
status: 200,
|
||||
headers: {
|
||||
'Content-Type': 'application/x-ndjson',
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
await expect(fetchGraph('big-repo')).rejects.toMatchObject({
|
||||
message: 'stream failed',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
287
gitnexus-web/test/unit/use-auto-scroll.test.tsx
Normal file
287
gitnexus-web/test/unit/use-auto-scroll.test.tsx
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
import { act, fireEvent, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { useAutoScroll } from '../../src/hooks/useAutoScroll';
|
||||
|
||||
interface HarnessProps {
|
||||
messages: unknown[];
|
||||
isChatLoading: boolean;
|
||||
}
|
||||
|
||||
function AutoScrollHarness({ messages, isChatLoading }: HarnessProps) {
|
||||
const { scrollContainerRef, messagesContainerRef, isAtBottom, scrollToBottom } = useAutoScroll(
|
||||
messages,
|
||||
isChatLoading,
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
<div data-testid="is-at-bottom">{String(isAtBottom)}</div>
|
||||
<div data-testid="container" ref={scrollContainerRef}>
|
||||
{messages.length > 0 ? (
|
||||
<div data-testid="messages-container" ref={messagesContainerRef}>
|
||||
{messages.map((message, index) => (
|
||||
<div key={index}>{String(message)}</div>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
<button type="button" onClick={() => scrollToBottom()}>
|
||||
Scroll to bottom
|
||||
</button>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function setScrollMetrics(
|
||||
element: HTMLDivElement,
|
||||
metrics: { scrollTop?: number; scrollHeight?: number; clientHeight?: number },
|
||||
) {
|
||||
if (metrics.scrollTop !== undefined) {
|
||||
Object.defineProperty(element, 'scrollTop', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: metrics.scrollTop,
|
||||
});
|
||||
}
|
||||
|
||||
if (metrics.scrollHeight !== undefined) {
|
||||
Object.defineProperty(element, 'scrollHeight', {
|
||||
configurable: true,
|
||||
value: metrics.scrollHeight,
|
||||
});
|
||||
}
|
||||
|
||||
if (metrics.clientHeight !== undefined) {
|
||||
Object.defineProperty(element, 'clientHeight', {
|
||||
configurable: true,
|
||||
value: metrics.clientHeight,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function flushAnimationFrame() {
|
||||
await act(async () => {
|
||||
vi.runAllTimers();
|
||||
});
|
||||
}
|
||||
|
||||
async function scrollContainer(element: HTMLDivElement, scrollTop: number) {
|
||||
setScrollMetrics(element, { scrollTop });
|
||||
fireEvent.scroll(element);
|
||||
await flushAnimationFrame();
|
||||
}
|
||||
|
||||
const resizeObserverInstances: ResizeObserverMock[] = [];
|
||||
|
||||
class ResizeObserverMock {
|
||||
callback: ResizeObserverCallback;
|
||||
observedElements: Element[] = [];
|
||||
observe = vi.fn((element: Element) => {
|
||||
this.observedElements.push(element);
|
||||
});
|
||||
unobserve = vi.fn();
|
||||
disconnect = vi.fn();
|
||||
|
||||
constructor(callback: ResizeObserverCallback) {
|
||||
this.callback = callback;
|
||||
resizeObserverInstances.push(this);
|
||||
}
|
||||
}
|
||||
|
||||
async function triggerResize(instance: ResizeObserverMock) {
|
||||
await act(async () => {
|
||||
instance.callback([], instance as unknown as ResizeObserver);
|
||||
});
|
||||
await flushAnimationFrame();
|
||||
}
|
||||
|
||||
describe('useAutoScroll', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
resizeObserverInstances.length = 0;
|
||||
vi.stubGlobal(
|
||||
'requestAnimationFrame',
|
||||
vi.fn((callback: FrameRequestCallback) => {
|
||||
return window.setTimeout(() => callback(performance.now()), 0);
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal(
|
||||
'cancelAnimationFrame',
|
||||
vi.fn((frameId: number) => {
|
||||
clearTimeout(frameId);
|
||||
}),
|
||||
);
|
||||
Object.defineProperty(HTMLElement.prototype, 'scrollTo', {
|
||||
configurable: true,
|
||||
value: function (options: ScrollToOptions) {
|
||||
if (options.top !== undefined) {
|
||||
Object.defineProperty(this, 'scrollTop', {
|
||||
configurable: true,
|
||||
writable: true,
|
||||
value: options.top,
|
||||
});
|
||||
}
|
||||
},
|
||||
});
|
||||
vi.stubGlobal('ResizeObserver', ResizeObserverMock);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('starts with isAtBottom true and auto-scrolls the very first message', () => {
|
||||
const { rerender } = render(<AutoScrollHarness messages={[]} isChatLoading={false} />);
|
||||
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
|
||||
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
setScrollMetrics(container, { scrollTop: 0, scrollHeight: 500, clientHeight: 200 });
|
||||
|
||||
rerender(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
|
||||
expect(container.scrollTop).toBe(500);
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
|
||||
});
|
||||
|
||||
it('follows streaming updates while the view stays pinned to the bottom', () => {
|
||||
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
|
||||
|
||||
rerender(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={true} />);
|
||||
|
||||
expect(container.scrollTop).toBe(1000);
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
|
||||
});
|
||||
|
||||
it('stops auto-scroll after the user scrolls up', async () => {
|
||||
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
|
||||
await scrollContainer(container, 700);
|
||||
await scrollContainer(container, 250);
|
||||
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false');
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1400, clientHeight: 200 });
|
||||
rerender(<AutoScrollHarness messages={[{ id: 1 }, { id: 2 }]} isChatLoading={true} />);
|
||||
|
||||
expect(container.scrollTop).toBe(250);
|
||||
});
|
||||
|
||||
it('re-enables auto-scroll once the user returns near the bottom', async () => {
|
||||
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
|
||||
await scrollContainer(container, 700);
|
||||
await scrollContainer(container, 250);
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 1120, scrollHeight: 1400, clientHeight: 200 });
|
||||
await scrollContainer(container, 1120);
|
||||
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 1120, scrollHeight: 1800, clientHeight: 200 });
|
||||
rerender(<AutoScrollHarness messages={[{ id: 1 }, { id: 2 }]} isChatLoading={true} />);
|
||||
|
||||
expect(container.scrollTop).toBe(1800);
|
||||
});
|
||||
|
||||
it('scrollToBottom re-engages auto-scroll and scrolls to the container bottom', async () => {
|
||||
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
const scrollTo = vi.spyOn(container, 'scrollTo');
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
|
||||
await scrollContainer(container, 700);
|
||||
await scrollContainer(container, 250);
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Scroll to bottom' }));
|
||||
|
||||
expect(scrollTo).toHaveBeenCalledWith({ top: 1000, behavior: 'smooth' });
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false');
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1600, clientHeight: 200 });
|
||||
rerender(<AutoScrollHarness messages={[{ id: 1 }, { id: 2 }]} isChatLoading={true} />);
|
||||
|
||||
expect(container.scrollTop).toBe(1600);
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
|
||||
});
|
||||
|
||||
it('re-pins to the latest bottom when inner content grows asynchronously', async () => {
|
||||
render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
|
||||
await scrollContainer(container, 700);
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 1000, scrollHeight: 1450, clientHeight: 200 });
|
||||
await triggerResize(resizeObserverInstances[0]);
|
||||
|
||||
expect(container.scrollTop).toBe(1450);
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
|
||||
});
|
||||
|
||||
it('does not auto-scroll on async growth after user intentionally scrolls away', async () => {
|
||||
render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
|
||||
await scrollContainer(container, 700);
|
||||
await scrollContainer(container, 250);
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1400, clientHeight: 200 });
|
||||
await triggerResize(resizeObserverInstances[0]);
|
||||
|
||||
expect(container.scrollTop).toBe(250);
|
||||
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false');
|
||||
});
|
||||
|
||||
it('cancels the pending ResizeObserver rAF when the component unmounts', () => {
|
||||
const cancelRAF = vi.mocked(cancelAnimationFrame);
|
||||
|
||||
const { unmount } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
const container = screen.getByTestId('container') as HTMLDivElement;
|
||||
|
||||
setScrollMetrics(container, { scrollTop: 950, scrollHeight: 1000, clientHeight: 200 });
|
||||
|
||||
const callsBefore = cancelRAF.mock.calls.length;
|
||||
|
||||
act(() => {
|
||||
resizeObserverInstances[0].callback(
|
||||
[],
|
||||
resizeObserverInstances[0] as unknown as ResizeObserver,
|
||||
);
|
||||
});
|
||||
|
||||
unmount();
|
||||
|
||||
expect(cancelRAF.mock.calls.length).toBeGreaterThan(callsBefore);
|
||||
|
||||
expect(() => vi.runAllTimers()).not.toThrow();
|
||||
});
|
||||
|
||||
it('attaches the observer when the messages wrapper first appears and disconnects on unmount', () => {
|
||||
const { rerender, unmount } = render(<AutoScrollHarness messages={[]} isChatLoading={false} />);
|
||||
|
||||
expect(screen.queryByTestId('messages-container')).toBeNull();
|
||||
expect(resizeObserverInstances).toHaveLength(0);
|
||||
|
||||
rerender(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
|
||||
|
||||
const messagesContainer = screen.getByTestId('messages-container');
|
||||
const resizeObserver = resizeObserverInstances[0];
|
||||
|
||||
expect(resizeObserverInstances).toHaveLength(1);
|
||||
expect(resizeObserver.observe).toHaveBeenCalledWith(messagesContainer);
|
||||
|
||||
unmount();
|
||||
|
||||
expect(resizeObserver.disconnect).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -9,6 +9,10 @@ tsconfig.json
|
|||
.gitignore
|
||||
node_modules/
|
||||
|
||||
# Vendor build artifacts (created during install, not shipped)
|
||||
vendor/**/node_modules
|
||||
vendor/**/build
|
||||
|
||||
# Package lock (consumers use their own)
|
||||
package-lock.json
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,81 @@
|
|||
|
||||
All notable changes to GitNexus will be documented in this file.
|
||||
|
||||
## [1.6.1] - 2026-04-13
|
||||
|
||||
### Added
|
||||
- **Service group extractor expansion** — manifest extractor and broader extractor coverage (2/4 of #606 split) (#796)
|
||||
- **Dart call patterns** for `await`, cascade, lambda, and widget-tree contexts (#801)
|
||||
|
||||
### Fixed
|
||||
- **Stack overflow and memory exhaustion** on large repository analysis (#814)
|
||||
- **`tree-sitter-dart` install crash** — switched from git URL to npm tarball (#811)
|
||||
- **Generic TypeScript awaited function calls** missing from the call graph (#804)
|
||||
- **Runtime dependency on `file:../gitnexus-shared`** removed from the published package (#803)
|
||||
- **Ruby `singleton_class` context** preserved during sequential parsing (#774)
|
||||
|
||||
### Changed
|
||||
- **DAG-based ingestion pipeline architecture** — pipeline phases now declare typed dependencies and run via a topologically sorted DAG; container-node logic extracted to `LanguageProvider`. Includes hardened lifecycle (try/finally cleanup, error wrapping, cycle reporting), tightened `ParseOutput.exportedTypeMap` immutability, and corrected phase dependencies (#809)
|
||||
|
||||
## [1.6.0] - 2026-04-12
|
||||
|
||||
### Added
|
||||
- **SemanticModel architecture refactor (SM-8 through SM-19)** — extracted registries into `model/` module with ISP-compliant interfaces: TypeRegistry, MethodRegistry, FieldRegistry, RegistrationTable, ResolutionContext (#786)
|
||||
- HeritageMap built from accumulated `ExtractedHeritage[]` for MRO-aware resolution (#739)
|
||||
- `lookupMethodByOwnerWithMRO` using HeritageMap for cross-class method dispatch (#740)
|
||||
- MRO fast path before D2 fuzzy widening in call resolution (#741)
|
||||
- BindingAccumulator for cross-file return type propagation (#743, #763)
|
||||
- Restructured `resolveUncached` replacing `lookupFuzzy` data source for all tiers (#764)
|
||||
- Deleted `lookupFuzzy`, `lookupFuzzyCallable`, `globalIndex`, `callableIndex` — replaced with structured lookups (#769)
|
||||
- Deleted `resolveCallTarget` god-method — replaced with thin dispatcher delegating to `resolveMemberCall` (#744), `resolveStaticCall` (#754), `resolveFreeCall` (#756) (#770)
|
||||
- **Service group infrastructure** — service boundary detection, contract extractors, sync pipeline, CLI/MCP tools, monorepo fixture; bridge.lbug storage and contract matching expansion (#795)
|
||||
- **C# interface-to-interface heritage** capture (#789)
|
||||
- **Vue SFC support** with destructured call result tracking (#604)
|
||||
- **Java method reference** resolution — `obj::method` as call sites (#622)
|
||||
- **C/C++ MethodExtractor** config with pure virtual detection (#617)
|
||||
- **MethodExtractor configs** for Python, PHP, Swift, Dart, Rust, Ruby (#624)
|
||||
- **METHOD_IMPLEMENTS edges** with overload disambiguation and MethodExtractor unification (#642)
|
||||
- **Same-arity overload disambiguation** via type-hash suffix (#658)
|
||||
- **`GITNEXUS_HOME` env var** to customize global directory (#746)
|
||||
- **Verbose analyze output** prints skipped large file paths (#745)
|
||||
- **Class name lookup index** for O(1) qualified lookups (#707, #716)
|
||||
- **`lookupMethodByOwner` index** for O(1) cross-class chain resolution (#665)
|
||||
- **Fuzzy lookup counters** for performance visibility (#708)
|
||||
|
||||
### Fixed
|
||||
- **Stack overflow on large PHP files** — iterative AST traversal (#783)
|
||||
- **Large repository graph loading** failure (#732)
|
||||
- **Windows multi-repo switching** — false 404 errors and stale repo context (#633)
|
||||
- **`detect_changes` diff mapping** — map diff hunks to symbol line ranges (#779)
|
||||
- **HTTP client vs Express route detection** and Spring interface attribution (#780)
|
||||
- **VECTOR extension** not loaded during DB init for semantic search (#782)
|
||||
- **tree-sitter-swift** postinstall patch for macOS ARM64 (#788)
|
||||
- **tree-sitter-c** peer dependency conflict pinned (#723)
|
||||
- **Constructor indexing** in methodByOwner (#694, #753)
|
||||
- **Named binding processor** — `lookupExact` replaced with `lookupExactAll` (#755)
|
||||
- **`.gitnexusignore` negation patterns** now respected (#654)
|
||||
- **MCP setup** prefers global gitnexus binary over npx (#653)
|
||||
- **CORS rejection** returns clean error instead of 500 (#646)
|
||||
- **Array.push stack overflow** — replaced spread with loop (#650)
|
||||
- **MCP stdout silencing** prevents embedder/pool-adapter conflicts (#645)
|
||||
- **Web heartbeat** — graceful reconnection replaces aggressive disconnect (#643)
|
||||
- **Web repo scoping** — backend calls scoped to active repo (#644)
|
||||
- **OpenCode config path** and FTS extension load order (#781)
|
||||
- **OnboardingGuide** dev-mode serve command corrected (#725)
|
||||
- **Security issues** and critical bugs from code review (#709)
|
||||
|
||||
### Changed
|
||||
- Replaced class-type fuzzy lookups with structured indices in type-env (#733, #734, #736)
|
||||
- Extracted `CLASS_LIKE_TYPES` constant (#693)
|
||||
|
||||
## [1.5.3] - 2026-04-01
|
||||
|
||||
### Added
|
||||
- **TypeScript/JavaScript MethodExtractor** config (#588)
|
||||
|
||||
### Fixed
|
||||
- **Wiki Azure OpenAI** compat and HTML viewer script injection (#618)
|
||||
|
||||
## [1.5.2] - 2026-04-01
|
||||
|
||||
### Fixed
|
||||
|
|
|
|||
|
|
@ -234,6 +234,79 @@ Installed automatically by both `gitnexus analyze` (per-repo) and `gitnexus setu
|
|||
- Node.js >= 18
|
||||
- Git repository (uses git for commit tracking)
|
||||
|
||||
## Release candidates
|
||||
|
||||
Stable releases publish to the default `latest` dist-tag. When a pull request
|
||||
with non-documentation changes merges into `main`, an automated workflow also
|
||||
publishes a prerelease build under the `rc` dist-tag, so early adopters can
|
||||
try in-flight fixes without waiting for the next stable cut. (Docs-only
|
||||
merges are skipped.)
|
||||
|
||||
```bash
|
||||
# Try the latest release candidate (pre-stable — may change at any time)
|
||||
npm install -g gitnexus@rc
|
||||
# — or —
|
||||
npx gitnexus@rc analyze
|
||||
```
|
||||
|
||||
Release-candidate versions follow the standard semver prerelease format
|
||||
`X.Y.Z-rc.N`, where `X.Y.Z` is the next stable target (bumped from the
|
||||
current `latest` by patch by default; `minor` or `major` when kicking off a
|
||||
bigger cycle) and `N` increments per published rc. Example sequence:
|
||||
`1.6.2-rc.1`, `1.6.2-rc.2`, …, then once `1.6.2` ships stable,
|
||||
`1.6.3-rc.1`. See the [Releases page](https://github.com/abhigyanpatwari/GitNexus/releases)
|
||||
for the full list; stable `latest` is unaffected.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### `Cannot destructure property 'package' of 'node.target' as it is null`
|
||||
|
||||
This crash was caused by a dependency URL format that is incompatible with
|
||||
certain npm/arborist versions ([npm/cli#8126](https://github.com/npm/cli/issues/8126)).
|
||||
It is fixed in **gitnexus v1.6.2+**. Upgrade to the latest version:
|
||||
|
||||
```bash
|
||||
npx gitnexus@latest analyze # always uses the newest release
|
||||
# — or —
|
||||
npm install -g gitnexus@latest # upgrade a global install
|
||||
```
|
||||
|
||||
If you still hit npm install issues after upgrading, these generic workarounds
|
||||
may help:
|
||||
|
||||
```bash
|
||||
npm install -g npm@latest # update npm itself
|
||||
npm cache clean --force # clear a possibly corrupt cache
|
||||
```
|
||||
|
||||
### Installation fails with native module errors
|
||||
|
||||
Some optional language grammars (Dart, Kotlin, Swift) require native compilation. If they fail, GitNexus still works — those languages will be skipped.
|
||||
|
||||
If `npm install -g gitnexus` fails on native modules:
|
||||
|
||||
```bash
|
||||
# Ensure build tools are available (Linux/macOS)
|
||||
# Ubuntu/Debian: sudo apt install python3 make g++
|
||||
# macOS: xcode-select --install
|
||||
|
||||
# Retry installation
|
||||
npm install -g gitnexus
|
||||
```
|
||||
|
||||
### Analysis runs out of memory
|
||||
|
||||
For very large repositories:
|
||||
|
||||
```bash
|
||||
# Increase Node.js heap size
|
||||
NODE_OPTIONS="--max-old-space-size=16384" npx gitnexus analyze
|
||||
|
||||
# Exclude large directories
|
||||
echo "vendor/" >> .gitnexusignore
|
||||
echo "dist/" >> .gitnexusignore
|
||||
```
|
||||
|
||||
## Privacy
|
||||
|
||||
- All processing happens locally on your machine
|
||||
|
|
|
|||
157
gitnexus/package-lock.json
generated
157
gitnexus/package-lock.json
generated
|
|
@ -1,12 +1,13 @@
|
|||
{
|
||||
"name": "gitnexus",
|
||||
"version": "1.5.3",
|
||||
"version": "1.6.1",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "gitnexus",
|
||||
"version": "1.5.3",
|
||||
"version": "1.6.1",
|
||||
"hasInstallScript": true,
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.0.0",
|
||||
|
|
@ -29,7 +30,7 @@
|
|||
"pandemonium": "^2.4.0",
|
||||
"tree-sitter": "^0.21.1",
|
||||
"tree-sitter-c": "0.23.2",
|
||||
"tree-sitter-c-sharp": "^0.23.1",
|
||||
"tree-sitter-c-sharp": "0.23.1",
|
||||
"tree-sitter-cpp": "^0.23.4",
|
||||
"tree-sitter-go": "^0.23.0",
|
||||
"tree-sitter-java": "^0.23.5",
|
||||
|
|
@ -61,8 +62,11 @@
|
|||
"node": ">=20.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4",
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.0",
|
||||
"tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
|
||||
"tree-sitter-kotlin": "^0.3.8",
|
||||
"tree-sitter-proto": "file:./vendor/tree-sitter-proto",
|
||||
"tree-sitter-swift": "^0.6.0"
|
||||
}
|
||||
},
|
||||
|
|
@ -1224,6 +1228,12 @@
|
|||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@ladybugdb/core/node_modules/node-addon-api": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
|
||||
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@modelcontextprotocol/sdk": {
|
||||
"version": "1.28.0",
|
||||
"resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.28.0.tgz",
|
||||
|
|
@ -4116,10 +4126,13 @@
|
|||
}
|
||||
},
|
||||
"node_modules/node-addon-api": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
|
||||
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==",
|
||||
"license": "MIT"
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/node-api-headers": {
|
||||
"version": "1.8.0",
|
||||
|
|
@ -5067,24 +5080,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-c-sharp/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-c/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-cli": {
|
||||
"version": "0.23.2",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.23.2.tgz",
|
||||
|
|
@ -5119,15 +5114,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-cpp/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-dart": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "git+ssh://git@github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
|
||||
|
|
@ -5174,15 +5160,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-go/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-java": {
|
||||
"version": "0.23.5",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.23.5.tgz",
|
||||
|
|
@ -5202,15 +5179,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-java/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-javascript": {
|
||||
"version": "0.23.1",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz",
|
||||
|
|
@ -5230,15 +5198,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-javascript/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-kotlin": {
|
||||
"version": "0.3.8",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-kotlin/-/tree-sitter-kotlin-0.3.8.tgz",
|
||||
|
|
@ -5285,14 +5244,9 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-php/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
"node_modules/tree-sitter-proto": {
|
||||
"resolved": "vendor/tree-sitter-proto",
|
||||
"link": true
|
||||
},
|
||||
"node_modules/tree-sitter-python": {
|
||||
"version": "0.23.4",
|
||||
|
|
@ -5313,15 +5267,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-python/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-ruby": {
|
||||
"version": "0.23.1",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-ruby/-/tree-sitter-ruby-0.23.1.tgz",
|
||||
|
|
@ -5341,15 +5286,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-ruby/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-rust": {
|
||||
"version": "0.23.1",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-rust/-/tree-sitter-rust-0.23.1.tgz",
|
||||
|
|
@ -5369,15 +5305,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-rust/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-swift": {
|
||||
"version": "0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/tree-sitter-swift/-/tree-sitter-swift-0.6.0.tgz",
|
||||
|
|
@ -5407,16 +5334,6 @@
|
|||
"license": "ISC",
|
||||
"optional": true
|
||||
},
|
||||
"node_modules/tree-sitter-swift/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-swift/node_modules/which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
|
|
@ -5453,24 +5370,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter-typescript/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tree-sitter/node_modules/node-addon-api": {
|
||||
"version": "8.7.0",
|
||||
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
|
||||
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "^18 || ^20 || >= 21"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
|
|
@ -5875,6 +5774,14 @@
|
|||
"peerDependencies": {
|
||||
"zod": "^3.25.28 || ^4"
|
||||
}
|
||||
},
|
||||
"vendor/tree-sitter-proto": {
|
||||
"version": "0.4.1",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"peerDependencies": {
|
||||
"tree-sitter": ">=0.21.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "gitnexus",
|
||||
"version": "1.5.3",
|
||||
"version": "1.6.1",
|
||||
"description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
|
||||
"author": "Abhigyan Patwari",
|
||||
"license": "PolyForm-Noncommercial-1.0.0",
|
||||
|
|
@ -46,6 +46,7 @@
|
|||
"test:integration": "vitest run test/integration",
|
||||
"test:watch": "vitest",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"postinstall": "node scripts/patch-tree-sitter-swift.cjs && node scripts/build-tree-sitter-proto.cjs",
|
||||
"prepare": "node scripts/build.js",
|
||||
"prepack": "node scripts/build.js"
|
||||
},
|
||||
|
|
@ -58,7 +59,6 @@
|
|||
"commander": "^12.0.0",
|
||||
"cors": "^2.8.5",
|
||||
"express": "^4.19.2",
|
||||
"gitnexus-shared": "file:../gitnexus-shared",
|
||||
"glob": "^11.0.0",
|
||||
"graphology": "^0.25.4",
|
||||
"graphology-indices": "^0.17.0",
|
||||
|
|
@ -71,7 +71,7 @@
|
|||
"pandemonium": "^2.4.0",
|
||||
"tree-sitter": "^0.21.1",
|
||||
"tree-sitter-c": "0.23.2",
|
||||
"tree-sitter-c-sharp": "^0.23.1",
|
||||
"tree-sitter-c-sharp": "0.23.1",
|
||||
"tree-sitter-cpp": "^0.23.4",
|
||||
"tree-sitter-go": "^0.23.0",
|
||||
"tree-sitter-java": "^0.23.5",
|
||||
|
|
@ -84,8 +84,11 @@
|
|||
"uuid": "^13.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4",
|
||||
"node-addon-api": "^8.0.0",
|
||||
"node-gyp-build": "^4.8.0",
|
||||
"tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
|
||||
"tree-sitter-kotlin": "^0.3.8",
|
||||
"tree-sitter-proto": "file:./vendor/tree-sitter-proto",
|
||||
"tree-sitter-swift": "^0.6.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
82
gitnexus/scripts/build-tree-sitter-proto.cjs
Normal file
82
gitnexus/scripts/build-tree-sitter-proto.cjs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build tree-sitter-proto native binding.
|
||||
*
|
||||
* Why this script exists:
|
||||
* tree-sitter-proto is vendored under gitnexus/vendor/tree-sitter-proto/
|
||||
* and declared as a `file:` optionalDependency. Previously, the vendored
|
||||
* package had its own `dependencies` and `install` script, which caused
|
||||
* npm to create `vendor/tree-sitter-proto/node_modules/` and
|
||||
* `vendor/tree-sitter-proto/build/` during install. Those directories
|
||||
* blocked `rmdir` on global-install upgrade, producing:
|
||||
*
|
||||
* ENOTEMPTY: directory not empty, rmdir
|
||||
* '.../gitnexus/vendor/tree-sitter-proto/node_modules/node-addon-api'
|
||||
*
|
||||
* (See https://github.com/abhigyanpatwari/GitNexus/issues/836.)
|
||||
*
|
||||
* We stripped `dependencies` and the `install` script from the vendored
|
||||
* package.json, hoisted `node-addon-api` and `node-gyp-build` into
|
||||
* gitnexus's own optionalDependencies, and moved native compilation here.
|
||||
*
|
||||
* What this does:
|
||||
* Runs `npx node-gyp rebuild` inside `node_modules/tree-sitter-proto/`
|
||||
* (which npm creates as a copy of vendor/tree-sitter-proto/ when
|
||||
* resolving the file: dep). Build output lands in
|
||||
* `node_modules/tree-sitter-proto/build/Release/tree_sitter_proto_binding.node`
|
||||
* — under npm-managed territory, safe on upgrade.
|
||||
*
|
||||
* Mirrors scripts/patch-tree-sitter-swift.cjs. Best-effort: if any
|
||||
* precondition fails (optional dep absent, no toolchain, --ignore-scripts),
|
||||
* warn and exit 0 so gitnexus install still succeeds.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto');
|
||||
const bindingGyp = path.join(protoDir, 'binding.gyp');
|
||||
const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node');
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(bindingGyp)) {
|
||||
// tree-sitter-proto is an optionalDependency; absent when install
|
||||
// skipped optional deps or the file: dep was not resolved.
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Skip if the native binding already exists (idempotent re-run).
|
||||
if (fs.existsSync(bindingNode)) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Pre-flight: the hoisted build deps must be resolvable.
|
||||
try {
|
||||
require.resolve('node-addon-api');
|
||||
require.resolve('node-gyp-build');
|
||||
} catch (resolveErr) {
|
||||
console.warn(
|
||||
'[tree-sitter-proto] Skipping build: hoisted build deps not resolvable (%s).',
|
||||
resolveErr.message,
|
||||
);
|
||||
console.warn(
|
||||
'[tree-sitter-proto] Proto parsing will be unavailable. Install without --no-optional and with scripts enabled to build.',
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
console.log('[tree-sitter-proto] Building native binding...');
|
||||
execSync('npx node-gyp rebuild', {
|
||||
cwd: protoDir,
|
||||
stdio: 'pipe',
|
||||
timeout: 180000,
|
||||
});
|
||||
console.log('[tree-sitter-proto] Native binding built successfully');
|
||||
} catch (err) {
|
||||
console.warn('[tree-sitter-proto] Could not build native binding:', err.message);
|
||||
console.warn(
|
||||
'[tree-sitter-proto] Proto (.proto) parsing will be unavailable. Non-proto gitnexus functionality is unaffected.',
|
||||
);
|
||||
// Exit 0: optionalDependency failures must not fail the gitnexus install.
|
||||
process.exit(0);
|
||||
}
|
||||
78
gitnexus/scripts/patch-tree-sitter-swift.cjs
Normal file
78
gitnexus/scripts/patch-tree-sitter-swift.cjs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* WORKAROUND: tree-sitter-swift@0.6.0 binding.gyp build failure
|
||||
*
|
||||
* Background:
|
||||
* tree-sitter-swift@0.6.0's binding.gyp contains an "actions" array that
|
||||
* invokes `tree-sitter generate` to regenerate parser.c from grammar.js.
|
||||
* This is intended for grammar developers, but the published npm package
|
||||
* already ships pre-generated parser files (parser.c, scanner.c), so the
|
||||
* actions are unnecessary for consumers. Since consumers don't have
|
||||
* tree-sitter-cli installed, the actions always fail during `npm install`.
|
||||
*
|
||||
* Why we can't just upgrade:
|
||||
* tree-sitter-swift@0.7.1 fixes this (removes postinstall, ships prebuilds),
|
||||
* but it requires tree-sitter@^0.22.1. The upstream project pins tree-sitter
|
||||
* to ^0.21.0 and all other grammar packages depend on that version.
|
||||
* Upgrading tree-sitter would be a separate breaking change.
|
||||
*
|
||||
* How this workaround works:
|
||||
* 1. tree-sitter-swift's own postinstall fails (npm warns but continues)
|
||||
* 2. This script runs as gitnexus's postinstall
|
||||
* 3. It removes the "actions" array from binding.gyp
|
||||
* 4. It rebuilds the native binding with the cleaned binding.gyp
|
||||
*
|
||||
* TODO: Remove this script when tree-sitter is upgraded to ^0.22.x,
|
||||
* which allows using tree-sitter-swift@0.7.1+ directly.
|
||||
*/
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { execSync } = require('child_process');
|
||||
|
||||
const swiftDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-swift');
|
||||
const bindingPath = path.join(swiftDir, 'binding.gyp');
|
||||
|
||||
try {
|
||||
if (!fs.existsSync(bindingPath)) {
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const content = fs.readFileSync(bindingPath, 'utf8');
|
||||
let needsRebuild = false;
|
||||
|
||||
if (content.includes('"actions"')) {
|
||||
// Strip Python-style comments (#) and trailing commas before JSON parsing
|
||||
const cleaned = content
|
||||
.replace(/#[^\n]*/g, '') // Remove # comments
|
||||
.replace(/,(\s*[\]}])/g, '$1'); // Remove trailing commas before ] or }
|
||||
const gyp = JSON.parse(cleaned);
|
||||
|
||||
if (gyp.targets && gyp.targets[0] && gyp.targets[0].actions) {
|
||||
delete gyp.targets[0].actions;
|
||||
fs.writeFileSync(bindingPath, JSON.stringify(gyp, null, 2) + '\n');
|
||||
console.log('[tree-sitter-swift] Patched binding.gyp (removed actions array)');
|
||||
needsRebuild = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if native binding exists
|
||||
const bindingNode = path.join(swiftDir, 'build', 'Release', 'tree_sitter_swift_binding.node');
|
||||
if (!fs.existsSync(bindingNode)) {
|
||||
needsRebuild = true;
|
||||
}
|
||||
|
||||
if (needsRebuild) {
|
||||
console.log('[tree-sitter-swift] Rebuilding native binding...');
|
||||
execSync('npx node-gyp rebuild', {
|
||||
cwd: swiftDir,
|
||||
stdio: 'pipe',
|
||||
timeout: 120000,
|
||||
});
|
||||
console.log('[tree-sitter-swift] Native binding built successfully');
|
||||
}
|
||||
} catch (err) {
|
||||
console.warn('[tree-sitter-swift] Could not build native binding:', err.message);
|
||||
console.warn(
|
||||
'[tree-sitter-swift] You may need to manually run: cd node_modules/tree-sitter-swift && npx node-gyp rebuild',
|
||||
);
|
||||
}
|
||||
|
|
@ -26,6 +26,7 @@ interface RepoStats {
|
|||
|
||||
export interface AIContextOptions {
|
||||
skipAgentsMd?: boolean;
|
||||
noStats?: boolean;
|
||||
}
|
||||
|
||||
const GITNEXUS_START_MARKER = '<!-- gitnexus:start -->';
|
||||
|
|
@ -64,6 +65,7 @@ function generateGitNexusContent(
|
|||
stats: RepoStats,
|
||||
generatedSkills?: GeneratedSkillInfo[],
|
||||
groupNames?: string[],
|
||||
noStats?: boolean,
|
||||
): string {
|
||||
const generatedRows =
|
||||
generatedSkills && generatedSkills.length > 0
|
||||
|
|
@ -87,7 +89,7 @@ function generateGitNexusContent(
|
|||
return `${GITNEXUS_START_MARKER}
|
||||
# GitNexus — Code Intelligence
|
||||
|
||||
This project is indexed by GitNexus as **${projectName}** (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
|
||||
This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows)`}. Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
|
||||
|
||||
> If any GitNexus tool warns the index is stale, run \`npx gitnexus analyze\` in terminal first.
|
||||
|
||||
|
|
@ -332,7 +334,13 @@ export async function generateAIContextFiles(
|
|||
options?: AIContextOptions,
|
||||
): Promise<{ files: string[] }> {
|
||||
const groupNames = await findGroupsContainingRegistryName(projectName);
|
||||
const content = generateGitNexusContent(projectName, stats, generatedSkills, groupNames);
|
||||
const content = generateGitNexusContent(
|
||||
projectName,
|
||||
stats,
|
||||
generatedSkills,
|
||||
groupNames,
|
||||
options?.noStats,
|
||||
);
|
||||
const createdFiles: string[] = [];
|
||||
|
||||
if (!options?.skipAgentsMd) {
|
||||
|
|
|
|||
|
|
@ -20,8 +20,11 @@ import fs from 'fs/promises';
|
|||
|
||||
const HEAP_MB = 8192;
|
||||
const HEAP_FLAG = `--max-old-space-size=${HEAP_MB}`;
|
||||
/** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */
|
||||
const STACK_KB = 4096;
|
||||
const STACK_FLAG = `--stack-size=${STACK_KB}`;
|
||||
|
||||
/** Re-exec the process with an 8GB heap if we're currently below that. */
|
||||
/** Re-exec the process with an 8GB heap and larger stack if we're currently below that. */
|
||||
function ensureHeap(): boolean {
|
||||
const nodeOpts = process.env.NODE_OPTIONS || '';
|
||||
if (nodeOpts.includes('--max-old-space-size')) return false;
|
||||
|
|
@ -29,8 +32,13 @@ function ensureHeap(): boolean {
|
|||
const v8Heap = v8.getHeapStatistics().heap_size_limit;
|
||||
if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false;
|
||||
|
||||
// --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+,
|
||||
// so pass it only as a direct CLI argument, not via the environment.
|
||||
const cliFlags = [HEAP_FLAG];
|
||||
if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG);
|
||||
|
||||
try {
|
||||
execFileSync(process.execPath, [HEAP_FLAG, ...process.argv.slice(1)], {
|
||||
execFileSync(process.execPath, [...cliFlags, ...process.argv.slice(1)], {
|
||||
stdio: 'inherit',
|
||||
env: { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim() },
|
||||
});
|
||||
|
|
@ -47,6 +55,8 @@ export interface AnalyzeOptions {
|
|||
verbose?: boolean;
|
||||
/** Skip AGENTS.md and CLAUDE.md gitnexus block updates. */
|
||||
skipAgentsMd?: boolean;
|
||||
/** Omit volatile symbol/relationship counts from AGENTS.md and CLAUDE.md. */
|
||||
noStats?: boolean;
|
||||
/** Index the folder even when no .git directory is present. */
|
||||
skipGit?: boolean;
|
||||
}
|
||||
|
|
@ -137,9 +147,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
|||
const origLog = console.log.bind(console);
|
||||
const origWarn = console.warn.bind(console);
|
||||
const origError = console.error.bind(console);
|
||||
let barCurrentValue = 0;
|
||||
const barLog = (...args: any[]) => {
|
||||
process.stdout.write('\x1b[2K\r');
|
||||
origLog(args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
|
||||
bar.update(barCurrentValue);
|
||||
};
|
||||
console.log = barLog;
|
||||
console.warn = barLog;
|
||||
|
|
@ -150,6 +162,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
|||
let phaseStart = Date.now();
|
||||
|
||||
const updateBar = (value: number, phaseLabel: string) => {
|
||||
barCurrentValue = value;
|
||||
if (phaseLabel !== lastPhaseLabel) {
|
||||
lastPhaseLabel = phaseLabel;
|
||||
phaseStart = Date.now();
|
||||
|
|
@ -177,6 +190,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
|||
embeddings: options?.embeddings,
|
||||
skipGit: options?.skipGit,
|
||||
skipAgentsMd: options?.skipAgentsMd,
|
||||
noStats: options?.noStats,
|
||||
},
|
||||
{
|
||||
onProgress: (_phase, percent, message) => {
|
||||
|
|
@ -240,7 +254,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
|||
processes: s.processes,
|
||||
},
|
||||
skillResult.skills,
|
||||
{ skipAgentsMd: options?.skipAgentsMd },
|
||||
{ skipAgentsMd: options?.skipAgentsMd, noStats: options?.noStats },
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
|
|
@ -282,7 +296,51 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
|||
console.warn = origWarn;
|
||||
console.error = origError;
|
||||
bar.stop();
|
||||
console.error(`\n Analysis failed: ${err.message}\n`);
|
||||
|
||||
const msg = err.message || String(err);
|
||||
console.error(`\n Analysis failed: ${msg}\n`);
|
||||
|
||||
// Provide helpful guidance for known failure modes
|
||||
if (
|
||||
msg.includes('Maximum call stack size exceeded') ||
|
||||
msg.includes('call stack') ||
|
||||
msg.includes('Map maximum size') ||
|
||||
msg.includes('Invalid array length') ||
|
||||
msg.includes('Invalid string length') ||
|
||||
msg.includes('allocation failed') ||
|
||||
msg.includes('heap out of memory') ||
|
||||
msg.includes('JavaScript heap')
|
||||
) {
|
||||
console.error(' This error typically occurs on very large repositories.');
|
||||
console.error(' Suggestions:');
|
||||
console.error(' 1. Add large vendored/generated directories to .gitnexusignore');
|
||||
console.error(' 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"');
|
||||
console.error(' 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"');
|
||||
console.error('');
|
||||
} else if (msg.includes('ERESOLVE') || msg.includes('Could not resolve dependency')) {
|
||||
// Note: the original arborist "Cannot destructure property 'package' of
|
||||
// 'node.target'" crash happens inside npm *before* gitnexus code runs,
|
||||
// so it can't be caught here. This branch handles dependency-resolution
|
||||
// errors that surface at runtime (e.g. dynamic require failures).
|
||||
console.error(' This looks like an npm dependency resolution issue.');
|
||||
console.error(' Suggestions:');
|
||||
console.error(' 1. Clear the npm cache: npm cache clean --force');
|
||||
console.error(' 2. Update npm: npm install -g npm@latest');
|
||||
console.error(' 3. Reinstall gitnexus: npm install -g gitnexus@latest');
|
||||
console.error(' 4. Or try npx directly: npx gitnexus@latest analyze');
|
||||
console.error('');
|
||||
} else if (
|
||||
msg.includes('MODULE_NOT_FOUND') ||
|
||||
msg.includes('Cannot find module') ||
|
||||
msg.includes('ERR_MODULE_NOT_FOUND')
|
||||
) {
|
||||
console.error(' A required module could not be loaded. The installation may be corrupt.');
|
||||
console.error(' Suggestions:');
|
||||
console.error(' 1. Reinstall: npm install -g gitnexus@latest');
|
||||
console.error(' 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze');
|
||||
console.error('');
|
||||
}
|
||||
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ program
|
|||
.option('--embeddings', 'Enable embedding generation for semantic search (off by default)')
|
||||
.option('--skills', 'Generate repo-specific skill files from detected communities')
|
||||
.option('--skip-agents-md', 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md')
|
||||
.option('--no-stats', 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md')
|
||||
.option('--skip-git', 'Index a folder without requiring a .git directory')
|
||||
.option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)')
|
||||
.addHelpText(
|
||||
|
|
|
|||
|
|
@ -265,7 +265,7 @@ async function setupOpenCode(result: SetupResult): Promise<void> {
|
|||
return;
|
||||
}
|
||||
|
||||
const configPath = path.join(opencodeDir, 'config.json');
|
||||
const configPath = path.join(opencodeDir, 'opencode.json');
|
||||
try {
|
||||
const existing = await readJsonFile(configPath);
|
||||
const config = existing || {};
|
||||
|
|
|
|||
112
gitnexus/src/core/embeddings/ast-utils.ts
Normal file
112
gitnexus/src/core/embeddings/ast-utils.ts
Normal file
|
|
@ -0,0 +1,112 @@
|
|||
/**
|
||||
* Shared AST utilities for the embedding pipeline.
|
||||
* Centralizes parser caching and tree-sitter node lookups
|
||||
* used by both chunker.ts and structural-extractor.ts.
|
||||
*/
|
||||
|
||||
import { getLanguageFromFilename } from 'gitnexus-shared';
|
||||
import {
|
||||
createParserForLanguage,
|
||||
isLanguageAvailable,
|
||||
resolveLanguageKey,
|
||||
} from '../tree-sitter/parser-loader.js';
|
||||
|
||||
const parserCache = new Map<string, any>();
|
||||
|
||||
/**
|
||||
* Ensure parser is initialized and language is loaded, then parse content.
|
||||
* Returns null if language is unavailable or parsing fails.
|
||||
*/
|
||||
export const ensureAndParse = async (content: string, filePath: string): Promise<any | null> => {
|
||||
const language = getLanguageFromFilename(filePath);
|
||||
if (!language) return null;
|
||||
if (!isLanguageAvailable(language)) return null;
|
||||
|
||||
const parserKey = resolveLanguageKey(language, filePath);
|
||||
let parserInstance = parserCache.get(parserKey);
|
||||
if (!parserInstance) {
|
||||
parserInstance = await createParserForLanguage(language, filePath);
|
||||
parserCache.set(parserKey, parserInstance);
|
||||
}
|
||||
|
||||
return parserInstance.parse(content);
|
||||
};
|
||||
|
||||
const FUNCTION_LIKE_TYPES = new Set([
|
||||
'function_declaration',
|
||||
'function_definition',
|
||||
'method_declaration',
|
||||
'method_definition',
|
||||
'function_item',
|
||||
'function_signature_item',
|
||||
'arrow_function',
|
||||
'function_expression',
|
||||
'generator_function_declaration',
|
||||
'generator_function',
|
||||
'async_function_declaration',
|
||||
'async_arrow_function',
|
||||
'constructor_declaration',
|
||||
'constructor_definition',
|
||||
'compact_constructor_declaration',
|
||||
'short_function_declaration',
|
||||
'proc_declaration',
|
||||
'func_literal',
|
||||
'local_function_statement',
|
||||
'anonymous_function',
|
||||
'lambda_literal',
|
||||
'init_declaration',
|
||||
'deinit_declaration',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Find the first function/method-like declaration in a snippet AST.
|
||||
* Used by the chunker when parsing node.content where absolute line
|
||||
* numbers don't apply.
|
||||
*/
|
||||
export const findFunctionNode = (root: any): any | null => {
|
||||
if (FUNCTION_LIKE_TYPES.has(root.type)) return root;
|
||||
|
||||
for (let i = 0; i < root.namedChildCount; i++) {
|
||||
const child = root.namedChild(i);
|
||||
if (!child) continue;
|
||||
if (FUNCTION_LIKE_TYPES.has(child.type)) return child;
|
||||
const found = findFunctionNode(child);
|
||||
if (found) return found;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
/**
|
||||
* Find the first class/struct/interface/enum-like declaration in an AST.
|
||||
* Used when parsing node.content (a snippet, not a full file) where
|
||||
* absolute line numbers don't apply.
|
||||
*/
|
||||
export const findDeclarationNode = (root: any): any | null => {
|
||||
const CLASS_LIKE_TYPES = new Set([
|
||||
'class_declaration',
|
||||
'class_definition',
|
||||
'struct_declaration',
|
||||
'struct_item',
|
||||
'interface_declaration',
|
||||
'interface_definition',
|
||||
'enum_declaration',
|
||||
'enum_item',
|
||||
'type_declaration', // Go: type X struct
|
||||
'declaration', // Go: type X struct
|
||||
'object_declaration', // Kotlin: object
|
||||
'impl_item', // Rust: impl
|
||||
]);
|
||||
|
||||
if (CLASS_LIKE_TYPES.has(root.type)) return root;
|
||||
|
||||
for (let i = 0; i < root.namedChildCount; i++) {
|
||||
const child = root.namedChild(i);
|
||||
if (!child) continue;
|
||||
if (CLASS_LIKE_TYPES.has(child.type)) return child;
|
||||
const found = findDeclarationNode(child);
|
||||
if (found) return found;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
63
gitnexus/src/core/embeddings/character-chunk.ts
Normal file
63
gitnexus/src/core/embeddings/character-chunk.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
/**
|
||||
* Character-based sliding window chunking (pure, no tree-sitter dependency)
|
||||
*/
|
||||
|
||||
import { buildLineIndex, resolveChunkLines } from './line-index.js';
|
||||
|
||||
export interface Chunk {
|
||||
text: string;
|
||||
chunkIndex: number;
|
||||
startOffset: number;
|
||||
endOffset: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
export const characterChunk = (
|
||||
content: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number = 1200,
|
||||
overlap: number = 120,
|
||||
): Chunk[] => {
|
||||
if (content.length <= chunkSize) {
|
||||
return [
|
||||
{
|
||||
text: content,
|
||||
chunkIndex: 0,
|
||||
startOffset: 0,
|
||||
endOffset: content.length,
|
||||
startLine,
|
||||
endLine,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const chunks: Chunk[] = [];
|
||||
let offset = 0;
|
||||
const lineOffsets = buildLineIndex(content);
|
||||
|
||||
while (offset < content.length) {
|
||||
const end = Math.min(offset + chunkSize, content.length);
|
||||
const chunkText = content.slice(offset, end);
|
||||
const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine);
|
||||
|
||||
chunks.push({
|
||||
text: chunkText,
|
||||
chunkIndex: chunks.length,
|
||||
startOffset: offset,
|
||||
endOffset: end,
|
||||
startLine: lineRange.startLine,
|
||||
endLine: lineRange.endLine,
|
||||
});
|
||||
|
||||
offset = end - overlap;
|
||||
if (offset >= content.length) break;
|
||||
if (end >= content.length) break;
|
||||
if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) {
|
||||
offset = end;
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
};
|
||||
363
gitnexus/src/core/embeddings/chunker.ts
Normal file
363
gitnexus/src/core/embeddings/chunker.ts
Normal file
|
|
@ -0,0 +1,363 @@
|
|||
/**
|
||||
* Chunker Module
|
||||
*
|
||||
* Splits code nodes into chunks for embedding.
|
||||
* - Function/Method: AST-aware chunking by statement boundaries
|
||||
* - Other types: character-based sliding window fallback
|
||||
* - Short content (≤ chunkSize): no chunking
|
||||
*/
|
||||
|
||||
export { type Chunk, characterChunk } from './character-chunk.js';
|
||||
|
||||
import { characterChunk } from './character-chunk.js';
|
||||
import type { Chunk } from './character-chunk.js';
|
||||
import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js';
|
||||
import { buildLineIndex, resolveChunkLines } from './line-index.js';
|
||||
|
||||
/**
|
||||
* Main chunkNode function: dispatches by label
|
||||
*/
|
||||
export const chunkNode = async (
|
||||
label: string,
|
||||
content: string,
|
||||
filePath: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number = 1200,
|
||||
overlap: number = 120,
|
||||
): Promise<Chunk[]> => {
|
||||
// Content fits in one chunk — no splitting needed
|
||||
if (content.length <= chunkSize) {
|
||||
return [
|
||||
{
|
||||
text: content,
|
||||
chunkIndex: 0,
|
||||
startOffset: 0,
|
||||
endOffset: content.length,
|
||||
startLine,
|
||||
endLine,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// Only function-like labels get AST chunking
|
||||
if (label === 'Function' || label === 'Method' || label === 'Constructor') {
|
||||
try {
|
||||
const astChunks = await astChunk(content, filePath, startLine, endLine, chunkSize, overlap);
|
||||
if (astChunks.length > 0) return astChunks;
|
||||
} catch {
|
||||
// AST parsing failed — fall through to character fallback
|
||||
}
|
||||
}
|
||||
|
||||
if (label === 'Class' || label === 'Interface') {
|
||||
try {
|
||||
const declarationChunks = await declarationChunk(
|
||||
label,
|
||||
content,
|
||||
filePath,
|
||||
startLine,
|
||||
endLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
);
|
||||
if (declarationChunks.length > 0) return declarationChunks;
|
||||
} catch {
|
||||
// AST parsing failed — fall through to character fallback
|
||||
}
|
||||
}
|
||||
|
||||
// Character-based fallback for everything else
|
||||
return characterChunk(content, startLine, endLine, chunkSize, overlap);
|
||||
};
|
||||
|
||||
/**
|
||||
* AST-based chunking for Function/Method
|
||||
* Parse snippet content, locate the function declaration node,
|
||||
* split body by statement boundaries.
|
||||
*/
|
||||
const astChunk = async (
|
||||
content: string,
|
||||
filePath: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number,
|
||||
overlap: number,
|
||||
): Promise<Chunk[]> => {
|
||||
const tree = await ensureAndParse(content, filePath);
|
||||
if (!tree) return [];
|
||||
|
||||
const root = tree.rootNode;
|
||||
const lineOffsets = buildLineIndex(content);
|
||||
|
||||
// Find the function/method declaration in the snippet AST.
|
||||
// tree-sitter parses node.content (a snippet), so rows are relative (0-based).
|
||||
const targetNode = findFunctionNode(root);
|
||||
if (!targetNode) return [];
|
||||
|
||||
// Get the body (statements) via childForFieldName('body')
|
||||
const bodyNode = targetNode.childForFieldName('body');
|
||||
if (!bodyNode) return [];
|
||||
|
||||
// Extract individual statements
|
||||
const statements: Array<{ startIndex: number; endIndex: number }> = [];
|
||||
for (let i = 0; i < bodyNode.namedChildCount; i++) {
|
||||
const child = bodyNode.namedChild(i);
|
||||
if (!child) continue;
|
||||
statements.push({
|
||||
startIndex: child.startIndex,
|
||||
endIndex: child.endIndex,
|
||||
});
|
||||
}
|
||||
|
||||
if (statements.length === 0) return [];
|
||||
|
||||
return chunkByUnits(
|
||||
content,
|
||||
lineOffsets,
|
||||
startLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
statements,
|
||||
targetNode.startIndex,
|
||||
targetNode.endIndex,
|
||||
true,
|
||||
true,
|
||||
);
|
||||
};
|
||||
|
||||
const DECLARATION_BODY_NODE_TYPES = new Set([
|
||||
'class_body',
|
||||
'object_type',
|
||||
'declaration_list',
|
||||
'interface_body',
|
||||
]);
|
||||
|
||||
const FIELD_LIKE_MEMBER_TYPES = new Set([
|
||||
'field_definition',
|
||||
'public_field_definition',
|
||||
'property_definition',
|
||||
'property_signature',
|
||||
'variable_declarator',
|
||||
'lexical_declaration',
|
||||
'pair',
|
||||
'enum_assignment',
|
||||
]);
|
||||
|
||||
const declarationChunk = async (
|
||||
label: 'Class' | 'Interface',
|
||||
content: string,
|
||||
filePath: string,
|
||||
startLine: number,
|
||||
endLine: number,
|
||||
chunkSize: number,
|
||||
overlap: number,
|
||||
): Promise<Chunk[]> => {
|
||||
const tree = await ensureAndParse(content, filePath);
|
||||
if (!tree) return [];
|
||||
|
||||
const targetNode = findDeclarationNode(tree.rootNode);
|
||||
if (!targetNode) return [];
|
||||
|
||||
const bodyNode = getDeclarationBodyNode(targetNode);
|
||||
if (!bodyNode) return [];
|
||||
|
||||
const members = collectDeclarationUnits(bodyNode, label);
|
||||
if (members.length === 0) return [];
|
||||
|
||||
return chunkByUnits(
|
||||
content,
|
||||
buildLineIndex(content),
|
||||
startLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
members,
|
||||
targetNode.startIndex,
|
||||
targetNode.endIndex,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
};
|
||||
|
||||
const buildChunk = (
|
||||
content: string,
|
||||
lineOffsets: Int32Array,
|
||||
chunkIndex: number,
|
||||
startOffset: number,
|
||||
endOffset: number,
|
||||
baseStartLine: number,
|
||||
): Chunk => {
|
||||
const lineRange = resolveChunkLines(lineOffsets, startOffset, endOffset, baseStartLine);
|
||||
return {
|
||||
text: content.slice(startOffset, endOffset),
|
||||
chunkIndex,
|
||||
startOffset,
|
||||
endOffset,
|
||||
startLine: lineRange.startLine,
|
||||
endLine: lineRange.endLine,
|
||||
};
|
||||
};
|
||||
|
||||
const chunkByUnits = (
|
||||
content: string,
|
||||
lineOffsets: Int32Array,
|
||||
baseStartLine: number,
|
||||
chunkSize: number,
|
||||
overlap: number,
|
||||
units: Array<{ startIndex: number; endIndex: number }>,
|
||||
containerStartOffset: number,
|
||||
containerEndOffset: number,
|
||||
includeContainerPrefixOnFirstChunk: boolean,
|
||||
includeContainerSuffixOnLastChunk: boolean,
|
||||
): Chunk[] => {
|
||||
const chunks: Chunk[] = [];
|
||||
let chunkStartUnitIdx = 0;
|
||||
|
||||
while (chunkStartUnitIdx < units.length) {
|
||||
const chunkStartOffset =
|
||||
chunkStartUnitIdx === 0 && includeContainerPrefixOnFirstChunk
|
||||
? containerStartOffset
|
||||
: units[chunkStartUnitIdx].startIndex;
|
||||
|
||||
let chunkEndUnitIdx = chunkStartUnitIdx;
|
||||
let candidateEndOffset =
|
||||
chunkEndUnitIdx === units.length - 1 && includeContainerSuffixOnLastChunk
|
||||
? containerEndOffset
|
||||
: units[chunkEndUnitIdx].endIndex;
|
||||
|
||||
while (chunkEndUnitIdx + 1 < units.length) {
|
||||
const nextEndOffset =
|
||||
chunkEndUnitIdx + 1 === units.length - 1 && includeContainerSuffixOnLastChunk
|
||||
? containerEndOffset
|
||||
: units[chunkEndUnitIdx + 1].endIndex;
|
||||
if (nextEndOffset - chunkStartOffset > chunkSize) break;
|
||||
chunkEndUnitIdx += 1;
|
||||
candidateEndOffset = nextEndOffset;
|
||||
}
|
||||
|
||||
if (candidateEndOffset - chunkStartOffset > chunkSize) {
|
||||
const oversizedUnit = units[chunkStartUnitIdx];
|
||||
const oversizedLineRange = resolveChunkLines(
|
||||
lineOffsets,
|
||||
oversizedUnit.startIndex,
|
||||
oversizedUnit.endIndex,
|
||||
baseStartLine,
|
||||
);
|
||||
const oversizedChunks = characterChunk(
|
||||
content.slice(oversizedUnit.startIndex, oversizedUnit.endIndex),
|
||||
oversizedLineRange.startLine,
|
||||
oversizedLineRange.endLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
).map((chunk, offsetIdx) => ({
|
||||
...chunk,
|
||||
chunkIndex: chunks.length + offsetIdx,
|
||||
startOffset: chunk.startOffset + oversizedUnit.startIndex,
|
||||
endOffset: chunk.endOffset + oversizedUnit.startIndex,
|
||||
}));
|
||||
chunks.push(...oversizedChunks);
|
||||
chunkStartUnitIdx += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
chunks.push(
|
||||
buildChunk(
|
||||
content,
|
||||
lineOffsets,
|
||||
chunks.length,
|
||||
chunkStartOffset,
|
||||
candidateEndOffset,
|
||||
baseStartLine,
|
||||
),
|
||||
);
|
||||
|
||||
if (chunkEndUnitIdx === units.length - 1) {
|
||||
break;
|
||||
}
|
||||
|
||||
const nextChunkStartUnitIdx = findOverlapStartIndex(
|
||||
units,
|
||||
chunkStartUnitIdx,
|
||||
chunkEndUnitIdx,
|
||||
overlap,
|
||||
);
|
||||
if (nextChunkStartUnitIdx <= chunkStartUnitIdx) {
|
||||
chunkStartUnitIdx = chunkEndUnitIdx + 1;
|
||||
} else {
|
||||
chunkStartUnitIdx = nextChunkStartUnitIdx;
|
||||
}
|
||||
}
|
||||
|
||||
return chunks;
|
||||
};
|
||||
|
||||
const findOverlapStartIndex = (
|
||||
statements: Array<{ startIndex: number; endIndex: number }>,
|
||||
chunkStartStmtIdx: number,
|
||||
chunkEndStmtIdx: number,
|
||||
overlapSize: number,
|
||||
): number => {
|
||||
if (overlapSize <= 0) return chunkEndStmtIdx + 1;
|
||||
|
||||
let overlapStartIdx = chunkEndStmtIdx;
|
||||
while (overlapStartIdx > chunkStartStmtIdx) {
|
||||
const overlapLength =
|
||||
statements[chunkEndStmtIdx].endIndex - statements[overlapStartIdx - 1].startIndex;
|
||||
if (overlapLength > overlapSize) break;
|
||||
overlapStartIdx -= 1;
|
||||
}
|
||||
|
||||
return overlapStartIdx;
|
||||
};
|
||||
|
||||
const getDeclarationBodyNode = (node: any): any | null => {
|
||||
const bodyNode = node.childForFieldName?.('body');
|
||||
if (bodyNode) return bodyNode;
|
||||
|
||||
for (let i = 0; i < node.namedChildCount; i++) {
|
||||
const child = node.namedChild(i);
|
||||
if (!child) continue;
|
||||
if (DECLARATION_BODY_NODE_TYPES.has(child.type)) return child;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const collectDeclarationUnits = (
|
||||
bodyNode: any,
|
||||
label: 'Class' | 'Interface',
|
||||
): Array<{ startIndex: number; endIndex: number }> => {
|
||||
const members: Array<{ startIndex: number; endIndex: number; groupable: boolean }> = [];
|
||||
|
||||
for (let i = 0; i < bodyNode.namedChildCount; i++) {
|
||||
const child = bodyNode.namedChild(i);
|
||||
if (!child) continue;
|
||||
members.push({
|
||||
startIndex: child.startIndex,
|
||||
endIndex: child.endIndex,
|
||||
groupable: label === 'Class' && FIELD_LIKE_MEMBER_TYPES.has(child.type),
|
||||
});
|
||||
}
|
||||
|
||||
if (members.length === 0) return [];
|
||||
|
||||
const grouped: Array<{ startIndex: number; endIndex: number }> = [];
|
||||
let current = members[0];
|
||||
|
||||
for (let i = 1; i < members.length; i++) {
|
||||
const next = members[i];
|
||||
if (current.groupable && next.groupable) {
|
||||
current = {
|
||||
startIndex: current.startIndex,
|
||||
endIndex: next.endIndex,
|
||||
groupable: true,
|
||||
};
|
||||
continue;
|
||||
}
|
||||
grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex });
|
||||
current = next;
|
||||
}
|
||||
|
||||
grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex });
|
||||
return grouped;
|
||||
};
|
||||
|
|
@ -157,6 +157,11 @@ export const initEmbedder = async (
|
|||
try {
|
||||
// Configure transformers.js environment
|
||||
env.allowLocalModels = false;
|
||||
// Default cache to user-writable location. transformers.js defaults to
|
||||
// ./node_modules/.cache inside its own install dir, which is unwritable
|
||||
// when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
|
||||
// Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
|
||||
env.cacheDir = process.env.HF_HOME ?? `${process.env.HOME}/.cache/huggingface`;
|
||||
|
||||
const isDev = process.env.NODE_ENV === 'development';
|
||||
if (isDev) {
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@
|
|||
*
|
||||
* Orchestrates the background embedding process:
|
||||
* 1. Query embeddable nodes from LadybugDB
|
||||
* 2. Generate text representations
|
||||
* 3. Batch embed using transformers.js
|
||||
* 4. Update LadybugDB with embeddings
|
||||
* 2. Generate text representations with enriched metadata
|
||||
* 3. Chunk long nodes, batch embed
|
||||
* 4. Update LadybugDB with chunk-aware embeddings
|
||||
* 5. Create vector index for semantic search
|
||||
*/
|
||||
|
||||
import { createHash } from 'crypto';
|
||||
import {
|
||||
initEmbedder,
|
||||
embedBatch,
|
||||
|
|
@ -16,19 +17,54 @@ import {
|
|||
embeddingToArray,
|
||||
isEmbedderReady,
|
||||
} from './embedder.js';
|
||||
import { generateBatchEmbeddingTexts } from './text-generator.js';
|
||||
import { generateEmbeddingText } from './text-generator.js';
|
||||
import { chunkNode, characterChunk } from './chunker.js';
|
||||
import { extractStructuralNames } from './structural-extractor.js';
|
||||
import {
|
||||
type EmbeddingProgress,
|
||||
type EmbeddingConfig,
|
||||
type EmbeddableNode,
|
||||
type SemanticSearchResult,
|
||||
type ModelProgress,
|
||||
type EmbeddingContext,
|
||||
DEFAULT_EMBEDDING_CONFIG,
|
||||
EMBEDDABLE_LABELS,
|
||||
isShortLabel,
|
||||
LABELS_WITH_EXPORTED,
|
||||
STRUCTURAL_LABELS,
|
||||
collectBestChunks,
|
||||
} from './types.js';
|
||||
import {
|
||||
EMBEDDING_TABLE_NAME,
|
||||
EMBEDDING_INDEX_NAME,
|
||||
CREATE_VECTOR_INDEX_QUERY,
|
||||
STALE_HASH_SENTINEL,
|
||||
} from '../lbug/schema.js';
|
||||
import { loadVectorExtension } from '../lbug/lbug-adapter.js';
|
||||
|
||||
const isDev = process.env.NODE_ENV === 'development';
|
||||
|
||||
/**
|
||||
* Compute a stable content fingerprint for an embeddable node.
|
||||
* Used to detect when the underlying text has changed so stale vectors
|
||||
* can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for
|
||||
* vector-indexed rows).
|
||||
*/
|
||||
export const contentHashForNode = (
|
||||
node: EmbeddableNode,
|
||||
config: Partial<EmbeddingConfig> = {},
|
||||
): string => {
|
||||
// Hash must be deterministic across runs, so exclude methodNames/fieldNames
|
||||
// which are populated during the batch loop via AST extraction.
|
||||
// Using only node.content ensures the hash stays stable.
|
||||
const text = generateEmbeddingText(
|
||||
{ ...node, methodNames: undefined, fieldNames: undefined },
|
||||
node.content,
|
||||
config,
|
||||
);
|
||||
return createHash('sha1').update(text).digest('hex');
|
||||
};
|
||||
|
||||
/**
|
||||
* Progress callback type
|
||||
*/
|
||||
|
|
@ -36,37 +72,50 @@ export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void;
|
|||
|
||||
/**
|
||||
* Query all embeddable nodes from LadybugDB
|
||||
* Uses table-specific queries (File has different schema than code elements)
|
||||
* Uses table-specific queries for different label types
|
||||
*/
|
||||
const queryEmbeddableNodes = async (
|
||||
executeQuery: (cypher: string) => Promise<any[]>,
|
||||
): Promise<EmbeddableNode[]> => {
|
||||
const allNodes: EmbeddableNode[] = [];
|
||||
|
||||
// Query each embeddable table with table-specific columns
|
||||
for (const label of EMBEDDABLE_LABELS) {
|
||||
try {
|
||||
let query: string;
|
||||
|
||||
if (label === 'File') {
|
||||
// File nodes don't have startLine/endLine
|
||||
if (label === 'Method') {
|
||||
// Method has parameterCount and returnType
|
||||
query = `
|
||||
MATCH (n:File)
|
||||
RETURN n.id AS id, n.name AS name, 'File' AS label,
|
||||
n.filePath AS filePath, n.content AS content
|
||||
MATCH (n:Method)
|
||||
RETURN n.id AS id, n.name AS name, 'Method' AS label,
|
||||
n.filePath AS filePath, n.content AS content,
|
||||
n.startLine AS startLine, n.endLine AS endLine,
|
||||
n.isExported AS isExported, n.description AS description,
|
||||
n.parameterCount AS parameterCount, n.returnType AS returnType
|
||||
`;
|
||||
} else if (LABELS_WITH_EXPORTED.has(label)) {
|
||||
// Function, Class, Interface have isExported and description
|
||||
query = `
|
||||
MATCH (n:\`${label}\`)
|
||||
RETURN n.id AS id, n.name AS name, '${label}' AS label,
|
||||
n.filePath AS filePath, n.content AS content,
|
||||
n.startLine AS startLine, n.endLine AS endLine,
|
||||
n.isExported AS isExported, n.description AS description
|
||||
`;
|
||||
} else {
|
||||
// Code elements have startLine/endLine
|
||||
// Multi-language tables (Struct, Enum, etc.) — have description but no isExported
|
||||
query = `
|
||||
MATCH (n:${label})
|
||||
RETURN n.id AS id, n.name AS name, '${label}' AS label,
|
||||
MATCH (n:\`${label}\`)
|
||||
RETURN n.id AS id, n.name AS name, '${label}' AS label,
|
||||
n.filePath AS filePath, n.content AS content,
|
||||
n.startLine AS startLine, n.endLine AS endLine
|
||||
n.startLine AS startLine, n.endLine AS endLine,
|
||||
n.description AS description
|
||||
`;
|
||||
}
|
||||
|
||||
const rows = await executeQuery(query);
|
||||
for (const row of rows) {
|
||||
const hasExportedColumn = label === 'Method' || LABELS_WITH_EXPORTED.has(label);
|
||||
allNodes.push({
|
||||
id: row.id ?? row[0],
|
||||
name: row.name ?? row[1],
|
||||
|
|
@ -75,10 +124,17 @@ const queryEmbeddableNodes = async (
|
|||
content: row.content ?? row[4] ?? '',
|
||||
startLine: row.startLine ?? row[5],
|
||||
endLine: row.endLine ?? row[6],
|
||||
isExported: hasExportedColumn ? (row.isExported ?? row[7]) : undefined,
|
||||
description: row.description ?? (hasExportedColumn ? row[8] : row[7]),
|
||||
...(label === 'Method'
|
||||
? {
|
||||
parameterCount: row.parameterCount ?? row[9],
|
||||
returnType: row.returnType ?? row[10],
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
// Table might not exist or be empty, continue
|
||||
if (isDev) {
|
||||
console.warn(`Query for ${label} nodes failed:`, error);
|
||||
}
|
||||
|
|
@ -89,52 +145,52 @@ const queryEmbeddableNodes = async (
|
|||
};
|
||||
|
||||
/**
|
||||
* Batch INSERT embeddings into separate CodeEmbedding table
|
||||
* Using a separate lightweight table avoids copy-on-write overhead
|
||||
* that occurs when UPDATEing nodes with large content fields
|
||||
* Batch INSERT chunk-aware embeddings into CodeEmbedding table
|
||||
*/
|
||||
const batchInsertEmbeddings = async (
|
||||
export const batchInsertEmbeddings = async (
|
||||
executeWithReusedStatement: (
|
||||
cypher: string,
|
||||
paramsList: Array<Record<string, any>>,
|
||||
) => Promise<void>,
|
||||
updates: Array<{ id: string; embedding: number[] }>,
|
||||
updates: Array<{
|
||||
nodeId: string;
|
||||
chunkIndex: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
embedding: number[];
|
||||
contentHash?: string;
|
||||
}>,
|
||||
): Promise<void> => {
|
||||
// INSERT into separate embedding table - much more memory efficient!
|
||||
const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`;
|
||||
const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding }));
|
||||
const cypher = `CREATE (e:${EMBEDDING_TABLE_NAME} {id: $id, nodeId: $nodeId, chunkIndex: $chunkIndex, startLine: $startLine, endLine: $endLine, embedding: $embedding, contentHash: $contentHash})`;
|
||||
const paramsList = updates.map((u) => ({
|
||||
id: `${u.nodeId}:${u.chunkIndex}`,
|
||||
nodeId: u.nodeId,
|
||||
chunkIndex: u.chunkIndex,
|
||||
startLine: u.startLine,
|
||||
endLine: u.endLine,
|
||||
embedding: u.embedding,
|
||||
contentHash: u.contentHash ?? STALE_HASH_SENTINEL,
|
||||
}));
|
||||
await executeWithReusedStatement(cypher, paramsList);
|
||||
};
|
||||
|
||||
/**
|
||||
* Create the vector index for semantic search
|
||||
* Now indexes the separate CodeEmbedding table
|
||||
*/
|
||||
let vectorExtensionLoaded = false;
|
||||
|
||||
* Now indexes the separate CodeEmbedding table.
|
||||
* Delegates extension loading to lbug-adapter's loadVectorExtension(),
|
||||
* which owns the VECTOR extension lifecycle and state tracking.
|
||||
|
||||
*/
|
||||
const createVectorIndex = async (
|
||||
executeQuery: (cypher: string) => Promise<any[]>,
|
||||
): Promise<void> => {
|
||||
// LadybugDB v0.15+ requires explicit VECTOR extension loading (once per session)
|
||||
if (!vectorExtensionLoaded) {
|
||||
try {
|
||||
await executeQuery('INSTALL VECTOR');
|
||||
await executeQuery('LOAD EXTENSION VECTOR');
|
||||
vectorExtensionLoaded = true;
|
||||
} catch {
|
||||
// Extension may already be loaded — CREATE_VECTOR_INDEX will fail clearly if not
|
||||
vectorExtensionLoaded = true;
|
||||
}
|
||||
}
|
||||
|
||||
const cypher = `
|
||||
CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')
|
||||
`;
|
||||
// Delegate to the adapter which tracks loaded state and handles DB reconnect resets
|
||||
await loadVectorExtension();
|
||||
|
||||
try {
|
||||
await executeQuery(cypher);
|
||||
await executeQuery(CREATE_VECTOR_INDEX_QUERY);
|
||||
} catch (error) {
|
||||
// Index might already exist
|
||||
if (isDev) {
|
||||
console.warn('Vector index creation warning:', error);
|
||||
}
|
||||
|
|
@ -149,6 +205,11 @@ const createVectorIndex = async (
|
|||
* @param onProgress - Callback for progress updates
|
||||
* @param config - Optional configuration override
|
||||
* @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
|
||||
* @param context - Optional repo/server context for metadata enrichment
|
||||
* @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
|
||||
* Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
|
||||
* and re-embedded; nodes not in the map are embedded fresh.
|
||||
|
||||
*/
|
||||
export const runEmbeddingPipeline = async (
|
||||
executeQuery: (cypher: string) => Promise<any[]>,
|
||||
|
|
@ -159,6 +220,8 @@ export const runEmbeddingPipeline = async (
|
|||
onProgress: EmbeddingProgressCallback,
|
||||
config: Partial<EmbeddingConfig> = {},
|
||||
skipNodeIds?: Set<string>,
|
||||
context?: EmbeddingContext,
|
||||
existingEmbeddings?: Map<string, string>,
|
||||
): Promise<void> => {
|
||||
const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
|
||||
|
||||
|
|
@ -194,13 +257,65 @@ export const runEmbeddingPipeline = async (
|
|||
// Phase 2: Query embeddable nodes
|
||||
let nodes = await queryEmbeddableNodes(executeQuery);
|
||||
|
||||
// Incremental mode: filter out nodes that already have embeddings
|
||||
if (skipNodeIds && skipNodeIds.size > 0) {
|
||||
// Apply context metadata
|
||||
if (context?.repoName) {
|
||||
for (const node of nodes) {
|
||||
node.repoName = context.repoName;
|
||||
node.serverName = context.serverName;
|
||||
}
|
||||
}
|
||||
|
||||
// Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
|
||||
// Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
|
||||
// (avoids double computation).
|
||||
const computedStaleHashes = new Map<string, string>();
|
||||
if (existingEmbeddings && existingEmbeddings.size > 0) {
|
||||
const beforeCount = nodes.length;
|
||||
nodes = nodes.filter((n) => !skipNodeIds.has(n.id));
|
||||
const staleNodeIds: string[] = [];
|
||||
nodes = nodes.filter((n) => {
|
||||
const existingHash = existingEmbeddings.get(n.id);
|
||||
if (existingHash === undefined) {
|
||||
// New node — needs embedding
|
||||
return true;
|
||||
}
|
||||
const currentHash = contentHashForNode(n, finalConfig);
|
||||
if (currentHash !== existingHash) {
|
||||
// Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
|
||||
computedStaleHashes.set(n.id, currentHash);
|
||||
staleNodeIds.push(n.id);
|
||||
return true;
|
||||
}
|
||||
// Hash matches — skip (fresh); no need to cache hash for skipped nodes
|
||||
return false;
|
||||
});
|
||||
|
||||
// DELETE stale embedding rows so they can be re-inserted
|
||||
// (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
|
||||
if (staleNodeIds.length > 0) {
|
||||
if (isDev) {
|
||||
console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
|
||||
}
|
||||
try {
|
||||
await executeWithReusedStatement(
|
||||
`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`,
|
||||
staleNodeIds.map((nodeId) => ({ nodeId })),
|
||||
);
|
||||
} catch (err) {
|
||||
// "does not exist" = rows already gone — safe to proceed.
|
||||
// All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT
|
||||
// for vector-indexed properties) — propagate so the pipeline aborts cleanly.
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (!msg.includes('does not exist')) {
|
||||
throw new Error(
|
||||
`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isDev) {
|
||||
console.log(
|
||||
`📦 Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`,
|
||||
`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -212,6 +327,11 @@ export const runEmbeddingPipeline = async (
|
|||
}
|
||||
|
||||
if (totalNodes === 0) {
|
||||
// Ensure the vector index exists even when no new nodes need embedding.
|
||||
// A prior crash or first-time incremental run may have left CodeEmbedding
|
||||
// rows without ever reaching index creation.
|
||||
await createVectorIndex(executeQuery);
|
||||
|
||||
onProgress({
|
||||
phase: 'ready',
|
||||
percent: 100,
|
||||
|
|
@ -221,10 +341,12 @@ export const runEmbeddingPipeline = async (
|
|||
return;
|
||||
}
|
||||
|
||||
// Phase 3: Batch embed nodes
|
||||
// Phase 3: Chunk + embed nodes
|
||||
const batchSize = finalConfig.batchSize;
|
||||
const totalBatches = Math.ceil(totalNodes / batchSize);
|
||||
const chunkSize = finalConfig.chunkSize;
|
||||
const overlap = finalConfig.overlap;
|
||||
let processedNodes = 0;
|
||||
let totalChunks = 0;
|
||||
|
||||
onProgress({
|
||||
phase: 'embedding',
|
||||
|
|
@ -232,39 +354,116 @@ export const runEmbeddingPipeline = async (
|
|||
nodesProcessed: 0,
|
||||
totalNodes,
|
||||
currentBatch: 0,
|
||||
totalBatches,
|
||||
totalBatches: Math.ceil(totalNodes / batchSize),
|
||||
});
|
||||
|
||||
for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
|
||||
const start = batchIndex * batchSize;
|
||||
const end = Math.min(start + batchSize, totalNodes);
|
||||
const batch = nodes.slice(start, end);
|
||||
// Process in batches of nodes
|
||||
for (let batchIndex = 0; batchIndex < totalNodes; batchIndex += batchSize) {
|
||||
const batch = nodes.slice(batchIndex, batchIndex + batchSize);
|
||||
|
||||
// Generate texts for this batch
|
||||
const texts = generateBatchEmbeddingTexts(batch, finalConfig);
|
||||
// Chunk each node and generate text
|
||||
const allTexts: string[] = [];
|
||||
const allUpdates: Array<{
|
||||
nodeId: string;
|
||||
chunkIndex: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
contentHash: string;
|
||||
}> = [];
|
||||
|
||||
// Embed the batch
|
||||
const embeddings = await embedBatch(texts);
|
||||
for (const node of batch) {
|
||||
const isShort = isShortLabel(node.label);
|
||||
const startLine = node.startLine ?? 0;
|
||||
const endLine = node.endLine ?? 0;
|
||||
|
||||
// Update LadybugDB with embeddings
|
||||
const updates = batch.map((node, i) => ({
|
||||
id: node.id,
|
||||
embedding: embeddingToArray(embeddings[i]),
|
||||
}));
|
||||
// Extract structural names for class-like nodes via AST extractors
|
||||
if (!isShort && STRUCTURAL_LABELS.has(node.label)) {
|
||||
try {
|
||||
const names = await extractStructuralNames(node.content, node.filePath);
|
||||
node.methodNames = names.methodNames;
|
||||
node.fieldNames = names.fieldNames;
|
||||
} catch {
|
||||
// AST extraction failed — names stay undefined, text-generator handles gracefully
|
||||
}
|
||||
}
|
||||
|
||||
await batchInsertEmbeddings(executeWithReusedStatement, updates);
|
||||
// Compute content hash once per node (re-use cached value for stale nodes)
|
||||
const hash = computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig);
|
||||
|
||||
let chunks: Array<{ text: string; chunkIndex: number; startLine: number; endLine: number }>;
|
||||
if (isShort) {
|
||||
chunks = [{ text: node.content, chunkIndex: 0, startLine, endLine }];
|
||||
} else {
|
||||
try {
|
||||
chunks = await chunkNode(
|
||||
node.label,
|
||||
node.content,
|
||||
node.filePath,
|
||||
startLine,
|
||||
endLine,
|
||||
chunkSize,
|
||||
overlap,
|
||||
);
|
||||
} catch (chunkErr) {
|
||||
if (isDev) {
|
||||
console.warn(
|
||||
`⚠️ AST chunking failed for ${node.label} "${node.name}" (${node.filePath}), falling back to character-based chunking:`,
|
||||
chunkErr,
|
||||
);
|
||||
}
|
||||
chunks = characterChunk(node.content, startLine, endLine, chunkSize, overlap);
|
||||
}
|
||||
}
|
||||
|
||||
for (const chunk of chunks) {
|
||||
const text = generateEmbeddingText(node, chunk.text, finalConfig);
|
||||
allTexts.push(text);
|
||||
allUpdates.push({
|
||||
nodeId: node.id,
|
||||
chunkIndex: chunk.chunkIndex,
|
||||
startLine: chunk.startLine,
|
||||
endLine: chunk.endLine,
|
||||
contentHash: hash,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Embed chunk texts in sub-batches to control memory
|
||||
const EMBED_SUB_BATCH = 8;
|
||||
for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) {
|
||||
const subTexts = allTexts.slice(si, si + EMBED_SUB_BATCH);
|
||||
const subUpdates = allUpdates.slice(si, si + EMBED_SUB_BATCH);
|
||||
|
||||
let embeddings: Float32Array[];
|
||||
try {
|
||||
embeddings = await embedBatch(subTexts);
|
||||
} catch (embedErr) {
|
||||
console.error(
|
||||
`❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`,
|
||||
embedErr,
|
||||
);
|
||||
throw embedErr;
|
||||
}
|
||||
|
||||
const dbUpdates = subUpdates.map((u, i) => ({
|
||||
...u,
|
||||
embedding: embeddingToArray(embeddings[i]),
|
||||
}));
|
||||
|
||||
await batchInsertEmbeddings(executeWithReusedStatement, dbUpdates);
|
||||
}
|
||||
|
||||
processedNodes += batch.length;
|
||||
totalChunks += allUpdates.length;
|
||||
|
||||
// Report progress (20-90% for embedding phase)
|
||||
const embeddingProgress = 20 + (processedNodes / totalNodes) * 70;
|
||||
onProgress({
|
||||
phase: 'embedding',
|
||||
percent: Math.round(embeddingProgress),
|
||||
nodesProcessed: processedNodes,
|
||||
totalNodes,
|
||||
currentBatch: batchIndex + 1,
|
||||
totalBatches,
|
||||
currentBatch: Math.floor(batchIndex / batchSize) + 1,
|
||||
totalBatches: Math.ceil(totalNodes / batchSize),
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -282,7 +481,6 @@ export const runEmbeddingPipeline = async (
|
|||
|
||||
await createVectorIndex(executeQuery);
|
||||
|
||||
// Complete
|
||||
onProgress({
|
||||
phase: 'ready',
|
||||
percent: 100,
|
||||
|
|
@ -291,7 +489,9 @@ export const runEmbeddingPipeline = async (
|
|||
});
|
||||
|
||||
if (isDev) {
|
||||
console.log('✅ Embedding pipeline complete!');
|
||||
console.log(
|
||||
`✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
|
||||
|
|
@ -311,15 +511,7 @@ export const runEmbeddingPipeline = async (
|
|||
};
|
||||
|
||||
/**
|
||||
* Perform semantic search using the vector index
|
||||
*
|
||||
* Uses CodeEmbedding table and queries each node table to get metadata
|
||||
*
|
||||
* @param executeQuery - Function to execute Cypher queries
|
||||
* @param query - Search query text
|
||||
* @param k - Number of results to return (default: 10)
|
||||
* @param maxDistance - Maximum distance threshold (default: 0.5)
|
||||
* @returns Array of search results ordered by relevance
|
||||
* Perform semantic search using the vector index with chunk deduplication
|
||||
*/
|
||||
export const semanticSearch = async (
|
||||
executeQuery: (cypher: string) => Promise<any[]>,
|
||||
|
|
@ -331,37 +523,46 @@ export const semanticSearch = async (
|
|||
throw new Error('Embedding model not initialized. Run embedding pipeline first.');
|
||||
}
|
||||
|
||||
// Embed the query
|
||||
const queryEmbedding = await embedText(query);
|
||||
const queryVec = embeddingToArray(queryEmbedding);
|
||||
const queryVecStr = `[${queryVec.join(',')}]`;
|
||||
|
||||
// Query the vector index on CodeEmbedding to get nodeIds and distances
|
||||
const vectorQuery = `
|
||||
CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx',
|
||||
CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${k})
|
||||
YIELD node AS emb, distance
|
||||
WITH emb, distance
|
||||
WHERE distance < ${maxDistance}
|
||||
RETURN emb.nodeId AS nodeId, distance
|
||||
ORDER BY distance
|
||||
`;
|
||||
const bestChunks = await collectBestChunks(k, async (fetchLimit) => {
|
||||
const vectorQuery = `
|
||||
CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}',
|
||||
CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit})
|
||||
YIELD node AS emb, distance
|
||||
WITH emb, distance
|
||||
WHERE distance < ${maxDistance}
|
||||
RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex,
|
||||
emb.startLine AS startLine, emb.endLine AS endLine, distance
|
||||
ORDER BY distance
|
||||
`;
|
||||
|
||||
const embResults = await executeQuery(vectorQuery);
|
||||
const embResults = await executeQuery(vectorQuery);
|
||||
return embResults.map((row) => ({
|
||||
nodeId: row.nodeId ?? row[0],
|
||||
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
|
||||
startLine: row.startLine ?? row[2] ?? 0,
|
||||
endLine: row.endLine ?? row[3] ?? 0,
|
||||
distance: row.distance ?? row[4],
|
||||
}));
|
||||
});
|
||||
|
||||
if (embResults.length === 0) {
|
||||
if (bestChunks.size === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// Group results by label for batched metadata queries
|
||||
const byLabel = new Map<string, Array<{ nodeId: string; distance: number }>>();
|
||||
for (const embRow of embResults) {
|
||||
const nodeId = embRow.nodeId ?? embRow[0];
|
||||
const distance = embRow.distance ?? embRow[1];
|
||||
const byLabel = new Map<
|
||||
string,
|
||||
Array<{ nodeId: string; distance: number } & Record<string, any>>
|
||||
>();
|
||||
for (const [nodeId, chunk] of Array.from(bestChunks.entries()).slice(0, k)) {
|
||||
const labelEndIdx = nodeId.indexOf(':');
|
||||
const label = labelEndIdx > 0 ? nodeId.substring(0, labelEndIdx) : 'Unknown';
|
||||
if (!byLabel.has(label)) byLabel.set(label, []);
|
||||
byLabel.get(label)!.push({ nodeId, distance });
|
||||
byLabel.get(label)!.push({ nodeId, ...chunk });
|
||||
}
|
||||
|
||||
// Batch-fetch metadata per label
|
||||
|
|
@ -370,19 +571,11 @@ export const semanticSearch = async (
|
|||
for (const [label, items] of byLabel) {
|
||||
const idList = items.map((i) => `'${i.nodeId.replace(/'/g, "''")}'`).join(', ');
|
||||
try {
|
||||
let nodeQuery: string;
|
||||
if (label === 'File') {
|
||||
nodeQuery = `
|
||||
MATCH (n:File) WHERE n.id IN [${idList}]
|
||||
RETURN n.id AS id, n.name AS name, n.filePath AS filePath
|
||||
`;
|
||||
} else {
|
||||
nodeQuery = `
|
||||
MATCH (n:${label}) WHERE n.id IN [${idList}]
|
||||
RETURN n.id AS id, n.name AS name, n.filePath AS filePath,
|
||||
n.startLine AS startLine, n.endLine AS endLine
|
||||
`;
|
||||
}
|
||||
const nodeQuery = `
|
||||
MATCH (n:\`${label}\`) WHERE n.id IN [${idList}]
|
||||
RETURN n.id AS id, n.name AS name, n.filePath AS filePath,
|
||||
n.startLine AS startLine, n.endLine AS endLine
|
||||
`;
|
||||
const nodeRows = await executeQuery(nodeQuery);
|
||||
const rowMap = new Map<string, any>();
|
||||
for (const row of nodeRows) {
|
||||
|
|
@ -398,8 +591,8 @@ export const semanticSearch = async (
|
|||
label,
|
||||
filePath: nodeRow.filePath ?? nodeRow[2] ?? '',
|
||||
distance: item.distance,
|
||||
startLine: label !== 'File' ? (nodeRow.startLine ?? nodeRow[3]) : undefined,
|
||||
endLine: label !== 'File' ? (nodeRow.endLine ?? nodeRow[4]) : undefined,
|
||||
startLine: item.startLine,
|
||||
endLine: item.endLine,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
@ -408,7 +601,6 @@ export const semanticSearch = async (
|
|||
}
|
||||
}
|
||||
|
||||
// Re-sort by distance since batch queries may have mixed order
|
||||
results.sort((a, b) => a.distance - b.distance);
|
||||
|
||||
return results;
|
||||
|
|
@ -416,16 +608,6 @@ export const semanticSearch = async (
|
|||
|
||||
/**
|
||||
* Semantic search with graph expansion (flattened results)
|
||||
*
|
||||
* Note: With multi-table schema, graph traversal is simplified.
|
||||
* Returns semantic matches with their metadata.
|
||||
* For full graph traversal, use execute_vector_cypher tool directly.
|
||||
*
|
||||
* @param executeQuery - Function to execute Cypher queries
|
||||
* @param query - Search query text
|
||||
* @param k - Number of initial semantic matches (default: 5)
|
||||
* @param _hops - Unused (kept for API compatibility).
|
||||
* @returns Semantic matches with metadata
|
||||
*/
|
||||
export const semanticSearchWithContext = async (
|
||||
executeQuery: (cypher: string) => Promise<any[]>,
|
||||
|
|
@ -433,8 +615,6 @@ export const semanticSearchWithContext = async (
|
|||
k: number = 5,
|
||||
_hops: number = 1,
|
||||
): Promise<any[]> => {
|
||||
// For multi-table schema, just return semantic search results
|
||||
// Graph traversal is complex with separate tables - use execute_vector_cypher instead
|
||||
const results = await semanticSearch(executeQuery, query, k, 0.5);
|
||||
|
||||
return results.map((r) => ({
|
||||
|
|
|
|||
50
gitnexus/src/core/embeddings/line-index.ts
Normal file
50
gitnexus/src/core/embeddings/line-index.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
export interface ResolvedLineRange {
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
}
|
||||
|
||||
export const buildLineIndex = (content: string): Int32Array => {
|
||||
const offsets: number[] = [0];
|
||||
for (let i = 0; i < content.length; i++) {
|
||||
if (content.charCodeAt(i) === 10) offsets.push(i + 1);
|
||||
}
|
||||
return new Int32Array(offsets);
|
||||
};
|
||||
|
||||
const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => {
|
||||
if (lineOffsets.length === 0) return 0;
|
||||
const maxOffset = lineOffsets[lineOffsets.length - 1];
|
||||
if (charOffset < 0) return 0;
|
||||
if (charOffset > maxOffset) return maxOffset;
|
||||
return charOffset;
|
||||
};
|
||||
|
||||
export const lineFromOffset = (lineOffsets: Int32Array, charOffset: number): number => {
|
||||
if (lineOffsets.length === 0) return 0;
|
||||
|
||||
const clamped = clampOffset(lineOffsets, charOffset);
|
||||
let lo = 0;
|
||||
let hi = lineOffsets.length - 1;
|
||||
while (lo < hi) {
|
||||
const mid = (lo + hi + 1) >> 1;
|
||||
if (lineOffsets[mid] <= clamped) lo = mid;
|
||||
else hi = mid - 1;
|
||||
}
|
||||
return lo;
|
||||
};
|
||||
|
||||
export const resolveChunkLines = (
|
||||
lineOffsets: Int32Array,
|
||||
startOffset: number,
|
||||
endOffset: number,
|
||||
baseStartLine: number,
|
||||
): ResolvedLineRange => {
|
||||
const relativeStartLine = lineFromOffset(lineOffsets, startOffset);
|
||||
const effectiveEndOffset = endOffset > startOffset ? endOffset - 1 : startOffset;
|
||||
const relativeEndLine = lineFromOffset(lineOffsets, effectiveEndOffset);
|
||||
|
||||
return {
|
||||
startLine: baseStartLine + relativeStartLine,
|
||||
endLine: baseStartLine + relativeEndLine,
|
||||
};
|
||||
};
|
||||
37
gitnexus/src/core/embeddings/server-mapping.ts
Normal file
37
gitnexus/src/core/embeddings/server-mapping.ts
Normal file
|
|
@ -0,0 +1,37 @@
|
|||
/**
|
||||
* Server Mapping Configuration
|
||||
*
|
||||
* Reads ~/.gitnexus/server-mapping.json to map repo names to service names.
|
||||
* Used in embedding text to enrich metadata with microservice context.
|
||||
*/
|
||||
|
||||
import fs from 'fs/promises';
|
||||
import path from 'path';
|
||||
import os from 'os';
|
||||
|
||||
const MAPPING_FILE = path.join(os.homedir(), '.gitnexus', 'server-mapping.json');
|
||||
|
||||
let cachedMapping: Record<string, string> | null = null;
|
||||
|
||||
/**
|
||||
* Read the server mapping file and return the serverName for a given repoName.
|
||||
* Returns undefined if no mapping exists.
|
||||
*/
|
||||
export const readServerMapping = async (repoName: string): Promise<string | undefined> => {
|
||||
try {
|
||||
if (!cachedMapping) {
|
||||
const raw = await fs.readFile(MAPPING_FILE, 'utf-8');
|
||||
cachedMapping = JSON.parse(raw);
|
||||
}
|
||||
return cachedMapping[repoName];
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Clear the cached mapping (useful for testing or after file changes)
|
||||
*/
|
||||
export const clearServerMappingCache = (): void => {
|
||||
cachedMapping = null;
|
||||
};
|
||||
89
gitnexus/src/core/embeddings/structural-extractor.ts
Normal file
89
gitnexus/src/core/embeddings/structural-extractor.ts
Normal file
|
|
@ -0,0 +1,89 @@
|
|||
/**
|
||||
* Structural Extractor Module
|
||||
*
|
||||
* Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor
|
||||
* to extract method and field names for embedding text generation.
|
||||
*/
|
||||
|
||||
import { getProviderForFile } from '../ingestion/languages/index.js';
|
||||
import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js';
|
||||
import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js';
|
||||
import type { LanguageProvider } from '../ingestion/language-provider.js';
|
||||
import { buildTypeEnv } from '../ingestion/type-env.js';
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import { ensureAndParse, findDeclarationNode } from './ast-utils.js';
|
||||
|
||||
export interface StructuralNames {
|
||||
methodNames: string[];
|
||||
fieldNames: string[];
|
||||
}
|
||||
|
||||
const NOOP_SYMBOL_TABLE = {
|
||||
lookupExactAll: () => [],
|
||||
lookupExact: () => undefined,
|
||||
lookupExactFull: () => undefined,
|
||||
} as any;
|
||||
|
||||
/**
|
||||
* Extract method and field names from a class/struct/interface node
|
||||
* using the ingestion pipeline's AST extractors.
|
||||
*/
|
||||
export const extractStructuralNames = async (
|
||||
content: string,
|
||||
filePath: string,
|
||||
): Promise<StructuralNames> => {
|
||||
const provider = getProviderForFile(filePath);
|
||||
if (!provider) return { methodNames: [], fieldNames: [] };
|
||||
|
||||
const tree = await ensureAndParse(content, filePath);
|
||||
if (!tree) return { methodNames: [], fieldNames: [] };
|
||||
|
||||
// Parse node.content (a snippet) — find declaration directly, not by range
|
||||
const classNode = findDeclarationNode(tree.rootNode);
|
||||
if (!classNode) return { methodNames: [], fieldNames: [] };
|
||||
|
||||
const language = provider.id;
|
||||
|
||||
const methodNames = extractMethodNames(classNode, provider, filePath, language);
|
||||
const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language);
|
||||
|
||||
return { methodNames, fieldNames };
|
||||
};
|
||||
|
||||
function extractMethodNames(
|
||||
classNode: any,
|
||||
provider: LanguageProvider,
|
||||
filePath: string,
|
||||
language: SupportedLanguages,
|
||||
): string[] {
|
||||
if (!provider.methodExtractor) return [];
|
||||
|
||||
const context: MethodExtractorContext = { filePath, language };
|
||||
const result: ExtractedMethods | null = provider.methodExtractor.extract(classNode, context);
|
||||
if (!result?.methods?.length) return [];
|
||||
|
||||
return result.methods.map((m) => m.name);
|
||||
}
|
||||
|
||||
function extractFieldNames(
|
||||
classNode: any,
|
||||
provider: LanguageProvider,
|
||||
tree: any,
|
||||
filePath: string,
|
||||
language: SupportedLanguages,
|
||||
): string[] {
|
||||
if (!provider.fieldExtractor) return [];
|
||||
|
||||
const typeEnv = buildTypeEnv(tree, language);
|
||||
|
||||
const context: FieldExtractorContext = {
|
||||
typeEnv,
|
||||
symbolTable: NOOP_SYMBOL_TABLE,
|
||||
filePath,
|
||||
language,
|
||||
};
|
||||
const result: ExtractedFields | null = provider.fieldExtractor.extract(classNode, context);
|
||||
if (!result?.fields?.length) return [];
|
||||
|
||||
return result.fields.map((f) => f.name);
|
||||
}
|
||||
|
|
@ -1,206 +1,253 @@
|
|||
/**
|
||||
* Text Generator Module
|
||||
*
|
||||
* Pure functions to generate embedding text from code nodes.
|
||||
* Combines node metadata with code snippets for semantic matching.
|
||||
* Generates enriched embedding text from code nodes with metadata.
|
||||
* Supports chunkable labels (Function/Method with AST chunking),
|
||||
* Class-specific structural text, and short-node direct embed.
|
||||
*
|
||||
* Method/field names for Class nodes are extracted by the ingestion
|
||||
* pipeline's AST extractors and passed via node.methodNames/node.fieldNames.
|
||||
*/
|
||||
|
||||
import type { EmbeddableNode, EmbeddingConfig } from './types.js';
|
||||
import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
|
||||
import { DEFAULT_EMBEDDING_CONFIG, isShortLabel } from './types.js';
|
||||
|
||||
/**
|
||||
* Extract the filename from a file path
|
||||
* Truncate description to max length at sentence/word boundary
|
||||
*/
|
||||
const getFileName = (filePath: string): string => {
|
||||
const parts = filePath.split('/');
|
||||
return parts[parts.length - 1] || filePath;
|
||||
};
|
||||
const truncateDescription = (text: string, maxLength: number): string => {
|
||||
if (text.length <= maxLength) return text;
|
||||
|
||||
/**
|
||||
* Extract the directory path from a file path
|
||||
*/
|
||||
const getDirectory = (filePath: string): string => {
|
||||
const parts = filePath.split('/');
|
||||
parts.pop();
|
||||
return parts.join('/') || '';
|
||||
};
|
||||
const truncated = text.slice(0, maxLength);
|
||||
|
||||
/**
|
||||
* Truncate content to max length, preserving word boundaries
|
||||
*/
|
||||
const truncateContent = (content: string, maxLength: number): string => {
|
||||
if (content.length <= maxLength) {
|
||||
return content;
|
||||
// Try sentence boundary (. ! ?)
|
||||
const sentenceEnd = Math.max(
|
||||
truncated.lastIndexOf('. '),
|
||||
truncated.lastIndexOf('! '),
|
||||
truncated.lastIndexOf('? '),
|
||||
);
|
||||
if (sentenceEnd > maxLength * 0.5) {
|
||||
return truncated.slice(0, sentenceEnd + 1);
|
||||
}
|
||||
|
||||
// Find last space before maxLength to avoid cutting words
|
||||
const truncated = content.slice(0, maxLength);
|
||||
// Try word boundary
|
||||
const lastSpace = truncated.lastIndexOf(' ');
|
||||
|
||||
if (lastSpace > maxLength * 0.8) {
|
||||
return truncated.slice(0, lastSpace) + '...';
|
||||
if (lastSpace > maxLength * 0.5) {
|
||||
return truncated.slice(0, lastSpace);
|
||||
}
|
||||
|
||||
return truncated + '...';
|
||||
return truncated;
|
||||
};
|
||||
|
||||
/**
|
||||
* Clean code content for embedding
|
||||
* Removes excessive whitespace while preserving structure
|
||||
*/
|
||||
const cleanContent = (content: string): string => {
|
||||
return (
|
||||
content
|
||||
// Normalize line endings
|
||||
.replace(/\r\n/g, '\n')
|
||||
// Remove excessive blank lines (more than 2)
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
// Trim each line
|
||||
.split('\n')
|
||||
.map((line) => line.trimEnd())
|
||||
.join('\n')
|
||||
.trim()
|
||||
);
|
||||
return content
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.split('\n')
|
||||
.map((line) => line.trimEnd())
|
||||
.join('\n')
|
||||
.trim();
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for a Function node
|
||||
* Build metadata header for a node
|
||||
*/
|
||||
const generateFunctionText = (node: EmbeddableNode, maxSnippetLength: number): string => {
|
||||
const parts: string[] = [`Function: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
||||
const buildMetadataHeader = (node: EmbeddableNode, config: Partial<EmbeddingConfig>): string => {
|
||||
const parts: string[] = [];
|
||||
|
||||
const dir = getDirectory(node.filePath);
|
||||
if (dir) {
|
||||
parts.push(`Directory: ${dir}`);
|
||||
// Label + name
|
||||
parts.push(`${node.label}: ${node.name}`);
|
||||
|
||||
// Repo name
|
||||
if (node.repoName) {
|
||||
parts.push(`Repo: ${node.repoName}`);
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
const cleanedContent = cleanContent(node.content);
|
||||
const snippet = truncateContent(cleanedContent, maxSnippetLength);
|
||||
parts.push('', snippet);
|
||||
// Server name (optional)
|
||||
if (node.serverName) {
|
||||
parts.push(`Server: ${node.serverName}`);
|
||||
}
|
||||
|
||||
// Full file path
|
||||
parts.push(`Path: ${node.filePath}`);
|
||||
|
||||
// Export status
|
||||
if (node.isExported !== undefined) {
|
||||
parts.push(`Export: ${node.isExported}`);
|
||||
}
|
||||
|
||||
// Description (truncated)
|
||||
if (node.description) {
|
||||
const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength;
|
||||
const truncated = truncateDescription(node.description, maxLen);
|
||||
if (truncated) {
|
||||
parts.push(truncated);
|
||||
}
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for a Class node
|
||||
*/
|
||||
const generateClassText = (node: EmbeddableNode, maxSnippetLength: number): string => {
|
||||
const parts: string[] = [`Class: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
||||
const generateCodeBodyText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig>,
|
||||
): string => {
|
||||
const header = buildMetadataHeader(node, config);
|
||||
const cleaned = cleanContent(codeBody);
|
||||
return `${header}\n\n${cleaned}`;
|
||||
};
|
||||
|
||||
const dir = getDirectory(node.filePath);
|
||||
if (dir) {
|
||||
parts.push(`Directory: ${dir}`);
|
||||
/**
|
||||
* Generate embedding text for Class nodes
|
||||
* Signature + properties + method name list only (no method bodies)
|
||||
* Method/field names come from AST extractors via node.methodNames/node.fieldNames.
|
||||
*/
|
||||
const generateClassText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig>,
|
||||
): string => {
|
||||
return generateStructuralTypeText(node, codeBody, config);
|
||||
};
|
||||
|
||||
const generateStructuralTypeText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig>,
|
||||
): string => {
|
||||
const header = buildMetadataHeader(node, config);
|
||||
const parts: string[] = [header];
|
||||
|
||||
if (node.methodNames?.length) {
|
||||
parts.push(`Methods: ${node.methodNames.join(', ')}`);
|
||||
}
|
||||
if (node.fieldNames?.length) {
|
||||
parts.push(`Properties: ${node.fieldNames.join(', ')}`);
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
const cleanedContent = cleanContent(node.content);
|
||||
const snippet = truncateContent(cleanedContent, maxSnippetLength);
|
||||
parts.push('', snippet);
|
||||
const declarationOnly = extractDeclarationOnly(cleanContent(node.content));
|
||||
if (declarationOnly) {
|
||||
parts.push('', declarationOnly);
|
||||
}
|
||||
|
||||
const cleanedChunk = cleanContent(codeBody);
|
||||
if (cleanedChunk && cleanedChunk !== cleanContent(node.content)) {
|
||||
parts.push('', cleanedChunk);
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for a Method node
|
||||
*/
|
||||
const generateMethodText = (node: EmbeddableNode, maxSnippetLength: number): string => {
|
||||
const parts: string[] = [`Method: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
||||
|
||||
const dir = getDirectory(node.filePath);
|
||||
if (dir) {
|
||||
parts.push(`Directory: ${dir}`);
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
const cleanedContent = cleanContent(node.content);
|
||||
const snippet = truncateContent(cleanedContent, maxSnippetLength);
|
||||
parts.push('', snippet);
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
};
|
||||
const DECL_START_RE =
|
||||
/^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/;
|
||||
|
||||
/**
|
||||
* Generate embedding text for an Interface node
|
||||
* Extract class/interface/struct declaration lines, skipping method bodies.
|
||||
* - Brace-based languages: detects method signatures (lines with `(` and `{`)
|
||||
* and skips until depth returns to class body level.
|
||||
* - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
|
||||
*/
|
||||
const generateInterfaceText = (node: EmbeddableNode, maxSnippetLength: number): string => {
|
||||
const parts: string[] = [`Interface: ${node.name}`, `File: ${getFileName(node.filePath)}`];
|
||||
export const extractDeclarationOnly = (content: string): string => {
|
||||
const lines = content.split('\n');
|
||||
const declLines: string[] = [];
|
||||
let depth = 0;
|
||||
let started = false;
|
||||
let classDepth = 0;
|
||||
let skipDepth = 0;
|
||||
|
||||
const dir = getDirectory(node.filePath);
|
||||
if (dir) {
|
||||
parts.push(`Directory: ${dir}`);
|
||||
for (const [idx, line] of lines.entries()) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
if (!started) {
|
||||
if (DECL_START_RE.test(trimmed)) {
|
||||
// Non-brace language check: current line or next 3 lines must have `{`
|
||||
const nextLines = lines.slice(idx + 1, idx + 4);
|
||||
if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) {
|
||||
return '';
|
||||
}
|
||||
started = true;
|
||||
declLines.push(trimmed);
|
||||
for (const ch of trimmed) {
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') depth--;
|
||||
}
|
||||
if (depth > 0) classDepth = depth;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Always update depth (even when skipping)
|
||||
const opens = (trimmed.match(/{/g) || []).length;
|
||||
const closes = (trimmed.match(/}/g) || []).length;
|
||||
const prevDepth = depth;
|
||||
depth += opens - closes;
|
||||
|
||||
if (skipDepth > 0) {
|
||||
if (depth <= classDepth) {
|
||||
skipDepth = 0;
|
||||
// Closing brace of class
|
||||
if (depth <= 0) {
|
||||
declLines.push(trimmed);
|
||||
break;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Detect method signature: line has both `(` and `{` and goes deeper than class body
|
||||
const hasParens = trimmed.includes('(');
|
||||
const hasOpenBrace = opens > 0;
|
||||
if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) {
|
||||
if (opens === closes && trimmed.endsWith(';')) {
|
||||
// Property with function/object initializer like `config = { timeout: 5000 };` — keep
|
||||
declLines.push(trimmed);
|
||||
}
|
||||
// else: single-line or multi-line method — skip entirely
|
||||
if (opens !== closes) {
|
||||
skipDepth = classDepth;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
declLines.push(trimmed);
|
||||
|
||||
if (depth <= 0 && declLines.length > 1) break;
|
||||
}
|
||||
|
||||
if (node.content) {
|
||||
const cleanedContent = cleanContent(node.content);
|
||||
const snippet = truncateContent(cleanedContent, maxSnippetLength);
|
||||
parts.push('', snippet);
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for a File node
|
||||
* Uses file name and first N characters of content
|
||||
*/
|
||||
const generateFileText = (node: EmbeddableNode, maxSnippetLength: number): string => {
|
||||
const parts: string[] = [`File: ${node.name}`, `Path: ${node.filePath}`];
|
||||
|
||||
if (node.content) {
|
||||
const cleanedContent = cleanContent(node.content);
|
||||
// For files, use a shorter snippet since they can be very long
|
||||
const snippet = truncateContent(cleanedContent, Math.min(maxSnippetLength, 300));
|
||||
parts.push('', snippet);
|
||||
}
|
||||
|
||||
return parts.join('\n');
|
||||
return declLines.join('\n').trim();
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding text for any embeddable node
|
||||
* Dispatches to the appropriate generator based on node label
|
||||
*
|
||||
* @param node - The node to generate text for
|
||||
* @param config - Optional configuration for max snippet length
|
||||
* @returns Text suitable for embedding
|
||||
*/
|
||||
export const generateEmbeddingText = (
|
||||
node: EmbeddableNode,
|
||||
codeBody: string,
|
||||
config: Partial<EmbeddingConfig> = {},
|
||||
): string => {
|
||||
const maxSnippetLength = config.maxSnippetLength ?? DEFAULT_EMBEDDING_CONFIG.maxSnippetLength;
|
||||
|
||||
switch (node.label) {
|
||||
case 'Function':
|
||||
return generateFunctionText(node, maxSnippetLength);
|
||||
case 'Class':
|
||||
return generateClassText(node, maxSnippetLength);
|
||||
case 'Method':
|
||||
return generateMethodText(node, maxSnippetLength);
|
||||
case 'Interface':
|
||||
return generateInterfaceText(node, maxSnippetLength);
|
||||
case 'File':
|
||||
return generateFileText(node, maxSnippetLength);
|
||||
default:
|
||||
// Fallback for any other embeddable type
|
||||
return `${node.label}: ${node.name}\nPath: ${node.filePath}`;
|
||||
if (isShortLabel(node.label)) {
|
||||
const header = buildMetadataHeader(node, config);
|
||||
const cleaned = cleanContent(node.content);
|
||||
return `${header}\n\n${cleaned}`;
|
||||
}
|
||||
|
||||
if (node.label === 'Class') {
|
||||
return generateClassText(node, codeBody, config);
|
||||
}
|
||||
|
||||
if (node.label === 'Interface') {
|
||||
return generateStructuralTypeText(node, codeBody, config);
|
||||
}
|
||||
|
||||
return generateCodeBodyText(node, codeBody, config);
|
||||
};
|
||||
|
||||
/**
|
||||
* Generate embedding texts for a batch of nodes
|
||||
*
|
||||
* @param nodes - Array of nodes to generate text for
|
||||
* @param config - Optional configuration
|
||||
* @returns Array of texts in the same order as input nodes
|
||||
* Export truncation helper for testing
|
||||
*/
|
||||
export const generateBatchEmbeddingTexts = (
|
||||
nodes: EmbeddableNode[],
|
||||
config: Partial<EmbeddingConfig> = {},
|
||||
): string[] => {
|
||||
return nodes.map((node) => generateEmbeddingText(node, config));
|
||||
};
|
||||
export { truncateDescription };
|
||||
|
|
|
|||
|
|
@ -5,10 +5,40 @@
|
|||
*/
|
||||
|
||||
/**
|
||||
* Node labels that should be embedded for semantic search
|
||||
* These are code elements that benefit from semantic matching
|
||||
* Node labels that need chunking (have code body, potentially long)
|
||||
*/
|
||||
export const EMBEDDABLE_LABELS = ['Function', 'Class', 'Method', 'Interface', 'File'] as const;
|
||||
export const CHUNKABLE_LABELS = [
|
||||
'Function',
|
||||
'Method',
|
||||
'Constructor',
|
||||
'Class',
|
||||
'Interface',
|
||||
'Struct',
|
||||
'Enum',
|
||||
'Trait',
|
||||
'Impl',
|
||||
'Macro',
|
||||
'Namespace',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Node labels that are short (no chunking needed, embed directly)
|
||||
*/
|
||||
export const SHORT_LABELS = [
|
||||
'TypeAlias',
|
||||
'Typedef',
|
||||
'Const',
|
||||
'Property',
|
||||
'Record',
|
||||
'Union',
|
||||
'Static',
|
||||
'Variable',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* All embeddable labels (union of CHUNKABLE + SHORT)
|
||||
*/
|
||||
export const EMBEDDABLE_LABELS = [...CHUNKABLE_LABELS, ...SHORT_LABELS] as const;
|
||||
|
||||
export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
|
||||
|
||||
|
|
@ -18,6 +48,39 @@ export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
|
|||
export const isEmbeddableLabel = (label: string): label is EmbeddableLabel =>
|
||||
EMBEDDABLE_LABELS.includes(label as EmbeddableLabel);
|
||||
|
||||
/**
|
||||
* Check if a label needs chunking
|
||||
*/
|
||||
export const isChunkableLabel = (label: string): boolean =>
|
||||
(CHUNKABLE_LABELS as readonly string[]).includes(label);
|
||||
|
||||
/**
|
||||
* Check if a label is a short type (no chunking)
|
||||
*/
|
||||
export const isShortLabel = (label: string): boolean =>
|
||||
(SHORT_LABELS as readonly string[]).includes(label);
|
||||
|
||||
/**
|
||||
* Node labels that have structural names (methods/fields) extractable via AST
|
||||
*/
|
||||
export const STRUCTURAL_LABELS: ReadonlySet<string> = new Set([
|
||||
'Class',
|
||||
'Struct',
|
||||
'Interface',
|
||||
'Enum',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Node labels that have isExported column in their schema
|
||||
*/
|
||||
export const LABELS_WITH_EXPORTED = new Set([
|
||||
'Function',
|
||||
'Class',
|
||||
'Interface',
|
||||
'Method',
|
||||
'CodeElement',
|
||||
]) as ReadonlySet<string>;
|
||||
|
||||
/**
|
||||
* Embedding pipeline phases
|
||||
*/
|
||||
|
|
@ -57,6 +120,12 @@ export interface EmbeddingConfig {
|
|||
device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm';
|
||||
/** Maximum characters of code snippet to include */
|
||||
maxSnippetLength: number;
|
||||
/** Maximum code chunk size in characters (for chunking long code) */
|
||||
chunkSize: number;
|
||||
/** Overlap between chunks in characters */
|
||||
overlap: number;
|
||||
/** Maximum description length in characters */
|
||||
maxDescriptionLength: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -70,6 +139,9 @@ export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = {
|
|||
dimensions: 384,
|
||||
device: 'auto',
|
||||
maxSnippetLength: 500,
|
||||
chunkSize: 1200,
|
||||
overlap: 120,
|
||||
maxDescriptionLength: 150,
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -96,6 +168,34 @@ export interface EmbeddableNode {
|
|||
content: string;
|
||||
startLine?: number;
|
||||
endLine?: number;
|
||||
isExported?: boolean;
|
||||
description?: string;
|
||||
parameterCount?: number;
|
||||
returnType?: string;
|
||||
repoName?: string;
|
||||
serverName?: string;
|
||||
methodNames?: string[];
|
||||
fieldNames?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Cached embedding entry restored from LadybugDB before a graph rebuild
|
||||
*/
|
||||
export interface CachedEmbedding {
|
||||
nodeId: string;
|
||||
chunkIndex: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
embedding: number[];
|
||||
contentHash?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Context info for embedding pipeline (repo/server metadata enrichment)
|
||||
*/
|
||||
export interface EmbeddingContext {
|
||||
repoName?: string;
|
||||
serverName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -108,3 +208,75 @@ export interface ModelProgress {
|
|||
loaded?: number;
|
||||
total?: number;
|
||||
}
|
||||
|
||||
export interface ChunkSearchRow {
|
||||
nodeId: string;
|
||||
chunkIndex: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
distance: number;
|
||||
}
|
||||
|
||||
export interface BestChunkMatch {
|
||||
chunkIndex: number;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
distance: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deduplicate vector search chunk results by nodeId,
|
||||
* keeping the chunk with smallest distance for each node.
|
||||
*/
|
||||
export const dedupBestChunks = (
|
||||
rows: ChunkSearchRow[],
|
||||
limit?: number,
|
||||
): Map<string, BestChunkMatch> => {
|
||||
const best = new Map<string, BestChunkMatch>();
|
||||
for (const row of rows) {
|
||||
const existing = best.get(row.nodeId);
|
||||
if (!existing || row.distance < existing.distance) {
|
||||
best.set(row.nodeId, {
|
||||
chunkIndex: row.chunkIndex,
|
||||
startLine: row.startLine,
|
||||
endLine: row.endLine,
|
||||
distance: row.distance,
|
||||
});
|
||||
}
|
||||
if (limit !== undefined && best.size >= limit) break;
|
||||
}
|
||||
return best;
|
||||
};
|
||||
|
||||
const DEFAULT_FETCH_MULTIPLIER = 4;
|
||||
const DEFAULT_FETCH_BUFFER = 8;
|
||||
const DEFAULT_MAX_FETCH = 200;
|
||||
|
||||
/**
|
||||
* Fetch vector-search chunks until we have enough unique nodeIds
|
||||
* or can tell the result set is exhausted.
|
||||
*/
|
||||
export const collectBestChunks = async (
|
||||
limit: number,
|
||||
fetchRows: (fetchLimit: number) => Promise<ChunkSearchRow[]>,
|
||||
maxFetch: number = DEFAULT_MAX_FETCH,
|
||||
): Promise<Map<string, BestChunkMatch>> => {
|
||||
if (limit <= 0) return new Map();
|
||||
|
||||
let fetchLimit = Math.max(limit * DEFAULT_FETCH_MULTIPLIER, limit + DEFAULT_FETCH_BUFFER);
|
||||
let previousFetchLimit = 0;
|
||||
|
||||
while (fetchLimit > previousFetchLimit) {
|
||||
const rows = await fetchRows(fetchLimit);
|
||||
const bestChunks = dedupBestChunks(rows, limit);
|
||||
|
||||
if (bestChunks.size >= limit || rows.length < fetchLimit) {
|
||||
return bestChunks;
|
||||
}
|
||||
|
||||
previousFetchLimit = fetchLimit;
|
||||
fetchLimit = fetchLimit >= maxFetch ? fetchLimit * 2 : Math.min(maxFetch, fetchLimit * 2);
|
||||
}
|
||||
|
||||
return new Map();
|
||||
};
|
||||
|
|
|
|||
139
gitnexus/src/core/group/PIPELINE.md
Normal file
139
gitnexus/src/core/group/PIPELINE.md
Normal file
|
|
@ -0,0 +1,139 @@
|
|||
# Group Analysis Pipeline
|
||||
|
||||
Flow chart of the cross-repo contract extraction + matching pipeline.
|
||||
This covers what runs **inside this PR** (extractors + manifest) and
|
||||
the downstream handoff to the bridge storage (PR #795) and
|
||||
cross-impact query (PR #606).
|
||||
|
||||
## High-level overview
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[group.yaml] --> B[GroupConfig parser]
|
||||
B --> C{For each repo<br/>in group}
|
||||
C --> D[Per-repo LadybugDB<br/>indexed by main pipeline]
|
||||
|
||||
D --> E1[TopicExtractor]
|
||||
D --> E2[HttpRouteExtractor]
|
||||
D --> E3[GrpcExtractor]
|
||||
|
||||
E1 --> F[ExtractedContract array<br/>per repo]
|
||||
E2 --> F
|
||||
E3 --> F
|
||||
|
||||
B --> M[ManifestExtractor]
|
||||
M --> G[Manifest contracts<br/>+ cross-links]
|
||||
|
||||
F --> H[Contract matching<br/>exact + wildcard]
|
||||
G --> H
|
||||
|
||||
H --> I[(bridge.lbug<br/>#795)]
|
||||
|
||||
I --> J[runGroupImpact<br/>#606]
|
||||
J --> K[CrossRepoImpact]
|
||||
```
|
||||
|
||||
## Per-repo extractor pipeline
|
||||
|
||||
Each extractor under `src/core/group/extractors/` follows the same
|
||||
two-strategy shape:
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
R[RepoHandle + CypherExecutor<br/>for this repo] --> S{Graph-assisted<br/>Strategy A<br/>available?}
|
||||
|
||||
S -->|yes| A1[Cypher query against<br/>per-repo LadybugDB]
|
||||
A1 --> A2{non-empty<br/>result?}
|
||||
A2 -->|yes| OUT[ExtractedContract array]
|
||||
A2 -->|no| B1
|
||||
|
||||
S -->|no| B1[Source-scan Strategy B]
|
||||
B1 --> B2[glob repo source files]
|
||||
B2 --> B3{ext in registry?}
|
||||
B3 -->|yes| B4[Per-language plugin<br/>scan parsed tree]
|
||||
B3 -->|no| SKIP[skip file]
|
||||
B4 --> OUT
|
||||
|
||||
SKIP --> B2
|
||||
```
|
||||
|
||||
**Strategy A** (graph-assisted) uses Cypher over edges already produced
|
||||
by the main ingestion pipeline:
|
||||
- HTTP: `HANDLES_ROUTE` / `FETCHES` edges from `(File)-[]->(Route)`
|
||||
- topic: none (pipeline doesn't yet produce topic nodes — Strategy B only)
|
||||
- gRPC: none (Strategy B + proto map only)
|
||||
|
||||
**Strategy B** (source-scan) is 100% tree-sitter based after this PR.
|
||||
Each `*-patterns/<lang>.ts` plugin owns its grammar + S-expression
|
||||
queries; the top-level orchestrator imports neither.
|
||||
|
||||
## Plugin architecture
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
O[Orchestrator<br/>topic|http|grpc-extractor.ts] --> REG[REGISTRY<br/>*-patterns/index.ts]
|
||||
REG --> P1[java.ts<br/>tree-sitter-java]
|
||||
REG --> P2[go.ts<br/>tree-sitter-go]
|
||||
REG --> P3[python.ts<br/>tree-sitter-python]
|
||||
REG --> P4[node.ts<br/>JS + TS + TSX]
|
||||
REG --> P5[php.ts<br/>tree-sitter-php<br/>HTTP only]
|
||||
REG --> P6[proto.ts<br/>tree-sitter-proto<br/>gRPC only, optional]
|
||||
|
||||
P1 --> SCAN[tree-sitter-scanner.ts<br/>compilePatterns + runCompiledPatterns]
|
||||
P2 --> SCAN
|
||||
P3 --> SCAN
|
||||
P4 --> SCAN
|
||||
P5 --> SCAN
|
||||
P6 --> SCAN
|
||||
|
||||
SCAN --> DET[Detection objects<br/>TopicMeta / HttpDetection / GrpcDetection]
|
||||
DET --> O
|
||||
O --> CT[ExtractedContract array]
|
||||
```
|
||||
|
||||
The orchestrator never imports a grammar. Adding a new language /
|
||||
framework = drop one file in `*-patterns/`, register it in
|
||||
`index.ts`. No orchestrator edits required.
|
||||
|
||||
## Manifest extraction
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Y[group.yaml links] --> ME[ManifestExtractor]
|
||||
ME --> LOOP{for each link}
|
||||
LOOP --> RES[resolveSymbol<br/>label-scoped Cypher]
|
||||
RES --> OK{found?}
|
||||
OK -->|yes| REF[real symbol uid + ref]
|
||||
OK -->|no| SYN[synthetic uid<br/>manifest::repo::cid]
|
||||
|
||||
REF --> EMIT[emit provider + consumer<br/>Contract objects<br/>+ CrossLink]
|
||||
SYN --> EMIT
|
||||
|
||||
EMIT --> BRIDGE[(bridge.lbug<br/>#795)]
|
||||
```
|
||||
|
||||
Label-scoped queries in `resolveSymbol` keep accidental cross-matches
|
||||
out:
|
||||
- `topic` → `(n:Function|Method|Class|Interface)`
|
||||
- `grpc` method → `(n:Function|Method)`, service → `(n:Class|Interface)`
|
||||
- `lib` → `(n:Package|Module)`
|
||||
|
||||
## Cross-impact query (PR #606)
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
U[User changes symbol S<br/>in repo R] --> LI[Local impact engine<br/>per-repo uid expansion]
|
||||
LI --> IDS[Affected uid set]
|
||||
|
||||
IDS --> BR[Bridge query<br/>MATCH Contract WHERE uid IN ids]
|
||||
BR --> CL[CrossLink traversal]
|
||||
CL --> OTHER[Matching contract in<br/>other repo]
|
||||
|
||||
OTHER --> FE[Fan-out impact<br/>to consuming repo]
|
||||
FE --> OUT[CrossRepoImpact<br/>per affected repo]
|
||||
```
|
||||
|
||||
The bridge stores every extracted contract keyed by `symbolUid`.
|
||||
Manifest-sourced contracts use the synthetic uid form so both sides
|
||||
of the `(local impact) ↔ (bridge query)` join derive the same uid
|
||||
without coordinating through any shared state.
|
||||
588
gitnexus/src/core/group/bridge-db.ts
Normal file
588
gitnexus/src/core/group/bridge-db.ts
Normal file
|
|
@ -0,0 +1,588 @@
|
|||
import fsp from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import type { LbugValue } from '@ladybugdb/core';
|
||||
import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js';
|
||||
import { BRIDGE_SCHEMA_QUERIES, BRIDGE_SCHEMA_VERSION } from './bridge-schema.js';
|
||||
import { dedupeContracts, dedupeCrossLinks } from './normalization.js';
|
||||
|
||||
export function contractNodeId(
|
||||
repo: string,
|
||||
contractId: string,
|
||||
role: string,
|
||||
filePath: string,
|
||||
): string {
|
||||
return createHash('sha256').update(`${repo}\0${contractId}\0${role}\0${filePath}`).digest('hex');
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* ContractLookupIndex — in-memory lookup for findContractNode */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/**
|
||||
* In-memory index of contract node IDs keyed three ways, mirroring the
|
||||
* three-tier fallback lookup in {@link findContractNode}. Built once per
|
||||
* `writeBridge` call after all contracts are successfully inserted, then
|
||||
* consulted for every cross-link — which eliminates the former N+1 query
|
||||
* pattern (up to `6 × cross-links` DB round-trips) and turns cross-link
|
||||
* resolution into constant-time per link.
|
||||
*
|
||||
* Keys are deliberately flat strings (not tuples) so `Map<string, ...>`
|
||||
* works; the separator `\0` can't occur in any legal repo path / file
|
||||
* path / symbol identifier, which makes the encoding injection-safe.
|
||||
*/
|
||||
export interface ContractLookupIndex {
|
||||
/** tier 1: `repo + role + symbolUid` → contract node id */
|
||||
byUid: Map<string, string>;
|
||||
/** tier 2: `repo + role + filePath + symbolName` → contract node id */
|
||||
byRef: Map<string, string>;
|
||||
/** tier 3: `repo + role + filePath` → list of contract node ids in that file */
|
||||
byFile: Map<string, string[]>;
|
||||
}
|
||||
|
||||
export function createContractLookupIndex(): ContractLookupIndex {
|
||||
return {
|
||||
byUid: new Map(),
|
||||
byRef: new Map(),
|
||||
byFile: new Map(),
|
||||
};
|
||||
}
|
||||
|
||||
function uidKey(repo: string, role: string, symbolUid: string): string {
|
||||
return `${repo}\0${role}\0${symbolUid}`;
|
||||
}
|
||||
|
||||
function refKey(repo: string, role: string, filePath: string, symbolName: string): string {
|
||||
return `${repo}\0${role}\0${filePath}\0${symbolName}`;
|
||||
}
|
||||
|
||||
function fileKey(repo: string, role: string, filePath: string): string {
|
||||
return `${repo}\0${role}\0${filePath}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a successfully-inserted contract to the lookup index. Must be called
|
||||
* AFTER the DB insert succeeds (not before) so failed inserts don't poison
|
||||
* the index and cause cross-links to point at non-existent rows.
|
||||
*/
|
||||
export function indexContract(
|
||||
index: ContractLookupIndex,
|
||||
contract: StoredContract,
|
||||
nodeId: string,
|
||||
): void {
|
||||
if (contract.symbolUid) {
|
||||
index.byUid.set(uidKey(contract.repo, contract.role, contract.symbolUid), nodeId);
|
||||
}
|
||||
index.byRef.set(
|
||||
refKey(contract.repo, contract.role, contract.symbolRef.filePath, contract.symbolRef.name),
|
||||
nodeId,
|
||||
);
|
||||
const fk = fileKey(contract.repo, contract.role, contract.symbolRef.filePath);
|
||||
const existing = index.byFile.get(fk);
|
||||
if (existing) {
|
||||
existing.push(nodeId);
|
||||
} else {
|
||||
index.byFile.set(fk, [nodeId]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a cross-link endpoint (consumer or provider reference) to an
|
||||
* already-inserted contract node id. Returns `null` if no match — the
|
||||
* caller is expected to count that as a dropped link in `WriteBridgeReport`.
|
||||
*
|
||||
* The resolution order matches the pre-cache DB-query behavior:
|
||||
* 1. exact `symbolUid` match in the same `(repo, role)` scope
|
||||
* 2. exact `(filePath, symbolName)` match
|
||||
* 3. if exactly one contract lives in the file → that one (fallback for
|
||||
* legacy graph-assisted extractors that couldn't resolve a symbol name)
|
||||
*
|
||||
* This is a pure function — no I/O, no DB — so it's trivial to unit-test
|
||||
* in isolation (which was the reviewer's main clean-code concern on the
|
||||
* original 35-line inner closure in `writeBridge`).
|
||||
*/
|
||||
export function findContractNode(
|
||||
index: ContractLookupIndex,
|
||||
repo: string,
|
||||
role: 'consumer' | 'provider',
|
||||
symbolUid: string,
|
||||
filePath: string,
|
||||
symbolName: string,
|
||||
): string | null {
|
||||
if (symbolUid) {
|
||||
const uidHit = index.byUid.get(uidKey(repo, role, symbolUid));
|
||||
if (uidHit !== undefined) return uidHit;
|
||||
}
|
||||
|
||||
const refHit = index.byRef.get(refKey(repo, role, filePath, symbolName));
|
||||
if (refHit !== undefined) return refHit;
|
||||
|
||||
const fileCandidates = index.byFile.get(fileKey(repo, role, filePath));
|
||||
if (fileCandidates && fileCandidates.length === 1) return fileCandidates[0];
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function openBridgeDb(dbPath: string): Promise<BridgeHandle> {
|
||||
const parentDir = path.dirname(dbPath);
|
||||
await fsp.mkdir(parentDir, { recursive: true });
|
||||
const db = new lbug.Database(dbPath, 0, false, false); // writable
|
||||
const conn = new lbug.Connection(db);
|
||||
return { _db: db, _conn: conn, groupDir: parentDir } as BridgeHandle;
|
||||
}
|
||||
|
||||
/**
|
||||
* LadybugDB returns an error whose message contains this substring when a
|
||||
* CREATE NODE TABLE or CREATE REL TABLE statement hits an already-existing
|
||||
* table. LadybugDB DDL doesn't support IF NOT EXISTS, and its JS driver
|
||||
* doesn't expose typed error codes, so we match on the message substring —
|
||||
* the same pattern used by `core/lbug/lbug-adapter.ts`. If a future
|
||||
* LadybugDB release changes the wording, update this constant.
|
||||
*/
|
||||
const LBUG_ALREADY_EXISTS_MSG = 'already exists';
|
||||
|
||||
export async function ensureBridgeSchema(handle: BridgeHandle): Promise<void> {
|
||||
const conn = handle._conn as lbug.Connection;
|
||||
for (const q of BRIDGE_SCHEMA_QUERIES) {
|
||||
try {
|
||||
await conn.query(q);
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (!msg.includes(LBUG_ALREADY_EXISTS_MSG)) throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function queryBridge<T>(
|
||||
handle: BridgeHandle,
|
||||
cypher: string,
|
||||
params?: Record<string, LbugValue>,
|
||||
): Promise<T[]> {
|
||||
const conn = handle._conn as lbug.Connection;
|
||||
if (params && Object.keys(params).length > 0) {
|
||||
const stmt = await conn.prepare(cypher);
|
||||
if (!stmt.isSuccess()) {
|
||||
const errMsg = await stmt.getErrorMessage();
|
||||
throw new Error(`Bridge query prepare failed: ${errMsg}`);
|
||||
}
|
||||
const queryResult = await conn.execute(stmt, params);
|
||||
const result = unwrapQueryResult(queryResult);
|
||||
return (await result.getAll()) as T[];
|
||||
}
|
||||
const queryResult = await conn.query(cypher);
|
||||
const result = unwrapQueryResult(queryResult);
|
||||
return (await result.getAll()) as T[];
|
||||
}
|
||||
|
||||
/**
|
||||
* LadybugDB's `conn.query` / `conn.execute` can return either a single
|
||||
* `QueryResult` (for a single statement) or an array of them (when a
|
||||
* multi-statement script is dispatched). We always pass a single statement,
|
||||
* so the array form is a wrapper we unwrap here — but an empty top-level
|
||||
* array would cause `.getAll()` on `undefined` and crash with a confusing
|
||||
* stack. Throwing an explicit error makes a driver-contract regression
|
||||
* visible immediately instead of masking it.
|
||||
*/
|
||||
function unwrapQueryResult(queryResult: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult {
|
||||
if (Array.isArray(queryResult)) {
|
||||
if (queryResult.length === 0) {
|
||||
throw new Error('Bridge query returned an empty QueryResult array');
|
||||
}
|
||||
return queryResult[0];
|
||||
}
|
||||
return queryResult;
|
||||
}
|
||||
|
||||
export async function closeBridgeDb(handle: BridgeHandle): Promise<void> {
|
||||
try {
|
||||
await (handle._conn as lbug.Connection).close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
try {
|
||||
await (handle._db as lbug.Database).close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* retryRename — handles transient EBUSY/EPERM/EACCES on Windows */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
const RETRY_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
|
||||
|
||||
export async function retryRename(src: string, dst: string, attempts = 3): Promise<void> {
|
||||
for (let i = 1; i <= attempts; i++) {
|
||||
try {
|
||||
await fsp.rename(src, dst);
|
||||
return;
|
||||
} catch (err: unknown) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (!code || !RETRY_CODES.has(code) || i === attempts) throw err;
|
||||
await new Promise((r) => setTimeout(r, 100 * Math.pow(2, i - 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* writeBridgeMeta / readBridgeMeta */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise<void> {
|
||||
const target = path.join(groupDir, 'meta.json');
|
||||
const tmp = `${target}.tmp.${Date.now()}`;
|
||||
await fsp.writeFile(tmp, JSON.stringify(meta, null, 2), 'utf-8');
|
||||
// Use retryRename for consistency with writeBridge's atomic swap — on
|
||||
// Windows a concurrent reader can cause EBUSY/EPERM even on a tiny
|
||||
// meta.json, and we don't want meta write to be less robust than the
|
||||
// bridge.lbug swap it accompanies.
|
||||
await retryRename(tmp, target);
|
||||
}
|
||||
|
||||
export async function readBridgeMeta(groupDir: string): Promise<BridgeMeta> {
|
||||
try {
|
||||
const content = await fsp.readFile(path.join(groupDir, 'meta.json'), 'utf-8');
|
||||
return JSON.parse(content) as BridgeMeta;
|
||||
} catch {
|
||||
return { version: 0, generatedAt: '', missingRepos: [] };
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* writeBridge — atomic write-to-temp-then-rename */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export interface WriteBridgeInput {
|
||||
contracts: StoredContract[];
|
||||
crossLinks: CrossLink[];
|
||||
repoSnapshots: Record<string, RepoSnapshot>;
|
||||
missingRepos: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Non-fatal issues encountered during writeBridge. Callers can log these to
|
||||
* surface partial-success state without aborting the whole sync.
|
||||
* `sampleErrors` is capped at MAX_SAMPLE_ERRORS per category to bound memory.
|
||||
*/
|
||||
export interface WriteBridgeReport {
|
||||
contractsInserted: number;
|
||||
contractsFailed: number;
|
||||
snapshotsInserted: number;
|
||||
snapshotsFailed: number;
|
||||
linksInserted: number;
|
||||
linksFailed: number;
|
||||
/** Cross-links skipped because their from/to contract nodes weren't found. */
|
||||
linksDroppedMissingNode: number;
|
||||
sampleErrors: Array<{
|
||||
kind: 'contract' | 'snapshot' | 'link';
|
||||
id: string;
|
||||
message: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
const MAX_SAMPLE_ERRORS = 10;
|
||||
|
||||
function errMessage(err: unknown): string {
|
||||
if (err instanceof Error) return err.message;
|
||||
try {
|
||||
return String(err);
|
||||
} catch {
|
||||
return 'unknown error';
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeBridge(
|
||||
groupDir: string,
|
||||
input: WriteBridgeInput,
|
||||
): Promise<WriteBridgeReport> {
|
||||
await fsp.mkdir(groupDir, { recursive: true });
|
||||
const contracts = dedupeContracts(input.contracts);
|
||||
const crossLinks = dedupeCrossLinks(input.crossLinks);
|
||||
|
||||
const finalPath = path.join(groupDir, 'bridge.lbug');
|
||||
const tmpPath = path.join(groupDir, 'bridge.lbug.tmp');
|
||||
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
|
||||
|
||||
const report: WriteBridgeReport = {
|
||||
contractsInserted: 0,
|
||||
contractsFailed: 0,
|
||||
snapshotsInserted: 0,
|
||||
snapshotsFailed: 0,
|
||||
linksInserted: 0,
|
||||
linksFailed: 0,
|
||||
linksDroppedMissingNode: 0,
|
||||
sampleErrors: [],
|
||||
};
|
||||
|
||||
const recordError = (kind: 'contract' | 'snapshot' | 'link', id: string, err: unknown) => {
|
||||
if (report.sampleErrors.length < MAX_SAMPLE_ERRORS) {
|
||||
report.sampleErrors.push({ kind, id, message: errMessage(err) });
|
||||
}
|
||||
};
|
||||
|
||||
// Clean up any leftover tmp
|
||||
try {
|
||||
await fsp.rm(tmpPath, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
// 1. Create temp DB, insert all data.
|
||||
//
|
||||
// Everything after `openBridgeDb` must run inside a try/finally so that
|
||||
// if ANY step before the explicit `closeBridgeDb` throws — schema
|
||||
// creation, a contract insert loop that rethrows, a snapshot write, the
|
||||
// cross-link loop, or anything else — the handle is still released. A
|
||||
// leaked handle holds the native LadybugDB file lock on tmpPath, which
|
||||
// (a) leaks a FD and (b) prevents the next writeBridge call from
|
||||
// reusing the same tmp slot.
|
||||
const handle = await openBridgeDb(tmpPath);
|
||||
let handleClosed = false;
|
||||
try {
|
||||
await ensureBridgeSchema(handle);
|
||||
|
||||
// Build the lookup index incrementally as contracts are inserted, so
|
||||
// failed inserts are never in the index (and therefore never resolved
|
||||
// by the cross-link loop below). This replaces a previous N+1 query
|
||||
// pattern where each link made up to 6 DB round-trips to find its
|
||||
// endpoints — see ContractLookupIndex.
|
||||
const lookupIndex = createContractLookupIndex();
|
||||
|
||||
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
|
||||
// that can't be serialized). The whole sync must not fail because one
|
||||
// contract is broken.
|
||||
for (const c of contracts) {
|
||||
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (n:Contract {
|
||||
id: $id,
|
||||
contractId: $contractId,
|
||||
type: $type,
|
||||
role: $role,
|
||||
repo: $repo,
|
||||
service: $service,
|
||||
symbolUid: $symbolUid,
|
||||
filePath: $filePath,
|
||||
symbolName: $symbolName,
|
||||
confidence: $confidence,
|
||||
meta: $meta
|
||||
})`,
|
||||
{
|
||||
id,
|
||||
contractId: c.contractId,
|
||||
type: c.type,
|
||||
role: c.role,
|
||||
repo: c.repo,
|
||||
service: c.service ?? '',
|
||||
symbolUid: c.symbolUid,
|
||||
filePath: c.symbolRef.filePath,
|
||||
symbolName: c.symbolName,
|
||||
confidence: c.confidence,
|
||||
meta: JSON.stringify(c.meta),
|
||||
},
|
||||
);
|
||||
report.contractsInserted++;
|
||||
// Only index on successful insert — the cross-link loop must never
|
||||
// resolve to a row that isn't actually in the DB.
|
||||
indexContract(lookupIndex, c, id);
|
||||
} catch (err) {
|
||||
report.contractsFailed++;
|
||||
recordError('contract', id, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert repo snapshots
|
||||
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (s:RepoSnapshot {
|
||||
id: $id,
|
||||
indexedAt: $indexedAt,
|
||||
lastCommit: $lastCommit
|
||||
})`,
|
||||
{
|
||||
id: repoId,
|
||||
indexedAt: snap.indexedAt,
|
||||
lastCommit: snap.lastCommit,
|
||||
},
|
||||
);
|
||||
report.snapshotsInserted++;
|
||||
} catch (err) {
|
||||
report.snapshotsFailed++;
|
||||
recordError('snapshot', repoId, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert cross-links (tolerating missing nodes).
|
||||
//
|
||||
// `findContractNode` consults the in-memory lookup index built above,
|
||||
// not the DB — that's an O(1) pure-function lookup per endpoint instead
|
||||
// of the previous 2-3 DB queries. For M cross-links, the previous code
|
||||
// issued up to 6M round-trips; this version issues zero.
|
||||
//
|
||||
// `link.contractId` may differ between the consumer and provider sides
|
||||
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
|
||||
// `grpc::Service/Method`) — that's why we resolve each endpoint
|
||||
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
|
||||
// tuple rather than matching on contractId.
|
||||
for (const link of crossLinks) {
|
||||
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
|
||||
try {
|
||||
const fromId = findContractNode(
|
||||
lookupIndex,
|
||||
link.from.repo,
|
||||
'consumer',
|
||||
link.from.symbolUid,
|
||||
link.from.symbolRef.filePath,
|
||||
link.from.symbolRef.name,
|
||||
);
|
||||
const toId = findContractNode(
|
||||
lookupIndex,
|
||||
link.to.repo,
|
||||
'provider',
|
||||
link.to.symbolUid,
|
||||
link.to.symbolRef.filePath,
|
||||
link.to.symbolRef.name,
|
||||
);
|
||||
if (!fromId || !toId) {
|
||||
report.linksDroppedMissingNode++;
|
||||
continue;
|
||||
}
|
||||
await queryBridge(
|
||||
handle,
|
||||
`
|
||||
MATCH (a:Contract), (b:Contract)
|
||||
WHERE a.id = $fromId AND b.id = $toId
|
||||
CREATE (a)-[:ContractLink {
|
||||
matchType: $matchType,
|
||||
confidence: $confidence,
|
||||
contractId: $contractId,
|
||||
fromRepo: $fromRepo,
|
||||
toRepo: $toRepo
|
||||
}]->(b)
|
||||
`,
|
||||
{
|
||||
fromId,
|
||||
toId,
|
||||
matchType: link.matchType,
|
||||
confidence: link.confidence,
|
||||
contractId: link.contractId,
|
||||
fromRepo: link.from.repo,
|
||||
toRepo: link.to.repo,
|
||||
},
|
||||
);
|
||||
report.linksInserted++;
|
||||
} catch (err) {
|
||||
report.linksFailed++;
|
||||
recordError('link', linkId, err);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Close temp DB (happy path). The finally block also calls
|
||||
// closeBridgeDb if we threw above; `handleClosed` prevents a
|
||||
// double-close on the native handle.
|
||||
await closeBridgeDb(handle);
|
||||
handleClosed = true;
|
||||
} finally {
|
||||
if (!handleClosed) {
|
||||
await closeBridgeDb(handle).catch(() => {
|
||||
/* ignore: cleanup path, best effort */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
|
||||
try {
|
||||
await fsp.access(finalPath);
|
||||
await retryRename(finalPath, bakPath);
|
||||
} catch {
|
||||
/* no existing db */
|
||||
}
|
||||
await retryRename(tmpPath, finalPath);
|
||||
try {
|
||||
await fsp.rm(bakPath, { recursive: true, force: true });
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
|
||||
// 4. Write meta.json
|
||||
await writeBridgeMeta(groupDir, {
|
||||
version: BRIDGE_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
missingRepos: input.missingRepos,
|
||||
});
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* openBridgeDbReadOnly */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export async function openBridgeDbReadOnly(groupDir: string): Promise<BridgeHandle | null> {
|
||||
const dbPath = path.join(groupDir, 'bridge.lbug');
|
||||
try {
|
||||
await fsp.access(dbPath);
|
||||
} catch {
|
||||
// Check for .bak recovery. Use `retryRename` (not `fsp.rename`) for the
|
||||
// exact same reason the rest of this file does: the scenario that
|
||||
// triggers bak recovery is an interrupted writer, which on Windows may
|
||||
// still be holding an open handle on `.bak` for a few milliseconds when
|
||||
// a reader races in. EBUSY/EPERM retries recover that case silently.
|
||||
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
|
||||
try {
|
||||
await fsp.access(bakPath);
|
||||
await retryRename(bakPath, dbPath);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// Version gate: check meta.json version compatibility
|
||||
const meta = await readBridgeMeta(groupDir);
|
||||
if (meta.version > 0 && meta.version !== BRIDGE_SCHEMA_VERSION) {
|
||||
return null; // incompatible schema version — fallback to JSON or re-sync
|
||||
}
|
||||
|
||||
// Open the native handle. If Connection construction throws AFTER
|
||||
// Database was successfully allocated, we'd leak the native Database
|
||||
// object. Wrap each step separately and tear down the partial handle.
|
||||
let db: lbug.Database | undefined;
|
||||
let conn: lbug.Connection | undefined;
|
||||
try {
|
||||
db = new lbug.Database(dbPath, 0, false, true); // readOnly
|
||||
conn = new lbug.Connection(db);
|
||||
return { _db: db, _conn: conn, groupDir } as BridgeHandle;
|
||||
} catch {
|
||||
if (conn) {
|
||||
try {
|
||||
await conn.close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
if (db) {
|
||||
try {
|
||||
await db.close();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* bridgeExists */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
export async function bridgeExists(groupDir: string): Promise<boolean> {
|
||||
const handle = await openBridgeDbReadOnly(groupDir);
|
||||
if (!handle) return false;
|
||||
await closeBridgeDb(handle);
|
||||
return true;
|
||||
}
|
||||
60
gitnexus/src/core/group/bridge-schema.ts
Normal file
60
gitnexus/src/core/group/bridge-schema.ts
Normal file
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Bridge LadybugDB schema for cross-repo Contract Registry.
|
||||
* Separate from per-repo schema in lbug/schema.ts.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Version of the bridge.lbug schema below. `openBridgeDbReadOnly` compares
|
||||
* this against `meta.json`'s version field and returns `null` on mismatch,
|
||||
* which trips the caller into either the JSON fallback path or a fresh
|
||||
* `group sync` that rebuilds `bridge.lbug` from scratch.
|
||||
*
|
||||
* Migration contract for contributors bumping this constant:
|
||||
* 1. Bump the number (e.g. `1` → `2`).
|
||||
* 2. Update the DDL below to match the new schema.
|
||||
* 3. DO NOT attempt an online migration in this file — the version gate
|
||||
* is intentionally a "discard and re-sync" strategy for V1. An old
|
||||
* bridge.lbug whose version doesn't match is treated as opaque and
|
||||
* rebuilt by the next `group sync`.
|
||||
* 4. If online migration becomes necessary (e.g. when groups accumulate
|
||||
* large amounts of embedding data), add a migration path as a
|
||||
* separate `bridge-migrations.ts` module rather than bloating this
|
||||
* file — keep schema and migration concerns separate.
|
||||
*/
|
||||
export const BRIDGE_SCHEMA_VERSION = 1;
|
||||
|
||||
export const CONTRACT_SCHEMA = `
|
||||
CREATE NODE TABLE Contract (
|
||||
id STRING,
|
||||
contractId STRING,
|
||||
type STRING,
|
||||
role STRING,
|
||||
repo STRING,
|
||||
service STRING DEFAULT '',
|
||||
symbolUid STRING DEFAULT '',
|
||||
filePath STRING DEFAULT '',
|
||||
symbolName STRING DEFAULT '',
|
||||
confidence DOUBLE DEFAULT 0.0,
|
||||
meta STRING DEFAULT '{}',
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const REPO_SNAPSHOT_SCHEMA = `
|
||||
CREATE NODE TABLE RepoSnapshot (
|
||||
id STRING,
|
||||
indexedAt STRING DEFAULT '',
|
||||
lastCommit STRING DEFAULT '',
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
|
||||
export const CONTRACT_LINK_SCHEMA = `
|
||||
CREATE REL TABLE ContractLink (
|
||||
FROM Contract TO Contract,
|
||||
matchType STRING,
|
||||
confidence DOUBLE,
|
||||
contractId STRING,
|
||||
fromRepo STRING,
|
||||
toRepo STRING
|
||||
)`;
|
||||
|
||||
export const BRIDGE_SCHEMA_QUERIES = [CONTRACT_SCHEMA, REPO_SNAPSHOT_SCHEMA, CONTRACT_LINK_SCHEMA];
|
||||
23
gitnexus/src/core/group/extractors/fs-utils.ts
Normal file
23
gitnexus/src/core/group/extractors/fs-utils.ts
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
|
||||
/**
|
||||
* Safely read a file inside a repo, rejecting any path that escapes
|
||||
* `repoPath` via `..` traversal or absolute segments. Returns `null` if
|
||||
* the path is outside the repo or the file can't be read.
|
||||
*
|
||||
* Used by every source-scan extractor under this directory. Kept as a
|
||||
* single shared implementation so the path-traversal guard (security-
|
||||
* sensitive) lives in exactly one place.
|
||||
*/
|
||||
export function readSafe(repoPath: string, rel: string): string | null {
|
||||
const abs = path.resolve(repoPath, rel);
|
||||
const base = path.resolve(repoPath);
|
||||
const relToBase = path.relative(base, abs);
|
||||
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
|
||||
try {
|
||||
return fs.readFileSync(abs, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,20 +1,38 @@
|
|||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { glob } from 'glob';
|
||||
import Parser from 'tree-sitter';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import {
|
||||
GRPC_SCAN_GLOB,
|
||||
getPluginForFile,
|
||||
hasProtoPlugin,
|
||||
type GrpcDetection,
|
||||
} from './grpc-patterns/index.js';
|
||||
|
||||
function readSafe(repoPath: string, rel: string): string | null {
|
||||
const abs = path.resolve(repoPath, rel);
|
||||
const base = path.resolve(repoPath);
|
||||
const relToBase = path.relative(base, abs);
|
||||
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
|
||||
try {
|
||||
return fs.readFileSync(abs, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Language-agnostic orchestrator for gRPC (provider + consumer) contract
|
||||
* extraction.
|
||||
*
|
||||
* Two parts:
|
||||
*
|
||||
* 1. **`.proto` parsing** — tree-sitter when `tree-sitter-proto` is
|
||||
* installed (optionalDependency vendored in `vendor/tree-sitter-proto/`),
|
||||
* via the `.proto` entry in `grpc-patterns/` and `hasProtoPlugin`.
|
||||
* When the grammar isn't available (platform incompatibility, native
|
||||
* build failure) the orchestrator falls back to the in-process
|
||||
* string-sanitizing parser defined below (`stripProtoCommentsAndStrings`
|
||||
* + `extractServiceBlocks`). The fallback preserves offsets so any
|
||||
* downstream regex scans run against a sanitized copy without
|
||||
* affecting line numbers of the original.
|
||||
*
|
||||
* 2. **Source-scan providers / consumers** — delegated to per-language
|
||||
* plugins in `./grpc-patterns/`. The orchestrator imports NO
|
||||
* tree-sitter grammars or query strings — each plugin owns its own.
|
||||
*/
|
||||
|
||||
// ─── .proto fallback parser (used only when tree-sitter-proto is absent) ───
|
||||
|
||||
function contractId(pkg: string, service: string, method: string): string {
|
||||
const prefix = pkg ? `${pkg}.${service}` : service;
|
||||
|
|
@ -25,20 +43,110 @@ function serviceOnlyContractId(serviceName: string): string {
|
|||
return `grpc::${serviceName}/*`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace all .proto comments and string literals with spaces, preserving the
|
||||
* original length and character offsets of the input. This lets downstream
|
||||
* regex / brace-depth parsers run on a "sanitized" copy without having to
|
||||
* understand proto syntax, while any RegExp.exec/index-based lookups that
|
||||
* were already positional against `content` continue to work against the
|
||||
* original string.
|
||||
*
|
||||
* Supported comment forms: `// line comment`, `/* block comment * /`.
|
||||
* Supported strings: double-quoted ("…") and single-quoted ('…') with `\`
|
||||
* escape handling. Raw/unterminated strings are not supported — we stop
|
||||
* on a line break for line-style comments and on EOF for unterminated
|
||||
* strings/blocks, which matches how most real proto files parse.
|
||||
*/
|
||||
function stripProtoCommentsAndStrings(content: string): string {
|
||||
const out = new Array<string>(content.length);
|
||||
let i = 0;
|
||||
while (i < content.length) {
|
||||
const ch = content[i];
|
||||
const next = content[i + 1];
|
||||
|
||||
// Line comment: // ... \n
|
||||
if (ch === '/' && next === '/') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
while (i < content.length && content[i] !== '\n') {
|
||||
out[i] = content[i] === '\r' ? '\r' : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Block comment: /* ... */
|
||||
if (ch === '/' && next === '*') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
while (i < content.length) {
|
||||
if (content[i] === '*' && content[i + 1] === '/') {
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
// Preserve newlines so line numbers stay stable for downstream code.
|
||||
out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// String literal: "..." or '...'
|
||||
if (ch === '"' || ch === "'") {
|
||||
const quote = ch;
|
||||
out[i] = ' '; // replace opening quote
|
||||
i++;
|
||||
while (i < content.length) {
|
||||
const c = content[i];
|
||||
if (c === '\\' && i + 1 < content.length) {
|
||||
// Skip escaped pair (e.g. \" \n \\)
|
||||
out[i] = ' ';
|
||||
out[i + 1] = ' ';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === quote) {
|
||||
out[i] = ' ';
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
// Preserve newlines; proto technically disallows unescaped newlines
|
||||
// inside strings, but real files occasionally have them.
|
||||
out[i] = c === '\n' || c === '\r' ? c : ' ';
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
out[i] = ch;
|
||||
i++;
|
||||
}
|
||||
return out.join('');
|
||||
}
|
||||
|
||||
function extractServiceBlocks(content: string): Array<{ name: string; body: string }> {
|
||||
const results: Array<{ name: string; body: string }> = [];
|
||||
// v1: brace-depth only — braces inside comments or string literals are not filtered (see spec Fix 2)
|
||||
// Sanitize comments and string literals so braces inside them don't
|
||||
// throw off the depth counter. The sanitized copy has the same length
|
||||
// and offsets as the original, so we use it ONLY to scan for service
|
||||
// headers and braces; the service body we return is sliced from the
|
||||
// ORIGINAL content to preserve exact source text for downstream use.
|
||||
const sanitized = stripProtoCommentsAndStrings(content);
|
||||
const headerRe = /service\s+(\w+)\s*\{/g;
|
||||
let headerMatch: RegExpExecArray | null;
|
||||
|
||||
while ((headerMatch = headerRe.exec(content)) !== null) {
|
||||
while ((headerMatch = headerRe.exec(sanitized)) !== null) {
|
||||
const serviceName = headerMatch[1];
|
||||
const bodyStart = headerMatch.index + headerMatch[0].length;
|
||||
let depth = 1;
|
||||
let pos = bodyStart;
|
||||
|
||||
while (pos < content.length && depth > 0) {
|
||||
const ch = content[pos];
|
||||
while (pos < sanitized.length && depth > 0) {
|
||||
const ch = sanitized[pos];
|
||||
if (ch === '{') depth++;
|
||||
else if (ch === '}') depth--;
|
||||
pos++;
|
||||
|
|
@ -75,6 +183,177 @@ function makeContract(
|
|||
};
|
||||
}
|
||||
|
||||
export interface ProtoServiceInfo {
|
||||
package: string;
|
||||
serviceName: string;
|
||||
methods: string[];
|
||||
protoPath: string;
|
||||
}
|
||||
|
||||
function normalizeProtoPath(rel: string): string {
|
||||
return rel.replace(/\\/g, '/');
|
||||
}
|
||||
|
||||
function extractProtoImports(content: string): string[] {
|
||||
const imports: string[] = [];
|
||||
const re = /^\s*import\s+"([^"]+)"\s*;/gm;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = re.exec(content)) !== null) {
|
||||
imports.push(match[1]);
|
||||
}
|
||||
return imports;
|
||||
}
|
||||
|
||||
function longestSharedSegmentRun(aPath: string, bPath: string): number {
|
||||
const a = aPath.split('/').filter(Boolean);
|
||||
const b = bPath.split('/').filter(Boolean);
|
||||
let best = 0;
|
||||
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
for (let j = 0; j < b.length; j++) {
|
||||
let run = 0;
|
||||
while (a[i + run] && b[j + run] && a[i + run] === b[j + run]) {
|
||||
run++;
|
||||
}
|
||||
if (run > best) best = run;
|
||||
}
|
||||
}
|
||||
|
||||
return best;
|
||||
}
|
||||
|
||||
async function buildProtoContext(repoPath: string): Promise<{
|
||||
packagesByProto: Map<string, string>;
|
||||
servicesByName: Map<string, ProtoServiceInfo[]>;
|
||||
}> {
|
||||
const servicesByName = new Map<string, ProtoServiceInfo[]>();
|
||||
const protoFiles = await glob('**/*.proto', {
|
||||
cwd: repoPath,
|
||||
absolute: false,
|
||||
nodir: true,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'],
|
||||
});
|
||||
const contents = new Map<string, string>();
|
||||
|
||||
for (const rel of protoFiles) {
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
contents.set(normalizeProtoPath(rel), content);
|
||||
}
|
||||
|
||||
const packagesByProto = new Map<string, string>();
|
||||
|
||||
const resolvePackage = (protoPath: string, seen = new Set<string>()): string => {
|
||||
if (packagesByProto.has(protoPath)) return packagesByProto.get(protoPath) ?? '';
|
||||
if (seen.has(protoPath)) return '';
|
||||
|
||||
const content = contents.get(protoPath);
|
||||
if (!content) return '';
|
||||
|
||||
seen.add(protoPath);
|
||||
const pkgMatch = content.match(/^\s*package\s+([\w.]+)\s*;/m);
|
||||
if (pkgMatch?.[1]) {
|
||||
packagesByProto.set(protoPath, pkgMatch[1]);
|
||||
return pkgMatch[1];
|
||||
}
|
||||
|
||||
for (const importPath of extractProtoImports(content)) {
|
||||
const normalizedImport = normalizeProtoPath(importPath);
|
||||
const candidates = [
|
||||
normalizeProtoPath(
|
||||
path.posix.normalize(path.posix.join(path.posix.dirname(protoPath), normalizedImport)),
|
||||
),
|
||||
normalizedImport,
|
||||
];
|
||||
for (const candidate of candidates) {
|
||||
if (!contents.has(candidate)) continue;
|
||||
const inheritedPackage = resolvePackage(candidate, seen);
|
||||
if (inheritedPackage) {
|
||||
packagesByProto.set(protoPath, inheritedPackage);
|
||||
return inheritedPackage;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
packagesByProto.set(protoPath, '');
|
||||
return '';
|
||||
};
|
||||
|
||||
for (const rel of protoFiles) {
|
||||
const normalizedRel = normalizeProtoPath(rel);
|
||||
const content = contents.get(normalizedRel);
|
||||
if (!content) continue;
|
||||
const pkg = resolvePackage(normalizedRel);
|
||||
|
||||
const serviceBlocks = extractServiceBlocks(content);
|
||||
for (const block of serviceBlocks) {
|
||||
const rpcRe = /rpc\s+(\w+)\s*\(/g;
|
||||
const methods: string[] = [];
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = rpcRe.exec(block.body)) !== null) {
|
||||
methods.push(m[1]);
|
||||
}
|
||||
const info: ProtoServiceInfo = {
|
||||
package: pkg,
|
||||
serviceName: block.name,
|
||||
methods,
|
||||
protoPath: normalizedRel,
|
||||
};
|
||||
const existing = servicesByName.get(block.name) ?? [];
|
||||
existing.push(info);
|
||||
servicesByName.set(block.name, existing);
|
||||
}
|
||||
}
|
||||
|
||||
return { packagesByProto, servicesByName };
|
||||
}
|
||||
|
||||
export async function buildProtoMap(repoPath: string): Promise<Map<string, ProtoServiceInfo[]>> {
|
||||
const { servicesByName } = await buildProtoContext(repoPath);
|
||||
return servicesByName;
|
||||
}
|
||||
|
||||
export function resolveProtoConflict(
|
||||
serviceName: string,
|
||||
sourceFilePath: string,
|
||||
candidates: ProtoServiceInfo[],
|
||||
): ProtoServiceInfo | null {
|
||||
if (candidates.length === 0) return null;
|
||||
if (candidates.length === 1) return candidates[0];
|
||||
|
||||
const sourceDir = normalizeProtoPath(path.dirname(sourceFilePath));
|
||||
const scored = candidates.map((c) => {
|
||||
const protoDir = normalizeProtoPath(path.dirname(c.protoPath));
|
||||
return { candidate: c, score: longestSharedSegmentRun(sourceDir, protoDir) };
|
||||
});
|
||||
|
||||
let maxScore = -1;
|
||||
for (const s of scored) {
|
||||
if (s.score > maxScore) maxScore = s.score;
|
||||
}
|
||||
const winners = scored.filter((s) => s.score === maxScore);
|
||||
|
||||
// Path heuristic cannot uniquely identify a winner — refuse to guess.
|
||||
// Ties (including all-zero ties) would otherwise silently merge unrelated
|
||||
// services under a fabricated package-qualified contract id.
|
||||
if (winners.length !== 1) {
|
||||
const paths = candidates.map((c) => c.protoPath).join(', ');
|
||||
console.warn(
|
||||
`[grpc-extractor] Ambiguous proto resolution for service "${serviceName}" from ${sourceFilePath}: ${winners.length} candidates tied at score ${maxScore} among [${paths}] — skipping canonical contract`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
return winners[0].candidate;
|
||||
}
|
||||
|
||||
export function serviceContractId(pkg: string, serviceName: string): string {
|
||||
const prefix = pkg ? `${pkg}.${serviceName}` : serviceName;
|
||||
return `grpc::${prefix}/*`;
|
||||
}
|
||||
|
||||
// ─── Orchestrator ────────────────────────────────────────────────────
|
||||
|
||||
export class GrpcExtractor implements ContractExtractor {
|
||||
type = 'grpc' as const;
|
||||
|
||||
|
|
@ -88,270 +367,116 @@ export class GrpcExtractor implements ContractExtractor {
|
|||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const out: ExtractedContract[] = [];
|
||||
const protoContext = await buildProtoContext(repoPath);
|
||||
const protoMap = protoContext.servicesByName;
|
||||
|
||||
// Proto files — definitive provider source
|
||||
const protoFiles = await glob('**/*.proto', {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'],
|
||||
nodir: true,
|
||||
});
|
||||
for (const rel of protoFiles) {
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (content) out.push(...this.parseProtoFile(content, rel));
|
||||
// ─── Proto files — definitive provider source ─────────────────
|
||||
// When tree-sitter-proto is available, .proto files are handled by
|
||||
// the plugin loop below (they're in GRPC_SCAN_GLOB). Otherwise
|
||||
// emit provider contracts directly from the proto map that
|
||||
// `buildProtoContext` already built — no second glob / parse pass.
|
||||
if (!hasProtoPlugin) {
|
||||
for (const infos of protoMap.values()) {
|
||||
for (const info of infos) {
|
||||
for (const methodName of info.methods) {
|
||||
const cid = contractId(info.package, info.serviceName, methodName);
|
||||
out.push(
|
||||
makeContract(
|
||||
cid,
|
||||
'provider',
|
||||
info.protoPath,
|
||||
`${info.serviceName}.${methodName}`,
|
||||
0.85,
|
||||
{
|
||||
package: info.package,
|
||||
service: info.serviceName,
|
||||
method: methodName,
|
||||
source: 'proto',
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Source files — server/client detection
|
||||
const sourceFiles = await glob('**/*.{go,java,py,ts,tsx,js,jsx}', {
|
||||
// ─── Source files (+ .proto when plugin available) ────────────
|
||||
const sourceFiles = await glob(GRPC_SCAN_GLOB, {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
nodir: true,
|
||||
});
|
||||
|
||||
const parser = new Parser();
|
||||
for (const rel of sourceFiles) {
|
||||
const plugin = getPluginForFile(rel);
|
||||
if (!plugin) continue;
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
const ext = path.extname(rel).toLowerCase();
|
||||
|
||||
if (ext === '.go') {
|
||||
out.push(...this.scanGoProviders(content, rel));
|
||||
out.push(...this.scanGoConsumers(content, rel));
|
||||
} else if (ext === '.java') {
|
||||
out.push(...this.scanJavaProviders(content, rel));
|
||||
out.push(...this.scanJavaConsumers(content, rel));
|
||||
} else if (ext === '.py') {
|
||||
out.push(...this.scanPythonProviders(content, rel));
|
||||
out.push(...this.scanPythonConsumers(content, rel));
|
||||
} else if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) {
|
||||
out.push(...this.scanTsProviders(content, rel));
|
||||
let detections: GrpcDetection[] = [];
|
||||
try {
|
||||
parser.setLanguage(plugin.language);
|
||||
const tree = parser.parse(content);
|
||||
detections = plugin.scan(tree);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const d of detections) {
|
||||
const contract = this.detectionToContract(d, rel, protoMap);
|
||||
if (contract) out.push(contract);
|
||||
}
|
||||
}
|
||||
|
||||
return this.dedupe(out);
|
||||
}
|
||||
|
||||
private parseProtoFile(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
|
||||
const pkgMatch = content.match(/^package\s+([\w.]+)\s*;/m);
|
||||
const pkg = pkgMatch ? pkgMatch[1] : '';
|
||||
|
||||
for (const { name: serviceName, body } of extractServiceBlocks(content)) {
|
||||
const rpcRe = /rpc\s+(\w+)\s*\(/g;
|
||||
let rpcMatch: RegExpExecArray | null;
|
||||
while ((rpcMatch = rpcRe.exec(body)) !== null) {
|
||||
const methodName = rpcMatch[1];
|
||||
const cid = contractId(pkg, serviceName, methodName);
|
||||
out.push(
|
||||
makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.85, {
|
||||
package: pkg,
|
||||
service: serviceName,
|
||||
method: methodName,
|
||||
source: 'proto',
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanGoProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
|
||||
// pb.RegisterXxxServer(
|
||||
const registerRe = /\w+\.Register(\w+)Server\s*\(/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = registerRe.exec(content)) !== null) {
|
||||
const serviceName = m[1];
|
||||
out.push(
|
||||
makeContract(
|
||||
serviceOnlyContractId(serviceName),
|
||||
'provider',
|
||||
filePath,
|
||||
`Register${serviceName}Server`,
|
||||
0.8,
|
||||
{ service: serviceName, source: 'go_register' },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// pb.UnimplementedXxxServer
|
||||
const unimplRe = /\w+\.Unimplemented(\w+)Server\b/g;
|
||||
while ((m = unimplRe.exec(content)) !== null) {
|
||||
const serviceName = m[1];
|
||||
out.push(
|
||||
makeContract(
|
||||
serviceOnlyContractId(serviceName),
|
||||
'provider',
|
||||
filePath,
|
||||
`Unimplemented${serviceName}Server`,
|
||||
0.8,
|
||||
{ service: serviceName, source: 'go_unimplemented' },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanGoConsumers(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
const re = /\w+\.New(\w+)Client\s*\(/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const serviceName = m[1];
|
||||
out.push(
|
||||
makeContract(
|
||||
serviceOnlyContractId(serviceName),
|
||||
'consumer',
|
||||
filePath,
|
||||
`New${serviceName}Client`,
|
||||
0.7,
|
||||
{ service: serviceName, source: 'go_client' },
|
||||
),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanJavaProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
|
||||
// @GrpcService
|
||||
if (content.includes('@GrpcService')) {
|
||||
const implBaseRe = /extends\s+(\w+)Grpc\.(\w+)ImplBase/;
|
||||
const m = content.match(implBaseRe);
|
||||
if (m) {
|
||||
out.push(
|
||||
makeContract(serviceOnlyContractId(m[1]), 'provider', filePath, m[2], 0.8, {
|
||||
service: m[1],
|
||||
source: 'java_grpc_service',
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
// Try extracting service name from class name
|
||||
const classRe =
|
||||
/class\s+(\w*?)(?:Grpc)?(?:Service)?\s+extends\s+(\w+)(?:Grpc\.(\w+))?ImplBase/;
|
||||
const cm = content.match(classRe);
|
||||
if (cm) {
|
||||
const svcName = cm[2].replace(/Grpc$/, '');
|
||||
out.push(
|
||||
makeContract(serviceOnlyContractId(svcName), 'provider', filePath, cm[1], 0.8, {
|
||||
service: svcName,
|
||||
source: 'java_grpc_service',
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extends XxxImplBase (without @GrpcService)
|
||||
if (!content.includes('@GrpcService')) {
|
||||
const implRe = /extends\s+(\w+?)(?:Grpc\.(\w+))?ImplBase/;
|
||||
const m = content.match(implRe);
|
||||
if (m) {
|
||||
const svcName = m[2] || m[1].replace(/Grpc$/, '');
|
||||
out.push(
|
||||
makeContract(serviceOnlyContractId(svcName), 'provider', filePath, svcName, 0.8, {
|
||||
service: svcName,
|
||||
source: 'java_impl_base',
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanJavaConsumers(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
// XxxGrpc.newBlockingStub( or XxxGrpc.newStub(
|
||||
const re = /(\w+)Grpc\.new(?:Blocking)?Stub\s*\(/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const serviceName = m[1];
|
||||
out.push(
|
||||
makeContract(
|
||||
serviceOnlyContractId(serviceName),
|
||||
'consumer',
|
||||
filePath,
|
||||
`${serviceName}Stub`,
|
||||
0.7,
|
||||
{ service: serviceName, source: 'java_stub' },
|
||||
),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanPythonProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
// add_XxxServicer_to_server(
|
||||
const re = /add_(\w+?)Servicer_to_server\s*\(/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const serviceName = m[1];
|
||||
out.push(
|
||||
makeContract(
|
||||
serviceOnlyContractId(serviceName),
|
||||
'provider',
|
||||
filePath,
|
||||
`add_${serviceName}Servicer_to_server`,
|
||||
0.8,
|
||||
{ service: serviceName, source: 'python_servicer' },
|
||||
),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanPythonConsumers(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
// XxxStub(
|
||||
const re = /(\w+)Stub\s*\(/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const name = m[1];
|
||||
// Filter out common false positives
|
||||
if (['Mock', 'Test', 'Fake', 'Stub'].includes(name)) continue;
|
||||
out.push(
|
||||
makeContract(serviceOnlyContractId(name), 'consumer', filePath, `${name}Stub`, 0.7, {
|
||||
service: name,
|
||||
source: 'python_stub',
|
||||
}),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanTsProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
// @GrpcMethod('ServiceName', 'MethodName')
|
||||
const re = /@GrpcMethod\s*\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const serviceName = m[1];
|
||||
const methodName = m[2];
|
||||
const cid = contractId('', serviceName, methodName);
|
||||
out.push(
|
||||
makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.8, {
|
||||
service: serviceName,
|
||||
method: methodName,
|
||||
source: 'ts_grpc_method',
|
||||
}),
|
||||
);
|
||||
}
|
||||
return out;
|
||||
/**
|
||||
* Convert a plugin `GrpcDetection` into a concrete `ExtractedContract`
|
||||
* by resolving the short service name against the proto map, building
|
||||
* either a service-level (`grpc::pkg.Svc/*`) or method-level
|
||||
* (`grpc::pkg.Svc/Method`) contract id, and selecting confidence
|
||||
* based on whether the proto map had an entry.
|
||||
*/
|
||||
private detectionToContract(
|
||||
d: GrpcDetection,
|
||||
filePath: string,
|
||||
protoMap: Map<string, ProtoServiceInfo[]>,
|
||||
): ExtractedContract | null {
|
||||
const candidates = protoMap.get(d.serviceName) ?? [];
|
||||
const proto = resolveProtoConflict(d.serviceName, filePath, candidates);
|
||||
// If there were proto candidates but resolution was ambiguous, skip
|
||||
// contract emission rather than fabricating a package-qualified id from
|
||||
// an arbitrary candidate. resolveProtoConflict already warned.
|
||||
if (candidates.length > 0 && proto === null) return null;
|
||||
const pkg = proto?.package ?? '';
|
||||
const cid = d.methodName
|
||||
? contractId(pkg, d.serviceName, d.methodName)
|
||||
: proto
|
||||
? serviceContractId(pkg, d.serviceName)
|
||||
: serviceOnlyContractId(d.serviceName);
|
||||
const confidence = proto ? d.confidenceWithProto : d.confidenceWithoutProto;
|
||||
const meta: Record<string, unknown> = {
|
||||
service: d.serviceName,
|
||||
source: d.source,
|
||||
};
|
||||
if (d.methodName) meta.method = d.methodName;
|
||||
return makeContract(cid, d.role, filePath, d.symbolName, confidence, meta);
|
||||
}
|
||||
|
||||
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const seen = new Set<string>();
|
||||
const out: ExtractedContract[] = [];
|
||||
const byKey = new Map<string, ExtractedContract>();
|
||||
for (const c of items) {
|
||||
const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`;
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push(c);
|
||||
const existing = byKey.get(k);
|
||||
if (
|
||||
!existing ||
|
||||
c.confidence > existing.confidence ||
|
||||
(c.confidence === existing.confidence &&
|
||||
String(c.meta.source) < String(existing.meta.source))
|
||||
) {
|
||||
byKey.set(k, c);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
return Array.from(byKey.values());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
109
gitnexus/src/core/group/extractors/grpc-patterns/go.ts
Normal file
109
gitnexus/src/core/group/extractors/grpc-patterns/go.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import Go from 'tree-sitter-go';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Go gRPC plugin. Detects:
|
||||
* - Provider: `pb.RegisterXxxServer(...)` calls
|
||||
* - Provider: `pb.UnimplementedXxxServer` embedded in a struct
|
||||
* - Consumer: `pb.NewXxxClient(conn)` calls
|
||||
*/
|
||||
|
||||
const REGISTER_RE = /^Register(\w+)Server$/;
|
||||
const UNIMPLEMENTED_RE = /^Unimplemented(\w+)Server$/;
|
||||
const NEW_CLIENT_RE = /^New(\w+)Client$/;
|
||||
|
||||
// Any `xxx.<fn>(...)` call — plugin filters the field identifier text.
|
||||
const SELECTOR_CALL_PATTERNS = compilePatterns({
|
||||
name: 'go-grpc-selector-call',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
field: (field_identifier) @fn))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// Any `qualified_type` used as a struct field — for `pb.UnimplementedXxxServer`.
|
||||
const STRUCT_EMBEDDING_PATTERNS = compilePatterns({
|
||||
name: 'go-grpc-struct-embedding',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(struct_type
|
||||
(field_declaration_list
|
||||
(field_declaration
|
||||
type: (qualified_type
|
||||
name: (type_identifier) @field_type))))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
export const GO_GRPC_PLUGIN: GrpcLanguagePlugin = {
|
||||
name: 'go-grpc',
|
||||
language: Go,
|
||||
scan(tree) {
|
||||
const out: GrpcDetection[] = [];
|
||||
|
||||
for (const match of runCompiledPatterns(SELECTOR_CALL_PATTERNS, tree)) {
|
||||
const fnNode = match.captures.fn;
|
||||
if (!fnNode) continue;
|
||||
const fnText = fnNode.text;
|
||||
|
||||
const registerMatch = REGISTER_RE.exec(fnText);
|
||||
if (registerMatch) {
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName: registerMatch[1],
|
||||
symbolName: fnText,
|
||||
source: 'go_register',
|
||||
confidenceWithProto: 0.8,
|
||||
confidenceWithoutProto: 0.65,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const newClientMatch = NEW_CLIENT_RE.exec(fnText);
|
||||
if (newClientMatch) {
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName: newClientMatch[1],
|
||||
symbolName: fnText,
|
||||
source: 'go_client',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
for (const match of runCompiledPatterns(STRUCT_EMBEDDING_PATTERNS, tree)) {
|
||||
const fieldNode = match.captures.field_type;
|
||||
if (!fieldNode) continue;
|
||||
const unimpl = UNIMPLEMENTED_RE.exec(fieldNode.text);
|
||||
if (!unimpl) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName: unimpl[1],
|
||||
symbolName: fieldNode.text,
|
||||
source: 'go_unimplemented',
|
||||
confidenceWithProto: 0.8,
|
||||
confidenceWithoutProto: 0.65,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
53
gitnexus/src/core/group/extractors/grpc-patterns/index.ts
Normal file
53
gitnexus/src/core/group/extractors/grpc-patterns/index.ts
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import * as path from 'node:path';
|
||||
import type { GrpcLanguagePlugin } from './types.js';
|
||||
import { GO_GRPC_PLUGIN } from './go.js';
|
||||
import { JAVA_GRPC_PLUGIN } from './java.js';
|
||||
import { PYTHON_GRPC_PLUGIN } from './python.js';
|
||||
import { JAVASCRIPT_GRPC_PLUGIN, TYPESCRIPT_GRPC_PLUGIN, TSX_GRPC_PLUGIN } from './node.js';
|
||||
import { PROTO_GRPC_PLUGIN } from './proto.js';
|
||||
|
||||
export type { GrpcDetection, GrpcLanguagePlugin, GrpcRole } from './types.js';
|
||||
export { PROTO_GRPC_PLUGIN, extractPackageFromTree } from './proto.js';
|
||||
|
||||
/**
|
||||
* File-extension → gRPC language plugin registry. Mirrors the shape
|
||||
* of `http-patterns/index.ts` and `topic-patterns/index.ts`.
|
||||
*
|
||||
* `.proto` files are registered only when `tree-sitter-proto` is
|
||||
* available (it's an optionalDependency). When absent, the orchestrator
|
||||
* falls back to the built-in manual proto parser.
|
||||
*/
|
||||
const REGISTRY: Record<string, GrpcLanguagePlugin> = {
|
||||
'.go': GO_GRPC_PLUGIN,
|
||||
'.java': JAVA_GRPC_PLUGIN,
|
||||
'.py': PYTHON_GRPC_PLUGIN,
|
||||
'.js': JAVASCRIPT_GRPC_PLUGIN,
|
||||
'.jsx': JAVASCRIPT_GRPC_PLUGIN,
|
||||
'.ts': TYPESCRIPT_GRPC_PLUGIN,
|
||||
'.tsx': TSX_GRPC_PLUGIN,
|
||||
...(PROTO_GRPC_PLUGIN ? { '.proto': PROTO_GRPC_PLUGIN } : {}),
|
||||
};
|
||||
|
||||
/**
|
||||
* Glob for source files worth scanning for gRPC server/client patterns.
|
||||
* Includes `.proto` when the grammar is available.
|
||||
*/
|
||||
export const GRPC_SCAN_GLOB = PROTO_GRPC_PLUGIN
|
||||
? '**/*.{go,java,py,ts,tsx,js,jsx,proto}'
|
||||
: '**/*.{go,java,py,ts,tsx,js,jsx}';
|
||||
|
||||
/**
|
||||
* Whether the tree-sitter proto plugin is available. The orchestrator
|
||||
* uses this to decide between the tree-sitter path and the fallback
|
||||
* manual parser for `.proto` files.
|
||||
*/
|
||||
export const hasProtoPlugin = PROTO_GRPC_PLUGIN !== null;
|
||||
|
||||
/**
|
||||
* Return the gRPC plugin registered for the given file's extension,
|
||||
* or `undefined` if the extension is not registered.
|
||||
*/
|
||||
export function getPluginForFile(rel: string): GrpcLanguagePlugin | undefined {
|
||||
const ext = path.extname(rel).toLowerCase();
|
||||
return REGISTRY[ext];
|
||||
}
|
||||
179
gitnexus/src/core/group/extractors/grpc-patterns/java.ts
Normal file
179
gitnexus/src/core/group/extractors/grpc-patterns/java.ts
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
import Parser from 'tree-sitter';
|
||||
import Java from 'tree-sitter-java';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Java gRPC plugin. Detects:
|
||||
* - Provider: classes extending `XxxServiceGrpc.XxxServiceImplBase`
|
||||
* (with or without a `@GrpcService` annotation; the annotation
|
||||
* only affects confidence labelling in the original regex version
|
||||
* — here we emit a single detection per class and pick the source
|
||||
* label based on whether the annotation is present).
|
||||
* - Consumer: `XxxServiceGrpc.newBlockingStub(ch)` /
|
||||
* `XxxServiceGrpc.newStub(ch)` calls.
|
||||
*/
|
||||
|
||||
const IMPL_BASE_RE = /^(\w+)ImplBase$/;
|
||||
const GRPC_SUFFIX_RE = /^(\w+)Grpc$/;
|
||||
|
||||
// Classes extending `ScopedType.ScopedType` where the inner name ends
|
||||
// in ImplBase. Covers `XxxServiceGrpc.XxxServiceImplBase`.
|
||||
// Note: tree-sitter-java's `scoped_type_identifier` exposes its two
|
||||
// segments as positional `type_identifier` children, NOT as named
|
||||
// `scope:`/`name:` fields. We match positionally here and rely on the
|
||||
// grammar's left-to-right ordering: first child = outer, second = inner.
|
||||
const SCOPED_IMPL_BASE_PATTERNS = compilePatterns({
|
||||
name: 'java-grpc-scoped-impl-base',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(class_declaration
|
||||
name: (identifier) @class_name
|
||||
superclass: (superclass
|
||||
(scoped_type_identifier
|
||||
(type_identifier) @outer
|
||||
(type_identifier) @inner (#match? @inner "ImplBase$")))) @class
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// Classes extending a simple `XxxImplBase` identifier (no scope).
|
||||
const PLAIN_IMPL_BASE_PATTERNS = compilePatterns({
|
||||
name: 'java-grpc-plain-impl-base',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(class_declaration
|
||||
name: (identifier) @class_name
|
||||
superclass: (superclass
|
||||
(type_identifier) @plain_type (#match? @plain_type "ImplBase$"))) @class
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// gRPC stub factories: `XxxGrpc.newStub(ch)` / `XxxGrpc.newBlockingStub(ch)`.
|
||||
const STUB_PATTERNS = compilePatterns({
|
||||
name: 'java-grpc-stub',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (identifier) @grpc_cls
|
||||
name: (identifier) @method (#match? @method "^new(Blocking)?Stub$"))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
/**
|
||||
* Check whether a `class_declaration` node has a `@GrpcService`
|
||||
* annotation in its modifiers list. In tree-sitter-java, class-level
|
||||
* annotations live under `(class_declaration (modifiers (marker_annotation|annotation)))`.
|
||||
*/
|
||||
function hasGrpcServiceAnnotation(classNode: Parser.SyntaxNode): boolean {
|
||||
for (let i = 0; i < classNode.namedChildCount; i++) {
|
||||
const child = classNode.namedChild(i);
|
||||
if (!child || child.type !== 'modifiers') continue;
|
||||
for (let j = 0; j < child.namedChildCount; j++) {
|
||||
const mod = child.namedChild(j);
|
||||
if (!mod) continue;
|
||||
if (mod.type !== 'marker_annotation' && mod.type !== 'annotation') continue;
|
||||
const nameNode = mod.childForFieldName('name');
|
||||
if (nameNode?.text === 'GrpcService') return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Given the inner type_identifier text like `AuthServiceImplBase`,
|
||||
* return the service name (`AuthService`), or null if the text
|
||||
* doesn't end in `ImplBase`.
|
||||
*/
|
||||
function extractServiceFromImplBase(text: string): string | null {
|
||||
const m = IMPL_BASE_RE.exec(text);
|
||||
if (!m) return null;
|
||||
// Strip a trailing `Grpc` on the service name too — the original
|
||||
// regex replaces `Grpc$` on the extracted prefix.
|
||||
return m[1].replace(/Grpc$/, '');
|
||||
}
|
||||
|
||||
export const JAVA_GRPC_PLUGIN: GrpcLanguagePlugin = {
|
||||
name: 'java-grpc',
|
||||
language: Java,
|
||||
scan(tree) {
|
||||
const out: GrpcDetection[] = [];
|
||||
const emittedClassIds = new Set<number>();
|
||||
|
||||
// ─── Providers: scoped form (`...Grpc.XxxImplBase`) ─────────────
|
||||
for (const match of runCompiledPatterns(SCOPED_IMPL_BASE_PATTERNS, tree)) {
|
||||
const classNode = match.captures.class;
|
||||
const innerNode = match.captures.inner;
|
||||
if (!classNode || !innerNode) continue;
|
||||
const serviceName = extractServiceFromImplBase(innerNode.text);
|
||||
if (!serviceName) continue;
|
||||
emittedClassIds.add(classNode.id);
|
||||
const annotated = hasGrpcServiceAnnotation(classNode);
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName,
|
||||
symbolName: serviceName,
|
||||
source: annotated ? 'java_grpc_service' : 'java_impl_base',
|
||||
confidenceWithProto: 0.8,
|
||||
confidenceWithoutProto: 0.65,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Providers: plain form (`XxxImplBase`) ──────────────────────
|
||||
for (const match of runCompiledPatterns(PLAIN_IMPL_BASE_PATTERNS, tree)) {
|
||||
const classNode = match.captures.class;
|
||||
const plainNode = match.captures.plain_type;
|
||||
if (!classNode || !plainNode) continue;
|
||||
if (emittedClassIds.has(classNode.id)) continue;
|
||||
const serviceName = extractServiceFromImplBase(plainNode.text);
|
||||
if (!serviceName) continue;
|
||||
emittedClassIds.add(classNode.id);
|
||||
const annotated = hasGrpcServiceAnnotation(classNode);
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName,
|
||||
symbolName: serviceName,
|
||||
source: annotated ? 'java_grpc_service' : 'java_impl_base',
|
||||
confidenceWithProto: 0.8,
|
||||
confidenceWithoutProto: 0.65,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumers: `XxxGrpc.newBlockingStub(...)` / `newStub(...)` ─
|
||||
for (const match of runCompiledPatterns(STUB_PATTERNS, tree)) {
|
||||
const grpcClsNode = match.captures.grpc_cls;
|
||||
if (!grpcClsNode) continue;
|
||||
const grpcMatch = GRPC_SUFFIX_RE.exec(grpcClsNode.text);
|
||||
if (!grpcMatch) continue;
|
||||
const serviceName = grpcMatch[1];
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName,
|
||||
symbolName: `${serviceName}Stub`,
|
||||
source: 'java_stub',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
314
gitnexus/src/core/group/extractors/grpc-patterns/node.ts
Normal file
314
gitnexus/src/core/group/extractors/grpc-patterns/node.ts
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
import Parser from 'tree-sitter';
|
||||
import JavaScript from 'tree-sitter-javascript';
|
||||
import TypeScript from 'tree-sitter-typescript';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
unquoteLiteral,
|
||||
type CompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
type PatternSpec,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Node.js / TypeScript gRPC plugin family. Detects:
|
||||
* - Provider: NestJS `@GrpcMethod('Service', 'Method')` decorators
|
||||
* - Consumer: NestJS `@GrpcClient(...) readonly x!: XxxServiceClient`
|
||||
* - Consumer: `client.getService<X>('AuthService')`
|
||||
* - Consumer: `new XxxServiceClient(...)` (generated client constructor)
|
||||
* - Consumer: `new foo.bar.Xxx(...)` when the file uses
|
||||
* `loadPackageDefinition` (gRPC dynamic proto loader)
|
||||
*
|
||||
* As with the HTTP `node.ts`, pattern sources are defined once and
|
||||
* compiled against three grammar variants (JS / TS / TSX) because
|
||||
* `Parser.Query` is not portable across grammar objects.
|
||||
*/
|
||||
|
||||
const SERVICE_CLIENT_RE = /^(\w+Service)Client$/;
|
||||
const CAPITALIZED_SERVICE_RE = /^[A-Z]\w+$/;
|
||||
|
||||
// @GrpcMethod('Service', 'Method')
|
||||
const GRPC_METHOD_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(decorator
|
||||
(call_expression
|
||||
function: (identifier) @dec (#eq? @dec "GrpcMethod")
|
||||
arguments: (arguments
|
||||
. [(string) (template_string)] @service
|
||||
. [(string) (template_string)] @method)))
|
||||
`,
|
||||
};
|
||||
|
||||
// @GrpcClient(...) standalone decorator — the plugin walks to the next
|
||||
// sibling (a field definition) to read its type annotation.
|
||||
const GRPC_CLIENT_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(decorator
|
||||
(call_expression
|
||||
function: (identifier) @dec (#eq? @dec "GrpcClient"))) @grpc_client_decorator
|
||||
`,
|
||||
};
|
||||
|
||||
// `.getService<X>('AuthService')` / `.getService('AuthService')`
|
||||
const GET_SERVICE_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
property: (property_identifier) @method (#eq? @method "getService"))
|
||||
arguments: (arguments . [(string) (template_string)] @service))
|
||||
`,
|
||||
};
|
||||
|
||||
// `new XxxServiceClient(...)` — bare identifier constructor.
|
||||
const NEW_SIMPLE_CTOR_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(new_expression
|
||||
constructor: (identifier) @ctor)
|
||||
`,
|
||||
};
|
||||
|
||||
// `new foo.bar.XxxService(...)` — qualified constructor.
|
||||
const NEW_QUALIFIED_CTOR_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(new_expression
|
||||
constructor: (member_expression
|
||||
property: (property_identifier) @ctor))
|
||||
`,
|
||||
};
|
||||
|
||||
// Detect whether the file uses `loadPackageDefinition` (gRPC dynamic
|
||||
// proto loader). Matches either a bare call or an `obj.loadPackageDefinition(...)`
|
||||
// call. Plugin gates the qualified-constructor consumer on this —
|
||||
// structural check avoids materializing `tree.rootNode.text` for every file.
|
||||
const LOAD_PACKAGE_DEFINITION_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: [
|
||||
(identifier) @fn (#eq? @fn "loadPackageDefinition")
|
||||
(member_expression property: (property_identifier) @fn (#eq? @fn "loadPackageDefinition"))
|
||||
])
|
||||
`,
|
||||
};
|
||||
|
||||
interface NodeGrpcPatternBundle {
|
||||
grpcMethod: CompiledPatterns<Record<string, never>>;
|
||||
grpcClient: CompiledPatterns<Record<string, never>>;
|
||||
getService: CompiledPatterns<Record<string, never>>;
|
||||
newSimpleCtor: CompiledPatterns<Record<string, never>>;
|
||||
newQualifiedCtor: CompiledPatterns<Record<string, never>>;
|
||||
loadPackageDefinition: CompiledPatterns<Record<string, never>>;
|
||||
}
|
||||
|
||||
function compileBundle(language: unknown, name: string): NodeGrpcPatternBundle {
|
||||
const mk = (spec: PatternSpec<Record<string, never>>, suffix: string) =>
|
||||
compilePatterns({
|
||||
name: `${name}-${suffix}`,
|
||||
language,
|
||||
patterns: [spec],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
return {
|
||||
grpcMethod: mk(GRPC_METHOD_SPEC, 'grpc-method'),
|
||||
grpcClient: mk(GRPC_CLIENT_SPEC, 'grpc-client'),
|
||||
getService: mk(GET_SERVICE_SPEC, 'get-service'),
|
||||
newSimpleCtor: mk(NEW_SIMPLE_CTOR_SPEC, 'new-simple-ctor'),
|
||||
newQualifiedCtor: mk(NEW_QUALIFIED_CTOR_SPEC, 'new-qualified-ctor'),
|
||||
loadPackageDefinition: mk(LOAD_PACKAGE_DEFINITION_SPEC, 'load-package-definition'),
|
||||
};
|
||||
}
|
||||
|
||||
const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-grpc');
|
||||
const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-grpc');
|
||||
const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-grpc');
|
||||
|
||||
/**
|
||||
* Given a `@GrpcClient(...)` decorator node, find the type annotation
|
||||
* text of the field it decorates (e.g. `AuthServiceClient`).
|
||||
*
|
||||
* In tree-sitter-typescript, decorators on class fields can appear in
|
||||
* two configurations:
|
||||
* - As a CHILD of `public_field_definition` alongside the field's
|
||||
* type annotation (the common case for NestJS `@GrpcClient`).
|
||||
* - As a SIBLING of the field in `class_body` (for method
|
||||
* decorators, but kept for resilience against grammar variants).
|
||||
* We walk the parent container and search for a type annotation.
|
||||
*/
|
||||
function resolveGrpcClientFieldType(decoratorNode: Parser.SyntaxNode): string | null {
|
||||
const parent = decoratorNode.parent;
|
||||
if (!parent) return null;
|
||||
|
||||
// Case 1: decorator is a child of the field definition — search
|
||||
// the parent itself (which is the field definition) for a
|
||||
// type_annotation child.
|
||||
if (parent.type === 'public_field_definition' || parent.type.endsWith('field_definition')) {
|
||||
return findFirstTypeAnnotationText(parent);
|
||||
}
|
||||
|
||||
// Case 2: decorator is a sibling of the field in a class_body — walk
|
||||
// forward through subsequent siblings until we find a node containing
|
||||
// a type annotation.
|
||||
for (let i = 0; i < parent.namedChildCount; i++) {
|
||||
const child = parent.namedChild(i);
|
||||
if (child && child.id === decoratorNode.id) {
|
||||
for (let j = i + 1; j < parent.namedChildCount; j++) {
|
||||
const next = parent.namedChild(j);
|
||||
if (!next) continue;
|
||||
if (next.type === 'decorator') continue;
|
||||
const typeText = findFirstTypeAnnotationText(next);
|
||||
if (typeText) return typeText;
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively search `node` for the first `type_annotation` child and
|
||||
* return the text of its inner `type_identifier`, or null. Handles
|
||||
* both `public_field_definition` and its variants.
|
||||
*/
|
||||
function findFirstTypeAnnotationText(node: Parser.SyntaxNode): string | null {
|
||||
if (node.type === 'type_annotation') {
|
||||
for (let i = 0; i < node.namedChildCount; i++) {
|
||||
const child = node.namedChild(i);
|
||||
if (!child) continue;
|
||||
if (child.type === 'type_identifier') return child.text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
for (let i = 0; i < node.namedChildCount; i++) {
|
||||
const child = node.namedChild(i);
|
||||
if (!child) continue;
|
||||
const found = findFirstTypeAnnotationText(child);
|
||||
if (found) return found;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function scanBundle(bundle: NodeGrpcPatternBundle, tree: Parser.Tree): GrpcDetection[] {
|
||||
const out: GrpcDetection[] = [];
|
||||
|
||||
// ─── Provider: @GrpcMethod('Service', 'Method') ──────────────────
|
||||
for (const match of runCompiledPatterns(bundle.grpcMethod, tree)) {
|
||||
const svcNode = match.captures.service;
|
||||
const methodNode = match.captures.method;
|
||||
if (!svcNode || !methodNode) continue;
|
||||
const svc = unquoteLiteral(svcNode.text);
|
||||
const mth = unquoteLiteral(methodNode.text);
|
||||
if (!svc || !mth) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName: svc,
|
||||
symbolName: `${svc}.${mth}`,
|
||||
source: 'ts_grpc_method',
|
||||
methodName: mth,
|
||||
// @GrpcMethod hard-coded confidence 0.8 in the original code
|
||||
// regardless of whether the proto map has a match.
|
||||
confidenceWithProto: 0.8,
|
||||
confidenceWithoutProto: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumer: @GrpcClient() field with XxxServiceClient type ────
|
||||
for (const match of runCompiledPatterns(bundle.grpcClient, tree)) {
|
||||
const decoratorNode = match.captures.grpc_client_decorator;
|
||||
if (!decoratorNode) continue;
|
||||
const typeText = resolveGrpcClientFieldType(decoratorNode);
|
||||
if (!typeText) continue;
|
||||
const svcMatch = SERVICE_CLIENT_RE.exec(typeText);
|
||||
if (!svcMatch) continue;
|
||||
const serviceName = svcMatch[1];
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName,
|
||||
symbolName: `${serviceName}Client`,
|
||||
source: 'ts_grpc_client_decorator',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumer: client.getService<X>('Service') ───────────────────
|
||||
for (const match of runCompiledPatterns(bundle.getService, tree)) {
|
||||
const svcNode = match.captures.service;
|
||||
if (!svcNode) continue;
|
||||
const svc = unquoteLiteral(svcNode.text);
|
||||
if (!svc) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName: svc,
|
||||
symbolName: `${svc}Client`,
|
||||
source: 'ts_client_grpc_get_service',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumer: new XxxServiceClient(...) ─────────────────────────
|
||||
for (const match of runCompiledPatterns(bundle.newSimpleCtor, tree)) {
|
||||
const ctorNode = match.captures.ctor;
|
||||
if (!ctorNode) continue;
|
||||
const svcMatch = SERVICE_CLIENT_RE.exec(ctorNode.text);
|
||||
if (!svcMatch) continue;
|
||||
const serviceName = svcMatch[1];
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName,
|
||||
symbolName: `${serviceName}Client`,
|
||||
source: 'ts_generated_client',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumer: loadPackageDefinition dynamic proto loader ────────
|
||||
// Only emit when the file uses loadPackageDefinition, otherwise a
|
||||
// generic `new foo.bar.Something()` in unrelated code would falsely
|
||||
// register as a gRPC consumer. Check structurally via a dedicated
|
||||
// query — avoids materializing `tree.rootNode.text` for the whole
|
||||
// file (expensive on large files).
|
||||
const usesLoadPackage = runCompiledPatterns(bundle.loadPackageDefinition, tree).length > 0;
|
||||
if (usesLoadPackage) {
|
||||
for (const match of runCompiledPatterns(bundle.newQualifiedCtor, tree)) {
|
||||
const ctorNode = match.captures.ctor;
|
||||
if (!ctorNode) continue;
|
||||
if (!CAPITALIZED_SERVICE_RE.test(ctorNode.text)) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName: ctorNode.text,
|
||||
symbolName: `${ctorNode.text}Client`,
|
||||
source: 'ts_load_package_definition',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
export const JAVASCRIPT_GRPC_PLUGIN: GrpcLanguagePlugin = {
|
||||
name: 'javascript-grpc',
|
||||
language: JavaScript,
|
||||
scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree),
|
||||
};
|
||||
|
||||
export const TYPESCRIPT_GRPC_PLUGIN: GrpcLanguagePlugin = {
|
||||
name: 'typescript-grpc',
|
||||
language: TypeScript.typescript,
|
||||
scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree),
|
||||
};
|
||||
|
||||
export const TSX_GRPC_PLUGIN: GrpcLanguagePlugin = {
|
||||
name: 'tsx-grpc',
|
||||
language: TypeScript.tsx,
|
||||
scan: (tree) => scanBundle(TSX_BUNDLE, tree),
|
||||
};
|
||||
147
gitnexus/src/core/group/extractors/grpc-patterns/proto.ts
Normal file
147
gitnexus/src/core/group/extractors/grpc-patterns/proto.ts
Normal file
|
|
@ -0,0 +1,147 @@
|
|||
import { createRequire } from 'node:module';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
type CompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Protobuf (.proto) tree-sitter plugin for gRPC contract extraction.
|
||||
*
|
||||
* Uses `tree-sitter-proto` (coder3101/tree-sitter-proto) as an
|
||||
* optionalDependency — if the grammar is not installed (e.g. native
|
||||
* compilation failed on an unusual platform), the plugin exports
|
||||
* `null` and the orchestrator falls back to the existing manual
|
||||
* string-sanitizing parser.
|
||||
*
|
||||
* The grammar is vendored in `vendor/tree-sitter-proto/` with
|
||||
* parser.c regenerated against tree-sitter-cli 0.24 (ABI version 14)
|
||||
* so it is compatible with the project's tree-sitter 0.25 runtime.
|
||||
*/
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
let ProtoGrammar: unknown = null;
|
||||
try {
|
||||
ProtoGrammar = _require('tree-sitter-proto');
|
||||
} catch {
|
||||
// Grammar not installed — PROTO_GRPC_PLUGIN will be null.
|
||||
}
|
||||
|
||||
let PACKAGE_PATTERNS: CompiledPatterns<Record<string, never>> | null = null;
|
||||
let SERVICE_PATTERNS: CompiledPatterns<Record<string, never>> | null = null;
|
||||
|
||||
if (ProtoGrammar) {
|
||||
try {
|
||||
// Validate that the grammar actually loads end-to-end: compile queries
|
||||
// AND parse + walk a trivial proto file. tree-sitter's internal
|
||||
// `initializeLanguageNodeClasses` can fail with a TDZ error in some
|
||||
// test runners (vitest forks) when SyntaxNode isn't fully initialized
|
||||
// yet. Catching that here ensures `PROTO_GRPC_PLUGIN` stays null and
|
||||
// the orchestrator falls back to the manual parser.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const _Parser = _require('tree-sitter') as any;
|
||||
// Smoke-test: parse + setLanguage to verify the grammar is
|
||||
// end-to-end compatible with this tree-sitter runtime.
|
||||
const _testParser = new _Parser();
|
||||
_testParser.setLanguage(ProtoGrammar);
|
||||
_testParser.parse('service X { rpc Y (R) returns (R); }');
|
||||
|
||||
PACKAGE_PATTERNS = compilePatterns({
|
||||
name: 'proto-package',
|
||||
language: ProtoGrammar,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `(package (full_ident) @pkg)`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
SERVICE_PATTERNS = compilePatterns({
|
||||
name: 'proto-service',
|
||||
language: ProtoGrammar,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(service
|
||||
(service_name) @service_name
|
||||
(rpc
|
||||
(rpc_name) @rpc_name))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
} catch {
|
||||
// Compilation failed (grammar ABI mismatch?) — fall back to null.
|
||||
PACKAGE_PATTERNS = null;
|
||||
SERVICE_PATTERNS = null;
|
||||
ProtoGrammar = null;
|
||||
}
|
||||
}
|
||||
|
||||
function buildPlugin(): GrpcLanguagePlugin | null {
|
||||
if (!ProtoGrammar || !PACKAGE_PATTERNS || !SERVICE_PATTERNS) return null;
|
||||
const pkgPatterns = PACKAGE_PATTERNS;
|
||||
const svcPatterns = SERVICE_PATTERNS;
|
||||
|
||||
return {
|
||||
name: 'proto-grpc',
|
||||
language: ProtoGrammar,
|
||||
scan(tree) {
|
||||
const out: GrpcDetection[] = [];
|
||||
|
||||
// Extract `package` declaration (first match wins).
|
||||
let pkg = '';
|
||||
for (const match of runCompiledPatterns(pkgPatterns, tree)) {
|
||||
const pkgNode = match.captures.pkg;
|
||||
if (pkgNode) {
|
||||
pkg = pkgNode.text;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract `service → rpc` pairs. The query returns one match per
|
||||
// (service, rpc) combination thanks to the nested structure.
|
||||
for (const match of runCompiledPatterns(svcPatterns, tree)) {
|
||||
const serviceNode = match.captures.service_name;
|
||||
const rpcNode = match.captures.rpc_name;
|
||||
if (!serviceNode || !rpcNode) continue;
|
||||
const serviceName = serviceNode.text;
|
||||
const methodName = rpcNode.text;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName,
|
||||
symbolName: `${serviceName}.${methodName}`,
|
||||
source: 'proto',
|
||||
methodName,
|
||||
// Proto definitions are the canonical source of truth — always
|
||||
// high confidence regardless of cross-referencing.
|
||||
confidenceWithProto: 0.85,
|
||||
confidenceWithoutProto: 0.85,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The proto plugin, or `null` if tree-sitter-proto is not available.
|
||||
* The orchestrator checks this at import time and decides whether to
|
||||
* use the tree-sitter path or the fallback manual parser.
|
||||
*/
|
||||
export const PROTO_GRPC_PLUGIN: GrpcLanguagePlugin | null = buildPlugin();
|
||||
|
||||
/** The package declaration text from a proto file's tree. */
|
||||
export function extractPackageFromTree(tree: import('tree-sitter').Tree): string {
|
||||
if (!PACKAGE_PATTERNS) return '';
|
||||
for (const match of runCompiledPatterns(PACKAGE_PATTERNS, tree)) {
|
||||
const pkgNode = match.captures.pkg;
|
||||
if (pkgNode) return pkgNode.text;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
77
gitnexus/src/core/group/extractors/grpc-patterns/python.ts
Normal file
77
gitnexus/src/core/group/extractors/grpc-patterns/python.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import Python from 'tree-sitter-python';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Python gRPC plugin. Detects:
|
||||
* - Provider: `add_XxxServicer_to_server(...)` calls (bare identifier
|
||||
* or qualified attribute form `auth_pb2_grpc.add_XxxServicer_to_server`)
|
||||
* - Consumer: `XxxStub(channel)` calls (bare or `auth_pb2_grpc.XxxStub`)
|
||||
*/
|
||||
|
||||
const ADD_SERVICER_RE = /^add_(\w+)Servicer_to_server$/;
|
||||
const STUB_RE = /^(\w+)Stub$/;
|
||||
/** Reserved names that would produce garbage service names. */
|
||||
const STUB_IGNORE = new Set(['Mock', 'Test', 'Fake', 'Stub']);
|
||||
|
||||
// Any call whose target is either a bare identifier or an attribute
|
||||
// access (`obj.method`). The plugin filters the function name in JS.
|
||||
const CALL_PATTERNS = compilePatterns({
|
||||
name: 'python-grpc-call',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call
|
||||
function: [
|
||||
(identifier) @fn
|
||||
(attribute attribute: (identifier) @fn)
|
||||
])
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
export const PYTHON_GRPC_PLUGIN: GrpcLanguagePlugin = {
|
||||
name: 'python-grpc',
|
||||
language: Python,
|
||||
scan(tree) {
|
||||
const out: GrpcDetection[] = [];
|
||||
for (const match of runCompiledPatterns(CALL_PATTERNS, tree)) {
|
||||
const fnNode = match.captures.fn;
|
||||
if (!fnNode) continue;
|
||||
const fnText = fnNode.text;
|
||||
|
||||
const addServicer = ADD_SERVICER_RE.exec(fnText);
|
||||
if (addServicer) {
|
||||
out.push({
|
||||
role: 'provider',
|
||||
serviceName: addServicer[1],
|
||||
symbolName: fnText,
|
||||
source: 'python_servicer',
|
||||
confidenceWithProto: 0.8,
|
||||
confidenceWithoutProto: 0.65,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const stubMatch = STUB_RE.exec(fnText);
|
||||
if (stubMatch && !STUB_IGNORE.has(stubMatch[1])) {
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
serviceName: stubMatch[1],
|
||||
symbolName: fnText,
|
||||
source: 'python_stub',
|
||||
confidenceWithProto: 0.75,
|
||||
confidenceWithoutProto: 0.55,
|
||||
});
|
||||
}
|
||||
}
|
||||
return out;
|
||||
},
|
||||
};
|
||||
54
gitnexus/src/core/group/extractors/grpc-patterns/types.ts
Normal file
54
gitnexus/src/core/group/extractors/grpc-patterns/types.ts
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
import type Parser from 'tree-sitter';
|
||||
|
||||
/**
|
||||
* Shared types for the grpc-extractor language plugins.
|
||||
*
|
||||
* Each plugin lives in its own file (java.ts, go.ts, ...) and owns the
|
||||
* tree-sitter grammar import + query sources. The top-level
|
||||
* `grpc-extractor.ts` orchestrator only knows about this type module
|
||||
* and the plugin registry (`./index.ts`). It MUST NOT import any
|
||||
* grammar or query text directly.
|
||||
*/
|
||||
|
||||
export type GrpcRole = 'provider' | 'consumer';
|
||||
|
||||
/**
|
||||
* One raw gRPC detection produced by a plugin's `scan()` function. The
|
||||
* orchestrator uses the proto map to resolve the full package-qualified
|
||||
* contract id and choose a confidence based on whether the proto was
|
||||
* found.
|
||||
*
|
||||
* Most patterns produce service-level detections; `TS @GrpcMethod` is
|
||||
* the only pattern that captures an explicit `methodName`, producing
|
||||
* a method-level contract (`grpc::pkg.Service/Method`).
|
||||
*/
|
||||
export interface GrpcDetection {
|
||||
role: GrpcRole;
|
||||
/** Short service name, e.g. `"AuthService"`. */
|
||||
serviceName: string;
|
||||
/** Symbol name emitted into the contract's symbolRef. */
|
||||
symbolName: string;
|
||||
/** Metadata source label (goes into `meta.source`). */
|
||||
source: string;
|
||||
/** Explicit method name; set only by TS `@GrpcMethod`. */
|
||||
methodName?: string;
|
||||
/** Confidence when the proto map resolves the service. */
|
||||
confidenceWithProto: number;
|
||||
/** Confidence when the proto map has no entry. */
|
||||
confidenceWithoutProto: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* One language-scoped gRPC plugin. Plugins own the tree-sitter grammar
|
||||
* and a `scan(tree)` function that returns zero or more
|
||||
* `GrpcDetection`s. The plugin is free to run multiple compiled query
|
||||
* bundles and walk the AST to cross-reference captures.
|
||||
*
|
||||
* `language` is typed `unknown` for the same reason as in
|
||||
* `tree-sitter-scanner.ts`.
|
||||
*/
|
||||
export interface GrpcLanguagePlugin {
|
||||
name: string;
|
||||
language: unknown;
|
||||
scan(tree: Parser.Tree): GrpcDetection[];
|
||||
}
|
||||
224
gitnexus/src/core/group/extractors/http-patterns/go.ts
Normal file
224
gitnexus/src/core/group/extractors/http-patterns/go.ts
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
import Go from 'tree-sitter-go';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
unquoteLiteral,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Go HTTP plugin. Handles:
|
||||
* - gin / echo / chi framework routing — `r.GET("/path", handler)`
|
||||
* - net/http stdlib — `http.HandleFunc("/path", handler)`
|
||||
* - net/http consumer — `http.Get(...)`, `http.NewRequest("METHOD", ...)`
|
||||
* - resty consumer — `client.R().Delete("/path")`
|
||||
*/
|
||||
|
||||
// ─── Provider: framework routing ──────────────────────────────────────
|
||||
// Matches `\w+\.GET(...)` etc. (gin, echo, chi all share this shape).
|
||||
// Captures the HTTP method (field name), path literal, and handler
|
||||
// identifier passed as the second argument.
|
||||
const FRAMEWORK_ROUTE_PATTERNS = compilePatterns({
|
||||
name: 'go-framework-route',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
field: (field_identifier) @http_method (#match? @http_method "^(GET|POST|PUT|DELETE|PATCH)$"))
|
||||
arguments: (argument_list
|
||||
(interpreted_string_literal) @path
|
||||
(identifier) @handler))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Provider: net/http `http.HandleFunc("/p", handler)` ─────────────
|
||||
const HANDLE_FUNC_PATTERNS = compilePatterns({
|
||||
name: 'go-handle-func',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
operand: (identifier) @pkg (#eq? @pkg "http")
|
||||
field: (field_identifier) @fn (#eq? @fn "HandleFunc"))
|
||||
arguments: (argument_list
|
||||
(interpreted_string_literal) @path
|
||||
(identifier) @handler))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: net/http stdlib Get / Post / Head ─────────────────────
|
||||
const HTTP_CLIENT_METHOD_TO_HTTP: Record<string, string> = {
|
||||
Get: 'GET',
|
||||
Post: 'POST',
|
||||
Head: 'GET', // HEAD has no body semantics we care about — treat as GET for contract matching
|
||||
};
|
||||
|
||||
const HTTP_CLIENT_PATTERNS = compilePatterns({
|
||||
name: 'go-http-client',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
operand: (identifier) @pkg (#eq? @pkg "http")
|
||||
field: (field_identifier) @fn (#match? @fn "^(Get|Post|Head)$"))
|
||||
arguments: (argument_list . (interpreted_string_literal) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: net/http `http.NewRequest("METHOD", "/path", ...)` ────
|
||||
const NEW_REQUEST_PATTERNS = compilePatterns({
|
||||
name: 'go-new-request',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
operand: (identifier) @pkg (#eq? @pkg "http")
|
||||
field: (field_identifier) @fn (#eq? @fn "NewRequest"))
|
||||
arguments: (argument_list
|
||||
.
|
||||
(interpreted_string_literal) @http_method
|
||||
(interpreted_string_literal) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: resty `client.R().Delete("/path")` ─────────────────────
|
||||
// Matches any chained call whose receiver is `something.R()` and whose
|
||||
// method name is an HTTP verb. This is how go-resty's fluent API looks.
|
||||
const RESTY_PATTERNS = compilePatterns({
|
||||
name: 'go-resty',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
operand: (call_expression
|
||||
function: (selector_expression
|
||||
field: (field_identifier) @r (#eq? @r "R")))
|
||||
field: (field_identifier) @http_method (#match? @http_method "^(Get|Post|Put|Delete|Patch)$"))
|
||||
arguments: (argument_list . (interpreted_string_literal) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
export const GO_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'go-http',
|
||||
language: Go,
|
||||
scan(tree) {
|
||||
const out: HttpDetection[] = [];
|
||||
|
||||
// Framework providers: r.GET/POST/... with handler identifier
|
||||
for (const match of runCompiledPatterns(FRAMEWORK_ROUTE_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
const handlerNode = match.captures.handler;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'go-framework',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: handlerNode?.text ?? null,
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// net/http HandleFunc: default method GET
|
||||
for (const match of runCompiledPatterns(HANDLE_FUNC_PATTERNS, tree)) {
|
||||
const pathNode = match.captures.path;
|
||||
const handlerNode = match.captures.handler;
|
||||
if (!pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'go-stdlib',
|
||||
method: 'GET',
|
||||
path,
|
||||
name: handlerNode?.text ?? null,
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// net/http client: http.Get/Post/Head
|
||||
for (const match of runCompiledPatterns(HTTP_CLIENT_PATTERNS, tree)) {
|
||||
const fnNode = match.captures.fn;
|
||||
const pathNode = match.captures.path;
|
||||
if (!fnNode || !pathNode) continue;
|
||||
const httpMethod = HTTP_CLIENT_METHOD_TO_HTTP[fnNode.text];
|
||||
if (!httpMethod) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'go-stdlib',
|
||||
method: httpMethod,
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// net/http NewRequest
|
||||
for (const match of runCompiledPatterns(NEW_REQUEST_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const method = unquoteLiteral(methodNode.text);
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (method === null || path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'go-stdlib',
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// resty
|
||||
for (const match of runCompiledPatterns(RESTY_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'go-resty',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
50
gitnexus/src/core/group/extractors/http-patterns/index.ts
Normal file
50
gitnexus/src/core/group/extractors/http-patterns/index.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import * as path from 'node:path';
|
||||
import type { HttpLanguagePlugin } from './types.js';
|
||||
import { JAVA_HTTP_PLUGIN } from './java.js';
|
||||
import { GO_HTTP_PLUGIN } from './go.js';
|
||||
import { PYTHON_HTTP_PLUGIN } from './python.js';
|
||||
import { PHP_HTTP_PLUGIN } from './php.js';
|
||||
import { JAVASCRIPT_HTTP_PLUGIN, TYPESCRIPT_HTTP_PLUGIN, TSX_HTTP_PLUGIN } from './node.js';
|
||||
|
||||
export type { HttpDetection, HttpLanguagePlugin, HttpRole } from './types.js';
|
||||
|
||||
/**
|
||||
* File-extension → HTTP language plugin registry. The top-level
|
||||
* orchestrator (`http-route-extractor.ts`) looks up the plugin for each
|
||||
* file it visits and delegates the tree-sitter scanning to the plugin.
|
||||
*
|
||||
* Keys are lowercase extensions including the leading dot. To add a
|
||||
* new language, drop a `http-patterns/<lang>.ts` that exports a
|
||||
* `HttpLanguagePlugin`, import it here and register the extension(s).
|
||||
* No edits to `http-route-extractor.ts` are required.
|
||||
*/
|
||||
const REGISTRY: Record<string, HttpLanguagePlugin> = {
|
||||
'.java': JAVA_HTTP_PLUGIN,
|
||||
'.go': GO_HTTP_PLUGIN,
|
||||
'.py': PYTHON_HTTP_PLUGIN,
|
||||
'.php': PHP_HTTP_PLUGIN,
|
||||
'.js': JAVASCRIPT_HTTP_PLUGIN,
|
||||
'.jsx': JAVASCRIPT_HTTP_PLUGIN,
|
||||
'.ts': TYPESCRIPT_HTTP_PLUGIN,
|
||||
'.tsx': TSX_HTTP_PLUGIN,
|
||||
};
|
||||
|
||||
/**
|
||||
* Glob for files worth scanning for HTTP routes. Kept alongside the
|
||||
* registry so adding a new language widens the glob in one edit.
|
||||
*
|
||||
* `.vue` / `.svelte` files are intentionally omitted for the source-scan
|
||||
* path — they need their own grammar-aware extraction and the existing
|
||||
* regex fallback for them was never very accurate. The graph-assisted
|
||||
* Strategy A still handles them via the ingestion pipeline.
|
||||
*/
|
||||
export const HTTP_SCAN_GLOB = '**/*.{ts,tsx,js,jsx,java,go,py,php}';
|
||||
|
||||
/**
|
||||
* Return the HTTP plugin registered for the given file's extension,
|
||||
* or `undefined` if the extension is not registered.
|
||||
*/
|
||||
export function getPluginForFile(rel: string): HttpLanguagePlugin | undefined {
|
||||
const ext = path.extname(rel).toLowerCase();
|
||||
return REGISTRY[ext];
|
||||
}
|
||||
267
gitnexus/src/core/group/extractors/http-patterns/java.ts
Normal file
267
gitnexus/src/core/group/extractors/http-patterns/java.ts
Normal file
|
|
@ -0,0 +1,267 @@
|
|||
import Parser from 'tree-sitter';
|
||||
import Java from 'tree-sitter-java';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
unquoteLiteral,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Java HTTP plugin. Handles:
|
||||
* - Spring `@RequestMapping` class prefixes + `@(Get|Post|...)Mapping` method annotations
|
||||
* - Spring `RestTemplate.getForObject/...`, `WebClient.method(HttpMethod.X, ...)`
|
||||
* - OkHttp `new Request.Builder().url("...")`
|
||||
*
|
||||
* The plugin runs two pattern bundles: one to collect class-level
|
||||
* `@RequestMapping` prefixes keyed by the enclosing class node, and a
|
||||
* second to match method-level annotations. The `scan` function walks
|
||||
* up from each matched annotation to find its enclosing class and
|
||||
* combines the prefix with the method path.
|
||||
*/
|
||||
|
||||
const METHOD_ANNOTATION_TO_HTTP: Record<string, string> = {
|
||||
GetMapping: 'GET',
|
||||
PostMapping: 'POST',
|
||||
PutMapping: 'PUT',
|
||||
DeleteMapping: 'DELETE',
|
||||
PatchMapping: 'PATCH',
|
||||
};
|
||||
|
||||
// ─── Provider: Spring class-level @RequestMapping prefix ──────────────
|
||||
const SPRING_CLASS_PREFIX_PATTERNS = compilePatterns({
|
||||
name: 'java-spring-class-prefix',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(class_declaration
|
||||
(modifiers
|
||||
(annotation
|
||||
name: (identifier) @ann (#eq? @ann "RequestMapping")
|
||||
arguments: (annotation_argument_list (string_literal) @prefix)))) @class
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Provider: Spring @(Get|Post|...)Mapping method annotations ───────
|
||||
const SPRING_METHOD_ROUTE_PATTERNS = compilePatterns({
|
||||
name: 'java-spring-method-route',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(method_declaration
|
||||
(modifiers
|
||||
(annotation
|
||||
name: (identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")
|
||||
arguments: (annotation_argument_list (string_literal) @path)))
|
||||
name: (identifier) @method_name) @method
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: Spring RestTemplate (object-named + method-named) ──────
|
||||
// RestTemplate.getForObject / getForEntity → GET
|
||||
// RestTemplate.postForObject / postForEntity → POST
|
||||
// RestTemplate.put → PUT
|
||||
// RestTemplate.delete → DELETE
|
||||
// RestTemplate.patchForObject → PATCH
|
||||
const REST_TEMPLATE_TO_HTTP: Record<string, string> = {
|
||||
getForObject: 'GET',
|
||||
getForEntity: 'GET',
|
||||
postForObject: 'POST',
|
||||
postForEntity: 'POST',
|
||||
put: 'PUT',
|
||||
delete: 'DELETE',
|
||||
patchForObject: 'PATCH',
|
||||
};
|
||||
|
||||
interface RestTemplateMeta {
|
||||
framework: 'spring-rest-template';
|
||||
}
|
||||
|
||||
const REST_TEMPLATE_PATTERNS = compilePatterns({
|
||||
name: 'java-rest-template',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: { framework: 'spring-rest-template' },
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (identifier) @obj (#eq? @obj "restTemplate")
|
||||
name: (identifier) @method
|
||||
arguments: (argument_list . (string_literal) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<RestTemplateMeta>);
|
||||
|
||||
// ─── Consumer: Spring WebClient — webClient.method(HttpMethod.X, "path") ─
|
||||
const WEB_CLIENT_PATTERNS = compilePatterns({
|
||||
name: 'java-web-client',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (identifier) @obj (#eq? @obj "webClient")
|
||||
name: (identifier) @method (#eq? @method "method")
|
||||
arguments: (argument_list
|
||||
(field_access
|
||||
object: (identifier) @httpMethodCls (#eq? @httpMethodCls "HttpMethod")
|
||||
field: (identifier) @http_method)
|
||||
(string_literal) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: OkHttp `new Request.Builder().url("path")` ─────────────
|
||||
// Note: `Request.Builder` is a `scoped_type_identifier` whose text includes
|
||||
// the dot, so `#eq?` against the literal string matches cleanly (no need
|
||||
// to escape a regex dot).
|
||||
const OK_HTTP_PATTERNS = compilePatterns({
|
||||
name: 'java-okhttp',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (object_creation_expression
|
||||
type: (scoped_type_identifier) @type (#eq? @type "Request.Builder"))
|
||||
name: (identifier) @method (#eq? @method "url")
|
||||
arguments: (argument_list . (string_literal) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
/**
|
||||
* Find the nearest enclosing class_declaration ancestor for a node, or
|
||||
* null if the node is top-level. Tree-sitter's SyntaxNode.parent walks
|
||||
* one level at a time.
|
||||
*/
|
||||
function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
|
||||
let cur: Parser.SyntaxNode | null = node.parent;
|
||||
while (cur) {
|
||||
if (cur.type === 'class_declaration') return cur;
|
||||
cur = cur.parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Join a class-level prefix and a method-level path into a single URL
|
||||
* path. Mirrors the semantics of the original regex implementation:
|
||||
* strip trailing slashes on the prefix, then ensure a single slash
|
||||
* between prefix and method path.
|
||||
*/
|
||||
function joinPath(prefix: string, methodPath: string): string {
|
||||
const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, '');
|
||||
const cleanSub = methodPath.replace(/^\/+/, '');
|
||||
if (!cleanPrefix) return `/${cleanSub}`;
|
||||
return `/${cleanPrefix}/${cleanSub}`;
|
||||
}
|
||||
|
||||
export const JAVA_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'java-http',
|
||||
language: Java,
|
||||
scan(tree) {
|
||||
const out: HttpDetection[] = [];
|
||||
|
||||
// ─── Providers: Spring class prefix + method annotations ────────
|
||||
const prefixByClassId = new Map<number, string>();
|
||||
for (const match of runCompiledPatterns(SPRING_CLASS_PREFIX_PATTERNS, tree)) {
|
||||
const prefixNode = match.captures.prefix;
|
||||
const classNode = match.captures.class;
|
||||
if (!prefixNode || !classNode) continue;
|
||||
const prefix = unquoteLiteral(prefixNode.text);
|
||||
if (prefix !== null) prefixByClassId.set(classNode.id, prefix);
|
||||
}
|
||||
|
||||
for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) {
|
||||
const annNode = match.captures.ann;
|
||||
const pathNode = match.captures.path;
|
||||
const nameNode = match.captures.method_name;
|
||||
const methodNode = match.captures.method;
|
||||
if (!annNode || !pathNode || !methodNode) continue;
|
||||
const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text];
|
||||
if (!httpMethod) continue;
|
||||
const rawPath = unquoteLiteral(pathNode.text);
|
||||
if (rawPath === null) continue;
|
||||
const enclosingClass = findEnclosingClass(methodNode);
|
||||
const prefix = enclosingClass ? (prefixByClassId.get(enclosingClass.id) ?? '') : '';
|
||||
const fullPath = joinPath(prefix, rawPath);
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'spring',
|
||||
method: httpMethod,
|
||||
path: fullPath,
|
||||
name: nameNode?.text ?? null,
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumers: RestTemplate ────────────────────────────────────
|
||||
for (const match of runCompiledPatterns(REST_TEMPLATE_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const httpMethod = REST_TEMPLATE_TO_HTTP[methodNode.text];
|
||||
if (!httpMethod) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'spring-rest-template',
|
||||
method: httpMethod,
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumers: WebClient.method(HttpMethod.X, "path") ──────────
|
||||
for (const match of runCompiledPatterns(WEB_CLIENT_PATTERNS, tree)) {
|
||||
const httpMethodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!httpMethodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'spring-web-client',
|
||||
method: httpMethodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Consumers: OkHttp Request.Builder().url("path") ────────────
|
||||
for (const match of runCompiledPatterns(OK_HTTP_PATTERNS, tree)) {
|
||||
const pathNode = match.captures.path;
|
||||
if (!pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'okhttp',
|
||||
method: 'GET',
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
502
gitnexus/src/core/group/extractors/http-patterns/node.ts
Normal file
502
gitnexus/src/core/group/extractors/http-patterns/node.ts
Normal file
|
|
@ -0,0 +1,502 @@
|
|||
import Parser from 'tree-sitter';
|
||||
import JavaScript from 'tree-sitter-javascript';
|
||||
import TypeScript from 'tree-sitter-typescript';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
unquoteLiteral,
|
||||
type CompiledPatterns,
|
||||
type LanguagePatterns,
|
||||
type PatternSpec,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Node.js / TypeScript HTTP plugin family. Handles:
|
||||
* - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods
|
||||
* - Express `router.get(...)` / `app.post(...)` providers
|
||||
* - `fetch(url)` / `fetch(url, { method: 'POST' })` consumers
|
||||
* - `axios.get(url)` / `axios.delete(url)` consumers
|
||||
* - `axios({ method, url })` object-form consumers
|
||||
* - jQuery `$.get(url)` / `$.post(url, ...)` shorthand consumers
|
||||
* - jQuery `$.ajax({ url, method | type })` consumers
|
||||
*
|
||||
* Because the JavaScript and TypeScript tree-sitter grammars share
|
||||
* node type names for every construct we query, pattern sources are
|
||||
* defined once and compiled against each grammar variant. The plugin
|
||||
* exports three `HttpLanguagePlugin`s (JS, TS, TSX) that share the
|
||||
* same `scan` function but bind to different grammars.
|
||||
*/
|
||||
|
||||
// ─── Provider: NestJS — class-level @Controller('prefix') ────────────
|
||||
// In tree-sitter-typescript decorators are NOT children of
|
||||
// class_declaration / method_definition — they're siblings in the
|
||||
// surrounding class_body / program node. We therefore match the
|
||||
// decorator standalone and walk to its related class/method in JS.
|
||||
const NEST_CONTROLLER_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(decorator
|
||||
(call_expression
|
||||
function: (identifier) @dec (#eq? @dec "Controller")
|
||||
arguments: (arguments . [(string) (template_string)] @prefix))) @ctrl_decorator
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Provider: NestJS — method-level @Get/@Post/... decorators ───────
|
||||
// Matches either `@Get('path')` or `@Get()`. The `@path` capture is
|
||||
// optional — when the first argument isn't a string, the plugin falls
|
||||
// back to '/' for the method-level path.
|
||||
const NEST_METHOD_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(decorator
|
||||
(call_expression
|
||||
function: (identifier) @dec (#match? @dec "^(Get|Post|Put|Delete|Patch)$")
|
||||
arguments: (arguments) @args)) @method_decorator
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Provider: Express — router.get/app.post/... ─────────────────────
|
||||
const EXPRESS_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#match? @obj "^(router|app)$")
|
||||
property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$"))
|
||||
arguments: (arguments . [(string) (template_string)] @path))
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Consumer: fetch(url) with NO options ─────────────────────────────
|
||||
const FETCH_NO_OPTIONS_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (identifier) @fn (#eq? @fn "fetch")
|
||||
arguments: (arguments . [(string) (template_string)] @path .))
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Consumer: fetch(url, { method: 'X', ... }) ──────────────────────
|
||||
const FETCH_WITH_OPTIONS_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (identifier) @fn (#eq? @fn "fetch")
|
||||
arguments: (arguments
|
||||
. [(string) (template_string)] @path
|
||||
(object
|
||||
(pair
|
||||
key: (property_identifier) @key (#eq? @key "method")
|
||||
value: (string) @http_method))))
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Consumer: axios.get/post/... ────────────────────────────────────
|
||||
const AXIOS_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "axios")
|
||||
property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$"))
|
||||
arguments: (arguments . [(string) (template_string)] @path))
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Consumer: jQuery shorthand $.get(url) / $.post(url, ...) ────────
|
||||
// `$` is a valid JS identifier, so tree-sitter parses `$.get(...)` as a
|
||||
// call_expression whose function is a member_expression on identifier `$`.
|
||||
const JQUERY_SHORTHAND_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "$")
|
||||
property: (property_identifier) @http_method (#match? @http_method "^(get|post)$"))
|
||||
arguments: (arguments . [(string) (template_string)] @path))
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Consumer: jQuery $.ajax({ url, method|type }) ───────────────────
|
||||
// The query captures the options object only; key/value pairs are read
|
||||
// programmatically via `readStringProp` below, which tolerates any key
|
||||
// order and accepts either `method:` or `type:` (jQuery supports both).
|
||||
const JQUERY_AJAX_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "$")
|
||||
property: (property_identifier) @fn (#eq? @fn "ajax"))
|
||||
arguments: (arguments (object) @options))
|
||||
`,
|
||||
};
|
||||
|
||||
// ─── Consumer: axios({ method, url }) object form ────────────────────
|
||||
// Distinct from AXIOS_SPEC above because the call target is an identifier
|
||||
// (`axios`) rather than a member expression (`axios.get`). As with the
|
||||
// jQuery ajax form, option keys are resolved programmatically.
|
||||
const AXIOS_OBJECT_SPEC: PatternSpec<Record<string, never>> = {
|
||||
meta: {},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (identifier) @fn (#eq? @fn "axios")
|
||||
arguments: (arguments (object) @options))
|
||||
`,
|
||||
};
|
||||
|
||||
interface NodePatternBundle {
|
||||
controller: CompiledPatterns<Record<string, never>>;
|
||||
methodDecorator: CompiledPatterns<Record<string, never>>;
|
||||
express: CompiledPatterns<Record<string, never>>;
|
||||
fetchNoOptions: CompiledPatterns<Record<string, never>>;
|
||||
fetchWithOptions: CompiledPatterns<Record<string, never>>;
|
||||
axios: CompiledPatterns<Record<string, never>>;
|
||||
jqueryShorthand: CompiledPatterns<Record<string, never>>;
|
||||
jqueryAjax: CompiledPatterns<Record<string, never>>;
|
||||
axiosObject: CompiledPatterns<Record<string, never>>;
|
||||
}
|
||||
|
||||
function compileBundle(language: unknown, name: string): NodePatternBundle {
|
||||
const mk = (spec: PatternSpec<Record<string, never>>, suffix: string) =>
|
||||
compilePatterns({
|
||||
name: `${name}-${suffix}`,
|
||||
language,
|
||||
patterns: [spec],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
return {
|
||||
controller: mk(NEST_CONTROLLER_SPEC, 'nest-controller'),
|
||||
methodDecorator: mk(NEST_METHOD_SPEC, 'nest-method-decorator'),
|
||||
express: mk(EXPRESS_SPEC, 'express'),
|
||||
fetchNoOptions: mk(FETCH_NO_OPTIONS_SPEC, 'fetch-no-options'),
|
||||
fetchWithOptions: mk(FETCH_WITH_OPTIONS_SPEC, 'fetch-with-options'),
|
||||
axios: mk(AXIOS_SPEC, 'axios'),
|
||||
jqueryShorthand: mk(JQUERY_SHORTHAND_SPEC, 'jquery-shorthand'),
|
||||
jqueryAjax: mk(JQUERY_AJAX_SPEC, 'jquery-ajax'),
|
||||
axiosObject: mk(AXIOS_OBJECT_SPEC, 'axios-object'),
|
||||
};
|
||||
}
|
||||
|
||||
const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-http');
|
||||
const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-http');
|
||||
const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-http');
|
||||
|
||||
const NEST_DECORATOR_TO_HTTP: Record<string, string> = {
|
||||
Get: 'GET',
|
||||
Post: 'POST',
|
||||
Put: 'PUT',
|
||||
Delete: 'DELETE',
|
||||
Patch: 'PATCH',
|
||||
};
|
||||
|
||||
/**
|
||||
* Find the nearest enclosing class_declaration for a node, or null.
|
||||
*/
|
||||
function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
|
||||
let cur: Parser.SyntaxNode | null = node.parent;
|
||||
while (cur) {
|
||||
if (cur.type === 'class_declaration') return cur;
|
||||
cur = cur.parent;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function joinPath(prefix: string, sub: string): string {
|
||||
const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, '');
|
||||
const cleanSub = sub.replace(/^\/+/, '');
|
||||
if (!cleanPrefix) return `/${cleanSub}`;
|
||||
return `/${cleanPrefix}/${cleanSub}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk `pair` children of an `object` literal and return the unquoted
|
||||
* string/template_string value for the first pair whose key matches one
|
||||
* of `keyNames`. Returns null when no matching pair is present or the
|
||||
* value is not a string literal. Used by the jQuery ajax / axios object
|
||||
* consumers to resolve `url` / `method` / `type` keys in any order.
|
||||
*/
|
||||
function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string[]): string | null {
|
||||
for (let i = 0; i < objectNode.namedChildCount; i++) {
|
||||
const pair = objectNode.namedChild(i);
|
||||
if (!pair || pair.type !== 'pair') continue;
|
||||
const keyNode = pair.childForFieldName('key');
|
||||
const valueNode = pair.childForFieldName('value');
|
||||
if (!keyNode || !valueNode) continue;
|
||||
if (!keyNames.includes(keyNode.text)) continue;
|
||||
if (valueNode.type !== 'string' && valueNode.type !== 'template_string') continue;
|
||||
const lit = unquoteLiteral(valueNode.text);
|
||||
if (lit !== null) return lit;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* For a standalone `decorator` node (child of class_body / program),
|
||||
* find the related `class_declaration` node that it decorates. In
|
||||
* tree-sitter-typescript the decorator is placed before the class
|
||||
* declaration as a sibling (when decorating a class) or inside the
|
||||
* class_body before a method_definition (when decorating a method);
|
||||
* we walk the parent chain until we find the enclosing class.
|
||||
*/
|
||||
function findDecoratedClass(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null {
|
||||
const parent = decoratorNode.parent;
|
||||
if (!parent) return null;
|
||||
// Case 1: decorator is a sibling of the class_declaration at program /
|
||||
// export_statement level. Walk forward through siblings until we find
|
||||
// the class_declaration this decorator belongs to.
|
||||
for (let i = 0; i < parent.namedChildCount; i++) {
|
||||
const child = parent.namedChild(i);
|
||||
if (child && child.id === decoratorNode.id) {
|
||||
for (let j = i + 1; j < parent.namedChildCount; j++) {
|
||||
const next = parent.namedChild(j);
|
||||
if (!next) continue;
|
||||
if (next.type === 'decorator') continue; // adjacent decorators stack
|
||||
if (next.type === 'class_declaration') return next;
|
||||
if (next.type === 'export_statement') {
|
||||
// `export class Foo { ... }` wraps the declaration.
|
||||
for (let k = 0; k < next.namedChildCount; k++) {
|
||||
const inner = next.namedChild(k);
|
||||
if (inner?.type === 'class_declaration') return inner;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Case 2: decorator is inside a class_body (decorating a method) —
|
||||
// walk up to the enclosing class_declaration.
|
||||
return findEnclosingClass(decoratorNode);
|
||||
}
|
||||
|
||||
/**
|
||||
* For a method-level decorator node (child of class_body before a
|
||||
* method_definition), find the method_definition it decorates.
|
||||
*/
|
||||
function findDecoratedMethod(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null {
|
||||
const parent = decoratorNode.parent;
|
||||
if (!parent || parent.type !== 'class_body') return null;
|
||||
for (let i = 0; i < parent.namedChildCount; i++) {
|
||||
const child = parent.namedChild(i);
|
||||
if (child && child.id === decoratorNode.id) {
|
||||
for (let j = i + 1; j < parent.namedChildCount; j++) {
|
||||
const next = parent.namedChild(j);
|
||||
if (!next) continue;
|
||||
if (next.type === 'decorator') continue;
|
||||
if (next.type === 'method_definition') return next;
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function scanBundle(bundle: NodePatternBundle, tree: Parser.Tree): HttpDetection[] {
|
||||
const out: HttpDetection[] = [];
|
||||
|
||||
// NestJS: collect `@Controller('prefix')` class decorators, keyed by
|
||||
// the `class_declaration` they decorate.
|
||||
const prefixByClassId = new Map<number, string>();
|
||||
for (const match of runCompiledPatterns(bundle.controller, tree)) {
|
||||
const prefixNode = match.captures.prefix;
|
||||
const decoratorNode = match.captures.ctrl_decorator;
|
||||
if (!prefixNode || !decoratorNode) continue;
|
||||
const prefix = unquoteLiteral(prefixNode.text);
|
||||
if (prefix === null) continue;
|
||||
const classNode = findDecoratedClass(decoratorNode);
|
||||
if (!classNode) continue;
|
||||
prefixByClassId.set(classNode.id, prefix);
|
||||
}
|
||||
|
||||
// NestJS: method-level @Get/@Post/... decorators. The decorator's
|
||||
// arguments list may be empty (`@Get()`), a string (`@Get('path')`),
|
||||
// or something else (which we skip).
|
||||
for (const match of runCompiledPatterns(bundle.methodDecorator, tree)) {
|
||||
const decNode = match.captures.dec;
|
||||
const argsNode = match.captures.args;
|
||||
const decoratorNode = match.captures.method_decorator;
|
||||
if (!decNode || !argsNode || !decoratorNode) continue;
|
||||
const httpMethod = NEST_DECORATOR_TO_HTTP[decNode.text];
|
||||
if (!httpMethod) continue;
|
||||
const methodNode = findDecoratedMethod(decoratorNode);
|
||||
if (!methodNode) continue;
|
||||
const enclosingClass = findEnclosingClass(methodNode);
|
||||
// Only emit NestJS detections when the class actually has a
|
||||
// @Controller decorator — without it, the match is almost certainly
|
||||
// something else (e.g. an unrelated library using similar names).
|
||||
if (!enclosingClass || !prefixByClassId.has(enclosingClass.id)) continue;
|
||||
const prefix = prefixByClassId.get(enclosingClass.id) ?? '';
|
||||
|
||||
let rawPath = '/';
|
||||
const firstArg = argsNode.namedChild(0);
|
||||
if (firstArg && (firstArg.type === 'string' || firstArg.type === 'template_string')) {
|
||||
const unquoted = unquoteLiteral(firstArg.text);
|
||||
if (unquoted !== null) rawPath = unquoted;
|
||||
}
|
||||
|
||||
// Get the method name from the decorated method_definition.
|
||||
const methodNameNode = methodNode.childForFieldName('name');
|
||||
const name = methodNameNode?.text ?? null;
|
||||
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'nest',
|
||||
method: httpMethod,
|
||||
path: joinPath(prefix, rawPath),
|
||||
name,
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// Express: router/app.<verb>(...)
|
||||
for (const match of runCompiledPatterns(bundle.express, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'express',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: 'handler',
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumer: fetch with options { method: 'X' }
|
||||
const fetchSeen = new Set<number>();
|
||||
for (const match of runCompiledPatterns(bundle.fetchWithOptions, tree)) {
|
||||
const pathNode = match.captures.path;
|
||||
const methodNode = match.captures.http_method;
|
||||
if (!pathNode || !methodNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
const method = unquoteLiteral(methodNode.text);
|
||||
if (path === null || method === null) continue;
|
||||
fetchSeen.add(pathNode.id);
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'fetch',
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumer: plain fetch(path) — default GET. Skip path nodes we already
|
||||
// matched with the options variant so we don't double-emit.
|
||||
for (const match of runCompiledPatterns(bundle.fetchNoOptions, tree)) {
|
||||
const pathNode = match.captures.path;
|
||||
if (!pathNode) continue;
|
||||
if (fetchSeen.has(pathNode.id)) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'fetch',
|
||||
method: 'GET',
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumer: axios.<verb>(url)
|
||||
for (const match of runCompiledPatterns(bundle.axios, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'axios',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumer: jQuery shorthand $.get(url) / $.post(url, ...)
|
||||
for (const match of runCompiledPatterns(bundle.jqueryShorthand, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'jquery',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumer: jQuery $.ajax({ url, method|type }). jQuery accepts either
|
||||
// `method:` or `type:`; both default to GET when absent.
|
||||
for (const match of runCompiledPatterns(bundle.jqueryAjax, tree)) {
|
||||
const optionsNode = match.captures.options;
|
||||
if (!optionsNode) continue;
|
||||
const path = readStringProp(optionsNode, ['url']);
|
||||
if (path === null) continue;
|
||||
const rawMethod = readStringProp(optionsNode, ['method', 'type']);
|
||||
const method = (rawMethod ?? 'GET').toUpperCase();
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'jquery',
|
||||
method,
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumer: axios({ method, url }) object form. Structurally distinct
|
||||
// from axios.<verb>(url) (identifier vs member_expression call), so no
|
||||
// dedup against the member-form loop above is required.
|
||||
for (const match of runCompiledPatterns(bundle.axiosObject, tree)) {
|
||||
const optionsNode = match.captures.options;
|
||||
if (!optionsNode) continue;
|
||||
const path = readStringProp(optionsNode, ['url']);
|
||||
if (path === null) continue;
|
||||
const rawMethod = readStringProp(optionsNode, ['method']);
|
||||
const method = (rawMethod ?? 'GET').toUpperCase();
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'axios',
|
||||
method,
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
export const JAVASCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'javascript-http',
|
||||
language: JavaScript,
|
||||
scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree),
|
||||
};
|
||||
|
||||
export const TYPESCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'typescript-http',
|
||||
language: TypeScript.typescript,
|
||||
scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree),
|
||||
};
|
||||
|
||||
export const TSX_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'tsx-http',
|
||||
language: TypeScript.tsx,
|
||||
scan: (tree) => scanBundle(TSX_BUNDLE, tree),
|
||||
};
|
||||
79
gitnexus/src/core/group/extractors/http-patterns/php.ts
Normal file
79
gitnexus/src/core/group/extractors/http-patterns/php.ts
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
import PHP from 'tree-sitter-php';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
unquoteLiteral,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* PHP HTTP plugin — Laravel `Route::get/post/...` declarations.
|
||||
*
|
||||
* The pipeline already uses `PHP.php_only` for ingesting plain `.php`
|
||||
* files (see `core/tree-sitter/parser-loader.ts`), and we do the same
|
||||
* here so Laravel route files are parsed with the right grammar dialect.
|
||||
*/
|
||||
|
||||
const LARAVEL_PATTERNS = compilePatterns({
|
||||
name: 'php-laravel',
|
||||
language: PHP.php_only,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(scoped_call_expression
|
||||
scope: (name) @scope (#eq? @scope "Route")
|
||||
name: (name) @method (#match? @method "^(get|post|put|delete|patch)$")
|
||||
arguments: (arguments . (argument (string) @path)))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
/**
|
||||
* Extract the inner text of a PHP `string` node. The tree-sitter-php
|
||||
* grammar wraps single / double-quoted literals differently depending
|
||||
* on content; we try both the raw `text` (with quotes) through
|
||||
* `unquoteLiteral`, and a fallback via the `string_value` / `string_content`
|
||||
* child nodes.
|
||||
*/
|
||||
function phpStringText(node: import('tree-sitter').SyntaxNode): string | null {
|
||||
// Most single-quoted strings expose their inner content through the
|
||||
// full node text (including quotes), which unquoteLiteral strips.
|
||||
const direct = unquoteLiteral(node.text);
|
||||
if (direct !== null && direct !== node.text) return direct;
|
||||
// Fall back to child string_content / string_value node if present.
|
||||
for (const child of node.children) {
|
||||
if (child.type === 'string_content' || child.type === 'string_value') {
|
||||
return child.text;
|
||||
}
|
||||
}
|
||||
return direct;
|
||||
}
|
||||
|
||||
export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'php-http',
|
||||
language: PHP.php_only,
|
||||
scan(tree) {
|
||||
const out: HttpDetection[] = [];
|
||||
|
||||
for (const match of runCompiledPatterns(LARAVEL_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = phpStringText(pathNode);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'laravel',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: 'route',
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
142
gitnexus/src/core/group/extractors/http-patterns/python.ts
Normal file
142
gitnexus/src/core/group/extractors/http-patterns/python.ts
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import Python from 'tree-sitter-python';
|
||||
import {
|
||||
compilePatterns,
|
||||
runCompiledPatterns,
|
||||
unquoteLiteral,
|
||||
type LanguagePatterns,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
|
||||
|
||||
/**
|
||||
* Python HTTP plugin. Handles:
|
||||
* - FastAPI `@app.get("/path")` provider decorators
|
||||
* - `requests.get/post/...("url")` consumer calls
|
||||
* - Generic `requests.request("METHOD", "url")` consumer calls
|
||||
*/
|
||||
|
||||
const FASTAPI_VERBS: Record<string, string> = {
|
||||
get: 'GET',
|
||||
post: 'POST',
|
||||
put: 'PUT',
|
||||
delete: 'DELETE',
|
||||
patch: 'PATCH',
|
||||
};
|
||||
|
||||
// ─── Provider: FastAPI @app.get/... ──────────────────────────────────
|
||||
const FASTAPI_PATTERNS = compilePatterns({
|
||||
name: 'python-fastapi',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(decorator
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "app")
|
||||
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
|
||||
arguments: (argument_list . (string) @path)))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: requests.get/post/... ──────────────────────────────────
|
||||
const REQUESTS_VERB_PATTERNS = compilePatterns({
|
||||
name: 'python-requests-verb',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "requests")
|
||||
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
|
||||
arguments: (argument_list . (string) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
// ─── Consumer: requests.request("METHOD", "url") ─────────────────────
|
||||
const REQUESTS_GENERIC_PATTERNS = compilePatterns({
|
||||
name: 'python-requests-generic',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "requests")
|
||||
attribute: (identifier) @method (#eq? @method "request"))
|
||||
arguments: (argument_list . (string) @http_method (string) @path))
|
||||
`,
|
||||
},
|
||||
],
|
||||
} satisfies LanguagePatterns<Record<string, never>>);
|
||||
|
||||
export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
|
||||
name: 'python-http',
|
||||
language: Python,
|
||||
scan(tree) {
|
||||
const out: HttpDetection[] = [];
|
||||
|
||||
// Providers: FastAPI
|
||||
for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const httpMethod = FASTAPI_VERBS[methodNode.text];
|
||||
if (!httpMethod) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'provider',
|
||||
framework: 'fastapi',
|
||||
method: httpMethod,
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.8,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumers: requests.<verb>
|
||||
for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'python-requests',
|
||||
method: methodNode.text.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
// Consumers: requests.request("METHOD", "url")
|
||||
for (const match of runCompiledPatterns(REQUESTS_GENERIC_PATTERNS, tree)) {
|
||||
const methodNode = match.captures.http_method;
|
||||
const pathNode = match.captures.path;
|
||||
if (!methodNode || !pathNode) continue;
|
||||
const methodRaw = unquoteLiteral(methodNode.text);
|
||||
const path = unquoteLiteral(pathNode.text);
|
||||
if (methodRaw === null || path === null) continue;
|
||||
out.push({
|
||||
role: 'consumer',
|
||||
framework: 'python-requests',
|
||||
method: methodRaw.toUpperCase(),
|
||||
path,
|
||||
name: null,
|
||||
confidence: 0.7,
|
||||
});
|
||||
}
|
||||
|
||||
return out;
|
||||
},
|
||||
};
|
||||
65
gitnexus/src/core/group/extractors/http-patterns/types.ts
Normal file
65
gitnexus/src/core/group/extractors/http-patterns/types.ts
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
import type Parser from 'tree-sitter';
|
||||
|
||||
/**
|
||||
* Shared types for the http-route-extractor language plugins.
|
||||
*
|
||||
* Each plugin lives in its own file (java.ts, node.ts, ...) and owns
|
||||
* the tree-sitter grammar import + queries. The top-level
|
||||
* `http-route-extractor.ts` orchestrator only knows about this type
|
||||
* module and the plugin registry (`./index.ts`). It MUST NOT import
|
||||
* any grammar or query text directly — language-specific knowledge
|
||||
* belongs in the plugins.
|
||||
*/
|
||||
|
||||
export type HttpRole = 'provider' | 'consumer';
|
||||
|
||||
/**
|
||||
* One raw HTTP detection produced by a plugin's `scan()` function. The
|
||||
* orchestrator converts this into a full `ExtractedContract` by running
|
||||
* path normalization and building the contract id.
|
||||
*
|
||||
* `path` is the raw literal string as it appeared in source (with
|
||||
* `${...}` template placeholders still in place); the orchestrator
|
||||
* runs the appropriate normalizer for provider vs. consumer paths.
|
||||
*/
|
||||
export interface HttpDetection {
|
||||
role: HttpRole;
|
||||
/** Short framework label, e.g. `'spring'`, `'nest'`, `'express'`. */
|
||||
framework: string;
|
||||
/** HTTP method in upper case (`'GET'`, `'POST'`, ...). */
|
||||
method: string;
|
||||
/** Raw path literal as seen in source (template placeholders intact). */
|
||||
path: string;
|
||||
/**
|
||||
* Symbol name of the handler (for providers) or calling function
|
||||
* (for consumers) when the plugin can determine it structurally.
|
||||
* Null when no good candidate is available.
|
||||
*/
|
||||
name: string | null;
|
||||
/** Confidence in (0, 1]. Source-scan plugins typically use 0.7–0.8. */
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* One language-scoped HTTP plugin. The plugin owns the tree-sitter
|
||||
* grammar and the `scan` function that translates a parsed tree into
|
||||
* zero or more `HttpDetection`s. Plugins are free to run multiple
|
||||
* compiled pattern bundles internally (see the shared scanner's
|
||||
* `runCompiledPatterns` helper).
|
||||
*
|
||||
* `language` is typed as `unknown` for the same reason as
|
||||
* `LanguagePatterns.language` in `tree-sitter-scanner.ts` — the
|
||||
* grammar modules export different shapes.
|
||||
*/
|
||||
export interface HttpLanguagePlugin {
|
||||
/** Human-readable plugin name for diagnostics. */
|
||||
name: string;
|
||||
/** tree-sitter grammar object (passed to the shared parser). */
|
||||
language: unknown;
|
||||
/**
|
||||
* Scan a parsed tree and return zero or more HTTP detections. Plugins
|
||||
* must not throw — they should swallow per-match errors so a single
|
||||
* malformed construct does not abort the whole file.
|
||||
*/
|
||||
scan(tree: Parser.Tree): HttpDetection[];
|
||||
}
|
||||
|
|
@ -1,8 +1,34 @@
|
|||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { glob } from 'glob';
|
||||
import Parser from 'tree-sitter';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import { getPluginForFile, HTTP_SCAN_GLOB, type HttpDetection } from './http-patterns/index.js';
|
||||
|
||||
/**
|
||||
* Language-agnostic orchestrator for HTTP route (provider + consumer)
|
||||
* contract extraction. Two strategies, in order of preference per role:
|
||||
*
|
||||
* 1. **Graph-assisted (Strategy A)** — if a per-repo LadybugDB executor
|
||||
* is available, read `HANDLES_ROUTE` / `FETCHES` Cypher edges that
|
||||
* the ingestion pipeline already produced via tree-sitter. This is
|
||||
* the preferred path because the graph has richer symbol metadata
|
||||
* (real uids, class/method structure, etc.).
|
||||
*
|
||||
* 2. **Source-scan fallback (Strategy B)** — parse files directly with
|
||||
* the per-language plugin registry in `./http-patterns/`. Used when
|
||||
* the graph has no routes/fetches for this repo (e.g. a repo that
|
||||
* hasn't been indexed yet, or whose indexer doesn't know the
|
||||
* framework). Each plugin owns its tree-sitter grammar and query
|
||||
* sources — this orchestrator imports NO grammars or query strings.
|
||||
*
|
||||
* Adding a new language for Strategy B is a one-file edit in
|
||||
* `http-patterns/index.ts`: register a new `HttpLanguagePlugin` and
|
||||
* widen `HTTP_SCAN_GLOB` if needed.
|
||||
*/
|
||||
|
||||
// ─── Graph-assisted queries ──────────────────────────────────────────
|
||||
|
||||
const HANDLES_ROUTE_QUERY = `
|
||||
MATCH (handlerFile:File)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route)
|
||||
|
|
@ -23,14 +49,56 @@ WHERE sym.startLine IS NOT NULL
|
|||
RETURN sym.id AS uid, sym.name AS name, sym.filePath AS filePath, labels(sym) AS labels
|
||||
ORDER BY sym.startLine`;
|
||||
|
||||
// ─── Path normalization (shared between provider / consumer paths) ──
|
||||
|
||||
/**
|
||||
* Canonicalize a provider-side HTTP path for contract-id generation:
|
||||
* - strip query string
|
||||
* - lower-case
|
||||
* - drop trailing slash
|
||||
* - collapse `:id`, `{id}`, `[id]` path params into a single `{param}`
|
||||
*/
|
||||
export function normalizeHttpPath(p: string): string {
|
||||
let s = p.trim().split('?')[0].toLowerCase().replace(/\/+$/, '');
|
||||
s = s.replace(/:\w+/g, '{param}');
|
||||
s = s.replace(/\{[^}]+\}/g, '{param}');
|
||||
s = s.replace(/\[[^\]]+\]/g, '{param}');
|
||||
return s;
|
||||
// Preserve root: after stripping trailing slashes, the root "/"
|
||||
// collapses to "" which would produce malformed contract ids like
|
||||
// `http::GET::`. Restore a single slash for the root case.
|
||||
return s === '' ? '/' : s;
|
||||
}
|
||||
|
||||
/**
|
||||
* Consumer-side normalization is more aggressive:
|
||||
* - template literals (`${x}`) → `{param}`
|
||||
* - strip protocol + host if the URL is absolute
|
||||
* - numeric segments → `{param}` (so `/api/orders/42` → `/api/orders/{param}`)
|
||||
*/
|
||||
function normalizeConsumerPath(url: string): string {
|
||||
const templated = url.replace(/\$\{[^}]+\}/g, '{param}').trim();
|
||||
let pathOnly = templated;
|
||||
if (/^https?:\/\//i.test(templated)) {
|
||||
try {
|
||||
pathOnly = new URL(templated).pathname;
|
||||
} catch {
|
||||
pathOnly = templated.replace(/^https?:\/\/[^/]+/i, '');
|
||||
}
|
||||
}
|
||||
const normalized = normalizeHttpPath(pathOnly || '/');
|
||||
const segments = normalized
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.map((segment) => (/^\d+$/.test(segment) ? '{param}' : segment));
|
||||
return `/${segments.join('/')}`.replace(/\/+$/, '') || '/';
|
||||
}
|
||||
|
||||
function contractIdFor(method: string, pathNorm: string): string {
|
||||
return `http::${method.toUpperCase()}::${pathNorm}`;
|
||||
}
|
||||
|
||||
// ─── Graph row helpers ───────────────────────────────────────────────
|
||||
|
||||
function methodFromRouteReason(reason: string): string | null {
|
||||
const r = reason || '';
|
||||
if (/GetMapping|decorator-Get/i.test(r)) return 'GET';
|
||||
|
|
@ -41,50 +109,6 @@ function methodFromRouteReason(reason: string): string | null {
|
|||
return null;
|
||||
}
|
||||
|
||||
function contractIdFor(method: string, pathNorm: string): string {
|
||||
return `http::${method.toUpperCase()}::${pathNorm}`;
|
||||
}
|
||||
|
||||
function readSafe(repoPath: string, rel: string): string | null {
|
||||
const abs = path.resolve(repoPath, rel);
|
||||
const base = path.resolve(repoPath);
|
||||
const relToBase = path.relative(base, abs);
|
||||
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
|
||||
try {
|
||||
return fs.readFileSync(abs, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function pickJavaHandlerName(
|
||||
content: string,
|
||||
routePath: string,
|
||||
httpMethod: string,
|
||||
): string | null {
|
||||
const tail = routePath.split('/').filter(Boolean).pop() || '';
|
||||
const mapNames: Record<string, string> = {
|
||||
GET: 'GetMapping',
|
||||
POST: 'PostMapping',
|
||||
PUT: 'PutMapping',
|
||||
DELETE: 'DeleteMapping',
|
||||
PATCH: 'PatchMapping',
|
||||
};
|
||||
const ann = mapNames[httpMethod] || 'GetMapping';
|
||||
const lines = content.split(/\r?\n/);
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (!line.includes(`@${ann}`)) continue;
|
||||
if (!line.includes(`"${tail}"`) && !line.includes(`'${tail}'`) && tail && !line.includes(tail))
|
||||
continue;
|
||||
for (let j = i + 1; j < Math.min(i + 8, lines.length); j++) {
|
||||
const m = lines[j].match(/(?:public|protected|private)\s+[\w<>,\s\[\]]+\s+(\w+)\s*\(/);
|
||||
if (m) return m[1];
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function pickSymbolUid(
|
||||
rows: Record<string, unknown>[],
|
||||
preferredName: string | null,
|
||||
|
|
@ -114,6 +138,8 @@ function pickSymbolUid(
|
|||
};
|
||||
}
|
||||
|
||||
// ─── Orchestrator ────────────────────────────────────────────────────
|
||||
|
||||
export class HttpRouteExtractor implements ContractExtractor {
|
||||
type = 'http' as const;
|
||||
|
||||
|
|
@ -124,20 +150,76 @@ export class HttpRouteExtractor implements ContractExtractor {
|
|||
async extract(
|
||||
dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
repo: RepoHandle,
|
||||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const graphP = dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, repoPath) : [];
|
||||
const providers = graphP.length > 0 ? graphP : await this.extractProvidersSourceScan(repoPath);
|
||||
// Parse each file at most once and reuse the plugin results across
|
||||
// both graph-assisted enrichment and source-scan emission.
|
||||
const parser = new Parser();
|
||||
const cachedDetections = new Map<string, HttpDetection[]>();
|
||||
const getDetections = (rel: string): HttpDetection[] => {
|
||||
const cached = cachedDetections.get(rel);
|
||||
if (cached) return cached;
|
||||
const plugin = getPluginForFile(rel);
|
||||
if (!plugin) {
|
||||
cachedDetections.set(rel, []);
|
||||
return [];
|
||||
}
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) {
|
||||
cachedDetections.set(rel, []);
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
parser.setLanguage(plugin.language);
|
||||
const tree = parser.parse(content);
|
||||
const detections = plugin.scan(tree);
|
||||
cachedDetections.set(rel, detections);
|
||||
return detections;
|
||||
} catch {
|
||||
cachedDetections.set(rel, []);
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const graphC = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, repoPath) : [];
|
||||
const consumers = graphC.length > 0 ? graphC : await this.extractConsumersSourceScan(repoPath);
|
||||
// Glob the source-scan file list at most once per extract() —
|
||||
// both provider and consumer fallback paths share the same list.
|
||||
let scannedFiles: string[] | null = null;
|
||||
const getScannedFiles = async (): Promise<string[]> => {
|
||||
if (scannedFiles) return scannedFiles;
|
||||
scannedFiles = await this.scanFiles(repoPath);
|
||||
return scannedFiles;
|
||||
};
|
||||
|
||||
const graphProviders =
|
||||
dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, getDetections) : [];
|
||||
const providers =
|
||||
graphProviders.length > 0
|
||||
? graphProviders
|
||||
: this.extractProvidersSourceScan(await getScannedFiles(), getDetections);
|
||||
|
||||
const graphConsumers =
|
||||
dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : [];
|
||||
const consumers =
|
||||
graphConsumers.length > 0
|
||||
? graphConsumers
|
||||
: this.extractConsumersSourceScan(await getScannedFiles(), getDetections);
|
||||
|
||||
return [...providers, ...consumers];
|
||||
}
|
||||
|
||||
private async scanFiles(repoPath: string): Promise<string[]> {
|
||||
return glob(HTTP_SCAN_GLOB, {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/vendor/**'],
|
||||
nodir: true,
|
||||
});
|
||||
}
|
||||
|
||||
// ─── Graph-assisted providers ──────────────────────────────────────
|
||||
|
||||
private async extractProvidersGraph(
|
||||
db: CypherExecutor,
|
||||
repoPath: string,
|
||||
getDetections: (rel: string) => HttpDetection[],
|
||||
): Promise<ExtractedContract[]> {
|
||||
const out: ExtractedContract[] = [];
|
||||
let rows: Record<string, unknown>[];
|
||||
|
|
@ -152,22 +234,55 @@ export class HttpRouteExtractor implements ContractExtractor {
|
|||
const routePath = String(row.routePath ?? '');
|
||||
const routeSource = String(row.routeSource ?? row.routeReason ?? '');
|
||||
let method = methodFromRouteReason(routeSource);
|
||||
const content = readSafe(repoPath, filePath);
|
||||
if (!method && content) {
|
||||
method = this.inferMethodFromFileScan(content, routePath, 'provider');
|
||||
|
||||
// Look up handler name (and backfill method if missing) from the
|
||||
// plugin's scan of the handler file. This replaces the old
|
||||
// regex-based `inferMethodFromFileScan` and `pickJavaHandlerName`
|
||||
// helpers — tree-sitter gives both pieces of information
|
||||
// structurally. Always run the lookup: even when method is set by
|
||||
// `methodFromRouteReason`, we still need the handler name.
|
||||
const detections = filePath ? getDetections(filePath) : [];
|
||||
const providerDetections = detections.filter((d) => d.role === 'provider');
|
||||
let handlerName: string | null = null;
|
||||
const normalizedRoute = normalizeHttpPath(routePath);
|
||||
// Candidates share the same normalized path. When multiple
|
||||
// detections at the same path exist (e.g. GET + POST /api/orders
|
||||
// in one router), a blind `.find()` silently returned the first
|
||||
// verb — attaching the wrong handler and, when method was not
|
||||
// already pinned by the route reason, the wrong method too.
|
||||
// Disambiguate by method when we know it; refuse to guess when
|
||||
// we don't.
|
||||
const candidates = providerDetections.filter(
|
||||
(d) => normalizeHttpPath(d.path) === normalizedRoute,
|
||||
);
|
||||
let match: (typeof candidates)[number] | undefined;
|
||||
const ambiguousCandidates = !method && candidates.length > 1;
|
||||
if (method) {
|
||||
match = candidates.find((d) => d.method === method);
|
||||
} else if (candidates.length === 1) {
|
||||
match = candidates[0];
|
||||
}
|
||||
// else: multiple candidates + unknown method → leave match
|
||||
// undefined so handlerName stays null and skip symbol
|
||||
// enrichment below, keeping the file-basename fallback instead
|
||||
// of letting pickSymbolUid silently pick the first Function /
|
||||
// Method in the file (which reintroduces the mis-attribution
|
||||
// we were trying to avoid). Method stays at the conservative
|
||||
// 'GET' default set below.
|
||||
if (match) {
|
||||
if (!method) method = match.method;
|
||||
handlerName = match.name;
|
||||
}
|
||||
if (!method) method = 'GET';
|
||||
|
||||
const pathNorm = normalizeHttpPath(routePath);
|
||||
const cid = contractIdFor(method, pathNorm);
|
||||
const handlerName =
|
||||
content && routePath ? pickJavaHandlerName(content, routePath, method) : null;
|
||||
|
||||
let symbolUid = '';
|
||||
let symbolName = path.basename(filePath) || 'handler';
|
||||
let symPath = filePath;
|
||||
const fileId = row.fileId ?? row[0];
|
||||
if (fileId) {
|
||||
if (fileId && !ambiguousCandidates) {
|
||||
try {
|
||||
const syms = await db(CONTAINS_QUERY, { fileId });
|
||||
if (syms.length > 0) {
|
||||
|
|
@ -201,145 +316,44 @@ export class HttpRouteExtractor implements ContractExtractor {
|
|||
return out;
|
||||
}
|
||||
|
||||
private inferMethodFromFileScan(
|
||||
content: string,
|
||||
routePath: string,
|
||||
_role: string,
|
||||
): string | null {
|
||||
const tail = routePath.split('/').filter(Boolean).pop() || '';
|
||||
for (const m of ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'] as const) {
|
||||
const mapNames: Record<string, string> = {
|
||||
GET: 'GetMapping',
|
||||
POST: 'PostMapping',
|
||||
PUT: 'PutMapping',
|
||||
DELETE: 'DeleteMapping',
|
||||
PATCH: 'PatchMapping',
|
||||
};
|
||||
if (
|
||||
content.includes(`@${mapNames[m]}`) &&
|
||||
(content.includes(tail) || routePath.includes(tail))
|
||||
) {
|
||||
return m;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
// ─── Source-scan providers ─────────────────────────────────────────
|
||||
|
||||
private async extractProvidersSourceScan(repoPath: string): Promise<ExtractedContract[]> {
|
||||
const files = await glob('**/*.{ts,tsx,js,jsx,java,vue,svelte,php,py}', {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'],
|
||||
nodir: true,
|
||||
});
|
||||
private extractProvidersSourceScan(
|
||||
files: string[],
|
||||
getDetections: (rel: string) => HttpDetection[],
|
||||
): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const rel of files) {
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
out.push(...this.scanSpringProviders(content, rel));
|
||||
out.push(...this.scanExpressProviders(content, rel));
|
||||
out.push(...this.scanLaravelProviders(content, rel));
|
||||
out.push(...this.scanFastApiProviders(content, rel));
|
||||
const detections = getDetections(rel);
|
||||
for (const d of detections) {
|
||||
if (d.role !== 'provider') continue;
|
||||
const pathNorm = normalizeHttpPath(d.path);
|
||||
out.push({
|
||||
contractId: contractIdFor(d.method, pathNorm),
|
||||
type: 'http',
|
||||
role: 'provider',
|
||||
symbolUid: '',
|
||||
symbolRef: { filePath: rel, name: d.name ?? 'handler' },
|
||||
symbolName: d.name ?? 'handler',
|
||||
confidence: d.confidence,
|
||||
meta: {
|
||||
method: d.method,
|
||||
path: pathNorm,
|
||||
pathSegments: pathNorm.split('/').filter(Boolean),
|
||||
extractionStrategy: 'source_scan',
|
||||
framework: d.framework,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
return this.dedupeContracts(out);
|
||||
}
|
||||
|
||||
private dedupeContracts(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const seen = new Set<string>();
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const c of items) {
|
||||
const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`;
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanSpringProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
let classPrefix = '';
|
||||
const classRm = content.match(/@RequestMapping\s*\(\s*"([^"]+)"/);
|
||||
if (classRm) classPrefix = classRm[1].replace(/\/+$/, '');
|
||||
|
||||
const re = /@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*"([^"]+)"/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const method = m[1].toUpperCase();
|
||||
let p = m[2];
|
||||
if (classPrefix) p = `${classPrefix}/${p.replace(/^\//, '')}`;
|
||||
const pathNorm = normalizeHttpPath(p);
|
||||
const sub = content.slice(m.index);
|
||||
const nameM = sub.match(/(?:public|protected|private)\s+[\w<>,\s\[\]]+\s+(\w+)\s*\(/);
|
||||
const name = nameM ? nameM[1] : m[0];
|
||||
out.push(this.makeProvider(filePath, method, pathNorm, name, 0.8));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanExpressProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
const re = /(?:router|app)\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const method = m[1].toUpperCase();
|
||||
const pathNorm = normalizeHttpPath(m[2]);
|
||||
out.push(this.makeProvider(filePath, method, pathNorm, 'handler', 0.8));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanLaravelProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
const re = /Route::(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const method = m[1].toUpperCase();
|
||||
const pathNorm = normalizeHttpPath(m[2]);
|
||||
out.push(this.makeProvider(filePath, method, pathNorm, 'route', 0.8));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private scanFastApiProviders(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
const re = /@app\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const method = m[1].toUpperCase();
|
||||
const pathNorm = normalizeHttpPath(m[2]);
|
||||
out.push(this.makeProvider(filePath, method, pathNorm, 'handler', 0.8));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private makeProvider(
|
||||
filePath: string,
|
||||
method: string,
|
||||
pathNorm: string,
|
||||
name: string,
|
||||
confidence: number,
|
||||
): ExtractedContract {
|
||||
const cid = contractIdFor(method, pathNorm);
|
||||
return {
|
||||
contractId: cid,
|
||||
type: 'http',
|
||||
role: 'provider',
|
||||
symbolUid: '',
|
||||
symbolRef: { filePath, name },
|
||||
symbolName: name,
|
||||
confidence,
|
||||
meta: {
|
||||
method,
|
||||
path: pathNorm,
|
||||
pathSegments: pathNorm.split('/').filter(Boolean),
|
||||
extractionStrategy: 'source_scan',
|
||||
},
|
||||
};
|
||||
}
|
||||
// ─── Graph-assisted consumers ──────────────────────────────────────
|
||||
|
||||
private async extractConsumersGraph(
|
||||
db: CypherExecutor,
|
||||
repoPath: string,
|
||||
getDetections: (rel: string) => HttpDetection[],
|
||||
): Promise<ExtractedContract[]> {
|
||||
const out: ExtractedContract[] = [];
|
||||
let rows: Record<string, unknown>[];
|
||||
|
|
@ -353,11 +367,23 @@ export class HttpRouteExtractor implements ContractExtractor {
|
|||
const routePath = String(row.routePath ?? '');
|
||||
const pathNorm = normalizeHttpPath(routePath);
|
||||
let method = 'GET';
|
||||
const content = readSafe(repoPath, filePath);
|
||||
if (content) {
|
||||
const inferred = this.inferFetchMethod(content, pathNorm);
|
||||
if (inferred) method = inferred;
|
||||
// Prefer the plugin's detected method if we can find a matching
|
||||
// fetch/axios call in the same file.
|
||||
const detections = filePath ? getDetections(filePath) : [];
|
||||
// Symmetric to the provider path: if multiple consumer calls in
|
||||
// the same file share the same normalized path (e.g. a GET
|
||||
// fetch AND a POST fetch to `/api/orders`), `.find()` silently
|
||||
// picked the first verb and keyed the contract id on the wrong
|
||||
// method. With no upstream method signal here, refuse to guess
|
||||
// when candidates are ambiguous — leave `method` at its
|
||||
// conservative 'GET' default.
|
||||
const consumerCandidates = detections.filter(
|
||||
(d) => d.role === 'consumer' && normalizeConsumerPath(d.path) === pathNorm,
|
||||
);
|
||||
if (consumerCandidates.length === 1) {
|
||||
method = consumerCandidates[0].method;
|
||||
}
|
||||
|
||||
const cid = contractIdFor(method, pathNorm);
|
||||
let symbolUid = '';
|
||||
let symbolName = 'fetch';
|
||||
|
|
@ -395,81 +421,47 @@ export class HttpRouteExtractor implements ContractExtractor {
|
|||
return out;
|
||||
}
|
||||
|
||||
private inferFetchMethod(content: string, pathNorm: string): string | null {
|
||||
const esc = pathNorm.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
const fetchRe = new RegExp(
|
||||
`fetch\\s*\\(\\s*['"\`]([^'"\`]*${esc}[^'"\`]*)['"\`]\\s*,\\s*\\{[^}]*method:\\s*['"](\\w+)['"]`,
|
||||
'i',
|
||||
);
|
||||
const m = content.match(fetchRe);
|
||||
if (m) return m[2].toUpperCase();
|
||||
return null;
|
||||
}
|
||||
// ─── Source-scan consumers ─────────────────────────────────────────
|
||||
|
||||
private async extractConsumersSourceScan(repoPath: string): Promise<ExtractedContract[]> {
|
||||
const files = await glob('**/*.{ts,tsx,js,jsx,vue,svelte}', {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**'],
|
||||
nodir: true,
|
||||
});
|
||||
private extractConsumersSourceScan(
|
||||
files: string[],
|
||||
getDetections: (rel: string) => HttpDetection[],
|
||||
): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const rel of files) {
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
out.push(...this.scanFetchConsumers(content, rel));
|
||||
out.push(...this.scanAxiosConsumers(content, rel));
|
||||
const detections = getDetections(rel);
|
||||
for (const d of detections) {
|
||||
if (d.role !== 'consumer') continue;
|
||||
const pathNorm = normalizeConsumerPath(d.path);
|
||||
out.push({
|
||||
contractId: contractIdFor(d.method, pathNorm),
|
||||
type: 'http',
|
||||
role: 'consumer',
|
||||
symbolUid: '',
|
||||
symbolRef: { filePath: rel, name: 'fetch' },
|
||||
symbolName: 'fetch',
|
||||
confidence: d.confidence,
|
||||
meta: {
|
||||
method: d.method,
|
||||
path: pathNorm,
|
||||
extractionStrategy: 'source_scan',
|
||||
framework: d.framework,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
return this.dedupeContracts(out);
|
||||
}
|
||||
|
||||
private scanFetchConsumers(content: string, filePath: string): ExtractedContract[] {
|
||||
private dedupeContracts(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const seen = new Set<string>();
|
||||
const out: ExtractedContract[] = [];
|
||||
const re =
|
||||
/fetch\s*\(\s*['"`]([^'"`]+)['"`](?:\s*,\s*\{[^}]*method:\s*['"](\w+)['"][^}]*\})?\s*\)/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const pathNorm = normalizeHttpPath(this.templateToPattern(m[1]));
|
||||
const method = (m[2] || 'GET').toUpperCase();
|
||||
out.push(this.makeConsumer(filePath, method, pathNorm, 0.7));
|
||||
for (const c of items) {
|
||||
const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`;
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private templateToPattern(url: string): string {
|
||||
return url.replace(/\$\{[^}]+\}/g, '{param}');
|
||||
}
|
||||
|
||||
private scanAxiosConsumers(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
const re = /axios\.(get|post|put|delete|patch)\s*\(\s*[`'"]([^`'"]+)[`'"]/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const method = m[1].toUpperCase();
|
||||
const pathNorm = normalizeHttpPath(this.templateToPattern(m[2]));
|
||||
out.push(this.makeConsumer(filePath, method, pathNorm, 0.7));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private makeConsumer(
|
||||
filePath: string,
|
||||
method: string,
|
||||
pathNorm: string,
|
||||
confidence: number,
|
||||
): ExtractedContract {
|
||||
return {
|
||||
contractId: contractIdFor(method, pathNorm),
|
||||
type: 'http',
|
||||
role: 'consumer',
|
||||
symbolUid: '',
|
||||
symbolRef: { filePath, name: 'fetch' },
|
||||
symbolName: 'fetch',
|
||||
confidence,
|
||||
meta: {
|
||||
method,
|
||||
path: pathNorm,
|
||||
extractionStrategy: 'source_scan',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
344
gitnexus/src/core/group/extractors/manifest-extractor.ts
Normal file
344
gitnexus/src/core/group/extractors/manifest-extractor.ts
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js';
|
||||
import type { CypherExecutor } from '../contract-extractor.js';
|
||||
|
||||
export interface ManifestExtractResult {
|
||||
contracts: StoredContract[];
|
||||
crossLinks: CrossLink[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonicalize an HTTP path for matching against Route.name in the graph.
|
||||
* Mirrors core/ingestion/pipeline.ts ensureSlash semantics:
|
||||
* - Ensures a leading slash.
|
||||
* - Strips trailing slashes (except the root "/").
|
||||
* - Normalizes consecutive slashes.
|
||||
* - Does NOT lowercase (route matching is case-sensitive).
|
||||
*/
|
||||
function normalizeRoutePath(raw: string): string {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) return '/';
|
||||
const withLeading = trimmed.startsWith('/') ? trimmed : `/${trimmed}`;
|
||||
const collapsed = withLeading.replace(/\/+/g, '/');
|
||||
if (collapsed === '/') return '/';
|
||||
return collapsed.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a manifest HTTP contract into its optional `METHOD::` prefix and
|
||||
* its path portion.
|
||||
*
|
||||
* `buildContractId` recommends the explicit-method form `GET::/api/orders`
|
||||
* in group.yaml; if we hand that raw string to `normalizeRoutePath` we get
|
||||
* `/GET::/api/orders`, which can never match `Route.name = "/api/orders"`
|
||||
* in the graph. This helper extracts the path portion so the Cypher
|
||||
* lookup uses the canonical route name.
|
||||
*
|
||||
* The method prefix regex mirrors `buildContractId` (line ~251) for
|
||||
* symmetry: case-insensitive `[A-Za-z]+` followed by `::`. The captured
|
||||
* method is upper-cased for downstream use; method-constrained matching
|
||||
* against `HANDLES_ROUTE` is a future enhancement (not yet wired).
|
||||
*
|
||||
* Edge cases:
|
||||
* - `"::/api/orders"` — empty method portion, no alpha prefix match, so
|
||||
* the whole string is treated as a bare path (matches buildContractId
|
||||
* which also requires `[A-Za-z]+`).
|
||||
* - `"GET::"` — method with empty path, returns `{ method: 'GET', path: '' }`;
|
||||
* `normalizeRoutePath('')` resolves to `/` for caller.
|
||||
*/
|
||||
function parseHttpContract(raw: string): { method: string | null; path: string } {
|
||||
const match = raw.match(/^([A-Za-z]+)::/);
|
||||
if (!match) return { method: null, path: raw };
|
||||
return { method: match[1].toUpperCase(), path: raw.slice(match[0].length) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable synthetic symbolUid for a manifest-declared contract whose target
|
||||
* symbol could not be resolved against the per-repo graph (resolveSymbol
|
||||
* returned null). Two reasons we don't leave the uid empty:
|
||||
*
|
||||
* 1. The bridge stores Contract nodes keyed in part by symbolUid; an empty
|
||||
* uid means downstream Cypher queries that anchor on `provider.symbolUid`
|
||||
* can't tell two different unresolved manifest contracts apart.
|
||||
* 2. The cross-impact bridge query in cross-impact.ts joins local impact
|
||||
* results to bridge contracts via `WHERE provider.symbolUid IN $localUids`.
|
||||
* If the local impact engine produces a deterministic identifier for the
|
||||
* unresolved target, it must agree with the value the bridge stored. A
|
||||
* synthetic uid keyed off (repo, contractId) is the only thing both sides
|
||||
* can derive without knowing about each other.
|
||||
*
|
||||
* Format: `manifest::<repo>::<contractId>`. Stable across syncs, scoped to a
|
||||
* single repo within a group, and never collides with real indexer uids
|
||||
* (which never start with `manifest::`).
|
||||
*/
|
||||
export function manifestSymbolUid(repo: string, contractId: string): string {
|
||||
return `manifest::${repo}::${contractId}`;
|
||||
}
|
||||
|
||||
export class ManifestExtractor {
|
||||
async extractFromManifest(
|
||||
links: GroupManifestLink[],
|
||||
dbExecutors?: Map<string, CypherExecutor>,
|
||||
): Promise<ManifestExtractResult> {
|
||||
// Resolve all (repo, link) pairs in parallel. The previous sequential
|
||||
// await-per-link produced 2N round-trips; parallel resolution uses the
|
||||
// per-repo executor pool directly and scales linearly with manifest size.
|
||||
//
|
||||
// Memoization: a manifest can list the same contract multiple times
|
||||
// (e.g. a consumer and provider declaration, or cross-referenced groups).
|
||||
// Key on (repo, type, contract) — the canonical input to the Cypher
|
||||
// query — so duplicate links resolve to one DB hit.
|
||||
type ResolvedSymbol = { filePath: string; name: string; uid: string } | null;
|
||||
const resolveCache = new Map<string, Promise<ResolvedSymbol>>();
|
||||
const resolveOnce = (repo: string, link: GroupManifestLink): Promise<ResolvedSymbol> => {
|
||||
const key = `${repo}\u0000${link.type}\u0000${link.contract}`;
|
||||
let pending = resolveCache.get(key);
|
||||
if (!pending) {
|
||||
pending = this.resolveSymbol(repo, link, dbExecutors);
|
||||
resolveCache.set(key, pending);
|
||||
}
|
||||
return pending;
|
||||
};
|
||||
|
||||
const perLink = await Promise.all(
|
||||
links.map(async (link) => {
|
||||
const contractId = this.buildContractId(link.type, link.contract);
|
||||
const providerRepo = link.role === 'provider' ? link.from : link.to;
|
||||
const consumerRepo = link.role === 'provider' ? link.to : link.from;
|
||||
const [providerSymbol, consumerSymbol] = await Promise.all([
|
||||
resolveOnce(providerRepo, link),
|
||||
resolveOnce(consumerRepo, link),
|
||||
]);
|
||||
return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol };
|
||||
}),
|
||||
);
|
||||
|
||||
const contracts: StoredContract[] = [];
|
||||
const crossLinks: CrossLink[] = [];
|
||||
|
||||
for (const {
|
||||
link,
|
||||
contractId,
|
||||
providerRepo,
|
||||
consumerRepo,
|
||||
providerSymbol,
|
||||
consumerSymbol,
|
||||
} of perLink) {
|
||||
const providerRef = providerSymbol || { filePath: '', name: link.contract };
|
||||
const consumerRef = consumerSymbol || { filePath: '', name: link.contract };
|
||||
// When the resolver finds a real graph symbol we keep its uid, otherwise
|
||||
// fall back to the deterministic synthetic uid (see manifestSymbolUid).
|
||||
const providerUid = providerSymbol?.uid || manifestSymbolUid(providerRepo, contractId);
|
||||
const consumerUid = consumerSymbol?.uid || manifestSymbolUid(consumerRepo, contractId);
|
||||
|
||||
contracts.push({
|
||||
contractId,
|
||||
type: link.type,
|
||||
role: 'provider',
|
||||
symbolUid: providerUid,
|
||||
symbolRef: providerRef,
|
||||
symbolName: link.contract,
|
||||
confidence: 1.0,
|
||||
meta: { source: 'manifest' },
|
||||
repo: providerRepo,
|
||||
});
|
||||
|
||||
contracts.push({
|
||||
contractId,
|
||||
type: link.type,
|
||||
role: 'consumer',
|
||||
symbolUid: consumerUid,
|
||||
symbolRef: consumerRef,
|
||||
symbolName: link.contract,
|
||||
confidence: 1.0,
|
||||
meta: { source: 'manifest' },
|
||||
repo: consumerRepo,
|
||||
});
|
||||
|
||||
crossLinks.push({
|
||||
from: { repo: consumerRepo, symbolUid: consumerUid, symbolRef: consumerRef },
|
||||
to: { repo: providerRepo, symbolUid: providerUid, symbolRef: providerRef },
|
||||
type: link.type,
|
||||
contractId,
|
||||
matchType: 'manifest',
|
||||
confidence: 1.0,
|
||||
});
|
||||
}
|
||||
|
||||
return { contracts, crossLinks };
|
||||
}
|
||||
|
||||
private async resolveSymbol(
|
||||
repoPathKey: string,
|
||||
link: GroupManifestLink,
|
||||
dbExecutors?: Map<string, CypherExecutor>,
|
||||
): Promise<{ filePath: string; name: string; uid: string } | null> {
|
||||
const executor = dbExecutors?.get(repoPathKey);
|
||||
if (!executor) return null;
|
||||
|
||||
// NOTE: All lookups use EXACT equality on the relevant name field and
|
||||
// deterministic ORDER BY before LIMIT 1. Previous versions used CONTAINS
|
||||
// for fuzzy matching (plus an unconditional ".proto" fallback for gRPC)
|
||||
// which produced silent false positives: e.g. manifest "/orders" would
|
||||
// match "/suborders", and a gRPC manifest entry in a repo with any
|
||||
// .proto file would attach to a random proto symbol.
|
||||
//
|
||||
// If resolveSymbol returns null, the extractor falls back to a
|
||||
// deterministic synthetic uid via `manifestSymbolUid(repo, contractId)`
|
||||
// (see the function's docstring for why synthetic rather than empty).
|
||||
// Cross-impact still works: the bridge query joins on the synthetic
|
||||
// uid, and the local impact engine derives the same uid for the
|
||||
// unresolved symbol — name-based hints are the additional safety net.
|
||||
try {
|
||||
let rows: Record<string, unknown>[];
|
||||
if (link.type === 'http') {
|
||||
// Route.name is the canonicalized URL path (see
|
||||
// core/ingestion/pipeline.ts ensureSlash + generateId('Route', ...)).
|
||||
// Normalize the manifest contract the same way so a user-written
|
||||
// "/api/orders" matches "api/orders" in the graph.
|
||||
//
|
||||
// The contract may also use the explicit-method form "GET::/api/orders"
|
||||
// recommended by buildContractId. Strip the METHOD:: prefix before
|
||||
// normalizing — otherwise `normalizeRoutePath('GET::/api/orders')`
|
||||
// returns `/GET::/api/orders` and never matches Route.name. The
|
||||
// captured method is not yet used to constrain the Cypher query
|
||||
// (method-aware HANDLES_ROUTE matching is a future enhancement).
|
||||
const parsed = parseHttpContract(link.contract);
|
||||
const normalized = normalizeRoutePath(parsed.path);
|
||||
rows = await executor(
|
||||
`MATCH (handler)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route)
|
||||
WHERE route.name = $normalized
|
||||
RETURN handler.id AS uid, handler.name AS name, handler.filePath AS filePath
|
||||
ORDER BY handler.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ normalized },
|
||||
);
|
||||
} else if (link.type === 'topic') {
|
||||
// Topic names aren't a first-class NodeLabel in the graph —
|
||||
// topics are referenced by function/method symbols (Kafka
|
||||
// listeners, publishers). Restrict to symbol-like labels to
|
||||
// avoid cross-matching Files/Variables/Imports that happen to
|
||||
// share the topic name.
|
||||
rows = await executor(
|
||||
`MATCH (n:Function|Method|Class|Interface) WHERE n.name = $contract
|
||||
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
|
||||
ORDER BY n.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else if (link.type === 'grpc') {
|
||||
// Contract is "Service/Method" or just "Service" (or package.Service
|
||||
// variants). Prefer matching by method name when present, otherwise
|
||||
// by service name. NO .proto path fallback — that's guaranteed to
|
||||
// return a wrong symbol in any repo with more than one proto file.
|
||||
// Label filters scope lookups: methods → Function|Method, services
|
||||
// → Class|Interface (no label match = no silent wrong hits on
|
||||
// File/Variable nodes that happen to share the name).
|
||||
const parts = link.contract.split('/');
|
||||
const serviceName = parts[0]?.trim() ?? '';
|
||||
const methodName = parts[1]?.trim() ?? '';
|
||||
if (methodName) {
|
||||
rows = await executor(
|
||||
`MATCH (n:Function|Method) WHERE n.name = $methodName
|
||||
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
|
||||
ORDER BY n.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ methodName },
|
||||
);
|
||||
} else if (serviceName) {
|
||||
rows = await executor(
|
||||
`MATCH (n:Class|Interface) WHERE n.name = $serviceName
|
||||
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
|
||||
ORDER BY n.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ serviceName },
|
||||
);
|
||||
} else {
|
||||
rows = [];
|
||||
}
|
||||
} else if (link.type === 'lib') {
|
||||
// Only exact match on the symbol's name. Previous fallback to
|
||||
// CONTAINS on n.filePath would promote "react" to "react-native"
|
||||
// or "@types/react" — silent wrong attribution. Restrict to
|
||||
// package-level labels so we don't return arbitrary symbols
|
||||
// named after a library.
|
||||
rows = await executor(
|
||||
`MATCH (n:Package|Module) WHERE n.name = $contract
|
||||
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
|
||||
ORDER BY n.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
if (rows.length > 0) {
|
||||
return {
|
||||
filePath: rows[0].filePath as string,
|
||||
name: rows[0].name as string,
|
||||
uid: String(rows[0].uid ?? ''),
|
||||
};
|
||||
}
|
||||
} catch (err) {
|
||||
// Log but don't throw: a broken graph query in one repo shouldn't
|
||||
// fail the whole manifest extraction. Unresolved contracts still
|
||||
// get a synthetic symbolUid below, so cross-impact can proceed.
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(
|
||||
`[manifest-extractor] resolveSymbol failed for ${link.type}:${link.contract} ` +
|
||||
`in ${repoPathKey}: ${message}`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a canonical contract id for a manifest link.
|
||||
*
|
||||
* HTTP is the only type with two valid forms:
|
||||
* - Explicit method: `"GET::/api/orders"` → `"http::GET::/api/orders"`
|
||||
* (matches exactly against `HttpRouteExtractor` provider/consumer
|
||||
* contracts, which are also keyed by `http::<METHOD>::<path>`).
|
||||
* - Method-agnostic: `"/api/orders"` → `"http::*::/api/orders"`
|
||||
* — the `*` is a wildcard and is intended to match any concrete
|
||||
* HTTP method on that path. Wildcard-aware matching is the
|
||||
* responsibility of the sync / cross-impact layer (see #793);
|
||||
* downstream code should treat `http::*::<path>` as matching
|
||||
* every `http::<METHOD>::<path>` for the same path.
|
||||
*
|
||||
* Recommend the explicit-method form in group.yaml whenever the
|
||||
* manifest author knows the method — it round-trips through exact
|
||||
* equality matching without requiring wildcard logic downstream.
|
||||
*
|
||||
* NOTE on exhaustiveness: the switch covers every current
|
||||
* `ContractType` variant and falls through to a `never` assertion so
|
||||
* TypeScript fails the build if a new variant is added without a
|
||||
* corresponding case.
|
||||
*/
|
||||
private buildContractId(type: ContractType, contract: string): string {
|
||||
switch (type) {
|
||||
case 'http': {
|
||||
// Canonicalize method casing and path separators so logically
|
||||
// equivalent inputs (`get::/api/orders` vs `GET::/api/orders`,
|
||||
// or trailing-slash variants) produce the same contractId and
|
||||
// matching `manifestSymbolUid` fallback. Without this, raw
|
||||
// user casing leaks into cross-impact join keys and fragments
|
||||
// matches across repos.
|
||||
const { method, path: rawPath } = parseHttpContract(contract);
|
||||
const normalizedPath = normalizeRoutePath(rawPath);
|
||||
return method ? `http::${method}::${normalizedPath}` : `http::*::${normalizedPath}`;
|
||||
}
|
||||
case 'grpc':
|
||||
return `grpc::${contract}`;
|
||||
case 'topic':
|
||||
return `topic::${contract}`;
|
||||
case 'lib':
|
||||
return `lib::${contract}`;
|
||||
case 'custom':
|
||||
return `custom::${contract}`;
|
||||
default: {
|
||||
const _exhaustive: never = type;
|
||||
throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,214 +1,49 @@
|
|||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { glob } from 'glob';
|
||||
import Parser from 'tree-sitter';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import { scanFile, unquoteLiteral } from './tree-sitter-scanner.js';
|
||||
import {
|
||||
TOPIC_SCAN_GLOB,
|
||||
getProviderForFile,
|
||||
type Broker,
|
||||
type TopicMeta,
|
||||
} from './topic-patterns/index.js';
|
||||
|
||||
type Broker = 'kafka' | 'rabbitmq' | 'nats';
|
||||
/**
|
||||
* Language-agnostic orchestrator for topic (message broker) contract
|
||||
* extraction. All grammar-specific knowledge lives in `topic-patterns/*`
|
||||
* — this file must not import any tree-sitter grammar directly.
|
||||
*
|
||||
* Flow per file:
|
||||
* 1. `getProviderForFile(rel)` → compiled plugin (or `undefined` if the
|
||||
* file's extension isn't registered, in which case we skip it).
|
||||
* 2. `scanFile(parser, provider, content)` → list of `{meta, valueText}`
|
||||
* pairs, one per matched literal.
|
||||
* 3. `unquoteLiteral(valueText)` → the raw topic string.
|
||||
* 4. `makeContract(topic, meta, relPath)` → `ExtractedContract`.
|
||||
*
|
||||
* Adding a new language is a one-file edit in `topic-patterns/index.ts`.
|
||||
*/
|
||||
|
||||
function readSafe(repoPath: string, rel: string): string | null {
|
||||
const abs = path.resolve(repoPath, rel);
|
||||
const base = path.resolve(repoPath);
|
||||
const relToBase = path.relative(base, abs);
|
||||
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
|
||||
try {
|
||||
return fs.readFileSync(abs, 'utf-8');
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function makeContract(
|
||||
topicName: string,
|
||||
role: 'provider' | 'consumer',
|
||||
filePath: string,
|
||||
symbolName: string,
|
||||
confidence: number,
|
||||
broker: Broker,
|
||||
): ExtractedContract {
|
||||
function makeContract(topicName: string, meta: TopicMeta, filePath: string): ExtractedContract {
|
||||
return {
|
||||
contractId: `topic::${topicName}`,
|
||||
type: 'topic',
|
||||
role,
|
||||
role: meta.role,
|
||||
symbolUid: '',
|
||||
symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: symbolName },
|
||||
symbolName,
|
||||
confidence,
|
||||
symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: meta.symbolName },
|
||||
symbolName: meta.symbolName,
|
||||
confidence: meta.confidence,
|
||||
meta: {
|
||||
broker,
|
||||
broker: meta.broker satisfies Broker,
|
||||
topicName,
|
||||
extractionStrategy: 'source_scan',
|
||||
extractionStrategy: 'tree_sitter',
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
interface PatternDef {
|
||||
regex: RegExp;
|
||||
role: 'provider' | 'consumer';
|
||||
broker: Broker;
|
||||
confidence: number;
|
||||
topicGroup: number;
|
||||
symbolName: string;
|
||||
}
|
||||
|
||||
// --- Kafka patterns ---
|
||||
const KAFKA_PATTERNS: PatternDef[] = [
|
||||
// Java: @KafkaListener(topics = "xxx")
|
||||
{
|
||||
regex: /@KafkaListener\s*\(\s*topics\s*=\s*"([^"]+)"/g,
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'kafkaListener',
|
||||
},
|
||||
// Java: kafkaTemplate.send("xxx"
|
||||
{
|
||||
regex: /kafkaTemplate\.send\s*\(\s*"([^"]+)"/gi,
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'kafkaTemplate.send',
|
||||
},
|
||||
// Node: producer.send({ topic: 'xxx'
|
||||
{
|
||||
regex: /producer\.send\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g,
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'producer.send',
|
||||
},
|
||||
// Node: consumer.subscribe({ topic: 'xxx'
|
||||
{
|
||||
regex: /consumer\.subscribe\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g,
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'consumer.subscribe',
|
||||
},
|
||||
// Go: consumer.ConsumePartition("xxx"
|
||||
{
|
||||
regex: /\.ConsumePartition\s*\(\s*"([^"]+)"/g,
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.7,
|
||||
topicGroup: 1,
|
||||
symbolName: 'ConsumePartition',
|
||||
},
|
||||
// Python: KafkaConsumer('xxx'
|
||||
{
|
||||
regex: /KafkaConsumer\s*\(\s*['"]([^'"]+)['"]/g,
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.7,
|
||||
topicGroup: 1,
|
||||
symbolName: 'KafkaConsumer',
|
||||
},
|
||||
// Python: producer.send('xxx' or producer.produce('xxx'
|
||||
{
|
||||
regex: /producer\.(?:send|produce)\s*\(\s*['"]([^'"]+)['"]/g,
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.7,
|
||||
topicGroup: 1,
|
||||
symbolName: 'producer.send',
|
||||
},
|
||||
];
|
||||
|
||||
// --- RabbitMQ patterns ---
|
||||
const RABBITMQ_PATTERNS: PatternDef[] = [
|
||||
// Java: @RabbitListener(queues = "xxx")
|
||||
{
|
||||
regex: /@RabbitListener\s*\(\s*queues\s*=\s*"([^"]+)"/g,
|
||||
role: 'consumer',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'rabbitListener',
|
||||
},
|
||||
// Java: rabbitTemplate.convertAndSend("xxx"
|
||||
{
|
||||
regex: /rabbitTemplate\.convertAndSend\s*\(\s*"([^"]+)"/gi,
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'rabbitTemplate.convertAndSend',
|
||||
},
|
||||
// Node: channel.consume("xxx"
|
||||
{
|
||||
regex: /channel\.consume\s*\(\s*"([^"]+)"/g,
|
||||
role: 'consumer',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'channel.consume',
|
||||
},
|
||||
// Node: channel.publish("xxx"
|
||||
{
|
||||
regex: /channel\.publish\s*\(\s*"([^"]+)"/g,
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'channel.publish',
|
||||
},
|
||||
// Node: channel.sendToQueue("xxx"
|
||||
{
|
||||
regex: /channel\.sendToQueue\s*\(\s*"([^"]+)"/g,
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'channel.sendToQueue',
|
||||
},
|
||||
// Python: channel.basic_consume(queue='xxx'
|
||||
{
|
||||
regex: /channel\.basic_consume\s*\(\s*queue\s*=\s*['"]([^'"]+)['"]/g,
|
||||
role: 'consumer',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.7,
|
||||
topicGroup: 1,
|
||||
symbolName: 'basic_consume',
|
||||
},
|
||||
// Python: channel.basic_publish(exchange='xxx'
|
||||
{
|
||||
regex: /channel\.basic_publish\s*\([^)]*exchange\s*=\s*['"]([^'"]+)['"]/g,
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.7,
|
||||
topicGroup: 1,
|
||||
symbolName: 'basic_publish',
|
||||
},
|
||||
];
|
||||
|
||||
// --- NATS patterns ---
|
||||
const NATS_PATTERNS: PatternDef[] = [
|
||||
// Go/Node: nc.Subscribe("xxx" or nc.subscribe("xxx"
|
||||
{
|
||||
regex: /nc\.(?:S|s)ubscribe\s*\(\s*"([^"]+)"/g,
|
||||
role: 'consumer',
|
||||
broker: 'nats',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'nc.Subscribe',
|
||||
},
|
||||
// Go/Node: nc.Publish("xxx" or nc.publish("xxx"
|
||||
{
|
||||
regex: /nc\.(?:P|p)ublish\s*\(\s*"([^"]+)"/g,
|
||||
role: 'provider',
|
||||
broker: 'nats',
|
||||
confidence: 0.8,
|
||||
topicGroup: 1,
|
||||
symbolName: 'nc.Publish',
|
||||
},
|
||||
];
|
||||
|
||||
const ALL_PATTERNS: PatternDef[] = [...KAFKA_PATTERNS, ...RABBITMQ_PATTERNS, ...NATS_PATTERNS];
|
||||
|
||||
export class TopicExtractor implements ContractExtractor {
|
||||
type = 'topic' as const;
|
||||
|
||||
|
|
@ -221,46 +56,48 @@ export class TopicExtractor implements ContractExtractor {
|
|||
repoPath: string,
|
||||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const files = await glob('**/*.{ts,tsx,js,jsx,java,go,py}', {
|
||||
const files = await glob(TOPIC_SCAN_GLOB, {
|
||||
cwd: repoPath,
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
ignore: [
|
||||
'**/node_modules/**',
|
||||
'**/.git/**',
|
||||
'**/vendor/**',
|
||||
'**/dist/**',
|
||||
'**/build/**',
|
||||
// Language-level test file conventions. Go test files
|
||||
// `*_test.go` live next to source; other languages either use
|
||||
// separate test directories (Python's `tests/`, Java's
|
||||
// `src/test/`) or are already covered by the dist/build ignores.
|
||||
// Pushed to the glob level so the orchestrator stays
|
||||
// language-agnostic.
|
||||
'**/*_test.go',
|
||||
],
|
||||
nodir: true,
|
||||
});
|
||||
|
||||
// One parser reused across files; the scanner calls `setLanguage` per
|
||||
// file based on which plugin the registry returns.
|
||||
const parser = new Parser();
|
||||
const out: ExtractedContract[] = [];
|
||||
|
||||
for (const rel of files) {
|
||||
const provider = getProviderForFile(rel);
|
||||
if (!provider) continue;
|
||||
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
out.push(...this.scanFile(content, rel));
|
||||
}
|
||||
|
||||
return this.dedupe(out);
|
||||
}
|
||||
|
||||
private scanFile(content: string, filePath: string): ExtractedContract[] {
|
||||
const out: ExtractedContract[] = [];
|
||||
|
||||
for (const pattern of ALL_PATTERNS) {
|
||||
// Reset regex state for each file
|
||||
const re = new RegExp(pattern.regex.source, pattern.regex.flags);
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(content)) !== null) {
|
||||
const topicName = m[pattern.topicGroup];
|
||||
const matches = scanFile(parser, provider, content);
|
||||
for (const match of matches) {
|
||||
const valueNode = match.captures.value;
|
||||
if (!valueNode) continue;
|
||||
const topicName = unquoteLiteral(valueNode.text);
|
||||
if (!topicName) continue;
|
||||
out.push(
|
||||
makeContract(
|
||||
topicName,
|
||||
pattern.role,
|
||||
filePath,
|
||||
pattern.symbolName,
|
||||
pattern.confidence,
|
||||
pattern.broker,
|
||||
),
|
||||
);
|
||||
out.push(makeContract(topicName, match.meta, rel));
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
return this.dedupe(out);
|
||||
}
|
||||
|
||||
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
|
||||
|
|
|
|||
123
gitnexus/src/core/group/extractors/topic-patterns/go.ts
Normal file
123
gitnexus/src/core/group/extractors/topic-patterns/go.ts
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
import Go from 'tree-sitter-go';
|
||||
import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js';
|
||||
import type { TopicMeta } from './types.js';
|
||||
|
||||
/**
|
||||
* Go topic extraction patterns.
|
||||
*
|
||||
* Detects Sarama, segmentio/kafka-go and nats.go producer/consumer APIs:
|
||||
* - `X.ConsumePartition("topic", ...)`
|
||||
* - `sarama.ProducerMessage{Topic: "xxx"}`
|
||||
* - `kafka.Writer{Topic: "xxx"}` / `kafka.WriterConfig{Topic: ...}`
|
||||
* - `kafka.Reader{Topic: "xxx"}` / `kafka.ReaderConfig{Topic: ...}`
|
||||
* - `nc.Subscribe("topic", ...)` / `js.Subscribe("topic", ...)`
|
||||
* - `nc.Publish("topic", ...)` / `js.Publish("topic", ...)`
|
||||
*
|
||||
* Every query MUST bind `@value` to the topic literal node.
|
||||
*/
|
||||
const GO_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
|
||||
name: 'go-topic',
|
||||
language: Go,
|
||||
patterns: [
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.7,
|
||||
symbolName: 'ConsumePartition',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
field: (field_identifier) @method (#eq? @method "ConsumePartition"))
|
||||
arguments: (argument_list . (interpreted_string_literal) @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.75,
|
||||
symbolName: 'sarama.ProducerMessage',
|
||||
},
|
||||
query: `
|
||||
(composite_literal
|
||||
type: (qualified_type
|
||||
package: (package_identifier) @pkg (#eq? @pkg "sarama")
|
||||
name: (type_identifier) @ty (#eq? @ty "ProducerMessage"))
|
||||
body: (literal_value
|
||||
(keyed_element
|
||||
(literal_element (identifier) @field (#eq? @field "Topic"))
|
||||
(literal_element (interpreted_string_literal) @value))))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.75,
|
||||
symbolName: 'kafka.Writer',
|
||||
},
|
||||
query: `
|
||||
(composite_literal
|
||||
type: (qualified_type
|
||||
package: (package_identifier) @pkg (#eq? @pkg "kafka")
|
||||
name: (type_identifier) @ty (#match? @ty "^(Writer|WriterConfig)$"))
|
||||
body: (literal_value
|
||||
(keyed_element
|
||||
(literal_element (identifier) @field (#eq? @field "Topic"))
|
||||
(literal_element (interpreted_string_literal) @value))))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.75,
|
||||
symbolName: 'kafka.Reader',
|
||||
},
|
||||
query: `
|
||||
(composite_literal
|
||||
type: (qualified_type
|
||||
package: (package_identifier) @pkg (#eq? @pkg "kafka")
|
||||
name: (type_identifier) @ty (#match? @ty "^(Reader|ReaderConfig)$"))
|
||||
body: (literal_value
|
||||
(keyed_element
|
||||
(literal_element (identifier) @field (#eq? @field "Topic"))
|
||||
(literal_element (interpreted_string_literal) @value))))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'nats',
|
||||
confidence: 0.8,
|
||||
symbolName: 'nc.Subscribe',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
operand: (identifier) @obj (#match? @obj "^(nc|js)$")
|
||||
field: (field_identifier) @method (#match? @method "^[Ss]ubscribe$"))
|
||||
arguments: (argument_list . (interpreted_string_literal) @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'nats',
|
||||
confidence: 0.8,
|
||||
symbolName: 'nc.Publish',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (selector_expression
|
||||
operand: (identifier) @obj (#match? @obj "^(nc|js)$")
|
||||
field: (field_identifier) @method (#match? @method "^[Pp]ublish$"))
|
||||
arguments: (argument_list . (interpreted_string_literal) @value))
|
||||
`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const GO_TOPIC_PROVIDER = compilePatterns(GO_TOPIC_SPEC);
|
||||
49
gitnexus/src/core/group/extractors/topic-patterns/index.ts
Normal file
49
gitnexus/src/core/group/extractors/topic-patterns/index.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import * as path from 'node:path';
|
||||
import type { CompiledPatterns } from '../tree-sitter-scanner.js';
|
||||
import type { TopicMeta } from './types.js';
|
||||
import { JAVA_TOPIC_PROVIDER } from './java.js';
|
||||
import { GO_TOPIC_PROVIDER } from './go.js';
|
||||
import { PYTHON_TOPIC_PROVIDER } from './python.js';
|
||||
import {
|
||||
JAVASCRIPT_TOPIC_PROVIDER,
|
||||
TYPESCRIPT_TOPIC_PROVIDER,
|
||||
TSX_TOPIC_PROVIDER,
|
||||
} from './node.js';
|
||||
|
||||
export type { TopicMeta, Broker } from './types.js';
|
||||
|
||||
/**
|
||||
* File-extension → compiled-plugin registry for topic extraction. The
|
||||
* top-level orchestrator (`topic-extractor.ts`) looks up the plugin for
|
||||
* each file it visits and delegates the scanning to `tree-sitter-scanner`.
|
||||
*
|
||||
* Keys are lowercase extensions including the leading dot. To add a new
|
||||
* language, drop a `topic-patterns/<lang>.ts` that exports a compiled
|
||||
* provider, import it here and register the extension(s). No edits to
|
||||
* `topic-extractor.ts` are required.
|
||||
*/
|
||||
const REGISTRY: Record<string, CompiledPatterns<TopicMeta>> = {
|
||||
'.java': JAVA_TOPIC_PROVIDER,
|
||||
'.go': GO_TOPIC_PROVIDER,
|
||||
'.py': PYTHON_TOPIC_PROVIDER,
|
||||
'.js': JAVASCRIPT_TOPIC_PROVIDER,
|
||||
'.jsx': JAVASCRIPT_TOPIC_PROVIDER,
|
||||
'.ts': TYPESCRIPT_TOPIC_PROVIDER,
|
||||
'.tsx': TSX_TOPIC_PROVIDER,
|
||||
};
|
||||
|
||||
/**
|
||||
* Glob pattern for files worth scanning. Kept here so adding a new
|
||||
* language to the registry also widens the glob automatically via a
|
||||
* single edit.
|
||||
*/
|
||||
export const TOPIC_SCAN_GLOB = '**/*.{ts,tsx,js,jsx,java,go,py}';
|
||||
|
||||
/**
|
||||
* Return the compiled provider registered for the given file's
|
||||
* extension, or `undefined` if the extension is not registered.
|
||||
*/
|
||||
export function getProviderForFile(rel: string): CompiledPatterns<TopicMeta> | undefined {
|
||||
const ext = path.extname(rel).toLowerCase();
|
||||
return REGISTRY[ext];
|
||||
}
|
||||
83
gitnexus/src/core/group/extractors/topic-patterns/java.ts
Normal file
83
gitnexus/src/core/group/extractors/topic-patterns/java.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import Java from 'tree-sitter-java';
|
||||
import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js';
|
||||
import type { TopicMeta } from './types.js';
|
||||
|
||||
/**
|
||||
* Java topic extraction patterns.
|
||||
*
|
||||
* Detects Kafka and RabbitMQ (Spring conventions) producer/consumer APIs:
|
||||
* - `@KafkaListener(topics = "xxx")`
|
||||
* - `@RabbitListener(queues = "xxx")`
|
||||
* - `kafkaTemplate.send("xxx", ...)`
|
||||
* - `rabbitTemplate.convertAndSend("xxx", ...)`
|
||||
*
|
||||
* Every query MUST bind `@value` to the topic literal node.
|
||||
*/
|
||||
const JAVA_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
|
||||
name: 'java-topic',
|
||||
language: Java,
|
||||
patterns: [
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
symbolName: 'kafkaListener',
|
||||
},
|
||||
query: `
|
||||
(annotation
|
||||
name: (identifier) @name (#eq? @name "KafkaListener")
|
||||
arguments: (annotation_argument_list
|
||||
(element_value_pair
|
||||
key: (identifier) @key (#eq? @key "topics")
|
||||
value: (string_literal) @value)))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
symbolName: 'rabbitListener',
|
||||
},
|
||||
query: `
|
||||
(annotation
|
||||
name: (identifier) @name (#eq? @name "RabbitListener")
|
||||
arguments: (annotation_argument_list
|
||||
(element_value_pair
|
||||
key: (identifier) @key (#eq? @key "queues")
|
||||
value: (string_literal) @value)))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
symbolName: 'kafkaTemplate.send',
|
||||
},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (identifier) @obj (#eq? @obj "kafkaTemplate")
|
||||
name: (identifier) @method (#eq? @method "send")
|
||||
arguments: (argument_list . (string_literal) @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
symbolName: 'rabbitTemplate.convertAndSend',
|
||||
},
|
||||
query: `
|
||||
(method_invocation
|
||||
object: (identifier) @obj (#eq? @obj "rabbitTemplate")
|
||||
name: (identifier) @method (#eq? @method "convertAndSend")
|
||||
arguments: (argument_list . (string_literal) @value))
|
||||
`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const JAVA_TOPIC_PROVIDER = compilePatterns(JAVA_TOPIC_SPEC);
|
||||
165
gitnexus/src/core/group/extractors/topic-patterns/node.ts
Normal file
165
gitnexus/src/core/group/extractors/topic-patterns/node.ts
Normal file
|
|
@ -0,0 +1,165 @@
|
|||
import JavaScript from 'tree-sitter-javascript';
|
||||
import TypeScript from 'tree-sitter-typescript';
|
||||
import {
|
||||
compilePatterns,
|
||||
type LanguagePatterns,
|
||||
type PatternSpec,
|
||||
} from '../tree-sitter-scanner.js';
|
||||
import type { TopicMeta } from './types.js';
|
||||
|
||||
/**
|
||||
* Node.js / TypeScript topic extraction patterns.
|
||||
*
|
||||
* Detects kafkajs, amqplib (RabbitMQ), and nats.js producer/consumer APIs:
|
||||
* - `producer.send({ topic: 'xxx', ... })` (kafkajs)
|
||||
* - `consumer.subscribe({ topic: 'xxx', ... })` (kafkajs)
|
||||
* - `channel.consume("queue", ...)` / `channel.publish(...)` / `channel.sendToQueue(...)`
|
||||
* - `nc.subscribe("topic")` / `js.subscribe("topic")`
|
||||
* - `nc.publish("topic", ...)` / `js.publish("topic", ...)`
|
||||
*
|
||||
* The JavaScript and TypeScript tree-sitter grammars share node type
|
||||
* names for every construct we query here, so the pattern sources are
|
||||
* defined once and compiled against each grammar variant. We export three
|
||||
* providers because Parser.Query objects are NOT portable across grammar
|
||||
* instances — `.js` files use the JavaScript grammar, `.ts` uses
|
||||
* TypeScript.typescript, and `.tsx` uses TypeScript.tsx.
|
||||
*
|
||||
* Every query MUST bind `@value` to the topic literal node.
|
||||
*/
|
||||
const NODE_TOPIC_PATTERNS: PatternSpec<TopicMeta>[] = [
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
symbolName: 'producer.send',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "producer")
|
||||
property: (property_identifier) @prop (#eq? @prop "send"))
|
||||
arguments: (arguments
|
||||
(object
|
||||
(pair
|
||||
key: (property_identifier) @key (#eq? @key "topic")
|
||||
value: [(string) (template_string)] @value))))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.8,
|
||||
symbolName: 'consumer.subscribe',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "consumer")
|
||||
property: (property_identifier) @prop (#eq? @prop "subscribe"))
|
||||
arguments: (arguments
|
||||
(object
|
||||
(pair
|
||||
key: (property_identifier) @key (#eq? @key "topic")
|
||||
value: [(string) (template_string)] @value))))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
symbolName: 'channel.consume',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "channel")
|
||||
property: (property_identifier) @prop (#eq? @prop "consume"))
|
||||
arguments: (arguments . [(string) (template_string)] @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
symbolName: 'channel.publish',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "channel")
|
||||
property: (property_identifier) @prop (#eq? @prop "publish"))
|
||||
arguments: (arguments . [(string) (template_string)] @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.8,
|
||||
symbolName: 'channel.sendToQueue',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#eq? @obj "channel")
|
||||
property: (property_identifier) @prop (#eq? @prop "sendToQueue"))
|
||||
arguments: (arguments . [(string) (template_string)] @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'nats',
|
||||
confidence: 0.8,
|
||||
symbolName: 'nc.subscribe',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#match? @obj "^(nc|js)$")
|
||||
property: (property_identifier) @prop (#match? @prop "^[Ss]ubscribe$"))
|
||||
arguments: (arguments . [(string) (template_string)] @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'nats',
|
||||
confidence: 0.8,
|
||||
symbolName: 'nc.publish',
|
||||
},
|
||||
query: `
|
||||
(call_expression
|
||||
function: (member_expression
|
||||
object: (identifier) @obj (#match? @obj "^(nc|js)$")
|
||||
property: (property_identifier) @prop (#match? @prop "^[Pp]ublish$"))
|
||||
arguments: (arguments . [(string) (template_string)] @value))
|
||||
`,
|
||||
},
|
||||
];
|
||||
|
||||
const JAVASCRIPT_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
|
||||
name: 'javascript-topic',
|
||||
language: JavaScript,
|
||||
patterns: NODE_TOPIC_PATTERNS,
|
||||
};
|
||||
|
||||
const TYPESCRIPT_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
|
||||
name: 'typescript-topic',
|
||||
language: TypeScript.typescript,
|
||||
patterns: NODE_TOPIC_PATTERNS,
|
||||
};
|
||||
|
||||
const TSX_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
|
||||
name: 'tsx-topic',
|
||||
language: TypeScript.tsx,
|
||||
patterns: NODE_TOPIC_PATTERNS,
|
||||
};
|
||||
|
||||
export const JAVASCRIPT_TOPIC_PROVIDER = compilePatterns(JAVASCRIPT_TOPIC_SPEC);
|
||||
export const TYPESCRIPT_TOPIC_PROVIDER = compilePatterns(TYPESCRIPT_TOPIC_SPEC);
|
||||
export const TSX_TOPIC_PROVIDER = compilePatterns(TSX_TOPIC_SPEC);
|
||||
119
gitnexus/src/core/group/extractors/topic-patterns/python.ts
Normal file
119
gitnexus/src/core/group/extractors/topic-patterns/python.ts
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
import Python from 'tree-sitter-python';
|
||||
import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js';
|
||||
import type { TopicMeta } from './types.js';
|
||||
|
||||
/**
|
||||
* Python topic extraction patterns.
|
||||
*
|
||||
* Detects kafka-python, pika (RabbitMQ), and nats-py producer/consumer APIs:
|
||||
* - `KafkaConsumer('topic', ...)`
|
||||
* - `producer.send('topic', ...)` / `producer.produce('topic', ...)`
|
||||
* - `channel.basic_consume(queue='xxx', ...)`
|
||||
* - `channel.basic_publish(exchange='xxx', ...)`
|
||||
* - `await nc.subscribe('topic')`
|
||||
* - `await nc.publish('topic', ...)`
|
||||
*
|
||||
* Every query MUST bind `@value` to the topic literal node.
|
||||
*/
|
||||
const PYTHON_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
|
||||
name: 'python-topic',
|
||||
language: Python,
|
||||
patterns: [
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'kafka',
|
||||
confidence: 0.7,
|
||||
symbolName: 'KafkaConsumer',
|
||||
},
|
||||
query: `
|
||||
(call
|
||||
function: (identifier) @func (#eq? @func "KafkaConsumer")
|
||||
arguments: (argument_list . (string) @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'kafka',
|
||||
confidence: 0.7,
|
||||
symbolName: 'producer.send',
|
||||
},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "producer")
|
||||
attribute: (identifier) @method (#match? @method "^(send|produce)$"))
|
||||
arguments: (argument_list . (string) @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.7,
|
||||
symbolName: 'basic_consume',
|
||||
},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "channel")
|
||||
attribute: (identifier) @method (#eq? @method "basic_consume"))
|
||||
arguments: (argument_list
|
||||
(keyword_argument
|
||||
name: (identifier) @kw (#eq? @kw "queue")
|
||||
value: (string) @value)))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'rabbitmq',
|
||||
confidence: 0.7,
|
||||
symbolName: 'basic_publish',
|
||||
},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "channel")
|
||||
attribute: (identifier) @method (#eq? @method "basic_publish"))
|
||||
arguments: (argument_list
|
||||
(keyword_argument
|
||||
name: (identifier) @kw (#eq? @kw "exchange")
|
||||
value: (string) @value)))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'consumer',
|
||||
broker: 'nats',
|
||||
confidence: 0.75,
|
||||
symbolName: 'nc.subscribe',
|
||||
},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "nc")
|
||||
attribute: (identifier) @method (#eq? @method "subscribe"))
|
||||
arguments: (argument_list . (string) @value))
|
||||
`,
|
||||
},
|
||||
{
|
||||
meta: {
|
||||
role: 'provider',
|
||||
broker: 'nats',
|
||||
confidence: 0.75,
|
||||
symbolName: 'nc.publish',
|
||||
},
|
||||
query: `
|
||||
(call
|
||||
function: (attribute
|
||||
object: (identifier) @obj (#eq? @obj "nc")
|
||||
attribute: (identifier) @method (#eq? @method "publish"))
|
||||
arguments: (argument_list . (string) @value))
|
||||
`,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const PYTHON_TOPIC_PROVIDER = compilePatterns(PYTHON_TOPIC_SPEC);
|
||||
27
gitnexus/src/core/group/extractors/topic-patterns/types.ts
Normal file
27
gitnexus/src/core/group/extractors/topic-patterns/types.ts
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
/**
|
||||
* Shared types for the topic-extractor language plugins.
|
||||
*
|
||||
* Each plugin lives in its own file (java.ts, go.ts, ...) and owns the
|
||||
* tree-sitter grammar import + query sources. The top-level
|
||||
* `topic-extractor.ts` orchestrator only knows about this type module and
|
||||
* the plugin registry (`./index.ts`). It MUST NOT import any grammar or
|
||||
* query text directly — that's the whole point of the split.
|
||||
*/
|
||||
|
||||
export type Broker = 'kafka' | 'rabbitmq' | 'nats';
|
||||
|
||||
/**
|
||||
* Per-pattern payload every topic plugin attaches to its query. Whatever
|
||||
* the pattern matches, the orchestrator receives this object verbatim
|
||||
* and uses it to build an `ExtractedContract`.
|
||||
*
|
||||
* Plugins produce one `TopicMeta` per pattern (not per match) because a
|
||||
* single query uniquely identifies its broker/role/confidence triple.
|
||||
*/
|
||||
export interface TopicMeta {
|
||||
role: 'provider' | 'consumer';
|
||||
broker: Broker;
|
||||
confidence: number;
|
||||
/** Short human-readable label of the API being detected. */
|
||||
symbolName: string;
|
||||
}
|
||||
193
gitnexus/src/core/group/extractors/tree-sitter-scanner.ts
Normal file
193
gitnexus/src/core/group/extractors/tree-sitter-scanner.ts
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
import Parser from 'tree-sitter';
|
||||
|
||||
/**
|
||||
* Shared, language-agnostic tree-sitter scanning utilities used by group
|
||||
* extractors (topic, http, grpc, ...).
|
||||
*
|
||||
* Design goals:
|
||||
* - The top-level extractors must not import any tree-sitter grammar.
|
||||
* - Per-language plugins own their grammar import, their query sources,
|
||||
* and the mapping from capture → meta.
|
||||
* - This module provides the plumbing: compile queries once per plugin,
|
||||
* parse a file with a given grammar, run all patterns, and return the
|
||||
* captured `string_literal`-style nodes together with the plugin's meta.
|
||||
*/
|
||||
|
||||
/**
|
||||
* One pattern owned by a language plugin. Each pattern owns a tree-sitter
|
||||
* S-expression query. Plugins can freely choose which capture names to
|
||||
* use — the scanner exposes every capture in the returned `captures`
|
||||
* map and does not privilege any particular name.
|
||||
*
|
||||
* `TMeta` is the plugin-specific payload the orchestrator receives back
|
||||
* when this pattern matches — e.g. for topic extraction it carries the
|
||||
* broker name, role, confidence, symbol name.
|
||||
*/
|
||||
export interface PatternSpec<TMeta> {
|
||||
/** Tree-sitter S-expression. */
|
||||
query: string;
|
||||
/** Plugin-specific payload returned on every match. */
|
||||
meta: TMeta;
|
||||
}
|
||||
|
||||
/**
|
||||
* A set of patterns owned by one language plugin, bound to a specific
|
||||
* tree-sitter grammar.
|
||||
*
|
||||
* `language` is typed as `unknown` because tree-sitter's TypeScript
|
||||
* declarations use `any` for the grammar object, and the grammar modules
|
||||
* export different shapes (plain grammar vs. namespace with `typescript`
|
||||
* / `tsx` members). Callers pass the concrete grammar object; this
|
||||
* module forwards it to `parser.setLanguage` / `new Parser.Query`.
|
||||
*/
|
||||
export interface LanguagePatterns<TMeta> {
|
||||
/** Human-readable plugin name for diagnostics. */
|
||||
name: string;
|
||||
/** tree-sitter grammar object. */
|
||||
language: unknown;
|
||||
/** Patterns authored against `language`. */
|
||||
patterns: PatternSpec<TMeta>[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compiled form of a `LanguagePatterns` bundle. Queries are compiled
|
||||
* eagerly at module load time so a broken grammar/query pair fails
|
||||
* loudly the first time the plugin is imported, instead of silently
|
||||
* at scan time when no contract is produced.
|
||||
*/
|
||||
export interface CompiledPatterns<TMeta> {
|
||||
name: string;
|
||||
language: unknown;
|
||||
patterns: CompiledPattern<TMeta>[];
|
||||
}
|
||||
|
||||
export interface CompiledPattern<TMeta> {
|
||||
query: Parser.Query;
|
||||
meta: TMeta;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map from capture name → syntax node. Every named capture the query
|
||||
* binds is exposed as an entry. If a query captures the same name more
|
||||
* than once (unusual), the first occurrence wins — plugins that need
|
||||
* all occurrences should use distinct capture names or fall back to
|
||||
* `match.captures` array directly by iterating `query.matches()`
|
||||
* themselves.
|
||||
*/
|
||||
export type CaptureMap = Record<string, Parser.SyntaxNode>;
|
||||
|
||||
/**
|
||||
* One match returned by `scanFile` / `runCompiledPatterns`. The caller
|
||||
* receives the full capture map plus the plugin meta, and is
|
||||
* responsible for turning it into a domain object.
|
||||
*/
|
||||
export interface ScanMatch<TMeta> {
|
||||
meta: TMeta;
|
||||
captures: CaptureMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile a LanguagePatterns bundle. Call this once per plugin, at
|
||||
* module load time, and export the result. Throws if any pattern
|
||||
* fails to compile against the grammar — that's a bug in the plugin
|
||||
* author's query, not a runtime condition.
|
||||
*/
|
||||
export function compilePatterns<TMeta>(bundle: LanguagePatterns<TMeta>): CompiledPatterns<TMeta> {
|
||||
const compiled: CompiledPattern<TMeta>[] = [];
|
||||
for (const spec of bundle.patterns) {
|
||||
try {
|
||||
const query = new Parser.Query(bundle.language, spec.query);
|
||||
compiled.push({ query, meta: spec.meta });
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(
|
||||
`[tree-sitter-scanner] Failed to compile pattern in ${bundle.name}: ${message}\n` +
|
||||
`Query source:\n${spec.query}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { name: bundle.name, language: bundle.language, patterns: compiled };
|
||||
}
|
||||
|
||||
/**
|
||||
* Run every compiled pattern in `plugin` against an already-parsed
|
||||
* tree. Use this when a plugin needs multiple query bundles against
|
||||
* the same file (e.g. one query for class-level prefixes and another
|
||||
* for method-level annotations) and wants to avoid re-parsing.
|
||||
*/
|
||||
export function runCompiledPatterns<TMeta>(
|
||||
plugin: CompiledPatterns<TMeta>,
|
||||
tree: Parser.Tree,
|
||||
): ScanMatch<TMeta>[] {
|
||||
const out: ScanMatch<TMeta>[] = [];
|
||||
for (const compiled of plugin.patterns) {
|
||||
let matches: Parser.QueryMatch[];
|
||||
try {
|
||||
matches = compiled.query.matches(tree.rootNode);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const match of matches) {
|
||||
const captures: CaptureMap = {};
|
||||
for (const cap of match.captures) {
|
||||
if (!(cap.name in captures)) captures[cap.name] = cap.node;
|
||||
}
|
||||
out.push({ meta: compiled.meta, captures });
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `content` with the plugin's grammar and run every compiled
|
||||
* pattern against the AST. Returns one `ScanMatch` per matched query
|
||||
* occurrence, carrying the plugin's meta payload.
|
||||
*
|
||||
* Errors are swallowed at the file level (malformed file must not abort
|
||||
* the whole extract). Individual pattern failures are swallowed too so
|
||||
* a single unusable query doesn't block the rest of the plugin.
|
||||
*/
|
||||
export function scanFile<TMeta>(
|
||||
parser: Parser,
|
||||
plugin: CompiledPatterns<TMeta>,
|
||||
content: string,
|
||||
): ScanMatch<TMeta>[] {
|
||||
let tree: Parser.Tree;
|
||||
try {
|
||||
parser.setLanguage(plugin.language);
|
||||
tree = parser.parse(content);
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
return runCompiledPatterns(plugin, tree);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip enclosing quotes from a tree-sitter string literal node's text.
|
||||
* Handles single / double / template quotes, Python triple-quoted strings,
|
||||
* and Go raw string literals (backticks).
|
||||
*
|
||||
* Returns null for empty/nullish input so callers can uniformly skip
|
||||
* captures whose value is missing.
|
||||
*/
|
||||
export function unquoteLiteral(raw: string): string | null {
|
||||
if (!raw) return null;
|
||||
|
||||
// Python triple-quoted
|
||||
if (
|
||||
(raw.startsWith('"""') && raw.endsWith('"""')) ||
|
||||
(raw.startsWith("'''") && raw.endsWith("'''"))
|
||||
) {
|
||||
return raw.slice(3, -3);
|
||||
}
|
||||
|
||||
const first = raw[0];
|
||||
const last = raw[raw.length - 1];
|
||||
if ((first === '"' || first === "'" || first === '`') && last === first && raw.length >= 2) {
|
||||
return raw.slice(1, -1);
|
||||
}
|
||||
|
||||
// Some grammars expose the string content without quotes already (e.g.
|
||||
// Python `string_content` child). Return as-is.
|
||||
return raw;
|
||||
}
|
||||
|
|
@ -5,6 +5,15 @@ export interface MatchResult {
|
|||
unmatched: StoredContract[];
|
||||
}
|
||||
|
||||
export interface WildcardMatchResult {
|
||||
matched: CrossLink[];
|
||||
remaining: StoredContract[];
|
||||
}
|
||||
|
||||
function isGrpcWildcard(cid: string): boolean {
|
||||
return cid.startsWith('grpc::') && cid.endsWith('/*');
|
||||
}
|
||||
|
||||
export function normalizeContractId(id: string): string {
|
||||
const colonIdx = id.indexOf('::');
|
||||
if (colonIdx === -1) return id;
|
||||
|
|
@ -24,6 +33,22 @@ export function normalizeContractId(id: string): string {
|
|||
return id;
|
||||
}
|
||||
case 'grpc': {
|
||||
// Canonical form: `grpc::<lowercased-package-or-service>[/<method>]`.
|
||||
//
|
||||
// The package/service segment is lowercased because gRPC package
|
||||
// names are effectively case-insensitive across language bindings
|
||||
// (`auth.AuthService`, `auth.authservice`, `AUTH.AUTHSERVICE` all
|
||||
// describe the same wire protocol service). The RPC method segment
|
||||
// is preserved as-is because the HTTP/2 path used on the wire is
|
||||
// case-sensitive per the gRPC spec (`/Service/MethodName`), and
|
||||
// method names in generated clients match the proto source exactly.
|
||||
//
|
||||
// A package-only id (no slash) and a package/method id are treated
|
||||
// as DISTINCT canonical forms: `grpc::userservice` does not match
|
||||
// `grpc::userservice/Login`. That's by design — callers that want
|
||||
// service-level manifest matching against method-level providers
|
||||
// should use the gRPC wildcard form `grpc::UserService/*` which is
|
||||
// handled by runWildcardMatch below.
|
||||
const slashIdx = rest.indexOf('/');
|
||||
if (slashIdx > 0) {
|
||||
const pkg = rest.substring(0, slashIdx).toLowerCase();
|
||||
|
|
@ -31,12 +56,12 @@ export function normalizeContractId(id: string): string {
|
|||
return `grpc::${pkg}${method}`;
|
||||
}
|
||||
if (slashIdx === 0) {
|
||||
// Malformed "package/method" with leading slash — do not lowercase the whole string
|
||||
// (method segment is case-sensitive per spec).
|
||||
// Malformed "/method" with leading slash — keep as-is so two
|
||||
// equally malformed ids can still match each other.
|
||||
return `grpc::${rest}`;
|
||||
}
|
||||
// No slash: spec is ambiguous (package-only vs full service.method). MVP: lowercase
|
||||
// the whole token; differs from pkg/method split above where RPC method keeps case.
|
||||
// No slash: package/service only. Lowercase to match the package
|
||||
// segment produced by the pkg/method branch above.
|
||||
return `grpc::${rest.toLowerCase()}`;
|
||||
}
|
||||
case 'topic':
|
||||
|
|
@ -66,27 +91,36 @@ function findMatchingKeys(contractId: string, index: Map<string, StoredContract[
|
|||
return [];
|
||||
}
|
||||
|
||||
export function runExactMatch(contracts: StoredContract[]): MatchResult {
|
||||
export function buildProviderIndex(contracts: StoredContract[]): Map<string, StoredContract[]> {
|
||||
const providers = contracts.filter((c) => c.role === 'provider');
|
||||
const consumers = contracts.filter((c) => c.role === 'consumer');
|
||||
|
||||
const providerIndex = new Map<string, StoredContract[]>();
|
||||
const index = new Map<string, StoredContract[]>();
|
||||
for (const p of providers) {
|
||||
const key = normalizeContractId(p.contractId);
|
||||
const list = providerIndex.get(key) || [];
|
||||
const list = index.get(key) || [];
|
||||
list.push(p);
|
||||
providerIndex.set(key, list);
|
||||
index.set(key, list);
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
export function runExactMatch(
|
||||
contracts: StoredContract[],
|
||||
providerIndex?: Map<string, StoredContract[]>,
|
||||
): MatchResult {
|
||||
const index = providerIndex ?? buildProviderIndex(contracts);
|
||||
|
||||
// Skip gRPC wildcard consumers — they go to wildcard pass only
|
||||
const consumers = contracts.filter((c) => c.role === 'consumer' && !isGrpcWildcard(c.contractId));
|
||||
|
||||
const matched: CrossLink[] = [];
|
||||
const matchedConsumerIds = new Set<string>();
|
||||
const matchedProviderIds = new Set<string>();
|
||||
|
||||
for (const consumer of consumers) {
|
||||
const matchingKeys = findMatchingKeys(consumer.contractId, providerIndex);
|
||||
const matchingKeys = findMatchingKeys(consumer.contractId, index);
|
||||
if (matchingKeys.length === 0) continue;
|
||||
|
||||
const allMatchingProviders = matchingKeys.flatMap((k) => providerIndex.get(k) || []);
|
||||
const allMatchingProviders = matchingKeys.flatMap((k) => index.get(k) || []);
|
||||
for (const provider of allMatchingProviders) {
|
||||
if (provider.repo === consumer.repo) {
|
||||
if (!provider.service || !consumer.service || provider.service === consumer.service) {
|
||||
|
|
@ -118,10 +152,86 @@ export function runExactMatch(contracts: StoredContract[]): MatchResult {
|
|||
}
|
||||
}
|
||||
|
||||
const unmatched = contracts.filter((c) => {
|
||||
// normalUnmatched: contracts that weren't matched in exact pass
|
||||
const normalUnmatched = contracts.filter((c) => {
|
||||
if (isGrpcWildcard(c.contractId)) return false; // excluded from exact, handled separately
|
||||
const id = `${c.repo}::${c.contractId}`;
|
||||
return c.role === 'provider' ? !matchedProviderIds.has(id) : !matchedConsumerIds.has(id);
|
||||
});
|
||||
|
||||
// Re-add gRPC wildcard contracts — they were never in exact matching
|
||||
const grpcWildcards = contracts.filter((c) => isGrpcWildcard(c.contractId));
|
||||
const unmatched = [...normalUnmatched, ...grpcWildcards];
|
||||
|
||||
return { matched, unmatched };
|
||||
}
|
||||
|
||||
export function runWildcardMatch(
|
||||
unmatched: StoredContract[],
|
||||
providerIndex: Map<string, StoredContract[]>,
|
||||
): WildcardMatchResult {
|
||||
const wildcardConsumers = unmatched.filter(
|
||||
(c) => c.role === 'consumer' && isGrpcWildcard(c.contractId),
|
||||
);
|
||||
const matched: CrossLink[] = [];
|
||||
const matchedConsumerIds = new Set<string>();
|
||||
|
||||
for (const consumer of wildcardConsumers) {
|
||||
const normalized = normalizeContractId(consumer.contractId);
|
||||
// "grpc::com.example.userservice/*" → "com.example.userservice"
|
||||
// "grpc::userservice/*" → "userservice"
|
||||
const fqService = normalized.slice(normalized.indexOf('::') + 2, -2); // strip "grpc::" and "/*"
|
||||
|
||||
for (const [key, providers] of providerIndex) {
|
||||
// Only match against non-wildcard gRPC providers (method-level IDs)
|
||||
if (!key.startsWith('grpc::') || key.endsWith('/*')) continue;
|
||||
const afterPrefix = key.slice(6); // strip "grpc::"
|
||||
const slashIdx = afterPrefix.indexOf('/');
|
||||
if (slashIdx < 0) continue;
|
||||
const providerFqService = afterPrefix.slice(0, slashIdx);
|
||||
|
||||
// Match: exact FQ service, or bare-name match when consumer has no package
|
||||
const isMatch =
|
||||
providerFqService === fqService ||
|
||||
(!fqService.includes('.') && providerFqService.endsWith('.' + fqService));
|
||||
|
||||
if (!isMatch) continue;
|
||||
|
||||
for (const provider of providers) {
|
||||
// Skip same-repo same-service (same logic as runExactMatch)
|
||||
if (provider.repo === consumer.repo) {
|
||||
if (!provider.service || !consumer.service || provider.service === consumer.service) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
matched.push({
|
||||
from: {
|
||||
repo: consumer.repo,
|
||||
service: consumer.service,
|
||||
symbolUid: consumer.symbolUid,
|
||||
symbolRef: consumer.symbolRef,
|
||||
},
|
||||
to: {
|
||||
repo: provider.repo,
|
||||
service: provider.service,
|
||||
symbolUid: provider.symbolUid,
|
||||
symbolRef: provider.symbolRef,
|
||||
},
|
||||
type: consumer.type,
|
||||
contractId: consumer.contractId, // consumer's wildcard ID
|
||||
matchType: 'wildcard',
|
||||
confidence: Math.min(provider.confidence, consumer.confidence),
|
||||
});
|
||||
matchedConsumerIds.add(`${consumer.repo}::${consumer.contractId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const remaining = unmatched.filter((c) => {
|
||||
if (c.role !== 'consumer' || !isGrpcWildcard(c.contractId)) return true;
|
||||
return !matchedConsumerIds.has(`${c.repo}::${c.contractId}`);
|
||||
});
|
||||
|
||||
return { matched, remaining };
|
||||
}
|
||||
|
|
|
|||
124
gitnexus/src/core/group/normalization.ts
Normal file
124
gitnexus/src/core/group/normalization.ts
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import type { CrossLink, CrossLinkEndpoint, StoredContract } from './types.js';
|
||||
|
||||
function contractKey(contract: StoredContract): string {
|
||||
return [contract.repo, contract.contractId, contract.role, contract.symbolRef.filePath].join(
|
||||
'\0',
|
||||
);
|
||||
}
|
||||
|
||||
function endpointKey(endpoint: CrossLinkEndpoint): string {
|
||||
return [
|
||||
endpoint.repo,
|
||||
endpoint.service ?? '',
|
||||
endpoint.symbolRef.filePath,
|
||||
endpoint.symbolRef.name,
|
||||
].join('\0');
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a contract by how much information it carries, so `dedupeContracts`
|
||||
* can prefer the "richer" record when two contracts collide on the same
|
||||
* `(repo, contractId, role, filePath)` key.
|
||||
*
|
||||
* Weights express a priority ordering, not calibrated probabilities:
|
||||
* +3 — `symbolUid` resolved (tier 1 of the downstream lookup — highest
|
||||
* signal because it's the strongest anchor for cross-impact traversal
|
||||
* and the only one that's robust to renames)
|
||||
* +2 — any of `filePath`, `symbolRef.name`, or `symbolName` that's more
|
||||
* specific than the contractId itself (tier 2 signal — resolves
|
||||
* uniquely in most cases and survives across syncs)
|
||||
* +1 — `service` tag (monorepo attribution — useful but not sufficient
|
||||
* on its own) or non-manifest origin (auto-extracted contracts are
|
||||
* preferred over manifest-declared synthetic ones because the former
|
||||
* are grounded in real source code)
|
||||
*
|
||||
* The absolute numbers don't matter, only their relative ordering.
|
||||
*/
|
||||
function contractRichness(contract: StoredContract): number {
|
||||
let score = 0;
|
||||
if (contract.symbolUid) score += 3;
|
||||
if (contract.symbolRef.filePath) score += 2;
|
||||
if (contract.symbolRef.name && contract.symbolRef.name !== contract.contractId) score += 2;
|
||||
if (contract.symbolName && contract.symbolName !== contract.contractId) score += 2;
|
||||
if (contract.service) score += 1;
|
||||
if (contract.meta.source !== 'manifest') score += 1;
|
||||
return score;
|
||||
}
|
||||
|
||||
function mergeContracts(existing: StoredContract, incoming: StoredContract): StoredContract {
|
||||
const [primary, secondary] =
|
||||
contractRichness(incoming) > contractRichness(existing)
|
||||
? [incoming, existing]
|
||||
: [existing, incoming];
|
||||
const symbolRefName = primary.symbolRef.name || secondary.symbolRef.name;
|
||||
return {
|
||||
...secondary,
|
||||
...primary,
|
||||
symbolUid: primary.symbolUid || secondary.symbolUid,
|
||||
symbolRef: {
|
||||
filePath: primary.symbolRef.filePath || secondary.symbolRef.filePath,
|
||||
name: symbolRefName,
|
||||
},
|
||||
symbolName: primary.symbolName || secondary.symbolName || symbolRefName,
|
||||
confidence: Math.max(existing.confidence, incoming.confidence),
|
||||
service: primary.service ?? secondary.service,
|
||||
meta: { ...secondary.meta, ...primary.meta },
|
||||
};
|
||||
}
|
||||
|
||||
function mergeEndpoints(
|
||||
existing: CrossLinkEndpoint,
|
||||
incoming: CrossLinkEndpoint,
|
||||
): CrossLinkEndpoint {
|
||||
return {
|
||||
repo: existing.repo,
|
||||
service: existing.service ?? incoming.service,
|
||||
symbolUid: existing.symbolUid || incoming.symbolUid,
|
||||
symbolRef: {
|
||||
filePath: existing.symbolRef.filePath || incoming.symbolRef.filePath,
|
||||
name: existing.symbolRef.name || incoming.symbolRef.name,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function crossLinkKey(link: CrossLink): string {
|
||||
return [
|
||||
link.type,
|
||||
link.contractId,
|
||||
link.matchType,
|
||||
endpointKey(link.from),
|
||||
endpointKey(link.to),
|
||||
].join('\0');
|
||||
}
|
||||
|
||||
export function dedupeContracts(items: StoredContract[]): StoredContract[] {
|
||||
const deduped = new Map<string, StoredContract>();
|
||||
for (const contract of items) {
|
||||
const key = contractKey(contract);
|
||||
const existing = deduped.get(key);
|
||||
deduped.set(key, existing ? mergeContracts(existing, contract) : contract);
|
||||
}
|
||||
return [...deduped.values()];
|
||||
}
|
||||
|
||||
export function dedupeCrossLinks(items: CrossLink[]): CrossLink[] {
|
||||
const deduped = new Map<string, CrossLink>();
|
||||
for (const link of items) {
|
||||
const key = crossLinkKey(link);
|
||||
const existing = deduped.get(key);
|
||||
if (!existing) {
|
||||
deduped.set(key, link);
|
||||
continue;
|
||||
}
|
||||
const keepIncoming = link.confidence > existing.confidence;
|
||||
const primary = keepIncoming ? link : existing;
|
||||
const secondary = keepIncoming ? existing : link;
|
||||
deduped.set(key, {
|
||||
...primary,
|
||||
confidence: Math.max(existing.confidence, link.confidence),
|
||||
from: mergeEndpoints(primary.from, secondary.from),
|
||||
to: mergeEndpoints(primary.to, secondary.to),
|
||||
});
|
||||
}
|
||||
return [...deduped.values()];
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ import type { GroupConfig, RepoHandle, RepoSnapshot, StoredContract, CrossLink }
|
|||
import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
|
||||
import { GrpcExtractor } from './extractors/grpc-extractor.js';
|
||||
import { TopicExtractor } from './extractors/topic-extractor.js';
|
||||
import { ManifestExtractor } from './extractors/manifest-extractor.js';
|
||||
import { runExactMatch } from './matching.js';
|
||||
import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
|
||||
import type { CypherExecutor } from './contract-extractor.js';
|
||||
|
|
@ -60,10 +61,28 @@ function defaultResolveHandle(allEntries: RegistryEntry[]) {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Dedupe cross-links that point from the same consumer endpoint to the same
|
||||
* provider endpoint for the same contract. Preserves first-seen order so the
|
||||
* caller controls precedence (e.g., pass manifest links first).
|
||||
*/
|
||||
function dedupeCrossLinks(links: CrossLink[]): CrossLink[] {
|
||||
const seen = new Set<string>();
|
||||
const out: CrossLink[] = [];
|
||||
for (const link of links) {
|
||||
const key = `${link.from.repo}::${link.from.symbolUid}|${link.to.repo}::${link.to.symbolUid}|${link.type}|${link.contractId}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(link);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promise<SyncResult> {
|
||||
const missingRepos: string[] = [];
|
||||
const repoSnapshots: Record<string, RepoSnapshot> = {};
|
||||
let autoContracts: StoredContract[] = [];
|
||||
let manifestCrossLinks: CrossLink[] = [];
|
||||
let dbExecutors: Map<string, CypherExecutor> | undefined;
|
||||
|
||||
const eo = opts?.extractorOverride;
|
||||
|
|
@ -158,8 +177,44 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
|||
}
|
||||
}
|
||||
|
||||
// Process manifest links declared in group.yaml.
|
||||
// ManifestExtractor is fully implemented but was never wired into this
|
||||
// pipeline — config.links were parsed and validated but silently dropped.
|
||||
// Placed after the DB try/finally: resolveSymbol falls back to synthetic
|
||||
// UIDs when dbExecutors is undefined or a pool is closed, so cross-links
|
||||
// are always generated regardless of whether real DB executors are available.
|
||||
if (config.links.length > 0) {
|
||||
// Warn about dangling links that reference repos not declared in config.repos.
|
||||
// They still generate cross-links via synthetic UIDs (determinism is preserved),
|
||||
// but the operator probably meant something that now silently does nothing useful.
|
||||
const knownRepos = new Set(Object.keys(config.repos));
|
||||
for (const link of config.links) {
|
||||
const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
|
||||
if (dangling.length > 0) {
|
||||
console.warn(
|
||||
`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const manifestEx = new ManifestExtractor();
|
||||
const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors);
|
||||
autoContracts.push(...manifestResult.contracts);
|
||||
manifestCrossLinks = manifestResult.crossLinks;
|
||||
if (opts?.verbose) {
|
||||
console.log(
|
||||
` manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const { matched, unmatched } = runExactMatch(autoContracts);
|
||||
const crossLinks: CrossLink[] = matched;
|
||||
|
||||
// Dedupe cross-links. Manifest contracts participate in runExactMatch, so a
|
||||
// manifest-declared link can also emit a matchType:'exact' CrossLink with the
|
||||
// same endpoints. Prefer the manifest version — it reflects operator intent
|
||||
// and carries matchType:'manifest' which downstream consumers may rely on.
|
||||
const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]);
|
||||
const allContracts: StoredContract[] = autoContracts;
|
||||
|
||||
const registry: ContractRegistry = {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
export type ContractType = 'http' | 'grpc' | 'topic' | 'lib' | 'custom';
|
||||
export type MatchType = 'exact' | 'manifest' | 'bm25' | 'embedding';
|
||||
export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding';
|
||||
export type ContractRole = 'provider' | 'consumer';
|
||||
|
||||
export interface GroupConfig {
|
||||
|
|
@ -131,3 +131,17 @@ export interface OutOfScopeLink {
|
|||
contractId: string;
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/** Opaque handle to an open bridge LadybugDB. */
|
||||
export interface BridgeHandle {
|
||||
/** Internal — do not access directly. */
|
||||
readonly _db: unknown;
|
||||
readonly _conn: unknown;
|
||||
readonly groupDir: string;
|
||||
}
|
||||
|
||||
export interface BridgeMeta {
|
||||
version: number;
|
||||
generatedAt: string;
|
||||
missingRepos: string[];
|
||||
}
|
||||
|
|
|
|||
391
gitnexus/src/core/ingestion/binding-accumulator.ts
Normal file
391
gitnexus/src/core/ingestion/binding-accumulator.ts
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
/**
|
||||
* BindingAccumulator — read-append-only accumulator that collects TypeEnv
|
||||
* bindings across files in the GitNexus analyzer pipeline.
|
||||
*
|
||||
* **Current behavior (both execution paths):** The accumulator carries only
|
||||
* file-scope (`scope = ''`) entries. Function-scope bindings are stripped
|
||||
* at both write sites:
|
||||
*
|
||||
* - **Worker path**: `parse-worker.ts` serializes only
|
||||
* `typeEnv.fileScope()` entries across the IPC boundary.
|
||||
* - **Sequential path**: `type-env.ts::flush()` iterates only the FILE_SCOPE
|
||||
* entry of the env map and writes `BindingEntry` records with
|
||||
* `scope: ''` hardcoded.
|
||||
*
|
||||
* The narrowing exists because function-scope bindings have zero downstream
|
||||
* consumers today and were previously costing ~4.9 MB of heap + IPC on
|
||||
* every pipeline run. See `type-env.ts::flush()` and the `FileScopeBindings`
|
||||
* JSDoc in `parse-worker.ts` for the paired Phase 9 reversion checklist.
|
||||
*
|
||||
* **Historical quality asymmetry (Phase 9 consideration):** Even though
|
||||
* both paths now carry only file-scope data, the two paths were built
|
||||
* under different resolution capabilities, and a future Phase 9 reverter
|
||||
* that widens them back to all scopes will inherit that asymmetry:
|
||||
*
|
||||
* - **Sequential path** had (and would regain) access to the full
|
||||
* `SymbolTable` and `importedBindings`, so its bindings benefit from
|
||||
* Tier 2 cross-file propagation.
|
||||
* - **Worker path** runs without `SymbolTable` / `importedBindings` and
|
||||
* can only produce Tier 0 (annotation-declared) and local Tier 1
|
||||
* (same-file constructor inference) bindings.
|
||||
*
|
||||
* Phase 9 consumers that trust every entry equally will silently produce
|
||||
* worse results for large repos (worker-dominant) than small ones
|
||||
* (sequential-dominant). If Phase 9 needs homogeneous quality, either
|
||||
* (a) tag entries with their tier at insert time so consumers can filter,
|
||||
* or (b) post-process worker-path entries through a follow-up resolution
|
||||
* pass after the main-thread `SymbolTable` is complete.
|
||||
*
|
||||
* **Lifecycle contract**: single-use — `append* → finalize → consume → dispose`.
|
||||
* After `dispose()` the accumulator is permanently dead: any mutating call
|
||||
* (`appendFile`) throws, and read methods return empty/undefined as if the
|
||||
* accumulator had never been appended to. The instance is not recyclable;
|
||||
* construct a new one for a new pipeline run. Finalization and disposal are
|
||||
* orthogonal state dimensions and may be invoked in either order.
|
||||
*/
|
||||
|
||||
export interface BindingEntry {
|
||||
readonly scope: string; // '' for file-level, 'funcName@startIndex' for function-local
|
||||
readonly varName: string;
|
||||
readonly typeName: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal graph-node shape required by `enrichExportedTypeMap()`. Intentionally
|
||||
* narrower than the full `GraphNode` type in `graph/types.ts` so tests can
|
||||
* construct a minimal mock without depending on the full graph module, and
|
||||
* so the enrichment logic is a pure function over this contract.
|
||||
*
|
||||
* Matches the shape of the real `KnowledgeGraph` node's `properties.isExported`
|
||||
* access path — tests that use a different shape silently pass while
|
||||
* production fails.
|
||||
*/
|
||||
export interface EnrichmentGraphNode {
|
||||
readonly id: string;
|
||||
readonly properties?: { readonly isExported?: boolean } | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Minimal graph lookup interface used by `enrichExportedTypeMap()`.
|
||||
* Consumes only the method the enrichment loop actually calls.
|
||||
*/
|
||||
export interface EnrichmentGraphLookup {
|
||||
getNode(id: string): EnrichmentGraphNode | undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge file-scope bindings from a (finalized) `BindingAccumulator` into an
|
||||
* `exportedTypeMap` for symbols whose graph nodes are marked as exported.
|
||||
*
|
||||
* This is the single source of truth for the worker-path ExportedTypeMap
|
||||
* enrichment loop. Previously the logic lived inline in `pipeline.ts` and
|
||||
* the test suite reimplemented it as a `runEnrichmentLoop` helper — a
|
||||
* drift-prone pattern that meant tests could pass while production regressed.
|
||||
* Extracting it here makes the production code call the same function the
|
||||
* tests call.
|
||||
*
|
||||
* **Node ID candidate order**: `Function:{filePath}:{name}` →
|
||||
* `Variable:{filePath}:{name}` → `Const:{filePath}:{name}`. First match wins.
|
||||
*
|
||||
* **Tier 0 priority**: if `exportedTypeMap` already has an entry for a
|
||||
* `(filePath, name)` pair, the accumulator entry does NOT overwrite it —
|
||||
* the SymbolTable tier-0 pass is authoritative. Without this guard, a
|
||||
* worker-path binding could clobber a higher-quality type from SymbolTable.
|
||||
*
|
||||
* **Finalize precondition**: the accumulator should be finalized before
|
||||
* calling this function. The lifecycle contract is
|
||||
* `append → finalize → enrich → dispose`. Finalization is not asserted
|
||||
* here (the test suite and pipeline both honor it separately), but any
|
||||
* append happening concurrently with this enrichment would be a lifecycle
|
||||
* bug at the caller level.
|
||||
*
|
||||
* @returns The number of new entries written into `exportedTypeMap`
|
||||
* (0 on empty accumulator or when every candidate was filtered
|
||||
* out by the export check or the Tier 0 guard).
|
||||
*/
|
||||
export function enrichExportedTypeMap(
|
||||
bindingAccumulator: BindingAccumulator,
|
||||
graph: EnrichmentGraphLookup,
|
||||
exportedTypeMap: Map<string, Map<string, string>>,
|
||||
): number {
|
||||
if (bindingAccumulator.fileCount === 0) return 0;
|
||||
let enriched = 0;
|
||||
for (const filePath of bindingAccumulator.files()) {
|
||||
for (const [name, type] of bindingAccumulator.fileScopeEntries(filePath)) {
|
||||
// Three-candidate-ID lookup mirrors the sequential-path export check
|
||||
// in `collectExportedBindings()` (call-processor.ts).
|
||||
const functionNodeId = `Function:${filePath}:${name}`;
|
||||
const variableNodeId = `Variable:${filePath}:${name}`;
|
||||
const constNodeId = `Const:${filePath}:${name}`;
|
||||
const node =
|
||||
graph.getNode(functionNodeId) ??
|
||||
graph.getNode(variableNodeId) ??
|
||||
graph.getNode(constNodeId);
|
||||
if (!node?.properties?.isExported) continue;
|
||||
|
||||
let fileExports = exportedTypeMap.get(filePath);
|
||||
if (!fileExports) {
|
||||
fileExports = new Map();
|
||||
exportedTypeMap.set(filePath, fileExports);
|
||||
}
|
||||
// Tier 0 priority: SymbolTable-populated entries are authoritative.
|
||||
if (!fileExports.has(name)) {
|
||||
fileExports.set(name, type);
|
||||
enriched++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return enriched;
|
||||
}
|
||||
|
||||
const ENTRY_OVERHEAD = 64; // bytes per entry (object overhead + property refs)
|
||||
const MAP_ENTRY_OVERHEAD = 80; // bytes per file entry in the map
|
||||
|
||||
export class BindingAccumulator {
|
||||
// Storage is split into two parallel maps so file-scope reads are fast.
|
||||
// - _allByFile holds every BindingEntry (used by getFile, memory estimate).
|
||||
// - _fileScopeByFile is a nested Map<filePath, Map<varName, typeName>> for
|
||||
// O(1) point-lookup via fileScopeGet(). For iteration-based consumers
|
||||
// (enrichExportedTypeMap), fileScopeEntries() iterates the inner Map.
|
||||
// Both maps carry the same key set modulo the `scope === ''` precondition:
|
||||
// _allByFile has a key as soon as any entry is appended; _fileScopeByFile
|
||||
// only has a key once a file-scope entry arrives. Code that iterates via
|
||||
// files() uses _allByFile so files with only function-scope entries
|
||||
// remain visible.
|
||||
//
|
||||
// Note: Map.set semantics mean a duplicate varName for the same file
|
||||
// overwrites the previous value (last-write-wins). This is the correct
|
||||
// behavior — duplicate top-level bindings in the same file shouldn't
|
||||
// happen in well-formed source, and if they do the last declaration
|
||||
// is typically the one the compiler sees.
|
||||
private readonly _allByFile = new Map<string, BindingEntry[]>();
|
||||
private readonly _fileScopeByFile = new Map<string, Map<string, string>>();
|
||||
private _totalBindings = 0;
|
||||
private _finalized = false;
|
||||
private _disposed = false;
|
||||
|
||||
/**
|
||||
* Append bindings for a file. Safe to call multiple times for the same file.
|
||||
* Throws if the accumulator has been finalized. Skips if entries is empty.
|
||||
*
|
||||
* The `entries` parameter is `readonly` — this method never mutates the
|
||||
* caller's array. Internally, the first `appendFile` call per filePath
|
||||
* makes a defensive copy (`slice()`), and subsequent calls push into the
|
||||
* accumulator's own storage.
|
||||
*/
|
||||
appendFile(filePath: string, entries: readonly BindingEntry[]): void {
|
||||
if (this._finalized) {
|
||||
throw new Error(
|
||||
'[BindingAccumulator] appendFile after finalize — no further appends allowed',
|
||||
);
|
||||
}
|
||||
// Single-use lifecycle: once disposed, the accumulator is dead. A
|
||||
// post-dispose append almost always indicates a missed wiring step
|
||||
// (the consumer is reading state that was supposed to be released),
|
||||
// so convert the silent use-after-dispose into a loud failure.
|
||||
if (this._disposed) {
|
||||
throw new Error('BindingAccumulator: use after dispose');
|
||||
}
|
||||
if (entries.length === 0) {
|
||||
return;
|
||||
}
|
||||
// Note on the file-scope-only invariant:
|
||||
// The accumulator does NOT reject function-scope entries at this
|
||||
// boundary. The narrowing contract is enforced by the two production
|
||||
// write sites — `parse-worker.ts` (which uses `typeEnv.fileScope()`
|
||||
// and hardcodes `scope: ''` in the pipeline adapter) and
|
||||
// `type-env.ts::flush()` (which iterates only `env.get(FILE_SCOPE)`).
|
||||
// The class JSDoc documents the invariant and the Phase 9 reversion
|
||||
// path. Making `appendFile` runtime-reject non-file-scope entries
|
||||
// would break the accumulator's own storage-split tests which
|
||||
// legitimately exercise mixed-scope entries. If a future write path
|
||||
// violates the invariant, tests should fail via missing exports in
|
||||
// the enrichment loop, not via an assertion here.
|
||||
// All-scope store.
|
||||
const existingAll = this._allByFile.get(filePath);
|
||||
if (existingAll !== undefined) {
|
||||
for (const e of entries) {
|
||||
existingAll.push(e);
|
||||
}
|
||||
} else {
|
||||
this._allByFile.set(filePath, entries.slice());
|
||||
}
|
||||
// File-scope fast-path store (nested Map for O(1) point-lookup via fileScopeGet).
|
||||
// Populated lazily on first file-scope entry per file.
|
||||
let fileScopeMap = this._fileScopeByFile.get(filePath);
|
||||
for (const e of entries) {
|
||||
if (e.scope === '') {
|
||||
if (fileScopeMap === undefined) {
|
||||
fileScopeMap = new Map();
|
||||
this._fileScopeByFile.set(filePath, fileScopeMap);
|
||||
}
|
||||
fileScopeMap.set(e.varName, e.typeName);
|
||||
}
|
||||
}
|
||||
this._totalBindings += entries.length;
|
||||
}
|
||||
|
||||
/** Lock the accumulator — no further appends. Idempotent. */
|
||||
finalize(): void {
|
||||
// Dev-mode invariant: verify the parallel storage split is consistent.
|
||||
// `_fileScopeByFile` must be a proper projection of `_allByFile`
|
||||
// where the outer key is a subset and the inner entries are exactly
|
||||
// the `scope === ''` subset of `_allByFile[key]`. A drift would
|
||||
// indicate a bug in `appendFile()` where one map was updated but
|
||||
// not the other.
|
||||
if (process.env.NODE_ENV !== 'production' && !this._finalized) {
|
||||
for (const [filePath, fileScopeMap] of this._fileScopeByFile) {
|
||||
const allEntries = this._allByFile.get(filePath);
|
||||
if (allEntries === undefined) {
|
||||
throw new Error(
|
||||
`[BindingAccumulator] storage split drift: file ${filePath} has file-scope entries ` +
|
||||
`but no _allByFile entry`,
|
||||
);
|
||||
}
|
||||
// Count unique file-scope varNames in _allByFile (to match Map dedup
|
||||
// semantics in _fileScopeByFile where Map.set deduplicates same-name).
|
||||
const projectedNames = new Set(
|
||||
allEntries.filter((e) => e.scope === '').map((e) => e.varName),
|
||||
);
|
||||
if (projectedNames.size !== fileScopeMap.size) {
|
||||
throw new Error(
|
||||
`[BindingAccumulator] storage split drift: file ${filePath} has ` +
|
||||
`${fileScopeMap.size} file-scope names in Map but ${projectedNames.size} unique ` +
|
||||
`file-scope varNames in _allByFile`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
this._finalized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the accumulator's heap footprint. Clears both internal storage
|
||||
* maps and resets `_totalBindings` to zero. Idempotent — calling twice
|
||||
* is a no-op. Orthogonal to `finalize()` — calling `dispose()` does not
|
||||
* change the finalized state.
|
||||
*
|
||||
* **Single-use lifecycle.** This is a one-way terminal transition: the
|
||||
* accumulator is not recyclable. Any subsequent `appendFile` call throws
|
||||
* (`'BindingAccumulator: use after dispose'`), regardless of whether
|
||||
* `finalize()` was called first. Post-dispose reads do not throw —
|
||||
* they return empty/undefined state matching a never-appended-to
|
||||
* accumulator:
|
||||
* - `fileCount === 0`
|
||||
* - `totalBindings === 0`
|
||||
* - `files()` yields an empty iterator
|
||||
* - `getFile(x)` returns `undefined` for all `x`
|
||||
* - `fileScopeEntries(x)` returns `[]` for all `x`
|
||||
* - `fileScopeGet(x, y)` returns `undefined` for all `x, y`
|
||||
* - `estimateMemoryBytes()` returns `0`
|
||||
*
|
||||
* Lifecycle note: the pipeline disposes the accumulator inside the
|
||||
* `finally` of the `crossFile` phase, which is scheduled after every
|
||||
* other accumulator consumer (Phase 9 call/assignment processing and
|
||||
* the ExportedTypeMap enrichment loop). The dispose call therefore
|
||||
* runs once, on both the happy path and the throw path of the
|
||||
* crossFile phase.
|
||||
*/
|
||||
dispose(): void {
|
||||
this._allByFile.clear();
|
||||
this._fileScopeByFile.clear();
|
||||
this._totalBindings = 0;
|
||||
this._disposed = true;
|
||||
}
|
||||
|
||||
/** Get all bindings for a file, or undefined if the file is unknown. */
|
||||
getFile(filePath: string): readonly BindingEntry[] | undefined {
|
||||
return this._allByFile.get(filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get only scope='' (file-level) entries as [varName, typeName] tuples.
|
||||
* For iteration-based consumers (e.g., `enrichExportedTypeMap`).
|
||||
* Returns an empty array for an unknown file.
|
||||
*
|
||||
* O(1) map lookup + O(n_file_scope) tuple reconstruction from the inner
|
||||
* Map. Does NOT walk function-scope entries.
|
||||
*
|
||||
* For point-lookup consumers (e.g., Phase 9 fallback), prefer
|
||||
* `fileScopeGet(filePath, name)` — O(1) with no allocation.
|
||||
*/
|
||||
fileScopeEntries(filePath: string): readonly (readonly [string, string])[] {
|
||||
const map = this._fileScopeByFile.get(filePath);
|
||||
return map ? [...map.entries()] : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* O(1) point-lookup for a single file-scope binding by (filePath, name).
|
||||
* Returns the typeName if found, `undefined` otherwise.
|
||||
*
|
||||
* This is the preferred lookup path for Phase 9 consumers that resolve
|
||||
* a single callee's return type — avoids the O(n_file_scope) iteration
|
||||
* and defensive-copy allocation of `fileScopeEntries()`.
|
||||
*/
|
||||
fileScopeGet(filePath: string, name: string): string | undefined {
|
||||
return this._fileScopeByFile.get(filePath)?.get(name);
|
||||
}
|
||||
|
||||
/** Iterate over all file paths in insertion order. */
|
||||
files(): IterableIterator<string> {
|
||||
return this._allByFile.keys();
|
||||
}
|
||||
|
||||
/** Number of distinct files with at least one binding. */
|
||||
get fileCount(): number {
|
||||
return this._allByFile.size;
|
||||
}
|
||||
|
||||
/** Total number of binding entries across all files. */
|
||||
get totalBindings(): number {
|
||||
return this._totalBindings;
|
||||
}
|
||||
|
||||
/** Whether the accumulator has been finalized. */
|
||||
get finalized(): boolean {
|
||||
return this._finalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the accumulator has been disposed. Exposed for symmetry with
|
||||
* `finalized` so debug tooling and future Phase 9 consumers can detect a
|
||||
* disposed accumulator without inspecting empty state heuristically.
|
||||
*
|
||||
* Disposal and finalization are orthogonal: a disposed accumulator may or
|
||||
* may not be finalized, and vice versa. See `dispose()` for the full
|
||||
* lifecycle contract.
|
||||
*/
|
||||
get disposed(): boolean {
|
||||
return this._disposed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rough memory estimate in bytes (intentionally pessimistic).
|
||||
* Formula: sum of (ENTRY_OVERHEAD + char bytes of scope+varName+typeName) per entry
|
||||
* + MAP_ENTRY_OVERHEAD + char bytes of filePath per file.
|
||||
*
|
||||
* Note: V8 stores all-ASCII strings as Latin-1 (1 byte/char) and only upgrades
|
||||
* to UCS-2 (2 bytes/char) for non-Latin-1 code points. Source paths and type names
|
||||
* are typically all-ASCII, so actual heap cost is roughly half what this returns.
|
||||
* The pessimistic factor is intentional — better to over-budget than under-budget.
|
||||
*
|
||||
* **⚠ Cost profile**: O(totalBindings) — iterates every entry in
|
||||
* `_allByFile` and reads three string `.length` properties per entry.
|
||||
* At a typical repo scale (10k files × ~20 file-scope bindings) this is
|
||||
* ~200k property reads per call. Call at most once per pipeline run,
|
||||
* NOT per file, per chunk, or per progress tick. The current single
|
||||
* call site is the dev-mode telemetry log at the pipeline finalize
|
||||
* seam. Adding a per-file-progress caller would silently make it
|
||||
* quadratic in repo size.
|
||||
*/
|
||||
estimateMemoryBytes(): number {
|
||||
let total = 0;
|
||||
for (const [filePath, entries] of this._allByFile) {
|
||||
total += MAP_ENTRY_OVERHEAD + filePath.length * 2;
|
||||
for (const e of entries) {
|
||||
total += ENTRY_OVERHEAD + (e.scope.length + e.varName.length + e.typeName.length) * 2;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
}
|
||||
12
gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts
Normal file
12
gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
// gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import type { CallExtractionConfig } from '../../call-types.js';
|
||||
|
||||
export const cCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.C,
|
||||
};
|
||||
|
||||
export const cppCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.CPlusPlus,
|
||||
};
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
// gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import type { CallExtractionConfig } from '../../call-types.js';
|
||||
|
||||
export const csharpCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.CSharp,
|
||||
typeAsReceiverHeuristic: true,
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
// gitnexus/src/core/ingestion/call-extractors/configs/dart.ts
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import type { CallExtractionConfig } from '../../call-types.js';
|
||||
|
||||
export const dartCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.Dart,
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
// gitnexus/src/core/ingestion/call-extractors/configs/go.ts
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import type { CallExtractionConfig } from '../../call-types.js';
|
||||
|
||||
export const goCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.Go,
|
||||
};
|
||||
59
gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts
Normal file
59
gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
// gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import type { CallExtractionConfig, ExtractedCallSite } from '../../call-types.js';
|
||||
import type { SyntaxNode } from '../../utils/ast-helpers.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Java method_reference (::) parsing — absorbs call-sites/java.ts
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Parse Java `method_reference` nodes (`expr::method`, `Type::new`,
|
||||
* `this::m`, `super::m`).
|
||||
*/
|
||||
function parseJavaMethodReference(callNode: SyntaxNode): ExtractedCallSite | null {
|
||||
if (callNode.type !== 'method_reference') return null;
|
||||
|
||||
const recv = callNode.namedChild(0);
|
||||
if (!recv) return null;
|
||||
|
||||
// Type::new → constructor call
|
||||
for (const c of callNode.children) {
|
||||
if (c.type === 'new') {
|
||||
if (recv.type !== 'identifier') return null;
|
||||
return { calledName: recv.text, callForm: 'constructor' };
|
||||
}
|
||||
}
|
||||
|
||||
// expr::method → member call with receiver
|
||||
const rhs = callNode.child(callNode.childCount - 1);
|
||||
if (!rhs || rhs.type !== 'identifier') return null;
|
||||
const methodName = rhs.text;
|
||||
|
||||
if (recv.type === 'identifier') {
|
||||
return { calledName: methodName, callForm: 'member', receiverName: recv.text };
|
||||
}
|
||||
if (recv.type === 'this') {
|
||||
return { calledName: methodName, callForm: 'member', receiverName: 'this' };
|
||||
}
|
||||
if (recv.type === 'super') {
|
||||
return { calledName: methodName, callForm: 'member', receiverName: 'super' };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Configs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const javaCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.Java,
|
||||
extractLanguageCallSite: parseJavaMethodReference,
|
||||
typeAsReceiverHeuristic: true,
|
||||
};
|
||||
|
||||
export const kotlinCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.Kotlin,
|
||||
typeAsReceiverHeuristic: true,
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
// gitnexus/src/core/ingestion/call-extractors/configs/php.ts
|
||||
|
||||
import { SupportedLanguages } from 'gitnexus-shared';
|
||||
import type { CallExtractionConfig } from '../../call-types.js';
|
||||
|
||||
export const phpCallConfig: CallExtractionConfig = {
|
||||
language: SupportedLanguages.PHP,
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue