Merge remote-tracking branch 'origin/main' into fix/stream-graph-response

# Conflicts:
#	gitnexus-web/src/services/backend-client.ts
#	gitnexus/src/server/api.ts
This commit is contained in:
Shyam 2026-04-18 01:29:43 +05:30
commit fbd9040647
No known key found for this signature in database
406 changed files with 52684 additions and 8014 deletions

75
.github/dependabot.yml vendored Normal file
View file

@ -0,0 +1,75 @@
version: 2
updates:
# Keep third-party Actions SHA pins current. See CONTRIBUTING.md — when
# reviewing these bumps, verify the SHA corresponds to the claimed tag by
# running `gh api repos/<owner>/<action>/git/refs/tags/<tag>` before merge.
- package-ecosystem: github-actions
directory: /
schedule:
interval: weekly
open-pull-requests-limit: 5
commit-message:
prefix: chore
include: scope
labels:
- dependencies
- ci
# Gitnexus npm deps — tree-sitter grammars checked daily so we catch
# new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars
# are grouped so lockstep bumps produce a single PR. The tree-sitter
# RUNTIME is pinned — upgrade deliberately via the drift check workflow.
# See .github/scripts/check-tree-sitter-upgrade-readiness.py for
# the upgrade readiness tracker.
- package-ecosystem: npm
directory: /gitnexus
schedule:
interval: daily
open-pull-requests-limit: 10
commit-message:
prefix: chore(deps)
include: scope
labels:
- dependencies
groups:
tree-sitter-grammars:
patterns:
- tree-sitter-*
exclude-patterns:
- tree-sitter
- tree-sitter-cli
ignore:
# Pin the tree-sitter runtime at 0.21.x until the drift check
# reports all grammars are peer-dep compatible with 0.25.
- dependency-name: tree-sitter
update-types:
- version-update:semver-major
- version-update:semver-minor
# tree-sitter-cli follows the runtime's version cadence. Bump when
# regenerating vendor/tree-sitter-proto/src/parser.c, not on a schedule.
- dependency-name: tree-sitter-cli
# gitnexus-web (thin frontend client).
- package-ecosystem: npm
directory: /gitnexus-web
schedule:
interval: weekly
open-pull-requests-limit: 5
commit-message:
prefix: chore(deps)
include: scope
labels:
- dependencies
- frontend
# Shared types package.
- package-ecosystem: npm
directory: /gitnexus-shared
schedule:
interval: weekly
open-pull-requests-limit: 5
commit-message:
prefix: chore(deps)
include: scope
labels:
- dependencies

53
.github/release-drafter.yml vendored Normal file
View file

@ -0,0 +1,53 @@
# release-drafter config — used only for PR autolabeling by
# `.github/workflows/pr-labeler.yml` (the workflow passes `disable-releaser: true`,
# so the draft-release side of release-drafter never runs).
#
# The labels applied here are the same ones `.github/release.yml` maps to
# categorized release-notes sections.
#
# `sync-labels: true` removes managed autolabels that no longer match the PR —
# critical for the breaking-change case: if a PR title drops the `!` or the body
# drops `BREAKING CHANGE:`, the `breaking` label is pulled off automatically.
# Required by release-drafter; not used because releaser is disabled.
name-template: 'unused'
tag-template: 'unused'
template: |
$CHANGES
sync-labels: true
autolabeler:
- label: enhancement
title:
- '/^feat(\([^)]+\))?!?:/i'
- label: bug
title:
- '/^fix(\([^)]+\))?!?:/i'
- label: performance
title:
- '/^perf(\([^)]+\))?!?:/i'
- label: refactor
title:
- '/^refactor(\([^)]+\))?!?:/i'
- label: documentation
title:
- '/^docs(\([^)]+\))?!?:/i'
- label: test
title:
- '/^test(\([^)]+\))?!?:/i'
- label: ci
title:
- '/^ci(\([^)]+\))?!?:/i'
- label: dependencies
title:
- '/^(build|deps)(\([^)]+\))?!?:/i'
- label: chore
title:
- '/^(chore|revert)(\([^)]+\))?!?:/i'
# Breaking-change marker: either `!` in the type prefix or `BREAKING CHANGE:` in body.
- label: breaking
title:
- '/^[a-z]+(\([^)]+\))?!:/i'
body:
- '/BREAKING[ -]CHANGE:/i'

View file

@ -0,0 +1,358 @@
#!/usr/bin/env python3
"""Monitor tree-sitter 0.25 upgrade readiness.
Tracks two things Dependabot cannot see:
1. Peer-dep compatibility. Each tree-sitter-* grammar declares a peer
dependency on the tree-sitter runtime. We want to know when every
grammar's *latest npm release* satisfies tree-sitter@0.25.0 so we
can upgrade without --legacy-peer-deps.
2. Vendored upstream drift. vendor/tree-sitter-proto/ is a snapshot of
coder3101/tree-sitter-proto's parser.c. When upstream moves, we want
to know whether we can pick it up.
Invoked from .github/workflows/tree-sitter-upgrade-readiness.yml daily.
Runs locally too:
python3 .github/scripts/check-tree-sitter-upgrade-readiness.py
Outputs Markdown to stdout. Exit 0 when every grammar is upgrade-ready
and the vendored proto is in sync. Exit 1 when blockers remain (the
workflow uses this to open or update a tracking issue).
No external deps -- stdlib only, so it runs on any vanilla runner.
"""
from __future__ import annotations
import json
import os
import pathlib
import re
import sys
import urllib.error
import urllib.request
REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
GITNEXUS_DIR = REPO_ROOT / "gitnexus"
VENDOR_PROTO_DIR = GITNEXUS_DIR / "vendor" / "tree-sitter-proto"
# ── Upgrade target ──────────────────────────────────────────────────────
# The runtime version we want to upgrade TO. Update this when the goal
# changes (e.g. once 0.25 lands and we target 0.26).
TARGET_RUNTIME = "0.25.0"
TARGET_RUNTIME_MAJOR_MINOR = ".".join(TARGET_RUNTIME.split(".")[:2])
# Tree-sitter runtime -> (min_abi, max_abi) it can load. Only the current
# and target entries matter; extend when changing TARGET_RUNTIME.
RUNTIME_ABI_RANGES: dict[str, tuple[int, int]] = {
"0.21": (13, 14),
"0.25": (13, 15),
}
assert TARGET_RUNTIME_MAJOR_MINOR in RUNTIME_ABI_RANGES, (
f"RUNTIME_ABI_RANGES has no entry for {TARGET_RUNTIME_MAJOR_MINOR!r}. "
f"Add the ABI range after auditing the upstream release notes."
)
# Grammars we use. Values are the upstream GitHub repos to check for
# unreleased ABI bumps (owner/repo, branch, parser.c path).
GRAMMARS: dict[str, tuple[str, str, str]] = {
"tree-sitter-c": ("tree-sitter/tree-sitter-c", "master", "src/parser.c"),
"tree-sitter-c-sharp": ("tree-sitter/tree-sitter-c-sharp", "master", "src/parser.c"),
"tree-sitter-cpp": ("tree-sitter/tree-sitter-cpp", "master", "src/parser.c"),
"tree-sitter-dart": ("UserNobody14/tree-sitter-dart", "master", "src/parser.c"),
"tree-sitter-go": ("tree-sitter/tree-sitter-go", "master", "src/parser.c"),
"tree-sitter-java": ("tree-sitter/tree-sitter-java", "master", "src/parser.c"),
"tree-sitter-javascript": ("tree-sitter/tree-sitter-javascript", "master", "src/parser.c"),
"tree-sitter-kotlin": ("fwcd/tree-sitter-kotlin", "main", "src/parser.c"),
"tree-sitter-php": ("tree-sitter/tree-sitter-php", "master", "php/src/parser.c"),
"tree-sitter-python": ("tree-sitter/tree-sitter-python", "master", "src/parser.c"),
"tree-sitter-ruby": ("tree-sitter/tree-sitter-ruby", "master", "src/parser.c"),
"tree-sitter-rust": ("tree-sitter/tree-sitter-rust", "master", "src/parser.c"),
"tree-sitter-swift": ("alex-pinkus/tree-sitter-swift", "main", "src/parser.c"),
"tree-sitter-typescript": ("tree-sitter/tree-sitter-typescript", "master", "typescript/src/parser.c"),
}
UPSTREAM_PROTO_OWNER = "coder3101"
UPSTREAM_PROTO_REPO = "tree-sitter-proto"
UPSTREAM_PROTO_BRANCH = "main"
# ── Helpers ─────────────────────────────────────────────────────────────
def read_current_runtime() -> str:
"""Return the tree-sitter runtime version pinned in package.json (e.g. '0.21')."""
pkg = json.loads((GITNEXUS_DIR / "package.json").read_text())
raw = pkg["dependencies"]["tree-sitter"]
match = re.search(r"(\d+)\.(\d+)", raw)
if not match:
raise SystemExit(f"could not parse tree-sitter version: {raw!r}")
return f"{match.group(1)}.{match.group(2)}"
def npm_view_json(pkg: str) -> dict | None:
"""Fetch package metadata from the npm registry via HTTPS.
Uses the registry API directly so we don't depend on the npm CLI
being available (it's a batch file on Windows which complicates
subprocess calls).
"""
url = f"https://registry.npmjs.org/{pkg}/latest"
try:
req = urllib.request.Request(url, headers={"Accept": "application/json"})
with urllib.request.urlopen(req, timeout=8) as resp:
return json.loads(resp.read().decode("utf-8"))
except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError):
return None
def satisfies_target(peer_range: str | None, target: str) -> bool:
"""Check if a semver range like '^0.22.4' or '^0.25.0' satisfies the target.
Simple heuristic: extract the minimum version from the range and check
if target >= min. For caret ranges (^X.Y.Z), the upper bound is the
next major (for X>0) or next minor (for X==0). We check both bounds.
"""
if peer_range is None:
# No peer dep declared = no constraint = compatible.
return True
match = re.search(r"(\d+)\.(\d+)\.(\d+)", peer_range)
if not match:
return False
min_major, min_minor, min_patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
t_match = re.search(r"(\d+)\.(\d+)\.(\d+)", target)
if not t_match:
return False
t_major, t_minor, t_patch = int(t_match.group(1)), int(t_match.group(2)), int(t_match.group(3))
# Target must be >= minimum.
target_tuple = (t_major, t_minor, t_patch)
min_tuple = (min_major, min_minor, min_patch)
if target_tuple < min_tuple:
return False
# For caret ranges with major 0: ^0.X.Y allows [0.X.Y, 0.(X+1).0).
if peer_range.startswith("^") and min_major == 0:
if t_major != 0 or t_minor >= min_minor + 1:
return False
# For caret ranges with major >0: ^X.Y.Z allows [X.Y.Z, (X+1).0.0).
elif peer_range.startswith("^") and min_major > 0:
if t_major >= min_major + 1:
return False
return True
_GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
def fetch_text(url: str, timeout: int = 8) -> str | None:
"""Fetch a URL and return its text, or None on failure.
Adds an Authorization header for github.com URLs when GITHUB_TOKEN is
set (raises the rate limit from 60 to 5 000 requests/hour).
"""
headers: dict[str, str] = {}
if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url):
headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}"
try:
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, timeout=timeout) as resp:
return resp.read().decode("utf-8", errors="ignore")
except (urllib.error.URLError, urllib.error.HTTPError):
return None
def extract_abi_from_text(text: str) -> int | None:
"""Extract LANGUAGE_VERSION from parser.c text."""
match = re.search(r"#define\s+LANGUAGE_VERSION\s+(\d+)", text[:4096])
return int(match.group(1)) if match else None
def extract_language_version(parser_c: pathlib.Path) -> int | None:
"""Return the LANGUAGE_VERSION defined in a parser.c, or None if absent."""
if not parser_c.is_file():
return None
with parser_c.open("r", encoding="utf-8", errors="ignore") as fh:
head = fh.read(4096)
return extract_abi_from_text(head)
def md_h(text: str, level: int = 2) -> str:
return f"{'#' * level} {text}\n"
# ── Main ────────────────────────────────────────────────────────────────
def main() -> int:
blockers: dict[str, str] = {}
lines: list[str] = []
lines.append(md_h("Tree-sitter 0.25 upgrade readiness", 1))
lines.append("")
current_runtime = read_current_runtime()
current_abi_range = RUNTIME_ABI_RANGES.get(current_runtime, (0, 0))
target_abi_range = RUNTIME_ABI_RANGES.get(TARGET_RUNTIME_MAJOR_MINOR, (0, 0))
lines.append(f"- Current runtime: `tree-sitter@{current_runtime}.x` (ABI {current_abi_range[0]}..{current_abi_range[1]})")
lines.append(f"- Target runtime: `tree-sitter@{TARGET_RUNTIME}` (ABI {target_abi_range[0]}..{target_abi_range[1]})")
lines.append("")
# ── Grammar peer-dep compatibility ───────────────────────────────
lines.append(md_h("Grammar compatibility", 2))
lines.append("| Grammar | npm latest | Peer dep | Satisfies 0.25? | ABI | Upstream ABI | Status |")
lines.append("|---|---|---|---|---|---|---|")
ready_count = 0
total_count = len(GRAMMARS)
for name, (upstream_repo, upstream_branch, parser_path) in sorted(GRAMMARS.items()):
# Fetch latest npm metadata.
info = npm_view_json(name)
fetch_failed = info is None
npm_version = "?"
peer_range = None
peer_optional = True
if info:
npm_version = info.get("version", "?")
peers = info.get("peerDependencies") or {}
peer_range = peers.get("tree-sitter")
meta = info.get("peerDependenciesMeta") or {}
ts_meta = meta.get("tree-sitter") or {}
peer_optional = ts_meta.get("optional", False) if peer_range else True
if fetch_failed:
peer_display = "? (fetch failed)"
compatible = False
else:
peer_display = peer_range or "none"
if peer_range and not peer_optional:
peer_display += " (required)"
compatible = satisfies_target(peer_range, TARGET_RUNTIME)
# Check installed ABI using the same parser_path from GRAMMARS.
installed_parser = GITNEXUS_DIR / "node_modules" / name / parser_path
if not installed_parser.is_file():
# Fallback to default location.
installed_parser = GITNEXUS_DIR / "node_modules" / name / "src" / "parser.c"
installed_abi = extract_language_version(installed_parser)
abi_display = str(installed_abi) if installed_abi else "?"
# Check upstream (main/master branch) ABI for unreleased work.
upstream_url = (
f"https://raw.githubusercontent.com/{upstream_repo}/"
f"{upstream_branch}/{parser_path}"
)
upstream_text = fetch_text(upstream_url)
upstream_abi = extract_abi_from_text(upstream_text) if upstream_text else None
upstream_abi_display = str(upstream_abi) if upstream_abi else "?"
# Determine status.
if fetch_failed:
status = "Unknown (fetch failed)"
blockers[name] = f"`{name}`: npm registry fetch failed — could not verify peer dep"
elif compatible:
status = "Ready"
ready_count += 1
elif upstream_abi and upstream_abi >= 15:
status = "Unreleased (ABI 15 on main)"
blockers[name] = f"`{name}`: ABI 15 on `{upstream_repo}` main but not published to npm"
else:
status = "Blocking"
blockers[name] = f"`{name}@{npm_version}`: peer `{peer_display}` incompatible with 0.25"
# Also check upstream package.json for relaxed peer dep.
if not compatible and not fetch_failed:
upstream_pkg_url = (
f"https://raw.githubusercontent.com/{upstream_repo}/"
f"{upstream_branch}/package.json"
)
upstream_pkg_text = fetch_text(upstream_pkg_url)
if upstream_pkg_text:
try:
upstream_pkg = json.loads(upstream_pkg_text)
upstream_peer = (upstream_pkg.get("peerDependencies") or {}).get("tree-sitter")
if upstream_peer and satisfies_target(upstream_peer, TARGET_RUNTIME):
status = "Unreleased (peer relaxed on main)"
blockers[name] = f"`{name}`: peer dep relaxed on `{upstream_repo}` main but not published to npm"
except json.JSONDecodeError:
pass
compat_icon = "Yes" if compatible else "**No**"
lines.append(
f"| `{name}` | {npm_version} | {peer_display} | {compat_icon} | {abi_display} | {upstream_abi_display} | {status} |"
)
lines.append("")
lines.append(f"**{ready_count}/{total_count}** grammars ready for `tree-sitter@{TARGET_RUNTIME}`.")
lines.append("")
# ── Vendored proto drift ─────────────────────────────────────────
lines.append(md_h("Vendored tree-sitter-proto", 2))
vendored_abi = extract_language_version(VENDOR_PROTO_DIR / "src" / "parser.c")
upstream_proto_url = (
f"https://raw.githubusercontent.com/{UPSTREAM_PROTO_OWNER}/"
f"{UPSTREAM_PROTO_REPO}/{UPSTREAM_PROTO_BRANCH}/src/parser.c"
)
upstream_proto_text = fetch_text(upstream_proto_url)
upstream_proto_abi = extract_abi_from_text(upstream_proto_text) if upstream_proto_text else None
sha_url = (
f"https://api.github.com/repos/{UPSTREAM_PROTO_OWNER}/"
f"{UPSTREAM_PROTO_REPO}/commits/{UPSTREAM_PROTO_BRANCH}"
)
sha_text = fetch_text(sha_url)
upstream_sha = "?"
if sha_text:
try:
upstream_sha = json.loads(sha_text).get("sha", "?")[:12]
except json.JSONDecodeError:
pass
local_proto_path = VENDOR_PROTO_DIR / "src" / "parser.c"
local_proto_text = local_proto_path.read_text(encoding="utf-8", errors="ignore") if local_proto_path.is_file() else ""
in_sync = bool(
upstream_proto_text
and local_proto_text.replace("\r\n", "\n")
== upstream_proto_text.replace("\r\n", "\n")
)
lines.append(f"- Upstream: `{UPSTREAM_PROTO_OWNER}/{UPSTREAM_PROTO_REPO}@{UPSTREAM_PROTO_BRANCH}` (HEAD `{upstream_sha}`)")
lines.append(f"- Upstream ABI: **{upstream_proto_abi}**")
lines.append(f"- Vendored ABI: **{vendored_abi}**")
lines.append(f"- In sync: {'yes' if in_sync else 'no — upstream has diverged'}")
if upstream_proto_abi and vendored_abi and upstream_proto_abi > vendored_abi:
can_upgrade = upstream_proto_abi <= target_abi_range[1]
lines.append(f"- Upstream ABI {upstream_proto_abi} {'is' if can_upgrade else 'is NOT'} within target runtime range ({target_abi_range[0]}..{target_abi_range[1]})")
if can_upgrade:
lines.append(f"- **Action:** after upgrading to tree-sitter@{TARGET_RUNTIME}, regenerate vendored parser.c from upstream `{upstream_sha}`")
else:
lines.append(f"- **Action:** wait for runtime upgrade beyond {TARGET_RUNTIME} that supports ABI {upstream_proto_abi}")
blockers["vendored-proto-abi"] = f"vendored tree-sitter-proto: upstream ABI {upstream_proto_abi} outside target range"
elif not in_sync:
lines.append("- **Action:** review upstream changes; vendored copy may need updating")
blockers["vendored-proto-sync"] = "vendored tree-sitter-proto: out of sync with upstream"
# ── Summary ──────────────────────────────────────────────────────
lines.append("")
lines.append(md_h("Summary", 2))
if blockers:
lines.append(f"**{len(blockers)} blocker(s) remaining:**\n")
for b in blockers.values():
lines.append(f"- {b}")
lines.append("")
lines.append("Upgrade to `tree-sitter@0.25` is **blocked**.")
else:
lines.append("All grammars are compatible. Upgrade to `tree-sitter@0.25` is **ready**.")
print("\n".join(lines))
return 1 if blockers else 0
if __name__ == "__main__":
sys.exit(main())

View file

@ -0,0 +1,173 @@
#!/usr/bin/env python3
"""Enforce the GitHub Actions concurrency convention.
See CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention" for the rules.
Invoked from .github/workflows/ci-quality.yml. Runs locally too:
python3 .github/scripts/check-workflow-concurrency.py .github/workflows
Rules:
1. Every entry-point (non-reusable) workflow declares a top-level
`concurrency:` block.
2. Reusable workflows (on: workflow_call ONLY) do NOT declare one.
3. The `concurrency.group` expression MUST reference either
`${{ github.workflow }}` or a literal `CI-` prefix (the documented
ci.yml reusable-workflow-safe exception). This is checked by substring
containment rather than prefix match because ci.yml's group is a
conditional expression that resolves to a `CI-` literal at runtime.
We deliberately do not use a YAML library keeps the script dependency-free
on any vanilla runner. `on:` block parsing is line-based and handles both the
flat (`on: workflow_call`) and mapping (`on:\n workflow_call:`) forms.
"""
from __future__ import annotations
import pathlib
import re
import sys
REQUIRED_TOKENS = ("${{ github.workflow }}", "CI-")
def is_reusable(lines: list[str]) -> bool:
"""Return True iff the workflow's `on:` block names only `workflow_call`."""
in_on = False
on_indent: int | None = None
keys: list[str] = []
for raw in lines:
# Skip blank lines and comments
stripped = raw.strip()
if not stripped or stripped.startswith("#"):
continue
indent = len(raw) - len(raw.lstrip(" "))
if not in_on:
if raw.startswith("on:"):
remainder = raw[len("on:"):].strip()
if not remainder:
# `on:` followed by indented mapping on next lines
in_on = True
on_indent = indent
continue
if remainder.startswith("[") and remainder.endswith("]"):
# Flow-style list: on: [workflow_call]
items = [
item.strip() for item in remainder.strip("[]").split(",")
]
return items == ["workflow_call"]
# Scalar form: on: workflow_call (or a single other event)
return remainder == "workflow_call"
continue
# Inside the `on:` block; stop when indentation returns to <= on_indent
if on_indent is not None and indent <= on_indent:
break
# Only consider keys at on_indent + indentation step (anything deeper
# is nested config like `types:`)
if ":" not in stripped:
continue
# Heuristic: first-level event keys are those with indent == on_indent + 2
# (the canonical step for a 2-space YAML doc). We collect all first-level
# keys by tracking the smallest indent seen inside the block.
keys.append((indent, stripped.split(":", 1)[0].strip()))
if not keys:
return False
# Take only the outermost-indented keys as the event list
min_indent = min(i for i, _ in keys)
events = [name for i, name in keys if i == min_indent]
return events == ["workflow_call"]
CONCURRENCY_RE = re.compile(r"^concurrency:\s*$")
GROUP_RE = re.compile(r"^\s+group:\s*(.+?)\s*$")
def extract_group_key(lines: list[str]) -> str | None:
"""Return the `group:` value of the top-level `concurrency:` block, or None."""
for idx, raw in enumerate(lines):
if CONCURRENCY_RE.match(raw):
# Scan forward until we leave the concurrency block (next top-level key
# is at column 0 and ends with `:`).
for follow in lines[idx + 1:]:
if follow and not follow.startswith(" ") and follow.rstrip().endswith(":"):
break
m = GROUP_RE.match(follow)
if m:
return m.group(1).strip().strip("'").strip('"')
break
return None
def has_top_level_concurrency(lines: list[str]) -> bool:
return any(CONCURRENCY_RE.match(raw) for raw in lines)
def check(workflows_dir: pathlib.Path) -> int:
fail = 0
files = sorted(
list(workflows_dir.glob("*.yml")) + list(workflows_dir.glob("*.yaml"))
)
for path in files:
lines = path.read_text(encoding="utf-8").splitlines()
reusable = is_reusable(lines)
has_conc = has_top_level_concurrency(lines)
if reusable:
if has_conc:
print(
f"::error file={path}::Reusable workflow (on: workflow_call) "
"must NOT declare its own concurrency block — it inherits "
"from the caller. See CONTRIBUTING.md -> GitHub Actions — "
"Concurrency Convention."
)
fail = 1
continue
if not has_conc:
print(
f"::error file={path}::Missing top-level concurrency block. "
"See CONTRIBUTING.md -> GitHub Actions — Concurrency Convention."
)
fail = 1
continue
group = extract_group_key(lines)
if group is None:
print(
f"::error file={path}::concurrency block is missing a "
"`group:` key."
)
fail = 1
continue
if not any(token in group for token in REQUIRED_TOKENS):
print(
f"::error file={path}::concurrency.group `{group}` must "
f"reference one of {REQUIRED_TOKENS}. See CONTRIBUTING.md -> "
"GitHub Actions — Concurrency Convention."
)
fail = 1
return fail
def main(argv: list[str]) -> int:
if len(argv) != 2:
print(f"usage: {argv[0]} <workflows-dir>", file=sys.stderr)
return 2
workflows_dir = pathlib.Path(argv[1])
if not workflows_dir.is_dir():
print(f"not a directory: {workflows_dir}", file=sys.stderr)
return 2
return check(workflows_dir)
if __name__ == "__main__":
sys.exit(main(sys.argv))

View file

@ -11,8 +11,8 @@ jobs:
outputs:
web_changed: ${{ steps.filter.outputs.web }}
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v3
id: filter
with:
filters: |
@ -26,7 +26,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: ./.github/actions/setup-gitnexus-web
@ -74,7 +74,7 @@ jobs:
- name: Upload test results
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-results
path: |

View file

@ -8,8 +8,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: 20
cache: npm
@ -21,8 +21,8 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: 20
cache: npm
@ -34,7 +34,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: ./.github/actions/setup-gitnexus
- run: npx tsc --noEmit
working-directory: gitnexus
@ -43,7 +43,30 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: ./.github/actions/setup-gitnexus-web
- run: npx tsc -b --noEmit
working-directory: gitnexus-web
# Enforces the convention documented in CONTRIBUTING.md → "GitHub Actions —
# Concurrency Convention":
# 1. Every entry-point (non-reusable) workflow declares a top-level
# `concurrency:` block.
# 2. Reusable workflows (`on: workflow_call` only) do NOT declare one —
# they inherit concurrency from the caller.
# 3. The concurrency group key starts with `${{ github.workflow }}` or
# the literal `CI-` prefix (the documented ci.yml exception for
# reusable-workflow-safe grouping).
# Reusability is detected by parsing each workflow's `on:` block, not an
# allowlist, so new reusable workflows never produce false positives.
workflow-convention:
name: Workflow concurrency convention
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Validate workflow concurrency convention
shell: bash
run: |
set -euo pipefail
python3 .github/scripts/check-workflow-concurrency.py .github/workflows

View file

@ -14,6 +14,16 @@ permissions:
contents: read # needed for sparse checkout of vitest.config.ts
pull-requests: write # needed to post sticky PR comment
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Serialize sticky-comment writes per PR so two rapid CI completions don't race.
# Internal PRs surface in `pull_requests[0].number`. Fork PRs leave that array empty,
# so we fall back to `<head-repo-full-name>/<head-branch>`, which is stable across
# reruns and subsequent pushes for the same fork PR (unlike `workflow_run.id` which
# is unique per run and therefore does not serialize anything).
concurrency:
group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}
cancel-in-progress: false
jobs:
pr-report:
name: PR Report
@ -26,7 +36,7 @@ jobs:
steps:
# ── Download artifacts from the CI run ────────────────────────
- name: Download artifacts
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
with:
script: |
const fs = require('fs');
@ -113,7 +123,7 @@ jobs:
- name: Checkout (for vitest config)
if: steps.meta.outputs.skip != 'true'
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: gitnexus/vitest.config.ts
sparse-checkout-cone-mode: false
@ -122,7 +132,7 @@ jobs:
- name: Fetch base branch coverage
if: steps.meta.outputs.skip != 'true'
id: base-coverage
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
with:
script: |
const fs = require('fs');
@ -406,7 +416,7 @@ jobs:
- name: Comment on PR
if: steps.meta.outputs.skip != 'true'
uses: marocchino/sticky-pull-request-comment@773744901bac0e8cbb5a0dc842800d45e9b2b405 # v2
uses: marocchino/sticky-pull-request-comment@0ea0beb66eb9baf113663a64ec522f60e49231c0 # v2
with:
header: ci-report
number: ${{ steps.meta.outputs.pr_number }}

View file

@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: ./.github/actions/setup-gitnexus
with:
build: 'true'
@ -43,7 +43,7 @@ jobs:
- name: Upload test reports
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: test-reports
path: |
@ -63,7 +63,7 @@ jobs:
runs-on: ${{ matrix.os }}
timeout-minutes: 25
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: ./.github/actions/setup-gitnexus
with:
build: 'true'

View file

@ -9,9 +9,20 @@ on:
paths-ignore: ['**.md', 'docs/**', 'LICENSE']
workflow_call:
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Hardcoded `CI-` prefix (not `${{ github.workflow }}`) because this workflow is
# invoked as a reusable workflow from publish.yml and release-candidate.yml. In
# called-workflow context `github.workflow` evaluation is ambiguous across GitHub
# Actions versions, and a prefix that could resolve to the caller's name would
# share a concurrency group with the caller → deadlock. A literal prefix is
# immune. Direct `push`/`pull_request` invocations use `CI-<ref>`; invocations
# from a reusable-workflow caller fall into a per-run-unique group that never
# serializes with the caller.
# cancel-in-progress is event-aware: cancel superseded PR runs, queue every other
# event (push to main, workflow_call from publish.yml, etc.).
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
group: ${{ (github.event_name == 'pull_request' || github.event_name == 'push') && format('CI-{0}', github.ref) || format('CI-nested-{0}', github.run_id) }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# ── Reusable workflow orchestration ─────────────────────────────────
# Each concern lives in its own workflow file for maintainability:
@ -74,7 +85,7 @@ jobs:
cp pr-meta/e2e_result pr-meta/e2e-result
- name: Upload PR metadata
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: pr-meta
path: pr-meta/
@ -96,9 +107,9 @@ jobs:
TESTS: ${{ needs.tests.result }}
E2E: ${{ needs.e2e.result }}
run: |
echo "Quality: $QUALITY"
echo "Tests: $TESTS"
echo "E2E: $E2E"
echo "Quality: $QUALITY"
echo "Tests: $TESTS"
echo "E2E: $E2E"
if [[ "$QUALITY" != "success" ]] ||
[[ "$TESTS" != "success" ]]; then
echo "::error::Quality or test jobs failed"

View file

@ -16,9 +16,10 @@ on:
issue_comment:
types: [created]
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Serialize per-PR to avoid racing review comments.
concurrency:
group: claude-review-${{ github.event.issue.number || github.event.pull_request.number }}
group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number }}
cancel-in-progress: false
jobs:
@ -56,7 +57,7 @@ jobs:
# For issue_comment triggers, resolve the PR number, head SHA, and fork repo
- name: Resolve PR context
id: pr
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
with:
script: |
let pr;
@ -76,7 +77,7 @@ jobs:
core.setOutput('branch', pr.head.ref);
- name: Checkout PR head
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ${{ steps.pr.outputs.repo }}
ref: ${{ steps.pr.outputs.sha }}

View file

@ -10,9 +10,10 @@ on:
pull_request_review:
types: [submitted]
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Serialize per-PR/issue to avoid racing comments.
concurrency:
group: claude-code-${{ github.event.issue.number || github.event.pull_request.number || github.event.issue.id }}
group: ${{ github.workflow }}-${{ github.event.issue.number || github.event.pull_request.number || github.event.issue.id }}
cancel-in-progress: false
jobs:
@ -58,7 +59,7 @@ jobs:
# For PR-related triggers, resolve the fork repo so we can checkout correctly.
- name: Resolve PR context
id: pr
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v7
with:
script: |
// Determine if this event is PR-related
@ -90,7 +91,7 @@ jobs:
core.setOutput('branch', pr.head.ref);
- name: Checkout repository
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
repository: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.repo || github.repository }}
ref: ${{ steps.pr.outputs.is_pr == 'true' && steps.pr.outputs.sha || '' }}

View file

@ -8,8 +8,9 @@ on:
permissions:
pull-requests: write
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
concurrency:
group: pr-desc-${{ github.event.pull_request.number }}
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
@ -18,7 +19,7 @@ jobs:
timeout-minutes: 5
steps:
- name: Check PR description quality
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const MIN_BODY_LENGTH = 50;

113
.github/workflows/pr-labeler.yml vendored Normal file
View file

@ -0,0 +1,113 @@
name: PR Conventional Labeler
# Two workflows in one file with different triggers, matched to the minimum
# privilege each needs:
#
# validate-title (on: pull_request)
# Fork-safe. Runs with the PR-head's read-only GITHUB_TOKEN. Uses
# `amannn/action-semantic-pull-request` to fail the check when the PR
# title doesn't follow the conventional-commit format. Because the
# action only reads the event payload, no fork-controlled code runs.
#
# autolabel (on: pull_request_target)
# Needs `pull-requests: write` to apply labels, so must be
# pull_request_target. Uses `release-drafter/release-drafter` with
# `dry-run: true` to only run the autolabeler against the
# `.github/release-drafter.yml` config from the BASE ref (release-
# drafter reads the config from the repository's default branch, NOT
# the PR head — verify with `gh api repos/release-drafter/release-drafter/contents/...`
# or a fork-test PR before merging if the repo is high-value).
# `sync-labels: true` in the config removes managed autolabels that no
# longer match (e.g. when `!` or `BREAKING CHANGE:` is dropped).
#
# Title format: <type>[(scope)][!]: <subject>
# Allowed types: feat, fix, perf, refactor, docs, test, ci, build, chore, revert, deps
# Trailing `!` on the type marks a breaking change.
# See CONTRIBUTING.md → "Pull request titles".
on:
pull_request:
# Title-only changes fire `edited`. `opened` and `reopened` cover creation.
# `synchronize` (push to the PR branch) is intentionally excluded — titles
# don't change on push, so it only wastes CI minutes and broadens the
# privileged-token exposure window on the autolabel job.
types: [opened, edited, reopened]
pull_request_target:
types: [opened, edited, reopened]
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Include `github.event_name` so `pull_request` (validate-title) and
# `pull_request_target` (autolabel) runs for the same PR do NOT share a slot
# and therefore cannot cancel each other — a cancelled required-check would
# permanently block merge until the next title edit.
# Within each trigger the latest title edit still supersedes the prior run.
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
validate-title:
# Fork-safe job — only runs on `pull_request` (not `pull_request_target`).
# Token is read-only; writes a commit status that branch protection can
# require before merge.
name: Validate PR title
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
pull-requests: read
steps:
# Pinned to v6.1.1. Verify SHA via:
# gh api repos/amannn/action-semantic-pull-request/git/refs/tags/v6.1.1
- uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
types: |
feat
fix
perf
refactor
docs
test
ci
build
chore
revert
deps
requireScope: false
# Subject must be non-empty. We DO allow capitalized proper nouns
# (MCP, GitHub, API, etc.) — the old `^(?![A-Z]).+$` pattern
# rejected legitimate titles like `fix: MCP tool schema`.
subjectPattern: ^\S.{2,}$
subjectPatternError: |
The subject "{subject}" in PR title "{title}" is invalid.
Subjects must be at least 3 characters and must not start with whitespace.
wip: false
autolabel:
# Privileged job — runs only on `pull_request_target` so it can write labels.
# Never checks out fork code, never executes fork-controlled input; only
# reads the PR metadata (title, body, labels) and calls the GitHub API.
name: Apply conventional label
if: github.event_name == 'pull_request_target'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
# `contents: read` is required — release-drafter's context.config() reads
# `.github/release-drafter.yml` from the repo's default branch via the
# repo-contents API. Without it the job silently 403s and no labels are
# applied. Job-level permissions nullify all unlisted scopes, so an
# explicit grant is necessary here.
contents: read
pull-requests: write
steps:
# Pinned to v7.2.0. Verify SHA via:
# gh api repos/release-drafter/release-drafter/git/refs/tags/v7.2.0
# v7 removed `disable-releaser`; use `dry-run: true` to only autolabel.
- uses: release-drafter/release-drafter@5de93583980a40bd78603b6dfdcda5b4df377b32 # v7.2.0
with:
config-name: release-drafter.yml
dry-run: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View file

@ -7,13 +7,22 @@ on:
# No workflow-level permissions — scoped per job below.
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Tag refs are unique per release, so distinct tags run in parallel. Re-pushes of the
# same tag serialize. cancel-in-progress: false — never cancel a publish mid-flight.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
ci:
uses: ./.github/workflows/ci.yml
permissions:
contents: read
actions: read
pull-requests: write
# No pull-requests:write — `ci.yml`'s save-pr-meta job is gated on
# `github.event_name == 'pull_request'`, so it never runs during a
# tag-triggered publish. Least-privilege for release-critical paths.
publish:
needs: ci
@ -23,8 +32,8 @@ jobs:
contents: write
id-token: write
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: 20
registry-url: https://registry.npmjs.org
@ -82,7 +91,7 @@ jobs:
fi
- name: Create GitHub Release
uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2
with:
body_path: ${{ steps.changelog.outputs.fallback == 'false' && '/tmp/release-notes.md' || '' }}
generate_release_notes: ${{ steps.changelog.outputs.fallback == 'true' }}

366
.github/workflows/release-candidate.yml vendored Normal file
View file

@ -0,0 +1,366 @@
name: Release Candidate
on:
# Publish a release-candidate build whenever a merge/commit lands on main.
# Docs/README-only changes are filtered out so prose updates don't
# cut a release.
push:
branches: [main]
paths-ignore:
- '**.md'
- 'docs/**'
- 'LICENSE'
workflow_dispatch:
inputs:
bump:
description: >-
Cycle policy. 'auto' (default) continues the active rc cycle on
this branch if there is one, otherwise bumps patch from latest.
Choose 'patch' / 'minor' / 'major' to explicitly start or reset
an rc cycle.
required: false
default: 'auto'
type: choice
options:
- auto
- patch
- minor
- major
force:
description: 'Publish even when HEAD already has an rc marker'
required: false
default: 'false'
type: choice
options:
- 'false'
- 'true'
# No workflow-level permissions — scoped per job below.
permissions: {}
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Serialize all runs on the same ref (push + workflow_dispatch) to prevent two publishes
# racing on the rc counter. cancel-in-progress: false — the earlier merge publishes first.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
# ── Skip when HEAD already has an rc marker (retry / duplicate dispatch) ──
# The marker is a lightweight tag `rc/<HEAD_SHA>` pushed *before* `npm
# publish`, so a failed publish leaves the marker in place and the guard
# refuses to re-publish. Recovery path after a partial failure:
# git push --delete origin rc/<HEAD_SHA> v<RC_VERSION>
# then redispatch with force=true.
guard:
name: Check if release candidate should run
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
outputs:
should_run: ${{ steps.decide.outputs.should_run }}
head_sha: ${{ steps.decide.outputs.head_sha }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
fetch-tags: true
- name: Decide
id: decide
shell: bash
env:
FORCE: ${{ inputs.force }}
BUMP_INPUT: ${{ inputs.bump }}
EVENT_NAME: ${{ github.event_name }}
run: |
set -euo pipefail
HEAD_SHA=$(git rev-parse HEAD)
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
if [ "$FORCE" = "true" ]; then
echo "Force flag set — running regardless of marker tag."
echo "should_run=true" >> "$GITHUB_OUTPUT"
exit 0
fi
# An explicit cycle reset on dispatch (bump != auto) also bypasses
# the dedup guard — the maintainer is deliberately asking for a
# new rc from the same commit.
if [ "$EVENT_NAME" = "workflow_dispatch" ] \
&& [ -n "${BUMP_INPUT:-}" ] \
&& [ "${BUMP_INPUT:-auto}" != "auto" ]; then
echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup."
echo "should_run=true" >> "$GITHUB_OUTPUT"
exit 0
fi
# Dedup: is there already an rc/<HEAD_SHA> marker pointing at HEAD?
MARKER="rc/${HEAD_SHA}"
if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then
echo "HEAD already has marker $MARKER — skipping."
echo "should_run=false" >> "$GITHUB_OUTPUT"
else
echo "No marker on HEAD — proceeding."
echo "should_run=true" >> "$GITHUB_OUTPUT"
fi
# ── Reuse the stable CI workflow ─────────────────────────────────────
ci:
needs: guard
if: needs.guard.outputs.should_run == 'true'
uses: ./.github/workflows/ci.yml
permissions:
contents: read
secrets: inherit
# ── Publish the rc build to npm + create GitHub prerelease ───────────
publish:
name: Publish release candidate to npm
needs: [guard, ci]
if: needs.guard.outputs.should_run == 'true'
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: write # push rc tag + marker
id-token: write # npm provenance
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
fetch-tags: true
- uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version: 20
registry-url: https://registry.npmjs.org
cache: npm
cache-dependency-path: gitnexus/package-lock.json
- name: Build gitnexus-shared
run: npm install && npm run build
working-directory: gitnexus-shared
- name: Install gitnexus dependencies
run: npm ci
working-directory: gitnexus
- name: Resolve rc version
id: version
shell: bash
working-directory: gitnexus
env:
BUMP_INPUT: ${{ inputs.bump }}
EVENT_NAME: ${{ github.event_name }}
PKG_NAME: gitnexus
run: |
set -euo pipefail
# 1. Current published `latest` — the floor for any new rc base.
# Only E404 ("never published") falls back to package.json; any
# other error (network, auth, malformed response) fails fast.
NPM_STDERR_LATEST="$(mktemp)"
if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then
:
else
if grep -q 'E404' "$NPM_STDERR_LATEST"; then
CURRENT_LATEST="$(node -p "require('./package.json').version")"
echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST"
else
echo "::error::npm registry unreachable for 'view version':" >&2
cat "$NPM_STDERR_LATEST" >&2
rm -f "$NPM_STDERR_LATEST"
exit 1
fi
fi
rm -f "$NPM_STDERR_LATEST"
CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}"
# 2. Full version list — needed for the counter and for active-cycle
# inference. Same E404-only fallback.
NPM_STDERR_VERSIONS="$(mktemp)"
if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then
:
else
if grep -q 'E404' "$NPM_STDERR_VERSIONS"; then
VERSIONS_JSON='[]'
echo "No published versions for $PKG_NAME yet (E404)."
else
echo "::error::npm registry unreachable for 'view versions':" >&2
cat "$NPM_STDERR_VERSIONS" >&2
rm -f "$NPM_STDERR_VERSIONS"
exit 1
fi
fi
rm -f "$NPM_STDERR_VERSIONS"
# 3. Base selection.
# - workflow_dispatch + bump ∈ {patch,minor,major} → explicit cycle
# reset from latest.
# - Everything else (push, or dispatch with bump=auto) → continue
# the highest active rc base > latest if one exists; else
# default to patch from latest.
if [ "$EVENT_NAME" = "workflow_dispatch" ] \
&& [ -n "${BUMP_INPUT:-}" ] \
&& [ "${BUMP_INPUT:-auto}" != "auto" ]; then
BASE="$(npx --yes -p semver@7 semver -i "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")"
echo "Explicit bump=$BUMP_INPUT → BASE=$BASE"
else
cat > /tmp/active_base.mjs <<'NODESCRIPT'
const latest = process.env.LATEST;
let v;
try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; }
if (!Array.isArray(v)) v = [v];
const parse = s => s.split(".").map(n => parseInt(n, 10));
const gt = (a, b) => {
const [A, B] = [parse(a), parse(b)];
for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i];
return false;
};
const bases = new Set();
for (const s of v) {
const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s);
if (m && gt(m[1], latest)) bases.add(m[1]);
}
if (!bases.size) { process.stdout.write(""); process.exit(0); }
const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1);
process.stdout.write(sorted[sorted.length - 1]);
NODESCRIPT
ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)"
if [ -n "$ACTIVE_BASE" ]; then
BASE="$ACTIVE_BASE"
echo "Continuing active rc cycle → BASE=$BASE"
else
BASE="$(npx --yes -p semver@7 semver -i patch "$CURRENT_LATEST_CLEAN")"
echo "No active rc cycle → patch bump from latest → BASE=$BASE"
fi
fi
# 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1.
cat > /tmp/next_rc.mjs <<'NODESCRIPT'
const base = process.env.BASE;
const prefix = base + "-rc.";
let v;
try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; }
if (!Array.isArray(v)) v = [v];
const ns = v
.filter(s => typeof s === "string" && s.startsWith(prefix))
.map(s => parseInt(s.slice(prefix.length), 10))
.filter(n => Number.isInteger(n) && n >= 0);
process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1));
NODESCRIPT
NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)"
RC_VERSION="${BASE}-rc.${NEXT_N}"
echo "Computed rc: $RC_VERSION"
# 5. Defensive: if the exact version already exists on the registry
# (e.g., race with another run), abort before re-publishing.
# Same E404-only pattern used above — a transient network
# failure must fail loudly, not pretend the version is missing.
NPM_STDERR_EXISTS="$(mktemp)"
if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then
rm -f "$NPM_STDERR_EXISTS"
echo "::error::Version $RC_VERSION already exists on npm — aborting."
exit 1
else
if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then
rm -f "$NPM_STDERR_EXISTS"
# Version doesn't exist — safe to proceed.
else
echo "::error::npm registry unreachable for existence check:" >&2
cat "$NPM_STDERR_EXISTS" >&2
rm -f "$NPM_STDERR_EXISTS"
exit 1
fi
fi
echo "base=$BASE" >> "$GITHUB_OUTPUT"
echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT"
echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT"
- name: Apply rc version in-CI
shell: bash
working-directory: gitnexus
run: |
set -euo pipefail
npm version "${{ steps.version.outputs.rc_version }}" \
--no-git-tag-version --allow-same-version
- name: Build gitnexus
run: npm run build
working-directory: gitnexus
- name: Dry-run publish
run: npm publish --dry-run --tag rc
working-directory: gitnexus
# ── Acquire the "rc lock" BEFORE publishing (fixes idempotency) ─────
# We create two tags and push them atomically:
# v<RC_VERSION> → annotated tag on a detached release commit
# whose tree contains the rewritten package.json
# (so the tag's source matches the npm tarball)
# rc/<HEAD_SHA> → lightweight tag on HEAD; the guard's dedup key
# If this push fails, nothing is published — safe.
# If this push succeeds but npm publish fails, the marker stays on
# the remote and blocks retries until an operator manually cleans up.
- name: Create and push rc tags
id: reltag
shell: bash
working-directory: gitnexus
env:
RC_VERSION: ${{ steps.version.outputs.rc_version }}
HEAD_SHA: ${{ needs.guard.outputs.head_sha }}
run: |
set -euo pipefail
VTAG="v${RC_VERSION}"
MARKER="rc/${HEAD_SHA}"
git config user.name 'github-actions[bot]'
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
# Detached release commit with the version bump — keeps `main`
# pristine but gives the v-tag a tree that matches the published
# package contents exactly (fixes release-integrity gap).
git add package.json package-lock.json 2>/dev/null || git add package.json
git commit -m "release: ${VTAG}" --allow-empty
RELEASE_SHA="$(git rev-parse HEAD)"
echo "Detached release commit: $RELEASE_SHA"
# Annotated release tag on the release commit.
git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG"
# Lightweight marker on the user-visible HEAD for the guard.
git tag "$MARKER" "$HEAD_SHA"
# Atomic push of both refs. If either would clobber an existing
# remote ref, the push fails and we stop before npm publish.
git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER"
echo "vtag=$VTAG" >> "$GITHUB_OUTPUT"
echo "marker=$MARKER" >> "$GITHUB_OUTPUT"
echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT"
- name: Publish to npm (rc dist-tag)
run: npm publish --provenance --access public --tag rc
working-directory: gitnexus
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
- name: Create GitHub prerelease
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2
with:
tag_name: ${{ steps.reltag.outputs.vtag }}
name: Release Candidate ${{ steps.reltag.outputs.vtag }}
prerelease: true
make_latest: 'false'
generate_release_notes: true
body: |
Automated release candidate build from `main`.
**npm:** `npm install gitnexus@rc`
**Version:** `${{ steps.version.outputs.rc_version }}`
**Target base:** `${{ steps.version.outputs.base }}` (rc #${{ steps.version.outputs.rc_n }})
**Source commit (main):** ${{ needs.guard.outputs.head_sha }}
**Release commit (versioned tree):** ${{ steps.reltag.outputs.release_sha }}
Release candidates are pre-stable builds intended for early testing.
Stable releases remain on the `latest` dist-tag.

View file

@ -0,0 +1,185 @@
name: Tree-sitter Upgrade Readiness
# Monitors readiness for upgrading tree-sitter to 0.25.x. Tracks:
# 1. Peer-dep compatibility — can each grammar install cleanly with
# tree-sitter@0.25.0 without --legacy-peer-deps?
# 2. Vendored proto drift — has coder3101/tree-sitter-proto moved
# ahead of our vendored snapshot?
# See .github/scripts/check-tree-sitter-upgrade-readiness.py for the logic.
#
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
on:
schedule:
# Daily at 09:00 UTC. Matches Dependabot's daily cadence so drift
# and dep PRs surface together.
- cron: '0 9 * * *'
workflow_dispatch:
pull_request:
paths:
- '.github/scripts/check-tree-sitter-upgrade-readiness.py'
- '.github/workflows/tree-sitter-upgrade-readiness.yml'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
permissions:
contents: read
jobs:
readiness:
name: Check upgrade readiness
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
# Needed to open/update the tracking issue on scheduled runs.
issues: write
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: ./.github/actions/setup-gitnexus
with:
build: 'false'
- name: Run upgrade readiness check
id: readiness
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set +e
python3 .github/scripts/check-tree-sitter-upgrade-readiness.py > drift-report.md
code=$?
set -e
echo "exit_code=$code" >> "$GITHUB_OUTPUT"
{
echo 'report<<DRIFT_EOF'
cat drift-report.md
echo 'DRIFT_EOF'
} >> "$GITHUB_OUTPUT"
echo "=== Report ==="
cat drift-report.md
# On PR runs, the script validates that it runs correctly. Blockers
# are informational — the scheduled run opens a tracking issue.
- name: Annotate PR with readiness status
if: github.event_name == 'pull_request' && steps.readiness.outputs.exit_code != '0'
run: |
echo "::warning::Tree-sitter 0.25 upgrade has blockers. See job output for the full readiness report."
- name: Upsert tracking issue on scheduled runs
if: >
github.event_name == 'schedule' &&
steps.readiness.outputs.exit_code != '0'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
REPORT: ${{ steps.readiness.outputs.report }}
with:
script: |
const title = 'Tree-sitter 0.25 upgrade readiness';
const report = process.env.REPORT;
const body = report + '\n\n' +
'<sub>Generated daily by `.github/workflows/tree-sitter-upgrade-readiness.yml`. ' +
'Closes automatically when all blockers are resolved.</sub>';
const { data: open } = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'open',
labels: 'tree-sitter-drift',
per_page: 10,
});
const existing = open.find(i => i.title === title);
if (existing) {
// Extract ready/total count for the changelog comment.
const readyMatch = report.match(/\*\*(\d+)\/(\d+)\*\* grammars ready/);
const blockerMatch = report.match(/\*\*(\d+) blocker/);
const ready = readyMatch ? readyMatch[1] : '?';
const total = readyMatch ? readyMatch[2] : '?';
const blockers = blockerMatch ? blockerMatch[1] : '?';
// Find grammars whose status changed by diffing the old and
// new table rows. Each row looks like:
// | `tree-sitter-foo` | ... | Ready |
// | `tree-sitter-foo` | ... | Blocking |
const parseRows = (md) => {
const map = {};
for (const m of md.matchAll(/\| `(tree-sitter-[^`]+)` \|.*?\| (\S+(?:\s\S+)*?) \|$/gm)) {
map[m[1]] = m[2].trim();
}
return map;
};
const oldRows = parseRows(existing.body || '');
const newRows = parseRows(report);
const changes = [];
for (const [name, newStatus] of Object.entries(newRows)) {
const oldStatus = oldRows[name];
if (oldStatus && oldStatus !== newStatus) {
changes.push(`\`${name}\`: ${oldStatus} → ${newStatus}`);
}
}
const today = new Date().toISOString().slice(0, 10);
let comment = `**${today}:** ${ready}/${total} ready. ${blockers} blocker(s) remaining.`;
if (changes.length > 0) {
comment += '\n\nChanges:\n' + changes.map(c => `- ${c}`).join('\n');
} else {
comment += ' No changes from previous run.';
}
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: existing.number,
body: comment,
});
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: existing.number,
body,
});
core.info(`Updated existing issue #${existing.number}`);
} else {
const { data: created } = await github.rest.issues.create({
owner: context.repo.owner,
repo: context.repo.repo,
title,
body,
labels: ['tree-sitter-drift', 'dependencies'],
});
core.info(`Opened issue #${created.number}`);
}
- name: Close tracking issue on clean scheduled runs
if: >
github.event_name == 'schedule' &&
steps.readiness.outputs.exit_code == '0'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const title = 'Tree-sitter 0.25 upgrade readiness';
const { data: open } = await github.rest.issues.listForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
state: 'open',
labels: 'tree-sitter-drift',
per_page: 10,
});
const existing = open.find(i => i.title === title);
if (existing) {
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: existing.number,
body: 'All grammars are now compatible with tree-sitter@0.25. Upgrade is ready! Closing automatically.',
});
await github.rest.issues.update({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: existing.number,
state: 'closed',
});
core.info(`Closed issue #${existing.number}`);
}

View file

@ -47,8 +47,10 @@ permissions:
issues: write
pull-requests: write
# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention".
# Single global slot — newest manual dispatch supersedes any in-flight run.
concurrency:
group: triage-sweep
group: ${{ github.workflow }}
cancel-in-progress: true
jobs:
@ -74,7 +76,7 @@ jobs:
run: pip install -r .github/scripts/triage/requirements.txt
- name: Cache FastEmbed model weights
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
with:
path: ${{ github.workspace }}/.fastembed_cache
key: fastembed-bge-small-en-v1.5

8
.gitignore vendored
View file

@ -81,6 +81,11 @@ GitNexus.sln
# Git worktrees
.worktrees/
# Vendored tree-sitter grammar build artifacts (created at install time,
# never committed). See docs/plans/2026-04-15-002-fix-tree-sitter-proto-vendor-deps-plan.md
gitnexus/vendor/**/build/
gitnexus/vendor/**/node_modules/
/github/scripts/triage/__pycache__/
.claude-flow/
@ -95,4 +100,5 @@ GitNexus.sln
.swarm/
local_docs/
local_docs/

205
AGENTS.md
View file

@ -1,116 +1,122 @@
<!-- version: 1.2.0 -->
<!--
Metadata: version, last reviewed, scope, model policy, reference docs, changelog.
Last updated: 2026-03-22
-->
<!-- version: 1.4.0 -->
<!-- Last updated: 2026-04-16 -->
Last reviewed: 2026-03-24
Last reviewed: 2026-04-16
**Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub)
This file uses a standard agent header (version, scope, model policy, reference docs, changelog), adapted for this **TypeScript/JavaScript monorepo**.
## Scope
| | |
|--|--|
| **Reads** | Repository tree as needed for the task: `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/` when present, and docs. |
| **Writes** | Only paths required for the requested change; keep diffs minimal. Update lockfiles when dependencies change. |
| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/` when applicable; shell utilities for documented CI/dev workflows. |
| **Off-limits** | User secrets (e.g. real `.env`), production deployment credentials, unrelated repositories, destructive git history operations without explicit human confirmation. |
| Boundary | Rule |
|----------|------|
| **Reads** | `gitnexus/`, `gitnexus-web/`, `eval/`, plugin packages, `.github/`, `.gitnexus/`, docs. |
| **Writes** | Only paths required for the change; keep diffs minimal. Update lockfiles when deps change. |
| **Executes** | `npm`, `npx`, `node` under `gitnexus/` and `gitnexus-web/`; `uv run` for Python under `eval/`; documented CI/dev workflows. |
| **Off-limits** | Real `.env` / secrets, production credentials, unrelated repos, destructive git ops without confirmation. |
## Model Configuration
- **Primary:** Pin in **Cursor** (Settings → model). Use a **named** model (e.g. GPT-5.2, Claude Sonnet 4.x). Avoid relying on **Auto** when reproducibility or audit trail matters.
- **Fallback:** As configured in Cursor or your organization (do not encode `latest` or wildcards in automation configs).
- **Notes:** The open-source GitNexus CLI indexer does not call an LLM. Optional Nexus AI in the web UI uses end-user provider keys and models.
- **Primary:** Use a named model (e.g. Claude Sonnet 4.x). Avoid `Auto` or unversioned `latest` when reproducibility matters.
- **Notes:** The GitNexus CLI indexer does not call an LLM.
## Execution Sequence (complex tasks)
Long sessions dilute instructions. For **multi-step** work, state up front:
For multi-step work, state up front:
1. Which rules in this file and **[GUARDRAILS.md](GUARDRAILS.md)** apply (and any relevant Signs).
2. Current **Scope** boundaries (Reads / Writes / Off-limits).
3. Which **validation commands** you will run (e.g. `cd gitnexus && npm test`, `npx tsc --noEmit`).
2. Current **Scope** boundaries.
3. Which **validation commands** you will run (`cd gitnexus && npm test`, `npx tsc --noEmit`).
On very long threads, the human may add *“Remember: apply all AGENTS.md rules”* to re-weight rule tokens against context dilution.
On long threads, *"Remember: apply all AGENTS.md rules"* re-weights these instructions against context dilution.
## Claude Code hooks
Hooks enforce gates that prompts cannot. In **Claude Code**, **PreToolUse** hooks can block tools such as `git_commit` until checks pass. Adapt to this repo: e.g. `cd gitnexus && npm test` before commit.
**PreToolUse** hooks can block tools (e.g. `git_commit`) until checks pass. Adapt to this repo: `cd gitnexus && npm test` before commit.
## Context budget (Cursor / standards)
## Context budget
Generic “core standards” playbooks are often long and stack-specific. For this monorepo, commands and gotchas live under **Cursor Cloud specific instructions** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split domain rules into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules live in **`.cursor/index.mdc`** (YAML frontmatter with `alwaysApply: true`). **Claude Code:** optionally load a **`STANDARDS.md`** only when needed (e.g. *“When writing new code, read STANDARDS.md”*) to save context.
Commands and gotchas live under **Repo reference** below and in **[CONTRIBUTING.md](CONTRIBUTING.md)**. If always-on rules grow, split into **`.cursor/rules/*.mdc`** (globs). **Cursor:** project-wide rules in `.cursor/index.mdc`. **Claude Code:** load `STANDARDS.md` only when needed.
## Reference Documentation
## Reference docs
- **This repository:** **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)**.
- **Cursor:** `.cursor/index.mdc` (always-on rules); optional `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` is deprecated — see `.cursor/index.mdc`.
- **Optional local files:** `NOTES.md` (short vendor-neutral project snapshot). For handoffs, keep notes local (e.g., a scratch file outside the repo) rather than committing `HANDOFF.md`.
- **GitNexus:** skills under `.claude/skills/gitnexus/`; machine-oriented rules in the `gitnexus:start``gitnexus:end` block below.
- **[ARCHITECTURE.md](ARCHITECTURE.md)**, **[CONTRIBUTING.md](CONTRIBUTING.md)**, **[GUARDRAILS.md](GUARDRAILS.md)**
- **Call-resolution DAG:** See ARCHITECTURE.md § Call-Resolution DAG. Typed 6-stage DAG inside the `parse` phase; language-specific behavior behind `inferImplicitReceiver` / `selectDispatch` hooks on `LanguageProvider`. Shared code in `gitnexus/src/core/ingestion/` must not name languages. Types: `gitnexus/src/core/ingestion/call-types.ts`.
- **Cursor:** `.cursor/index.mdc` (always-on); `.cursor/rules/*.mdc` (glob-scoped). Legacy `.cursorrules` deprecated.
- **GitNexus:** skills in `.claude/skills/gitnexus/`; MCP rules in `gitnexus:start` block below.
## Changelog
| Date | Version | Change |
|------|---------|--------|
| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication (was inlined in Reference Docs bullet). |
| 2026-03-23 | 1.1.0 | Updated agent instructions (sections, references, Cursor layout). |
| 2026-03-22 | 1.0.0 | Added structured agent header and changelog. |
| 2026-04-16 | 1.4.0 | Fixed: web UI description, pre-commit behavior, MCP tools (7->16), added gitnexus-shared, removed stale vite-plugin-wasm gotcha. |
| 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. |
| 2026-03-24 | 1.2.0 | Fixed gitnexus:start block duplication. |
| 2026-03-23 | 1.1.0 | Updated agent instructions, references, Cursor layout. |
| 2026-03-22 | 1.0.0 | Initial structured header and changelog. |
---
<!-- gitnexus:start -->
# GitNexus — Code Intelligence
This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relationships, 185 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
Indexed as **GitNexus** (4325 symbols, 10556 relationships, 300 execution flows). Use MCP tools to understand code, assess impact, and navigate safely.
> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first.
> If any tool warns the index is stale, run `npx gitnexus analyze` first.
## Always Do
- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user.
- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows.
- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits.
- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance.
- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`.
- **MUST run impact analysis before editing any symbol.** `gitnexus_impact({target: "symbolName", direction: "upstream"})` — report blast radius to the user.
- **MUST run `gitnexus_detect_changes()` before committing** — verify only expected symbols and flows are affected.
- **MUST warn the user** if impact returns HIGH or CRITICAL risk.
- Explore unfamiliar code with `gitnexus_query({query: "concept"})` (process-grouped, ranked) instead of grepping.
- Full context on a symbol: `gitnexus_context({name: "symbolName"})`.
## When Debugging
1. `gitnexus_query({query: "<error or symptom>"})` — find execution flows related to the issue
2. `gitnexus_context({name: "<suspect function>"})` see all callers, callees, and process participation
3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step
4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed
1. `gitnexus_query({query: "<error or symptom>"})` — find related execution flows
2. `gitnexus_context({name: "<suspect function>"})` — callers, callees, process participation
3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace flow step by step
4. Regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})`
## When Refactoring
- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`.
- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code.
- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed.
- **Rename:** `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Graph edits are safe; text_search edits need manual review.
- **Extract/Split:** `gitnexus_context` (incoming/outgoing refs) then `gitnexus_impact` (upstream callers) before moving code.
- **After any refactor:** `gitnexus_detect_changes({scope: "all"})` to verify scope.
## Never Do
- NEVER edit a function, class, or method without first running `gitnexus_impact` on it.
- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis.
- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph.
- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope.
- Edit a symbol without running `gitnexus_impact` first.
- Ignore HIGH/CRITICAL risk warnings.
- Rename with find-and-replace — use `gitnexus_rename`.
- Commit without `gitnexus_detect_changes()`.
- Add language-specific behavior to shared ingestion code (`gitnexus/src/core/ingestion/`) — use a `LanguageProvider` hook. Seeing `provider.mroStrategy === 'xxx'` or an import from `languages/xxx.ts` in shared code means stop and add a hook.
## Tools Quick Reference
| Tool | When to use | Command |
| Tool | When to use | Example |
|------|-------------|---------|
| `list_repos` | Discover indexed repos | `gitnexus_list_repos({})` |
| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` |
| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` |
| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` |
| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` |
| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` |
| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` |
| `api_impact` | Pre-change API route impact | `gitnexus_api_impact({route: "/api/users", method: "GET"})` |
| `route_map` | Route → handler → consumer map | `gitnexus_route_map({})` |
| `tool_map` | MCP/RPC tool definitions | `gitnexus_tool_map({})` |
| `shape_check` | Response shape vs consumer access | `gitnexus_shape_check({route: "/api/users"})` |
| `group_list` | List repo groups | `gitnexus_group_list({})` |
| `group_query` | Cross-repo search in a group | `gitnexus_group_query({name: "myGroup", query: "auth"})` |
| `group_sync` | Rebuild group Contract Registry | `gitnexus_group_sync({name: "myGroup"})` |
| `group_contracts` | Inspect group contracts | `gitnexus_group_contracts({name: "myGroup"})` |
| `group_status` | Group staleness report | `gitnexus_group_status({name: "myGroup"})` |
## Impact Risk Levels
| Depth | Meaning | Action |
|-------|---------|--------|
| d=1 | WILL BREAK — direct callers/importers | MUST update these |
| d=1 | WILL BREAK — direct callers/importers | MUST update |
| d=2 | LIKELY AFFECTED — indirect deps | Should test |
| d=3 | MAY NEED TESTING — transitive | Test if critical path |
@ -118,87 +124,80 @@ This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relation
| Resource | Use for |
|----------|---------|
| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness |
| `gitnexus://repo/GitNexus/context` | Codebase overview, index freshness |
| `gitnexus://repo/GitNexus/clusters` | All functional areas |
| `gitnexus://repo/GitNexus/processes` | All execution flows |
| `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace |
## Self-Check Before Finishing
Before completing any code modification task, verify:
1. `gitnexus_impact` was run for all modified symbols
2. No HIGH/CRITICAL risk warnings were ignored
3. `gitnexus_detect_changes()` confirms changes match expected scope
4. All d=1 (WILL BREAK) dependents were updated
2. No HIGH/CRITICAL warnings were ignored
3. `gitnexus_detect_changes()` confirms expected scope
4. All d=1 dependents were updated
## Keeping the Index Fresh
After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it:
```bash
npx gitnexus analyze
npx gitnexus analyze # basic refresh
npx gitnexus analyze --embeddings # preserve embeddings
```
If the index previously included embeddings, preserve them by adding `--embeddings`:
Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). Running without `--embeddings` deletes existing vectors.
```bash
npx gitnexus analyze --embeddings
```
> Claude Code: PostToolUse hook handles this after `git commit` and `git merge`.
To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.**
## CLI Skills
> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`.
## CLI
| Task | Read this skill file |
|------|---------------------|
| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
| Task | Skill file |
|------|-----------|
| Architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
| Blast radius / "What breaks?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
| Debugging / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
| Refactoring | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
| Tools/resources/schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
| CLI commands (index, status, clean, wiki) | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
<!-- gitnexus:end -->
## Cursor Cloud specific instructions
## Repo reference
### Repository structure
### Packages
This is a monorepo with two main products and supporting config packages:
| Component | Path | Purpose |
|-----------|------|---------|
| **GitNexus CLI/Core** | `gitnexus/` | Main product — TypeScript CLI, indexing pipeline, MCP server. Published to npm. |
| **GitNexus Web UI** | `gitnexus-web/` | React/Vite browser app — graph explorer + AI chat. Runs entirely in WASM. |
| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace (no build). |
| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor (no build). |
| SWE-bench Eval | `eval/` | Python evaluation harness (optional; needs Docker + LLM API keys). |
| Package | Path | Purpose |
|---------|------|---------|
| **CLI/Core** | `gitnexus/` | TypeScript CLI, indexing pipeline, MCP server. Published to npm. |
| **Web UI** | `gitnexus-web/` | React/Vite thin client. All queries via `gitnexus serve` HTTP API. |
| **Shared** | `gitnexus-shared/` | Shared TypeScript types and constants. |
| Claude Plugin | `gitnexus-claude-plugin/` | Static config for Claude marketplace. |
| Cursor Integration | `gitnexus-cursor-integration/` | Static config for Cursor editor. |
| Eval | `eval/` | Python evaluation harness (Docker + LLM API keys). |
### Running services
- **CLI/Core**: `cd gitnexus && npm run dev` (tsx watch mode) or `npm run build && node dist/cli/index.js <command>`
- **Web UI**: `cd gitnexus-web && npm run dev` (Vite on port 5173)
- **Backend mode**: `cd <indexed-repo> && node /workspace/gitnexus/dist/cli/index.js serve` (HTTP API on port 3741 by default)
```bash
cd gitnexus && npm run dev # CLI: tsx watch mode
cd gitnexus-web && npm run dev # Web UI: Vite on port 5173
npx gitnexus serve # HTTP API on port 4747 (from any indexed repo)
```
### Testing
**CLI / Core (`gitnexus/`)**
- **Unit tests**: `cd gitnexus && npm test` (vitest, ~2000 tests)
- **Integration tests**: `cd gitnexus && npm run test:integration` (vitest, ~1850 tests). Two LadybugDB file-locking tests (`lbug-core-adapter`, `search-core`) may fail in containerized environments due to `/tmp` locking limitations — this is a known environment issue, not a code bug.
- **TypeScript check**: `cd gitnexus && npx tsc --noEmit`
- `npm test` — full vitest suite (~2000 tests)
- `npm run test:unit` — unit tests only
- `npm run test:integration` — integration (~1850 tests). LadybugDB file-locking tests may fail in containers (known env issue).
- `npx tsc --noEmit` — typecheck
**Web UI (`gitnexus-web/`)**
- **Unit tests**: `cd gitnexus-web && npm test` (vitest, ~200 tests)
- **E2E tests**: `cd gitnexus-web && E2E=1 npx playwright test` (Playwright, 5 tests — requires `gitnexus serve` + `npm run dev` running)
- **TypeScript check**: `cd gitnexus-web && npx tsc -b --noEmit`
- `npm test` — vitest (~200 tests)
- `npm run test:e2e` — Playwright (7 spec files; requires `gitnexus serve` + `npm run dev`)
- `npx tsc -b --noEmit` — typecheck
No separate lint command is configured; TypeScript strict checking serves as the primary static analysis.
**Pre-commit hook** (`.husky/pre-commit`): formatting (prettier via lint-staged) + typecheck for staged packages. Tests do **not** run in pre-commit — CI only.
### Gotchas
- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift). Native tree-sitter bindings require `python3`, `make`, and `g++` to be present.
- `tree-sitter-kotlin` and `tree-sitter-swift` are optional dependencies — install warnings for these are expected and non-blocking.
- The Web UI uses `vite-plugin-wasm` and requires `Cross-Origin-Opener-Policy`/`Cross-Origin-Embedder-Policy` headers for `SharedArrayBuffer` (handled automatically by Vite dev server).
- There is no ESLint/Prettier configuration in this repo.
- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (patches tree-sitter-swift, builds tree-sitter-proto). Native bindings need `python3`, `make`, `g++`.
- `tree-sitter-kotlin` and `tree-sitter-swift` are optional — install warnings expected.
- ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`.

View file

@ -1,123 +1,365 @@
# Architecture — GitNexus
This repository is a **monorepo** with two main products: the **CLI / MCP package** (`gitnexus/`) and the **browser UI** (`gitnexus-web/`). Supporting folders ship editor integrations and plugins without changing the core graph engine.
Monorepo: **CLI/MCP** (`gitnexus/`) + **browser UI** (`gitnexus-web/`).
## Repository layout
| Path | Role |
|------|------|
| `gitnexus/` | Published npm package `gitnexus`: CLI, MCP server (stdio), local HTTP API for bridge mode, ingestion pipeline, LadybugDB graph, embeddings (optional). |
| `gitnexus-web/` | Vite + React UI: in-browser indexing (WASM), graph visualization, optional connection to `gitnexus serve`. |
| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Packaged **skills** and plugin metadata so agents discover the same workflows as documented in `AGENTS.md`. |
| `eval/` | Evaluation harnesses and docs for benchmarking tool usage. |
| `.github/` | CI workflows (quality, unit, integration, E2E) and composite actions. |
| `gitnexus/` | npm package `gitnexus`: CLI, MCP server (stdio), HTTP API, ingestion pipeline, LadybugDB graph, embeddings. |
| `gitnexus-web/` | Vite + React thin client: graph explorer + AI chat. All queries via `gitnexus serve` HTTP API. |
| `gitnexus-shared/` | Shared TypeScript types and constants (consumed by CLI and Web). |
| `.claude/`, `gitnexus-claude-plugin/`, `gitnexus-cursor-integration/` | Agent skills and plugin metadata. |
| `eval/` | Evaluation harnesses for benchmarking tool usage. |
| `.github/` | CI workflows + composite actions (`setup-gitnexus/`, `setup-gitnexus-web/`). |
## End-to-end flow: index → graph → tools
1. **Ingestion** (`gitnexus analyze`)
- Entry: `gitnexus/src/cli/analyze.ts``runPipelineFromRepo` in `gitnexus/src/core/ingestion/pipeline.ts`.
- Walks the git working tree, parses supported languages via **Tree-sitter**, resolves imports/calls/inheritance, detects **communities** and **processes** (execution flows), and builds an in-memory **knowledge graph** (`gitnexus/src/core/graph/`).
- Output is loaded into **LadybugDB** under **`.gitnexus/`** at the repo root (`lbug/`, `meta.json`, etc.). Optional **FTS** indexes and **embeddings** attach to the same store.
- The repo is registered in **`~/.gitnexus/registry.json`** so MCP can find it from any working directory.
1. **Ingestion**`analyze.ts``runFullAnalysis` (`run-analyze.ts`) → `runPipelineFromRepo` (`pipeline.ts`). DAG of 12 phases builds a `KnowledgeGraph` in memory, then loads into LadybugDB under `.gitnexus/`. Repo registered in `~/.gitnexus/registry.json` for MCP discovery.
2. **Persistence & metadata**
- `gitnexus/src/storage/repo-manager.ts` — paths, registry, cleanup of legacy Kuzu artifacts.
- `gitnexus/src/core/lbug/lbug-adapter.ts` — graph load, queries, embedding restore batches.
2. **Persistence**`repo-manager.ts` (paths, registry, KuzuDB cleanup). `lbug-adapter.ts` (graph load, queries, embedding batches).
3. **Query & agents**
- **MCP (stdio):** `gitnexus/src/cli/mcp.ts` → `startMCPServer``LocalBackend` (`gitnexus/src/mcp/local/local-backend.ts`) opens registered repos and serves **tools** from `gitnexus/src/mcp/tools.ts` and **resources** from `gitnexus/src/mcp/resources.ts`.
- **Bridge HTTP:** `gitnexus/src/cli/serve.ts` → Express app in `gitnexus/src/server/api.ts` (CORS-limited) exposes REST + MCP-over-HTTP for the web UI.
- **CLI tools (no MCP):** `gitnexus query`, `context`, `impact`, `cypher` in `gitnexus/src/cli/tool.ts` call the same backend for scripts and CI.
3. **Query layer** — three interfaces to the same backend:
- **MCP (stdio):** `mcp.ts` → `LocalBackend` → tools (`tools.ts`) + resources (`resources.ts`)
- **HTTP bridge:** `serve.ts` → Express (`api.ts`, `mcp-http.ts`) for web UI
- **CLI direct:** `gitnexus query|context|impact|cypher` in `tool.ts`
4. **Staleness**
- `gitnexus/src/mcp/staleness.ts` compares indexed `lastCommit` to `HEAD` and surfaces hints when the graph is behind git.
4. **Staleness**`staleness.ts` compares indexed `lastCommit` to `HEAD`, surfaces hints.
## MCP tools (summary)
## MCP tools
| Tool | Purpose |
|------|---------|
| `list_repos` | Discover indexed repositories when more than one is registered. |
| `query` | Natural-language / keyword search over the graph (hybrid BM25 + optional vectors). |
| `cypher` | Ad hoc **Cypher** against the schema (see resource `gitnexus://repo/{name}/schema`). |
| `context` | Callers, callees, processes for one symbol (with disambiguation). |
| `impact` | Blast radius (upstream/downstream) with depth and risk summary. |
| `detect_changes` | Map git diffs to affected symbols and processes. |
| `rename` | Graph-assisted rename with `dry_run` preview (`graph` vs `text_search` confidence). |
| `list_repos` | Discover indexed repos |
| `query` | Hybrid BM25 + vector search over the graph |
| `cypher` | Ad hoc Cypher against the schema |
| `context` | Callers, callees, processes for one symbol |
| `impact` | Blast radius (upstream/downstream) with risk summary |
| `detect_changes` | Map git diffs to affected symbols and processes |
| `rename` | Graph-assisted multi-file rename with `dry_run` preview |
| `api_impact` | Pre-change impact report for an API route handler |
| `route_map` | API route → handler → consumer mappings |
| `tool_map` | MCP/RPC tool definitions and handlers |
| `shape_check` | Response shape vs consumer property access mismatches |
| `group_list` | List repo groups or details for one group |
| `group_query` | Cross-repo search in a group (reciprocal rank fusion) |
| `group_sync` | Rebuild group Contract Registry (`contracts.json`) |
| `group_contracts` | Inspect group contracts and cross-links |
| `group_status` | Index and Contract Registry staleness per repo in a group |
## Where to change what
| If you are changing… | Start in… |
|----------------------|-----------|
| CLI commands / flags | `gitnexus/src/cli/` (`index.ts`, per-command modules). |
| Parsing or graph construction | `gitnexus/src/core/ingestion/` (pipeline, processors, resolvers, type-extractors). |
| Graph schema / DB access | `gitnexus/src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`), `gitnexus/src/mcp/core/lbug-adapter.ts` if MCP-specific. |
| MCP protocol, tools, resources | `gitnexus/src/mcp/server.ts`, `tools.ts`, `resources.ts`. |
| Search ranking | `gitnexus/src/core/search/` (BM25, hybrid fusion). |
| Embeddings | `gitnexus/src/core/embeddings/`, phases in `analyze.ts`. |
| Wiki generation | `gitnexus/src/core/wiki/`. |
| Web UI behavior | `gitnexus-web/src/` (components, workers, graph client). |
| CI | `.github/workflows/*.yml`, `.github/actions/setup-gitnexus/`. |
| Concern | Start in |
|---------|----------|
| CLI commands/flags | `src/cli/` (`index.ts`, per-command modules) |
| Parsing/graph construction | `src/core/ingestion/pipeline-phases/` + `pipeline.ts` |
| Graph schema/DB | `src/core/lbug/` (`schema.ts`, `lbug-adapter.ts`) |
| MCP tools/resources | `src/mcp/server.ts`, `tools.ts`, `resources.ts` |
| Search ranking | `src/core/search/` (BM25, hybrid fusion) |
| Embeddings | `src/core/embeddings/` + `src/core/run-analyze.ts` |
| Wiki generation | `src/core/wiki/` |
| Language support | `src/core/ingestion/languages/` + `tree-sitter-queries.ts` + `gitnexus-shared/src/languages.ts` |
| Import resolution | `src/core/ingestion/import-processor.ts` + `import-resolvers/configs/` + `model/resolution-context.ts` |
| Call resolution/MRO | `src/core/ingestion/call-processor.ts` + `model/resolve.ts` |
| Type extraction | `src/core/ingestion/type-extractors/` |
| Worker pool | `src/core/ingestion/workers/` |
| Web UI | `gitnexus-web/src/` |
| CI | `.github/workflows/*.yml`, `.github/actions/` |
> Paths above are relative to `gitnexus/` unless they start with `gitnexus-web/` or `.github/`.
---
## Pipeline Phase DAG
12 phases defined in `gitnexus/src/core/ingestion/pipeline-phases/`, each with explicit `deps` and typed output.
```
scan → structure → [markdown, cobol] → parse → [routes, tools, orm]
→ crossFile → mro → communities → processes
```
| Phase | File | Deps | Output |
|-------|------|------|--------|
| `scan` | `scan.ts` | (root) | File paths + sizes |
| `structure` | `structure.ts` | `scan` | File/Folder nodes, CONTAINS edges, `allPathSet` |
| `markdown` | `markdown.ts` | `structure` | Section nodes, cross-link edges from .md/.mdx |
| `cobol` | `cobol.ts` | `structure` | COBOL program/paragraph/section nodes (regex, no tree-sitter) |
| `parse` | `parse.ts` + `parse-impl.ts` | `structure`, `markdown`, `cobol` | Symbol nodes, IMPORTS/CALLS/EXTENDS edges, extracted routes/tools/ORM queries |
| `routes` | `routes.ts` | `parse` | Route nodes + HANDLES_ROUTE edges (Next.js, Expo, PHP, decorators) |
| `tools` | `tools.ts` | `parse` | Tool nodes + HANDLES_TOOL edges |
| `orm` | `orm.ts` | `parse` | QUERIES edges (Prisma, Supabase) |
| `crossFile` | `cross-file.ts` + `cross-file-impl.ts` | `parse`, `routes`, `tools`, `orm` | Cross-file type propagation in topological import order |
| `mro` | `mro.ts` | `crossFile`, `structure` | METHOD_OVERRIDES + METHOD_IMPLEMENTS edges |
| `communities` | `communities.ts` | `mro`, `structure` | Community nodes + MEMBER_OF edges (Leiden algorithm) |
| `processes` | `processes.ts` | `communities`, `routes`, `tools`, `structure` | Process nodes + STEP_IN_PROCESS edges |
**Non-phase files in the same directory:** `parse-impl.ts`, `cross-file-impl.ts` (implementation), `wildcard-synthesis.ts` (whole-module import expansion), `orm-extraction.ts` (sequential ORM fallback), `types.ts`, `runner.ts`, `index.ts`.
### DAG runner
`runner.ts` — static phase graph, no plugins, compile-time type safety.
1. **Validation** — Kahn's topological sort. Rejects on: duplicate names, missing deps, cycles (DFS traces the concrete cycle path, e.g., `A -> B -> C -> A`, plus count of transitively blocked dependents).
2. **Execution** — sequential in topological order. Each phase receives:
- `ctx: PipelineContext` — shared mutable `KnowledgeGraph`, `repoPath`, progress callback, options
- `deps: ReadonlyMap<string, PhaseResult>`**declared deps only** (runner filters the results map to prevent hidden coupling)
3. **Error handling** — wraps phase errors with the phase name, emits terminal `error` progress event, swallows progress handler errors to preserve the original cause.
4. **Timing** — per-phase `durationMs` in `PhaseResult`, dev-mode console logging.
**Design patterns:**
- **Single graph accumulator** — all phases mutate the same `KnowledgeGraph` in `ctx`; the graph is the primary output.
- **Typed phase access**`getPhaseOutput<T>(deps, 'name')` for type-safe upstream results.
- **Binding accumulator lifecycle** — created in `parse`, disposed by `crossFile` (in `finally`). No other phase should take ownership.
- **Skippable phases**`skipGraphPhases` omits MRO/communities/processes (faster tests). `skipWorkers` forces sequential parsing.
### How to add a new phase
1. Create `pipeline-phases/my-phase.ts` with a `PipelinePhase<MyOutput>` (name, deps, execute)
2. Export from `pipeline-phases/index.ts`
3. Add to `buildPhaseList()` in `pipeline.ts`
```typescript
import type { PipelinePhase, PhaseResult } from './types.js';
import { getPhaseOutput } from './types.js';
import type { ParseOutput } from './parse.js';
export interface MyPhaseOutput { /* ... */ }
export const myPhase: PipelinePhase<MyPhaseOutput> = {
name: 'myPhase',
deps: ['parse'],
async execute(ctx, deps) {
const { allPaths } = getPhaseOutput<ParseOutput>(deps, 'parse');
// ... write to ctx.graph ...
return { /* typed output */ };
},
};
```
---
## Call-Resolution DAG
Typed 6-stage pipeline in `call-processor.ts` (inside the `parse` phase) that resolves method/function calls and emits CALLS edges. Language behavior plugs in at two `LanguageProvider` hook points (stages 34); shared code names no languages. Scope: call resolution only — import resolution, type extraction, heritage, and symbol-table population live in other phases.
### Stages
```
extract-call ──▶ classify-form ──▶ infer-receiver ──▶ select-dispatch ──▶ resolve-target ──▶ emit-edge
(1) (2) (3) [hook] (4) [hook] (5) (6)
```
| Stage | Produces | Location |
|-------|----------|----------|
| **extract-call** | `ExtractedCallSite` (name, form, receiver, argCount) | `call-extractors/` (per-language); runs in worker |
| **classify-form** | callForm (`free`/`member`/`constructor`) + arity | `call-analysis.ts``inferCallForm`; shared, runs in worker |
| **infer-receiver** | `ReceiverEnriched` (receiver type finalized) | `call-processor.ts`; shared default chain, then `inferImplicitReceiver` hook |
| **select-dispatch** | `DispatchDecision` (primary, fallback, ancestryView) | `selectDispatch` hook, falls back to shared default |
| **resolve-target** | `TieredCandidates` | `model/resolve.ts``lookupMethodByOwnerWithMRO` (MRO walk) |
| **emit-edge** | CALLS edge in graph | `call-processor.ts`; writes edge with confidence tier |
### Provider hooks
Both hooks are optional on `LanguageProvider`. Ruby is the only current implementer.
**`inferImplicitReceiver`** — called after shared infer-receiver defaults. Returns `ImplicitReceiverOverride | null`.
| | |
|---|---|
| Inputs | `calledName`, `callForm`, `receiverName`, `receiverTypeName`, `callNode` (AST), `filePath` |
| Non-null fields | `callForm`, `receiverName`, `receiverTypeName` (required); `receiverSource: 'implicit-self'` (fixed); `hint?` (opaque, passed to `selectDispatch`) |
| Null | Keep existing `ReceiverEnriched` state |
**`selectDispatch`** — called after infer-receiver (including hook). Returns `DispatchDecision | null`; null uses shared default (constructor → `primary:'constructor'`; typed receiver → `primary:'owner-scoped'`; else → `primary:'free'`).
| | |
|---|---|
| Inputs | `calledName`, `callForm`, `receiverName`, `receiverTypeName`, `receiverSource`, `hint` |
| Non-null fields | `primary: 'owner-scoped' \| 'free' \| 'constructor'`; `fallback?: 'free-arity-narrowed'`; `ancestryView?: 'instance' \| 'singleton'`; `hint?` |
**`DispatchDecision` field semantics:**
- `primary: 'owner-scoped'` — MRO walk from receiver's type; used when receiver type is known.
- `fallback: 'free-arity-narrowed'` — after owner-scoped miss, search free-call candidates by arity only (Ruby uses this for implicit-self calls that miss their owner's MRO).
- `ancestryView: 'singleton'` — walk singleton/class ancestry instead of instance ancestry (Ruby `def self.foo` bodies, so `extend`-ed methods are found).
### Adding language behavior
1. **Implicit receivers** — implement `inferImplicitReceiver`: return null if call already has a receiver; otherwise use `findEnclosingClassInfo` (`ast-helpers.ts`) to find the enclosing context, return `ImplicitReceiverOverride` with `receiverSource: 'implicit-self'`, and optionally set `hint` for `selectDispatch`.
2. **Custom dispatch** — implement `selectDispatch`: inspect `receiverSource` and `hint`, return `DispatchDecision` with `primary`, optional `fallback`, optional `ancestryView`; return null to keep shared defaults.
3. **MRO strategy** — confirm `mroStrategy` is `'first-wins'`, `'c3'`, `'ruby-mixin'`, or `'none'`; consumed by `lookupMethodByOwnerWithMRO`.
**Ruby example** (`languages/ruby.ts` + `utils/ruby-self-call.ts`): `inferImplicitReceiver` rewrites bare-identifier calls to `self.method` and sets `hint` to `'instance'`/`'singleton'`; `selectDispatch` uses hint for `ancestryView` and adds `fallback: 'free-arity-narrowed'` for implicit-self calls.
### Code references
| Module | Purpose |
|--------|---------|
| `core/ingestion/call-types.ts` | DAG types: `ReceiverEnriched`, `DispatchDecision`, `ImplicitReceiverOverride` |
| `core/ingestion/language-provider.ts` | Hook signatures: `inferImplicitReceiver`, `selectDispatch` |
| `core/ingestion/call-processor.ts` | `processCalls`: stages 36 |
| `core/ingestion/model/resolve.ts` | `lookupMethodByOwnerWithMRO`: stage 5 MRO walk |
| `core/ingestion/languages/ruby.ts` | Both hooks + `mroStrategy: 'ruby-mixin'` |
| `core/ingestion/utils/ruby-self-call.ts` | Bare-call rewrite for `inferImplicitReceiver` |
---
## Language-agnostic graph feeding
16 languages → single unified graph. Four abstraction layers:
```
Unified Graph Schema (44 node types, 21 relationship types)
Unified Resolution (3-tier name lookup + MRO walk)
Language Providers (import semantics, type config, export checker, MRO strategy)
Tree-Sitter Queries (per-language S-expressions, unified capture tags)
```
### Language providers
Each language implements `LanguageProvider` (`language-provider.ts`). Key fields:
| Field | Purpose |
|-------|---------|
| `id`, `extensions` | Language identity and file matching |
| `treeSitterQueries` | S-expression queries for AST extraction |
| `importSemantics` | `named` / `wildcard-leaf` / `wildcard-transitive` / `namespace` |
| `importResolver` | Language-specific path → file resolution |
| `exportChecker` | Public/exported symbol detection |
| `typeConfig` | Type annotation extraction rules |
| `mroStrategy` | `first-wins` / `c3` / `none` |
16 providers in `languages/index.ts` via `satisfies Record<SupportedLanguages, LanguageProvider>` — missing a language is a compile error.
### Unified capture tags
Per-language tree-sitter queries use different AST node names but produce the **same semantic capture tags**: `@definition.class`, `@definition.function`, `@call.name`, `@import.source`, `@heritage.extends`. Downstream extraction needs no language branching. Defined in `tree-sitter-queries.ts`.
### Import resolution
Per-language import resolution uses the **configs + factory** pattern (like call/method/class extractors). Each language declares an `ImportResolutionConfig` in `import-resolvers/configs/`, listing an ordered chain of `ImportResolverStrategy` functions. `createImportResolver()` (in `resolver-factory.ts`) composes them: first non-null result wins. Low-level helpers shared across strategies live alongside the configs in `import-resolvers/` (e.g. `go.ts`, `rust.ts`, `python.ts`).
Unified 3-tier algorithm (`model/resolution-context.ts`), per-language `importSemantics` controls which tier activates:
| Tier | Confidence | Mechanism |
|------|-----------|-----------|
| 1 — same-file | 0.95 | Symbol table for caller's file |
| 2 — import-scoped | 0.9 | `NamedImportMap` chains (named) or all files in `importMap` (wildcard) |
| 3 — global | 0.5 | O(1) index lookups: class, impl, callable. Fallback only |
| Import strategy | Languages | Behavior |
|----------------|-----------|----------|
| `named` | TS, JS, Java, C#, Rust, PHP, Kotlin | Only explicitly imported names visible |
| `wildcard-leaf` | Go, Ruby, Swift, Dart | Whole-package import, no transitive re-exports |
| `wildcard-transitive` | C, C++ | `#include` closure chains through re-exports |
| `namespace` | Python | Module aliases resolved at call site |
### Chunked parse-and-resolve
`parse` processes files in ~20 MB byte-budget chunks to bound memory. Per chunk:
1. Worker pool dispatches files (or sequential fallback via `skipWorkers`)
2. Each worker: detect language → load grammar → run queries → return unified `ParseWorkerResult`
3. Synthesize wildcard bindings (`wildcard-synthesis.ts`)
4. Resolve imports and heritage
5. Collect `BindingAccumulator` entries for cross-file propagation
Workers: `workers/worker-pool.ts`, `workers/parse-worker.ts`.
### Heritage and MRO
All languages emit unified `ExtractedHeritage` (child, parent, `EXTENDS`/`IMPLEMENTS`). MRO phase walks the heritage graph using per-language strategy:
- **`first-wins`** — Java, C#, C++, TS, Ruby, Go
- **`c3`** — Python (C3 linearization)
- **`none`** — single-inheritance languages
Unified walk: `lookupMethodByOwnerWithMRO()` in `model/resolve.ts`.
---
## Full analysis flow
`runFullAnalysis` in `run-analyze.ts` orchestrates everything around the pipeline:
```
CLI (analyze.ts) → runFullAnalysis(repoPath, options, callbacks)
1. Early exit if lastCommit == HEAD (unless --force) [0%]
2. Cache existing embeddings from prior index [0%]
3. runPipelineFromRepo() → KnowledgeGraph [0-60%]
4. Clean up legacy KuzuDB files [60%]
5. initLbug() → loadGraphToLbug() via CSV streaming [60-85%]
6. Create FTS indexes (File, Function, Class, Method...) [85-90%]
7. Restore cached embeddings (batch insert) [88%]
8. Generate new embeddings if --embeddings [90-98%]
9. Save metadata + register repo + update .gitignore [98-100%]
10. Generate AI context files (AGENTS.md, CLAUDE.md) [100%]
```
**Options:** `--force` (rebuild regardless), `--embeddings` (opt-in, skipped if >50k nodes), `--skipGit`, `--noStats`.
## Storage
```
<repo>/.gitnexus/
├── lbug # LadybugDB database
├── lbug.wal # Write-ahead log
├── lbug.lock # Single-writer lock
└── meta.json # lastCommit, indexedAt, stats
~/.gitnexus/
└── registry.json # Global repo registry (MCP discovery)
```
Managed by `repo-manager.ts`.
## LadybugDB schema
Defined in `lbug/schema.ts`. Separate node tables per type, single `CodeRelation` table.
**Node tables:** File, Folder, Function, Class, Interface, Method, Constructor, CodeElement, Struct, Enum, Macro, Typedef, Union, Namespace, Trait, Impl, TypeAlias, Const, Static, Property, Record, Delegate, Annotation, Template, Module, Community, Process, Route, Tool, Section, Embedding.
**Relation types** (`CodeRelation.type`): CONTAINS, DEFINES, CALLS, IMPORTS, EXTENDS, IMPLEMENTS, HAS_METHOD, HAS_PROPERTY, ACCESSES, METHOD_OVERRIDES, METHOD_IMPLEMENTS, MEMBER_OF, STEP_IN_PROCESS, HANDLES_ROUTE, FETCHES, HANDLES_TOOL, ENTRY_POINT_OF.
## Embeddings and search
**Embeddings** (`src/core/embeddings/`): Snowflake arctic-embed-xs (384D). Embeddable: File, Function, Class, Method, Interface. Incremental via SHA1 content hash. Separate `Embedding` table.
**Search** (`src/core/search/`): Hybrid BM25 + semantic vector, merged via Reciprocal Rank Fusion (K=60).
## Known limitations
### Overloaded method resolution
Method and Constructor node IDs include an arity suffix (`#<paramCount>`) to
disambiguate overloaded methods. Two overloads with different parameter counts
produce distinct graph nodes: `Method:file:Class.method#1` vs
`Method:file:Class.method#2`.
Node IDs use arity suffix (`#<paramCount>`): `Method:file:Class.method#1` vs `#2`.
**Same-arity overload disambiguation:** When two overloads share the same
parameter count but differ in types (e.g. `save(int)` vs `save(String)`), a
type-hash suffix `~type1,type2` is appended to produce distinct node IDs:
`Method:file:Class.save#1~int` vs `Method:file:Class.save#1~String`. The suffix
is only added when a same-arity collision is detected within a class and all
parameters have non-null type annotations. Languages without type info (Python,
Ruby, JS) fall back to arity-only IDs. TypeScript/JavaScript overload signatures
are intentionally excluded from type-hashing because they are declaration-only
contracts that should collapse to the implementation body's node ID. See issue
\#651.
**Same-arity disambiguation:** type-hash suffix `~type1,type2` when collision detected and type annotations present. Languages without types (Python, Ruby, JS) use arity-only. TS/JS overload signatures excluded (collapse to implementation body). See #651.
**C++ const-qualified overload disambiguation:** Methods overloaded by const
qualification (e.g. `begin()` vs `begin() const`) are disambiguated via an
`isConst` property and a `$const` ID suffix appended to the const-qualified
variant when a non-const collision exists. The `$const` suffix appears after the
type-hash suffix: e.g. `Method:file:Container.begin#0$const`.
**C++ const-qualified:** `$const` suffix after type-hash when non-const collision exists: `Method:file:Container.begin#0$const`.
**Generic/template type preservation in type-hash:** The type-hash suffix uses
`rawType` (full AST text including generic/template args) rather than the
simplified `type` from `extractSimpleTypeName`. This means C++ template overloads
like `process(vector<int>)` vs `process(vector<string>)` produce distinct IDs:
`~vector<int>` vs `~vector<std::string>`. Java generic overloads like
`process(List<String>)` vs `process(List<Integer>)` are a compile error due to
type erasure, so this gap is theoretical for Java.
**Generic/template types:** type-hash uses `rawType` (full AST text including generics): `~vector<int>` vs `~vector<std::string>`.
**ID stability on first overload:** Type and const tags are collision-only. When
a class has `save(int)` as its only `save` method, the ID is `save#1` (no tag).
Adding `save(String)` changes the original to `save#1~int`. This is correct for
fresh analysis but means IDs are not stable across overload additions. Future
incremental re-analysis should account for this.
**ID stability:** collision-only tags mean IDs change when overloads are added. `save#1` becomes `save#1~int` when `save(String)` is added.
**Variadic method matching:** When one side is variadic (`parameterCount`
undefined) and the other has a fixed count, `METHOD_IMPLEMENTS` edges are
emitted with confidence 0.7 instead of 1.0. Variadic methods like
`foo(String... args)` may superficially match `foo(String s)` by type but
are not guaranteed to be interchangeable across all languages (Java/Kotlin
accept this via varargs sugar; TypeScript, C#, Rust do not).
**Variadic matching:** confidence 0.7 when one side is variadic and the other has fixed count.
**Confidence tiering** for `METHOD_IMPLEMENTS` edges:
**METHOD_IMPLEMENTS confidence tiering:**
| Match quality | Confidence | When |
|---|---|---|
| Exact parameter types match | 1.0 | Both sides have `parameterTypes` arrays and they match |
| Arity (count) matches | 1.0 | Both sides have `parameterCount`, types unavailable |
| Variadic vs fixed | 0.7 | One side is variadic, other has fixed count |
| Lenient (insufficient info) | 0.7 | One or both sides lack type and count data |
| Match quality | Confidence |
|---|---|
| Exact parameter types match | 1.0 |
| Arity match, types unavailable | 1.0 |
| Variadic vs fixed | 0.7 |
| Insufficient info | 0.7 |
## Related docs
- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance.
- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery.
- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents.
- [TESTING.md](TESTING.md) — how to run tests.
- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage expectations for **this** repo when indexed by GitNexus.
- [MIGRATION.md](MIGRATION.md) — breaking changes and migration guidance
- [RUNBOOK.md](RUNBOOK.md) — operational commands and recovery
- [GUARDRAILS.md](GUARDRAILS.md) — safety boundaries for humans and agents
- [TESTING.md](TESTING.md) — how to run tests
- `AGENTS.md` / `CLAUDE.md` — agent workflows and tool usage

110
CLAUDE.md
View file

@ -1,10 +1,10 @@
<!-- version: 1.2.0 -->
<!-- version: 1.3.0 -->
<!--
Metadata: version, last reviewed, scope, model policy, reference docs, changelog.
Last updated: 2026-03-22
-->
Last reviewed: 2026-03-24
Last reviewed: 2026-04-13
**Project:** GitNexus · **Environment:** dev · **Maintainer:** repository maintainers (see GitHub)
@ -35,12 +35,14 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g
## Reference Documentation
- **This repository:** [AGENTS.md](AGENTS.md) (Cursor + monorepo notes), [ARCHITECTURE.md](ARCHITECTURE.md), [CONTRIBUTING.md](CONTRIBUTING.md), [GUARDRAILS.md](GUARDRAILS.md).
- **Call-resolution DAG:** See ARCHITECTURE.md § Call-Resolution DAG. Shared pipeline code in `gitnexus/src/core/ingestion/` must not name languages — use `LanguageProvider` hooks instead (see AGENTS.md).
- **GitNexus:** `.claude/skills/gitnexus/`; MCP and indexed-repo rules live only in [AGENTS.md](AGENTS.md) (`gitnexus:start``gitnexus:end`). See **GitNexus rules** below.
## Changelog
| Date | Version | Change |
|------|---------|--------|
| 2026-04-13 | 1.3.0 | Updated GitNexus index stats after DAG refactor. |
| 2026-03-24 | 1.2.0 | Removed duplicated gitnexus:start block and scope table; replaced with pointers to AGENTS.md. |
| 2026-03-23 | 1.1.0 | Updated agent instructions to match AGENTS.md. |
| 2026-03-22 | 1.0.0 | Added structured header and changelog. |
@ -49,106 +51,4 @@ If always-on instructions grow, load deep conventions via conditional reads (e.g
## GitNexus rules
GitNexus MCP rules are in the `<!-- gitnexus:start -->``<!-- gitnexus:end -->` block in **[AGENTS.md](AGENTS.md)** — load that section when working with MCP tools or the graph index.
<!-- gitnexus:start -->
# GitNexus — Code Intelligence
This project is indexed by GitNexus as **GitNexus** (3298 symbols, 7954 relationships, 185 execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
> If any GitNexus tool warns the index is stale, run `npx gitnexus analyze` in terminal first.
## Always Do
- **MUST run impact analysis before editing any symbol.** Before modifying a function, class, or method, run `gitnexus_impact({target: "symbolName", direction: "upstream"})` and report the blast radius (direct callers, affected processes, risk level) to the user.
- **MUST run `gitnexus_detect_changes()` before committing** to verify your changes only affect expected symbols and execution flows.
- **MUST warn the user** if impact analysis returns HIGH or CRITICAL risk before proceeding with edits.
- When exploring unfamiliar code, use `gitnexus_query({query: "concept"})` to find execution flows instead of grepping. It returns process-grouped results ranked by relevance.
- When you need full context on a specific symbol — callers, callees, which execution flows it participates in — use `gitnexus_context({name: "symbolName"})`.
## When Debugging
1. `gitnexus_query({query: "<error or symptom>"})` — find execution flows related to the issue
2. `gitnexus_context({name: "<suspect function>"})` — see all callers, callees, and process participation
3. `READ gitnexus://repo/GitNexus/process/{processName}` — trace the full execution flow step by step
4. For regressions: `gitnexus_detect_changes({scope: "compare", base_ref: "main"})` — see what your branch changed
## When Refactoring
- **Renaming**: MUST use `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` first. Review the preview — graph edits are safe, text_search edits need manual review. Then run with `dry_run: false`.
- **Extracting/Splitting**: MUST run `gitnexus_context({name: "target"})` to see all incoming/outgoing refs, then `gitnexus_impact({target: "target", direction: "upstream"})` to find all external callers before moving code.
- After any refactor: run `gitnexus_detect_changes({scope: "all"})` to verify only expected files changed.
## Never Do
- NEVER edit a function, class, or method without first running `gitnexus_impact` on it.
- NEVER ignore HIGH or CRITICAL risk warnings from impact analysis.
- NEVER rename symbols with find-and-replace — use `gitnexus_rename` which understands the call graph.
- NEVER commit changes without running `gitnexus_detect_changes()` to check affected scope.
## Tools Quick Reference
| Tool | When to use | Command |
|------|-------------|---------|
| `query` | Find code by concept | `gitnexus_query({query: "auth validation"})` |
| `context` | 360-degree view of one symbol | `gitnexus_context({name: "validateUser"})` |
| `impact` | Blast radius before editing | `gitnexus_impact({target: "X", direction: "upstream"})` |
| `detect_changes` | Pre-commit scope check | `gitnexus_detect_changes({scope: "staged"})` |
| `rename` | Safe multi-file rename | `gitnexus_rename({symbol_name: "old", new_name: "new", dry_run: true})` |
| `cypher` | Custom graph queries | `gitnexus_cypher({query: "MATCH ..."})` |
## Impact Risk Levels
| Depth | Meaning | Action |
|-------|---------|--------|
| d=1 | WILL BREAK — direct callers/importers | MUST update these |
| d=2 | LIKELY AFFECTED — indirect deps | Should test |
| d=3 | MAY NEED TESTING — transitive | Test if critical path |
## Resources
| Resource | Use for |
|----------|---------|
| `gitnexus://repo/GitNexus/context` | Codebase overview, check index freshness |
| `gitnexus://repo/GitNexus/clusters` | All functional areas |
| `gitnexus://repo/GitNexus/processes` | All execution flows |
| `gitnexus://repo/GitNexus/process/{name}` | Step-by-step execution trace |
## Self-Check Before Finishing
Before completing any code modification task, verify:
1. `gitnexus_impact` was run for all modified symbols
2. No HIGH/CRITICAL risk warnings were ignored
3. `gitnexus_detect_changes()` confirms changes match expected scope
4. All d=1 (WILL BREAK) dependents were updated
## Keeping the Index Fresh
After committing code changes, the GitNexus index becomes stale. Re-run analyze to update it:
```bash
npx gitnexus analyze
```
If the index previously included embeddings, preserve them by adding `--embeddings`:
```bash
npx gitnexus analyze --embeddings
```
To check whether embeddings exist, inspect `.gitnexus/meta.json` — the `stats.embeddings` field shows the count (0 means no embeddings). **Running analyze without `--embeddings` will delete any previously generated embeddings.**
> Claude Code users: A PostToolUse hook handles this automatically after `git commit` and `git merge`.
## CLI
| Task | Read this skill file |
|------|---------------------|
| Understand architecture / "How does X work?" | `.claude/skills/gitnexus/gitnexus-exploring/SKILL.md` |
| Blast radius / "What breaks if I change X?" | `.claude/skills/gitnexus/gitnexus-impact-analysis/SKILL.md` |
| Trace bugs / "Why is X failing?" | `.claude/skills/gitnexus/gitnexus-debugging/SKILL.md` |
| Rename / extract / split / refactor | `.claude/skills/gitnexus/gitnexus-refactoring/SKILL.md` |
| Tools, resources, schema reference | `.claude/skills/gitnexus/gitnexus-guide/SKILL.md` |
| Index, status, clean, wiki CLI commands | `.claude/skills/gitnexus/gitnexus-cli/SKILL.md` |
<!-- gitnexus:end -->
See the `<!-- gitnexus:start --> … <!-- gitnexus:end -->` block in **[AGENTS.md](AGENTS.md)** for the canonical MCP tools, impact analysis rules, and index instructions.

View file

@ -21,30 +21,127 @@ This project uses the [PolyForm Noncommercial License 1.0.0](https://polyformpro
## Branch and pull requests
- Use short-lived branches off the default branch of the repo you are targeting.
- Prefer **conventional commits** (short prefix + description), for example:
```text
feat: add graph export option
fix: correct MCP tool schema for query
test: cover cluster merge edge case
docs: clarify analyze flags
```
- **PR title:** `[area] Short description` (e.g. `[cli] Fix index refresh race`).
- **PR titles MUST follow the conventional-commit format**`pr-labeler.yml` enforces this on every PR and auto-applies the matching label so release notes group the change correctly.
- **PR description:** what changed, why, how to verify (commands), and any risk or rollback notes.
### Pull request titles
Format: `<type>[(scope)][!]: <subject>`
Allowed types and the release-notes section each one lands in (defined in `.github/release.yml`):
| Type | Label applied | Release-notes section |
|------|---------------|-----------------------|
| `feat` | `enhancement` | 🚀 Features |
| `fix` | `bug` | 🐛 Bug Fixes |
| `perf` | `performance` | 🏎️ Performance |
| `refactor` | `refactor` | 🔄 Refactoring |
| `test` | `test` | 🧪 Tests |
| `ci` | `ci` | 👷 CI/CD |
| `build` / `deps` | `dependencies` | 📦 Dependencies |
| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) |
| `chore` / `revert` | `chore` | (excluded from release notes) |
Append `!` to the type (e.g. `feat(api)!: drop /v1 endpoint`) or include `BREAKING CHANGE:` in the PR body to flag a breaking change — the labeler then adds the `breaking` label and the 💥 Breaking Changes section is rendered first.
Examples:
```text
feat(web): add smart chat scroll
fix(extractors): resolve silent contract mis-resolution
perf: avoid O(n²) traversal in heritage walker
chore(deps): bump vitest to 3.0.0
ci: standardize workflow concurrency
```
Commits within a PR may use any style — only the **merged PR title** shows up in release notes, so that's the one the convention applies to.
## Before you open a PR
- [ ] Tests pass for the packages you touched (`gitnexus` and/or `gitnexus-web`).
- [ ] Typecheck passes: `npx tsc --noEmit` in `gitnexus/` and `npx tsc -b --noEmit` in `gitnexus-web/`.
- [ ] No secrets, tokens, or machine-specific paths committed.
- [ ] Documentation updated if behavior or public CLI/MCP contract changes.
- [ ] Pre-commit hook runs clean (`.husky/pre-commit` — typecheck + unit tests for staged packages).
- [ ] Pre-commit hook runs clean (`.husky/pre-commit`formatting via lint-staged + typecheck for staged packages; tests run in CI only).
## Code review
Maintainers may request changes for correctness, tests, performance, or consistency with existing patterns. Keeping diffs focused makes review faster.
## GitHub Actions — Concurrency Convention
Every workflow under `.github/workflows/` MUST declare a top-level `concurrency:` block using this convention:
- **Group key** starts with `${{ github.workflow }}` so no two workflows can collide on the same group name. The discriminator that follows is chosen per event shape:
- Branch/tag scope: `${{ github.workflow }}-${{ github.ref }}`
- Per-PR scope (for `issue_comment`, `pull_request_review*`, `pull_request` meta events): `${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}`
- `workflow_run` scope (e.g. `ci-report.yml`): `${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}` — the fork fallback must be stable across reruns (never `workflow_run.id`, which is per-run-unique and defeats serialization).
- Global single-slot (manual dispatch utilities): `${{ github.workflow }}`
- **Reusable workflows invoked via `workflow_call`:** do NOT use `${{ github.workflow }}` in the group key — in called-workflow context its evaluation is ambiguous and can resolve to the caller's name, which would deadlock against the caller's own group. Use a hardcoded literal prefix and a `github.event_name`-aware expression that falls through to `github.run_id` for reusable invocations (see `ci.yml` for the canonical form).
- **Merge queue (`merge_group`)**: when this event is added, use `${{ github.workflow }}-${{ github.event.merge_group.head_ref }}` with `cancel-in-progress: false` (every queue entry is a distinct ref; never cancel).
- **`cancel-in-progress` policy:**
| Event | `cancel-in-progress` | Why |
|-------|----------------------|-----|
| `pull_request` CI run | `true` | New push supersedes old run |
| `push` to `main` | `false` | Every main commit gets validated |
| Tag push (`v*` publish) | `false` | Never cancel mid-publish |
| `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish |
| `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional |
| `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race |
| Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR |
| PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins |
| Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes |
- For workflows that serve multiple events at once (e.g. `ci.yml` handles `pull_request`, `push`, and `workflow_call`), make `cancel-in-progress` event-aware:
```yaml
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
```
- When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape.
## AI-assisted contributions
If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes.
## Releases
Two publish workflows ship `gitnexus` to npm:
- **Stable** (`.github/workflows/publish.yml`) — triggered by pushing any `v*`
tag. Publishes to the `latest` dist-tag with a changelog-backed GitHub
release. Maintainers are expected to tag from `main` as a convention; the
workflow itself does not enforce branch reachability.
- **Release Candidate** (`.github/workflows/release-candidate.yml`) — runs on
every push to `main` (typically a merged PR) plus manual dispatch. Docs-only
changes are skipped via `paths-ignore`. Publishes to the `rc` dist-tag with
version `X.Y.Z-rc.N` and a GitHub prerelease, where:
- `X.Y.Z` is selected automatically. On push (and on dispatch with
`bump: auto`, the default) the workflow **continues the active rc cycle**:
if the registry already has `X.Y.Z-rc.*` versions with `X.Y.Z` > current
`latest`, it reuses the highest such base; otherwise it patch-bumps
from `latest`. Dispatching with `bump: patch|minor|major` **resets**
the cycle from `latest`.
- `N` is auto-incremented against existing `X.Y.Z-rc.*` entries on the
registry. First rc for a given base is `rc.1`.
Idempotency: the workflow pushes an `rc/<HEAD_SHA>` marker tag and a
`v<RC>` release tag **atomically, before** calling `npm publish`. The guard
refuses to re-run once the marker exists, so a post-publish failure will
not mint a duplicate rc for the same commit. The `v<RC>` tag points at a
detached release commit whose `package.json` matches the npm tarball
exactly (traceable releases). Recovery after a partial failure:
```bash
git push --delete origin rc/<HEAD_SHA> v<RC>
# then redispatch the workflow with force: true
```
The rc workflow never moves `latest`. To verify after a change, inspect dist-tags:
```bash
npm view gitnexus dist-tags
```

View file

@ -1,72 +1,69 @@
# Guardrails — GitNexus (repo + agents)
# Guardrails — GitNexus
Rules for **human contributors** and **AI agents** working on this codebase or publishing artifacts. These complement `AGENTS.md` / `CLAUDE.md` (which focus on GitNexus-in-GitNexus workflows).
Rules for **human contributors** and **AI agents**. Complements `AGENTS.md` (workflows) and `CONTRIBUTING.md` (PR process).
## Scope (typical agent session)
## Scope (least privilege)
When automating changes in this repository, treat scope as **least privilege**:
- **Read:** Source, tests, docs, public config as needed.
- **Write:** Only files required for the fix or feature; no unrelated formatting or refactors.
- **Execute:** Tests, typecheck, documented CLI commands. No destructive commands on user data without approval.
- **Off-limits:** Other people's machines, production deployments you don't own, credentials you lack permission to use.
- **Read:** Source, tests, docs, public config as needed for the task.
- **Write:** Only files required for the requested fix or feature; avoid unrelated formatting or refactors.
- **Execute:** Tests, typecheck, and documented CLI commands; do not run destructive commands on user data outside the repo without explicit approval.
- **Off-limits:** Other peoples machines, production deployments you dont own, and credentials you didnt receive permission to use.
Adjust explicitly if the maintainer defines a different scope for a task.
Maintainer may widen scope per task.
---
## Non-negotiables
1. **Never commit secrets** — API keys, tokens, `.env` with real values, private URLs, or session cookies. Use `.env.example` with placeholders only.
2. **Never rename symbols with blind find-and-replace** when working in a GitNexus-indexed project — use the **`rename` MCP tool** with **`dry_run: true` first**, then review `graph` vs `text_search` edits. (There is no separate `gitnexus rename` CLI; renaming goes through MCP or editor integration.)
3. **Run impact analysis before editing shared symbols**use **`impact`** (upstream) for functions/classes/methods others call; do not ignore **HIGH** / **CRITICAL** risk without maintainer sign-off.
4. **Prefer `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available.
5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, run `npx gitnexus analyze --embeddings` when refreshing the index; plain `analyze` can drop them.
1. **Never commit secrets** — API keys, tokens, real `.env` values, private URLs, session cookies. Use `.env.example` with placeholders.
2. **Never rename with find-and-replace** in GitNexus-indexed projects — use `rename` MCP tool with `dry_run: true` first, review `graph` vs `text_search` edits. No separate `gitnexus rename` CLI exists.
3. **Run impact analysis before editing shared symbols**`impact` (upstream) for functions/classes/methods others call. Do not ignore HIGH/CRITICAL without maintainer sign-off.
4. **Run `detect_changes` before commit** — confirm diffs map to expected symbols/processes when the graph is available.
5. **Preserve embeddings** — if `.gitnexus/meta.json` shows embeddings, use `npx gitnexus analyze --embeddings`; plain `analyze` drops them.
---
## Signs (recurring failure patterns)
Use this format: **Trigger → Instruction → Reason**.
Append new Signs here when the same mistake repeats (e.g. CI broken twice the same way).
Format: **Trigger → Instruction → Reason**. Append new Signs when the same mistake repeats.
### Sign: Stale graph after edits
### Stale graph after edits
- **Trigger:** MCP or resources warn the index is behind `HEAD`, or code search doesnt match latest commit.
- **Instruction:** Run `npx gitnexus analyze` from the repo root (plus `--embeddings` if the project used them).
- **Reason:** Tools query LadybugDB built at last analyze; git changes are invisible until re-indexed.
- **Trigger:** MCP warns index is behind `HEAD`, or search doesn't match latest commit.
- **Do:** `npx gitnexus analyze` (plus `--embeddings` if used).
- **Why:** Tools query LadybugDB from last analyze; git changes are invisible until re-indexed.
### Sign: Embeddings vanished after analyze
### Embeddings vanished after analyze
- **Trigger:** Semantic search quality drops; `stats.embeddings` in `.gitnexus/meta.json` is 0 after a refresh.
- **Instruction:** Re-run `npx gitnexus analyze --embeddings` and confirm `meta.json` reflects stored embeddings.
- **Reason:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors.
- **Trigger:** Semantic search quality drops; `stats.embeddings` in `meta.json` is 0 after refresh.
- **Do:** `npx gitnexus analyze --embeddings`, confirm `meta.json` reflects stored embeddings.
- **Why:** Embedding generation is opt-in; analyze without the flag does not preserve prior vectors.
### Sign: MCP lists no repos
### MCP lists no repos
- **Trigger:** MCP stderr says no indexed repos.
- **Instruction:** Run `npx gitnexus analyze` in the target repository; verify `npx gitnexus list` shows it.
- **Reason:** The MCP server discovers repos via `~/.gitnexus/registry.json`, populated by analyze.
- **Trigger:** MCP stderr says no indexed repos.
- **Do:** `npx gitnexus analyze` in the target repo; verify `npx gitnexus list` shows it.
- **Why:** MCP discovers repos via `~/.gitnexus/registry.json`, populated by analyze.
### Sign: Wrong repo in multi-repo setups
### Wrong repo in multi-repo setups
- **Trigger:** Query/impact results clearly belong to another project.
- **Instruction:** Call `list_repos`, then pass **`repo`** on subsequent tools (or use per-workspace MCP config).
- **Reason:** Default target may be ambiguous when multiple repos are registered.
- **Trigger:** Query/impact results belong to another project.
- **Do:** Call `list_repos`, then pass `repo` on subsequent tools.
- **Why:** Default target is ambiguous when multiple repos are registered.
### Sign: LadybugDB lock / “database busy”
### LadybugDB lock / "database busy"
- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run.
- **Instruction:** Stop overlapping processes; one writer at a time. Retry analyze or restart MCP.
- **Reason:** Embedded DB expects single-process ownership of the store.
- **Trigger:** Errors opening `.gitnexus/lbug` while MCP and analyze both run.
- **Do:** Stop overlapping processes (one writer at a time). Retry analyze or restart MCP.
- **Why:** Embedded DB expects single-process ownership.
---
## Publishing & supply chain
- **npm:** Do not publish from unreviewed automation; follow maintainer release process. Bump version intentionally; tag releases to match `package.json`.
- **Dependencies:** Prefer minimal, auditable changes to `package.json`; run tests and CI after lockfile updates.
- **License:** This project ships under **PolyForm Noncommercial 1.0.0** — do not relicense or imply a different license in docs or metadata without maintainer approval.
- **npm:** Do not publish from unreviewed automation. Bump version intentionally; tag releases to match `package.json`.
- **Dependencies:** Minimal, auditable `package.json` changes; run tests and CI after lockfile updates.
- **License:** PolyForm Noncommercial 1.0.0 — do not relicense without maintainer approval.
---
@ -74,15 +71,15 @@ Append new Signs here when the same mistake repeats (e.g. CI broken twice the sa
Stop and ask a **human maintainer** when:
- Impact analysis shows **HIGH** / **CRITICAL** risk and the task still requires the change.
- You need to alter **CI**, **release**, or **security-sensitive** config.
- Requirements conflict (e.g. “speed up analyze” vs “must keep all embeddings on huge repo”).
- Impact analysis shows HIGH/CRITICAL risk and the task still requires the change.
- You need to alter CI, release, or security-sensitive config.
- Requirements conflict (e.g. "speed up analyze" vs "must keep all embeddings on huge repo").
- You are unsure whether data loss is acceptable (`clean`, forced migrations, schema changes).
---
## Related docs
- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow.
- [RUNBOOK.md](RUNBOOK.md) — commands for recovery.
- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations.
- [ARCHITECTURE.md](ARCHITECTURE.md) — components and data flow
- [RUNBOOK.md](RUNBOOK.md) — commands for recovery
- [CONTRIBUTING.md](CONTRIBUTING.md) — PR and commit expectations

View file

@ -20,9 +20,9 @@ From repository root, unless noted:
cd gitnexus
npm install
npm run build
npm test # unit: vitest run test/unit
npm test # full suite: vitest run
npm run test:unit # unit only: vitest run test/unit
npm run test:integration # integration suite
npm run test:all
npm run test:coverage
npx tsc --noEmit # typecheck (matches CI)
```
@ -42,8 +42,11 @@ npm run test:e2e # Playwright (requires gitnexus serve + npm run dev)
A husky pre-commit hook (`.husky/pre-commit`) runs automatically on every `git commit`:
- **`gitnexus-web/` files staged** → `tsc -b --noEmit` + `vitest run`
- **`gitnexus/` files staged** → `tsc --noEmit` + `vitest run --project default`
1. **Formatting**`lint-staged` runs prettier on staged files
2. **`gitnexus-web/` files staged** → `tsc -b --noEmit`
3. **`gitnexus/` files staged** → `tsc --noEmit`
Tests do **not** run in the pre-commit hook — they run in CI (`ci-tests.yml`) only.
Skip with `git commit --no-verify` (use sparingly).
@ -77,7 +80,7 @@ Re-run the full relevant suite when:
GitHub Actions (`.github/workflows/ci.yml`) orchestrate:
- **`ci-quality.yml`** — `tsc --noEmit` for `gitnexus/` + `tsc -b --noEmit` for `gitnexus-web/`
- **`ci-quality.yml`** — prettier format check, eslint lint, `tsc --noEmit` for `gitnexus/`, `tsc -b --noEmit` for `gitnexus-web/`
- **`ci-tests.yml`** — `vitest run` with coverage (ubuntu) + cross-platform (macOS, Windows)
- **`ci-e2e.yml`** — Playwright E2E tests, gated on `gitnexus-web/**` changes

View file

@ -19,6 +19,7 @@ export type { NodeTableName, RelType } from './lbug/schema-constants.js';
// Language support
export { SupportedLanguages } from './languages.js';
export { getLanguageFromFilename, getSyntaxLanguageFromFilename } from './language-detection.js';
export type { MroStrategy } from './mro-strategy.js';
// Pipeline progress
export type { PipelinePhase, PipelineProgress } from './pipeline.js';

View file

@ -30,6 +30,7 @@ export const NODE_TABLES = [
'TypeAlias',
'Const',
'Static',
'Variable',
'Property',
'Record',
'Delegate',

View file

@ -0,0 +1,46 @@
/**
* MRO (Method Resolution Order) strategy shared canonical definition.
*
* Lives in `gitnexus-shared` so `model/resolve.ts` and `mro-processor.ts` share
* the type without importing the language registry (avoids circular coupling).
*
* `first-wins` (default, Java/C#/Kotlin/Go/Swift/Dart):
* BFS ancestor walk in declaration order; first match wins.
*
* `leftmost-base` (C++):
* BFS walk; HeritageMap preserves source insertion order, so BFS naturally
* picks the leftmost base in diamond inheritance.
*
* `c3` (Python):
* C3-linearization; falls back to BFS on cyclic/inconsistent hierarchy.
* See model/resolve.ts § c3Linearize.
*
* `implements-split` (Java/C#/Kotlin):
* Low-level lookup is BFS; graph-level mro-processor detects and warns on
* interface-default method ambiguity.
*
* `qualified-syntax` (Rust):
* No auto-resolution `lookupMethodByOwnerWithMRO` returns undefined immediately.
* Rust requires explicit `<Type as Trait>::method` syntax.
*
* `ruby-mixin` (Ruby):
* Kind-aware walk that does NOT short-circuit on direct owner first (`prepend`
* must beat the class's own method). Walk order:
* 1. Prepend providers (reverse declaration last-prepended wins)
* 2. Direct owner's own methods
* 3. Include providers (reverse declaration)
* 4. Transitive ancestors (BFS fallback)
* Singleton dispatch: caller passes `ancestryOverride` (extend providers only);
* becomes a simple left-to-right scan. Miss NEVER falls through to file-scoped
* lookup null-routes or honors `fallback`.
*
* @see model/resolve.ts § lookupMethodByOwnerWithMRO
* @see languages/ruby.ts § selectDispatch
*/
export type MroStrategy =
| 'first-wins'
| 'c3'
| 'leftmost-base'
| 'implements-split'
| 'qualified-syntax'
| 'ruby-mixin';

View file

@ -0,0 +1,168 @@
import { test, expect } from '@playwright/test';
/**
* E2E tests for the repo-switching and false-404 fixes.
*
* Most tests use the live backend (same pattern as multi-repo-scoping.spec.ts).
* The 503 hold-queue test uses route interception to simulate a slow analysis.
*/
const BACKEND_URL = process.env.BACKEND_URL ?? 'http://localhost:4747';
const FRONTEND_URL = process.env.FRONTEND_URL ?? 'http://localhost:5173';
let firstRepoName: string;
test.beforeAll(async () => {
if (process.env.E2E) {
try {
const res = await fetch(`${BACKEND_URL}/api/repos`);
const repos = await res.json();
firstRepoName = repos[0]?.name ?? '';
} catch {
firstRepoName = '';
}
return;
}
try {
const [backendRes, frontendRes] = await Promise.allSettled([
fetch(`${BACKEND_URL}/api/repos`),
fetch(FRONTEND_URL),
]);
if (
backendRes.status === 'rejected' ||
(backendRes.status === 'fulfilled' && !backendRes.value.ok)
) {
test.skip(true, 'gitnexus serve not available');
return;
}
if (
frontendRes.status === 'rejected' ||
(frontendRes.status === 'fulfilled' && !frontendRes.value.ok)
) {
test.skip(true, 'Vite dev server not available');
return;
}
if (backendRes.status === 'fulfilled') {
const repos = await backendRes.value.json();
if (!repos.length) {
test.skip(true, 'No indexed repos');
return;
}
firstRepoName = repos[0].name;
}
} catch {
test.skip(true, 'servers not available');
}
});
// ── 1. Hold-queue: 503 → descriptive user message ────────────────────────────
test.describe('Hold-queue timeout error', () => {
test('shows descriptive message when /api/repo returns 503', async ({ page }, testInfo) => {
// Intercept only /api/repo (singular) — not /api/repos — to return a 503
// regex: /api/repo followed by end, ?, or # — NOT /api/repos
await page.route(/\/api\/repo(?!s)(\?.*)?$/, (route) =>
route.fulfill({
status: 503,
contentType: 'application/json',
body: JSON.stringify({
error: `Repository analysis for "${firstRepoName}" is taking longer than expected. Please try again in a moment.`,
}),
}),
);
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
// UI should show the 503 error message
await expect(page.getByText(/taking longer than expected/i)).toBeVisible({
timeout: 20_000,
});
await page.screenshot({ path: testInfo.outputPath('hold-queue-503.png') });
});
});
// ── 2. ?project= URL persistence ─────────────────────────────────────────────
test.describe('?project= URL persistence', () => {
test('?project= is set in URL after connecting via ?server=', async ({ page }) => {
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
const url = new URL(page.url());
const project = url.searchParams.get('project');
expect(project).toBeTruthy();
// first repo returned by the live backend
if (firstRepoName) expect(project).toBe(firstRepoName);
});
test('?project= is still present after F5 reload', async ({ page }) => {
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
// After connect, URL has ?server=&project= — F5 re-uses both params
await page.reload();
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
const url = new URL(page.url());
expect(url.searchParams.get('project')).toBeTruthy();
});
});
// ── 3. ?project= + ?server= combined auto-connect ────────────────────────────
test.describe('?project= auto-connect', () => {
test('navigating with ?server=&project= connects to the correct repo', async ({
page,
}, testInfo) => {
if (!firstRepoName) test.skip(true, 'no repo name available');
await page.goto(
`/?server=${encodeURIComponent(BACKEND_URL)}&project=${encodeURIComponent(firstRepoName)}`,
);
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
// ?project= in URL should match what we passed in
const url = new URL(page.url());
expect(url.searchParams.get('project')).toBe(firstRepoName);
await page.screenshot({ path: testInfo.outputPath('project-param-connect.png') });
});
});
// ── 4. Windows path normalization ─────────────────────────────────────────────
test.describe('Windows path normalization', () => {
test('project name uses basename when /api/repo returns a Windows-style repoPath', async ({
page,
}) => {
const repoName = firstRepoName || 'test-repo';
const windowsPath = `C:\\Users\\LENOVO\\.gitnexus\\repos\\${repoName}`;
// Mock /api/repo to return a Windows backslash path while keeping name correct
await page.route(/\/api\/repo(?!s)(\?.*)?$/, (route) =>
route.fulfill({
contentType: 'application/json',
body: JSON.stringify({
// intentionally omit `name` to force path-based extraction
path: windowsPath,
repoPath: windowsPath,
}),
}),
);
await page.goto(`/?server=${encodeURIComponent(BACKEND_URL)}`);
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
// URL ?project= must be the short basename, NOT the full Windows path
const url = new URL(page.url());
const project = url.searchParams.get('project');
expect(project).toBeTruthy();
expect(project).not.toContain('\\');
expect(project).not.toContain('LENOVO');
expect(project).toBe(repoName);
});
});

View file

@ -1,4 +1,4 @@
import { test, expect, type TestInfo } from '@playwright/test';
import { test, expect } from '@playwright/test';
/**
* E2E tests for the GitNexus web UI exploring view features.
@ -58,36 +58,41 @@ test.beforeAll(async () => {
* For these tests we require at least one indexed repo, so pick the first
* landing card when present and then wait for the exploring view.
*/
async function waitForGraphLoaded(page: import('@playwright/test').Page, testInfo: TestInfo) {
async function waitForGraphLoaded(page: import('@playwright/test').Page) {
await page.goto('/');
const landingCard = page.locator('[data-testid="landing-repo-card"]').first();
const landingCards = page.locator('[data-testid="landing-repo-card"]');
const preferredLandingCard = landingCards
.filter({ hasText: /GitNexus|local-integration/ })
.first();
try {
await landingCard.waitFor({ state: 'visible', timeout: 15_000 });
await landingCards.first().waitFor({ state: 'visible', timeout: 15_000 });
const landingCard =
(await preferredLandingCard.count()) > 0 ? preferredLandingCard : landingCards.first();
await landingCard.click();
} catch {
// Landing screen may not appear (e.g. ?server auto-connect)
}
await expect(page.locator('[data-testid="status-ready"]')).toBeVisible({ timeout: 30_000 });
await expect(page.getByText(/\d+ nodes/).first()).toBeVisible();
await page.screenshot({ path: testInfo.outputPath('graph-loaded.png') });
const statusBar = page.getByRole('contentinfo');
await expect(statusBar.getByText('Ready', { exact: true })).toBeVisible({ timeout: 45_000 });
await expect(statusBar).toContainText(/nodes/, {
timeout: 20_000,
});
}
test.describe('Server Connection & Graph Loading', () => {
test('selects a repo from landing and loads graph', async ({ page }, testInfo) => {
await waitForGraphLoaded(page, testInfo);
await page.screenshot({ path: testInfo.outputPath('graph-loaded-full.png'), fullPage: true });
test('selects a repo from landing and loads graph', async ({ page }) => {
await waitForGraphLoaded(page);
});
});
test.describe('Nexus AI', () => {
test('panel opens and agent initializes without error', async ({ page }, testInfo) => {
await waitForGraphLoaded(page, testInfo);
test('panel opens and agent initializes without error', async ({ page }) => {
await waitForGraphLoaded(page);
await page.getByRole('button', { name: 'Nexus AI' }).click();
await expect(page.getByText('Ask me anything')).toBeVisible({ timeout: 15_000 });
await page.screenshot({ path: testInfo.outputPath('nexus-ai-panel.png'), fullPage: true });
const errorBanner = page.getByText('Database not ready');
expect(await errorBanner.isVisible().catch(() => false)).toBe(false);
@ -95,8 +100,8 @@ test.describe('Nexus AI', () => {
});
test.describe('Processes Panel', () => {
test('shows process list and View button works', async ({ page }, testInfo) => {
await waitForGraphLoaded(page, testInfo);
test('shows process list and View button works', async ({ page }) => {
await waitForGraphLoaded(page);
await page.getByRole('button', { name: 'Nexus AI' }).click();
await page.getByText('Processes').click();
@ -104,7 +109,6 @@ test.describe('Processes Panel', () => {
await expect(page.locator('[data-testid="process-list-loaded"]')).toBeVisible({
timeout: 15_000,
});
await page.screenshot({ path: testInfo.outputPath('processes-panel.png'), fullPage: true });
const processRow = page.locator('[data-testid="process-row"]').first();
await expect(processRow).toBeVisible({ timeout: 10_000 });
@ -114,14 +118,10 @@ test.describe('Processes Panel', () => {
await viewBtn.waitFor({ state: 'visible', timeout: 5_000 });
await viewBtn.click();
await expect(page.locator('[data-testid="process-modal"]')).toBeVisible({ timeout: 5_000 });
await page.screenshot({
path: testInfo.outputPath('process-view-clicked.png'),
fullPage: true,
});
});
test('lightbulb highlights nodes in graph', async ({ page }, testInfo) => {
await waitForGraphLoaded(page, testInfo);
test('lightbulb highlights nodes in graph', async ({ page }) => {
await waitForGraphLoaded(page);
await page.getByRole('button', { name: 'Nexus AI' }).click();
await page.getByText('Processes').click();
@ -137,13 +137,12 @@ test.describe('Processes Panel', () => {
await lightbulb.waitFor({ state: 'visible', timeout: 5_000 });
await lightbulb.click();
await expect(processRow).toHaveClass(/bg-amber-950/, { timeout: 5_000 });
await page.screenshot({ path: testInfo.outputPath('after-highlight.png'), fullPage: true });
});
});
test.describe('Turn Off All Highlights', () => {
test('selecting a node dims others, button clears it', async ({ page }, testInfo) => {
await waitForGraphLoaded(page, testInfo);
test('selecting a node dims others, button clears it', async ({ page }) => {
await waitForGraphLoaded(page);
await expect(page.locator('canvas').first()).toBeVisible({ timeout: 10_000 });
@ -160,6 +159,5 @@ test.describe('Turn Off All Highlights', () => {
await expect(highlightToggle).toHaveAttribute('title', 'Turn on AI highlights', {
timeout: 5_000,
});
await page.screenshot({ path: testInfo.outputPath('highlights-cleared.png'), fullPage: true });
});
});

View file

@ -56,16 +56,14 @@ const AppContent = () => {
// backend calls (queries, search, grep, readFile) scope to this repo.
const repoName = result.repoInfo.name;
const repoPath = result.repoInfo.repoPath ?? result.repoInfo.path;
// Normalize both Windows (\) and Unix (/) path separators before splitting
const projectName =
repoName || repoPath?.split('/').filter(Boolean).pop() || 'server-project';
result.repoInfo.name ||
(repoPath || '').replace(/\\/g, '/').split('/').filter(Boolean).pop() ||
'server-project';
setProjectName(projectName);
setCurrentRepo(projectName);
// Update URL so F5 / bookmarks preserve which repo is open
const url = new URL(window.location.href);
url.searchParams.set('project', projectName);
window.history.replaceState(null, '', url.toString());
// Build KnowledgeGraph from server data for visualization
const graph = createKnowledgeGraph();
for (const node of result.nodes) {
@ -76,6 +74,11 @@ const AppContent = () => {
}
setGraph(graph);
// Persist the active project in the URL for bookmarkability and F5 refresh resilience
const urlObj = new URL(window.location.href);
urlObj.searchParams.set('project', projectName);
window.history.replaceState(null, '', urlObj.toString());
// Transition directly to exploring view
setViewMode('exploring');
@ -99,22 +102,17 @@ const AppContent = () => {
],
);
// Auto-connect when ?server query param is present (bookmarkable shortcut).
// Also reads ?project= to connect to a specific repo.
// Auto-connect when ?server or ?project query param is present (bookmarkable shortcut)
const autoConnectRan = useRef(false);
useEffect(() => {
if (autoConnectRan.current) return;
const params = new URLSearchParams(window.location.search);
if (!params.has('server')) return;
const serverUrlParam = params.get('server');
const projectParam = params.get('project');
if (!serverUrlParam && !projectParam) return;
autoConnectRan.current = true;
const serverUrl = params.get('server') || window.location.origin;
const projectParam = params.get('project') || undefined;
// Keep ?server= in the URL so F5 reconnects to the same server.
// autoConnectRan.current prevents re-trigger within the same session.
// handleServerConnect() will add/update ?project= after connecting.
setProgress({
phase: 'extracting',
percent: 0,
@ -123,39 +121,45 @@ const AppContent = () => {
});
setViewMode('loading');
const serverUrl = serverUrlParam || window.location.origin;
const baseUrl = normalizeServerUrl(serverUrl);
connectToServer(
serverUrl,
(phase, downloaded, total) => {
if (phase === 'validating') {
setProgress({
phase: 'extracting',
percent: 5,
message: 'Connecting to server...',
detail: 'Validating server',
});
} else if (phase === 'downloading') {
const pct = total ? Math.round((downloaded / total) * 90) + 5 : 50;
const mb = (downloaded / (1024 * 1024)).toFixed(1);
setProgress({
phase: 'extracting',
percent: pct,
message: 'Downloading graph...',
detail: `${mb} MB downloaded`,
});
} else if (phase === 'extracting') {
setProgress({
phase: 'extracting',
percent: 97,
message: 'Processing...',
detail: 'Extracting file contents',
});
}
},
undefined,
projectParam,
)
const tryConnect = async () => {
return await connectToServer(
serverUrl,
(phase, downloaded, total) => {
if (phase === 'validating') {
setProgress({
phase: 'extracting',
percent: 5,
message: 'Connecting to server...',
detail: 'Validating server',
});
} else if (phase === 'downloading') {
const pct = total ? Math.round((downloaded / total) * 90) + 5 : 50;
const mb = (downloaded / (1024 * 1024)).toFixed(1);
setProgress({
phase: 'extracting',
percent: pct,
message: 'Downloading graph...',
detail: `${mb} MB downloaded`,
});
} else if (phase === 'extracting') {
setProgress({
phase: 'extracting',
percent: 97,
message: 'Processing...',
detail: 'Extracting file contents',
});
}
},
undefined,
projectParam || undefined,
{ awaitAnalysis: true }, // enable backend hold-queue for repos still being analyzed
);
};
tryConnect()
.then(async (result) => {
await handleServerConnect(result);
setProgress(null);

View file

@ -231,7 +231,7 @@ export const CodeReferencesPanel = ({ onFocusNode }: CodeReferencesPanelProps) =
repo: projectName,
};
readFile(selectedFilePath, options)
readFile(selectedFilePath, { ...options, repo: projectName || undefined })
.then((result) => {
if (!cancelled) {
setFileResult(result);

View file

@ -8,8 +8,10 @@ import {
Loader2,
AlertTriangle,
GitBranch,
ArrowDown,
} from '@/lib/lucide-icons';
import { useAppState } from '../hooks/useAppState';
import { useAutoScroll } from '../hooks/useAutoScroll';
import { ToolCallCard } from './ToolCallCard';
import { isProviderConfigured } from '../core/llm/settings-service';
import { MarkdownRenderer } from './MarkdownRenderer';
@ -35,14 +37,11 @@ export const RightPanel = () => {
const [chatInput, setChatInput] = useState('');
const [activeTab, setActiveTab] = useState<'chat' | 'processes'>('chat');
const textareaRef = useRef<HTMLTextAreaElement>(null);
const messagesEndRef = useRef<HTMLDivElement>(null);
// Auto-scroll to bottom when messages update or while streaming
useEffect(() => {
if (messagesEndRef.current) {
messagesEndRef.current.scrollIntoView({ behavior: 'smooth' });
}
}, [chatMessages, isChatLoading]);
// Keep streamed replies pinned unless the user intentionally scrolls away from the bottom.
const { scrollContainerRef, messagesContainerRef, isAtBottom, scrollToBottom } = useAutoScroll(
chatMessages,
isChatLoading,
);
const resolveFilePathForUI = useCallback((_requestedPath: string): string | null => {
return null;
@ -265,7 +264,7 @@ export const RightPanel = () => {
{/* Chat Content - only show when chat tab is active */}
{activeTab === 'chat' && (
<div className="flex flex-1 flex-col overflow-hidden">
<div className="relative flex flex-1 flex-col overflow-hidden">
{/* Status bar */}
<div className="flex items-center gap-2.5 border-b border-border-subtle bg-elevated/50 px-4 py-3">
<div className="ml-auto flex items-center gap-2">
@ -291,7 +290,7 @@ export const RightPanel = () => {
)}
{/* Messages */}
<div className="scrollbar-thin flex-1 overflow-y-auto p-4">
<div ref={scrollContainerRef} className="scrollbar-thin flex-1 overflow-y-auto p-4">
{chatMessages.length === 0 ? (
<div className="flex h-full flex-col items-center justify-center px-4 text-center">
<div className="mb-4 flex h-14 w-14 items-center justify-center rounded-xl bg-gradient-to-br from-accent to-node-interface text-2xl shadow-glow">
@ -315,7 +314,7 @@ export const RightPanel = () => {
</div>
</div>
) : (
<div className="flex flex-col gap-6">
<div ref={messagesContainerRef} className="flex flex-col gap-6">
{chatMessages.map((message) => (
<div key={message.id} className="animate-fade-in">
{/* User message - compact label style */}
@ -391,10 +390,22 @@ export const RightPanel = () => {
))}
</div>
)}
{/* Scroll anchor for auto-scroll */}
<div ref={messagesEndRef} />
</div>
{/* Scroll to bottom */}
<button
aria-label="Scroll to bottom"
onClick={() => scrollToBottom()}
className={`absolute bottom-20 left-1/2 z-10 -translate-x-1/2 rounded-full border border-border-subtle bg-elevated px-3 py-1.5 text-xs text-text-secondary shadow-lg transition-all duration-200 hover:border-accent hover:text-accent ${
!isAtBottom && chatMessages.length > 0
? 'translate-y-0 opacity-100'
: 'pointer-events-none translate-y-2 opacity-0'
}`}
>
<ArrowDown className="mr-1 inline h-3.5 w-3.5" />
Scroll to bottom
</button>
{/* Input */}
<div className="border-t border-border-subtle bg-surface p-3">
<div className="flex items-end gap-2 rounded-xl border border-border-subtle bg-elevated px-3 py-2 transition-all focus-within:border-accent focus-within:ring-2 focus-within:ring-accent/20">

View file

@ -64,7 +64,7 @@ export const StatusBar = () => {
</a>
{/* Right - Stats */}
<div className="flex items-center gap-3">
<div className="flex items-center gap-3" data-testid="graph-stats">
{graph && (
<>
<span>{nodeCount} nodes</span>

View file

@ -579,6 +579,13 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
try {
const effectiveProjectName = overrideProjectName || projectName || 'project';
// Sync repoRef so all agent backend calls target the correct repo.
// initializeAgent can be called from App.tsx (handleServerConnect) which
// never sets repoRef.current directly — without this, queries default to repo[0].
if (overrideProjectName) {
repoRef.current = overrideProjectName;
}
const repo = repoRef.current;
// Build backend interface for Graph RAG tools
@ -610,7 +617,8 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
setIsAgentInitializing(false);
}
},
[projectName],
// eslint-disable-next-line react-hooks/exhaustive-deps
[], // repoRef is a stable ref — we sync it explicitly on entry; no state deps needed
);
const sendChatMessage = useCallback(
@ -1042,6 +1050,9 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
setCodePanelOpen(false);
setCodeReferenceFocus(null);
let connectedRepo: BackendRepo | undefined;
let pNameStr = repoName || 'server-project';
try {
const result: ConnectResult = await connectToServer(
serverBaseUrl,
@ -1073,44 +1084,28 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
},
undefined,
repoName,
{ awaitAnalysis: true }, // enable backend hold-queue for repos still being analyzed
);
// Build graph for visualization
const repoPath = result.repoInfo.repoPath ?? result.repoInfo.path;
// Prefer the registry name, then normalize Windows \ and Unix / paths
const pName =
repoName || result.repoInfo.name || repoPath?.split('/').pop() || 'server-project';
repoName ||
result.repoInfo.name ||
(repoPath || '').replace(/\\/g, '/').split('/').filter(Boolean).pop() ||
'server-project';
setProjectName(pName);
repoRef.current = pName;
// Update URL so F5 / bookmarks open the correct repo
const url = new URL(window.location.href);
url.searchParams.set('project', pName);
window.history.replaceState(null, '', url.toString());
connectedRepo = result.repoInfo;
pNameStr = pName;
const newGraph = createKnowledgeGraph();
for (const node of result.nodes) newGraph.addNode(node);
for (const rel of result.relationships) newGraph.addRelationship(rel);
setGraph(newGraph);
// No fileContents needed — grep/read tools use backend HTTP
// Initialize agent with backend queries, then start embeddings
try {
if (getActiveProviderConfig()) {
await initializeAgent(pName);
}
setViewMode('exploring');
startEmbeddingsWithFallback();
setProgress(null);
} catch (err) {
console.warn('Failed to initialize agent:', err);
setIsAgentReady(false);
agentRef.current = null;
setAgentError('Failed to initialize agent');
setViewMode('exploring');
setProgress(null);
}
} catch (err) {
} catch (err: unknown) {
console.error('Repo switch failed:', err);
setProgress({
phase: 'error',
@ -1124,6 +1119,36 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
setViewMode('exploring');
setProgress(null);
}, ERROR_RESET_DELAY_MS);
return; // Abort the whole switchRepo process
}
if (pNameStr) {
// Persist the selected project in the URL so a refresh re-opens it
const urlObj = new URL(window.location.href);
urlObj.searchParams.set('project', pNameStr);
window.history.replaceState(null, '', urlObj.toString());
}
// Reset the agent and clear chat history so the AI starts fresh for the new repo
agentRef.current = null;
setIsAgentReady(false);
setChatMessages([]);
// Re-initialize agent with the new repo's graph context
try {
if (getActiveProviderConfig()) {
await initializeAgent(pNameStr);
}
setViewMode('exploring');
startEmbeddingsWithFallback();
setProgress(null);
} catch (err) {
console.warn('Failed to initialize agent:', err);
setIsAgentReady(false);
agentRef.current = null;
setAgentError('Failed to initialize agent');
setViewMode('exploring');
setProgress(null);
}
},
[
@ -1143,6 +1168,7 @@ const AppStateProviderInner = ({ children }: { children: ReactNode }) => {
setCodeReferences,
setCodePanelOpen,
setCodeReferenceFocus,
setChatMessages,
],
);

View file

@ -0,0 +1,145 @@
import { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react';
const DEFAULT_BOTTOM_THRESHOLD = 100;
const USER_SCROLL_EPSILON = 5;
export interface UseAutoScrollResult {
scrollContainerRef: React.RefObject<HTMLDivElement>;
messagesContainerRef: React.RefObject<HTMLDivElement>;
isAtBottom: boolean;
scrollToBottom: (behavior?: ScrollBehavior) => void;
}
function isNearBottom(element: HTMLElement, threshold: number): boolean {
return element.scrollHeight - element.scrollTop - element.clientHeight <= threshold;
}
export function useAutoScroll<T>(
chatMessages: T[],
isChatLoading: boolean,
bottomThreshold = DEFAULT_BOTTOM_THRESHOLD,
): UseAutoScrollResult {
const scrollContainerRef = useRef<HTMLDivElement>(null);
const messagesContainerRef = useRef<HTMLDivElement>(null);
const [isAtBottom, setIsAtBottom] = useState(true);
const shouldStickToBottomRef = useRef(true);
const lastScrollTopRef = useRef(0);
const scrollFrameIdRef = useRef<number | null>(null);
const syncScrollState = useCallback(() => {
const element = scrollContainerRef.current;
if (!element) return;
const currentScrollTop = element.scrollTop;
const nearBottom = isNearBottom(element, bottomThreshold);
if (nearBottom) {
shouldStickToBottomRef.current = true;
} else if (currentScrollTop < lastScrollTopRef.current - USER_SCROLL_EPSILON) {
shouldStickToBottomRef.current = false;
}
lastScrollTopRef.current = currentScrollTop;
setIsAtBottom(nearBottom);
}, [bottomThreshold]);
const scrollToBottom = useCallback(
(behavior: ScrollBehavior = 'smooth') => {
const element = scrollContainerRef.current;
if (!element) return;
shouldStickToBottomRef.current = true;
if (behavior === 'auto') {
element.scrollTop = element.scrollHeight;
lastScrollTopRef.current = element.scrollTop;
setIsAtBottom(isNearBottom(element, bottomThreshold));
return;
}
element.scrollTo({
top: element.scrollHeight,
behavior,
});
},
[bottomThreshold],
);
useEffect(() => {
const element = scrollContainerRef.current;
if (!element) return;
lastScrollTopRef.current = element.scrollTop;
const handleScroll = () => {
if (scrollFrameIdRef.current !== null) {
cancelAnimationFrame(scrollFrameIdRef.current);
}
scrollFrameIdRef.current = requestAnimationFrame(() => {
scrollFrameIdRef.current = null;
syncScrollState();
});
};
element.addEventListener('scroll', handleScroll, { passive: true });
syncScrollState();
return () => {
element.removeEventListener('scroll', handleScroll);
if (scrollFrameIdRef.current !== null) {
cancelAnimationFrame(scrollFrameIdRef.current);
scrollFrameIdRef.current = null;
}
};
}, [syncScrollState]);
useEffect(() => {
const content = messagesContainerRef.current;
const scrollEl = scrollContainerRef.current;
if (!content || !scrollEl || typeof ResizeObserver === 'undefined') return;
let resizeFrameId: number | null = null;
const observer = new ResizeObserver(() => {
if (shouldStickToBottomRef.current) {
if (resizeFrameId !== null) {
cancelAnimationFrame(resizeFrameId);
}
resizeFrameId = requestAnimationFrame(() => {
resizeFrameId = null;
scrollToBottom('auto');
});
} else {
syncScrollState();
}
});
observer.observe(content);
return () => {
observer.disconnect();
if (resizeFrameId !== null) {
cancelAnimationFrame(resizeFrameId);
resizeFrameId = null;
}
};
}, [chatMessages.length, scrollToBottom, syncScrollState]);
useLayoutEffect(() => {
if (!shouldStickToBottomRef.current) return;
scrollToBottom('auto');
}, [chatMessages.length, isChatLoading, scrollToBottom]);
return {
scrollContainerRef,
messagesContainerRef,
isAtBottom,
scrollToBottom,
};
}

View file

@ -9,6 +9,7 @@
export {
AlertCircle,
AlertTriangle,
ArrowDown,
ArrowRight,
AtSign,
Brain,

View file

@ -222,7 +222,7 @@ export function normalizeServerUrl(input: string): string {
// ── Internal Helpers ───────────────────────────────────────────────────────
const DEFAULT_TIMEOUT_MS = 10_000;
const DEFAULT_TIMEOUT_MS = 30_000;
const PROBE_TIMEOUT_MS = 2_000;
const fetchWithTimeout = async (
@ -264,11 +264,13 @@ const fetchWithTimeout = async (
const assertOk = async (response: Response): Promise<void> => {
if (response.ok) return;
let message = `Backend returned ${response.status} ${response.statusText}`;
let message = response.statusText;
try {
const body = await response.json();
if (body && typeof body.error === 'string') {
message = body.error;
} else if (body && typeof body.message === 'string') {
message = body.message;
}
} catch {
// Response body was not JSON
@ -386,10 +388,22 @@ export const fetchRepos = async (): Promise<BackendRepo[]> => {
return response.json() as Promise<BackendRepo[]>;
};
/** Fetch repo metadata. */
export const fetchRepoInfo = async (repo?: string): Promise<BackendRepo> => {
/** Fetch repo metadata.
* Pass `awaitAnalysis: true` when connecting to a repo that may still be cloning/analyzing
* this enables the backend's hold-queue and uses a 5-minute timeout to match.
* Normal calls (e.g. repo switching between already-indexed repos) use the default 10s timeout.
*
* Must stay in sync with HOLD_QUEUE_TIMEOUT_SECS in gitnexus/src/server/api.ts.
*/
const HOLD_QUEUE_TIMEOUT_MS = 300_000; // 5 minutes — matches backend HOLD_QUEUE_TIMEOUT_SECS
export const fetchRepoInfo = async (
repo?: string,
opts?: { awaitAnalysis?: boolean },
): Promise<BackendRepo> => {
const url = `${_backendUrl}/api/repo${repo ? `?${repoParam(repo)}` : ''}`;
const response = await fetchWithTimeout(url);
const timeout = opts?.awaitAnalysis ? HOLD_QUEUE_TIMEOUT_MS : undefined;
const response = await fetchWithTimeout(url, {}, timeout);
await assertOk(response);
const data = await response.json();
return { ...data, repoPath: data.repoPath ?? data.path };
@ -404,13 +418,19 @@ export const fetchGraph = async (
onProgress?: (downloaded: number, total: number | null) => void;
},
): Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }> => {
const params = [repoParam(repo), opts?.includeContent ? 'includeContent=true' : '']
const params = [repoParam(repo), opts?.includeContent ? 'includeContent=true' : '', 'stream=true']
.filter(Boolean)
.join('&');
const url = `${_backendUrl}/api/graph${params ? `?${params}` : ''}`;
// Large repos can take a while to serialize the graph — use an elevated timeout
const response = await fetchWithTimeout(url, { signal: opts?.signal }, 300_000);
await assertOk(response);
const contentType = response.headers.get('Content-Type') || '';
if (contentType.includes('application/x-ndjson')) {
return parseNdjsonGraphResponse(response, opts?.onProgress);
}
if (!opts?.onProgress || !response.body) {
return response.json() as Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }>;
}
@ -439,6 +459,66 @@ export const fetchGraph = async (
return JSON.parse(new TextDecoder().decode(combined));
};
const parseNdjsonGraphResponse = async (
response: Response,
onProgress?: (downloaded: number, total: number | null) => void,
): Promise<{ nodes: GraphNode[]; relationships: GraphRelationship[] }> => {
if (!response.body) {
throw new BackendError('No response body', response.status, 'server');
}
const contentLength = response.headers.get('Content-Length');
const total = contentLength ? parseInt(contentLength, 10) : null;
const reader = response.body.getReader();
const decoder = new TextDecoder();
const nodes: GraphNode[] = [];
const relationships: GraphRelationship[] = [];
let buffer = '';
let downloaded = 0;
const parseLine = (line: string) => {
const trimmed = line.trim();
if (!trimmed) return;
const record = JSON.parse(trimmed) as
| { type: 'node'; data: GraphNode }
| { type: 'relationship'; data: GraphRelationship }
| { type: 'error'; error: string };
if (record.type === 'node') {
nodes.push(record.data);
return;
}
if (record.type === 'relationship') {
relationships.push(record.data);
return;
}
if (record.type === 'error') {
throw new BackendError(record.error, response.status || 500, 'server');
}
};
while (true) {
const { done, value } = await reader.read();
if (done) break;
downloaded += value.length;
onProgress?.(downloaded, total);
buffer += decoder.decode(value, { stream: true });
const lines = buffer.split('\n');
buffer = lines.pop() || '';
for (const line of lines) {
parseLine(line);
}
}
buffer += decoder.decode();
parseLine(buffer);
return { nodes, relationships };
};
/** Execute a Cypher query. Returns rows. */
export const runQuery = async (
cypher: string,
@ -670,18 +750,21 @@ export interface ConnectResult {
/**
* Connect to a server: validate, fetch repo info, download graph.
* Content is NOT included (use readFile/grep for file access).
* Pass `awaitAnalysis: true` when the repo may still be cloning/analyzing
* this enables the backend hold-queue and a 5-minute fetch timeout.
*/
export async function connectToServer(
url: string,
onProgress?: (phase: string, downloaded: number, total: number | null) => void,
signal?: AbortSignal,
repoName?: string,
opts?: { awaitAnalysis?: boolean },
): Promise<ConnectResult> {
const baseUrl = normalizeServerUrl(url);
setBackendUrl(baseUrl);
onProgress?.('validating', 0, null);
const repoInfo = await fetchRepoInfo(repoName);
const repoInfo = await fetchRepoInfo(repoName, { awaitAnalysis: opts?.awaitAnalysis });
onProgress?.('downloading', 0, null);
const { nodes, relationships } = await fetchGraph(repoName, {

View file

@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { normalizeServerUrl } from '../../src/services/backend-client';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { fetchGraph, normalizeServerUrl, setBackendUrl } from '../../src/services/backend-client';
describe('normalizeServerUrl', () => {
it('adds http:// to localhost', () => {
@ -31,3 +31,137 @@ describe('normalizeServerUrl', () => {
expect(normalizeServerUrl('https://gitnexus.example.com')).toBe('https://gitnexus.example.com');
});
});
afterEach(() => {
vi.restoreAllMocks();
});
describe('fetchGraph', () => {
it('requests streamed graph responses from the backend', async () => {
setBackendUrl('http://localhost:4747');
const fetchMock = vi.fn().mockResolvedValue(
new Response('{"nodes":[],"relationships":[]}', {
status: 200,
headers: {
'Content-Type': 'application/json',
},
}),
);
vi.stubGlobal('fetch', fetchMock);
await fetchGraph('big-repo');
expect(fetchMock).toHaveBeenCalledWith(
expect.stringContaining('/api/graph?repo=big-repo&stream=true'),
expect.any(Object),
);
});
it('parses NDJSON graph streams incrementally', async () => {
setBackendUrl('http://localhost:4747');
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(
encoder.encode(
[
'{"type":"node","data":{"id":"File:src/app.ts","label":"File","properties":{"name":"app.ts","filePath":"src/app.ts"}}}\n',
'{"type":"relationship","data":{"id":"File:src/app.ts_CONTAINS_Function:src/app.ts:main","type":"CONTAINS","sourceId":"File:src/app.ts","targetId":"Function:src/app.ts:main"}}\n',
].join(''),
),
);
controller.close();
},
});
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(
new Response(stream, {
status: 200,
headers: {
'Content-Type': 'application/x-ndjson',
},
}),
),
);
const progress = vi.fn();
const result = await fetchGraph('big-repo', { onProgress: progress });
expect(result.nodes).toHaveLength(1);
expect(result.relationships).toHaveLength(1);
expect(result.nodes[0].id).toBe('File:src/app.ts');
expect(result.relationships[0].type).toBe('CONTAINS');
expect(progress).toHaveBeenCalled();
});
it('parses NDJSON graph lines split across chunks', async () => {
setBackendUrl('http://localhost:4747');
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(
encoder.encode(
'{"type":"node","data":{"id":"File:src/app.ts","label":"File","properties":{"name":"app.ts"',
),
);
controller.enqueue(
encoder.encode(
',"filePath":"src/app.ts"}}}\n{"type":"relationship","data":{"id":"File:src/app.ts_CONTAINS_Function:src/app.ts:main","type":"CONTAINS","sourceId":"File:src/app.ts","targetId":"Function:src/app.ts:main"}}\n',
),
);
controller.close();
},
});
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(
new Response(stream, {
status: 200,
headers: {
'Content-Type': 'application/x-ndjson',
},
}),
),
);
const result = await fetchGraph('big-repo');
expect(result.nodes).toHaveLength(1);
expect(result.relationships).toHaveLength(1);
expect(result.nodes[0].properties.filePath).toBe('src/app.ts');
});
it('throws backend errors emitted in the NDJSON stream', async () => {
setBackendUrl('http://localhost:4747');
const encoder = new TextEncoder();
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(encoder.encode('{"type":"error","error":"stream failed"}\n'));
controller.close();
},
});
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(
new Response(stream, {
status: 200,
headers: {
'Content-Type': 'application/x-ndjson',
},
}),
),
);
await expect(fetchGraph('big-repo')).rejects.toMatchObject({
message: 'stream failed',
});
});
});

View file

@ -0,0 +1,287 @@
import { act, fireEvent, render, screen } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { useAutoScroll } from '../../src/hooks/useAutoScroll';
interface HarnessProps {
messages: unknown[];
isChatLoading: boolean;
}
function AutoScrollHarness({ messages, isChatLoading }: HarnessProps) {
const { scrollContainerRef, messagesContainerRef, isAtBottom, scrollToBottom } = useAutoScroll(
messages,
isChatLoading,
);
return (
<>
<div data-testid="is-at-bottom">{String(isAtBottom)}</div>
<div data-testid="container" ref={scrollContainerRef}>
{messages.length > 0 ? (
<div data-testid="messages-container" ref={messagesContainerRef}>
{messages.map((message, index) => (
<div key={index}>{String(message)}</div>
))}
</div>
) : null}
</div>
<button type="button" onClick={() => scrollToBottom()}>
Scroll to bottom
</button>
</>
);
}
function setScrollMetrics(
element: HTMLDivElement,
metrics: { scrollTop?: number; scrollHeight?: number; clientHeight?: number },
) {
if (metrics.scrollTop !== undefined) {
Object.defineProperty(element, 'scrollTop', {
configurable: true,
writable: true,
value: metrics.scrollTop,
});
}
if (metrics.scrollHeight !== undefined) {
Object.defineProperty(element, 'scrollHeight', {
configurable: true,
value: metrics.scrollHeight,
});
}
if (metrics.clientHeight !== undefined) {
Object.defineProperty(element, 'clientHeight', {
configurable: true,
value: metrics.clientHeight,
});
}
}
async function flushAnimationFrame() {
await act(async () => {
vi.runAllTimers();
});
}
async function scrollContainer(element: HTMLDivElement, scrollTop: number) {
setScrollMetrics(element, { scrollTop });
fireEvent.scroll(element);
await flushAnimationFrame();
}
const resizeObserverInstances: ResizeObserverMock[] = [];
class ResizeObserverMock {
callback: ResizeObserverCallback;
observedElements: Element[] = [];
observe = vi.fn((element: Element) => {
this.observedElements.push(element);
});
unobserve = vi.fn();
disconnect = vi.fn();
constructor(callback: ResizeObserverCallback) {
this.callback = callback;
resizeObserverInstances.push(this);
}
}
async function triggerResize(instance: ResizeObserverMock) {
await act(async () => {
instance.callback([], instance as unknown as ResizeObserver);
});
await flushAnimationFrame();
}
describe('useAutoScroll', () => {
beforeEach(() => {
vi.useFakeTimers();
resizeObserverInstances.length = 0;
vi.stubGlobal(
'requestAnimationFrame',
vi.fn((callback: FrameRequestCallback) => {
return window.setTimeout(() => callback(performance.now()), 0);
}),
);
vi.stubGlobal(
'cancelAnimationFrame',
vi.fn((frameId: number) => {
clearTimeout(frameId);
}),
);
Object.defineProperty(HTMLElement.prototype, 'scrollTo', {
configurable: true,
value: function (options: ScrollToOptions) {
if (options.top !== undefined) {
Object.defineProperty(this, 'scrollTop', {
configurable: true,
writable: true,
value: options.top,
});
}
},
});
vi.stubGlobal('ResizeObserver', ResizeObserverMock);
});
afterEach(() => {
vi.useRealTimers();
vi.unstubAllGlobals();
});
it('starts with isAtBottom true and auto-scrolls the very first message', () => {
const { rerender } = render(<AutoScrollHarness messages={[]} isChatLoading={false} />);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 0, scrollHeight: 500, clientHeight: 200 });
rerender(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
expect(container.scrollTop).toBe(500);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
});
it('follows streaming updates while the view stays pinned to the bottom', () => {
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
rerender(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={true} />);
expect(container.scrollTop).toBe(1000);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
});
it('stops auto-scroll after the user scrolls up', async () => {
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
await scrollContainer(container, 700);
await scrollContainer(container, 250);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false');
setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1400, clientHeight: 200 });
rerender(<AutoScrollHarness messages={[{ id: 1 }, { id: 2 }]} isChatLoading={true} />);
expect(container.scrollTop).toBe(250);
});
it('re-enables auto-scroll once the user returns near the bottom', async () => {
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
await scrollContainer(container, 700);
await scrollContainer(container, 250);
setScrollMetrics(container, { scrollTop: 1120, scrollHeight: 1400, clientHeight: 200 });
await scrollContainer(container, 1120);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
setScrollMetrics(container, { scrollTop: 1120, scrollHeight: 1800, clientHeight: 200 });
rerender(<AutoScrollHarness messages={[{ id: 1 }, { id: 2 }]} isChatLoading={true} />);
expect(container.scrollTop).toBe(1800);
});
it('scrollToBottom re-engages auto-scroll and scrolls to the container bottom', async () => {
const { rerender } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
const scrollTo = vi.spyOn(container, 'scrollTo');
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
await scrollContainer(container, 700);
await scrollContainer(container, 250);
fireEvent.click(screen.getByRole('button', { name: 'Scroll to bottom' }));
expect(scrollTo).toHaveBeenCalledWith({ top: 1000, behavior: 'smooth' });
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false');
setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1600, clientHeight: 200 });
rerender(<AutoScrollHarness messages={[{ id: 1 }, { id: 2 }]} isChatLoading={true} />);
expect(container.scrollTop).toBe(1600);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
});
it('re-pins to the latest bottom when inner content grows asynchronously', async () => {
render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
await scrollContainer(container, 700);
setScrollMetrics(container, { scrollTop: 1000, scrollHeight: 1450, clientHeight: 200 });
await triggerResize(resizeObserverInstances[0]);
expect(container.scrollTop).toBe(1450);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('true');
});
it('does not auto-scroll on async growth after user intentionally scrolls away', async () => {
render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 700, scrollHeight: 1000, clientHeight: 200 });
await scrollContainer(container, 700);
await scrollContainer(container, 250);
setScrollMetrics(container, { scrollTop: 250, scrollHeight: 1400, clientHeight: 200 });
await triggerResize(resizeObserverInstances[0]);
expect(container.scrollTop).toBe(250);
expect(screen.getByTestId('is-at-bottom')).toHaveTextContent('false');
});
it('cancels the pending ResizeObserver rAF when the component unmounts', () => {
const cancelRAF = vi.mocked(cancelAnimationFrame);
const { unmount } = render(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const container = screen.getByTestId('container') as HTMLDivElement;
setScrollMetrics(container, { scrollTop: 950, scrollHeight: 1000, clientHeight: 200 });
const callsBefore = cancelRAF.mock.calls.length;
act(() => {
resizeObserverInstances[0].callback(
[],
resizeObserverInstances[0] as unknown as ResizeObserver,
);
});
unmount();
expect(cancelRAF.mock.calls.length).toBeGreaterThan(callsBefore);
expect(() => vi.runAllTimers()).not.toThrow();
});
it('attaches the observer when the messages wrapper first appears and disconnects on unmount', () => {
const { rerender, unmount } = render(<AutoScrollHarness messages={[]} isChatLoading={false} />);
expect(screen.queryByTestId('messages-container')).toBeNull();
expect(resizeObserverInstances).toHaveLength(0);
rerender(<AutoScrollHarness messages={[{ id: 1 }]} isChatLoading={false} />);
const messagesContainer = screen.getByTestId('messages-container');
const resizeObserver = resizeObserverInstances[0];
expect(resizeObserverInstances).toHaveLength(1);
expect(resizeObserver.observe).toHaveBeenCalledWith(messagesContainer);
unmount();
expect(resizeObserver.disconnect).toHaveBeenCalledTimes(1);
});
});

View file

@ -9,6 +9,10 @@ tsconfig.json
.gitignore
node_modules/
# Vendor build artifacts (created during install, not shipped)
vendor/**/node_modules
vendor/**/build
# Package lock (consumers use their own)
package-lock.json

View file

@ -2,6 +2,81 @@
All notable changes to GitNexus will be documented in this file.
## [1.6.1] - 2026-04-13
### Added
- **Service group extractor expansion** — manifest extractor and broader extractor coverage (2/4 of #606 split) (#796)
- **Dart call patterns** for `await`, cascade, lambda, and widget-tree contexts (#801)
### Fixed
- **Stack overflow and memory exhaustion** on large repository analysis (#814)
- **`tree-sitter-dart` install crash** — switched from git URL to npm tarball (#811)
- **Generic TypeScript awaited function calls** missing from the call graph (#804)
- **Runtime dependency on `file:../gitnexus-shared`** removed from the published package (#803)
- **Ruby `singleton_class` context** preserved during sequential parsing (#774)
### Changed
- **DAG-based ingestion pipeline architecture** — pipeline phases now declare typed dependencies and run via a topologically sorted DAG; container-node logic extracted to `LanguageProvider`. Includes hardened lifecycle (try/finally cleanup, error wrapping, cycle reporting), tightened `ParseOutput.exportedTypeMap` immutability, and corrected phase dependencies (#809)
## [1.6.0] - 2026-04-12
### Added
- **SemanticModel architecture refactor (SM-8 through SM-19)** — extracted registries into `model/` module with ISP-compliant interfaces: TypeRegistry, MethodRegistry, FieldRegistry, RegistrationTable, ResolutionContext (#786)
- HeritageMap built from accumulated `ExtractedHeritage[]` for MRO-aware resolution (#739)
- `lookupMethodByOwnerWithMRO` using HeritageMap for cross-class method dispatch (#740)
- MRO fast path before D2 fuzzy widening in call resolution (#741)
- BindingAccumulator for cross-file return type propagation (#743, #763)
- Restructured `resolveUncached` replacing `lookupFuzzy` data source for all tiers (#764)
- Deleted `lookupFuzzy`, `lookupFuzzyCallable`, `globalIndex`, `callableIndex` — replaced with structured lookups (#769)
- Deleted `resolveCallTarget` god-method — replaced with thin dispatcher delegating to `resolveMemberCall` (#744), `resolveStaticCall` (#754), `resolveFreeCall` (#756) (#770)
- **Service group infrastructure** — service boundary detection, contract extractors, sync pipeline, CLI/MCP tools, monorepo fixture; bridge.lbug storage and contract matching expansion (#795)
- **C# interface-to-interface heritage** capture (#789)
- **Vue SFC support** with destructured call result tracking (#604)
- **Java method reference** resolution — `obj::method` as call sites (#622)
- **C/C++ MethodExtractor** config with pure virtual detection (#617)
- **MethodExtractor configs** for Python, PHP, Swift, Dart, Rust, Ruby (#624)
- **METHOD_IMPLEMENTS edges** with overload disambiguation and MethodExtractor unification (#642)
- **Same-arity overload disambiguation** via type-hash suffix (#658)
- **`GITNEXUS_HOME` env var** to customize global directory (#746)
- **Verbose analyze output** prints skipped large file paths (#745)
- **Class name lookup index** for O(1) qualified lookups (#707, #716)
- **`lookupMethodByOwner` index** for O(1) cross-class chain resolution (#665)
- **Fuzzy lookup counters** for performance visibility (#708)
### Fixed
- **Stack overflow on large PHP files** — iterative AST traversal (#783)
- **Large repository graph loading** failure (#732)
- **Windows multi-repo switching** — false 404 errors and stale repo context (#633)
- **`detect_changes` diff mapping** — map diff hunks to symbol line ranges (#779)
- **HTTP client vs Express route detection** and Spring interface attribution (#780)
- **VECTOR extension** not loaded during DB init for semantic search (#782)
- **tree-sitter-swift** postinstall patch for macOS ARM64 (#788)
- **tree-sitter-c** peer dependency conflict pinned (#723)
- **Constructor indexing** in methodByOwner (#694, #753)
- **Named binding processor**`lookupExact` replaced with `lookupExactAll` (#755)
- **`.gitnexusignore` negation patterns** now respected (#654)
- **MCP setup** prefers global gitnexus binary over npx (#653)
- **CORS rejection** returns clean error instead of 500 (#646)
- **Array.push stack overflow** — replaced spread with loop (#650)
- **MCP stdout silencing** prevents embedder/pool-adapter conflicts (#645)
- **Web heartbeat** — graceful reconnection replaces aggressive disconnect (#643)
- **Web repo scoping** — backend calls scoped to active repo (#644)
- **OpenCode config path** and FTS extension load order (#781)
- **OnboardingGuide** dev-mode serve command corrected (#725)
- **Security issues** and critical bugs from code review (#709)
### Changed
- Replaced class-type fuzzy lookups with structured indices in type-env (#733, #734, #736)
- Extracted `CLASS_LIKE_TYPES` constant (#693)
## [1.5.3] - 2026-04-01
### Added
- **TypeScript/JavaScript MethodExtractor** config (#588)
### Fixed
- **Wiki Azure OpenAI** compat and HTML viewer script injection (#618)
## [1.5.2] - 2026-04-01
### Fixed

View file

@ -234,6 +234,79 @@ Installed automatically by both `gitnexus analyze` (per-repo) and `gitnexus setu
- Node.js >= 18
- Git repository (uses git for commit tracking)
## Release candidates
Stable releases publish to the default `latest` dist-tag. When a pull request
with non-documentation changes merges into `main`, an automated workflow also
publishes a prerelease build under the `rc` dist-tag, so early adopters can
try in-flight fixes without waiting for the next stable cut. (Docs-only
merges are skipped.)
```bash
# Try the latest release candidate (pre-stable — may change at any time)
npm install -g gitnexus@rc
# — or —
npx gitnexus@rc analyze
```
Release-candidate versions follow the standard semver prerelease format
`X.Y.Z-rc.N`, where `X.Y.Z` is the next stable target (bumped from the
current `latest` by patch by default; `minor` or `major` when kicking off a
bigger cycle) and `N` increments per published rc. Example sequence:
`1.6.2-rc.1`, `1.6.2-rc.2`, …, then once `1.6.2` ships stable,
`1.6.3-rc.1`. See the [Releases page](https://github.com/abhigyanpatwari/GitNexus/releases)
for the full list; stable `latest` is unaffected.
## Troubleshooting
### `Cannot destructure property 'package' of 'node.target' as it is null`
This crash was caused by a dependency URL format that is incompatible with
certain npm/arborist versions ([npm/cli#8126](https://github.com/npm/cli/issues/8126)).
It is fixed in **gitnexus v1.6.2+**. Upgrade to the latest version:
```bash
npx gitnexus@latest analyze # always uses the newest release
# — or —
npm install -g gitnexus@latest # upgrade a global install
```
If you still hit npm install issues after upgrading, these generic workarounds
may help:
```bash
npm install -g npm@latest # update npm itself
npm cache clean --force # clear a possibly corrupt cache
```
### Installation fails with native module errors
Some optional language grammars (Dart, Kotlin, Swift) require native compilation. If they fail, GitNexus still works — those languages will be skipped.
If `npm install -g gitnexus` fails on native modules:
```bash
# Ensure build tools are available (Linux/macOS)
# Ubuntu/Debian: sudo apt install python3 make g++
# macOS: xcode-select --install
# Retry installation
npm install -g gitnexus
```
### Analysis runs out of memory
For very large repositories:
```bash
# Increase Node.js heap size
NODE_OPTIONS="--max-old-space-size=16384" npx gitnexus analyze
# Exclude large directories
echo "vendor/" >> .gitnexusignore
echo "dist/" >> .gitnexusignore
```
## Privacy
- All processing happens locally on your machine

View file

@ -1,12 +1,13 @@
{
"name": "gitnexus",
"version": "1.5.3",
"version": "1.6.1",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "gitnexus",
"version": "1.5.3",
"version": "1.6.1",
"hasInstallScript": true,
"license": "PolyForm-Noncommercial-1.0.0",
"dependencies": {
"@huggingface/transformers": "^3.0.0",
@ -29,7 +30,7 @@
"pandemonium": "^2.4.0",
"tree-sitter": "^0.21.1",
"tree-sitter-c": "0.23.2",
"tree-sitter-c-sharp": "^0.23.1",
"tree-sitter-c-sharp": "0.23.1",
"tree-sitter-cpp": "^0.23.4",
"tree-sitter-go": "^0.23.0",
"tree-sitter-java": "^0.23.5",
@ -61,8 +62,11 @@
"node": ">=20.0.0"
},
"optionalDependencies": {
"tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4",
"node-addon-api": "^8.0.0",
"node-gyp-build": "^4.8.0",
"tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
"tree-sitter-kotlin": "^0.3.8",
"tree-sitter-proto": "file:./vendor/tree-sitter-proto",
"tree-sitter-swift": "^0.6.0"
}
},
@ -1224,6 +1228,12 @@
"win32"
]
},
"node_modules/@ladybugdb/core/node_modules/node-addon-api": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==",
"license": "MIT"
},
"node_modules/@modelcontextprotocol/sdk": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.28.0.tgz",
@ -4116,10 +4126,13 @@
}
},
"node_modules/node-addon-api": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
"integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==",
"license": "MIT"
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/node-api-headers": {
"version": "1.8.0",
@ -5067,24 +5080,6 @@
}
}
},
"node_modules/tree-sitter-c-sharp/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-c/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-cli": {
"version": "0.23.2",
"resolved": "https://registry.npmjs.org/tree-sitter-cli/-/tree-sitter-cli-0.23.2.tgz",
@ -5119,15 +5114,6 @@
}
}
},
"node_modules/tree-sitter-cpp/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-dart": {
"version": "1.0.0",
"resolved": "git+ssh://git@github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
@ -5174,15 +5160,6 @@
}
}
},
"node_modules/tree-sitter-go/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-java": {
"version": "0.23.5",
"resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.23.5.tgz",
@ -5202,15 +5179,6 @@
}
}
},
"node_modules/tree-sitter-java/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-javascript": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz",
@ -5230,15 +5198,6 @@
}
}
},
"node_modules/tree-sitter-javascript/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-kotlin": {
"version": "0.3.8",
"resolved": "https://registry.npmjs.org/tree-sitter-kotlin/-/tree-sitter-kotlin-0.3.8.tgz",
@ -5285,14 +5244,9 @@
}
}
},
"node_modules/tree-sitter-php/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
"node_modules/tree-sitter-proto": {
"resolved": "vendor/tree-sitter-proto",
"link": true
},
"node_modules/tree-sitter-python": {
"version": "0.23.4",
@ -5313,15 +5267,6 @@
}
}
},
"node_modules/tree-sitter-python/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-ruby": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/tree-sitter-ruby/-/tree-sitter-ruby-0.23.1.tgz",
@ -5341,15 +5286,6 @@
}
}
},
"node_modules/tree-sitter-ruby/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-rust": {
"version": "0.23.1",
"resolved": "https://registry.npmjs.org/tree-sitter-rust/-/tree-sitter-rust-0.23.1.tgz",
@ -5369,15 +5305,6 @@
}
}
},
"node_modules/tree-sitter-rust/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-swift": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tree-sitter-swift/-/tree-sitter-swift-0.6.0.tgz",
@ -5407,16 +5334,6 @@
"license": "ISC",
"optional": true
},
"node_modules/tree-sitter-swift/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"optional": true,
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter-swift/node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@ -5453,24 +5370,6 @@
}
}
},
"node_modules/tree-sitter-typescript/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tree-sitter/node_modules/node-addon-api": {
"version": "8.7.0",
"resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.7.0.tgz",
"integrity": "sha512-9MdFxmkKaOYVTV+XVRG8ArDwwQ77XIgIPyKASB1k3JPq3M8fGQQQE3YpMOrKm6g//Ktx8ivZr8xo1Qmtqub+GA==",
"license": "MIT",
"engines": {
"node": "^18 || ^20 || >= 21"
}
},
"node_modules/tslib": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
@ -5875,6 +5774,14 @@
"peerDependencies": {
"zod": "^3.25.28 || ^4"
}
},
"vendor/tree-sitter-proto": {
"version": "0.4.1",
"license": "MIT",
"optional": true,
"peerDependencies": {
"tree-sitter": ">=0.21.0"
}
}
}
}

View file

@ -1,6 +1,6 @@
{
"name": "gitnexus",
"version": "1.5.3",
"version": "1.6.1",
"description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
"author": "Abhigyan Patwari",
"license": "PolyForm-Noncommercial-1.0.0",
@ -46,6 +46,7 @@
"test:integration": "vitest run test/integration",
"test:watch": "vitest",
"test:coverage": "vitest run --coverage",
"postinstall": "node scripts/patch-tree-sitter-swift.cjs && node scripts/build-tree-sitter-proto.cjs",
"prepare": "node scripts/build.js",
"prepack": "node scripts/build.js"
},
@ -58,7 +59,6 @@
"commander": "^12.0.0",
"cors": "^2.8.5",
"express": "^4.19.2",
"gitnexus-shared": "file:../gitnexus-shared",
"glob": "^11.0.0",
"graphology": "^0.25.4",
"graphology-indices": "^0.17.0",
@ -71,7 +71,7 @@
"pandemonium": "^2.4.0",
"tree-sitter": "^0.21.1",
"tree-sitter-c": "0.23.2",
"tree-sitter-c-sharp": "^0.23.1",
"tree-sitter-c-sharp": "0.23.1",
"tree-sitter-cpp": "^0.23.4",
"tree-sitter-go": "^0.23.0",
"tree-sitter-java": "^0.23.5",
@ -84,8 +84,11 @@
"uuid": "^13.0.0"
},
"optionalDependencies": {
"tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4",
"node-addon-api": "^8.0.0",
"node-gyp-build": "^4.8.0",
"tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
"tree-sitter-kotlin": "^0.3.8",
"tree-sitter-proto": "file:./vendor/tree-sitter-proto",
"tree-sitter-swift": "^0.6.0"
},
"devDependencies": {

View file

@ -0,0 +1,82 @@
#!/usr/bin/env node
/**
* Build tree-sitter-proto native binding.
*
* Why this script exists:
* tree-sitter-proto is vendored under gitnexus/vendor/tree-sitter-proto/
* and declared as a `file:` optionalDependency. Previously, the vendored
* package had its own `dependencies` and `install` script, which caused
* npm to create `vendor/tree-sitter-proto/node_modules/` and
* `vendor/tree-sitter-proto/build/` during install. Those directories
* blocked `rmdir` on global-install upgrade, producing:
*
* ENOTEMPTY: directory not empty, rmdir
* '.../gitnexus/vendor/tree-sitter-proto/node_modules/node-addon-api'
*
* (See https://github.com/abhigyanpatwari/GitNexus/issues/836.)
*
* We stripped `dependencies` and the `install` script from the vendored
* package.json, hoisted `node-addon-api` and `node-gyp-build` into
* gitnexus's own optionalDependencies, and moved native compilation here.
*
* What this does:
* Runs `npx node-gyp rebuild` inside `node_modules/tree-sitter-proto/`
* (which npm creates as a copy of vendor/tree-sitter-proto/ when
* resolving the file: dep). Build output lands in
* `node_modules/tree-sitter-proto/build/Release/tree_sitter_proto_binding.node`
* under npm-managed territory, safe on upgrade.
*
* Mirrors scripts/patch-tree-sitter-swift.cjs. Best-effort: if any
* precondition fails (optional dep absent, no toolchain, --ignore-scripts),
* warn and exit 0 so gitnexus install still succeeds.
*/
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto');
const bindingGyp = path.join(protoDir, 'binding.gyp');
const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node');
try {
if (!fs.existsSync(bindingGyp)) {
// tree-sitter-proto is an optionalDependency; absent when install
// skipped optional deps or the file: dep was not resolved.
process.exit(0);
}
// Skip if the native binding already exists (idempotent re-run).
if (fs.existsSync(bindingNode)) {
process.exit(0);
}
// Pre-flight: the hoisted build deps must be resolvable.
try {
require.resolve('node-addon-api');
require.resolve('node-gyp-build');
} catch (resolveErr) {
console.warn(
'[tree-sitter-proto] Skipping build: hoisted build deps not resolvable (%s).',
resolveErr.message,
);
console.warn(
'[tree-sitter-proto] Proto parsing will be unavailable. Install without --no-optional and with scripts enabled to build.',
);
process.exit(0);
}
console.log('[tree-sitter-proto] Building native binding...');
execSync('npx node-gyp rebuild', {
cwd: protoDir,
stdio: 'pipe',
timeout: 180000,
});
console.log('[tree-sitter-proto] Native binding built successfully');
} catch (err) {
console.warn('[tree-sitter-proto] Could not build native binding:', err.message);
console.warn(
'[tree-sitter-proto] Proto (.proto) parsing will be unavailable. Non-proto gitnexus functionality is unaffected.',
);
// Exit 0: optionalDependency failures must not fail the gitnexus install.
process.exit(0);
}

View file

@ -0,0 +1,78 @@
#!/usr/bin/env node
/**
* WORKAROUND: tree-sitter-swift@0.6.0 binding.gyp build failure
*
* Background:
* tree-sitter-swift@0.6.0's binding.gyp contains an "actions" array that
* invokes `tree-sitter generate` to regenerate parser.c from grammar.js.
* This is intended for grammar developers, but the published npm package
* already ships pre-generated parser files (parser.c, scanner.c), so the
* actions are unnecessary for consumers. Since consumers don't have
* tree-sitter-cli installed, the actions always fail during `npm install`.
*
* Why we can't just upgrade:
* tree-sitter-swift@0.7.1 fixes this (removes postinstall, ships prebuilds),
* but it requires tree-sitter@^0.22.1. The upstream project pins tree-sitter
* to ^0.21.0 and all other grammar packages depend on that version.
* Upgrading tree-sitter would be a separate breaking change.
*
* How this workaround works:
* 1. tree-sitter-swift's own postinstall fails (npm warns but continues)
* 2. This script runs as gitnexus's postinstall
* 3. It removes the "actions" array from binding.gyp
* 4. It rebuilds the native binding with the cleaned binding.gyp
*
* TODO: Remove this script when tree-sitter is upgraded to ^0.22.x,
* which allows using tree-sitter-swift@0.7.1+ directly.
*/
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const swiftDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-swift');
const bindingPath = path.join(swiftDir, 'binding.gyp');
try {
if (!fs.existsSync(bindingPath)) {
process.exit(0);
}
const content = fs.readFileSync(bindingPath, 'utf8');
let needsRebuild = false;
if (content.includes('"actions"')) {
// Strip Python-style comments (#) and trailing commas before JSON parsing
const cleaned = content
.replace(/#[^\n]*/g, '') // Remove # comments
.replace(/,(\s*[\]}])/g, '$1'); // Remove trailing commas before ] or }
const gyp = JSON.parse(cleaned);
if (gyp.targets && gyp.targets[0] && gyp.targets[0].actions) {
delete gyp.targets[0].actions;
fs.writeFileSync(bindingPath, JSON.stringify(gyp, null, 2) + '\n');
console.log('[tree-sitter-swift] Patched binding.gyp (removed actions array)');
needsRebuild = true;
}
}
// Check if native binding exists
const bindingNode = path.join(swiftDir, 'build', 'Release', 'tree_sitter_swift_binding.node');
if (!fs.existsSync(bindingNode)) {
needsRebuild = true;
}
if (needsRebuild) {
console.log('[tree-sitter-swift] Rebuilding native binding...');
execSync('npx node-gyp rebuild', {
cwd: swiftDir,
stdio: 'pipe',
timeout: 120000,
});
console.log('[tree-sitter-swift] Native binding built successfully');
}
} catch (err) {
console.warn('[tree-sitter-swift] Could not build native binding:', err.message);
console.warn(
'[tree-sitter-swift] You may need to manually run: cd node_modules/tree-sitter-swift && npx node-gyp rebuild',
);
}

View file

@ -26,6 +26,7 @@ interface RepoStats {
export interface AIContextOptions {
skipAgentsMd?: boolean;
noStats?: boolean;
}
const GITNEXUS_START_MARKER = '<!-- gitnexus:start -->';
@ -64,6 +65,7 @@ function generateGitNexusContent(
stats: RepoStats,
generatedSkills?: GeneratedSkillInfo[],
groupNames?: string[],
noStats?: boolean,
): string {
const generatedRows =
generatedSkills && generatedSkills.length > 0
@ -87,7 +89,7 @@ function generateGitNexusContent(
return `${GITNEXUS_START_MARKER}
# GitNexus Code Intelligence
This project is indexed by GitNexus as **${projectName}** (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows). Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
This project is indexed by GitNexus as **${projectName}**${noStats ? '' : ` (${stats.nodes || 0} symbols, ${stats.edges || 0} relationships, ${stats.processes || 0} execution flows)`}. Use the GitNexus MCP tools to understand code, assess impact, and navigate safely.
> If any GitNexus tool warns the index is stale, run \`npx gitnexus analyze\` in terminal first.
@ -332,7 +334,13 @@ export async function generateAIContextFiles(
options?: AIContextOptions,
): Promise<{ files: string[] }> {
const groupNames = await findGroupsContainingRegistryName(projectName);
const content = generateGitNexusContent(projectName, stats, generatedSkills, groupNames);
const content = generateGitNexusContent(
projectName,
stats,
generatedSkills,
groupNames,
options?.noStats,
);
const createdFiles: string[] = [];
if (!options?.skipAgentsMd) {

View file

@ -20,8 +20,11 @@ import fs from 'fs/promises';
const HEAP_MB = 8192;
const HEAP_FLAG = `--max-old-space-size=${HEAP_MB}`;
/** Increase default stack size (KB) to prevent stack overflow on deep class hierarchies. */
const STACK_KB = 4096;
const STACK_FLAG = `--stack-size=${STACK_KB}`;
/** Re-exec the process with an 8GB heap if we're currently below that. */
/** Re-exec the process with an 8GB heap and larger stack if we're currently below that. */
function ensureHeap(): boolean {
const nodeOpts = process.env.NODE_OPTIONS || '';
if (nodeOpts.includes('--max-old-space-size')) return false;
@ -29,8 +32,13 @@ function ensureHeap(): boolean {
const v8Heap = v8.getHeapStatistics().heap_size_limit;
if (v8Heap >= HEAP_MB * 1024 * 1024 * 0.9) return false;
// --stack-size is a V8 flag not allowed in NODE_OPTIONS on Node 24+,
// so pass it only as a direct CLI argument, not via the environment.
const cliFlags = [HEAP_FLAG];
if (!nodeOpts.includes('--stack-size')) cliFlags.push(STACK_FLAG);
try {
execFileSync(process.execPath, [HEAP_FLAG, ...process.argv.slice(1)], {
execFileSync(process.execPath, [...cliFlags, ...process.argv.slice(1)], {
stdio: 'inherit',
env: { ...process.env, NODE_OPTIONS: `${nodeOpts} ${HEAP_FLAG}`.trim() },
});
@ -47,6 +55,8 @@ export interface AnalyzeOptions {
verbose?: boolean;
/** Skip AGENTS.md and CLAUDE.md gitnexus block updates. */
skipAgentsMd?: boolean;
/** Omit volatile symbol/relationship counts from AGENTS.md and CLAUDE.md. */
noStats?: boolean;
/** Index the folder even when no .git directory is present. */
skipGit?: boolean;
}
@ -137,9 +147,11 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
const origLog = console.log.bind(console);
const origWarn = console.warn.bind(console);
const origError = console.error.bind(console);
let barCurrentValue = 0;
const barLog = (...args: any[]) => {
process.stdout.write('\x1b[2K\r');
origLog(args.map((a) => (typeof a === 'string' ? a : String(a))).join(' '));
bar.update(barCurrentValue);
};
console.log = barLog;
console.warn = barLog;
@ -150,6 +162,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
let phaseStart = Date.now();
const updateBar = (value: number, phaseLabel: string) => {
barCurrentValue = value;
if (phaseLabel !== lastPhaseLabel) {
lastPhaseLabel = phaseLabel;
phaseStart = Date.now();
@ -177,6 +190,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
embeddings: options?.embeddings,
skipGit: options?.skipGit,
skipAgentsMd: options?.skipAgentsMd,
noStats: options?.noStats,
},
{
onProgress: (_phase, percent, message) => {
@ -240,7 +254,7 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
processes: s.processes,
},
skillResult.skills,
{ skipAgentsMd: options?.skipAgentsMd },
{ skipAgentsMd: options?.skipAgentsMd, noStats: options?.noStats },
);
}
} catch {
@ -282,7 +296,51 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
console.warn = origWarn;
console.error = origError;
bar.stop();
console.error(`\n Analysis failed: ${err.message}\n`);
const msg = err.message || String(err);
console.error(`\n Analysis failed: ${msg}\n`);
// Provide helpful guidance for known failure modes
if (
msg.includes('Maximum call stack size exceeded') ||
msg.includes('call stack') ||
msg.includes('Map maximum size') ||
msg.includes('Invalid array length') ||
msg.includes('Invalid string length') ||
msg.includes('allocation failed') ||
msg.includes('heap out of memory') ||
msg.includes('JavaScript heap')
) {
console.error(' This error typically occurs on very large repositories.');
console.error(' Suggestions:');
console.error(' 1. Add large vendored/generated directories to .gitnexusignore');
console.error(' 2. Increase Node.js heap: NODE_OPTIONS="--max-old-space-size=16384"');
console.error(' 3. Increase stack size: NODE_OPTIONS="--stack-size=4096"');
console.error('');
} else if (msg.includes('ERESOLVE') || msg.includes('Could not resolve dependency')) {
// Note: the original arborist "Cannot destructure property 'package' of
// 'node.target'" crash happens inside npm *before* gitnexus code runs,
// so it can't be caught here. This branch handles dependency-resolution
// errors that surface at runtime (e.g. dynamic require failures).
console.error(' This looks like an npm dependency resolution issue.');
console.error(' Suggestions:');
console.error(' 1. Clear the npm cache: npm cache clean --force');
console.error(' 2. Update npm: npm install -g npm@latest');
console.error(' 3. Reinstall gitnexus: npm install -g gitnexus@latest');
console.error(' 4. Or try npx directly: npx gitnexus@latest analyze');
console.error('');
} else if (
msg.includes('MODULE_NOT_FOUND') ||
msg.includes('Cannot find module') ||
msg.includes('ERR_MODULE_NOT_FOUND')
) {
console.error(' A required module could not be loaded. The installation may be corrupt.');
console.error(' Suggestions:');
console.error(' 1. Reinstall: npm install -g gitnexus@latest');
console.error(' 2. Clear cache: npm cache clean --force && npx gitnexus@latest analyze');
console.error('');
}
process.exitCode = 1;
return;
}

View file

@ -26,6 +26,7 @@ program
.option('--embeddings', 'Enable embedding generation for semantic search (off by default)')
.option('--skills', 'Generate repo-specific skill files from detected communities')
.option('--skip-agents-md', 'Skip updating the gitnexus section in AGENTS.md and CLAUDE.md')
.option('--no-stats', 'Omit volatile file/symbol counts from AGENTS.md and CLAUDE.md')
.option('--skip-git', 'Index a folder without requiring a .git directory')
.option('-v, --verbose', 'Enable verbose ingestion warnings (default: false)')
.addHelpText(

View file

@ -265,7 +265,7 @@ async function setupOpenCode(result: SetupResult): Promise<void> {
return;
}
const configPath = path.join(opencodeDir, 'config.json');
const configPath = path.join(opencodeDir, 'opencode.json');
try {
const existing = await readJsonFile(configPath);
const config = existing || {};

View file

@ -0,0 +1,112 @@
/**
* Shared AST utilities for the embedding pipeline.
* Centralizes parser caching and tree-sitter node lookups
* used by both chunker.ts and structural-extractor.ts.
*/
import { getLanguageFromFilename } from 'gitnexus-shared';
import {
createParserForLanguage,
isLanguageAvailable,
resolveLanguageKey,
} from '../tree-sitter/parser-loader.js';
const parserCache = new Map<string, any>();
/**
* Ensure parser is initialized and language is loaded, then parse content.
* Returns null if language is unavailable or parsing fails.
*/
export const ensureAndParse = async (content: string, filePath: string): Promise<any | null> => {
const language = getLanguageFromFilename(filePath);
if (!language) return null;
if (!isLanguageAvailable(language)) return null;
const parserKey = resolveLanguageKey(language, filePath);
let parserInstance = parserCache.get(parserKey);
if (!parserInstance) {
parserInstance = await createParserForLanguage(language, filePath);
parserCache.set(parserKey, parserInstance);
}
return parserInstance.parse(content);
};
const FUNCTION_LIKE_TYPES = new Set([
'function_declaration',
'function_definition',
'method_declaration',
'method_definition',
'function_item',
'function_signature_item',
'arrow_function',
'function_expression',
'generator_function_declaration',
'generator_function',
'async_function_declaration',
'async_arrow_function',
'constructor_declaration',
'constructor_definition',
'compact_constructor_declaration',
'short_function_declaration',
'proc_declaration',
'func_literal',
'local_function_statement',
'anonymous_function',
'lambda_literal',
'init_declaration',
'deinit_declaration',
]);
/**
* Find the first function/method-like declaration in a snippet AST.
* Used by the chunker when parsing node.content where absolute line
* numbers don't apply.
*/
export const findFunctionNode = (root: any): any | null => {
if (FUNCTION_LIKE_TYPES.has(root.type)) return root;
for (let i = 0; i < root.namedChildCount; i++) {
const child = root.namedChild(i);
if (!child) continue;
if (FUNCTION_LIKE_TYPES.has(child.type)) return child;
const found = findFunctionNode(child);
if (found) return found;
}
return null;
};
/**
* Find the first class/struct/interface/enum-like declaration in an AST.
* Used when parsing node.content (a snippet, not a full file) where
* absolute line numbers don't apply.
*/
export const findDeclarationNode = (root: any): any | null => {
const CLASS_LIKE_TYPES = new Set([
'class_declaration',
'class_definition',
'struct_declaration',
'struct_item',
'interface_declaration',
'interface_definition',
'enum_declaration',
'enum_item',
'type_declaration', // Go: type X struct
'declaration', // Go: type X struct
'object_declaration', // Kotlin: object
'impl_item', // Rust: impl
]);
if (CLASS_LIKE_TYPES.has(root.type)) return root;
for (let i = 0; i < root.namedChildCount; i++) {
const child = root.namedChild(i);
if (!child) continue;
if (CLASS_LIKE_TYPES.has(child.type)) return child;
const found = findDeclarationNode(child);
if (found) return found;
}
return null;
};

View file

@ -0,0 +1,63 @@
/**
* Character-based sliding window chunking (pure, no tree-sitter dependency)
*/
import { buildLineIndex, resolveChunkLines } from './line-index.js';
export interface Chunk {
text: string;
chunkIndex: number;
startOffset: number;
endOffset: number;
startLine: number;
endLine: number;
}
export const characterChunk = (
content: string,
startLine: number,
endLine: number,
chunkSize: number = 1200,
overlap: number = 120,
): Chunk[] => {
if (content.length <= chunkSize) {
return [
{
text: content,
chunkIndex: 0,
startOffset: 0,
endOffset: content.length,
startLine,
endLine,
},
];
}
const chunks: Chunk[] = [];
let offset = 0;
const lineOffsets = buildLineIndex(content);
while (offset < content.length) {
const end = Math.min(offset + chunkSize, content.length);
const chunkText = content.slice(offset, end);
const lineRange = resolveChunkLines(lineOffsets, offset, end, startLine);
chunks.push({
text: chunkText,
chunkIndex: chunks.length,
startOffset: offset,
endOffset: end,
startLine: lineRange.startLine,
endLine: lineRange.endLine,
});
offset = end - overlap;
if (offset >= content.length) break;
if (end >= content.length) break;
if (offset <= (chunks.length > 1 ? end - chunkSize : 0)) {
offset = end;
}
}
return chunks;
};

View file

@ -0,0 +1,363 @@
/**
* Chunker Module
*
* Splits code nodes into chunks for embedding.
* - Function/Method: AST-aware chunking by statement boundaries
* - Other types: character-based sliding window fallback
* - Short content ( chunkSize): no chunking
*/
export { type Chunk, characterChunk } from './character-chunk.js';
import { characterChunk } from './character-chunk.js';
import type { Chunk } from './character-chunk.js';
import { ensureAndParse, findDeclarationNode, findFunctionNode } from './ast-utils.js';
import { buildLineIndex, resolveChunkLines } from './line-index.js';
/**
* Main chunkNode function: dispatches by label
*/
export const chunkNode = async (
label: string,
content: string,
filePath: string,
startLine: number,
endLine: number,
chunkSize: number = 1200,
overlap: number = 120,
): Promise<Chunk[]> => {
// Content fits in one chunk — no splitting needed
if (content.length <= chunkSize) {
return [
{
text: content,
chunkIndex: 0,
startOffset: 0,
endOffset: content.length,
startLine,
endLine,
},
];
}
// Only function-like labels get AST chunking
if (label === 'Function' || label === 'Method' || label === 'Constructor') {
try {
const astChunks = await astChunk(content, filePath, startLine, endLine, chunkSize, overlap);
if (astChunks.length > 0) return astChunks;
} catch {
// AST parsing failed — fall through to character fallback
}
}
if (label === 'Class' || label === 'Interface') {
try {
const declarationChunks = await declarationChunk(
label,
content,
filePath,
startLine,
endLine,
chunkSize,
overlap,
);
if (declarationChunks.length > 0) return declarationChunks;
} catch {
// AST parsing failed — fall through to character fallback
}
}
// Character-based fallback for everything else
return characterChunk(content, startLine, endLine, chunkSize, overlap);
};
/**
* AST-based chunking for Function/Method
* Parse snippet content, locate the function declaration node,
* split body by statement boundaries.
*/
const astChunk = async (
content: string,
filePath: string,
startLine: number,
endLine: number,
chunkSize: number,
overlap: number,
): Promise<Chunk[]> => {
const tree = await ensureAndParse(content, filePath);
if (!tree) return [];
const root = tree.rootNode;
const lineOffsets = buildLineIndex(content);
// Find the function/method declaration in the snippet AST.
// tree-sitter parses node.content (a snippet), so rows are relative (0-based).
const targetNode = findFunctionNode(root);
if (!targetNode) return [];
// Get the body (statements) via childForFieldName('body')
const bodyNode = targetNode.childForFieldName('body');
if (!bodyNode) return [];
// Extract individual statements
const statements: Array<{ startIndex: number; endIndex: number }> = [];
for (let i = 0; i < bodyNode.namedChildCount; i++) {
const child = bodyNode.namedChild(i);
if (!child) continue;
statements.push({
startIndex: child.startIndex,
endIndex: child.endIndex,
});
}
if (statements.length === 0) return [];
return chunkByUnits(
content,
lineOffsets,
startLine,
chunkSize,
overlap,
statements,
targetNode.startIndex,
targetNode.endIndex,
true,
true,
);
};
const DECLARATION_BODY_NODE_TYPES = new Set([
'class_body',
'object_type',
'declaration_list',
'interface_body',
]);
const FIELD_LIKE_MEMBER_TYPES = new Set([
'field_definition',
'public_field_definition',
'property_definition',
'property_signature',
'variable_declarator',
'lexical_declaration',
'pair',
'enum_assignment',
]);
const declarationChunk = async (
label: 'Class' | 'Interface',
content: string,
filePath: string,
startLine: number,
endLine: number,
chunkSize: number,
overlap: number,
): Promise<Chunk[]> => {
const tree = await ensureAndParse(content, filePath);
if (!tree) return [];
const targetNode = findDeclarationNode(tree.rootNode);
if (!targetNode) return [];
const bodyNode = getDeclarationBodyNode(targetNode);
if (!bodyNode) return [];
const members = collectDeclarationUnits(bodyNode, label);
if (members.length === 0) return [];
return chunkByUnits(
content,
buildLineIndex(content),
startLine,
chunkSize,
overlap,
members,
targetNode.startIndex,
targetNode.endIndex,
false,
false,
);
};
const buildChunk = (
content: string,
lineOffsets: Int32Array,
chunkIndex: number,
startOffset: number,
endOffset: number,
baseStartLine: number,
): Chunk => {
const lineRange = resolveChunkLines(lineOffsets, startOffset, endOffset, baseStartLine);
return {
text: content.slice(startOffset, endOffset),
chunkIndex,
startOffset,
endOffset,
startLine: lineRange.startLine,
endLine: lineRange.endLine,
};
};
const chunkByUnits = (
content: string,
lineOffsets: Int32Array,
baseStartLine: number,
chunkSize: number,
overlap: number,
units: Array<{ startIndex: number; endIndex: number }>,
containerStartOffset: number,
containerEndOffset: number,
includeContainerPrefixOnFirstChunk: boolean,
includeContainerSuffixOnLastChunk: boolean,
): Chunk[] => {
const chunks: Chunk[] = [];
let chunkStartUnitIdx = 0;
while (chunkStartUnitIdx < units.length) {
const chunkStartOffset =
chunkStartUnitIdx === 0 && includeContainerPrefixOnFirstChunk
? containerStartOffset
: units[chunkStartUnitIdx].startIndex;
let chunkEndUnitIdx = chunkStartUnitIdx;
let candidateEndOffset =
chunkEndUnitIdx === units.length - 1 && includeContainerSuffixOnLastChunk
? containerEndOffset
: units[chunkEndUnitIdx].endIndex;
while (chunkEndUnitIdx + 1 < units.length) {
const nextEndOffset =
chunkEndUnitIdx + 1 === units.length - 1 && includeContainerSuffixOnLastChunk
? containerEndOffset
: units[chunkEndUnitIdx + 1].endIndex;
if (nextEndOffset - chunkStartOffset > chunkSize) break;
chunkEndUnitIdx += 1;
candidateEndOffset = nextEndOffset;
}
if (candidateEndOffset - chunkStartOffset > chunkSize) {
const oversizedUnit = units[chunkStartUnitIdx];
const oversizedLineRange = resolveChunkLines(
lineOffsets,
oversizedUnit.startIndex,
oversizedUnit.endIndex,
baseStartLine,
);
const oversizedChunks = characterChunk(
content.slice(oversizedUnit.startIndex, oversizedUnit.endIndex),
oversizedLineRange.startLine,
oversizedLineRange.endLine,
chunkSize,
overlap,
).map((chunk, offsetIdx) => ({
...chunk,
chunkIndex: chunks.length + offsetIdx,
startOffset: chunk.startOffset + oversizedUnit.startIndex,
endOffset: chunk.endOffset + oversizedUnit.startIndex,
}));
chunks.push(...oversizedChunks);
chunkStartUnitIdx += 1;
continue;
}
chunks.push(
buildChunk(
content,
lineOffsets,
chunks.length,
chunkStartOffset,
candidateEndOffset,
baseStartLine,
),
);
if (chunkEndUnitIdx === units.length - 1) {
break;
}
const nextChunkStartUnitIdx = findOverlapStartIndex(
units,
chunkStartUnitIdx,
chunkEndUnitIdx,
overlap,
);
if (nextChunkStartUnitIdx <= chunkStartUnitIdx) {
chunkStartUnitIdx = chunkEndUnitIdx + 1;
} else {
chunkStartUnitIdx = nextChunkStartUnitIdx;
}
}
return chunks;
};
const findOverlapStartIndex = (
statements: Array<{ startIndex: number; endIndex: number }>,
chunkStartStmtIdx: number,
chunkEndStmtIdx: number,
overlapSize: number,
): number => {
if (overlapSize <= 0) return chunkEndStmtIdx + 1;
let overlapStartIdx = chunkEndStmtIdx;
while (overlapStartIdx > chunkStartStmtIdx) {
const overlapLength =
statements[chunkEndStmtIdx].endIndex - statements[overlapStartIdx - 1].startIndex;
if (overlapLength > overlapSize) break;
overlapStartIdx -= 1;
}
return overlapStartIdx;
};
const getDeclarationBodyNode = (node: any): any | null => {
const bodyNode = node.childForFieldName?.('body');
if (bodyNode) return bodyNode;
for (let i = 0; i < node.namedChildCount; i++) {
const child = node.namedChild(i);
if (!child) continue;
if (DECLARATION_BODY_NODE_TYPES.has(child.type)) return child;
}
return null;
};
const collectDeclarationUnits = (
bodyNode: any,
label: 'Class' | 'Interface',
): Array<{ startIndex: number; endIndex: number }> => {
const members: Array<{ startIndex: number; endIndex: number; groupable: boolean }> = [];
for (let i = 0; i < bodyNode.namedChildCount; i++) {
const child = bodyNode.namedChild(i);
if (!child) continue;
members.push({
startIndex: child.startIndex,
endIndex: child.endIndex,
groupable: label === 'Class' && FIELD_LIKE_MEMBER_TYPES.has(child.type),
});
}
if (members.length === 0) return [];
const grouped: Array<{ startIndex: number; endIndex: number }> = [];
let current = members[0];
for (let i = 1; i < members.length; i++) {
const next = members[i];
if (current.groupable && next.groupable) {
current = {
startIndex: current.startIndex,
endIndex: next.endIndex,
groupable: true,
};
continue;
}
grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex });
current = next;
}
grouped.push({ startIndex: current.startIndex, endIndex: current.endIndex });
return grouped;
};

View file

@ -157,6 +157,11 @@ export const initEmbedder = async (
try {
// Configure transformers.js environment
env.allowLocalModels = false;
// Default cache to user-writable location. transformers.js defaults to
// ./node_modules/.cache inside its own install dir, which is unwritable
// when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
// Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
env.cacheDir = process.env.HF_HOME ?? `${process.env.HOME}/.cache/huggingface`;
const isDev = process.env.NODE_ENV === 'development';
if (isDev) {

View file

@ -3,12 +3,13 @@
*
* Orchestrates the background embedding process:
* 1. Query embeddable nodes from LadybugDB
* 2. Generate text representations
* 3. Batch embed using transformers.js
* 4. Update LadybugDB with embeddings
* 2. Generate text representations with enriched metadata
* 3. Chunk long nodes, batch embed
* 4. Update LadybugDB with chunk-aware embeddings
* 5. Create vector index for semantic search
*/
import { createHash } from 'crypto';
import {
initEmbedder,
embedBatch,
@ -16,19 +17,54 @@ import {
embeddingToArray,
isEmbedderReady,
} from './embedder.js';
import { generateBatchEmbeddingTexts } from './text-generator.js';
import { generateEmbeddingText } from './text-generator.js';
import { chunkNode, characterChunk } from './chunker.js';
import { extractStructuralNames } from './structural-extractor.js';
import {
type EmbeddingProgress,
type EmbeddingConfig,
type EmbeddableNode,
type SemanticSearchResult,
type ModelProgress,
type EmbeddingContext,
DEFAULT_EMBEDDING_CONFIG,
EMBEDDABLE_LABELS,
isShortLabel,
LABELS_WITH_EXPORTED,
STRUCTURAL_LABELS,
collectBestChunks,
} from './types.js';
import {
EMBEDDING_TABLE_NAME,
EMBEDDING_INDEX_NAME,
CREATE_VECTOR_INDEX_QUERY,
STALE_HASH_SENTINEL,
} from '../lbug/schema.js';
import { loadVectorExtension } from '../lbug/lbug-adapter.js';
const isDev = process.env.NODE_ENV === 'development';
/**
* Compute a stable content fingerprint for an embeddable node.
* Used to detect when the underlying text has changed so stale vectors
* can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for
* vector-indexed rows).
*/
export const contentHashForNode = (
node: EmbeddableNode,
config: Partial<EmbeddingConfig> = {},
): string => {
// Hash must be deterministic across runs, so exclude methodNames/fieldNames
// which are populated during the batch loop via AST extraction.
// Using only node.content ensures the hash stays stable.
const text = generateEmbeddingText(
{ ...node, methodNames: undefined, fieldNames: undefined },
node.content,
config,
);
return createHash('sha1').update(text).digest('hex');
};
/**
* Progress callback type
*/
@ -36,37 +72,50 @@ export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void;
/**
* Query all embeddable nodes from LadybugDB
* Uses table-specific queries (File has different schema than code elements)
* Uses table-specific queries for different label types
*/
const queryEmbeddableNodes = async (
executeQuery: (cypher: string) => Promise<any[]>,
): Promise<EmbeddableNode[]> => {
const allNodes: EmbeddableNode[] = [];
// Query each embeddable table with table-specific columns
for (const label of EMBEDDABLE_LABELS) {
try {
let query: string;
if (label === 'File') {
// File nodes don't have startLine/endLine
if (label === 'Method') {
// Method has parameterCount and returnType
query = `
MATCH (n:File)
RETURN n.id AS id, n.name AS name, 'File' AS label,
n.filePath AS filePath, n.content AS content
MATCH (n:Method)
RETURN n.id AS id, n.name AS name, 'Method' AS label,
n.filePath AS filePath, n.content AS content,
n.startLine AS startLine, n.endLine AS endLine,
n.isExported AS isExported, n.description AS description,
n.parameterCount AS parameterCount, n.returnType AS returnType
`;
} else if (LABELS_WITH_EXPORTED.has(label)) {
// Function, Class, Interface have isExported and description
query = `
MATCH (n:\`${label}\`)
RETURN n.id AS id, n.name AS name, '${label}' AS label,
n.filePath AS filePath, n.content AS content,
n.startLine AS startLine, n.endLine AS endLine,
n.isExported AS isExported, n.description AS description
`;
} else {
// Code elements have startLine/endLine
// Multi-language tables (Struct, Enum, etc.) — have description but no isExported
query = `
MATCH (n:${label})
RETURN n.id AS id, n.name AS name, '${label}' AS label,
MATCH (n:\`${label}\`)
RETURN n.id AS id, n.name AS name, '${label}' AS label,
n.filePath AS filePath, n.content AS content,
n.startLine AS startLine, n.endLine AS endLine
n.startLine AS startLine, n.endLine AS endLine,
n.description AS description
`;
}
const rows = await executeQuery(query);
for (const row of rows) {
const hasExportedColumn = label === 'Method' || LABELS_WITH_EXPORTED.has(label);
allNodes.push({
id: row.id ?? row[0],
name: row.name ?? row[1],
@ -75,10 +124,17 @@ const queryEmbeddableNodes = async (
content: row.content ?? row[4] ?? '',
startLine: row.startLine ?? row[5],
endLine: row.endLine ?? row[6],
isExported: hasExportedColumn ? (row.isExported ?? row[7]) : undefined,
description: row.description ?? (hasExportedColumn ? row[8] : row[7]),
...(label === 'Method'
? {
parameterCount: row.parameterCount ?? row[9],
returnType: row.returnType ?? row[10],
}
: {}),
});
}
} catch (error) {
// Table might not exist or be empty, continue
if (isDev) {
console.warn(`Query for ${label} nodes failed:`, error);
}
@ -89,52 +145,52 @@ const queryEmbeddableNodes = async (
};
/**
* Batch INSERT embeddings into separate CodeEmbedding table
* Using a separate lightweight table avoids copy-on-write overhead
* that occurs when UPDATEing nodes with large content fields
* Batch INSERT chunk-aware embeddings into CodeEmbedding table
*/
const batchInsertEmbeddings = async (
export const batchInsertEmbeddings = async (
executeWithReusedStatement: (
cypher: string,
paramsList: Array<Record<string, any>>,
) => Promise<void>,
updates: Array<{ id: string; embedding: number[] }>,
updates: Array<{
nodeId: string;
chunkIndex: number;
startLine: number;
endLine: number;
embedding: number[];
contentHash?: string;
}>,
): Promise<void> => {
// INSERT into separate embedding table - much more memory efficient!
const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`;
const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding }));
const cypher = `CREATE (e:${EMBEDDING_TABLE_NAME} {id: $id, nodeId: $nodeId, chunkIndex: $chunkIndex, startLine: $startLine, endLine: $endLine, embedding: $embedding, contentHash: $contentHash})`;
const paramsList = updates.map((u) => ({
id: `${u.nodeId}:${u.chunkIndex}`,
nodeId: u.nodeId,
chunkIndex: u.chunkIndex,
startLine: u.startLine,
endLine: u.endLine,
embedding: u.embedding,
contentHash: u.contentHash ?? STALE_HASH_SENTINEL,
}));
await executeWithReusedStatement(cypher, paramsList);
};
/**
* Create the vector index for semantic search
* Now indexes the separate CodeEmbedding table
*/
let vectorExtensionLoaded = false;
* Now indexes the separate CodeEmbedding table.
* Delegates extension loading to lbug-adapter's loadVectorExtension(),
* which owns the VECTOR extension lifecycle and state tracking.
*/
const createVectorIndex = async (
executeQuery: (cypher: string) => Promise<any[]>,
): Promise<void> => {
// LadybugDB v0.15+ requires explicit VECTOR extension loading (once per session)
if (!vectorExtensionLoaded) {
try {
await executeQuery('INSTALL VECTOR');
await executeQuery('LOAD EXTENSION VECTOR');
vectorExtensionLoaded = true;
} catch {
// Extension may already be loaded — CREATE_VECTOR_INDEX will fail clearly if not
vectorExtensionLoaded = true;
}
}
const cypher = `
CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')
`;
// Delegate to the adapter which tracks loaded state and handles DB reconnect resets
await loadVectorExtension();
try {
await executeQuery(cypher);
await executeQuery(CREATE_VECTOR_INDEX_QUERY);
} catch (error) {
// Index might already exist
if (isDev) {
console.warn('Vector index creation warning:', error);
}
@ -149,6 +205,11 @@ const createVectorIndex = async (
* @param onProgress - Callback for progress updates
* @param config - Optional configuration override
* @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
* @param context - Optional repo/server context for metadata enrichment
* @param existingEmbeddings - Optional map of nodeId contentHash for incremental mode.
* Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
* and re-embedded; nodes not in the map are embedded fresh.
*/
export const runEmbeddingPipeline = async (
executeQuery: (cypher: string) => Promise<any[]>,
@ -159,6 +220,8 @@ export const runEmbeddingPipeline = async (
onProgress: EmbeddingProgressCallback,
config: Partial<EmbeddingConfig> = {},
skipNodeIds?: Set<string>,
context?: EmbeddingContext,
existingEmbeddings?: Map<string, string>,
): Promise<void> => {
const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
@ -194,13 +257,65 @@ export const runEmbeddingPipeline = async (
// Phase 2: Query embeddable nodes
let nodes = await queryEmbeddableNodes(executeQuery);
// Incremental mode: filter out nodes that already have embeddings
if (skipNodeIds && skipNodeIds.size > 0) {
// Apply context metadata
if (context?.repoName) {
for (const node of nodes) {
node.repoName = context.repoName;
node.serverName = context.serverName;
}
}
// Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
// Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
// (avoids double computation).
const computedStaleHashes = new Map<string, string>();
if (existingEmbeddings && existingEmbeddings.size > 0) {
const beforeCount = nodes.length;
nodes = nodes.filter((n) => !skipNodeIds.has(n.id));
const staleNodeIds: string[] = [];
nodes = nodes.filter((n) => {
const existingHash = existingEmbeddings.get(n.id);
if (existingHash === undefined) {
// New node — needs embedding
return true;
}
const currentHash = contentHashForNode(n, finalConfig);
if (currentHash !== existingHash) {
// Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
computedStaleHashes.set(n.id, currentHash);
staleNodeIds.push(n.id);
return true;
}
// Hash matches — skip (fresh); no need to cache hash for skipped nodes
return false;
});
// DELETE stale embedding rows so they can be re-inserted
// (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
if (staleNodeIds.length > 0) {
if (isDev) {
console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
}
try {
await executeWithReusedStatement(
`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`,
staleNodeIds.map((nodeId) => ({ nodeId })),
);
} catch (err) {
// "does not exist" = rows already gone — safe to proceed.
// All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT
// for vector-indexed properties) — propagate so the pipeline aborts cleanly.
const msg = err instanceof Error ? err.message : String(err);
if (!msg.includes('does not exist')) {
throw new Error(
`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`,
);
}
}
}
if (isDev) {
console.log(
`📦 Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`,
`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`,
);
}
}
@ -212,6 +327,11 @@ export const runEmbeddingPipeline = async (
}
if (totalNodes === 0) {
// Ensure the vector index exists even when no new nodes need embedding.
// A prior crash or first-time incremental run may have left CodeEmbedding
// rows without ever reaching index creation.
await createVectorIndex(executeQuery);
onProgress({
phase: 'ready',
percent: 100,
@ -221,10 +341,12 @@ export const runEmbeddingPipeline = async (
return;
}
// Phase 3: Batch embed nodes
// Phase 3: Chunk + embed nodes
const batchSize = finalConfig.batchSize;
const totalBatches = Math.ceil(totalNodes / batchSize);
const chunkSize = finalConfig.chunkSize;
const overlap = finalConfig.overlap;
let processedNodes = 0;
let totalChunks = 0;
onProgress({
phase: 'embedding',
@ -232,39 +354,116 @@ export const runEmbeddingPipeline = async (
nodesProcessed: 0,
totalNodes,
currentBatch: 0,
totalBatches,
totalBatches: Math.ceil(totalNodes / batchSize),
});
for (let batchIndex = 0; batchIndex < totalBatches; batchIndex++) {
const start = batchIndex * batchSize;
const end = Math.min(start + batchSize, totalNodes);
const batch = nodes.slice(start, end);
// Process in batches of nodes
for (let batchIndex = 0; batchIndex < totalNodes; batchIndex += batchSize) {
const batch = nodes.slice(batchIndex, batchIndex + batchSize);
// Generate texts for this batch
const texts = generateBatchEmbeddingTexts(batch, finalConfig);
// Chunk each node and generate text
const allTexts: string[] = [];
const allUpdates: Array<{
nodeId: string;
chunkIndex: number;
startLine: number;
endLine: number;
contentHash: string;
}> = [];
// Embed the batch
const embeddings = await embedBatch(texts);
for (const node of batch) {
const isShort = isShortLabel(node.label);
const startLine = node.startLine ?? 0;
const endLine = node.endLine ?? 0;
// Update LadybugDB with embeddings
const updates = batch.map((node, i) => ({
id: node.id,
embedding: embeddingToArray(embeddings[i]),
}));
// Extract structural names for class-like nodes via AST extractors
if (!isShort && STRUCTURAL_LABELS.has(node.label)) {
try {
const names = await extractStructuralNames(node.content, node.filePath);
node.methodNames = names.methodNames;
node.fieldNames = names.fieldNames;
} catch {
// AST extraction failed — names stay undefined, text-generator handles gracefully
}
}
await batchInsertEmbeddings(executeWithReusedStatement, updates);
// Compute content hash once per node (re-use cached value for stale nodes)
const hash = computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig);
let chunks: Array<{ text: string; chunkIndex: number; startLine: number; endLine: number }>;
if (isShort) {
chunks = [{ text: node.content, chunkIndex: 0, startLine, endLine }];
} else {
try {
chunks = await chunkNode(
node.label,
node.content,
node.filePath,
startLine,
endLine,
chunkSize,
overlap,
);
} catch (chunkErr) {
if (isDev) {
console.warn(
`⚠️ AST chunking failed for ${node.label} "${node.name}" (${node.filePath}), falling back to character-based chunking:`,
chunkErr,
);
}
chunks = characterChunk(node.content, startLine, endLine, chunkSize, overlap);
}
}
for (const chunk of chunks) {
const text = generateEmbeddingText(node, chunk.text, finalConfig);
allTexts.push(text);
allUpdates.push({
nodeId: node.id,
chunkIndex: chunk.chunkIndex,
startLine: chunk.startLine,
endLine: chunk.endLine,
contentHash: hash,
});
}
}
// Embed chunk texts in sub-batches to control memory
const EMBED_SUB_BATCH = 8;
for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) {
const subTexts = allTexts.slice(si, si + EMBED_SUB_BATCH);
const subUpdates = allUpdates.slice(si, si + EMBED_SUB_BATCH);
let embeddings: Float32Array[];
try {
embeddings = await embedBatch(subTexts);
} catch (embedErr) {
console.error(
`❌ embedBatch failed for ${subTexts.length} texts (first: "${subTexts[0]?.substring(0, 80)}..."):`,
embedErr,
);
throw embedErr;
}
const dbUpdates = subUpdates.map((u, i) => ({
...u,
embedding: embeddingToArray(embeddings[i]),
}));
await batchInsertEmbeddings(executeWithReusedStatement, dbUpdates);
}
processedNodes += batch.length;
totalChunks += allUpdates.length;
// Report progress (20-90% for embedding phase)
const embeddingProgress = 20 + (processedNodes / totalNodes) * 70;
onProgress({
phase: 'embedding',
percent: Math.round(embeddingProgress),
nodesProcessed: processedNodes,
totalNodes,
currentBatch: batchIndex + 1,
totalBatches,
currentBatch: Math.floor(batchIndex / batchSize) + 1,
totalBatches: Math.ceil(totalNodes / batchSize),
});
}
@ -282,7 +481,6 @@ export const runEmbeddingPipeline = async (
await createVectorIndex(executeQuery);
// Complete
onProgress({
phase: 'ready',
percent: 100,
@ -291,7 +489,9 @@ export const runEmbeddingPipeline = async (
});
if (isDev) {
console.log('✅ Embedding pipeline complete!');
console.log(
`✅ Embedding pipeline complete! (${totalChunks} chunks from ${totalNodes} nodes)`,
);
}
} catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
@ -311,15 +511,7 @@ export const runEmbeddingPipeline = async (
};
/**
* Perform semantic search using the vector index
*
* Uses CodeEmbedding table and queries each node table to get metadata
*
* @param executeQuery - Function to execute Cypher queries
* @param query - Search query text
* @param k - Number of results to return (default: 10)
* @param maxDistance - Maximum distance threshold (default: 0.5)
* @returns Array of search results ordered by relevance
* Perform semantic search using the vector index with chunk deduplication
*/
export const semanticSearch = async (
executeQuery: (cypher: string) => Promise<any[]>,
@ -331,37 +523,46 @@ export const semanticSearch = async (
throw new Error('Embedding model not initialized. Run embedding pipeline first.');
}
// Embed the query
const queryEmbedding = await embedText(query);
const queryVec = embeddingToArray(queryEmbedding);
const queryVecStr = `[${queryVec.join(',')}]`;
// Query the vector index on CodeEmbedding to get nodeIds and distances
const vectorQuery = `
CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx',
CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${k})
YIELD node AS emb, distance
WITH emb, distance
WHERE distance < ${maxDistance}
RETURN emb.nodeId AS nodeId, distance
ORDER BY distance
`;
const bestChunks = await collectBestChunks(k, async (fetchLimit) => {
const vectorQuery = `
CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}',
CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${fetchLimit})
YIELD node AS emb, distance
WITH emb, distance
WHERE distance < ${maxDistance}
RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex,
emb.startLine AS startLine, emb.endLine AS endLine, distance
ORDER BY distance
`;
const embResults = await executeQuery(vectorQuery);
const embResults = await executeQuery(vectorQuery);
return embResults.map((row) => ({
nodeId: row.nodeId ?? row[0],
chunkIndex: row.chunkIndex ?? row[1] ?? 0,
startLine: row.startLine ?? row[2] ?? 0,
endLine: row.endLine ?? row[3] ?? 0,
distance: row.distance ?? row[4],
}));
});
if (embResults.length === 0) {
if (bestChunks.size === 0) {
return [];
}
// Group results by label for batched metadata queries
const byLabel = new Map<string, Array<{ nodeId: string; distance: number }>>();
for (const embRow of embResults) {
const nodeId = embRow.nodeId ?? embRow[0];
const distance = embRow.distance ?? embRow[1];
const byLabel = new Map<
string,
Array<{ nodeId: string; distance: number } & Record<string, any>>
>();
for (const [nodeId, chunk] of Array.from(bestChunks.entries()).slice(0, k)) {
const labelEndIdx = nodeId.indexOf(':');
const label = labelEndIdx > 0 ? nodeId.substring(0, labelEndIdx) : 'Unknown';
if (!byLabel.has(label)) byLabel.set(label, []);
byLabel.get(label)!.push({ nodeId, distance });
byLabel.get(label)!.push({ nodeId, ...chunk });
}
// Batch-fetch metadata per label
@ -370,19 +571,11 @@ export const semanticSearch = async (
for (const [label, items] of byLabel) {
const idList = items.map((i) => `'${i.nodeId.replace(/'/g, "''")}'`).join(', ');
try {
let nodeQuery: string;
if (label === 'File') {
nodeQuery = `
MATCH (n:File) WHERE n.id IN [${idList}]
RETURN n.id AS id, n.name AS name, n.filePath AS filePath
`;
} else {
nodeQuery = `
MATCH (n:${label}) WHERE n.id IN [${idList}]
RETURN n.id AS id, n.name AS name, n.filePath AS filePath,
n.startLine AS startLine, n.endLine AS endLine
`;
}
const nodeQuery = `
MATCH (n:\`${label}\`) WHERE n.id IN [${idList}]
RETURN n.id AS id, n.name AS name, n.filePath AS filePath,
n.startLine AS startLine, n.endLine AS endLine
`;
const nodeRows = await executeQuery(nodeQuery);
const rowMap = new Map<string, any>();
for (const row of nodeRows) {
@ -398,8 +591,8 @@ export const semanticSearch = async (
label,
filePath: nodeRow.filePath ?? nodeRow[2] ?? '',
distance: item.distance,
startLine: label !== 'File' ? (nodeRow.startLine ?? nodeRow[3]) : undefined,
endLine: label !== 'File' ? (nodeRow.endLine ?? nodeRow[4]) : undefined,
startLine: item.startLine,
endLine: item.endLine,
});
}
}
@ -408,7 +601,6 @@ export const semanticSearch = async (
}
}
// Re-sort by distance since batch queries may have mixed order
results.sort((a, b) => a.distance - b.distance);
return results;
@ -416,16 +608,6 @@ export const semanticSearch = async (
/**
* Semantic search with graph expansion (flattened results)
*
* Note: With multi-table schema, graph traversal is simplified.
* Returns semantic matches with their metadata.
* For full graph traversal, use execute_vector_cypher tool directly.
*
* @param executeQuery - Function to execute Cypher queries
* @param query - Search query text
* @param k - Number of initial semantic matches (default: 5)
* @param _hops - Unused (kept for API compatibility).
* @returns Semantic matches with metadata
*/
export const semanticSearchWithContext = async (
executeQuery: (cypher: string) => Promise<any[]>,
@ -433,8 +615,6 @@ export const semanticSearchWithContext = async (
k: number = 5,
_hops: number = 1,
): Promise<any[]> => {
// For multi-table schema, just return semantic search results
// Graph traversal is complex with separate tables - use execute_vector_cypher instead
const results = await semanticSearch(executeQuery, query, k, 0.5);
return results.map((r) => ({

View file

@ -0,0 +1,50 @@
export interface ResolvedLineRange {
startLine: number;
endLine: number;
}
export const buildLineIndex = (content: string): Int32Array => {
const offsets: number[] = [0];
for (let i = 0; i < content.length; i++) {
if (content.charCodeAt(i) === 10) offsets.push(i + 1);
}
return new Int32Array(offsets);
};
const clampOffset = (lineOffsets: Int32Array, charOffset: number): number => {
if (lineOffsets.length === 0) return 0;
const maxOffset = lineOffsets[lineOffsets.length - 1];
if (charOffset < 0) return 0;
if (charOffset > maxOffset) return maxOffset;
return charOffset;
};
export const lineFromOffset = (lineOffsets: Int32Array, charOffset: number): number => {
if (lineOffsets.length === 0) return 0;
const clamped = clampOffset(lineOffsets, charOffset);
let lo = 0;
let hi = lineOffsets.length - 1;
while (lo < hi) {
const mid = (lo + hi + 1) >> 1;
if (lineOffsets[mid] <= clamped) lo = mid;
else hi = mid - 1;
}
return lo;
};
export const resolveChunkLines = (
lineOffsets: Int32Array,
startOffset: number,
endOffset: number,
baseStartLine: number,
): ResolvedLineRange => {
const relativeStartLine = lineFromOffset(lineOffsets, startOffset);
const effectiveEndOffset = endOffset > startOffset ? endOffset - 1 : startOffset;
const relativeEndLine = lineFromOffset(lineOffsets, effectiveEndOffset);
return {
startLine: baseStartLine + relativeStartLine,
endLine: baseStartLine + relativeEndLine,
};
};

View file

@ -0,0 +1,37 @@
/**
* Server Mapping Configuration
*
* Reads ~/.gitnexus/server-mapping.json to map repo names to service names.
* Used in embedding text to enrich metadata with microservice context.
*/
import fs from 'fs/promises';
import path from 'path';
import os from 'os';
const MAPPING_FILE = path.join(os.homedir(), '.gitnexus', 'server-mapping.json');
let cachedMapping: Record<string, string> | null = null;
/**
* Read the server mapping file and return the serverName for a given repoName.
* Returns undefined if no mapping exists.
*/
export const readServerMapping = async (repoName: string): Promise<string | undefined> => {
try {
if (!cachedMapping) {
const raw = await fs.readFile(MAPPING_FILE, 'utf-8');
cachedMapping = JSON.parse(raw);
}
return cachedMapping[repoName];
} catch {
return undefined;
}
};
/**
* Clear the cached mapping (useful for testing or after file changes)
*/
export const clearServerMappingCache = (): void => {
cachedMapping = null;
};

View file

@ -0,0 +1,89 @@
/**
* Structural Extractor Module
*
* Reuses ingestion pipeline's AST-based MethodExtractor / FieldExtractor
* to extract method and field names for embedding text generation.
*/
import { getProviderForFile } from '../ingestion/languages/index.js';
import type { MethodExtractorContext, ExtractedMethods } from '../ingestion/method-types.js';
import type { FieldExtractorContext, ExtractedFields } from '../ingestion/field-types.js';
import type { LanguageProvider } from '../ingestion/language-provider.js';
import { buildTypeEnv } from '../ingestion/type-env.js';
import { SupportedLanguages } from 'gitnexus-shared';
import { ensureAndParse, findDeclarationNode } from './ast-utils.js';
export interface StructuralNames {
methodNames: string[];
fieldNames: string[];
}
const NOOP_SYMBOL_TABLE = {
lookupExactAll: () => [],
lookupExact: () => undefined,
lookupExactFull: () => undefined,
} as any;
/**
* Extract method and field names from a class/struct/interface node
* using the ingestion pipeline's AST extractors.
*/
export const extractStructuralNames = async (
content: string,
filePath: string,
): Promise<StructuralNames> => {
const provider = getProviderForFile(filePath);
if (!provider) return { methodNames: [], fieldNames: [] };
const tree = await ensureAndParse(content, filePath);
if (!tree) return { methodNames: [], fieldNames: [] };
// Parse node.content (a snippet) — find declaration directly, not by range
const classNode = findDeclarationNode(tree.rootNode);
if (!classNode) return { methodNames: [], fieldNames: [] };
const language = provider.id;
const methodNames = extractMethodNames(classNode, provider, filePath, language);
const fieldNames = extractFieldNames(classNode, provider, tree, filePath, language);
return { methodNames, fieldNames };
};
function extractMethodNames(
classNode: any,
provider: LanguageProvider,
filePath: string,
language: SupportedLanguages,
): string[] {
if (!provider.methodExtractor) return [];
const context: MethodExtractorContext = { filePath, language };
const result: ExtractedMethods | null = provider.methodExtractor.extract(classNode, context);
if (!result?.methods?.length) return [];
return result.methods.map((m) => m.name);
}
function extractFieldNames(
classNode: any,
provider: LanguageProvider,
tree: any,
filePath: string,
language: SupportedLanguages,
): string[] {
if (!provider.fieldExtractor) return [];
const typeEnv = buildTypeEnv(tree, language);
const context: FieldExtractorContext = {
typeEnv,
symbolTable: NOOP_SYMBOL_TABLE,
filePath,
language,
};
const result: ExtractedFields | null = provider.fieldExtractor.extract(classNode, context);
if (!result?.fields?.length) return [];
return result.fields.map((f) => f.name);
}

View file

@ -1,206 +1,253 @@
/**
* Text Generator Module
*
* Pure functions to generate embedding text from code nodes.
* Combines node metadata with code snippets for semantic matching.
* Generates enriched embedding text from code nodes with metadata.
* Supports chunkable labels (Function/Method with AST chunking),
* Class-specific structural text, and short-node direct embed.
*
* Method/field names for Class nodes are extracted by the ingestion
* pipeline's AST extractors and passed via node.methodNames/node.fieldNames.
*/
import type { EmbeddableNode, EmbeddingConfig } from './types.js';
import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
import { DEFAULT_EMBEDDING_CONFIG, isShortLabel } from './types.js';
/**
* Extract the filename from a file path
* Truncate description to max length at sentence/word boundary
*/
const getFileName = (filePath: string): string => {
const parts = filePath.split('/');
return parts[parts.length - 1] || filePath;
};
const truncateDescription = (text: string, maxLength: number): string => {
if (text.length <= maxLength) return text;
/**
* Extract the directory path from a file path
*/
const getDirectory = (filePath: string): string => {
const parts = filePath.split('/');
parts.pop();
return parts.join('/') || '';
};
const truncated = text.slice(0, maxLength);
/**
* Truncate content to max length, preserving word boundaries
*/
const truncateContent = (content: string, maxLength: number): string => {
if (content.length <= maxLength) {
return content;
// Try sentence boundary (. ! ?)
const sentenceEnd = Math.max(
truncated.lastIndexOf('. '),
truncated.lastIndexOf('! '),
truncated.lastIndexOf('? '),
);
if (sentenceEnd > maxLength * 0.5) {
return truncated.slice(0, sentenceEnd + 1);
}
// Find last space before maxLength to avoid cutting words
const truncated = content.slice(0, maxLength);
// Try word boundary
const lastSpace = truncated.lastIndexOf(' ');
if (lastSpace > maxLength * 0.8) {
return truncated.slice(0, lastSpace) + '...';
if (lastSpace > maxLength * 0.5) {
return truncated.slice(0, lastSpace);
}
return truncated + '...';
return truncated;
};
/**
* Clean code content for embedding
* Removes excessive whitespace while preserving structure
*/
const cleanContent = (content: string): string => {
return (
content
// Normalize line endings
.replace(/\r\n/g, '\n')
// Remove excessive blank lines (more than 2)
.replace(/\n{3,}/g, '\n\n')
// Trim each line
.split('\n')
.map((line) => line.trimEnd())
.join('\n')
.trim()
);
return content
.replace(/\r\n/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.split('\n')
.map((line) => line.trimEnd())
.join('\n')
.trim();
};
/**
* Generate embedding text for a Function node
* Build metadata header for a node
*/
const generateFunctionText = (node: EmbeddableNode, maxSnippetLength: number): string => {
const parts: string[] = [`Function: ${node.name}`, `File: ${getFileName(node.filePath)}`];
const buildMetadataHeader = (node: EmbeddableNode, config: Partial<EmbeddingConfig>): string => {
const parts: string[] = [];
const dir = getDirectory(node.filePath);
if (dir) {
parts.push(`Directory: ${dir}`);
// Label + name
parts.push(`${node.label}: ${node.name}`);
// Repo name
if (node.repoName) {
parts.push(`Repo: ${node.repoName}`);
}
if (node.content) {
const cleanedContent = cleanContent(node.content);
const snippet = truncateContent(cleanedContent, maxSnippetLength);
parts.push('', snippet);
// Server name (optional)
if (node.serverName) {
parts.push(`Server: ${node.serverName}`);
}
// Full file path
parts.push(`Path: ${node.filePath}`);
// Export status
if (node.isExported !== undefined) {
parts.push(`Export: ${node.isExported}`);
}
// Description (truncated)
if (node.description) {
const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength;
const truncated = truncateDescription(node.description, maxLen);
if (truncated) {
parts.push(truncated);
}
}
return parts.join('\n');
};
/**
* Generate embedding text for a Class node
*/
const generateClassText = (node: EmbeddableNode, maxSnippetLength: number): string => {
const parts: string[] = [`Class: ${node.name}`, `File: ${getFileName(node.filePath)}`];
const generateCodeBodyText = (
node: EmbeddableNode,
codeBody: string,
config: Partial<EmbeddingConfig>,
): string => {
const header = buildMetadataHeader(node, config);
const cleaned = cleanContent(codeBody);
return `${header}\n\n${cleaned}`;
};
const dir = getDirectory(node.filePath);
if (dir) {
parts.push(`Directory: ${dir}`);
/**
* Generate embedding text for Class nodes
* Signature + properties + method name list only (no method bodies)
* Method/field names come from AST extractors via node.methodNames/node.fieldNames.
*/
const generateClassText = (
node: EmbeddableNode,
codeBody: string,
config: Partial<EmbeddingConfig>,
): string => {
return generateStructuralTypeText(node, codeBody, config);
};
const generateStructuralTypeText = (
node: EmbeddableNode,
codeBody: string,
config: Partial<EmbeddingConfig>,
): string => {
const header = buildMetadataHeader(node, config);
const parts: string[] = [header];
if (node.methodNames?.length) {
parts.push(`Methods: ${node.methodNames.join(', ')}`);
}
if (node.fieldNames?.length) {
parts.push(`Properties: ${node.fieldNames.join(', ')}`);
}
if (node.content) {
const cleanedContent = cleanContent(node.content);
const snippet = truncateContent(cleanedContent, maxSnippetLength);
parts.push('', snippet);
const declarationOnly = extractDeclarationOnly(cleanContent(node.content));
if (declarationOnly) {
parts.push('', declarationOnly);
}
const cleanedChunk = cleanContent(codeBody);
if (cleanedChunk && cleanedChunk !== cleanContent(node.content)) {
parts.push('', cleanedChunk);
}
return parts.join('\n');
};
/**
* Generate embedding text for a Method node
*/
const generateMethodText = (node: EmbeddableNode, maxSnippetLength: number): string => {
const parts: string[] = [`Method: ${node.name}`, `File: ${getFileName(node.filePath)}`];
const dir = getDirectory(node.filePath);
if (dir) {
parts.push(`Directory: ${dir}`);
}
if (node.content) {
const cleanedContent = cleanContent(node.content);
const snippet = truncateContent(cleanedContent, maxSnippetLength);
parts.push('', snippet);
}
return parts.join('\n');
};
const DECL_START_RE =
/^(?:(?:export|pub|data|abstract)\s+)*(?:type\s+\w+\s+struct|(?:class|struct|enum|interface)\s)/;
/**
* Generate embedding text for an Interface node
* Extract class/interface/struct declaration lines, skipping method bodies.
* - Brace-based languages: detects method signatures (lines with `(` and `{`)
* and skips until depth returns to class body level.
* - Non-brace languages (Python/Ruby): returns empty string (patterns handle extraction).
*/
const generateInterfaceText = (node: EmbeddableNode, maxSnippetLength: number): string => {
const parts: string[] = [`Interface: ${node.name}`, `File: ${getFileName(node.filePath)}`];
export const extractDeclarationOnly = (content: string): string => {
const lines = content.split('\n');
const declLines: string[] = [];
let depth = 0;
let started = false;
let classDepth = 0;
let skipDepth = 0;
const dir = getDirectory(node.filePath);
if (dir) {
parts.push(`Directory: ${dir}`);
for (const [idx, line] of lines.entries()) {
const trimmed = line.trim();
if (!started) {
if (DECL_START_RE.test(trimmed)) {
// Non-brace language check: current line or next 3 lines must have `{`
const nextLines = lines.slice(idx + 1, idx + 4);
if (!trimmed.includes('{') && !nextLines.some((l) => l.includes('{'))) {
return '';
}
started = true;
declLines.push(trimmed);
for (const ch of trimmed) {
if (ch === '{') depth++;
else if (ch === '}') depth--;
}
if (depth > 0) classDepth = depth;
}
continue;
}
// Always update depth (even when skipping)
const opens = (trimmed.match(/{/g) || []).length;
const closes = (trimmed.match(/}/g) || []).length;
const prevDepth = depth;
depth += opens - closes;
if (skipDepth > 0) {
if (depth <= classDepth) {
skipDepth = 0;
// Closing brace of class
if (depth <= 0) {
declLines.push(trimmed);
break;
}
}
continue;
}
// Detect method signature: line has both `(` and `{` and goes deeper than class body
const hasParens = trimmed.includes('(');
const hasOpenBrace = opens > 0;
if (hasParens && hasOpenBrace && prevDepth + opens > classDepth) {
if (opens === closes && trimmed.endsWith(';')) {
// Property with function/object initializer like `config = { timeout: 5000 };` — keep
declLines.push(trimmed);
}
// else: single-line or multi-line method — skip entirely
if (opens !== closes) {
skipDepth = classDepth;
}
continue;
}
declLines.push(trimmed);
if (depth <= 0 && declLines.length > 1) break;
}
if (node.content) {
const cleanedContent = cleanContent(node.content);
const snippet = truncateContent(cleanedContent, maxSnippetLength);
parts.push('', snippet);
}
return parts.join('\n');
};
/**
* Generate embedding text for a File node
* Uses file name and first N characters of content
*/
const generateFileText = (node: EmbeddableNode, maxSnippetLength: number): string => {
const parts: string[] = [`File: ${node.name}`, `Path: ${node.filePath}`];
if (node.content) {
const cleanedContent = cleanContent(node.content);
// For files, use a shorter snippet since they can be very long
const snippet = truncateContent(cleanedContent, Math.min(maxSnippetLength, 300));
parts.push('', snippet);
}
return parts.join('\n');
return declLines.join('\n').trim();
};
/**
* Generate embedding text for any embeddable node
* Dispatches to the appropriate generator based on node label
*
* @param node - The node to generate text for
* @param config - Optional configuration for max snippet length
* @returns Text suitable for embedding
*/
export const generateEmbeddingText = (
node: EmbeddableNode,
codeBody: string,
config: Partial<EmbeddingConfig> = {},
): string => {
const maxSnippetLength = config.maxSnippetLength ?? DEFAULT_EMBEDDING_CONFIG.maxSnippetLength;
switch (node.label) {
case 'Function':
return generateFunctionText(node, maxSnippetLength);
case 'Class':
return generateClassText(node, maxSnippetLength);
case 'Method':
return generateMethodText(node, maxSnippetLength);
case 'Interface':
return generateInterfaceText(node, maxSnippetLength);
case 'File':
return generateFileText(node, maxSnippetLength);
default:
// Fallback for any other embeddable type
return `${node.label}: ${node.name}\nPath: ${node.filePath}`;
if (isShortLabel(node.label)) {
const header = buildMetadataHeader(node, config);
const cleaned = cleanContent(node.content);
return `${header}\n\n${cleaned}`;
}
if (node.label === 'Class') {
return generateClassText(node, codeBody, config);
}
if (node.label === 'Interface') {
return generateStructuralTypeText(node, codeBody, config);
}
return generateCodeBodyText(node, codeBody, config);
};
/**
* Generate embedding texts for a batch of nodes
*
* @param nodes - Array of nodes to generate text for
* @param config - Optional configuration
* @returns Array of texts in the same order as input nodes
* Export truncation helper for testing
*/
export const generateBatchEmbeddingTexts = (
nodes: EmbeddableNode[],
config: Partial<EmbeddingConfig> = {},
): string[] => {
return nodes.map((node) => generateEmbeddingText(node, config));
};
export { truncateDescription };

View file

@ -5,10 +5,40 @@
*/
/**
* Node labels that should be embedded for semantic search
* These are code elements that benefit from semantic matching
* Node labels that need chunking (have code body, potentially long)
*/
export const EMBEDDABLE_LABELS = ['Function', 'Class', 'Method', 'Interface', 'File'] as const;
export const CHUNKABLE_LABELS = [
'Function',
'Method',
'Constructor',
'Class',
'Interface',
'Struct',
'Enum',
'Trait',
'Impl',
'Macro',
'Namespace',
] as const;
/**
* Node labels that are short (no chunking needed, embed directly)
*/
export const SHORT_LABELS = [
'TypeAlias',
'Typedef',
'Const',
'Property',
'Record',
'Union',
'Static',
'Variable',
] as const;
/**
* All embeddable labels (union of CHUNKABLE + SHORT)
*/
export const EMBEDDABLE_LABELS = [...CHUNKABLE_LABELS, ...SHORT_LABELS] as const;
export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
@ -18,6 +48,39 @@ export type EmbeddableLabel = (typeof EMBEDDABLE_LABELS)[number];
export const isEmbeddableLabel = (label: string): label is EmbeddableLabel =>
EMBEDDABLE_LABELS.includes(label as EmbeddableLabel);
/**
* Check if a label needs chunking
*/
export const isChunkableLabel = (label: string): boolean =>
(CHUNKABLE_LABELS as readonly string[]).includes(label);
/**
* Check if a label is a short type (no chunking)
*/
export const isShortLabel = (label: string): boolean =>
(SHORT_LABELS as readonly string[]).includes(label);
/**
* Node labels that have structural names (methods/fields) extractable via AST
*/
export const STRUCTURAL_LABELS: ReadonlySet<string> = new Set([
'Class',
'Struct',
'Interface',
'Enum',
]);
/**
* Node labels that have isExported column in their schema
*/
export const LABELS_WITH_EXPORTED = new Set([
'Function',
'Class',
'Interface',
'Method',
'CodeElement',
]) as ReadonlySet<string>;
/**
* Embedding pipeline phases
*/
@ -57,6 +120,12 @@ export interface EmbeddingConfig {
device: 'auto' | 'dml' | 'cuda' | 'cpu' | 'wasm';
/** Maximum characters of code snippet to include */
maxSnippetLength: number;
/** Maximum code chunk size in characters (for chunking long code) */
chunkSize: number;
/** Overlap between chunks in characters */
overlap: number;
/** Maximum description length in characters */
maxDescriptionLength: number;
}
/**
@ -70,6 +139,9 @@ export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = {
dimensions: 384,
device: 'auto',
maxSnippetLength: 500,
chunkSize: 1200,
overlap: 120,
maxDescriptionLength: 150,
};
/**
@ -96,6 +168,34 @@ export interface EmbeddableNode {
content: string;
startLine?: number;
endLine?: number;
isExported?: boolean;
description?: string;
parameterCount?: number;
returnType?: string;
repoName?: string;
serverName?: string;
methodNames?: string[];
fieldNames?: string[];
}
/**
* Cached embedding entry restored from LadybugDB before a graph rebuild
*/
export interface CachedEmbedding {
nodeId: string;
chunkIndex: number;
startLine: number;
endLine: number;
embedding: number[];
contentHash?: string;
}
/**
* Context info for embedding pipeline (repo/server metadata enrichment)
*/
export interface EmbeddingContext {
repoName?: string;
serverName?: string;
}
/**
@ -108,3 +208,75 @@ export interface ModelProgress {
loaded?: number;
total?: number;
}
export interface ChunkSearchRow {
nodeId: string;
chunkIndex: number;
startLine: number;
endLine: number;
distance: number;
}
export interface BestChunkMatch {
chunkIndex: number;
startLine: number;
endLine: number;
distance: number;
}
/**
* Deduplicate vector search chunk results by nodeId,
* keeping the chunk with smallest distance for each node.
*/
export const dedupBestChunks = (
rows: ChunkSearchRow[],
limit?: number,
): Map<string, BestChunkMatch> => {
const best = new Map<string, BestChunkMatch>();
for (const row of rows) {
const existing = best.get(row.nodeId);
if (!existing || row.distance < existing.distance) {
best.set(row.nodeId, {
chunkIndex: row.chunkIndex,
startLine: row.startLine,
endLine: row.endLine,
distance: row.distance,
});
}
if (limit !== undefined && best.size >= limit) break;
}
return best;
};
const DEFAULT_FETCH_MULTIPLIER = 4;
const DEFAULT_FETCH_BUFFER = 8;
const DEFAULT_MAX_FETCH = 200;
/**
* Fetch vector-search chunks until we have enough unique nodeIds
* or can tell the result set is exhausted.
*/
export const collectBestChunks = async (
limit: number,
fetchRows: (fetchLimit: number) => Promise<ChunkSearchRow[]>,
maxFetch: number = DEFAULT_MAX_FETCH,
): Promise<Map<string, BestChunkMatch>> => {
if (limit <= 0) return new Map();
let fetchLimit = Math.max(limit * DEFAULT_FETCH_MULTIPLIER, limit + DEFAULT_FETCH_BUFFER);
let previousFetchLimit = 0;
while (fetchLimit > previousFetchLimit) {
const rows = await fetchRows(fetchLimit);
const bestChunks = dedupBestChunks(rows, limit);
if (bestChunks.size >= limit || rows.length < fetchLimit) {
return bestChunks;
}
previousFetchLimit = fetchLimit;
fetchLimit = fetchLimit >= maxFetch ? fetchLimit * 2 : Math.min(maxFetch, fetchLimit * 2);
}
return new Map();
};

View file

@ -0,0 +1,139 @@
# Group Analysis Pipeline
Flow chart of the cross-repo contract extraction + matching pipeline.
This covers what runs **inside this PR** (extractors + manifest) and
the downstream handoff to the bridge storage (PR #795) and
cross-impact query (PR #606).
## High-level overview
```mermaid
flowchart TD
A[group.yaml] --> B[GroupConfig parser]
B --> C{For each repo<br/>in group}
C --> D[Per-repo LadybugDB<br/>indexed by main pipeline]
D --> E1[TopicExtractor]
D --> E2[HttpRouteExtractor]
D --> E3[GrpcExtractor]
E1 --> F[ExtractedContract array<br/>per repo]
E2 --> F
E3 --> F
B --> M[ManifestExtractor]
M --> G[Manifest contracts<br/>+ cross-links]
F --> H[Contract matching<br/>exact + wildcard]
G --> H
H --> I[(bridge.lbug<br/>#795)]
I --> J[runGroupImpact<br/>#606]
J --> K[CrossRepoImpact]
```
## Per-repo extractor pipeline
Each extractor under `src/core/group/extractors/` follows the same
two-strategy shape:
```mermaid
flowchart TD
R[RepoHandle + CypherExecutor<br/>for this repo] --> S{Graph-assisted<br/>Strategy A<br/>available?}
S -->|yes| A1[Cypher query against<br/>per-repo LadybugDB]
A1 --> A2{non-empty<br/>result?}
A2 -->|yes| OUT[ExtractedContract array]
A2 -->|no| B1
S -->|no| B1[Source-scan Strategy B]
B1 --> B2[glob repo source files]
B2 --> B3{ext in registry?}
B3 -->|yes| B4[Per-language plugin<br/>scan parsed tree]
B3 -->|no| SKIP[skip file]
B4 --> OUT
SKIP --> B2
```
**Strategy A** (graph-assisted) uses Cypher over edges already produced
by the main ingestion pipeline:
- HTTP: `HANDLES_ROUTE` / `FETCHES` edges from `(File)-[]->(Route)`
- topic: none (pipeline doesn't yet produce topic nodes — Strategy B only)
- gRPC: none (Strategy B + proto map only)
**Strategy B** (source-scan) is 100% tree-sitter based after this PR.
Each `*-patterns/<lang>.ts` plugin owns its grammar + S-expression
queries; the top-level orchestrator imports neither.
## Plugin architecture
```mermaid
flowchart LR
O[Orchestrator<br/>topic|http|grpc-extractor.ts] --> REG[REGISTRY<br/>*-patterns/index.ts]
REG --> P1[java.ts<br/>tree-sitter-java]
REG --> P2[go.ts<br/>tree-sitter-go]
REG --> P3[python.ts<br/>tree-sitter-python]
REG --> P4[node.ts<br/>JS + TS + TSX]
REG --> P5[php.ts<br/>tree-sitter-php<br/>HTTP only]
REG --> P6[proto.ts<br/>tree-sitter-proto<br/>gRPC only, optional]
P1 --> SCAN[tree-sitter-scanner.ts<br/>compilePatterns + runCompiledPatterns]
P2 --> SCAN
P3 --> SCAN
P4 --> SCAN
P5 --> SCAN
P6 --> SCAN
SCAN --> DET[Detection objects<br/>TopicMeta / HttpDetection / GrpcDetection]
DET --> O
O --> CT[ExtractedContract array]
```
The orchestrator never imports a grammar. Adding a new language /
framework = drop one file in `*-patterns/`, register it in
`index.ts`. No orchestrator edits required.
## Manifest extraction
```mermaid
flowchart TD
Y[group.yaml links] --> ME[ManifestExtractor]
ME --> LOOP{for each link}
LOOP --> RES[resolveSymbol<br/>label-scoped Cypher]
RES --> OK{found?}
OK -->|yes| REF[real symbol uid + ref]
OK -->|no| SYN[synthetic uid<br/>manifest::repo::cid]
REF --> EMIT[emit provider + consumer<br/>Contract objects<br/>+ CrossLink]
SYN --> EMIT
EMIT --> BRIDGE[(bridge.lbug<br/>#795)]
```
Label-scoped queries in `resolveSymbol` keep accidental cross-matches
out:
- `topic``(n:Function|Method|Class|Interface)`
- `grpc` method → `(n:Function|Method)`, service → `(n:Class|Interface)`
- `lib``(n:Package|Module)`
## Cross-impact query (PR #606)
```mermaid
flowchart TD
U[User changes symbol S<br/>in repo R] --> LI[Local impact engine<br/>per-repo uid expansion]
LI --> IDS[Affected uid set]
IDS --> BR[Bridge query<br/>MATCH Contract WHERE uid IN ids]
BR --> CL[CrossLink traversal]
CL --> OTHER[Matching contract in<br/>other repo]
OTHER --> FE[Fan-out impact<br/>to consuming repo]
FE --> OUT[CrossRepoImpact<br/>per affected repo]
```
The bridge stores every extracted contract keyed by `symbolUid`.
Manifest-sourced contracts use the synthetic uid form so both sides
of the `(local impact) ↔ (bridge query)` join derive the same uid
without coordinating through any shared state.

View file

@ -0,0 +1,588 @@
import fsp from 'node:fs/promises';
import path from 'node:path';
import { createHash } from 'node:crypto';
import lbug from '@ladybugdb/core';
import type { LbugValue } from '@ladybugdb/core';
import type { BridgeHandle, BridgeMeta, StoredContract, CrossLink, RepoSnapshot } from './types.js';
import { BRIDGE_SCHEMA_QUERIES, BRIDGE_SCHEMA_VERSION } from './bridge-schema.js';
import { dedupeContracts, dedupeCrossLinks } from './normalization.js';
export function contractNodeId(
repo: string,
contractId: string,
role: string,
filePath: string,
): string {
return createHash('sha256').update(`${repo}\0${contractId}\0${role}\0${filePath}`).digest('hex');
}
/* ------------------------------------------------------------------ */
/* ContractLookupIndex — in-memory lookup for findContractNode */
/* ------------------------------------------------------------------ */
/**
* In-memory index of contract node IDs keyed three ways, mirroring the
* three-tier fallback lookup in {@link findContractNode}. Built once per
* `writeBridge` call after all contracts are successfully inserted, then
* consulted for every cross-link which eliminates the former N+1 query
* pattern (up to `6 × cross-links` DB round-trips) and turns cross-link
* resolution into constant-time per link.
*
* Keys are deliberately flat strings (not tuples) so `Map<string, ...>`
* works; the separator `\0` can't occur in any legal repo path / file
* path / symbol identifier, which makes the encoding injection-safe.
*/
export interface ContractLookupIndex {
/** tier 1: `repo + role + symbolUid` → contract node id */
byUid: Map<string, string>;
/** tier 2: `repo + role + filePath + symbolName` → contract node id */
byRef: Map<string, string>;
/** tier 3: `repo + role + filePath` → list of contract node ids in that file */
byFile: Map<string, string[]>;
}
export function createContractLookupIndex(): ContractLookupIndex {
return {
byUid: new Map(),
byRef: new Map(),
byFile: new Map(),
};
}
function uidKey(repo: string, role: string, symbolUid: string): string {
return `${repo}\0${role}\0${symbolUid}`;
}
function refKey(repo: string, role: string, filePath: string, symbolName: string): string {
return `${repo}\0${role}\0${filePath}\0${symbolName}`;
}
function fileKey(repo: string, role: string, filePath: string): string {
return `${repo}\0${role}\0${filePath}`;
}
/**
* Add a successfully-inserted contract to the lookup index. Must be called
* AFTER the DB insert succeeds (not before) so failed inserts don't poison
* the index and cause cross-links to point at non-existent rows.
*/
export function indexContract(
index: ContractLookupIndex,
contract: StoredContract,
nodeId: string,
): void {
if (contract.symbolUid) {
index.byUid.set(uidKey(contract.repo, contract.role, contract.symbolUid), nodeId);
}
index.byRef.set(
refKey(contract.repo, contract.role, contract.symbolRef.filePath, contract.symbolRef.name),
nodeId,
);
const fk = fileKey(contract.repo, contract.role, contract.symbolRef.filePath);
const existing = index.byFile.get(fk);
if (existing) {
existing.push(nodeId);
} else {
index.byFile.set(fk, [nodeId]);
}
}
/**
* Resolve a cross-link endpoint (consumer or provider reference) to an
* already-inserted contract node id. Returns `null` if no match the
* caller is expected to count that as a dropped link in `WriteBridgeReport`.
*
* The resolution order matches the pre-cache DB-query behavior:
* 1. exact `symbolUid` match in the same `(repo, role)` scope
* 2. exact `(filePath, symbolName)` match
* 3. if exactly one contract lives in the file that one (fallback for
* legacy graph-assisted extractors that couldn't resolve a symbol name)
*
* This is a pure function no I/O, no DB so it's trivial to unit-test
* in isolation (which was the reviewer's main clean-code concern on the
* original 35-line inner closure in `writeBridge`).
*/
export function findContractNode(
index: ContractLookupIndex,
repo: string,
role: 'consumer' | 'provider',
symbolUid: string,
filePath: string,
symbolName: string,
): string | null {
if (symbolUid) {
const uidHit = index.byUid.get(uidKey(repo, role, symbolUid));
if (uidHit !== undefined) return uidHit;
}
const refHit = index.byRef.get(refKey(repo, role, filePath, symbolName));
if (refHit !== undefined) return refHit;
const fileCandidates = index.byFile.get(fileKey(repo, role, filePath));
if (fileCandidates && fileCandidates.length === 1) return fileCandidates[0];
return null;
}
export async function openBridgeDb(dbPath: string): Promise<BridgeHandle> {
const parentDir = path.dirname(dbPath);
await fsp.mkdir(parentDir, { recursive: true });
const db = new lbug.Database(dbPath, 0, false, false); // writable
const conn = new lbug.Connection(db);
return { _db: db, _conn: conn, groupDir: parentDir } as BridgeHandle;
}
/**
* LadybugDB returns an error whose message contains this substring when a
* CREATE NODE TABLE or CREATE REL TABLE statement hits an already-existing
* table. LadybugDB DDL doesn't support IF NOT EXISTS, and its JS driver
* doesn't expose typed error codes, so we match on the message substring
* the same pattern used by `core/lbug/lbug-adapter.ts`. If a future
* LadybugDB release changes the wording, update this constant.
*/
const LBUG_ALREADY_EXISTS_MSG = 'already exists';
export async function ensureBridgeSchema(handle: BridgeHandle): Promise<void> {
const conn = handle._conn as lbug.Connection;
for (const q of BRIDGE_SCHEMA_QUERIES) {
try {
await conn.query(q);
} catch (err: unknown) {
const msg = err instanceof Error ? err.message : String(err);
if (!msg.includes(LBUG_ALREADY_EXISTS_MSG)) throw err;
}
}
}
export async function queryBridge<T>(
handle: BridgeHandle,
cypher: string,
params?: Record<string, LbugValue>,
): Promise<T[]> {
const conn = handle._conn as lbug.Connection;
if (params && Object.keys(params).length > 0) {
const stmt = await conn.prepare(cypher);
if (!stmt.isSuccess()) {
const errMsg = await stmt.getErrorMessage();
throw new Error(`Bridge query prepare failed: ${errMsg}`);
}
const queryResult = await conn.execute(stmt, params);
const result = unwrapQueryResult(queryResult);
return (await result.getAll()) as T[];
}
const queryResult = await conn.query(cypher);
const result = unwrapQueryResult(queryResult);
return (await result.getAll()) as T[];
}
/**
* LadybugDB's `conn.query` / `conn.execute` can return either a single
* `QueryResult` (for a single statement) or an array of them (when a
* multi-statement script is dispatched). We always pass a single statement,
* so the array form is a wrapper we unwrap here but an empty top-level
* array would cause `.getAll()` on `undefined` and crash with a confusing
* stack. Throwing an explicit error makes a driver-contract regression
* visible immediately instead of masking it.
*/
function unwrapQueryResult(queryResult: lbug.QueryResult | lbug.QueryResult[]): lbug.QueryResult {
if (Array.isArray(queryResult)) {
if (queryResult.length === 0) {
throw new Error('Bridge query returned an empty QueryResult array');
}
return queryResult[0];
}
return queryResult;
}
export async function closeBridgeDb(handle: BridgeHandle): Promise<void> {
try {
await (handle._conn as lbug.Connection).close();
} catch {
/* ignore */
}
try {
await (handle._db as lbug.Database).close();
} catch {
/* ignore */
}
}
/* ------------------------------------------------------------------ */
/* retryRename — handles transient EBUSY/EPERM/EACCES on Windows */
/* ------------------------------------------------------------------ */
const RETRY_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
export async function retryRename(src: string, dst: string, attempts = 3): Promise<void> {
for (let i = 1; i <= attempts; i++) {
try {
await fsp.rename(src, dst);
return;
} catch (err: unknown) {
const code = (err as NodeJS.ErrnoException).code;
if (!code || !RETRY_CODES.has(code) || i === attempts) throw err;
await new Promise((r) => setTimeout(r, 100 * Math.pow(2, i - 1)));
}
}
}
/* ------------------------------------------------------------------ */
/* writeBridgeMeta / readBridgeMeta */
/* ------------------------------------------------------------------ */
export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise<void> {
const target = path.join(groupDir, 'meta.json');
const tmp = `${target}.tmp.${Date.now()}`;
await fsp.writeFile(tmp, JSON.stringify(meta, null, 2), 'utf-8');
// Use retryRename for consistency with writeBridge's atomic swap — on
// Windows a concurrent reader can cause EBUSY/EPERM even on a tiny
// meta.json, and we don't want meta write to be less robust than the
// bridge.lbug swap it accompanies.
await retryRename(tmp, target);
}
export async function readBridgeMeta(groupDir: string): Promise<BridgeMeta> {
try {
const content = await fsp.readFile(path.join(groupDir, 'meta.json'), 'utf-8');
return JSON.parse(content) as BridgeMeta;
} catch {
return { version: 0, generatedAt: '', missingRepos: [] };
}
}
/* ------------------------------------------------------------------ */
/* writeBridge — atomic write-to-temp-then-rename */
/* ------------------------------------------------------------------ */
export interface WriteBridgeInput {
contracts: StoredContract[];
crossLinks: CrossLink[];
repoSnapshots: Record<string, RepoSnapshot>;
missingRepos: string[];
}
/**
* Non-fatal issues encountered during writeBridge. Callers can log these to
* surface partial-success state without aborting the whole sync.
* `sampleErrors` is capped at MAX_SAMPLE_ERRORS per category to bound memory.
*/
export interface WriteBridgeReport {
contractsInserted: number;
contractsFailed: number;
snapshotsInserted: number;
snapshotsFailed: number;
linksInserted: number;
linksFailed: number;
/** Cross-links skipped because their from/to contract nodes weren't found. */
linksDroppedMissingNode: number;
sampleErrors: Array<{
kind: 'contract' | 'snapshot' | 'link';
id: string;
message: string;
}>;
}
const MAX_SAMPLE_ERRORS = 10;
function errMessage(err: unknown): string {
if (err instanceof Error) return err.message;
try {
return String(err);
} catch {
return 'unknown error';
}
}
export async function writeBridge(
groupDir: string,
input: WriteBridgeInput,
): Promise<WriteBridgeReport> {
await fsp.mkdir(groupDir, { recursive: true });
const contracts = dedupeContracts(input.contracts);
const crossLinks = dedupeCrossLinks(input.crossLinks);
const finalPath = path.join(groupDir, 'bridge.lbug');
const tmpPath = path.join(groupDir, 'bridge.lbug.tmp');
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
const report: WriteBridgeReport = {
contractsInserted: 0,
contractsFailed: 0,
snapshotsInserted: 0,
snapshotsFailed: 0,
linksInserted: 0,
linksFailed: 0,
linksDroppedMissingNode: 0,
sampleErrors: [],
};
const recordError = (kind: 'contract' | 'snapshot' | 'link', id: string, err: unknown) => {
if (report.sampleErrors.length < MAX_SAMPLE_ERRORS) {
report.sampleErrors.push({ kind, id, message: errMessage(err) });
}
};
// Clean up any leftover tmp
try {
await fsp.rm(tmpPath, { recursive: true, force: true });
} catch {
/* ignore */
}
// 1. Create temp DB, insert all data.
//
// Everything after `openBridgeDb` must run inside a try/finally so that
// if ANY step before the explicit `closeBridgeDb` throws — schema
// creation, a contract insert loop that rethrows, a snapshot write, the
// cross-link loop, or anything else — the handle is still released. A
// leaked handle holds the native LadybugDB file lock on tmpPath, which
// (a) leaks a FD and (b) prevents the next writeBridge call from
// reusing the same tmp slot.
const handle = await openBridgeDb(tmpPath);
let handleClosed = false;
try {
await ensureBridgeSchema(handle);
// Build the lookup index incrementally as contracts are inserted, so
// failed inserts are never in the index (and therefore never resolved
// by the cross-link loop below). This replaces a previous N+1 query
// pattern where each link made up to 6 DB round-trips to find its
// endpoints — see ContractLookupIndex.
const lookupIndex = createContractLookupIndex();
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
// that can't be serialized). The whole sync must not fail because one
// contract is broken.
for (const c of contracts) {
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
try {
await queryBridge(
handle,
`CREATE (n:Contract {
id: $id,
contractId: $contractId,
type: $type,
role: $role,
repo: $repo,
service: $service,
symbolUid: $symbolUid,
filePath: $filePath,
symbolName: $symbolName,
confidence: $confidence,
meta: $meta
})`,
{
id,
contractId: c.contractId,
type: c.type,
role: c.role,
repo: c.repo,
service: c.service ?? '',
symbolUid: c.symbolUid,
filePath: c.symbolRef.filePath,
symbolName: c.symbolName,
confidence: c.confidence,
meta: JSON.stringify(c.meta),
},
);
report.contractsInserted++;
// Only index on successful insert — the cross-link loop must never
// resolve to a row that isn't actually in the DB.
indexContract(lookupIndex, c, id);
} catch (err) {
report.contractsFailed++;
recordError('contract', id, err);
}
}
// Insert repo snapshots
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
try {
await queryBridge(
handle,
`CREATE (s:RepoSnapshot {
id: $id,
indexedAt: $indexedAt,
lastCommit: $lastCommit
})`,
{
id: repoId,
indexedAt: snap.indexedAt,
lastCommit: snap.lastCommit,
},
);
report.snapshotsInserted++;
} catch (err) {
report.snapshotsFailed++;
recordError('snapshot', repoId, err);
}
}
// Insert cross-links (tolerating missing nodes).
//
// `findContractNode` consults the in-memory lookup index built above,
// not the DB — that's an O(1) pure-function lookup per endpoint instead
// of the previous 2-3 DB queries. For M cross-links, the previous code
// issued up to 6M round-trips; this version issues zero.
//
// `link.contractId` may differ between the consumer and provider sides
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
// `grpc::Service/Method`) — that's why we resolve each endpoint
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
// tuple rather than matching on contractId.
for (const link of crossLinks) {
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
try {
const fromId = findContractNode(
lookupIndex,
link.from.repo,
'consumer',
link.from.symbolUid,
link.from.symbolRef.filePath,
link.from.symbolRef.name,
);
const toId = findContractNode(
lookupIndex,
link.to.repo,
'provider',
link.to.symbolUid,
link.to.symbolRef.filePath,
link.to.symbolRef.name,
);
if (!fromId || !toId) {
report.linksDroppedMissingNode++;
continue;
}
await queryBridge(
handle,
`
MATCH (a:Contract), (b:Contract)
WHERE a.id = $fromId AND b.id = $toId
CREATE (a)-[:ContractLink {
matchType: $matchType,
confidence: $confidence,
contractId: $contractId,
fromRepo: $fromRepo,
toRepo: $toRepo
}]->(b)
`,
{
fromId,
toId,
matchType: link.matchType,
confidence: link.confidence,
contractId: link.contractId,
fromRepo: link.from.repo,
toRepo: link.to.repo,
},
);
report.linksInserted++;
} catch (err) {
report.linksFailed++;
recordError('link', linkId, err);
}
}
// 2. Close temp DB (happy path). The finally block also calls
// closeBridgeDb if we threw above; `handleClosed` prevents a
// double-close on the native handle.
await closeBridgeDb(handle);
handleClosed = true;
} finally {
if (!handleClosed) {
await closeBridgeDb(handle).catch(() => {
/* ignore: cleanup path, best effort */
});
}
}
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
try {
await fsp.access(finalPath);
await retryRename(finalPath, bakPath);
} catch {
/* no existing db */
}
await retryRename(tmpPath, finalPath);
try {
await fsp.rm(bakPath, { recursive: true, force: true });
} catch {
/* ignore */
}
// 4. Write meta.json
await writeBridgeMeta(groupDir, {
version: BRIDGE_SCHEMA_VERSION,
generatedAt: new Date().toISOString(),
missingRepos: input.missingRepos,
});
return report;
}
/* ------------------------------------------------------------------ */
/* openBridgeDbReadOnly */
/* ------------------------------------------------------------------ */
export async function openBridgeDbReadOnly(groupDir: string): Promise<BridgeHandle | null> {
const dbPath = path.join(groupDir, 'bridge.lbug');
try {
await fsp.access(dbPath);
} catch {
// Check for .bak recovery. Use `retryRename` (not `fsp.rename`) for the
// exact same reason the rest of this file does: the scenario that
// triggers bak recovery is an interrupted writer, which on Windows may
// still be holding an open handle on `.bak` for a few milliseconds when
// a reader races in. EBUSY/EPERM retries recover that case silently.
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
try {
await fsp.access(bakPath);
await retryRename(bakPath, dbPath);
} catch {
return null;
}
}
// Version gate: check meta.json version compatibility
const meta = await readBridgeMeta(groupDir);
if (meta.version > 0 && meta.version !== BRIDGE_SCHEMA_VERSION) {
return null; // incompatible schema version — fallback to JSON or re-sync
}
// Open the native handle. If Connection construction throws AFTER
// Database was successfully allocated, we'd leak the native Database
// object. Wrap each step separately and tear down the partial handle.
let db: lbug.Database | undefined;
let conn: lbug.Connection | undefined;
try {
db = new lbug.Database(dbPath, 0, false, true); // readOnly
conn = new lbug.Connection(db);
return { _db: db, _conn: conn, groupDir } as BridgeHandle;
} catch {
if (conn) {
try {
await conn.close();
} catch {
/* ignore */
}
}
if (db) {
try {
await db.close();
} catch {
/* ignore */
}
}
return null;
}
}
/* ------------------------------------------------------------------ */
/* bridgeExists */
/* ------------------------------------------------------------------ */
export async function bridgeExists(groupDir: string): Promise<boolean> {
const handle = await openBridgeDbReadOnly(groupDir);
if (!handle) return false;
await closeBridgeDb(handle);
return true;
}

View file

@ -0,0 +1,60 @@
/**
* Bridge LadybugDB schema for cross-repo Contract Registry.
* Separate from per-repo schema in lbug/schema.ts.
*/
/**
* Version of the bridge.lbug schema below. `openBridgeDbReadOnly` compares
* this against `meta.json`'s version field and returns `null` on mismatch,
* which trips the caller into either the JSON fallback path or a fresh
* `group sync` that rebuilds `bridge.lbug` from scratch.
*
* Migration contract for contributors bumping this constant:
* 1. Bump the number (e.g. `1` `2`).
* 2. Update the DDL below to match the new schema.
* 3. DO NOT attempt an online migration in this file the version gate
* is intentionally a "discard and re-sync" strategy for V1. An old
* bridge.lbug whose version doesn't match is treated as opaque and
* rebuilt by the next `group sync`.
* 4. If online migration becomes necessary (e.g. when groups accumulate
* large amounts of embedding data), add a migration path as a
* separate `bridge-migrations.ts` module rather than bloating this
* file keep schema and migration concerns separate.
*/
export const BRIDGE_SCHEMA_VERSION = 1;
export const CONTRACT_SCHEMA = `
CREATE NODE TABLE Contract (
id STRING,
contractId STRING,
type STRING,
role STRING,
repo STRING,
service STRING DEFAULT '',
symbolUid STRING DEFAULT '',
filePath STRING DEFAULT '',
symbolName STRING DEFAULT '',
confidence DOUBLE DEFAULT 0.0,
meta STRING DEFAULT '{}',
PRIMARY KEY (id)
)`;
export const REPO_SNAPSHOT_SCHEMA = `
CREATE NODE TABLE RepoSnapshot (
id STRING,
indexedAt STRING DEFAULT '',
lastCommit STRING DEFAULT '',
PRIMARY KEY (id)
)`;
export const CONTRACT_LINK_SCHEMA = `
CREATE REL TABLE ContractLink (
FROM Contract TO Contract,
matchType STRING,
confidence DOUBLE,
contractId STRING,
fromRepo STRING,
toRepo STRING
)`;
export const BRIDGE_SCHEMA_QUERIES = [CONTRACT_SCHEMA, REPO_SNAPSHOT_SCHEMA, CONTRACT_LINK_SCHEMA];

View file

@ -0,0 +1,23 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
/**
* Safely read a file inside a repo, rejecting any path that escapes
* `repoPath` via `..` traversal or absolute segments. Returns `null` if
* the path is outside the repo or the file can't be read.
*
* Used by every source-scan extractor under this directory. Kept as a
* single shared implementation so the path-traversal guard (security-
* sensitive) lives in exactly one place.
*/
export function readSafe(repoPath: string, rel: string): string | null {
const abs = path.resolve(repoPath, rel);
const base = path.resolve(repoPath);
const relToBase = path.relative(base, abs);
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
try {
return fs.readFileSync(abs, 'utf-8');
} catch {
return null;
}
}

View file

@ -1,20 +1,38 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { glob } from 'glob';
import Parser from 'tree-sitter';
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
import type { ExtractedContract, RepoHandle } from '../types.js';
import { readSafe } from './fs-utils.js';
import {
GRPC_SCAN_GLOB,
getPluginForFile,
hasProtoPlugin,
type GrpcDetection,
} from './grpc-patterns/index.js';
function readSafe(repoPath: string, rel: string): string | null {
const abs = path.resolve(repoPath, rel);
const base = path.resolve(repoPath);
const relToBase = path.relative(base, abs);
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
try {
return fs.readFileSync(abs, 'utf-8');
} catch {
return null;
}
}
/**
* Language-agnostic orchestrator for gRPC (provider + consumer) contract
* extraction.
*
* Two parts:
*
* 1. **`.proto` parsing** tree-sitter when `tree-sitter-proto` is
* installed (optionalDependency vendored in `vendor/tree-sitter-proto/`),
* via the `.proto` entry in `grpc-patterns/` and `hasProtoPlugin`.
* When the grammar isn't available (platform incompatibility, native
* build failure) the orchestrator falls back to the in-process
* string-sanitizing parser defined below (`stripProtoCommentsAndStrings`
* + `extractServiceBlocks`). The fallback preserves offsets so any
* downstream regex scans run against a sanitized copy without
* affecting line numbers of the original.
*
* 2. **Source-scan providers / consumers** delegated to per-language
* plugins in `./grpc-patterns/`. The orchestrator imports NO
* tree-sitter grammars or query strings each plugin owns its own.
*/
// ─── .proto fallback parser (used only when tree-sitter-proto is absent) ───
function contractId(pkg: string, service: string, method: string): string {
const prefix = pkg ? `${pkg}.${service}` : service;
@ -25,20 +43,110 @@ function serviceOnlyContractId(serviceName: string): string {
return `grpc::${serviceName}/*`;
}
/**
* Replace all .proto comments and string literals with spaces, preserving the
* original length and character offsets of the input. This lets downstream
* regex / brace-depth parsers run on a "sanitized" copy without having to
* understand proto syntax, while any RegExp.exec/index-based lookups that
* were already positional against `content` continue to work against the
* original string.
*
* Supported comment forms: `// line comment`, `/* block comment * /`.
* Supported strings: double-quoted ("…") and single-quoted ('…') with `\`
* escape handling. Raw/unterminated strings are not supported we stop
* on a line break for line-style comments and on EOF for unterminated
* strings/blocks, which matches how most real proto files parse.
*/
function stripProtoCommentsAndStrings(content: string): string {
const out = new Array<string>(content.length);
let i = 0;
while (i < content.length) {
const ch = content[i];
const next = content[i + 1];
// Line comment: // ... \n
if (ch === '/' && next === '/') {
out[i] = ' ';
out[i + 1] = ' ';
i += 2;
while (i < content.length && content[i] !== '\n') {
out[i] = content[i] === '\r' ? '\r' : ' ';
i++;
}
continue;
}
// Block comment: /* ... */
if (ch === '/' && next === '*') {
out[i] = ' ';
out[i + 1] = ' ';
i += 2;
while (i < content.length) {
if (content[i] === '*' && content[i + 1] === '/') {
out[i] = ' ';
out[i + 1] = ' ';
i += 2;
break;
}
// Preserve newlines so line numbers stay stable for downstream code.
out[i] = content[i] === '\n' || content[i] === '\r' ? content[i] : ' ';
i++;
}
continue;
}
// String literal: "..." or '...'
if (ch === '"' || ch === "'") {
const quote = ch;
out[i] = ' '; // replace opening quote
i++;
while (i < content.length) {
const c = content[i];
if (c === '\\' && i + 1 < content.length) {
// Skip escaped pair (e.g. \" \n \\)
out[i] = ' ';
out[i + 1] = ' ';
i += 2;
continue;
}
if (c === quote) {
out[i] = ' ';
i++;
break;
}
// Preserve newlines; proto technically disallows unescaped newlines
// inside strings, but real files occasionally have them.
out[i] = c === '\n' || c === '\r' ? c : ' ';
i++;
}
continue;
}
out[i] = ch;
i++;
}
return out.join('');
}
function extractServiceBlocks(content: string): Array<{ name: string; body: string }> {
const results: Array<{ name: string; body: string }> = [];
// v1: brace-depth only — braces inside comments or string literals are not filtered (see spec Fix 2)
// Sanitize comments and string literals so braces inside them don't
// throw off the depth counter. The sanitized copy has the same length
// and offsets as the original, so we use it ONLY to scan for service
// headers and braces; the service body we return is sliced from the
// ORIGINAL content to preserve exact source text for downstream use.
const sanitized = stripProtoCommentsAndStrings(content);
const headerRe = /service\s+(\w+)\s*\{/g;
let headerMatch: RegExpExecArray | null;
while ((headerMatch = headerRe.exec(content)) !== null) {
while ((headerMatch = headerRe.exec(sanitized)) !== null) {
const serviceName = headerMatch[1];
const bodyStart = headerMatch.index + headerMatch[0].length;
let depth = 1;
let pos = bodyStart;
while (pos < content.length && depth > 0) {
const ch = content[pos];
while (pos < sanitized.length && depth > 0) {
const ch = sanitized[pos];
if (ch === '{') depth++;
else if (ch === '}') depth--;
pos++;
@ -75,6 +183,177 @@ function makeContract(
};
}
export interface ProtoServiceInfo {
package: string;
serviceName: string;
methods: string[];
protoPath: string;
}
function normalizeProtoPath(rel: string): string {
return rel.replace(/\\/g, '/');
}
function extractProtoImports(content: string): string[] {
const imports: string[] = [];
const re = /^\s*import\s+"([^"]+)"\s*;/gm;
let match: RegExpExecArray | null;
while ((match = re.exec(content)) !== null) {
imports.push(match[1]);
}
return imports;
}
function longestSharedSegmentRun(aPath: string, bPath: string): number {
const a = aPath.split('/').filter(Boolean);
const b = bPath.split('/').filter(Boolean);
let best = 0;
for (let i = 0; i < a.length; i++) {
for (let j = 0; j < b.length; j++) {
let run = 0;
while (a[i + run] && b[j + run] && a[i + run] === b[j + run]) {
run++;
}
if (run > best) best = run;
}
}
return best;
}
async function buildProtoContext(repoPath: string): Promise<{
packagesByProto: Map<string, string>;
servicesByName: Map<string, ProtoServiceInfo[]>;
}> {
const servicesByName = new Map<string, ProtoServiceInfo[]>();
const protoFiles = await glob('**/*.proto', {
cwd: repoPath,
absolute: false,
nodir: true,
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'],
});
const contents = new Map<string, string>();
for (const rel of protoFiles) {
const content = readSafe(repoPath, rel);
if (!content) continue;
contents.set(normalizeProtoPath(rel), content);
}
const packagesByProto = new Map<string, string>();
const resolvePackage = (protoPath: string, seen = new Set<string>()): string => {
if (packagesByProto.has(protoPath)) return packagesByProto.get(protoPath) ?? '';
if (seen.has(protoPath)) return '';
const content = contents.get(protoPath);
if (!content) return '';
seen.add(protoPath);
const pkgMatch = content.match(/^\s*package\s+([\w.]+)\s*;/m);
if (pkgMatch?.[1]) {
packagesByProto.set(protoPath, pkgMatch[1]);
return pkgMatch[1];
}
for (const importPath of extractProtoImports(content)) {
const normalizedImport = normalizeProtoPath(importPath);
const candidates = [
normalizeProtoPath(
path.posix.normalize(path.posix.join(path.posix.dirname(protoPath), normalizedImport)),
),
normalizedImport,
];
for (const candidate of candidates) {
if (!contents.has(candidate)) continue;
const inheritedPackage = resolvePackage(candidate, seen);
if (inheritedPackage) {
packagesByProto.set(protoPath, inheritedPackage);
return inheritedPackage;
}
}
}
packagesByProto.set(protoPath, '');
return '';
};
for (const rel of protoFiles) {
const normalizedRel = normalizeProtoPath(rel);
const content = contents.get(normalizedRel);
if (!content) continue;
const pkg = resolvePackage(normalizedRel);
const serviceBlocks = extractServiceBlocks(content);
for (const block of serviceBlocks) {
const rpcRe = /rpc\s+(\w+)\s*\(/g;
const methods: string[] = [];
let m: RegExpExecArray | null;
while ((m = rpcRe.exec(block.body)) !== null) {
methods.push(m[1]);
}
const info: ProtoServiceInfo = {
package: pkg,
serviceName: block.name,
methods,
protoPath: normalizedRel,
};
const existing = servicesByName.get(block.name) ?? [];
existing.push(info);
servicesByName.set(block.name, existing);
}
}
return { packagesByProto, servicesByName };
}
export async function buildProtoMap(repoPath: string): Promise<Map<string, ProtoServiceInfo[]>> {
const { servicesByName } = await buildProtoContext(repoPath);
return servicesByName;
}
export function resolveProtoConflict(
serviceName: string,
sourceFilePath: string,
candidates: ProtoServiceInfo[],
): ProtoServiceInfo | null {
if (candidates.length === 0) return null;
if (candidates.length === 1) return candidates[0];
const sourceDir = normalizeProtoPath(path.dirname(sourceFilePath));
const scored = candidates.map((c) => {
const protoDir = normalizeProtoPath(path.dirname(c.protoPath));
return { candidate: c, score: longestSharedSegmentRun(sourceDir, protoDir) };
});
let maxScore = -1;
for (const s of scored) {
if (s.score > maxScore) maxScore = s.score;
}
const winners = scored.filter((s) => s.score === maxScore);
// Path heuristic cannot uniquely identify a winner — refuse to guess.
// Ties (including all-zero ties) would otherwise silently merge unrelated
// services under a fabricated package-qualified contract id.
if (winners.length !== 1) {
const paths = candidates.map((c) => c.protoPath).join(', ');
console.warn(
`[grpc-extractor] Ambiguous proto resolution for service "${serviceName}" from ${sourceFilePath}: ${winners.length} candidates tied at score ${maxScore} among [${paths}] — skipping canonical contract`,
);
return null;
}
return winners[0].candidate;
}
export function serviceContractId(pkg: string, serviceName: string): string {
const prefix = pkg ? `${pkg}.${serviceName}` : serviceName;
return `grpc::${prefix}/*`;
}
// ─── Orchestrator ────────────────────────────────────────────────────
export class GrpcExtractor implements ContractExtractor {
type = 'grpc' as const;
@ -88,270 +367,116 @@ export class GrpcExtractor implements ContractExtractor {
_repo: RepoHandle,
): Promise<ExtractedContract[]> {
const out: ExtractedContract[] = [];
const protoContext = await buildProtoContext(repoPath);
const protoMap = protoContext.servicesByName;
// Proto files — definitive provider source
const protoFiles = await glob('**/*.proto', {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**'],
nodir: true,
});
for (const rel of protoFiles) {
const content = readSafe(repoPath, rel);
if (content) out.push(...this.parseProtoFile(content, rel));
// ─── Proto files — definitive provider source ─────────────────
// When tree-sitter-proto is available, .proto files are handled by
// the plugin loop below (they're in GRPC_SCAN_GLOB). Otherwise
// emit provider contracts directly from the proto map that
// `buildProtoContext` already built — no second glob / parse pass.
if (!hasProtoPlugin) {
for (const infos of protoMap.values()) {
for (const info of infos) {
for (const methodName of info.methods) {
const cid = contractId(info.package, info.serviceName, methodName);
out.push(
makeContract(
cid,
'provider',
info.protoPath,
`${info.serviceName}.${methodName}`,
0.85,
{
package: info.package,
service: info.serviceName,
method: methodName,
source: 'proto',
},
),
);
}
}
}
}
// Source files — server/client detection
const sourceFiles = await glob('**/*.{go,java,py,ts,tsx,js,jsx}', {
// ─── Source files (+ .proto when plugin available) ────────────
const sourceFiles = await glob(GRPC_SCAN_GLOB, {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
nodir: true,
});
const parser = new Parser();
for (const rel of sourceFiles) {
const plugin = getPluginForFile(rel);
if (!plugin) continue;
const content = readSafe(repoPath, rel);
if (!content) continue;
const ext = path.extname(rel).toLowerCase();
if (ext === '.go') {
out.push(...this.scanGoProviders(content, rel));
out.push(...this.scanGoConsumers(content, rel));
} else if (ext === '.java') {
out.push(...this.scanJavaProviders(content, rel));
out.push(...this.scanJavaConsumers(content, rel));
} else if (ext === '.py') {
out.push(...this.scanPythonProviders(content, rel));
out.push(...this.scanPythonConsumers(content, rel));
} else if (['.ts', '.tsx', '.js', '.jsx'].includes(ext)) {
out.push(...this.scanTsProviders(content, rel));
let detections: GrpcDetection[] = [];
try {
parser.setLanguage(plugin.language);
const tree = parser.parse(content);
detections = plugin.scan(tree);
} catch {
continue;
}
for (const d of detections) {
const contract = this.detectionToContract(d, rel, protoMap);
if (contract) out.push(contract);
}
}
return this.dedupe(out);
}
private parseProtoFile(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
const pkgMatch = content.match(/^package\s+([\w.]+)\s*;/m);
const pkg = pkgMatch ? pkgMatch[1] : '';
for (const { name: serviceName, body } of extractServiceBlocks(content)) {
const rpcRe = /rpc\s+(\w+)\s*\(/g;
let rpcMatch: RegExpExecArray | null;
while ((rpcMatch = rpcRe.exec(body)) !== null) {
const methodName = rpcMatch[1];
const cid = contractId(pkg, serviceName, methodName);
out.push(
makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.85, {
package: pkg,
service: serviceName,
method: methodName,
source: 'proto',
}),
);
}
}
return out;
}
private scanGoProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
// pb.RegisterXxxServer(
const registerRe = /\w+\.Register(\w+)Server\s*\(/g;
let m: RegExpExecArray | null;
while ((m = registerRe.exec(content)) !== null) {
const serviceName = m[1];
out.push(
makeContract(
serviceOnlyContractId(serviceName),
'provider',
filePath,
`Register${serviceName}Server`,
0.8,
{ service: serviceName, source: 'go_register' },
),
);
}
// pb.UnimplementedXxxServer
const unimplRe = /\w+\.Unimplemented(\w+)Server\b/g;
while ((m = unimplRe.exec(content)) !== null) {
const serviceName = m[1];
out.push(
makeContract(
serviceOnlyContractId(serviceName),
'provider',
filePath,
`Unimplemented${serviceName}Server`,
0.8,
{ service: serviceName, source: 'go_unimplemented' },
),
);
}
return out;
}
private scanGoConsumers(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
const re = /\w+\.New(\w+)Client\s*\(/g;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const serviceName = m[1];
out.push(
makeContract(
serviceOnlyContractId(serviceName),
'consumer',
filePath,
`New${serviceName}Client`,
0.7,
{ service: serviceName, source: 'go_client' },
),
);
}
return out;
}
private scanJavaProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
// @GrpcService
if (content.includes('@GrpcService')) {
const implBaseRe = /extends\s+(\w+)Grpc\.(\w+)ImplBase/;
const m = content.match(implBaseRe);
if (m) {
out.push(
makeContract(serviceOnlyContractId(m[1]), 'provider', filePath, m[2], 0.8, {
service: m[1],
source: 'java_grpc_service',
}),
);
} else {
// Try extracting service name from class name
const classRe =
/class\s+(\w*?)(?:Grpc)?(?:Service)?\s+extends\s+(\w+)(?:Grpc\.(\w+))?ImplBase/;
const cm = content.match(classRe);
if (cm) {
const svcName = cm[2].replace(/Grpc$/, '');
out.push(
makeContract(serviceOnlyContractId(svcName), 'provider', filePath, cm[1], 0.8, {
service: svcName,
source: 'java_grpc_service',
}),
);
}
}
}
// extends XxxImplBase (without @GrpcService)
if (!content.includes('@GrpcService')) {
const implRe = /extends\s+(\w+?)(?:Grpc\.(\w+))?ImplBase/;
const m = content.match(implRe);
if (m) {
const svcName = m[2] || m[1].replace(/Grpc$/, '');
out.push(
makeContract(serviceOnlyContractId(svcName), 'provider', filePath, svcName, 0.8, {
service: svcName,
source: 'java_impl_base',
}),
);
}
}
return out;
}
private scanJavaConsumers(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
// XxxGrpc.newBlockingStub( or XxxGrpc.newStub(
const re = /(\w+)Grpc\.new(?:Blocking)?Stub\s*\(/g;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const serviceName = m[1];
out.push(
makeContract(
serviceOnlyContractId(serviceName),
'consumer',
filePath,
`${serviceName}Stub`,
0.7,
{ service: serviceName, source: 'java_stub' },
),
);
}
return out;
}
private scanPythonProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
// add_XxxServicer_to_server(
const re = /add_(\w+?)Servicer_to_server\s*\(/g;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const serviceName = m[1];
out.push(
makeContract(
serviceOnlyContractId(serviceName),
'provider',
filePath,
`add_${serviceName}Servicer_to_server`,
0.8,
{ service: serviceName, source: 'python_servicer' },
),
);
}
return out;
}
private scanPythonConsumers(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
// XxxStub(
const re = /(\w+)Stub\s*\(/g;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const name = m[1];
// Filter out common false positives
if (['Mock', 'Test', 'Fake', 'Stub'].includes(name)) continue;
out.push(
makeContract(serviceOnlyContractId(name), 'consumer', filePath, `${name}Stub`, 0.7, {
service: name,
source: 'python_stub',
}),
);
}
return out;
}
private scanTsProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
// @GrpcMethod('ServiceName', 'MethodName')
const re = /@GrpcMethod\s*\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)/g;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const serviceName = m[1];
const methodName = m[2];
const cid = contractId('', serviceName, methodName);
out.push(
makeContract(cid, 'provider', filePath, `${serviceName}.${methodName}`, 0.8, {
service: serviceName,
method: methodName,
source: 'ts_grpc_method',
}),
);
}
return out;
/**
* Convert a plugin `GrpcDetection` into a concrete `ExtractedContract`
* by resolving the short service name against the proto map, building
* either a service-level (`grpc::pkg.Svc/*`) or method-level
* (`grpc::pkg.Svc/Method`) contract id, and selecting confidence
* based on whether the proto map had an entry.
*/
private detectionToContract(
d: GrpcDetection,
filePath: string,
protoMap: Map<string, ProtoServiceInfo[]>,
): ExtractedContract | null {
const candidates = protoMap.get(d.serviceName) ?? [];
const proto = resolveProtoConflict(d.serviceName, filePath, candidates);
// If there were proto candidates but resolution was ambiguous, skip
// contract emission rather than fabricating a package-qualified id from
// an arbitrary candidate. resolveProtoConflict already warned.
if (candidates.length > 0 && proto === null) return null;
const pkg = proto?.package ?? '';
const cid = d.methodName
? contractId(pkg, d.serviceName, d.methodName)
: proto
? serviceContractId(pkg, d.serviceName)
: serviceOnlyContractId(d.serviceName);
const confidence = proto ? d.confidenceWithProto : d.confidenceWithoutProto;
const meta: Record<string, unknown> = {
service: d.serviceName,
source: d.source,
};
if (d.methodName) meta.method = d.methodName;
return makeContract(cid, d.role, filePath, d.symbolName, confidence, meta);
}
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
const seen = new Set<string>();
const out: ExtractedContract[] = [];
const byKey = new Map<string, ExtractedContract>();
for (const c of items) {
const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`;
if (seen.has(k)) continue;
seen.add(k);
out.push(c);
const existing = byKey.get(k);
if (
!existing ||
c.confidence > existing.confidence ||
(c.confidence === existing.confidence &&
String(c.meta.source) < String(existing.meta.source))
) {
byKey.set(k, c);
}
}
return out;
return Array.from(byKey.values());
}
}

View file

@ -0,0 +1,109 @@
import Go from 'tree-sitter-go';
import {
compilePatterns,
runCompiledPatterns,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
/**
* Go gRPC plugin. Detects:
* - Provider: `pb.RegisterXxxServer(...)` calls
* - Provider: `pb.UnimplementedXxxServer` embedded in a struct
* - Consumer: `pb.NewXxxClient(conn)` calls
*/
const REGISTER_RE = /^Register(\w+)Server$/;
const UNIMPLEMENTED_RE = /^Unimplemented(\w+)Server$/;
const NEW_CLIENT_RE = /^New(\w+)Client$/;
// Any `xxx.<fn>(...)` call — plugin filters the field identifier text.
const SELECTOR_CALL_PATTERNS = compilePatterns({
name: 'go-grpc-selector-call',
language: Go,
patterns: [
{
meta: {},
query: `
(call_expression
function: (selector_expression
field: (field_identifier) @fn))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// Any `qualified_type` used as a struct field — for `pb.UnimplementedXxxServer`.
const STRUCT_EMBEDDING_PATTERNS = compilePatterns({
name: 'go-grpc-struct-embedding',
language: Go,
patterns: [
{
meta: {},
query: `
(struct_type
(field_declaration_list
(field_declaration
type: (qualified_type
name: (type_identifier) @field_type))))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
export const GO_GRPC_PLUGIN: GrpcLanguagePlugin = {
name: 'go-grpc',
language: Go,
scan(tree) {
const out: GrpcDetection[] = [];
for (const match of runCompiledPatterns(SELECTOR_CALL_PATTERNS, tree)) {
const fnNode = match.captures.fn;
if (!fnNode) continue;
const fnText = fnNode.text;
const registerMatch = REGISTER_RE.exec(fnText);
if (registerMatch) {
out.push({
role: 'provider',
serviceName: registerMatch[1],
symbolName: fnText,
source: 'go_register',
confidenceWithProto: 0.8,
confidenceWithoutProto: 0.65,
});
continue;
}
const newClientMatch = NEW_CLIENT_RE.exec(fnText);
if (newClientMatch) {
out.push({
role: 'consumer',
serviceName: newClientMatch[1],
symbolName: fnText,
source: 'go_client',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
continue;
}
}
for (const match of runCompiledPatterns(STRUCT_EMBEDDING_PATTERNS, tree)) {
const fieldNode = match.captures.field_type;
if (!fieldNode) continue;
const unimpl = UNIMPLEMENTED_RE.exec(fieldNode.text);
if (!unimpl) continue;
out.push({
role: 'provider',
serviceName: unimpl[1],
symbolName: fieldNode.text,
source: 'go_unimplemented',
confidenceWithProto: 0.8,
confidenceWithoutProto: 0.65,
});
}
return out;
},
};

View file

@ -0,0 +1,53 @@
import * as path from 'node:path';
import type { GrpcLanguagePlugin } from './types.js';
import { GO_GRPC_PLUGIN } from './go.js';
import { JAVA_GRPC_PLUGIN } from './java.js';
import { PYTHON_GRPC_PLUGIN } from './python.js';
import { JAVASCRIPT_GRPC_PLUGIN, TYPESCRIPT_GRPC_PLUGIN, TSX_GRPC_PLUGIN } from './node.js';
import { PROTO_GRPC_PLUGIN } from './proto.js';
export type { GrpcDetection, GrpcLanguagePlugin, GrpcRole } from './types.js';
export { PROTO_GRPC_PLUGIN, extractPackageFromTree } from './proto.js';
/**
* File-extension gRPC language plugin registry. Mirrors the shape
* of `http-patterns/index.ts` and `topic-patterns/index.ts`.
*
* `.proto` files are registered only when `tree-sitter-proto` is
* available (it's an optionalDependency). When absent, the orchestrator
* falls back to the built-in manual proto parser.
*/
const REGISTRY: Record<string, GrpcLanguagePlugin> = {
'.go': GO_GRPC_PLUGIN,
'.java': JAVA_GRPC_PLUGIN,
'.py': PYTHON_GRPC_PLUGIN,
'.js': JAVASCRIPT_GRPC_PLUGIN,
'.jsx': JAVASCRIPT_GRPC_PLUGIN,
'.ts': TYPESCRIPT_GRPC_PLUGIN,
'.tsx': TSX_GRPC_PLUGIN,
...(PROTO_GRPC_PLUGIN ? { '.proto': PROTO_GRPC_PLUGIN } : {}),
};
/**
* Glob for source files worth scanning for gRPC server/client patterns.
* Includes `.proto` when the grammar is available.
*/
export const GRPC_SCAN_GLOB = PROTO_GRPC_PLUGIN
? '**/*.{go,java,py,ts,tsx,js,jsx,proto}'
: '**/*.{go,java,py,ts,tsx,js,jsx}';
/**
* Whether the tree-sitter proto plugin is available. The orchestrator
* uses this to decide between the tree-sitter path and the fallback
* manual parser for `.proto` files.
*/
export const hasProtoPlugin = PROTO_GRPC_PLUGIN !== null;
/**
* Return the gRPC plugin registered for the given file's extension,
* or `undefined` if the extension is not registered.
*/
export function getPluginForFile(rel: string): GrpcLanguagePlugin | undefined {
const ext = path.extname(rel).toLowerCase();
return REGISTRY[ext];
}

View file

@ -0,0 +1,179 @@
import Parser from 'tree-sitter';
import Java from 'tree-sitter-java';
import {
compilePatterns,
runCompiledPatterns,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
/**
* Java gRPC plugin. Detects:
* - Provider: classes extending `XxxServiceGrpc.XxxServiceImplBase`
* (with or without a `@GrpcService` annotation; the annotation
* only affects confidence labelling in the original regex version
* here we emit a single detection per class and pick the source
* label based on whether the annotation is present).
* - Consumer: `XxxServiceGrpc.newBlockingStub(ch)` /
* `XxxServiceGrpc.newStub(ch)` calls.
*/
const IMPL_BASE_RE = /^(\w+)ImplBase$/;
const GRPC_SUFFIX_RE = /^(\w+)Grpc$/;
// Classes extending `ScopedType.ScopedType` where the inner name ends
// in ImplBase. Covers `XxxServiceGrpc.XxxServiceImplBase`.
// Note: tree-sitter-java's `scoped_type_identifier` exposes its two
// segments as positional `type_identifier` children, NOT as named
// `scope:`/`name:` fields. We match positionally here and rely on the
// grammar's left-to-right ordering: first child = outer, second = inner.
const SCOPED_IMPL_BASE_PATTERNS = compilePatterns({
name: 'java-grpc-scoped-impl-base',
language: Java,
patterns: [
{
meta: {},
query: `
(class_declaration
name: (identifier) @class_name
superclass: (superclass
(scoped_type_identifier
(type_identifier) @outer
(type_identifier) @inner (#match? @inner "ImplBase$")))) @class
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// Classes extending a simple `XxxImplBase` identifier (no scope).
const PLAIN_IMPL_BASE_PATTERNS = compilePatterns({
name: 'java-grpc-plain-impl-base',
language: Java,
patterns: [
{
meta: {},
query: `
(class_declaration
name: (identifier) @class_name
superclass: (superclass
(type_identifier) @plain_type (#match? @plain_type "ImplBase$"))) @class
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// gRPC stub factories: `XxxGrpc.newStub(ch)` / `XxxGrpc.newBlockingStub(ch)`.
const STUB_PATTERNS = compilePatterns({
name: 'java-grpc-stub',
language: Java,
patterns: [
{
meta: {},
query: `
(method_invocation
object: (identifier) @grpc_cls
name: (identifier) @method (#match? @method "^new(Blocking)?Stub$"))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
/**
* Check whether a `class_declaration` node has a `@GrpcService`
* annotation in its modifiers list. In tree-sitter-java, class-level
* annotations live under `(class_declaration (modifiers (marker_annotation|annotation)))`.
*/
function hasGrpcServiceAnnotation(classNode: Parser.SyntaxNode): boolean {
for (let i = 0; i < classNode.namedChildCount; i++) {
const child = classNode.namedChild(i);
if (!child || child.type !== 'modifiers') continue;
for (let j = 0; j < child.namedChildCount; j++) {
const mod = child.namedChild(j);
if (!mod) continue;
if (mod.type !== 'marker_annotation' && mod.type !== 'annotation') continue;
const nameNode = mod.childForFieldName('name');
if (nameNode?.text === 'GrpcService') return true;
}
}
return false;
}
/**
* Given the inner type_identifier text like `AuthServiceImplBase`,
* return the service name (`AuthService`), or null if the text
* doesn't end in `ImplBase`.
*/
function extractServiceFromImplBase(text: string): string | null {
const m = IMPL_BASE_RE.exec(text);
if (!m) return null;
// Strip a trailing `Grpc` on the service name too — the original
// regex replaces `Grpc$` on the extracted prefix.
return m[1].replace(/Grpc$/, '');
}
export const JAVA_GRPC_PLUGIN: GrpcLanguagePlugin = {
name: 'java-grpc',
language: Java,
scan(tree) {
const out: GrpcDetection[] = [];
const emittedClassIds = new Set<number>();
// ─── Providers: scoped form (`...Grpc.XxxImplBase`) ─────────────
for (const match of runCompiledPatterns(SCOPED_IMPL_BASE_PATTERNS, tree)) {
const classNode = match.captures.class;
const innerNode = match.captures.inner;
if (!classNode || !innerNode) continue;
const serviceName = extractServiceFromImplBase(innerNode.text);
if (!serviceName) continue;
emittedClassIds.add(classNode.id);
const annotated = hasGrpcServiceAnnotation(classNode);
out.push({
role: 'provider',
serviceName,
symbolName: serviceName,
source: annotated ? 'java_grpc_service' : 'java_impl_base',
confidenceWithProto: 0.8,
confidenceWithoutProto: 0.65,
});
}
// ─── Providers: plain form (`XxxImplBase`) ──────────────────────
for (const match of runCompiledPatterns(PLAIN_IMPL_BASE_PATTERNS, tree)) {
const classNode = match.captures.class;
const plainNode = match.captures.plain_type;
if (!classNode || !plainNode) continue;
if (emittedClassIds.has(classNode.id)) continue;
const serviceName = extractServiceFromImplBase(plainNode.text);
if (!serviceName) continue;
emittedClassIds.add(classNode.id);
const annotated = hasGrpcServiceAnnotation(classNode);
out.push({
role: 'provider',
serviceName,
symbolName: serviceName,
source: annotated ? 'java_grpc_service' : 'java_impl_base',
confidenceWithProto: 0.8,
confidenceWithoutProto: 0.65,
});
}
// ─── Consumers: `XxxGrpc.newBlockingStub(...)` / `newStub(...)` ─
for (const match of runCompiledPatterns(STUB_PATTERNS, tree)) {
const grpcClsNode = match.captures.grpc_cls;
if (!grpcClsNode) continue;
const grpcMatch = GRPC_SUFFIX_RE.exec(grpcClsNode.text);
if (!grpcMatch) continue;
const serviceName = grpcMatch[1];
out.push({
role: 'consumer',
serviceName,
symbolName: `${serviceName}Stub`,
source: 'java_stub',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
}
return out;
},
};

View file

@ -0,0 +1,314 @@
import Parser from 'tree-sitter';
import JavaScript from 'tree-sitter-javascript';
import TypeScript from 'tree-sitter-typescript';
import {
compilePatterns,
runCompiledPatterns,
unquoteLiteral,
type CompiledPatterns,
type LanguagePatterns,
type PatternSpec,
} from '../tree-sitter-scanner.js';
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
/**
* Node.js / TypeScript gRPC plugin family. Detects:
* - Provider: NestJS `@GrpcMethod('Service', 'Method')` decorators
* - Consumer: NestJS `@GrpcClient(...) readonly x!: XxxServiceClient`
* - Consumer: `client.getService<X>('AuthService')`
* - Consumer: `new XxxServiceClient(...)` (generated client constructor)
* - Consumer: `new foo.bar.Xxx(...)` when the file uses
* `loadPackageDefinition` (gRPC dynamic proto loader)
*
* As with the HTTP `node.ts`, pattern sources are defined once and
* compiled against three grammar variants (JS / TS / TSX) because
* `Parser.Query` is not portable across grammar objects.
*/
const SERVICE_CLIENT_RE = /^(\w+Service)Client$/;
const CAPITALIZED_SERVICE_RE = /^[A-Z]\w+$/;
// @GrpcMethod('Service', 'Method')
const GRPC_METHOD_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(decorator
(call_expression
function: (identifier) @dec (#eq? @dec "GrpcMethod")
arguments: (arguments
. [(string) (template_string)] @service
. [(string) (template_string)] @method)))
`,
};
// @GrpcClient(...) standalone decorator — the plugin walks to the next
// sibling (a field definition) to read its type annotation.
const GRPC_CLIENT_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(decorator
(call_expression
function: (identifier) @dec (#eq? @dec "GrpcClient"))) @grpc_client_decorator
`,
};
// `.getService<X>('AuthService')` / `.getService('AuthService')`
const GET_SERVICE_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (member_expression
property: (property_identifier) @method (#eq? @method "getService"))
arguments: (arguments . [(string) (template_string)] @service))
`,
};
// `new XxxServiceClient(...)` — bare identifier constructor.
const NEW_SIMPLE_CTOR_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(new_expression
constructor: (identifier) @ctor)
`,
};
// `new foo.bar.XxxService(...)` — qualified constructor.
const NEW_QUALIFIED_CTOR_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(new_expression
constructor: (member_expression
property: (property_identifier) @ctor))
`,
};
// Detect whether the file uses `loadPackageDefinition` (gRPC dynamic
// proto loader). Matches either a bare call or an `obj.loadPackageDefinition(...)`
// call. Plugin gates the qualified-constructor consumer on this —
// structural check avoids materializing `tree.rootNode.text` for every file.
const LOAD_PACKAGE_DEFINITION_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: [
(identifier) @fn (#eq? @fn "loadPackageDefinition")
(member_expression property: (property_identifier) @fn (#eq? @fn "loadPackageDefinition"))
])
`,
};
interface NodeGrpcPatternBundle {
grpcMethod: CompiledPatterns<Record<string, never>>;
grpcClient: CompiledPatterns<Record<string, never>>;
getService: CompiledPatterns<Record<string, never>>;
newSimpleCtor: CompiledPatterns<Record<string, never>>;
newQualifiedCtor: CompiledPatterns<Record<string, never>>;
loadPackageDefinition: CompiledPatterns<Record<string, never>>;
}
function compileBundle(language: unknown, name: string): NodeGrpcPatternBundle {
const mk = (spec: PatternSpec<Record<string, never>>, suffix: string) =>
compilePatterns({
name: `${name}-${suffix}`,
language,
patterns: [spec],
} satisfies LanguagePatterns<Record<string, never>>);
return {
grpcMethod: mk(GRPC_METHOD_SPEC, 'grpc-method'),
grpcClient: mk(GRPC_CLIENT_SPEC, 'grpc-client'),
getService: mk(GET_SERVICE_SPEC, 'get-service'),
newSimpleCtor: mk(NEW_SIMPLE_CTOR_SPEC, 'new-simple-ctor'),
newQualifiedCtor: mk(NEW_QUALIFIED_CTOR_SPEC, 'new-qualified-ctor'),
loadPackageDefinition: mk(LOAD_PACKAGE_DEFINITION_SPEC, 'load-package-definition'),
};
}
const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-grpc');
const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-grpc');
const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-grpc');
/**
* Given a `@GrpcClient(...)` decorator node, find the type annotation
* text of the field it decorates (e.g. `AuthServiceClient`).
*
* In tree-sitter-typescript, decorators on class fields can appear in
* two configurations:
* - As a CHILD of `public_field_definition` alongside the field's
* type annotation (the common case for NestJS `@GrpcClient`).
* - As a SIBLING of the field in `class_body` (for method
* decorators, but kept for resilience against grammar variants).
* We walk the parent container and search for a type annotation.
*/
function resolveGrpcClientFieldType(decoratorNode: Parser.SyntaxNode): string | null {
const parent = decoratorNode.parent;
if (!parent) return null;
// Case 1: decorator is a child of the field definition — search
// the parent itself (which is the field definition) for a
// type_annotation child.
if (parent.type === 'public_field_definition' || parent.type.endsWith('field_definition')) {
return findFirstTypeAnnotationText(parent);
}
// Case 2: decorator is a sibling of the field in a class_body — walk
// forward through subsequent siblings until we find a node containing
// a type annotation.
for (let i = 0; i < parent.namedChildCount; i++) {
const child = parent.namedChild(i);
if (child && child.id === decoratorNode.id) {
for (let j = i + 1; j < parent.namedChildCount; j++) {
const next = parent.namedChild(j);
if (!next) continue;
if (next.type === 'decorator') continue;
const typeText = findFirstTypeAnnotationText(next);
if (typeText) return typeText;
return null;
}
return null;
}
}
return null;
}
/**
* Recursively search `node` for the first `type_annotation` child and
* return the text of its inner `type_identifier`, or null. Handles
* both `public_field_definition` and its variants.
*/
function findFirstTypeAnnotationText(node: Parser.SyntaxNode): string | null {
if (node.type === 'type_annotation') {
for (let i = 0; i < node.namedChildCount; i++) {
const child = node.namedChild(i);
if (!child) continue;
if (child.type === 'type_identifier') return child.text;
}
return null;
}
for (let i = 0; i < node.namedChildCount; i++) {
const child = node.namedChild(i);
if (!child) continue;
const found = findFirstTypeAnnotationText(child);
if (found) return found;
}
return null;
}
function scanBundle(bundle: NodeGrpcPatternBundle, tree: Parser.Tree): GrpcDetection[] {
const out: GrpcDetection[] = [];
// ─── Provider: @GrpcMethod('Service', 'Method') ──────────────────
for (const match of runCompiledPatterns(bundle.grpcMethod, tree)) {
const svcNode = match.captures.service;
const methodNode = match.captures.method;
if (!svcNode || !methodNode) continue;
const svc = unquoteLiteral(svcNode.text);
const mth = unquoteLiteral(methodNode.text);
if (!svc || !mth) continue;
out.push({
role: 'provider',
serviceName: svc,
symbolName: `${svc}.${mth}`,
source: 'ts_grpc_method',
methodName: mth,
// @GrpcMethod hard-coded confidence 0.8 in the original code
// regardless of whether the proto map has a match.
confidenceWithProto: 0.8,
confidenceWithoutProto: 0.8,
});
}
// ─── Consumer: @GrpcClient() field with XxxServiceClient type ────
for (const match of runCompiledPatterns(bundle.grpcClient, tree)) {
const decoratorNode = match.captures.grpc_client_decorator;
if (!decoratorNode) continue;
const typeText = resolveGrpcClientFieldType(decoratorNode);
if (!typeText) continue;
const svcMatch = SERVICE_CLIENT_RE.exec(typeText);
if (!svcMatch) continue;
const serviceName = svcMatch[1];
out.push({
role: 'consumer',
serviceName,
symbolName: `${serviceName}Client`,
source: 'ts_grpc_client_decorator',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
}
// ─── Consumer: client.getService<X>('Service') ───────────────────
for (const match of runCompiledPatterns(bundle.getService, tree)) {
const svcNode = match.captures.service;
if (!svcNode) continue;
const svc = unquoteLiteral(svcNode.text);
if (!svc) continue;
out.push({
role: 'consumer',
serviceName: svc,
symbolName: `${svc}Client`,
source: 'ts_client_grpc_get_service',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
}
// ─── Consumer: new XxxServiceClient(...) ─────────────────────────
for (const match of runCompiledPatterns(bundle.newSimpleCtor, tree)) {
const ctorNode = match.captures.ctor;
if (!ctorNode) continue;
const svcMatch = SERVICE_CLIENT_RE.exec(ctorNode.text);
if (!svcMatch) continue;
const serviceName = svcMatch[1];
out.push({
role: 'consumer',
serviceName,
symbolName: `${serviceName}Client`,
source: 'ts_generated_client',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
}
// ─── Consumer: loadPackageDefinition dynamic proto loader ────────
// Only emit when the file uses loadPackageDefinition, otherwise a
// generic `new foo.bar.Something()` in unrelated code would falsely
// register as a gRPC consumer. Check structurally via a dedicated
// query — avoids materializing `tree.rootNode.text` for the whole
// file (expensive on large files).
const usesLoadPackage = runCompiledPatterns(bundle.loadPackageDefinition, tree).length > 0;
if (usesLoadPackage) {
for (const match of runCompiledPatterns(bundle.newQualifiedCtor, tree)) {
const ctorNode = match.captures.ctor;
if (!ctorNode) continue;
if (!CAPITALIZED_SERVICE_RE.test(ctorNode.text)) continue;
out.push({
role: 'consumer',
serviceName: ctorNode.text,
symbolName: `${ctorNode.text}Client`,
source: 'ts_load_package_definition',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
}
}
return out;
}
export const JAVASCRIPT_GRPC_PLUGIN: GrpcLanguagePlugin = {
name: 'javascript-grpc',
language: JavaScript,
scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree),
};
export const TYPESCRIPT_GRPC_PLUGIN: GrpcLanguagePlugin = {
name: 'typescript-grpc',
language: TypeScript.typescript,
scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree),
};
export const TSX_GRPC_PLUGIN: GrpcLanguagePlugin = {
name: 'tsx-grpc',
language: TypeScript.tsx,
scan: (tree) => scanBundle(TSX_BUNDLE, tree),
};

View file

@ -0,0 +1,147 @@
import { createRequire } from 'node:module';
import {
compilePatterns,
runCompiledPatterns,
type CompiledPatterns,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
/**
* Protobuf (.proto) tree-sitter plugin for gRPC contract extraction.
*
* Uses `tree-sitter-proto` (coder3101/tree-sitter-proto) as an
* optionalDependency if the grammar is not installed (e.g. native
* compilation failed on an unusual platform), the plugin exports
* `null` and the orchestrator falls back to the existing manual
* string-sanitizing parser.
*
* The grammar is vendored in `vendor/tree-sitter-proto/` with
* parser.c regenerated against tree-sitter-cli 0.24 (ABI version 14)
* so it is compatible with the project's tree-sitter 0.25 runtime.
*/
const _require = createRequire(import.meta.url);
let ProtoGrammar: unknown = null;
try {
ProtoGrammar = _require('tree-sitter-proto');
} catch {
// Grammar not installed — PROTO_GRPC_PLUGIN will be null.
}
let PACKAGE_PATTERNS: CompiledPatterns<Record<string, never>> | null = null;
let SERVICE_PATTERNS: CompiledPatterns<Record<string, never>> | null = null;
if (ProtoGrammar) {
try {
// Validate that the grammar actually loads end-to-end: compile queries
// AND parse + walk a trivial proto file. tree-sitter's internal
// `initializeLanguageNodeClasses` can fail with a TDZ error in some
// test runners (vitest forks) when SyntaxNode isn't fully initialized
// yet. Catching that here ensures `PROTO_GRPC_PLUGIN` stays null and
// the orchestrator falls back to the manual parser.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const _Parser = _require('tree-sitter') as any;
// Smoke-test: parse + setLanguage to verify the grammar is
// end-to-end compatible with this tree-sitter runtime.
const _testParser = new _Parser();
_testParser.setLanguage(ProtoGrammar);
_testParser.parse('service X { rpc Y (R) returns (R); }');
PACKAGE_PATTERNS = compilePatterns({
name: 'proto-package',
language: ProtoGrammar,
patterns: [
{
meta: {},
query: `(package (full_ident) @pkg)`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
SERVICE_PATTERNS = compilePatterns({
name: 'proto-service',
language: ProtoGrammar,
patterns: [
{
meta: {},
query: `
(service
(service_name) @service_name
(rpc
(rpc_name) @rpc_name))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
} catch {
// Compilation failed (grammar ABI mismatch?) — fall back to null.
PACKAGE_PATTERNS = null;
SERVICE_PATTERNS = null;
ProtoGrammar = null;
}
}
function buildPlugin(): GrpcLanguagePlugin | null {
if (!ProtoGrammar || !PACKAGE_PATTERNS || !SERVICE_PATTERNS) return null;
const pkgPatterns = PACKAGE_PATTERNS;
const svcPatterns = SERVICE_PATTERNS;
return {
name: 'proto-grpc',
language: ProtoGrammar,
scan(tree) {
const out: GrpcDetection[] = [];
// Extract `package` declaration (first match wins).
let pkg = '';
for (const match of runCompiledPatterns(pkgPatterns, tree)) {
const pkgNode = match.captures.pkg;
if (pkgNode) {
pkg = pkgNode.text;
break;
}
}
// Extract `service → rpc` pairs. The query returns one match per
// (service, rpc) combination thanks to the nested structure.
for (const match of runCompiledPatterns(svcPatterns, tree)) {
const serviceNode = match.captures.service_name;
const rpcNode = match.captures.rpc_name;
if (!serviceNode || !rpcNode) continue;
const serviceName = serviceNode.text;
const methodName = rpcNode.text;
out.push({
role: 'provider',
serviceName,
symbolName: `${serviceName}.${methodName}`,
source: 'proto',
methodName,
// Proto definitions are the canonical source of truth — always
// high confidence regardless of cross-referencing.
confidenceWithProto: 0.85,
confidenceWithoutProto: 0.85,
});
}
return out;
},
};
}
/**
* The proto plugin, or `null` if tree-sitter-proto is not available.
* The orchestrator checks this at import time and decides whether to
* use the tree-sitter path or the fallback manual parser.
*/
export const PROTO_GRPC_PLUGIN: GrpcLanguagePlugin | null = buildPlugin();
/** The package declaration text from a proto file's tree. */
export function extractPackageFromTree(tree: import('tree-sitter').Tree): string {
if (!PACKAGE_PATTERNS) return '';
for (const match of runCompiledPatterns(PACKAGE_PATTERNS, tree)) {
const pkgNode = match.captures.pkg;
if (pkgNode) return pkgNode.text;
}
return '';
}

View file

@ -0,0 +1,77 @@
import Python from 'tree-sitter-python';
import {
compilePatterns,
runCompiledPatterns,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { GrpcDetection, GrpcLanguagePlugin } from './types.js';
/**
* Python gRPC plugin. Detects:
* - Provider: `add_XxxServicer_to_server(...)` calls (bare identifier
* or qualified attribute form `auth_pb2_grpc.add_XxxServicer_to_server`)
* - Consumer: `XxxStub(channel)` calls (bare or `auth_pb2_grpc.XxxStub`)
*/
const ADD_SERVICER_RE = /^add_(\w+)Servicer_to_server$/;
const STUB_RE = /^(\w+)Stub$/;
/** Reserved names that would produce garbage service names. */
const STUB_IGNORE = new Set(['Mock', 'Test', 'Fake', 'Stub']);
// Any call whose target is either a bare identifier or an attribute
// access (`obj.method`). The plugin filters the function name in JS.
const CALL_PATTERNS = compilePatterns({
name: 'python-grpc-call',
language: Python,
patterns: [
{
meta: {},
query: `
(call
function: [
(identifier) @fn
(attribute attribute: (identifier) @fn)
])
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
export const PYTHON_GRPC_PLUGIN: GrpcLanguagePlugin = {
name: 'python-grpc',
language: Python,
scan(tree) {
const out: GrpcDetection[] = [];
for (const match of runCompiledPatterns(CALL_PATTERNS, tree)) {
const fnNode = match.captures.fn;
if (!fnNode) continue;
const fnText = fnNode.text;
const addServicer = ADD_SERVICER_RE.exec(fnText);
if (addServicer) {
out.push({
role: 'provider',
serviceName: addServicer[1],
symbolName: fnText,
source: 'python_servicer',
confidenceWithProto: 0.8,
confidenceWithoutProto: 0.65,
});
continue;
}
const stubMatch = STUB_RE.exec(fnText);
if (stubMatch && !STUB_IGNORE.has(stubMatch[1])) {
out.push({
role: 'consumer',
serviceName: stubMatch[1],
symbolName: fnText,
source: 'python_stub',
confidenceWithProto: 0.75,
confidenceWithoutProto: 0.55,
});
}
}
return out;
},
};

View file

@ -0,0 +1,54 @@
import type Parser from 'tree-sitter';
/**
* Shared types for the grpc-extractor language plugins.
*
* Each plugin lives in its own file (java.ts, go.ts, ...) and owns the
* tree-sitter grammar import + query sources. The top-level
* `grpc-extractor.ts` orchestrator only knows about this type module
* and the plugin registry (`./index.ts`). It MUST NOT import any
* grammar or query text directly.
*/
export type GrpcRole = 'provider' | 'consumer';
/**
* One raw gRPC detection produced by a plugin's `scan()` function. The
* orchestrator uses the proto map to resolve the full package-qualified
* contract id and choose a confidence based on whether the proto was
* found.
*
* Most patterns produce service-level detections; `TS @GrpcMethod` is
* the only pattern that captures an explicit `methodName`, producing
* a method-level contract (`grpc::pkg.Service/Method`).
*/
export interface GrpcDetection {
role: GrpcRole;
/** Short service name, e.g. `"AuthService"`. */
serviceName: string;
/** Symbol name emitted into the contract's symbolRef. */
symbolName: string;
/** Metadata source label (goes into `meta.source`). */
source: string;
/** Explicit method name; set only by TS `@GrpcMethod`. */
methodName?: string;
/** Confidence when the proto map resolves the service. */
confidenceWithProto: number;
/** Confidence when the proto map has no entry. */
confidenceWithoutProto: number;
}
/**
* One language-scoped gRPC plugin. Plugins own the tree-sitter grammar
* and a `scan(tree)` function that returns zero or more
* `GrpcDetection`s. The plugin is free to run multiple compiled query
* bundles and walk the AST to cross-reference captures.
*
* `language` is typed `unknown` for the same reason as in
* `tree-sitter-scanner.ts`.
*/
export interface GrpcLanguagePlugin {
name: string;
language: unknown;
scan(tree: Parser.Tree): GrpcDetection[];
}

View file

@ -0,0 +1,224 @@
import Go from 'tree-sitter-go';
import {
compilePatterns,
runCompiledPatterns,
unquoteLiteral,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
/**
* Go HTTP plugin. Handles:
* - gin / echo / chi framework routing `r.GET("/path", handler)`
* - net/http stdlib `http.HandleFunc("/path", handler)`
* - net/http consumer `http.Get(...)`, `http.NewRequest("METHOD", ...)`
* - resty consumer `client.R().Delete("/path")`
*/
// ─── Provider: framework routing ──────────────────────────────────────
// Matches `\w+\.GET(...)` etc. (gin, echo, chi all share this shape).
// Captures the HTTP method (field name), path literal, and handler
// identifier passed as the second argument.
const FRAMEWORK_ROUTE_PATTERNS = compilePatterns({
name: 'go-framework-route',
language: Go,
patterns: [
{
meta: {},
query: `
(call_expression
function: (selector_expression
field: (field_identifier) @http_method (#match? @http_method "^(GET|POST|PUT|DELETE|PATCH)$"))
arguments: (argument_list
(interpreted_string_literal) @path
(identifier) @handler))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Provider: net/http `http.HandleFunc("/p", handler)` ─────────────
const HANDLE_FUNC_PATTERNS = compilePatterns({
name: 'go-handle-func',
language: Go,
patterns: [
{
meta: {},
query: `
(call_expression
function: (selector_expression
operand: (identifier) @pkg (#eq? @pkg "http")
field: (field_identifier) @fn (#eq? @fn "HandleFunc"))
arguments: (argument_list
(interpreted_string_literal) @path
(identifier) @handler))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: net/http stdlib Get / Post / Head ─────────────────────
const HTTP_CLIENT_METHOD_TO_HTTP: Record<string, string> = {
Get: 'GET',
Post: 'POST',
Head: 'GET', // HEAD has no body semantics we care about — treat as GET for contract matching
};
const HTTP_CLIENT_PATTERNS = compilePatterns({
name: 'go-http-client',
language: Go,
patterns: [
{
meta: {},
query: `
(call_expression
function: (selector_expression
operand: (identifier) @pkg (#eq? @pkg "http")
field: (field_identifier) @fn (#match? @fn "^(Get|Post|Head)$"))
arguments: (argument_list . (interpreted_string_literal) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: net/http `http.NewRequest("METHOD", "/path", ...)` ────
const NEW_REQUEST_PATTERNS = compilePatterns({
name: 'go-new-request',
language: Go,
patterns: [
{
meta: {},
query: `
(call_expression
function: (selector_expression
operand: (identifier) @pkg (#eq? @pkg "http")
field: (field_identifier) @fn (#eq? @fn "NewRequest"))
arguments: (argument_list
.
(interpreted_string_literal) @http_method
(interpreted_string_literal) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: resty `client.R().Delete("/path")` ─────────────────────
// Matches any chained call whose receiver is `something.R()` and whose
// method name is an HTTP verb. This is how go-resty's fluent API looks.
const RESTY_PATTERNS = compilePatterns({
name: 'go-resty',
language: Go,
patterns: [
{
meta: {},
query: `
(call_expression
function: (selector_expression
operand: (call_expression
function: (selector_expression
field: (field_identifier) @r (#eq? @r "R")))
field: (field_identifier) @http_method (#match? @http_method "^(Get|Post|Put|Delete|Patch)$"))
arguments: (argument_list . (interpreted_string_literal) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
export const GO_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'go-http',
language: Go,
scan(tree) {
const out: HttpDetection[] = [];
// Framework providers: r.GET/POST/... with handler identifier
for (const match of runCompiledPatterns(FRAMEWORK_ROUTE_PATTERNS, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
const handlerNode = match.captures.handler;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'provider',
framework: 'go-framework',
method: methodNode.text.toUpperCase(),
path,
name: handlerNode?.text ?? null,
confidence: 0.8,
});
}
// net/http HandleFunc: default method GET
for (const match of runCompiledPatterns(HANDLE_FUNC_PATTERNS, tree)) {
const pathNode = match.captures.path;
const handlerNode = match.captures.handler;
if (!pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'provider',
framework: 'go-stdlib',
method: 'GET',
path,
name: handlerNode?.text ?? null,
confidence: 0.8,
});
}
// net/http client: http.Get/Post/Head
for (const match of runCompiledPatterns(HTTP_CLIENT_PATTERNS, tree)) {
const fnNode = match.captures.fn;
const pathNode = match.captures.path;
if (!fnNode || !pathNode) continue;
const httpMethod = HTTP_CLIENT_METHOD_TO_HTTP[fnNode.text];
if (!httpMethod) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'go-stdlib',
method: httpMethod,
path,
name: null,
confidence: 0.7,
});
}
// net/http NewRequest
for (const match of runCompiledPatterns(NEW_REQUEST_PATTERNS, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const method = unquoteLiteral(methodNode.text);
const path = unquoteLiteral(pathNode.text);
if (method === null || path === null) continue;
out.push({
role: 'consumer',
framework: 'go-stdlib',
method: method.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// resty
for (const match of runCompiledPatterns(RESTY_PATTERNS, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'go-resty',
method: methodNode.text.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
return out;
},
};

View file

@ -0,0 +1,50 @@
import * as path from 'node:path';
import type { HttpLanguagePlugin } from './types.js';
import { JAVA_HTTP_PLUGIN } from './java.js';
import { GO_HTTP_PLUGIN } from './go.js';
import { PYTHON_HTTP_PLUGIN } from './python.js';
import { PHP_HTTP_PLUGIN } from './php.js';
import { JAVASCRIPT_HTTP_PLUGIN, TYPESCRIPT_HTTP_PLUGIN, TSX_HTTP_PLUGIN } from './node.js';
export type { HttpDetection, HttpLanguagePlugin, HttpRole } from './types.js';
/**
* File-extension HTTP language plugin registry. The top-level
* orchestrator (`http-route-extractor.ts`) looks up the plugin for each
* file it visits and delegates the tree-sitter scanning to the plugin.
*
* Keys are lowercase extensions including the leading dot. To add a
* new language, drop a `http-patterns/<lang>.ts` that exports a
* `HttpLanguagePlugin`, import it here and register the extension(s).
* No edits to `http-route-extractor.ts` are required.
*/
const REGISTRY: Record<string, HttpLanguagePlugin> = {
'.java': JAVA_HTTP_PLUGIN,
'.go': GO_HTTP_PLUGIN,
'.py': PYTHON_HTTP_PLUGIN,
'.php': PHP_HTTP_PLUGIN,
'.js': JAVASCRIPT_HTTP_PLUGIN,
'.jsx': JAVASCRIPT_HTTP_PLUGIN,
'.ts': TYPESCRIPT_HTTP_PLUGIN,
'.tsx': TSX_HTTP_PLUGIN,
};
/**
* Glob for files worth scanning for HTTP routes. Kept alongside the
* registry so adding a new language widens the glob in one edit.
*
* `.vue` / `.svelte` files are intentionally omitted for the source-scan
* path they need their own grammar-aware extraction and the existing
* regex fallback for them was never very accurate. The graph-assisted
* Strategy A still handles them via the ingestion pipeline.
*/
export const HTTP_SCAN_GLOB = '**/*.{ts,tsx,js,jsx,java,go,py,php}';
/**
* Return the HTTP plugin registered for the given file's extension,
* or `undefined` if the extension is not registered.
*/
export function getPluginForFile(rel: string): HttpLanguagePlugin | undefined {
const ext = path.extname(rel).toLowerCase();
return REGISTRY[ext];
}

View file

@ -0,0 +1,267 @@
import Parser from 'tree-sitter';
import Java from 'tree-sitter-java';
import {
compilePatterns,
runCompiledPatterns,
unquoteLiteral,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
/**
* Java HTTP plugin. Handles:
* - Spring `@RequestMapping` class prefixes + `@(Get|Post|...)Mapping` method annotations
* - Spring `RestTemplate.getForObject/...`, `WebClient.method(HttpMethod.X, ...)`
* - OkHttp `new Request.Builder().url("...")`
*
* The plugin runs two pattern bundles: one to collect class-level
* `@RequestMapping` prefixes keyed by the enclosing class node, and a
* second to match method-level annotations. The `scan` function walks
* up from each matched annotation to find its enclosing class and
* combines the prefix with the method path.
*/
const METHOD_ANNOTATION_TO_HTTP: Record<string, string> = {
GetMapping: 'GET',
PostMapping: 'POST',
PutMapping: 'PUT',
DeleteMapping: 'DELETE',
PatchMapping: 'PATCH',
};
// ─── Provider: Spring class-level @RequestMapping prefix ──────────────
const SPRING_CLASS_PREFIX_PATTERNS = compilePatterns({
name: 'java-spring-class-prefix',
language: Java,
patterns: [
{
meta: {},
query: `
(class_declaration
(modifiers
(annotation
name: (identifier) @ann (#eq? @ann "RequestMapping")
arguments: (annotation_argument_list (string_literal) @prefix)))) @class
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Provider: Spring @(Get|Post|...)Mapping method annotations ───────
const SPRING_METHOD_ROUTE_PATTERNS = compilePatterns({
name: 'java-spring-method-route',
language: Java,
patterns: [
{
meta: {},
query: `
(method_declaration
(modifiers
(annotation
name: (identifier) @ann (#match? @ann "^(Get|Post|Put|Delete|Patch)Mapping$")
arguments: (annotation_argument_list (string_literal) @path)))
name: (identifier) @method_name) @method
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: Spring RestTemplate (object-named + method-named) ──────
// RestTemplate.getForObject / getForEntity → GET
// RestTemplate.postForObject / postForEntity → POST
// RestTemplate.put → PUT
// RestTemplate.delete → DELETE
// RestTemplate.patchForObject → PATCH
const REST_TEMPLATE_TO_HTTP: Record<string, string> = {
getForObject: 'GET',
getForEntity: 'GET',
postForObject: 'POST',
postForEntity: 'POST',
put: 'PUT',
delete: 'DELETE',
patchForObject: 'PATCH',
};
interface RestTemplateMeta {
framework: 'spring-rest-template';
}
const REST_TEMPLATE_PATTERNS = compilePatterns({
name: 'java-rest-template',
language: Java,
patterns: [
{
meta: { framework: 'spring-rest-template' },
query: `
(method_invocation
object: (identifier) @obj (#eq? @obj "restTemplate")
name: (identifier) @method
arguments: (argument_list . (string_literal) @path))
`,
},
],
} satisfies LanguagePatterns<RestTemplateMeta>);
// ─── Consumer: Spring WebClient — webClient.method(HttpMethod.X, "path") ─
const WEB_CLIENT_PATTERNS = compilePatterns({
name: 'java-web-client',
language: Java,
patterns: [
{
meta: {},
query: `
(method_invocation
object: (identifier) @obj (#eq? @obj "webClient")
name: (identifier) @method (#eq? @method "method")
arguments: (argument_list
(field_access
object: (identifier) @httpMethodCls (#eq? @httpMethodCls "HttpMethod")
field: (identifier) @http_method)
(string_literal) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: OkHttp `new Request.Builder().url("path")` ─────────────
// Note: `Request.Builder` is a `scoped_type_identifier` whose text includes
// the dot, so `#eq?` against the literal string matches cleanly (no need
// to escape a regex dot).
const OK_HTTP_PATTERNS = compilePatterns({
name: 'java-okhttp',
language: Java,
patterns: [
{
meta: {},
query: `
(method_invocation
object: (object_creation_expression
type: (scoped_type_identifier) @type (#eq? @type "Request.Builder"))
name: (identifier) @method (#eq? @method "url")
arguments: (argument_list . (string_literal) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
/**
* Find the nearest enclosing class_declaration ancestor for a node, or
* null if the node is top-level. Tree-sitter's SyntaxNode.parent walks
* one level at a time.
*/
function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
let cur: Parser.SyntaxNode | null = node.parent;
while (cur) {
if (cur.type === 'class_declaration') return cur;
cur = cur.parent;
}
return null;
}
/**
* Join a class-level prefix and a method-level path into a single URL
* path. Mirrors the semantics of the original regex implementation:
* strip trailing slashes on the prefix, then ensure a single slash
* between prefix and method path.
*/
function joinPath(prefix: string, methodPath: string): string {
const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, '');
const cleanSub = methodPath.replace(/^\/+/, '');
if (!cleanPrefix) return `/${cleanSub}`;
return `/${cleanPrefix}/${cleanSub}`;
}
export const JAVA_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'java-http',
language: Java,
scan(tree) {
const out: HttpDetection[] = [];
// ─── Providers: Spring class prefix + method annotations ────────
const prefixByClassId = new Map<number, string>();
for (const match of runCompiledPatterns(SPRING_CLASS_PREFIX_PATTERNS, tree)) {
const prefixNode = match.captures.prefix;
const classNode = match.captures.class;
if (!prefixNode || !classNode) continue;
const prefix = unquoteLiteral(prefixNode.text);
if (prefix !== null) prefixByClassId.set(classNode.id, prefix);
}
for (const match of runCompiledPatterns(SPRING_METHOD_ROUTE_PATTERNS, tree)) {
const annNode = match.captures.ann;
const pathNode = match.captures.path;
const nameNode = match.captures.method_name;
const methodNode = match.captures.method;
if (!annNode || !pathNode || !methodNode) continue;
const httpMethod = METHOD_ANNOTATION_TO_HTTP[annNode.text];
if (!httpMethod) continue;
const rawPath = unquoteLiteral(pathNode.text);
if (rawPath === null) continue;
const enclosingClass = findEnclosingClass(methodNode);
const prefix = enclosingClass ? (prefixByClassId.get(enclosingClass.id) ?? '') : '';
const fullPath = joinPath(prefix, rawPath);
out.push({
role: 'provider',
framework: 'spring',
method: httpMethod,
path: fullPath,
name: nameNode?.text ?? null,
confidence: 0.8,
});
}
// ─── Consumers: RestTemplate ────────────────────────────────────
for (const match of runCompiledPatterns(REST_TEMPLATE_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const httpMethod = REST_TEMPLATE_TO_HTTP[methodNode.text];
if (!httpMethod) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'spring-rest-template',
method: httpMethod,
path,
name: null,
confidence: 0.7,
});
}
// ─── Consumers: WebClient.method(HttpMethod.X, "path") ──────────
for (const match of runCompiledPatterns(WEB_CLIENT_PATTERNS, tree)) {
const httpMethodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!httpMethodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'spring-web-client',
method: httpMethodNode.text.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// ─── Consumers: OkHttp Request.Builder().url("path") ────────────
for (const match of runCompiledPatterns(OK_HTTP_PATTERNS, tree)) {
const pathNode = match.captures.path;
if (!pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'okhttp',
method: 'GET',
path,
name: null,
confidence: 0.7,
});
}
return out;
},
};

View file

@ -0,0 +1,502 @@
import Parser from 'tree-sitter';
import JavaScript from 'tree-sitter-javascript';
import TypeScript from 'tree-sitter-typescript';
import {
compilePatterns,
runCompiledPatterns,
unquoteLiteral,
type CompiledPatterns,
type LanguagePatterns,
type PatternSpec,
} from '../tree-sitter-scanner.js';
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
/**
* Node.js / TypeScript HTTP plugin family. Handles:
* - NestJS `@Controller('prefix')` classes with `@Get(':id')` methods
* - Express `router.get(...)` / `app.post(...)` providers
* - `fetch(url)` / `fetch(url, { method: 'POST' })` consumers
* - `axios.get(url)` / `axios.delete(url)` consumers
* - `axios({ method, url })` object-form consumers
* - jQuery `$.get(url)` / `$.post(url, ...)` shorthand consumers
* - jQuery `$.ajax({ url, method | type })` consumers
*
* Because the JavaScript and TypeScript tree-sitter grammars share
* node type names for every construct we query, pattern sources are
* defined once and compiled against each grammar variant. The plugin
* exports three `HttpLanguagePlugin`s (JS, TS, TSX) that share the
* same `scan` function but bind to different grammars.
*/
// ─── Provider: NestJS — class-level @Controller('prefix') ────────────
// In tree-sitter-typescript decorators are NOT children of
// class_declaration / method_definition — they're siblings in the
// surrounding class_body / program node. We therefore match the
// decorator standalone and walk to its related class/method in JS.
const NEST_CONTROLLER_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(decorator
(call_expression
function: (identifier) @dec (#eq? @dec "Controller")
arguments: (arguments . [(string) (template_string)] @prefix))) @ctrl_decorator
`,
};
// ─── Provider: NestJS — method-level @Get/@Post/... decorators ───────
// Matches either `@Get('path')` or `@Get()`. The `@path` capture is
// optional — when the first argument isn't a string, the plugin falls
// back to '/' for the method-level path.
const NEST_METHOD_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(decorator
(call_expression
function: (identifier) @dec (#match? @dec "^(Get|Post|Put|Delete|Patch)$")
arguments: (arguments) @args)) @method_decorator
`,
};
// ─── Provider: Express — router.get/app.post/... ─────────────────────
const EXPRESS_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#match? @obj "^(router|app)$")
property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$"))
arguments: (arguments . [(string) (template_string)] @path))
`,
};
// ─── Consumer: fetch(url) with NO options ─────────────────────────────
const FETCH_NO_OPTIONS_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (identifier) @fn (#eq? @fn "fetch")
arguments: (arguments . [(string) (template_string)] @path .))
`,
};
// ─── Consumer: fetch(url, { method: 'X', ... }) ──────────────────────
const FETCH_WITH_OPTIONS_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (identifier) @fn (#eq? @fn "fetch")
arguments: (arguments
. [(string) (template_string)] @path
(object
(pair
key: (property_identifier) @key (#eq? @key "method")
value: (string) @http_method))))
`,
};
// ─── Consumer: axios.get/post/... ────────────────────────────────────
const AXIOS_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "axios")
property: (property_identifier) @http_method (#match? @http_method "^(get|post|put|delete|patch)$"))
arguments: (arguments . [(string) (template_string)] @path))
`,
};
// ─── Consumer: jQuery shorthand $.get(url) / $.post(url, ...) ────────
// `$` is a valid JS identifier, so tree-sitter parses `$.get(...)` as a
// call_expression whose function is a member_expression on identifier `$`.
const JQUERY_SHORTHAND_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "$")
property: (property_identifier) @http_method (#match? @http_method "^(get|post)$"))
arguments: (arguments . [(string) (template_string)] @path))
`,
};
// ─── Consumer: jQuery $.ajax({ url, method|type }) ───────────────────
// The query captures the options object only; key/value pairs are read
// programmatically via `readStringProp` below, which tolerates any key
// order and accepts either `method:` or `type:` (jQuery supports both).
const JQUERY_AJAX_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "$")
property: (property_identifier) @fn (#eq? @fn "ajax"))
arguments: (arguments (object) @options))
`,
};
// ─── Consumer: axios({ method, url }) object form ────────────────────
// Distinct from AXIOS_SPEC above because the call target is an identifier
// (`axios`) rather than a member expression (`axios.get`). As with the
// jQuery ajax form, option keys are resolved programmatically.
const AXIOS_OBJECT_SPEC: PatternSpec<Record<string, never>> = {
meta: {},
query: `
(call_expression
function: (identifier) @fn (#eq? @fn "axios")
arguments: (arguments (object) @options))
`,
};
interface NodePatternBundle {
controller: CompiledPatterns<Record<string, never>>;
methodDecorator: CompiledPatterns<Record<string, never>>;
express: CompiledPatterns<Record<string, never>>;
fetchNoOptions: CompiledPatterns<Record<string, never>>;
fetchWithOptions: CompiledPatterns<Record<string, never>>;
axios: CompiledPatterns<Record<string, never>>;
jqueryShorthand: CompiledPatterns<Record<string, never>>;
jqueryAjax: CompiledPatterns<Record<string, never>>;
axiosObject: CompiledPatterns<Record<string, never>>;
}
function compileBundle(language: unknown, name: string): NodePatternBundle {
const mk = (spec: PatternSpec<Record<string, never>>, suffix: string) =>
compilePatterns({
name: `${name}-${suffix}`,
language,
patterns: [spec],
} satisfies LanguagePatterns<Record<string, never>>);
return {
controller: mk(NEST_CONTROLLER_SPEC, 'nest-controller'),
methodDecorator: mk(NEST_METHOD_SPEC, 'nest-method-decorator'),
express: mk(EXPRESS_SPEC, 'express'),
fetchNoOptions: mk(FETCH_NO_OPTIONS_SPEC, 'fetch-no-options'),
fetchWithOptions: mk(FETCH_WITH_OPTIONS_SPEC, 'fetch-with-options'),
axios: mk(AXIOS_SPEC, 'axios'),
jqueryShorthand: mk(JQUERY_SHORTHAND_SPEC, 'jquery-shorthand'),
jqueryAjax: mk(JQUERY_AJAX_SPEC, 'jquery-ajax'),
axiosObject: mk(AXIOS_OBJECT_SPEC, 'axios-object'),
};
}
const JAVASCRIPT_BUNDLE = compileBundle(JavaScript, 'javascript-http');
const TYPESCRIPT_BUNDLE = compileBundle(TypeScript.typescript, 'typescript-http');
const TSX_BUNDLE = compileBundle(TypeScript.tsx, 'tsx-http');
const NEST_DECORATOR_TO_HTTP: Record<string, string> = {
Get: 'GET',
Post: 'POST',
Put: 'PUT',
Delete: 'DELETE',
Patch: 'PATCH',
};
/**
* Find the nearest enclosing class_declaration for a node, or null.
*/
function findEnclosingClass(node: Parser.SyntaxNode): Parser.SyntaxNode | null {
let cur: Parser.SyntaxNode | null = node.parent;
while (cur) {
if (cur.type === 'class_declaration') return cur;
cur = cur.parent;
}
return null;
}
function joinPath(prefix: string, sub: string): string {
const cleanPrefix = prefix.replace(/^\/+/, '').replace(/\/+$/, '');
const cleanSub = sub.replace(/^\/+/, '');
if (!cleanPrefix) return `/${cleanSub}`;
return `/${cleanPrefix}/${cleanSub}`;
}
/**
* Walk `pair` children of an `object` literal and return the unquoted
* string/template_string value for the first pair whose key matches one
* of `keyNames`. Returns null when no matching pair is present or the
* value is not a string literal. Used by the jQuery ajax / axios object
* consumers to resolve `url` / `method` / `type` keys in any order.
*/
function readStringProp(objectNode: Parser.SyntaxNode, keyNames: readonly string[]): string | null {
for (let i = 0; i < objectNode.namedChildCount; i++) {
const pair = objectNode.namedChild(i);
if (!pair || pair.type !== 'pair') continue;
const keyNode = pair.childForFieldName('key');
const valueNode = pair.childForFieldName('value');
if (!keyNode || !valueNode) continue;
if (!keyNames.includes(keyNode.text)) continue;
if (valueNode.type !== 'string' && valueNode.type !== 'template_string') continue;
const lit = unquoteLiteral(valueNode.text);
if (lit !== null) return lit;
}
return null;
}
/**
* For a standalone `decorator` node (child of class_body / program),
* find the related `class_declaration` node that it decorates. In
* tree-sitter-typescript the decorator is placed before the class
* declaration as a sibling (when decorating a class) or inside the
* class_body before a method_definition (when decorating a method);
* we walk the parent chain until we find the enclosing class.
*/
function findDecoratedClass(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null {
const parent = decoratorNode.parent;
if (!parent) return null;
// Case 1: decorator is a sibling of the class_declaration at program /
// export_statement level. Walk forward through siblings until we find
// the class_declaration this decorator belongs to.
for (let i = 0; i < parent.namedChildCount; i++) {
const child = parent.namedChild(i);
if (child && child.id === decoratorNode.id) {
for (let j = i + 1; j < parent.namedChildCount; j++) {
const next = parent.namedChild(j);
if (!next) continue;
if (next.type === 'decorator') continue; // adjacent decorators stack
if (next.type === 'class_declaration') return next;
if (next.type === 'export_statement') {
// `export class Foo { ... }` wraps the declaration.
for (let k = 0; k < next.namedChildCount; k++) {
const inner = next.namedChild(k);
if (inner?.type === 'class_declaration') return inner;
}
}
break;
}
break;
}
}
// Case 2: decorator is inside a class_body (decorating a method) —
// walk up to the enclosing class_declaration.
return findEnclosingClass(decoratorNode);
}
/**
* For a method-level decorator node (child of class_body before a
* method_definition), find the method_definition it decorates.
*/
function findDecoratedMethod(decoratorNode: Parser.SyntaxNode): Parser.SyntaxNode | null {
const parent = decoratorNode.parent;
if (!parent || parent.type !== 'class_body') return null;
for (let i = 0; i < parent.namedChildCount; i++) {
const child = parent.namedChild(i);
if (child && child.id === decoratorNode.id) {
for (let j = i + 1; j < parent.namedChildCount; j++) {
const next = parent.namedChild(j);
if (!next) continue;
if (next.type === 'decorator') continue;
if (next.type === 'method_definition') return next;
return null;
}
return null;
}
}
return null;
}
function scanBundle(bundle: NodePatternBundle, tree: Parser.Tree): HttpDetection[] {
const out: HttpDetection[] = [];
// NestJS: collect `@Controller('prefix')` class decorators, keyed by
// the `class_declaration` they decorate.
const prefixByClassId = new Map<number, string>();
for (const match of runCompiledPatterns(bundle.controller, tree)) {
const prefixNode = match.captures.prefix;
const decoratorNode = match.captures.ctrl_decorator;
if (!prefixNode || !decoratorNode) continue;
const prefix = unquoteLiteral(prefixNode.text);
if (prefix === null) continue;
const classNode = findDecoratedClass(decoratorNode);
if (!classNode) continue;
prefixByClassId.set(classNode.id, prefix);
}
// NestJS: method-level @Get/@Post/... decorators. The decorator's
// arguments list may be empty (`@Get()`), a string (`@Get('path')`),
// or something else (which we skip).
for (const match of runCompiledPatterns(bundle.methodDecorator, tree)) {
const decNode = match.captures.dec;
const argsNode = match.captures.args;
const decoratorNode = match.captures.method_decorator;
if (!decNode || !argsNode || !decoratorNode) continue;
const httpMethod = NEST_DECORATOR_TO_HTTP[decNode.text];
if (!httpMethod) continue;
const methodNode = findDecoratedMethod(decoratorNode);
if (!methodNode) continue;
const enclosingClass = findEnclosingClass(methodNode);
// Only emit NestJS detections when the class actually has a
// @Controller decorator — without it, the match is almost certainly
// something else (e.g. an unrelated library using similar names).
if (!enclosingClass || !prefixByClassId.has(enclosingClass.id)) continue;
const prefix = prefixByClassId.get(enclosingClass.id) ?? '';
let rawPath = '/';
const firstArg = argsNode.namedChild(0);
if (firstArg && (firstArg.type === 'string' || firstArg.type === 'template_string')) {
const unquoted = unquoteLiteral(firstArg.text);
if (unquoted !== null) rawPath = unquoted;
}
// Get the method name from the decorated method_definition.
const methodNameNode = methodNode.childForFieldName('name');
const name = methodNameNode?.text ?? null;
out.push({
role: 'provider',
framework: 'nest',
method: httpMethod,
path: joinPath(prefix, rawPath),
name,
confidence: 0.8,
});
}
// Express: router/app.<verb>(...)
for (const match of runCompiledPatterns(bundle.express, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'provider',
framework: 'express',
method: methodNode.text.toUpperCase(),
path,
name: 'handler',
confidence: 0.8,
});
}
// Consumer: fetch with options { method: 'X' }
const fetchSeen = new Set<number>();
for (const match of runCompiledPatterns(bundle.fetchWithOptions, tree)) {
const pathNode = match.captures.path;
const methodNode = match.captures.http_method;
if (!pathNode || !methodNode) continue;
const path = unquoteLiteral(pathNode.text);
const method = unquoteLiteral(methodNode.text);
if (path === null || method === null) continue;
fetchSeen.add(pathNode.id);
out.push({
role: 'consumer',
framework: 'fetch',
method: method.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// Consumer: plain fetch(path) — default GET. Skip path nodes we already
// matched with the options variant so we don't double-emit.
for (const match of runCompiledPatterns(bundle.fetchNoOptions, tree)) {
const pathNode = match.captures.path;
if (!pathNode) continue;
if (fetchSeen.has(pathNode.id)) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'fetch',
method: 'GET',
path,
name: null,
confidence: 0.7,
});
}
// Consumer: axios.<verb>(url)
for (const match of runCompiledPatterns(bundle.axios, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'axios',
method: methodNode.text.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// Consumer: jQuery shorthand $.get(url) / $.post(url, ...)
for (const match of runCompiledPatterns(bundle.jqueryShorthand, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'jquery',
method: methodNode.text.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// Consumer: jQuery $.ajax({ url, method|type }). jQuery accepts either
// `method:` or `type:`; both default to GET when absent.
for (const match of runCompiledPatterns(bundle.jqueryAjax, tree)) {
const optionsNode = match.captures.options;
if (!optionsNode) continue;
const path = readStringProp(optionsNode, ['url']);
if (path === null) continue;
const rawMethod = readStringProp(optionsNode, ['method', 'type']);
const method = (rawMethod ?? 'GET').toUpperCase();
out.push({
role: 'consumer',
framework: 'jquery',
method,
path,
name: null,
confidence: 0.7,
});
}
// Consumer: axios({ method, url }) object form. Structurally distinct
// from axios.<verb>(url) (identifier vs member_expression call), so no
// dedup against the member-form loop above is required.
for (const match of runCompiledPatterns(bundle.axiosObject, tree)) {
const optionsNode = match.captures.options;
if (!optionsNode) continue;
const path = readStringProp(optionsNode, ['url']);
if (path === null) continue;
const rawMethod = readStringProp(optionsNode, ['method']);
const method = (rawMethod ?? 'GET').toUpperCase();
out.push({
role: 'consumer',
framework: 'axios',
method,
path,
name: null,
confidence: 0.7,
});
}
return out;
}
export const JAVASCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'javascript-http',
language: JavaScript,
scan: (tree) => scanBundle(JAVASCRIPT_BUNDLE, tree),
};
export const TYPESCRIPT_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'typescript-http',
language: TypeScript.typescript,
scan: (tree) => scanBundle(TYPESCRIPT_BUNDLE, tree),
};
export const TSX_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'tsx-http',
language: TypeScript.tsx,
scan: (tree) => scanBundle(TSX_BUNDLE, tree),
};

View file

@ -0,0 +1,79 @@
import PHP from 'tree-sitter-php';
import {
compilePatterns,
runCompiledPatterns,
unquoteLiteral,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
/**
* PHP HTTP plugin Laravel `Route::get/post/...` declarations.
*
* The pipeline already uses `PHP.php_only` for ingesting plain `.php`
* files (see `core/tree-sitter/parser-loader.ts`), and we do the same
* here so Laravel route files are parsed with the right grammar dialect.
*/
const LARAVEL_PATTERNS = compilePatterns({
name: 'php-laravel',
language: PHP.php_only,
patterns: [
{
meta: {},
query: `
(scoped_call_expression
scope: (name) @scope (#eq? @scope "Route")
name: (name) @method (#match? @method "^(get|post|put|delete|patch)$")
arguments: (arguments . (argument (string) @path)))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
/**
* Extract the inner text of a PHP `string` node. The tree-sitter-php
* grammar wraps single / double-quoted literals differently depending
* on content; we try both the raw `text` (with quotes) through
* `unquoteLiteral`, and a fallback via the `string_value` / `string_content`
* child nodes.
*/
function phpStringText(node: import('tree-sitter').SyntaxNode): string | null {
// Most single-quoted strings expose their inner content through the
// full node text (including quotes), which unquoteLiteral strips.
const direct = unquoteLiteral(node.text);
if (direct !== null && direct !== node.text) return direct;
// Fall back to child string_content / string_value node if present.
for (const child of node.children) {
if (child.type === 'string_content' || child.type === 'string_value') {
return child.text;
}
}
return direct;
}
export const PHP_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'php-http',
language: PHP.php_only,
scan(tree) {
const out: HttpDetection[] = [];
for (const match of runCompiledPatterns(LARAVEL_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = phpStringText(pathNode);
if (path === null) continue;
out.push({
role: 'provider',
framework: 'laravel',
method: methodNode.text.toUpperCase(),
path,
name: 'route',
confidence: 0.8,
});
}
return out;
},
};

View file

@ -0,0 +1,142 @@
import Python from 'tree-sitter-python';
import {
compilePatterns,
runCompiledPatterns,
unquoteLiteral,
type LanguagePatterns,
} from '../tree-sitter-scanner.js';
import type { HttpDetection, HttpLanguagePlugin } from './types.js';
/**
* Python HTTP plugin. Handles:
* - FastAPI `@app.get("/path")` provider decorators
* - `requests.get/post/...("url")` consumer calls
* - Generic `requests.request("METHOD", "url")` consumer calls
*/
const FASTAPI_VERBS: Record<string, string> = {
get: 'GET',
post: 'POST',
put: 'PUT',
delete: 'DELETE',
patch: 'PATCH',
};
// ─── Provider: FastAPI @app.get/... ──────────────────────────────────
const FASTAPI_PATTERNS = compilePatterns({
name: 'python-fastapi',
language: Python,
patterns: [
{
meta: {},
query: `
(decorator
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "app")
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
arguments: (argument_list . (string) @path)))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: requests.get/post/... ──────────────────────────────────
const REQUESTS_VERB_PATTERNS = compilePatterns({
name: 'python-requests-verb',
language: Python,
patterns: [
{
meta: {},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "requests")
attribute: (identifier) @method (#match? @method "^(get|post|put|delete|patch)$"))
arguments: (argument_list . (string) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
// ─── Consumer: requests.request("METHOD", "url") ─────────────────────
const REQUESTS_GENERIC_PATTERNS = compilePatterns({
name: 'python-requests-generic',
language: Python,
patterns: [
{
meta: {},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "requests")
attribute: (identifier) @method (#eq? @method "request"))
arguments: (argument_list . (string) @http_method (string) @path))
`,
},
],
} satisfies LanguagePatterns<Record<string, never>>);
export const PYTHON_HTTP_PLUGIN: HttpLanguagePlugin = {
name: 'python-http',
language: Python,
scan(tree) {
const out: HttpDetection[] = [];
// Providers: FastAPI
for (const match of runCompiledPatterns(FASTAPI_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const httpMethod = FASTAPI_VERBS[methodNode.text];
if (!httpMethod) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'provider',
framework: 'fastapi',
method: httpMethod,
path,
name: null,
confidence: 0.8,
});
}
// Consumers: requests.<verb>
for (const match of runCompiledPatterns(REQUESTS_VERB_PATTERNS, tree)) {
const methodNode = match.captures.method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const path = unquoteLiteral(pathNode.text);
if (path === null) continue;
out.push({
role: 'consumer',
framework: 'python-requests',
method: methodNode.text.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
// Consumers: requests.request("METHOD", "url")
for (const match of runCompiledPatterns(REQUESTS_GENERIC_PATTERNS, tree)) {
const methodNode = match.captures.http_method;
const pathNode = match.captures.path;
if (!methodNode || !pathNode) continue;
const methodRaw = unquoteLiteral(methodNode.text);
const path = unquoteLiteral(pathNode.text);
if (methodRaw === null || path === null) continue;
out.push({
role: 'consumer',
framework: 'python-requests',
method: methodRaw.toUpperCase(),
path,
name: null,
confidence: 0.7,
});
}
return out;
},
};

View file

@ -0,0 +1,65 @@
import type Parser from 'tree-sitter';
/**
* Shared types for the http-route-extractor language plugins.
*
* Each plugin lives in its own file (java.ts, node.ts, ...) and owns
* the tree-sitter grammar import + queries. The top-level
* `http-route-extractor.ts` orchestrator only knows about this type
* module and the plugin registry (`./index.ts`). It MUST NOT import
* any grammar or query text directly language-specific knowledge
* belongs in the plugins.
*/
export type HttpRole = 'provider' | 'consumer';
/**
* One raw HTTP detection produced by a plugin's `scan()` function. The
* orchestrator converts this into a full `ExtractedContract` by running
* path normalization and building the contract id.
*
* `path` is the raw literal string as it appeared in source (with
* `${...}` template placeholders still in place); the orchestrator
* runs the appropriate normalizer for provider vs. consumer paths.
*/
export interface HttpDetection {
role: HttpRole;
/** Short framework label, e.g. `'spring'`, `'nest'`, `'express'`. */
framework: string;
/** HTTP method in upper case (`'GET'`, `'POST'`, ...). */
method: string;
/** Raw path literal as seen in source (template placeholders intact). */
path: string;
/**
* Symbol name of the handler (for providers) or calling function
* (for consumers) when the plugin can determine it structurally.
* Null when no good candidate is available.
*/
name: string | null;
/** Confidence in (0, 1]. Source-scan plugins typically use 0.70.8. */
confidence: number;
}
/**
* One language-scoped HTTP plugin. The plugin owns the tree-sitter
* grammar and the `scan` function that translates a parsed tree into
* zero or more `HttpDetection`s. Plugins are free to run multiple
* compiled pattern bundles internally (see the shared scanner's
* `runCompiledPatterns` helper).
*
* `language` is typed as `unknown` for the same reason as
* `LanguagePatterns.language` in `tree-sitter-scanner.ts` the
* grammar modules export different shapes.
*/
export interface HttpLanguagePlugin {
/** Human-readable plugin name for diagnostics. */
name: string;
/** tree-sitter grammar object (passed to the shared parser). */
language: unknown;
/**
* Scan a parsed tree and return zero or more HTTP detections. Plugins
* must not throw they should swallow per-match errors so a single
* malformed construct does not abort the whole file.
*/
scan(tree: Parser.Tree): HttpDetection[];
}

View file

@ -1,8 +1,34 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { glob } from 'glob';
import Parser from 'tree-sitter';
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
import type { ExtractedContract, RepoHandle } from '../types.js';
import { readSafe } from './fs-utils.js';
import { getPluginForFile, HTTP_SCAN_GLOB, type HttpDetection } from './http-patterns/index.js';
/**
* Language-agnostic orchestrator for HTTP route (provider + consumer)
* contract extraction. Two strategies, in order of preference per role:
*
* 1. **Graph-assisted (Strategy A)** if a per-repo LadybugDB executor
* is available, read `HANDLES_ROUTE` / `FETCHES` Cypher edges that
* the ingestion pipeline already produced via tree-sitter. This is
* the preferred path because the graph has richer symbol metadata
* (real uids, class/method structure, etc.).
*
* 2. **Source-scan fallback (Strategy B)** parse files directly with
* the per-language plugin registry in `./http-patterns/`. Used when
* the graph has no routes/fetches for this repo (e.g. a repo that
* hasn't been indexed yet, or whose indexer doesn't know the
* framework). Each plugin owns its tree-sitter grammar and query
* sources this orchestrator imports NO grammars or query strings.
*
* Adding a new language for Strategy B is a one-file edit in
* `http-patterns/index.ts`: register a new `HttpLanguagePlugin` and
* widen `HTTP_SCAN_GLOB` if needed.
*/
// ─── Graph-assisted queries ──────────────────────────────────────────
const HANDLES_ROUTE_QUERY = `
MATCH (handlerFile:File)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route)
@ -23,14 +49,56 @@ WHERE sym.startLine IS NOT NULL
RETURN sym.id AS uid, sym.name AS name, sym.filePath AS filePath, labels(sym) AS labels
ORDER BY sym.startLine`;
// ─── Path normalization (shared between provider / consumer paths) ──
/**
* Canonicalize a provider-side HTTP path for contract-id generation:
* - strip query string
* - lower-case
* - drop trailing slash
* - collapse `:id`, `{id}`, `[id]` path params into a single `{param}`
*/
export function normalizeHttpPath(p: string): string {
let s = p.trim().split('?')[0].toLowerCase().replace(/\/+$/, '');
s = s.replace(/:\w+/g, '{param}');
s = s.replace(/\{[^}]+\}/g, '{param}');
s = s.replace(/\[[^\]]+\]/g, '{param}');
return s;
// Preserve root: after stripping trailing slashes, the root "/"
// collapses to "" which would produce malformed contract ids like
// `http::GET::`. Restore a single slash for the root case.
return s === '' ? '/' : s;
}
/**
* Consumer-side normalization is more aggressive:
* - template literals (`${x}`) `{param}`
* - strip protocol + host if the URL is absolute
* - numeric segments `{param}` (so `/api/orders/42` `/api/orders/{param}`)
*/
function normalizeConsumerPath(url: string): string {
const templated = url.replace(/\$\{[^}]+\}/g, '{param}').trim();
let pathOnly = templated;
if (/^https?:\/\//i.test(templated)) {
try {
pathOnly = new URL(templated).pathname;
} catch {
pathOnly = templated.replace(/^https?:\/\/[^/]+/i, '');
}
}
const normalized = normalizeHttpPath(pathOnly || '/');
const segments = normalized
.split('/')
.filter(Boolean)
.map((segment) => (/^\d+$/.test(segment) ? '{param}' : segment));
return `/${segments.join('/')}`.replace(/\/+$/, '') || '/';
}
function contractIdFor(method: string, pathNorm: string): string {
return `http::${method.toUpperCase()}::${pathNorm}`;
}
// ─── Graph row helpers ───────────────────────────────────────────────
function methodFromRouteReason(reason: string): string | null {
const r = reason || '';
if (/GetMapping|decorator-Get/i.test(r)) return 'GET';
@ -41,50 +109,6 @@ function methodFromRouteReason(reason: string): string | null {
return null;
}
function contractIdFor(method: string, pathNorm: string): string {
return `http::${method.toUpperCase()}::${pathNorm}`;
}
function readSafe(repoPath: string, rel: string): string | null {
const abs = path.resolve(repoPath, rel);
const base = path.resolve(repoPath);
const relToBase = path.relative(base, abs);
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
try {
return fs.readFileSync(abs, 'utf-8');
} catch {
return null;
}
}
function pickJavaHandlerName(
content: string,
routePath: string,
httpMethod: string,
): string | null {
const tail = routePath.split('/').filter(Boolean).pop() || '';
const mapNames: Record<string, string> = {
GET: 'GetMapping',
POST: 'PostMapping',
PUT: 'PutMapping',
DELETE: 'DeleteMapping',
PATCH: 'PatchMapping',
};
const ann = mapNames[httpMethod] || 'GetMapping';
const lines = content.split(/\r?\n/);
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (!line.includes(`@${ann}`)) continue;
if (!line.includes(`"${tail}"`) && !line.includes(`'${tail}'`) && tail && !line.includes(tail))
continue;
for (let j = i + 1; j < Math.min(i + 8, lines.length); j++) {
const m = lines[j].match(/(?:public|protected|private)\s+[\w<>,\s\[\]]+\s+(\w+)\s*\(/);
if (m) return m[1];
}
}
return null;
}
function pickSymbolUid(
rows: Record<string, unknown>[],
preferredName: string | null,
@ -114,6 +138,8 @@ function pickSymbolUid(
};
}
// ─── Orchestrator ────────────────────────────────────────────────────
export class HttpRouteExtractor implements ContractExtractor {
type = 'http' as const;
@ -124,20 +150,76 @@ export class HttpRouteExtractor implements ContractExtractor {
async extract(
dbExecutor: CypherExecutor | null,
repoPath: string,
repo: RepoHandle,
_repo: RepoHandle,
): Promise<ExtractedContract[]> {
const graphP = dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, repoPath) : [];
const providers = graphP.length > 0 ? graphP : await this.extractProvidersSourceScan(repoPath);
// Parse each file at most once and reuse the plugin results across
// both graph-assisted enrichment and source-scan emission.
const parser = new Parser();
const cachedDetections = new Map<string, HttpDetection[]>();
const getDetections = (rel: string): HttpDetection[] => {
const cached = cachedDetections.get(rel);
if (cached) return cached;
const plugin = getPluginForFile(rel);
if (!plugin) {
cachedDetections.set(rel, []);
return [];
}
const content = readSafe(repoPath, rel);
if (!content) {
cachedDetections.set(rel, []);
return [];
}
try {
parser.setLanguage(plugin.language);
const tree = parser.parse(content);
const detections = plugin.scan(tree);
cachedDetections.set(rel, detections);
return detections;
} catch {
cachedDetections.set(rel, []);
return [];
}
};
const graphC = dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, repoPath) : [];
const consumers = graphC.length > 0 ? graphC : await this.extractConsumersSourceScan(repoPath);
// Glob the source-scan file list at most once per extract() —
// both provider and consumer fallback paths share the same list.
let scannedFiles: string[] | null = null;
const getScannedFiles = async (): Promise<string[]> => {
if (scannedFiles) return scannedFiles;
scannedFiles = await this.scanFiles(repoPath);
return scannedFiles;
};
const graphProviders =
dbExecutor != null ? await this.extractProvidersGraph(dbExecutor, getDetections) : [];
const providers =
graphProviders.length > 0
? graphProviders
: this.extractProvidersSourceScan(await getScannedFiles(), getDetections);
const graphConsumers =
dbExecutor != null ? await this.extractConsumersGraph(dbExecutor, getDetections) : [];
const consumers =
graphConsumers.length > 0
? graphConsumers
: this.extractConsumersSourceScan(await getScannedFiles(), getDetections);
return [...providers, ...consumers];
}
private async scanFiles(repoPath: string): Promise<string[]> {
return glob(HTTP_SCAN_GLOB, {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**', '**/vendor/**'],
nodir: true,
});
}
// ─── Graph-assisted providers ──────────────────────────────────────
private async extractProvidersGraph(
db: CypherExecutor,
repoPath: string,
getDetections: (rel: string) => HttpDetection[],
): Promise<ExtractedContract[]> {
const out: ExtractedContract[] = [];
let rows: Record<string, unknown>[];
@ -152,22 +234,55 @@ export class HttpRouteExtractor implements ContractExtractor {
const routePath = String(row.routePath ?? '');
const routeSource = String(row.routeSource ?? row.routeReason ?? '');
let method = methodFromRouteReason(routeSource);
const content = readSafe(repoPath, filePath);
if (!method && content) {
method = this.inferMethodFromFileScan(content, routePath, 'provider');
// Look up handler name (and backfill method if missing) from the
// plugin's scan of the handler file. This replaces the old
// regex-based `inferMethodFromFileScan` and `pickJavaHandlerName`
// helpers — tree-sitter gives both pieces of information
// structurally. Always run the lookup: even when method is set by
// `methodFromRouteReason`, we still need the handler name.
const detections = filePath ? getDetections(filePath) : [];
const providerDetections = detections.filter((d) => d.role === 'provider');
let handlerName: string | null = null;
const normalizedRoute = normalizeHttpPath(routePath);
// Candidates share the same normalized path. When multiple
// detections at the same path exist (e.g. GET + POST /api/orders
// in one router), a blind `.find()` silently returned the first
// verb — attaching the wrong handler and, when method was not
// already pinned by the route reason, the wrong method too.
// Disambiguate by method when we know it; refuse to guess when
// we don't.
const candidates = providerDetections.filter(
(d) => normalizeHttpPath(d.path) === normalizedRoute,
);
let match: (typeof candidates)[number] | undefined;
const ambiguousCandidates = !method && candidates.length > 1;
if (method) {
match = candidates.find((d) => d.method === method);
} else if (candidates.length === 1) {
match = candidates[0];
}
// else: multiple candidates + unknown method → leave match
// undefined so handlerName stays null and skip symbol
// enrichment below, keeping the file-basename fallback instead
// of letting pickSymbolUid silently pick the first Function /
// Method in the file (which reintroduces the mis-attribution
// we were trying to avoid). Method stays at the conservative
// 'GET' default set below.
if (match) {
if (!method) method = match.method;
handlerName = match.name;
}
if (!method) method = 'GET';
const pathNorm = normalizeHttpPath(routePath);
const cid = contractIdFor(method, pathNorm);
const handlerName =
content && routePath ? pickJavaHandlerName(content, routePath, method) : null;
let symbolUid = '';
let symbolName = path.basename(filePath) || 'handler';
let symPath = filePath;
const fileId = row.fileId ?? row[0];
if (fileId) {
if (fileId && !ambiguousCandidates) {
try {
const syms = await db(CONTAINS_QUERY, { fileId });
if (syms.length > 0) {
@ -201,145 +316,44 @@ export class HttpRouteExtractor implements ContractExtractor {
return out;
}
private inferMethodFromFileScan(
content: string,
routePath: string,
_role: string,
): string | null {
const tail = routePath.split('/').filter(Boolean).pop() || '';
for (const m of ['GET', 'POST', 'PUT', 'DELETE', 'PATCH'] as const) {
const mapNames: Record<string, string> = {
GET: 'GetMapping',
POST: 'PostMapping',
PUT: 'PutMapping',
DELETE: 'DeleteMapping',
PATCH: 'PatchMapping',
};
if (
content.includes(`@${mapNames[m]}`) &&
(content.includes(tail) || routePath.includes(tail))
) {
return m;
}
}
return null;
}
// ─── Source-scan providers ─────────────────────────────────────────
private async extractProvidersSourceScan(repoPath: string): Promise<ExtractedContract[]> {
const files = await glob('**/*.{ts,tsx,js,jsx,java,vue,svelte,php,py}', {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**', '**/dist/**', '**/build/**'],
nodir: true,
});
private extractProvidersSourceScan(
files: string[],
getDetections: (rel: string) => HttpDetection[],
): ExtractedContract[] {
const out: ExtractedContract[] = [];
for (const rel of files) {
const content = readSafe(repoPath, rel);
if (!content) continue;
out.push(...this.scanSpringProviders(content, rel));
out.push(...this.scanExpressProviders(content, rel));
out.push(...this.scanLaravelProviders(content, rel));
out.push(...this.scanFastApiProviders(content, rel));
const detections = getDetections(rel);
for (const d of detections) {
if (d.role !== 'provider') continue;
const pathNorm = normalizeHttpPath(d.path);
out.push({
contractId: contractIdFor(d.method, pathNorm),
type: 'http',
role: 'provider',
symbolUid: '',
symbolRef: { filePath: rel, name: d.name ?? 'handler' },
symbolName: d.name ?? 'handler',
confidence: d.confidence,
meta: {
method: d.method,
path: pathNorm,
pathSegments: pathNorm.split('/').filter(Boolean),
extractionStrategy: 'source_scan',
framework: d.framework,
},
});
}
}
return this.dedupeContracts(out);
}
private dedupeContracts(items: ExtractedContract[]): ExtractedContract[] {
const seen = new Set<string>();
const out: ExtractedContract[] = [];
for (const c of items) {
const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`;
if (seen.has(k)) continue;
seen.add(k);
out.push(c);
}
return out;
}
private scanSpringProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
let classPrefix = '';
const classRm = content.match(/@RequestMapping\s*\(\s*"([^"]+)"/);
if (classRm) classPrefix = classRm[1].replace(/\/+$/, '');
const re = /@(Get|Post|Put|Delete|Patch)Mapping\s*\(\s*"([^"]+)"/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const method = m[1].toUpperCase();
let p = m[2];
if (classPrefix) p = `${classPrefix}/${p.replace(/^\//, '')}`;
const pathNorm = normalizeHttpPath(p);
const sub = content.slice(m.index);
const nameM = sub.match(/(?:public|protected|private)\s+[\w<>,\s\[\]]+\s+(\w+)\s*\(/);
const name = nameM ? nameM[1] : m[0];
out.push(this.makeProvider(filePath, method, pathNorm, name, 0.8));
}
return out;
}
private scanExpressProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
const re = /(?:router|app)\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const method = m[1].toUpperCase();
const pathNorm = normalizeHttpPath(m[2]);
out.push(this.makeProvider(filePath, method, pathNorm, 'handler', 0.8));
}
return out;
}
private scanLaravelProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
const re = /Route::(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const method = m[1].toUpperCase();
const pathNorm = normalizeHttpPath(m[2]);
out.push(this.makeProvider(filePath, method, pathNorm, 'route', 0.8));
}
return out;
}
private scanFastApiProviders(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
const re = /@app\.(get|post|put|delete|patch)\s*\(\s*['"]([^'"]+)['"]/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const method = m[1].toUpperCase();
const pathNorm = normalizeHttpPath(m[2]);
out.push(this.makeProvider(filePath, method, pathNorm, 'handler', 0.8));
}
return out;
}
private makeProvider(
filePath: string,
method: string,
pathNorm: string,
name: string,
confidence: number,
): ExtractedContract {
const cid = contractIdFor(method, pathNorm);
return {
contractId: cid,
type: 'http',
role: 'provider',
symbolUid: '',
symbolRef: { filePath, name },
symbolName: name,
confidence,
meta: {
method,
path: pathNorm,
pathSegments: pathNorm.split('/').filter(Boolean),
extractionStrategy: 'source_scan',
},
};
}
// ─── Graph-assisted consumers ──────────────────────────────────────
private async extractConsumersGraph(
db: CypherExecutor,
repoPath: string,
getDetections: (rel: string) => HttpDetection[],
): Promise<ExtractedContract[]> {
const out: ExtractedContract[] = [];
let rows: Record<string, unknown>[];
@ -353,11 +367,23 @@ export class HttpRouteExtractor implements ContractExtractor {
const routePath = String(row.routePath ?? '');
const pathNorm = normalizeHttpPath(routePath);
let method = 'GET';
const content = readSafe(repoPath, filePath);
if (content) {
const inferred = this.inferFetchMethod(content, pathNorm);
if (inferred) method = inferred;
// Prefer the plugin's detected method if we can find a matching
// fetch/axios call in the same file.
const detections = filePath ? getDetections(filePath) : [];
// Symmetric to the provider path: if multiple consumer calls in
// the same file share the same normalized path (e.g. a GET
// fetch AND a POST fetch to `/api/orders`), `.find()` silently
// picked the first verb and keyed the contract id on the wrong
// method. With no upstream method signal here, refuse to guess
// when candidates are ambiguous — leave `method` at its
// conservative 'GET' default.
const consumerCandidates = detections.filter(
(d) => d.role === 'consumer' && normalizeConsumerPath(d.path) === pathNorm,
);
if (consumerCandidates.length === 1) {
method = consumerCandidates[0].method;
}
const cid = contractIdFor(method, pathNorm);
let symbolUid = '';
let symbolName = 'fetch';
@ -395,81 +421,47 @@ export class HttpRouteExtractor implements ContractExtractor {
return out;
}
private inferFetchMethod(content: string, pathNorm: string): string | null {
const esc = pathNorm.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
const fetchRe = new RegExp(
`fetch\\s*\\(\\s*['"\`]([^'"\`]*${esc}[^'"\`]*)['"\`]\\s*,\\s*\\{[^}]*method:\\s*['"](\\w+)['"]`,
'i',
);
const m = content.match(fetchRe);
if (m) return m[2].toUpperCase();
return null;
}
// ─── Source-scan consumers ─────────────────────────────────────────
private async extractConsumersSourceScan(repoPath: string): Promise<ExtractedContract[]> {
const files = await glob('**/*.{ts,tsx,js,jsx,vue,svelte}', {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**'],
nodir: true,
});
private extractConsumersSourceScan(
files: string[],
getDetections: (rel: string) => HttpDetection[],
): ExtractedContract[] {
const out: ExtractedContract[] = [];
for (const rel of files) {
const content = readSafe(repoPath, rel);
if (!content) continue;
out.push(...this.scanFetchConsumers(content, rel));
out.push(...this.scanAxiosConsumers(content, rel));
const detections = getDetections(rel);
for (const d of detections) {
if (d.role !== 'consumer') continue;
const pathNorm = normalizeConsumerPath(d.path);
out.push({
contractId: contractIdFor(d.method, pathNorm),
type: 'http',
role: 'consumer',
symbolUid: '',
symbolRef: { filePath: rel, name: 'fetch' },
symbolName: 'fetch',
confidence: d.confidence,
meta: {
method: d.method,
path: pathNorm,
extractionStrategy: 'source_scan',
framework: d.framework,
},
});
}
}
return this.dedupeContracts(out);
}
private scanFetchConsumers(content: string, filePath: string): ExtractedContract[] {
private dedupeContracts(items: ExtractedContract[]): ExtractedContract[] {
const seen = new Set<string>();
const out: ExtractedContract[] = [];
const re =
/fetch\s*\(\s*['"`]([^'"`]+)['"`](?:\s*,\s*\{[^}]*method:\s*['"](\w+)['"][^}]*\})?\s*\)/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const pathNorm = normalizeHttpPath(this.templateToPattern(m[1]));
const method = (m[2] || 'GET').toUpperCase();
out.push(this.makeConsumer(filePath, method, pathNorm, 0.7));
for (const c of items) {
const k = `${c.contractId}|${c.symbolRef.filePath}|${c.symbolRef.name}`;
if (seen.has(k)) continue;
seen.add(k);
out.push(c);
}
return out;
}
private templateToPattern(url: string): string {
return url.replace(/\$\{[^}]+\}/g, '{param}');
}
private scanAxiosConsumers(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
const re = /axios\.(get|post|put|delete|patch)\s*\(\s*[`'"]([^`'"]+)[`'"]/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const method = m[1].toUpperCase();
const pathNorm = normalizeHttpPath(this.templateToPattern(m[2]));
out.push(this.makeConsumer(filePath, method, pathNorm, 0.7));
}
return out;
}
private makeConsumer(
filePath: string,
method: string,
pathNorm: string,
confidence: number,
): ExtractedContract {
return {
contractId: contractIdFor(method, pathNorm),
type: 'http',
role: 'consumer',
symbolUid: '',
symbolRef: { filePath, name: 'fetch' },
symbolName: 'fetch',
confidence,
meta: {
method,
path: pathNorm,
extractionStrategy: 'source_scan',
},
};
}
}

View file

@ -0,0 +1,344 @@
import type { ContractType, CrossLink, GroupManifestLink, StoredContract } from '../types.js';
import type { CypherExecutor } from '../contract-extractor.js';
export interface ManifestExtractResult {
contracts: StoredContract[];
crossLinks: CrossLink[];
}
/**
* Canonicalize an HTTP path for matching against Route.name in the graph.
* Mirrors core/ingestion/pipeline.ts ensureSlash semantics:
* - Ensures a leading slash.
* - Strips trailing slashes (except the root "/").
* - Normalizes consecutive slashes.
* - Does NOT lowercase (route matching is case-sensitive).
*/
function normalizeRoutePath(raw: string): string {
const trimmed = raw.trim();
if (!trimmed) return '/';
const withLeading = trimmed.startsWith('/') ? trimmed : `/${trimmed}`;
const collapsed = withLeading.replace(/\/+/g, '/');
if (collapsed === '/') return '/';
return collapsed.replace(/\/+$/, '');
}
/**
* Split a manifest HTTP contract into its optional `METHOD::` prefix and
* its path portion.
*
* `buildContractId` recommends the explicit-method form `GET::/api/orders`
* in group.yaml; if we hand that raw string to `normalizeRoutePath` we get
* `/GET::/api/orders`, which can never match `Route.name = "/api/orders"`
* in the graph. This helper extracts the path portion so the Cypher
* lookup uses the canonical route name.
*
* The method prefix regex mirrors `buildContractId` (line ~251) for
* symmetry: case-insensitive `[A-Za-z]+` followed by `::`. The captured
* method is upper-cased for downstream use; method-constrained matching
* against `HANDLES_ROUTE` is a future enhancement (not yet wired).
*
* Edge cases:
* - `"::/api/orders"` empty method portion, no alpha prefix match, so
* the whole string is treated as a bare path (matches buildContractId
* which also requires `[A-Za-z]+`).
* - `"GET::"` method with empty path, returns `{ method: 'GET', path: '' }`;
* `normalizeRoutePath('')` resolves to `/` for caller.
*/
function parseHttpContract(raw: string): { method: string | null; path: string } {
const match = raw.match(/^([A-Za-z]+)::/);
if (!match) return { method: null, path: raw };
return { method: match[1].toUpperCase(), path: raw.slice(match[0].length) };
}
/**
* Stable synthetic symbolUid for a manifest-declared contract whose target
* symbol could not be resolved against the per-repo graph (resolveSymbol
* returned null). Two reasons we don't leave the uid empty:
*
* 1. The bridge stores Contract nodes keyed in part by symbolUid; an empty
* uid means downstream Cypher queries that anchor on `provider.symbolUid`
* can't tell two different unresolved manifest contracts apart.
* 2. The cross-impact bridge query in cross-impact.ts joins local impact
* results to bridge contracts via `WHERE provider.symbolUid IN $localUids`.
* If the local impact engine produces a deterministic identifier for the
* unresolved target, it must agree with the value the bridge stored. A
* synthetic uid keyed off (repo, contractId) is the only thing both sides
* can derive without knowing about each other.
*
* Format: `manifest::<repo>::<contractId>`. Stable across syncs, scoped to a
* single repo within a group, and never collides with real indexer uids
* (which never start with `manifest::`).
*/
export function manifestSymbolUid(repo: string, contractId: string): string {
return `manifest::${repo}::${contractId}`;
}
export class ManifestExtractor {
async extractFromManifest(
links: GroupManifestLink[],
dbExecutors?: Map<string, CypherExecutor>,
): Promise<ManifestExtractResult> {
// Resolve all (repo, link) pairs in parallel. The previous sequential
// await-per-link produced 2N round-trips; parallel resolution uses the
// per-repo executor pool directly and scales linearly with manifest size.
//
// Memoization: a manifest can list the same contract multiple times
// (e.g. a consumer and provider declaration, or cross-referenced groups).
// Key on (repo, type, contract) — the canonical input to the Cypher
// query — so duplicate links resolve to one DB hit.
type ResolvedSymbol = { filePath: string; name: string; uid: string } | null;
const resolveCache = new Map<string, Promise<ResolvedSymbol>>();
const resolveOnce = (repo: string, link: GroupManifestLink): Promise<ResolvedSymbol> => {
const key = `${repo}\u0000${link.type}\u0000${link.contract}`;
let pending = resolveCache.get(key);
if (!pending) {
pending = this.resolveSymbol(repo, link, dbExecutors);
resolveCache.set(key, pending);
}
return pending;
};
const perLink = await Promise.all(
links.map(async (link) => {
const contractId = this.buildContractId(link.type, link.contract);
const providerRepo = link.role === 'provider' ? link.from : link.to;
const consumerRepo = link.role === 'provider' ? link.to : link.from;
const [providerSymbol, consumerSymbol] = await Promise.all([
resolveOnce(providerRepo, link),
resolveOnce(consumerRepo, link),
]);
return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol };
}),
);
const contracts: StoredContract[] = [];
const crossLinks: CrossLink[] = [];
for (const {
link,
contractId,
providerRepo,
consumerRepo,
providerSymbol,
consumerSymbol,
} of perLink) {
const providerRef = providerSymbol || { filePath: '', name: link.contract };
const consumerRef = consumerSymbol || { filePath: '', name: link.contract };
// When the resolver finds a real graph symbol we keep its uid, otherwise
// fall back to the deterministic synthetic uid (see manifestSymbolUid).
const providerUid = providerSymbol?.uid || manifestSymbolUid(providerRepo, contractId);
const consumerUid = consumerSymbol?.uid || manifestSymbolUid(consumerRepo, contractId);
contracts.push({
contractId,
type: link.type,
role: 'provider',
symbolUid: providerUid,
symbolRef: providerRef,
symbolName: link.contract,
confidence: 1.0,
meta: { source: 'manifest' },
repo: providerRepo,
});
contracts.push({
contractId,
type: link.type,
role: 'consumer',
symbolUid: consumerUid,
symbolRef: consumerRef,
symbolName: link.contract,
confidence: 1.0,
meta: { source: 'manifest' },
repo: consumerRepo,
});
crossLinks.push({
from: { repo: consumerRepo, symbolUid: consumerUid, symbolRef: consumerRef },
to: { repo: providerRepo, symbolUid: providerUid, symbolRef: providerRef },
type: link.type,
contractId,
matchType: 'manifest',
confidence: 1.0,
});
}
return { contracts, crossLinks };
}
private async resolveSymbol(
repoPathKey: string,
link: GroupManifestLink,
dbExecutors?: Map<string, CypherExecutor>,
): Promise<{ filePath: string; name: string; uid: string } | null> {
const executor = dbExecutors?.get(repoPathKey);
if (!executor) return null;
// NOTE: All lookups use EXACT equality on the relevant name field and
// deterministic ORDER BY before LIMIT 1. Previous versions used CONTAINS
// for fuzzy matching (plus an unconditional ".proto" fallback for gRPC)
// which produced silent false positives: e.g. manifest "/orders" would
// match "/suborders", and a gRPC manifest entry in a repo with any
// .proto file would attach to a random proto symbol.
//
// If resolveSymbol returns null, the extractor falls back to a
// deterministic synthetic uid via `manifestSymbolUid(repo, contractId)`
// (see the function's docstring for why synthetic rather than empty).
// Cross-impact still works: the bridge query joins on the synthetic
// uid, and the local impact engine derives the same uid for the
// unresolved symbol — name-based hints are the additional safety net.
try {
let rows: Record<string, unknown>[];
if (link.type === 'http') {
// Route.name is the canonicalized URL path (see
// core/ingestion/pipeline.ts ensureSlash + generateId('Route', ...)).
// Normalize the manifest contract the same way so a user-written
// "/api/orders" matches "api/orders" in the graph.
//
// The contract may also use the explicit-method form "GET::/api/orders"
// recommended by buildContractId. Strip the METHOD:: prefix before
// normalizing — otherwise `normalizeRoutePath('GET::/api/orders')`
// returns `/GET::/api/orders` and never matches Route.name. The
// captured method is not yet used to constrain the Cypher query
// (method-aware HANDLES_ROUTE matching is a future enhancement).
const parsed = parseHttpContract(link.contract);
const normalized = normalizeRoutePath(parsed.path);
rows = await executor(
`MATCH (handler)-[r:CodeRelation {type: 'HANDLES_ROUTE'}]->(route:Route)
WHERE route.name = $normalized
RETURN handler.id AS uid, handler.name AS name, handler.filePath AS filePath
ORDER BY handler.filePath ASC
LIMIT 1`,
{ normalized },
);
} else if (link.type === 'topic') {
// Topic names aren't a first-class NodeLabel in the graph —
// topics are referenced by function/method symbols (Kafka
// listeners, publishers). Restrict to symbol-like labels to
// avoid cross-matching Files/Variables/Imports that happen to
// share the topic name.
rows = await executor(
`MATCH (n:Function|Method|Class|Interface) WHERE n.name = $contract
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
ORDER BY n.filePath ASC
LIMIT 1`,
{ contract: link.contract },
);
} else if (link.type === 'grpc') {
// Contract is "Service/Method" or just "Service" (or package.Service
// variants). Prefer matching by method name when present, otherwise
// by service name. NO .proto path fallback — that's guaranteed to
// return a wrong symbol in any repo with more than one proto file.
// Label filters scope lookups: methods → Function|Method, services
// → Class|Interface (no label match = no silent wrong hits on
// File/Variable nodes that happen to share the name).
const parts = link.contract.split('/');
const serviceName = parts[0]?.trim() ?? '';
const methodName = parts[1]?.trim() ?? '';
if (methodName) {
rows = await executor(
`MATCH (n:Function|Method) WHERE n.name = $methodName
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
ORDER BY n.filePath ASC
LIMIT 1`,
{ methodName },
);
} else if (serviceName) {
rows = await executor(
`MATCH (n:Class|Interface) WHERE n.name = $serviceName
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
ORDER BY n.filePath ASC
LIMIT 1`,
{ serviceName },
);
} else {
rows = [];
}
} else if (link.type === 'lib') {
// Only exact match on the symbol's name. Previous fallback to
// CONTAINS on n.filePath would promote "react" to "react-native"
// or "@types/react" — silent wrong attribution. Restrict to
// package-level labels so we don't return arbitrary symbols
// named after a library.
rows = await executor(
`MATCH (n:Package|Module) WHERE n.name = $contract
RETURN n.id AS uid, n.name AS name, n.filePath AS filePath
ORDER BY n.filePath ASC
LIMIT 1`,
{ contract: link.contract },
);
} else {
return null;
}
if (rows.length > 0) {
return {
filePath: rows[0].filePath as string,
name: rows[0].name as string,
uid: String(rows[0].uid ?? ''),
};
}
} catch (err) {
// Log but don't throw: a broken graph query in one repo shouldn't
// fail the whole manifest extraction. Unresolved contracts still
// get a synthetic symbolUid below, so cross-impact can proceed.
const message = err instanceof Error ? err.message : String(err);
console.warn(
`[manifest-extractor] resolveSymbol failed for ${link.type}:${link.contract} ` +
`in ${repoPathKey}: ${message}`,
);
}
return null;
}
/**
* Build a canonical contract id for a manifest link.
*
* HTTP is the only type with two valid forms:
* - Explicit method: `"GET::/api/orders"` `"http::GET::/api/orders"`
* (matches exactly against `HttpRouteExtractor` provider/consumer
* contracts, which are also keyed by `http::<METHOD>::<path>`).
* - Method-agnostic: `"/api/orders"` `"http::*::/api/orders"`
* the `*` is a wildcard and is intended to match any concrete
* HTTP method on that path. Wildcard-aware matching is the
* responsibility of the sync / cross-impact layer (see #793);
* downstream code should treat `http::*::<path>` as matching
* every `http::<METHOD>::<path>` for the same path.
*
* Recommend the explicit-method form in group.yaml whenever the
* manifest author knows the method it round-trips through exact
* equality matching without requiring wildcard logic downstream.
*
* NOTE on exhaustiveness: the switch covers every current
* `ContractType` variant and falls through to a `never` assertion so
* TypeScript fails the build if a new variant is added without a
* corresponding case.
*/
private buildContractId(type: ContractType, contract: string): string {
switch (type) {
case 'http': {
// Canonicalize method casing and path separators so logically
// equivalent inputs (`get::/api/orders` vs `GET::/api/orders`,
// or trailing-slash variants) produce the same contractId and
// matching `manifestSymbolUid` fallback. Without this, raw
// user casing leaks into cross-impact join keys and fragments
// matches across repos.
const { method, path: rawPath } = parseHttpContract(contract);
const normalizedPath = normalizeRoutePath(rawPath);
return method ? `http::${method}::${normalizedPath}` : `http::*::${normalizedPath}`;
}
case 'grpc':
return `grpc::${contract}`;
case 'topic':
return `topic::${contract}`;
case 'lib':
return `lib::${contract}`;
case 'custom':
return `custom::${contract}`;
default: {
const _exhaustive: never = type;
throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`);
}
}
}
}

View file

@ -1,214 +1,49 @@
import * as fs from 'node:fs';
import * as path from 'node:path';
import { glob } from 'glob';
import Parser from 'tree-sitter';
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
import type { ExtractedContract, RepoHandle } from '../types.js';
import { readSafe } from './fs-utils.js';
import { scanFile, unquoteLiteral } from './tree-sitter-scanner.js';
import {
TOPIC_SCAN_GLOB,
getProviderForFile,
type Broker,
type TopicMeta,
} from './topic-patterns/index.js';
type Broker = 'kafka' | 'rabbitmq' | 'nats';
/**
* Language-agnostic orchestrator for topic (message broker) contract
* extraction. All grammar-specific knowledge lives in `topic-patterns/*`
* this file must not import any tree-sitter grammar directly.
*
* Flow per file:
* 1. `getProviderForFile(rel)` compiled plugin (or `undefined` if the
* file's extension isn't registered, in which case we skip it).
* 2. `scanFile(parser, provider, content)` list of `{meta, valueText}`
* pairs, one per matched literal.
* 3. `unquoteLiteral(valueText)` the raw topic string.
* 4. `makeContract(topic, meta, relPath)` `ExtractedContract`.
*
* Adding a new language is a one-file edit in `topic-patterns/index.ts`.
*/
function readSafe(repoPath: string, rel: string): string | null {
const abs = path.resolve(repoPath, rel);
const base = path.resolve(repoPath);
const relToBase = path.relative(base, abs);
if (relToBase.startsWith('..') || path.isAbsolute(relToBase)) return null;
try {
return fs.readFileSync(abs, 'utf-8');
} catch {
return null;
}
}
function makeContract(
topicName: string,
role: 'provider' | 'consumer',
filePath: string,
symbolName: string,
confidence: number,
broker: Broker,
): ExtractedContract {
function makeContract(topicName: string, meta: TopicMeta, filePath: string): ExtractedContract {
return {
contractId: `topic::${topicName}`,
type: 'topic',
role,
role: meta.role,
symbolUid: '',
symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: symbolName },
symbolName,
confidence,
symbolRef: { filePath: filePath.replace(/\\/g, '/'), name: meta.symbolName },
symbolName: meta.symbolName,
confidence: meta.confidence,
meta: {
broker,
broker: meta.broker satisfies Broker,
topicName,
extractionStrategy: 'source_scan',
extractionStrategy: 'tree_sitter',
},
};
}
interface PatternDef {
regex: RegExp;
role: 'provider' | 'consumer';
broker: Broker;
confidence: number;
topicGroup: number;
symbolName: string;
}
// --- Kafka patterns ---
const KAFKA_PATTERNS: PatternDef[] = [
// Java: @KafkaListener(topics = "xxx")
{
regex: /@KafkaListener\s*\(\s*topics\s*=\s*"([^"]+)"/g,
role: 'consumer',
broker: 'kafka',
confidence: 0.8,
topicGroup: 1,
symbolName: 'kafkaListener',
},
// Java: kafkaTemplate.send("xxx"
{
regex: /kafkaTemplate\.send\s*\(\s*"([^"]+)"/gi,
role: 'provider',
broker: 'kafka',
confidence: 0.8,
topicGroup: 1,
symbolName: 'kafkaTemplate.send',
},
// Node: producer.send({ topic: 'xxx'
{
regex: /producer\.send\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g,
role: 'provider',
broker: 'kafka',
confidence: 0.8,
topicGroup: 1,
symbolName: 'producer.send',
},
// Node: consumer.subscribe({ topic: 'xxx'
{
regex: /consumer\.subscribe\s*\(\s*\{\s*topic:\s*['"]([^'"]+)['"]/g,
role: 'consumer',
broker: 'kafka',
confidence: 0.8,
topicGroup: 1,
symbolName: 'consumer.subscribe',
},
// Go: consumer.ConsumePartition("xxx"
{
regex: /\.ConsumePartition\s*\(\s*"([^"]+)"/g,
role: 'consumer',
broker: 'kafka',
confidence: 0.7,
topicGroup: 1,
symbolName: 'ConsumePartition',
},
// Python: KafkaConsumer('xxx'
{
regex: /KafkaConsumer\s*\(\s*['"]([^'"]+)['"]/g,
role: 'consumer',
broker: 'kafka',
confidence: 0.7,
topicGroup: 1,
symbolName: 'KafkaConsumer',
},
// Python: producer.send('xxx' or producer.produce('xxx'
{
regex: /producer\.(?:send|produce)\s*\(\s*['"]([^'"]+)['"]/g,
role: 'provider',
broker: 'kafka',
confidence: 0.7,
topicGroup: 1,
symbolName: 'producer.send',
},
];
// --- RabbitMQ patterns ---
const RABBITMQ_PATTERNS: PatternDef[] = [
// Java: @RabbitListener(queues = "xxx")
{
regex: /@RabbitListener\s*\(\s*queues\s*=\s*"([^"]+)"/g,
role: 'consumer',
broker: 'rabbitmq',
confidence: 0.8,
topicGroup: 1,
symbolName: 'rabbitListener',
},
// Java: rabbitTemplate.convertAndSend("xxx"
{
regex: /rabbitTemplate\.convertAndSend\s*\(\s*"([^"]+)"/gi,
role: 'provider',
broker: 'rabbitmq',
confidence: 0.8,
topicGroup: 1,
symbolName: 'rabbitTemplate.convertAndSend',
},
// Node: channel.consume("xxx"
{
regex: /channel\.consume\s*\(\s*"([^"]+)"/g,
role: 'consumer',
broker: 'rabbitmq',
confidence: 0.8,
topicGroup: 1,
symbolName: 'channel.consume',
},
// Node: channel.publish("xxx"
{
regex: /channel\.publish\s*\(\s*"([^"]+)"/g,
role: 'provider',
broker: 'rabbitmq',
confidence: 0.8,
topicGroup: 1,
symbolName: 'channel.publish',
},
// Node: channel.sendToQueue("xxx"
{
regex: /channel\.sendToQueue\s*\(\s*"([^"]+)"/g,
role: 'provider',
broker: 'rabbitmq',
confidence: 0.8,
topicGroup: 1,
symbolName: 'channel.sendToQueue',
},
// Python: channel.basic_consume(queue='xxx'
{
regex: /channel\.basic_consume\s*\(\s*queue\s*=\s*['"]([^'"]+)['"]/g,
role: 'consumer',
broker: 'rabbitmq',
confidence: 0.7,
topicGroup: 1,
symbolName: 'basic_consume',
},
// Python: channel.basic_publish(exchange='xxx'
{
regex: /channel\.basic_publish\s*\([^)]*exchange\s*=\s*['"]([^'"]+)['"]/g,
role: 'provider',
broker: 'rabbitmq',
confidence: 0.7,
topicGroup: 1,
symbolName: 'basic_publish',
},
];
// --- NATS patterns ---
const NATS_PATTERNS: PatternDef[] = [
// Go/Node: nc.Subscribe("xxx" or nc.subscribe("xxx"
{
regex: /nc\.(?:S|s)ubscribe\s*\(\s*"([^"]+)"/g,
role: 'consumer',
broker: 'nats',
confidence: 0.8,
topicGroup: 1,
symbolName: 'nc.Subscribe',
},
// Go/Node: nc.Publish("xxx" or nc.publish("xxx"
{
regex: /nc\.(?:P|p)ublish\s*\(\s*"([^"]+)"/g,
role: 'provider',
broker: 'nats',
confidence: 0.8,
topicGroup: 1,
symbolName: 'nc.Publish',
},
];
const ALL_PATTERNS: PatternDef[] = [...KAFKA_PATTERNS, ...RABBITMQ_PATTERNS, ...NATS_PATTERNS];
export class TopicExtractor implements ContractExtractor {
type = 'topic' as const;
@ -221,46 +56,48 @@ export class TopicExtractor implements ContractExtractor {
repoPath: string,
_repo: RepoHandle,
): Promise<ExtractedContract[]> {
const files = await glob('**/*.{ts,tsx,js,jsx,java,go,py}', {
const files = await glob(TOPIC_SCAN_GLOB, {
cwd: repoPath,
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
ignore: [
'**/node_modules/**',
'**/.git/**',
'**/vendor/**',
'**/dist/**',
'**/build/**',
// Language-level test file conventions. Go test files
// `*_test.go` live next to source; other languages either use
// separate test directories (Python's `tests/`, Java's
// `src/test/`) or are already covered by the dist/build ignores.
// Pushed to the glob level so the orchestrator stays
// language-agnostic.
'**/*_test.go',
],
nodir: true,
});
// One parser reused across files; the scanner calls `setLanguage` per
// file based on which plugin the registry returns.
const parser = new Parser();
const out: ExtractedContract[] = [];
for (const rel of files) {
const provider = getProviderForFile(rel);
if (!provider) continue;
const content = readSafe(repoPath, rel);
if (!content) continue;
out.push(...this.scanFile(content, rel));
}
return this.dedupe(out);
}
private scanFile(content: string, filePath: string): ExtractedContract[] {
const out: ExtractedContract[] = [];
for (const pattern of ALL_PATTERNS) {
// Reset regex state for each file
const re = new RegExp(pattern.regex.source, pattern.regex.flags);
let m: RegExpExecArray | null;
while ((m = re.exec(content)) !== null) {
const topicName = m[pattern.topicGroup];
const matches = scanFile(parser, provider, content);
for (const match of matches) {
const valueNode = match.captures.value;
if (!valueNode) continue;
const topicName = unquoteLiteral(valueNode.text);
if (!topicName) continue;
out.push(
makeContract(
topicName,
pattern.role,
filePath,
pattern.symbolName,
pattern.confidence,
pattern.broker,
),
);
out.push(makeContract(topicName, match.meta, rel));
}
}
return out;
return this.dedupe(out);
}
private dedupe(items: ExtractedContract[]): ExtractedContract[] {

View file

@ -0,0 +1,123 @@
import Go from 'tree-sitter-go';
import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js';
import type { TopicMeta } from './types.js';
/**
* Go topic extraction patterns.
*
* Detects Sarama, segmentio/kafka-go and nats.go producer/consumer APIs:
* - `X.ConsumePartition("topic", ...)`
* - `sarama.ProducerMessage{Topic: "xxx"}`
* - `kafka.Writer{Topic: "xxx"}` / `kafka.WriterConfig{Topic: ...}`
* - `kafka.Reader{Topic: "xxx"}` / `kafka.ReaderConfig{Topic: ...}`
* - `nc.Subscribe("topic", ...)` / `js.Subscribe("topic", ...)`
* - `nc.Publish("topic", ...)` / `js.Publish("topic", ...)`
*
* Every query MUST bind `@value` to the topic literal node.
*/
const GO_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
name: 'go-topic',
language: Go,
patterns: [
{
meta: {
role: 'consumer',
broker: 'kafka',
confidence: 0.7,
symbolName: 'ConsumePartition',
},
query: `
(call_expression
function: (selector_expression
field: (field_identifier) @method (#eq? @method "ConsumePartition"))
arguments: (argument_list . (interpreted_string_literal) @value))
`,
},
{
meta: {
role: 'provider',
broker: 'kafka',
confidence: 0.75,
symbolName: 'sarama.ProducerMessage',
},
query: `
(composite_literal
type: (qualified_type
package: (package_identifier) @pkg (#eq? @pkg "sarama")
name: (type_identifier) @ty (#eq? @ty "ProducerMessage"))
body: (literal_value
(keyed_element
(literal_element (identifier) @field (#eq? @field "Topic"))
(literal_element (interpreted_string_literal) @value))))
`,
},
{
meta: {
role: 'provider',
broker: 'kafka',
confidence: 0.75,
symbolName: 'kafka.Writer',
},
query: `
(composite_literal
type: (qualified_type
package: (package_identifier) @pkg (#eq? @pkg "kafka")
name: (type_identifier) @ty (#match? @ty "^(Writer|WriterConfig)$"))
body: (literal_value
(keyed_element
(literal_element (identifier) @field (#eq? @field "Topic"))
(literal_element (interpreted_string_literal) @value))))
`,
},
{
meta: {
role: 'consumer',
broker: 'kafka',
confidence: 0.75,
symbolName: 'kafka.Reader',
},
query: `
(composite_literal
type: (qualified_type
package: (package_identifier) @pkg (#eq? @pkg "kafka")
name: (type_identifier) @ty (#match? @ty "^(Reader|ReaderConfig)$"))
body: (literal_value
(keyed_element
(literal_element (identifier) @field (#eq? @field "Topic"))
(literal_element (interpreted_string_literal) @value))))
`,
},
{
meta: {
role: 'consumer',
broker: 'nats',
confidence: 0.8,
symbolName: 'nc.Subscribe',
},
query: `
(call_expression
function: (selector_expression
operand: (identifier) @obj (#match? @obj "^(nc|js)$")
field: (field_identifier) @method (#match? @method "^[Ss]ubscribe$"))
arguments: (argument_list . (interpreted_string_literal) @value))
`,
},
{
meta: {
role: 'provider',
broker: 'nats',
confidence: 0.8,
symbolName: 'nc.Publish',
},
query: `
(call_expression
function: (selector_expression
operand: (identifier) @obj (#match? @obj "^(nc|js)$")
field: (field_identifier) @method (#match? @method "^[Pp]ublish$"))
arguments: (argument_list . (interpreted_string_literal) @value))
`,
},
],
};
export const GO_TOPIC_PROVIDER = compilePatterns(GO_TOPIC_SPEC);

View file

@ -0,0 +1,49 @@
import * as path from 'node:path';
import type { CompiledPatterns } from '../tree-sitter-scanner.js';
import type { TopicMeta } from './types.js';
import { JAVA_TOPIC_PROVIDER } from './java.js';
import { GO_TOPIC_PROVIDER } from './go.js';
import { PYTHON_TOPIC_PROVIDER } from './python.js';
import {
JAVASCRIPT_TOPIC_PROVIDER,
TYPESCRIPT_TOPIC_PROVIDER,
TSX_TOPIC_PROVIDER,
} from './node.js';
export type { TopicMeta, Broker } from './types.js';
/**
* File-extension compiled-plugin registry for topic extraction. The
* top-level orchestrator (`topic-extractor.ts`) looks up the plugin for
* each file it visits and delegates the scanning to `tree-sitter-scanner`.
*
* Keys are lowercase extensions including the leading dot. To add a new
* language, drop a `topic-patterns/<lang>.ts` that exports a compiled
* provider, import it here and register the extension(s). No edits to
* `topic-extractor.ts` are required.
*/
const REGISTRY: Record<string, CompiledPatterns<TopicMeta>> = {
'.java': JAVA_TOPIC_PROVIDER,
'.go': GO_TOPIC_PROVIDER,
'.py': PYTHON_TOPIC_PROVIDER,
'.js': JAVASCRIPT_TOPIC_PROVIDER,
'.jsx': JAVASCRIPT_TOPIC_PROVIDER,
'.ts': TYPESCRIPT_TOPIC_PROVIDER,
'.tsx': TSX_TOPIC_PROVIDER,
};
/**
* Glob pattern for files worth scanning. Kept here so adding a new
* language to the registry also widens the glob automatically via a
* single edit.
*/
export const TOPIC_SCAN_GLOB = '**/*.{ts,tsx,js,jsx,java,go,py}';
/**
* Return the compiled provider registered for the given file's
* extension, or `undefined` if the extension is not registered.
*/
export function getProviderForFile(rel: string): CompiledPatterns<TopicMeta> | undefined {
const ext = path.extname(rel).toLowerCase();
return REGISTRY[ext];
}

View file

@ -0,0 +1,83 @@
import Java from 'tree-sitter-java';
import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js';
import type { TopicMeta } from './types.js';
/**
* Java topic extraction patterns.
*
* Detects Kafka and RabbitMQ (Spring conventions) producer/consumer APIs:
* - `@KafkaListener(topics = "xxx")`
* - `@RabbitListener(queues = "xxx")`
* - `kafkaTemplate.send("xxx", ...)`
* - `rabbitTemplate.convertAndSend("xxx", ...)`
*
* Every query MUST bind `@value` to the topic literal node.
*/
const JAVA_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
name: 'java-topic',
language: Java,
patterns: [
{
meta: {
role: 'consumer',
broker: 'kafka',
confidence: 0.8,
symbolName: 'kafkaListener',
},
query: `
(annotation
name: (identifier) @name (#eq? @name "KafkaListener")
arguments: (annotation_argument_list
(element_value_pair
key: (identifier) @key (#eq? @key "topics")
value: (string_literal) @value)))
`,
},
{
meta: {
role: 'consumer',
broker: 'rabbitmq',
confidence: 0.8,
symbolName: 'rabbitListener',
},
query: `
(annotation
name: (identifier) @name (#eq? @name "RabbitListener")
arguments: (annotation_argument_list
(element_value_pair
key: (identifier) @key (#eq? @key "queues")
value: (string_literal) @value)))
`,
},
{
meta: {
role: 'provider',
broker: 'kafka',
confidence: 0.8,
symbolName: 'kafkaTemplate.send',
},
query: `
(method_invocation
object: (identifier) @obj (#eq? @obj "kafkaTemplate")
name: (identifier) @method (#eq? @method "send")
arguments: (argument_list . (string_literal) @value))
`,
},
{
meta: {
role: 'provider',
broker: 'rabbitmq',
confidence: 0.8,
symbolName: 'rabbitTemplate.convertAndSend',
},
query: `
(method_invocation
object: (identifier) @obj (#eq? @obj "rabbitTemplate")
name: (identifier) @method (#eq? @method "convertAndSend")
arguments: (argument_list . (string_literal) @value))
`,
},
],
};
export const JAVA_TOPIC_PROVIDER = compilePatterns(JAVA_TOPIC_SPEC);

View file

@ -0,0 +1,165 @@
import JavaScript from 'tree-sitter-javascript';
import TypeScript from 'tree-sitter-typescript';
import {
compilePatterns,
type LanguagePatterns,
type PatternSpec,
} from '../tree-sitter-scanner.js';
import type { TopicMeta } from './types.js';
/**
* Node.js / TypeScript topic extraction patterns.
*
* Detects kafkajs, amqplib (RabbitMQ), and nats.js producer/consumer APIs:
* - `producer.send({ topic: 'xxx', ... })` (kafkajs)
* - `consumer.subscribe({ topic: 'xxx', ... })` (kafkajs)
* - `channel.consume("queue", ...)` / `channel.publish(...)` / `channel.sendToQueue(...)`
* - `nc.subscribe("topic")` / `js.subscribe("topic")`
* - `nc.publish("topic", ...)` / `js.publish("topic", ...)`
*
* The JavaScript and TypeScript tree-sitter grammars share node type
* names for every construct we query here, so the pattern sources are
* defined once and compiled against each grammar variant. We export three
* providers because Parser.Query objects are NOT portable across grammar
* instances `.js` files use the JavaScript grammar, `.ts` uses
* TypeScript.typescript, and `.tsx` uses TypeScript.tsx.
*
* Every query MUST bind `@value` to the topic literal node.
*/
const NODE_TOPIC_PATTERNS: PatternSpec<TopicMeta>[] = [
{
meta: {
role: 'provider',
broker: 'kafka',
confidence: 0.8,
symbolName: 'producer.send',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "producer")
property: (property_identifier) @prop (#eq? @prop "send"))
arguments: (arguments
(object
(pair
key: (property_identifier) @key (#eq? @key "topic")
value: [(string) (template_string)] @value))))
`,
},
{
meta: {
role: 'consumer',
broker: 'kafka',
confidence: 0.8,
symbolName: 'consumer.subscribe',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "consumer")
property: (property_identifier) @prop (#eq? @prop "subscribe"))
arguments: (arguments
(object
(pair
key: (property_identifier) @key (#eq? @key "topic")
value: [(string) (template_string)] @value))))
`,
},
{
meta: {
role: 'consumer',
broker: 'rabbitmq',
confidence: 0.8,
symbolName: 'channel.consume',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "channel")
property: (property_identifier) @prop (#eq? @prop "consume"))
arguments: (arguments . [(string) (template_string)] @value))
`,
},
{
meta: {
role: 'provider',
broker: 'rabbitmq',
confidence: 0.8,
symbolName: 'channel.publish',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "channel")
property: (property_identifier) @prop (#eq? @prop "publish"))
arguments: (arguments . [(string) (template_string)] @value))
`,
},
{
meta: {
role: 'provider',
broker: 'rabbitmq',
confidence: 0.8,
symbolName: 'channel.sendToQueue',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#eq? @obj "channel")
property: (property_identifier) @prop (#eq? @prop "sendToQueue"))
arguments: (arguments . [(string) (template_string)] @value))
`,
},
{
meta: {
role: 'consumer',
broker: 'nats',
confidence: 0.8,
symbolName: 'nc.subscribe',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#match? @obj "^(nc|js)$")
property: (property_identifier) @prop (#match? @prop "^[Ss]ubscribe$"))
arguments: (arguments . [(string) (template_string)] @value))
`,
},
{
meta: {
role: 'provider',
broker: 'nats',
confidence: 0.8,
symbolName: 'nc.publish',
},
query: `
(call_expression
function: (member_expression
object: (identifier) @obj (#match? @obj "^(nc|js)$")
property: (property_identifier) @prop (#match? @prop "^[Pp]ublish$"))
arguments: (arguments . [(string) (template_string)] @value))
`,
},
];
const JAVASCRIPT_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
name: 'javascript-topic',
language: JavaScript,
patterns: NODE_TOPIC_PATTERNS,
};
const TYPESCRIPT_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
name: 'typescript-topic',
language: TypeScript.typescript,
patterns: NODE_TOPIC_PATTERNS,
};
const TSX_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
name: 'tsx-topic',
language: TypeScript.tsx,
patterns: NODE_TOPIC_PATTERNS,
};
export const JAVASCRIPT_TOPIC_PROVIDER = compilePatterns(JAVASCRIPT_TOPIC_SPEC);
export const TYPESCRIPT_TOPIC_PROVIDER = compilePatterns(TYPESCRIPT_TOPIC_SPEC);
export const TSX_TOPIC_PROVIDER = compilePatterns(TSX_TOPIC_SPEC);

View file

@ -0,0 +1,119 @@
import Python from 'tree-sitter-python';
import { compilePatterns, type LanguagePatterns } from '../tree-sitter-scanner.js';
import type { TopicMeta } from './types.js';
/**
* Python topic extraction patterns.
*
* Detects kafka-python, pika (RabbitMQ), and nats-py producer/consumer APIs:
* - `KafkaConsumer('topic', ...)`
* - `producer.send('topic', ...)` / `producer.produce('topic', ...)`
* - `channel.basic_consume(queue='xxx', ...)`
* - `channel.basic_publish(exchange='xxx', ...)`
* - `await nc.subscribe('topic')`
* - `await nc.publish('topic', ...)`
*
* Every query MUST bind `@value` to the topic literal node.
*/
const PYTHON_TOPIC_SPEC: LanguagePatterns<TopicMeta> = {
name: 'python-topic',
language: Python,
patterns: [
{
meta: {
role: 'consumer',
broker: 'kafka',
confidence: 0.7,
symbolName: 'KafkaConsumer',
},
query: `
(call
function: (identifier) @func (#eq? @func "KafkaConsumer")
arguments: (argument_list . (string) @value))
`,
},
{
meta: {
role: 'provider',
broker: 'kafka',
confidence: 0.7,
symbolName: 'producer.send',
},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "producer")
attribute: (identifier) @method (#match? @method "^(send|produce)$"))
arguments: (argument_list . (string) @value))
`,
},
{
meta: {
role: 'consumer',
broker: 'rabbitmq',
confidence: 0.7,
symbolName: 'basic_consume',
},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "channel")
attribute: (identifier) @method (#eq? @method "basic_consume"))
arguments: (argument_list
(keyword_argument
name: (identifier) @kw (#eq? @kw "queue")
value: (string) @value)))
`,
},
{
meta: {
role: 'provider',
broker: 'rabbitmq',
confidence: 0.7,
symbolName: 'basic_publish',
},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "channel")
attribute: (identifier) @method (#eq? @method "basic_publish"))
arguments: (argument_list
(keyword_argument
name: (identifier) @kw (#eq? @kw "exchange")
value: (string) @value)))
`,
},
{
meta: {
role: 'consumer',
broker: 'nats',
confidence: 0.75,
symbolName: 'nc.subscribe',
},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "nc")
attribute: (identifier) @method (#eq? @method "subscribe"))
arguments: (argument_list . (string) @value))
`,
},
{
meta: {
role: 'provider',
broker: 'nats',
confidence: 0.75,
symbolName: 'nc.publish',
},
query: `
(call
function: (attribute
object: (identifier) @obj (#eq? @obj "nc")
attribute: (identifier) @method (#eq? @method "publish"))
arguments: (argument_list . (string) @value))
`,
},
],
};
export const PYTHON_TOPIC_PROVIDER = compilePatterns(PYTHON_TOPIC_SPEC);

View file

@ -0,0 +1,27 @@
/**
* Shared types for the topic-extractor language plugins.
*
* Each plugin lives in its own file (java.ts, go.ts, ...) and owns the
* tree-sitter grammar import + query sources. The top-level
* `topic-extractor.ts` orchestrator only knows about this type module and
* the plugin registry (`./index.ts`). It MUST NOT import any grammar or
* query text directly that's the whole point of the split.
*/
export type Broker = 'kafka' | 'rabbitmq' | 'nats';
/**
* Per-pattern payload every topic plugin attaches to its query. Whatever
* the pattern matches, the orchestrator receives this object verbatim
* and uses it to build an `ExtractedContract`.
*
* Plugins produce one `TopicMeta` per pattern (not per match) because a
* single query uniquely identifies its broker/role/confidence triple.
*/
export interface TopicMeta {
role: 'provider' | 'consumer';
broker: Broker;
confidence: number;
/** Short human-readable label of the API being detected. */
symbolName: string;
}

View file

@ -0,0 +1,193 @@
import Parser from 'tree-sitter';
/**
* Shared, language-agnostic tree-sitter scanning utilities used by group
* extractors (topic, http, grpc, ...).
*
* Design goals:
* - The top-level extractors must not import any tree-sitter grammar.
* - Per-language plugins own their grammar import, their query sources,
* and the mapping from capture meta.
* - This module provides the plumbing: compile queries once per plugin,
* parse a file with a given grammar, run all patterns, and return the
* captured `string_literal`-style nodes together with the plugin's meta.
*/
/**
* One pattern owned by a language plugin. Each pattern owns a tree-sitter
* S-expression query. Plugins can freely choose which capture names to
* use the scanner exposes every capture in the returned `captures`
* map and does not privilege any particular name.
*
* `TMeta` is the plugin-specific payload the orchestrator receives back
* when this pattern matches e.g. for topic extraction it carries the
* broker name, role, confidence, symbol name.
*/
export interface PatternSpec<TMeta> {
/** Tree-sitter S-expression. */
query: string;
/** Plugin-specific payload returned on every match. */
meta: TMeta;
}
/**
* A set of patterns owned by one language plugin, bound to a specific
* tree-sitter grammar.
*
* `language` is typed as `unknown` because tree-sitter's TypeScript
* declarations use `any` for the grammar object, and the grammar modules
* export different shapes (plain grammar vs. namespace with `typescript`
* / `tsx` members). Callers pass the concrete grammar object; this
* module forwards it to `parser.setLanguage` / `new Parser.Query`.
*/
export interface LanguagePatterns<TMeta> {
/** Human-readable plugin name for diagnostics. */
name: string;
/** tree-sitter grammar object. */
language: unknown;
/** Patterns authored against `language`. */
patterns: PatternSpec<TMeta>[];
}
/**
* Compiled form of a `LanguagePatterns` bundle. Queries are compiled
* eagerly at module load time so a broken grammar/query pair fails
* loudly the first time the plugin is imported, instead of silently
* at scan time when no contract is produced.
*/
export interface CompiledPatterns<TMeta> {
name: string;
language: unknown;
patterns: CompiledPattern<TMeta>[];
}
export interface CompiledPattern<TMeta> {
query: Parser.Query;
meta: TMeta;
}
/**
* Map from capture name syntax node. Every named capture the query
* binds is exposed as an entry. If a query captures the same name more
* than once (unusual), the first occurrence wins plugins that need
* all occurrences should use distinct capture names or fall back to
* `match.captures` array directly by iterating `query.matches()`
* themselves.
*/
export type CaptureMap = Record<string, Parser.SyntaxNode>;
/**
* One match returned by `scanFile` / `runCompiledPatterns`. The caller
* receives the full capture map plus the plugin meta, and is
* responsible for turning it into a domain object.
*/
export interface ScanMatch<TMeta> {
meta: TMeta;
captures: CaptureMap;
}
/**
* Compile a LanguagePatterns bundle. Call this once per plugin, at
* module load time, and export the result. Throws if any pattern
* fails to compile against the grammar that's a bug in the plugin
* author's query, not a runtime condition.
*/
export function compilePatterns<TMeta>(bundle: LanguagePatterns<TMeta>): CompiledPatterns<TMeta> {
const compiled: CompiledPattern<TMeta>[] = [];
for (const spec of bundle.patterns) {
try {
const query = new Parser.Query(bundle.language, spec.query);
compiled.push({ query, meta: spec.meta });
} catch (err) {
const message = err instanceof Error ? err.message : String(err);
throw new Error(
`[tree-sitter-scanner] Failed to compile pattern in ${bundle.name}: ${message}\n` +
`Query source:\n${spec.query}`,
);
}
}
return { name: bundle.name, language: bundle.language, patterns: compiled };
}
/**
* Run every compiled pattern in `plugin` against an already-parsed
* tree. Use this when a plugin needs multiple query bundles against
* the same file (e.g. one query for class-level prefixes and another
* for method-level annotations) and wants to avoid re-parsing.
*/
export function runCompiledPatterns<TMeta>(
plugin: CompiledPatterns<TMeta>,
tree: Parser.Tree,
): ScanMatch<TMeta>[] {
const out: ScanMatch<TMeta>[] = [];
for (const compiled of plugin.patterns) {
let matches: Parser.QueryMatch[];
try {
matches = compiled.query.matches(tree.rootNode);
} catch {
continue;
}
for (const match of matches) {
const captures: CaptureMap = {};
for (const cap of match.captures) {
if (!(cap.name in captures)) captures[cap.name] = cap.node;
}
out.push({ meta: compiled.meta, captures });
}
}
return out;
}
/**
* Parse `content` with the plugin's grammar and run every compiled
* pattern against the AST. Returns one `ScanMatch` per matched query
* occurrence, carrying the plugin's meta payload.
*
* Errors are swallowed at the file level (malformed file must not abort
* the whole extract). Individual pattern failures are swallowed too so
* a single unusable query doesn't block the rest of the plugin.
*/
export function scanFile<TMeta>(
parser: Parser,
plugin: CompiledPatterns<TMeta>,
content: string,
): ScanMatch<TMeta>[] {
let tree: Parser.Tree;
try {
parser.setLanguage(plugin.language);
tree = parser.parse(content);
} catch {
return [];
}
return runCompiledPatterns(plugin, tree);
}
/**
* Strip enclosing quotes from a tree-sitter string literal node's text.
* Handles single / double / template quotes, Python triple-quoted strings,
* and Go raw string literals (backticks).
*
* Returns null for empty/nullish input so callers can uniformly skip
* captures whose value is missing.
*/
export function unquoteLiteral(raw: string): string | null {
if (!raw) return null;
// Python triple-quoted
if (
(raw.startsWith('"""') && raw.endsWith('"""')) ||
(raw.startsWith("'''") && raw.endsWith("'''"))
) {
return raw.slice(3, -3);
}
const first = raw[0];
const last = raw[raw.length - 1];
if ((first === '"' || first === "'" || first === '`') && last === first && raw.length >= 2) {
return raw.slice(1, -1);
}
// Some grammars expose the string content without quotes already (e.g.
// Python `string_content` child). Return as-is.
return raw;
}

View file

@ -5,6 +5,15 @@ export interface MatchResult {
unmatched: StoredContract[];
}
export interface WildcardMatchResult {
matched: CrossLink[];
remaining: StoredContract[];
}
function isGrpcWildcard(cid: string): boolean {
return cid.startsWith('grpc::') && cid.endsWith('/*');
}
export function normalizeContractId(id: string): string {
const colonIdx = id.indexOf('::');
if (colonIdx === -1) return id;
@ -24,6 +33,22 @@ export function normalizeContractId(id: string): string {
return id;
}
case 'grpc': {
// Canonical form: `grpc::<lowercased-package-or-service>[/<method>]`.
//
// The package/service segment is lowercased because gRPC package
// names are effectively case-insensitive across language bindings
// (`auth.AuthService`, `auth.authservice`, `AUTH.AUTHSERVICE` all
// describe the same wire protocol service). The RPC method segment
// is preserved as-is because the HTTP/2 path used on the wire is
// case-sensitive per the gRPC spec (`/Service/MethodName`), and
// method names in generated clients match the proto source exactly.
//
// A package-only id (no slash) and a package/method id are treated
// as DISTINCT canonical forms: `grpc::userservice` does not match
// `grpc::userservice/Login`. That's by design — callers that want
// service-level manifest matching against method-level providers
// should use the gRPC wildcard form `grpc::UserService/*` which is
// handled by runWildcardMatch below.
const slashIdx = rest.indexOf('/');
if (slashIdx > 0) {
const pkg = rest.substring(0, slashIdx).toLowerCase();
@ -31,12 +56,12 @@ export function normalizeContractId(id: string): string {
return `grpc::${pkg}${method}`;
}
if (slashIdx === 0) {
// Malformed "package/method" with leading slash — do not lowercase the whole string
// (method segment is case-sensitive per spec).
// Malformed "/method" with leading slash — keep as-is so two
// equally malformed ids can still match each other.
return `grpc::${rest}`;
}
// No slash: spec is ambiguous (package-only vs full service.method). MVP: lowercase
// the whole token; differs from pkg/method split above where RPC method keeps case.
// No slash: package/service only. Lowercase to match the package
// segment produced by the pkg/method branch above.
return `grpc::${rest.toLowerCase()}`;
}
case 'topic':
@ -66,27 +91,36 @@ function findMatchingKeys(contractId: string, index: Map<string, StoredContract[
return [];
}
export function runExactMatch(contracts: StoredContract[]): MatchResult {
export function buildProviderIndex(contracts: StoredContract[]): Map<string, StoredContract[]> {
const providers = contracts.filter((c) => c.role === 'provider');
const consumers = contracts.filter((c) => c.role === 'consumer');
const providerIndex = new Map<string, StoredContract[]>();
const index = new Map<string, StoredContract[]>();
for (const p of providers) {
const key = normalizeContractId(p.contractId);
const list = providerIndex.get(key) || [];
const list = index.get(key) || [];
list.push(p);
providerIndex.set(key, list);
index.set(key, list);
}
return index;
}
export function runExactMatch(
contracts: StoredContract[],
providerIndex?: Map<string, StoredContract[]>,
): MatchResult {
const index = providerIndex ?? buildProviderIndex(contracts);
// Skip gRPC wildcard consumers — they go to wildcard pass only
const consumers = contracts.filter((c) => c.role === 'consumer' && !isGrpcWildcard(c.contractId));
const matched: CrossLink[] = [];
const matchedConsumerIds = new Set<string>();
const matchedProviderIds = new Set<string>();
for (const consumer of consumers) {
const matchingKeys = findMatchingKeys(consumer.contractId, providerIndex);
const matchingKeys = findMatchingKeys(consumer.contractId, index);
if (matchingKeys.length === 0) continue;
const allMatchingProviders = matchingKeys.flatMap((k) => providerIndex.get(k) || []);
const allMatchingProviders = matchingKeys.flatMap((k) => index.get(k) || []);
for (const provider of allMatchingProviders) {
if (provider.repo === consumer.repo) {
if (!provider.service || !consumer.service || provider.service === consumer.service) {
@ -118,10 +152,86 @@ export function runExactMatch(contracts: StoredContract[]): MatchResult {
}
}
const unmatched = contracts.filter((c) => {
// normalUnmatched: contracts that weren't matched in exact pass
const normalUnmatched = contracts.filter((c) => {
if (isGrpcWildcard(c.contractId)) return false; // excluded from exact, handled separately
const id = `${c.repo}::${c.contractId}`;
return c.role === 'provider' ? !matchedProviderIds.has(id) : !matchedConsumerIds.has(id);
});
// Re-add gRPC wildcard contracts — they were never in exact matching
const grpcWildcards = contracts.filter((c) => isGrpcWildcard(c.contractId));
const unmatched = [...normalUnmatched, ...grpcWildcards];
return { matched, unmatched };
}
export function runWildcardMatch(
unmatched: StoredContract[],
providerIndex: Map<string, StoredContract[]>,
): WildcardMatchResult {
const wildcardConsumers = unmatched.filter(
(c) => c.role === 'consumer' && isGrpcWildcard(c.contractId),
);
const matched: CrossLink[] = [];
const matchedConsumerIds = new Set<string>();
for (const consumer of wildcardConsumers) {
const normalized = normalizeContractId(consumer.contractId);
// "grpc::com.example.userservice/*" → "com.example.userservice"
// "grpc::userservice/*" → "userservice"
const fqService = normalized.slice(normalized.indexOf('::') + 2, -2); // strip "grpc::" and "/*"
for (const [key, providers] of providerIndex) {
// Only match against non-wildcard gRPC providers (method-level IDs)
if (!key.startsWith('grpc::') || key.endsWith('/*')) continue;
const afterPrefix = key.slice(6); // strip "grpc::"
const slashIdx = afterPrefix.indexOf('/');
if (slashIdx < 0) continue;
const providerFqService = afterPrefix.slice(0, slashIdx);
// Match: exact FQ service, or bare-name match when consumer has no package
const isMatch =
providerFqService === fqService ||
(!fqService.includes('.') && providerFqService.endsWith('.' + fqService));
if (!isMatch) continue;
for (const provider of providers) {
// Skip same-repo same-service (same logic as runExactMatch)
if (provider.repo === consumer.repo) {
if (!provider.service || !consumer.service || provider.service === consumer.service) {
continue;
}
}
matched.push({
from: {
repo: consumer.repo,
service: consumer.service,
symbolUid: consumer.symbolUid,
symbolRef: consumer.symbolRef,
},
to: {
repo: provider.repo,
service: provider.service,
symbolUid: provider.symbolUid,
symbolRef: provider.symbolRef,
},
type: consumer.type,
contractId: consumer.contractId, // consumer's wildcard ID
matchType: 'wildcard',
confidence: Math.min(provider.confidence, consumer.confidence),
});
matchedConsumerIds.add(`${consumer.repo}::${consumer.contractId}`);
}
}
}
const remaining = unmatched.filter((c) => {
if (c.role !== 'consumer' || !isGrpcWildcard(c.contractId)) return true;
return !matchedConsumerIds.has(`${c.repo}::${c.contractId}`);
});
return { matched, remaining };
}

View file

@ -0,0 +1,124 @@
import type { CrossLink, CrossLinkEndpoint, StoredContract } from './types.js';
function contractKey(contract: StoredContract): string {
return [contract.repo, contract.contractId, contract.role, contract.symbolRef.filePath].join(
'\0',
);
}
function endpointKey(endpoint: CrossLinkEndpoint): string {
return [
endpoint.repo,
endpoint.service ?? '',
endpoint.symbolRef.filePath,
endpoint.symbolRef.name,
].join('\0');
}
/**
* Score a contract by how much information it carries, so `dedupeContracts`
* can prefer the "richer" record when two contracts collide on the same
* `(repo, contractId, role, filePath)` key.
*
* Weights express a priority ordering, not calibrated probabilities:
* +3 `symbolUid` resolved (tier 1 of the downstream lookup highest
* signal because it's the strongest anchor for cross-impact traversal
* and the only one that's robust to renames)
* +2 any of `filePath`, `symbolRef.name`, or `symbolName` that's more
* specific than the contractId itself (tier 2 signal resolves
* uniquely in most cases and survives across syncs)
* +1 `service` tag (monorepo attribution useful but not sufficient
* on its own) or non-manifest origin (auto-extracted contracts are
* preferred over manifest-declared synthetic ones because the former
* are grounded in real source code)
*
* The absolute numbers don't matter, only their relative ordering.
*/
function contractRichness(contract: StoredContract): number {
let score = 0;
if (contract.symbolUid) score += 3;
if (contract.symbolRef.filePath) score += 2;
if (contract.symbolRef.name && contract.symbolRef.name !== contract.contractId) score += 2;
if (contract.symbolName && contract.symbolName !== contract.contractId) score += 2;
if (contract.service) score += 1;
if (contract.meta.source !== 'manifest') score += 1;
return score;
}
function mergeContracts(existing: StoredContract, incoming: StoredContract): StoredContract {
const [primary, secondary] =
contractRichness(incoming) > contractRichness(existing)
? [incoming, existing]
: [existing, incoming];
const symbolRefName = primary.symbolRef.name || secondary.symbolRef.name;
return {
...secondary,
...primary,
symbolUid: primary.symbolUid || secondary.symbolUid,
symbolRef: {
filePath: primary.symbolRef.filePath || secondary.symbolRef.filePath,
name: symbolRefName,
},
symbolName: primary.symbolName || secondary.symbolName || symbolRefName,
confidence: Math.max(existing.confidence, incoming.confidence),
service: primary.service ?? secondary.service,
meta: { ...secondary.meta, ...primary.meta },
};
}
function mergeEndpoints(
existing: CrossLinkEndpoint,
incoming: CrossLinkEndpoint,
): CrossLinkEndpoint {
return {
repo: existing.repo,
service: existing.service ?? incoming.service,
symbolUid: existing.symbolUid || incoming.symbolUid,
symbolRef: {
filePath: existing.symbolRef.filePath || incoming.symbolRef.filePath,
name: existing.symbolRef.name || incoming.symbolRef.name,
},
};
}
function crossLinkKey(link: CrossLink): string {
return [
link.type,
link.contractId,
link.matchType,
endpointKey(link.from),
endpointKey(link.to),
].join('\0');
}
export function dedupeContracts(items: StoredContract[]): StoredContract[] {
const deduped = new Map<string, StoredContract>();
for (const contract of items) {
const key = contractKey(contract);
const existing = deduped.get(key);
deduped.set(key, existing ? mergeContracts(existing, contract) : contract);
}
return [...deduped.values()];
}
export function dedupeCrossLinks(items: CrossLink[]): CrossLink[] {
const deduped = new Map<string, CrossLink>();
for (const link of items) {
const key = crossLinkKey(link);
const existing = deduped.get(key);
if (!existing) {
deduped.set(key, link);
continue;
}
const keepIncoming = link.confidence > existing.confidence;
const primary = keepIncoming ? link : existing;
const secondary = keepIncoming ? existing : link;
deduped.set(key, {
...primary,
confidence: Math.max(existing.confidence, link.confidence),
from: mergeEndpoints(primary.from, secondary.from),
to: mergeEndpoints(primary.to, secondary.to),
});
}
return [...deduped.values()];
}

View file

@ -7,6 +7,7 @@ import type { GroupConfig, RepoHandle, RepoSnapshot, StoredContract, CrossLink }
import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
import { GrpcExtractor } from './extractors/grpc-extractor.js';
import { TopicExtractor } from './extractors/topic-extractor.js';
import { ManifestExtractor } from './extractors/manifest-extractor.js';
import { runExactMatch } from './matching.js';
import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
import type { CypherExecutor } from './contract-extractor.js';
@ -60,10 +61,28 @@ function defaultResolveHandle(allEntries: RegistryEntry[]) {
};
}
/**
* Dedupe cross-links that point from the same consumer endpoint to the same
* provider endpoint for the same contract. Preserves first-seen order so the
* caller controls precedence (e.g., pass manifest links first).
*/
function dedupeCrossLinks(links: CrossLink[]): CrossLink[] {
const seen = new Set<string>();
const out: CrossLink[] = [];
for (const link of links) {
const key = `${link.from.repo}::${link.from.symbolUid}|${link.to.repo}::${link.to.symbolUid}|${link.type}|${link.contractId}`;
if (seen.has(key)) continue;
seen.add(key);
out.push(link);
}
return out;
}
export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promise<SyncResult> {
const missingRepos: string[] = [];
const repoSnapshots: Record<string, RepoSnapshot> = {};
let autoContracts: StoredContract[] = [];
let manifestCrossLinks: CrossLink[] = [];
let dbExecutors: Map<string, CypherExecutor> | undefined;
const eo = opts?.extractorOverride;
@ -158,8 +177,44 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
}
}
// Process manifest links declared in group.yaml.
// ManifestExtractor is fully implemented but was never wired into this
// pipeline — config.links were parsed and validated but silently dropped.
// Placed after the DB try/finally: resolveSymbol falls back to synthetic
// UIDs when dbExecutors is undefined or a pool is closed, so cross-links
// are always generated regardless of whether real DB executors are available.
if (config.links.length > 0) {
// Warn about dangling links that reference repos not declared in config.repos.
// They still generate cross-links via synthetic UIDs (determinism is preserved),
// but the operator probably meant something that now silently does nothing useful.
const knownRepos = new Set(Object.keys(config.repos));
for (const link of config.links) {
const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
if (dangling.length > 0) {
console.warn(
`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`,
);
}
}
const manifestEx = new ManifestExtractor();
const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors);
autoContracts.push(...manifestResult.contracts);
manifestCrossLinks = manifestResult.crossLinks;
if (opts?.verbose) {
console.log(
` manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`,
);
}
}
const { matched, unmatched } = runExactMatch(autoContracts);
const crossLinks: CrossLink[] = matched;
// Dedupe cross-links. Manifest contracts participate in runExactMatch, so a
// manifest-declared link can also emit a matchType:'exact' CrossLink with the
// same endpoints. Prefer the manifest version — it reflects operator intent
// and carries matchType:'manifest' which downstream consumers may rely on.
const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]);
const allContracts: StoredContract[] = autoContracts;
const registry: ContractRegistry = {

View file

@ -1,5 +1,5 @@
export type ContractType = 'http' | 'grpc' | 'topic' | 'lib' | 'custom';
export type MatchType = 'exact' | 'manifest' | 'bm25' | 'embedding';
export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding';
export type ContractRole = 'provider' | 'consumer';
export interface GroupConfig {
@ -131,3 +131,17 @@ export interface OutOfScopeLink {
contractId: string;
confidence: number;
}
/** Opaque handle to an open bridge LadybugDB. */
export interface BridgeHandle {
/** Internal — do not access directly. */
readonly _db: unknown;
readonly _conn: unknown;
readonly groupDir: string;
}
export interface BridgeMeta {
version: number;
generatedAt: string;
missingRepos: string[];
}

View file

@ -0,0 +1,391 @@
/**
* BindingAccumulator read-append-only accumulator that collects TypeEnv
* bindings across files in the GitNexus analyzer pipeline.
*
* **Current behavior (both execution paths):** The accumulator carries only
* file-scope (`scope = ''`) entries. Function-scope bindings are stripped
* at both write sites:
*
* - **Worker path**: `parse-worker.ts` serializes only
* `typeEnv.fileScope()` entries across the IPC boundary.
* - **Sequential path**: `type-env.ts::flush()` iterates only the FILE_SCOPE
* entry of the env map and writes `BindingEntry` records with
* `scope: ''` hardcoded.
*
* The narrowing exists because function-scope bindings have zero downstream
* consumers today and were previously costing ~4.9 MB of heap + IPC on
* every pipeline run. See `type-env.ts::flush()` and the `FileScopeBindings`
* JSDoc in `parse-worker.ts` for the paired Phase 9 reversion checklist.
*
* **Historical quality asymmetry (Phase 9 consideration):** Even though
* both paths now carry only file-scope data, the two paths were built
* under different resolution capabilities, and a future Phase 9 reverter
* that widens them back to all scopes will inherit that asymmetry:
*
* - **Sequential path** had (and would regain) access to the full
* `SymbolTable` and `importedBindings`, so its bindings benefit from
* Tier 2 cross-file propagation.
* - **Worker path** runs without `SymbolTable` / `importedBindings` and
* can only produce Tier 0 (annotation-declared) and local Tier 1
* (same-file constructor inference) bindings.
*
* Phase 9 consumers that trust every entry equally will silently produce
* worse results for large repos (worker-dominant) than small ones
* (sequential-dominant). If Phase 9 needs homogeneous quality, either
* (a) tag entries with their tier at insert time so consumers can filter,
* or (b) post-process worker-path entries through a follow-up resolution
* pass after the main-thread `SymbolTable` is complete.
*
* **Lifecycle contract**: single-use `append* → finalize → consume → dispose`.
* After `dispose()` the accumulator is permanently dead: any mutating call
* (`appendFile`) throws, and read methods return empty/undefined as if the
* accumulator had never been appended to. The instance is not recyclable;
* construct a new one for a new pipeline run. Finalization and disposal are
* orthogonal state dimensions and may be invoked in either order.
*/
export interface BindingEntry {
readonly scope: string; // '' for file-level, 'funcName@startIndex' for function-local
readonly varName: string;
readonly typeName: string;
}
/**
* Minimal graph-node shape required by `enrichExportedTypeMap()`. Intentionally
* narrower than the full `GraphNode` type in `graph/types.ts` so tests can
* construct a minimal mock without depending on the full graph module, and
* so the enrichment logic is a pure function over this contract.
*
* Matches the shape of the real `KnowledgeGraph` node's `properties.isExported`
* access path tests that use a different shape silently pass while
* production fails.
*/
export interface EnrichmentGraphNode {
readonly id: string;
readonly properties?: { readonly isExported?: boolean } | undefined;
}
/**
* Minimal graph lookup interface used by `enrichExportedTypeMap()`.
* Consumes only the method the enrichment loop actually calls.
*/
export interface EnrichmentGraphLookup {
getNode(id: string): EnrichmentGraphNode | undefined;
}
/**
* Merge file-scope bindings from a (finalized) `BindingAccumulator` into an
* `exportedTypeMap` for symbols whose graph nodes are marked as exported.
*
* This is the single source of truth for the worker-path ExportedTypeMap
* enrichment loop. Previously the logic lived inline in `pipeline.ts` and
* the test suite reimplemented it as a `runEnrichmentLoop` helper a
* drift-prone pattern that meant tests could pass while production regressed.
* Extracting it here makes the production code call the same function the
* tests call.
*
* **Node ID candidate order**: `Function:{filePath}:{name}`
* `Variable:{filePath}:{name}` `Const:{filePath}:{name}`. First match wins.
*
* **Tier 0 priority**: if `exportedTypeMap` already has an entry for a
* `(filePath, name)` pair, the accumulator entry does NOT overwrite it
* the SymbolTable tier-0 pass is authoritative. Without this guard, a
* worker-path binding could clobber a higher-quality type from SymbolTable.
*
* **Finalize precondition**: the accumulator should be finalized before
* calling this function. The lifecycle contract is
* `append → finalize → enrich → dispose`. Finalization is not asserted
* here (the test suite and pipeline both honor it separately), but any
* append happening concurrently with this enrichment would be a lifecycle
* bug at the caller level.
*
* @returns The number of new entries written into `exportedTypeMap`
* (0 on empty accumulator or when every candidate was filtered
* out by the export check or the Tier 0 guard).
*/
export function enrichExportedTypeMap(
bindingAccumulator: BindingAccumulator,
graph: EnrichmentGraphLookup,
exportedTypeMap: Map<string, Map<string, string>>,
): number {
if (bindingAccumulator.fileCount === 0) return 0;
let enriched = 0;
for (const filePath of bindingAccumulator.files()) {
for (const [name, type] of bindingAccumulator.fileScopeEntries(filePath)) {
// Three-candidate-ID lookup mirrors the sequential-path export check
// in `collectExportedBindings()` (call-processor.ts).
const functionNodeId = `Function:${filePath}:${name}`;
const variableNodeId = `Variable:${filePath}:${name}`;
const constNodeId = `Const:${filePath}:${name}`;
const node =
graph.getNode(functionNodeId) ??
graph.getNode(variableNodeId) ??
graph.getNode(constNodeId);
if (!node?.properties?.isExported) continue;
let fileExports = exportedTypeMap.get(filePath);
if (!fileExports) {
fileExports = new Map();
exportedTypeMap.set(filePath, fileExports);
}
// Tier 0 priority: SymbolTable-populated entries are authoritative.
if (!fileExports.has(name)) {
fileExports.set(name, type);
enriched++;
}
}
}
return enriched;
}
const ENTRY_OVERHEAD = 64; // bytes per entry (object overhead + property refs)
const MAP_ENTRY_OVERHEAD = 80; // bytes per file entry in the map
export class BindingAccumulator {
// Storage is split into two parallel maps so file-scope reads are fast.
// - _allByFile holds every BindingEntry (used by getFile, memory estimate).
// - _fileScopeByFile is a nested Map<filePath, Map<varName, typeName>> for
// O(1) point-lookup via fileScopeGet(). For iteration-based consumers
// (enrichExportedTypeMap), fileScopeEntries() iterates the inner Map.
// Both maps carry the same key set modulo the `scope === ''` precondition:
// _allByFile has a key as soon as any entry is appended; _fileScopeByFile
// only has a key once a file-scope entry arrives. Code that iterates via
// files() uses _allByFile so files with only function-scope entries
// remain visible.
//
// Note: Map.set semantics mean a duplicate varName for the same file
// overwrites the previous value (last-write-wins). This is the correct
// behavior — duplicate top-level bindings in the same file shouldn't
// happen in well-formed source, and if they do the last declaration
// is typically the one the compiler sees.
private readonly _allByFile = new Map<string, BindingEntry[]>();
private readonly _fileScopeByFile = new Map<string, Map<string, string>>();
private _totalBindings = 0;
private _finalized = false;
private _disposed = false;
/**
* Append bindings for a file. Safe to call multiple times for the same file.
* Throws if the accumulator has been finalized. Skips if entries is empty.
*
* The `entries` parameter is `readonly` this method never mutates the
* caller's array. Internally, the first `appendFile` call per filePath
* makes a defensive copy (`slice()`), and subsequent calls push into the
* accumulator's own storage.
*/
appendFile(filePath: string, entries: readonly BindingEntry[]): void {
if (this._finalized) {
throw new Error(
'[BindingAccumulator] appendFile after finalize — no further appends allowed',
);
}
// Single-use lifecycle: once disposed, the accumulator is dead. A
// post-dispose append almost always indicates a missed wiring step
// (the consumer is reading state that was supposed to be released),
// so convert the silent use-after-dispose into a loud failure.
if (this._disposed) {
throw new Error('BindingAccumulator: use after dispose');
}
if (entries.length === 0) {
return;
}
// Note on the file-scope-only invariant:
// The accumulator does NOT reject function-scope entries at this
// boundary. The narrowing contract is enforced by the two production
// write sites — `parse-worker.ts` (which uses `typeEnv.fileScope()`
// and hardcodes `scope: ''` in the pipeline adapter) and
// `type-env.ts::flush()` (which iterates only `env.get(FILE_SCOPE)`).
// The class JSDoc documents the invariant and the Phase 9 reversion
// path. Making `appendFile` runtime-reject non-file-scope entries
// would break the accumulator's own storage-split tests which
// legitimately exercise mixed-scope entries. If a future write path
// violates the invariant, tests should fail via missing exports in
// the enrichment loop, not via an assertion here.
// All-scope store.
const existingAll = this._allByFile.get(filePath);
if (existingAll !== undefined) {
for (const e of entries) {
existingAll.push(e);
}
} else {
this._allByFile.set(filePath, entries.slice());
}
// File-scope fast-path store (nested Map for O(1) point-lookup via fileScopeGet).
// Populated lazily on first file-scope entry per file.
let fileScopeMap = this._fileScopeByFile.get(filePath);
for (const e of entries) {
if (e.scope === '') {
if (fileScopeMap === undefined) {
fileScopeMap = new Map();
this._fileScopeByFile.set(filePath, fileScopeMap);
}
fileScopeMap.set(e.varName, e.typeName);
}
}
this._totalBindings += entries.length;
}
/** Lock the accumulator — no further appends. Idempotent. */
finalize(): void {
// Dev-mode invariant: verify the parallel storage split is consistent.
// `_fileScopeByFile` must be a proper projection of `_allByFile`
// where the outer key is a subset and the inner entries are exactly
// the `scope === ''` subset of `_allByFile[key]`. A drift would
// indicate a bug in `appendFile()` where one map was updated but
// not the other.
if (process.env.NODE_ENV !== 'production' && !this._finalized) {
for (const [filePath, fileScopeMap] of this._fileScopeByFile) {
const allEntries = this._allByFile.get(filePath);
if (allEntries === undefined) {
throw new Error(
`[BindingAccumulator] storage split drift: file ${filePath} has file-scope entries ` +
`but no _allByFile entry`,
);
}
// Count unique file-scope varNames in _allByFile (to match Map dedup
// semantics in _fileScopeByFile where Map.set deduplicates same-name).
const projectedNames = new Set(
allEntries.filter((e) => e.scope === '').map((e) => e.varName),
);
if (projectedNames.size !== fileScopeMap.size) {
throw new Error(
`[BindingAccumulator] storage split drift: file ${filePath} has ` +
`${fileScopeMap.size} file-scope names in Map but ${projectedNames.size} unique ` +
`file-scope varNames in _allByFile`,
);
}
}
}
this._finalized = true;
}
/**
* Release the accumulator's heap footprint. Clears both internal storage
* maps and resets `_totalBindings` to zero. Idempotent calling twice
* is a no-op. Orthogonal to `finalize()` calling `dispose()` does not
* change the finalized state.
*
* **Single-use lifecycle.** This is a one-way terminal transition: the
* accumulator is not recyclable. Any subsequent `appendFile` call throws
* (`'BindingAccumulator: use after dispose'`), regardless of whether
* `finalize()` was called first. Post-dispose reads do not throw
* they return empty/undefined state matching a never-appended-to
* accumulator:
* - `fileCount === 0`
* - `totalBindings === 0`
* - `files()` yields an empty iterator
* - `getFile(x)` returns `undefined` for all `x`
* - `fileScopeEntries(x)` returns `[]` for all `x`
* - `fileScopeGet(x, y)` returns `undefined` for all `x, y`
* - `estimateMemoryBytes()` returns `0`
*
* Lifecycle note: the pipeline disposes the accumulator inside the
* `finally` of the `crossFile` phase, which is scheduled after every
* other accumulator consumer (Phase 9 call/assignment processing and
* the ExportedTypeMap enrichment loop). The dispose call therefore
* runs once, on both the happy path and the throw path of the
* crossFile phase.
*/
dispose(): void {
this._allByFile.clear();
this._fileScopeByFile.clear();
this._totalBindings = 0;
this._disposed = true;
}
/** Get all bindings for a file, or undefined if the file is unknown. */
getFile(filePath: string): readonly BindingEntry[] | undefined {
return this._allByFile.get(filePath);
}
/**
* Get only scope='' (file-level) entries as [varName, typeName] tuples.
* For iteration-based consumers (e.g., `enrichExportedTypeMap`).
* Returns an empty array for an unknown file.
*
* O(1) map lookup + O(n_file_scope) tuple reconstruction from the inner
* Map. Does NOT walk function-scope entries.
*
* For point-lookup consumers (e.g., Phase 9 fallback), prefer
* `fileScopeGet(filePath, name)` O(1) with no allocation.
*/
fileScopeEntries(filePath: string): readonly (readonly [string, string])[] {
const map = this._fileScopeByFile.get(filePath);
return map ? [...map.entries()] : [];
}
/**
* O(1) point-lookup for a single file-scope binding by (filePath, name).
* Returns the typeName if found, `undefined` otherwise.
*
* This is the preferred lookup path for Phase 9 consumers that resolve
* a single callee's return type avoids the O(n_file_scope) iteration
* and defensive-copy allocation of `fileScopeEntries()`.
*/
fileScopeGet(filePath: string, name: string): string | undefined {
return this._fileScopeByFile.get(filePath)?.get(name);
}
/** Iterate over all file paths in insertion order. */
files(): IterableIterator<string> {
return this._allByFile.keys();
}
/** Number of distinct files with at least one binding. */
get fileCount(): number {
return this._allByFile.size;
}
/** Total number of binding entries across all files. */
get totalBindings(): number {
return this._totalBindings;
}
/** Whether the accumulator has been finalized. */
get finalized(): boolean {
return this._finalized;
}
/**
* Whether the accumulator has been disposed. Exposed for symmetry with
* `finalized` so debug tooling and future Phase 9 consumers can detect a
* disposed accumulator without inspecting empty state heuristically.
*
* Disposal and finalization are orthogonal: a disposed accumulator may or
* may not be finalized, and vice versa. See `dispose()` for the full
* lifecycle contract.
*/
get disposed(): boolean {
return this._disposed;
}
/**
* Rough memory estimate in bytes (intentionally pessimistic).
* Formula: sum of (ENTRY_OVERHEAD + char bytes of scope+varName+typeName) per entry
* + MAP_ENTRY_OVERHEAD + char bytes of filePath per file.
*
* Note: V8 stores all-ASCII strings as Latin-1 (1 byte/char) and only upgrades
* to UCS-2 (2 bytes/char) for non-Latin-1 code points. Source paths and type names
* are typically all-ASCII, so actual heap cost is roughly half what this returns.
* The pessimistic factor is intentional better to over-budget than under-budget.
*
* ** Cost profile**: O(totalBindings) iterates every entry in
* `_allByFile` and reads three string `.length` properties per entry.
* At a typical repo scale (10k files × ~20 file-scope bindings) this is
* ~200k property reads per call. Call at most once per pipeline run,
* NOT per file, per chunk, or per progress tick. The current single
* call site is the dev-mode telemetry log at the pipeline finalize
* seam. Adding a per-file-progress caller would silently make it
* quadratic in repo size.
*/
estimateMemoryBytes(): number {
let total = 0;
for (const [filePath, entries] of this._allByFile) {
total += MAP_ENTRY_OVERHEAD + filePath.length * 2;
for (const e of entries) {
total += ENTRY_OVERHEAD + (e.scope.length + e.varName.length + e.typeName.length) * 2;
}
}
return total;
}
}

View file

@ -0,0 +1,12 @@
// gitnexus/src/core/ingestion/call-extractors/configs/c-cpp.ts
import { SupportedLanguages } from 'gitnexus-shared';
import type { CallExtractionConfig } from '../../call-types.js';
export const cCallConfig: CallExtractionConfig = {
language: SupportedLanguages.C,
};
export const cppCallConfig: CallExtractionConfig = {
language: SupportedLanguages.CPlusPlus,
};

View file

@ -0,0 +1,9 @@
// gitnexus/src/core/ingestion/call-extractors/configs/csharp.ts
import { SupportedLanguages } from 'gitnexus-shared';
import type { CallExtractionConfig } from '../../call-types.js';
export const csharpCallConfig: CallExtractionConfig = {
language: SupportedLanguages.CSharp,
typeAsReceiverHeuristic: true,
};

View file

@ -0,0 +1,8 @@
// gitnexus/src/core/ingestion/call-extractors/configs/dart.ts
import { SupportedLanguages } from 'gitnexus-shared';
import type { CallExtractionConfig } from '../../call-types.js';
export const dartCallConfig: CallExtractionConfig = {
language: SupportedLanguages.Dart,
};

View file

@ -0,0 +1,8 @@
// gitnexus/src/core/ingestion/call-extractors/configs/go.ts
import { SupportedLanguages } from 'gitnexus-shared';
import type { CallExtractionConfig } from '../../call-types.js';
export const goCallConfig: CallExtractionConfig = {
language: SupportedLanguages.Go,
};

View file

@ -0,0 +1,59 @@
// gitnexus/src/core/ingestion/call-extractors/configs/jvm.ts
import { SupportedLanguages } from 'gitnexus-shared';
import type { CallExtractionConfig, ExtractedCallSite } from '../../call-types.js';
import type { SyntaxNode } from '../../utils/ast-helpers.js';
// ---------------------------------------------------------------------------
// Java method_reference (::) parsing — absorbs call-sites/java.ts
// ---------------------------------------------------------------------------
/**
* Parse Java `method_reference` nodes (`expr::method`, `Type::new`,
* `this::m`, `super::m`).
*/
function parseJavaMethodReference(callNode: SyntaxNode): ExtractedCallSite | null {
if (callNode.type !== 'method_reference') return null;
const recv = callNode.namedChild(0);
if (!recv) return null;
// Type::new → constructor call
for (const c of callNode.children) {
if (c.type === 'new') {
if (recv.type !== 'identifier') return null;
return { calledName: recv.text, callForm: 'constructor' };
}
}
// expr::method → member call with receiver
const rhs = callNode.child(callNode.childCount - 1);
if (!rhs || rhs.type !== 'identifier') return null;
const methodName = rhs.text;
if (recv.type === 'identifier') {
return { calledName: methodName, callForm: 'member', receiverName: recv.text };
}
if (recv.type === 'this') {
return { calledName: methodName, callForm: 'member', receiverName: 'this' };
}
if (recv.type === 'super') {
return { calledName: methodName, callForm: 'member', receiverName: 'super' };
}
return null;
}
// ---------------------------------------------------------------------------
// Configs
// ---------------------------------------------------------------------------
export const javaCallConfig: CallExtractionConfig = {
language: SupportedLanguages.Java,
extractLanguageCallSite: parseJavaMethodReference,
typeAsReceiverHeuristic: true,
};
export const kotlinCallConfig: CallExtractionConfig = {
language: SupportedLanguages.Kotlin,
typeAsReceiverHeuristic: true,
};

View file

@ -0,0 +1,8 @@
// gitnexus/src/core/ingestion/call-extractors/configs/php.ts
import { SupportedLanguages } from 'gitnexus-shared';
import type { CallExtractionConfig } from '../../call-types.js';
export const phpCallConfig: CallExtractionConfig = {
language: SupportedLanguages.PHP,
};

Some files were not shown because too many files have changed in this diff Show more