mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-09-22 00:31:17 +00:00
Merge branch 'main' into feat/Desktop-app
This commit is contained in:
commit
ea313579c6
123 changed files with 10142 additions and 645 deletions
|
|
@ -1,5 +1,5 @@
|
|||
name: Setup GitNexus Web
|
||||
description: Setup Node.js 20.19+ (vite 7 floor), build gitnexus-shared, install web dependencies
|
||||
description: Setup Node.js 22, build gitnexus-shared, install web dependencies
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
|
|
@ -7,9 +7,7 @@ runs:
|
|||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
# Vite 7 requires Node ^20.19.0 || >=22.12.0 (require(esm) support).
|
||||
# Pin explicitly so we don't depend on the floating "20" alias resolving
|
||||
# to a high enough patch version on every runner image.
|
||||
node-version: '20.19.0'
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: gitnexus-web/package-lock.json
|
||||
|
||||
|
|
|
|||
4
.github/actions/setup-gitnexus/action.yml
vendored
4
.github/actions/setup-gitnexus/action.yml
vendored
|
|
@ -1,5 +1,5 @@
|
|||
name: Setup GitNexus
|
||||
description: Setup Node.js 20, install dependencies, and optionally build
|
||||
description: Setup Node.js 22, install dependencies, and optionally build
|
||||
|
||||
inputs:
|
||||
build:
|
||||
|
|
@ -12,7 +12,7 @@ runs:
|
|||
steps:
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: gitnexus/package-lock.json
|
||||
|
||||
|
|
|
|||
47
.github/dependabot.yml
vendored
47
.github/dependabot.yml
vendored
|
|
@ -7,6 +7,8 @@ updates:
|
|||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore
|
||||
|
|
@ -15,6 +17,36 @@ updates:
|
|||
- dependencies
|
||||
- ci
|
||||
|
||||
# Keep pinned Docker base-image digests current for the root Dockerfiles.
|
||||
- package-ecosystem: docker
|
||||
directory: /
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
labels:
|
||||
- dependencies
|
||||
- ci
|
||||
|
||||
# Keep the nested test-image Docker base digest current as well.
|
||||
- package-ecosystem: docker
|
||||
directory: /gitnexus
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
include: scope
|
||||
labels:
|
||||
- dependencies
|
||||
- ci
|
||||
|
||||
# Gitnexus npm deps — tree-sitter grammars checked daily so we catch
|
||||
# new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars
|
||||
# are grouped so lockstep bumps produce a single PR. The tree-sitter
|
||||
|
|
@ -25,6 +57,11 @@ updates:
|
|||
directory: /gitnexus
|
||||
schedule:
|
||||
interval: daily
|
||||
cooldown:
|
||||
default-days: 7
|
||||
semver-major-days: 30
|
||||
semver-minor-days: 7
|
||||
semver-patch-days: 3
|
||||
open-pull-requests-limit: 10
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
|
|
@ -54,6 +91,11 @@ updates:
|
|||
directory: /gitnexus-web
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
semver-major-days: 30
|
||||
semver-minor-days: 7
|
||||
semver-patch-days: 3
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
|
|
@ -67,6 +109,11 @@ updates:
|
|||
directory: /gitnexus-shared
|
||||
schedule:
|
||||
interval: weekly
|
||||
cooldown:
|
||||
default-days: 7
|
||||
semver-major-days: 30
|
||||
semver-minor-days: 7
|
||||
semver-patch-days: 3
|
||||
open-pull-requests-limit: 5
|
||||
commit-message:
|
||||
prefix: chore(deps)
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ import pathlib
|
|||
import re
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
|
||||
|
|
@ -190,7 +191,17 @@ def fetch_text(url: str, timeout: int = 8) -> str | None:
|
|||
set (raises the rate limit from 60 to 5 000 requests/hour).
|
||||
"""
|
||||
headers: dict[str, str] = {}
|
||||
if _GITHUB_TOKEN and ("github.com" in url or "githubusercontent.com" in url):
|
||||
# Parse the URL and check the hostname rather than substring-matching
|
||||
# on the full URL string (CodeQL py/incomplete-url-substring-sanitization).
|
||||
# `https://evil.com/?u=github.com` would have passed the substring check.
|
||||
try:
|
||||
parsed_host = urllib.parse.urlparse(url).hostname or ""
|
||||
except ValueError:
|
||||
parsed_host = ""
|
||||
is_github_host = parsed_host == "github.com" or parsed_host.endswith(
|
||||
(".github.com", ".githubusercontent.com")
|
||||
) or parsed_host == "githubusercontent.com"
|
||||
if _GITHUB_TOKEN and is_github_host:
|
||||
headers["Authorization"] = f"Bearer {_GITHUB_TOKEN}"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
|
|
|
|||
4
.github/workflows/ci-quality.yml
vendored
4
.github/workflows/ci-quality.yml
vendored
|
|
@ -11,7 +11,7 @@ jobs:
|
|||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: package-lock.json
|
||||
- run: npm ci
|
||||
|
|
@ -24,7 +24,7 @@ jobs:
|
|||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: package-lock.json
|
||||
- run: npm ci
|
||||
|
|
|
|||
25
.github/workflows/ci-report.yml
vendored
25
.github/workflows/ci-report.yml
vendored
|
|
@ -95,31 +95,33 @@ jobs:
|
|||
|
||||
# Validate PR number is a positive integer (artifact comes from
|
||||
# untrusted fork code, so treat contents defensively).
|
||||
PR_NUM=$(cat "$DIR/pr_number" | tr -d '[:space:]')
|
||||
PR_NUM=$(tr -d '[:space:]' < "$DIR/pr_number")
|
||||
if ! [[ "$PR_NUM" =~ ^[0-9]+$ ]]; then
|
||||
echo "skip=true" >> "$GITHUB_OUTPUT"
|
||||
echo "::error::Invalid PR number in artifact: '$PR_NUM'"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "skip=false" >> "$GITHUB_OUTPUT"
|
||||
echo "pr_number=$PR_NUM" >> "$GITHUB_OUTPUT"
|
||||
# Validate job-result strings against known GitHub Actions values.
|
||||
# Artifact contents come from the PR workflow (potentially untrusted
|
||||
# fork code), so we whitelist to prevent newline injection into
|
||||
# GITHUB_OUTPUT.
|
||||
validate_result() {
|
||||
local val
|
||||
val=$(cat "$1" | tr -d '[:space:]')
|
||||
val=$(tr -d '[:space:]' < "$1")
|
||||
case "$val" in
|
||||
success|failure|cancelled|skipped) echo "$val" ;;
|
||||
*) echo "unknown" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
echo "quality=$(validate_result "$DIR/quality_result")" >> "$GITHUB_OUTPUT"
|
||||
echo "tests=$(validate_result "$DIR/tests_result")" >> "$GITHUB_OUTPUT"
|
||||
echo "e2e=$(validate_result "$DIR/e2e_result")" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "skip=false"
|
||||
echo "pr_number=$PR_NUM"
|
||||
echo "quality=$(validate_result "$DIR/quality_result")"
|
||||
echo "tests=$(validate_result "$DIR/tests_result")"
|
||||
echo "e2e=$(validate_result "$DIR/e2e_result")"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout (for vitest config)
|
||||
if: steps.meta.outputs.skip != 'true'
|
||||
|
|
@ -279,14 +281,17 @@ jobs:
|
|||
fi
|
||||
}
|
||||
|
||||
read CLI_T CLI_P CLI_F CLI_S CLI_SU CLI_D <<< "$(sum_results "$RESULTS_FILE")"
|
||||
read WEB_T WEB_P WEB_F WEB_S WEB_SU WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")"
|
||||
# `_` placeholder for the suite-count column — positional
|
||||
# readability for sum_results' 6-field output, but the value
|
||||
# isn't surfaced in the report (suites are tracked per-test
|
||||
# framework, not as a top-line metric).
|
||||
read -r CLI_T CLI_P CLI_F CLI_S _ CLI_D <<< "$(sum_results "$RESULTS_FILE")"
|
||||
read -r WEB_T WEB_P WEB_F WEB_S _ WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")"
|
||||
|
||||
TOTAL=$((CLI_T + WEB_T))
|
||||
PASSED=$((CLI_P + WEB_P))
|
||||
FAILED=$((CLI_F + WEB_F))
|
||||
SKIPPED=$((CLI_S + WEB_S))
|
||||
SUITES=$((CLI_SU + WEB_SU))
|
||||
DURATION=$((CLI_D > WEB_D ? CLI_D : WEB_D))
|
||||
|
||||
# ── Status helpers ──
|
||||
|
|
|
|||
595
.github/workflows/pr-autofix-apply.yml
vendored
Normal file
595
.github/workflows/pr-autofix-apply.yml
vendored
Normal file
|
|
@ -0,0 +1,595 @@
|
|||
name: PR Autofix (apply)
|
||||
|
||||
# CHATOPS HALF of the autofix pipeline.
|
||||
#
|
||||
# Triggered when a contributor comments `/autofix` on a PR. Validates
|
||||
# permission, locates the most recent successful `pr-autofix.yml`
|
||||
# artifact for the PR's current head SHA, applies the patch to the PR
|
||||
# head, and pushes a commit back to the PR branch.
|
||||
#
|
||||
# This workflow runs from the default branch's copy of the file
|
||||
# regardless of where the comment originates -- that's the trust
|
||||
# anchor. Comment body and author login are untrusted; both flow
|
||||
# through env vars and pattern-matched, never interpolated into shell.
|
||||
#
|
||||
# Fork PR support: `git push` with the GITHUB_TOKEN succeeds against
|
||||
# fork branches only when the contributor enabled "Allow edits by
|
||||
# maintainers" on the PR (the default). When they disabled it, we
|
||||
# fail loud with a 👎 reaction and an explanation comment.
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
concurrency:
|
||||
# Per-PR scope. issue_comment events expose `github.event.issue.number`
|
||||
# for both PR and Issue comments; the `pull_request != null` guard on
|
||||
# the job ensures we only run on PRs, so this number is the PR number.
|
||||
# cancel-in-progress: false — a second `/autofix` should wait for the
|
||||
# first to finish (idempotency check on the second invocation handles
|
||||
# the no-op case).
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
apply:
|
||||
name: apply-autofix
|
||||
# Pre-filter at the workflow level so non-PR comments and unrelated
|
||||
# comments don't even spawn a runner. The job-level body re-check
|
||||
# below (Step 1) is the strict gate.
|
||||
if: >-
|
||||
github.event.issue.pull_request != null
|
||||
&& startsWith(github.event.comment.body, '/autofix')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
# React on the triggering comment + post reply comments.
|
||||
pull-requests: write
|
||||
# Push the apply commit to the PR head branch.
|
||||
contents: write
|
||||
# Required by actions/download-artifact to fetch artifacts produced
|
||||
# by a different workflow run.
|
||||
actions: read
|
||||
steps:
|
||||
- name: Validate comment body precisely
|
||||
id: body
|
||||
env:
|
||||
BODY: ${{ github.event.comment.body }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Whole-line, case-sensitive match: `^/autofix\s*$`. The
|
||||
# workflow-level startsWith guard is coarse — `please don't
|
||||
# /autofix this code` would pass that filter but fail this one.
|
||||
# We exit silently (no reaction) on body mismatch so quoted
|
||||
# text in unrelated discussions doesn't get a visible response.
|
||||
if [[ ! "${BODY}" =~ ^/autofix[[:space:]]*$ ]]; then
|
||||
echo "Body did not match strict /autofix regex — exiting silently."
|
||||
echo "match=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
echo "match=true" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate commenter permission
|
||||
id: perm
|
||||
if: steps.body.outputs.match == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
COMMENTER: ${{ github.event.comment.user.login }}
|
||||
PR_AUTHOR: ${{ github.event.issue.user.login }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Retry wrapper for transient 5xx / 429 / network blips.
|
||||
# Mirrors the helper in pr-autofix-publish.yml. Used on
|
||||
# idempotent GETs only; reactions/comment-POSTs are NOT
|
||||
# wrapped (retrying a POST would dupe the resource).
|
||||
gh_retry() {
|
||||
local n=0 max=3
|
||||
while true; do
|
||||
if gh "$@"; then return 0; fi
|
||||
n=$((n+1))
|
||||
if [ "$n" -ge "$max" ]; then return 1; fi
|
||||
sleep $((n * 2))
|
||||
done
|
||||
}
|
||||
|
||||
# Allowlist the commenter login before it flows into a URL.
|
||||
# GitHub usernames: alphanumeric + dashes, max 39 chars.
|
||||
if ! [[ "${COMMENTER}" =~ ^[A-Za-z0-9-]{1,39}$ ]]; then
|
||||
echo "::error::Invalid commenter login format: $(printf '%q' "${COMMENTER}")"
|
||||
echo "allowed=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Self-comparison: PR author can always /autofix their own PR.
|
||||
if [ "${COMMENTER}" = "${PR_AUTHOR}" ]; then
|
||||
echo "Commenter is PR author — granting access."
|
||||
echo "allowed=true" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Repo permission lookup. admin/write/maintain are sufficient.
|
||||
# Distinguish API failure (5xx, 429, network) from genuine
|
||||
# permission denial (404 = not a collaborator). Conflating them
|
||||
# would silently refuse a legitimate maintainer with a public
|
||||
# 👎 every time GitHub blips. gh_retry handles transient blips;
|
||||
# the stderr-grep distinguishes 404 from persistent failure.
|
||||
perm_stderr=$(mktemp)
|
||||
if permission=$(gh_retry api "repos/${GH_REPO}/collaborators/${COMMENTER}/permission" \
|
||||
--jq '.permission' 2>"$perm_stderr"); then
|
||||
echo "Commenter permission: ${permission}"
|
||||
case "${permission}" in
|
||||
admin|write|maintain)
|
||||
echo "allowed=true" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
*)
|
||||
echo "allowed=false" >> "$GITHUB_OUTPUT"
|
||||
;;
|
||||
esac
|
||||
else
|
||||
err=$(cat "$perm_stderr")
|
||||
echo "Permission lookup stderr: ${err}" >&2
|
||||
# 404 (not a collaborator) is a genuine deny.
|
||||
# Anything else is a transient API/network failure.
|
||||
if grep -qE "HTTP 404|Not Found" "$perm_stderr"; then
|
||||
echo "allowed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::error::Permission lookup failed transiently — refusing to act."
|
||||
echo "allowed=api-failed" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
fi
|
||||
|
||||
- name: React 😕 on transient permission-API failure
|
||||
if: steps.body.outputs.match == 'true' && steps.perm.outputs.allowed == 'api-failed'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
COMMENT_ID: ${{ github.event.comment.id }}
|
||||
PR: ${{ github.event.issue.number }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="confused" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⚠️ Couldn't verify your repo permission (transient GitHub API failure). Please comment \`/autofix\` again. ([apply run](https://github.com/${GH_REPO}/actions/runs/${RUN_ID}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
|
||||
- name: React 👎 on permission denial
|
||||
if: steps.body.outputs.match == 'true' && steps.perm.outputs.allowed == 'false'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
COMMENT_ID: ${{ github.event.comment.id }}
|
||||
PR: ${{ github.event.issue.number }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="🚫 \`/autofix\` is restricted to users with write access or the PR author. Comment ignored." \
|
||||
>/dev/null
|
||||
# Hard exit so the rest of the job is skipped.
|
||||
exit 1
|
||||
|
||||
- name: React 👀 to acknowledge
|
||||
if: steps.perm.outputs.allowed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
COMMENT_ID: ${{ github.event.comment.id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="eyes" >/dev/null
|
||||
|
||||
- name: Resolve PR head and locate autofix run
|
||||
id: locate
|
||||
if: steps.perm.outputs.allowed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
PR: ${{ github.event.issue.number }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Same retry wrapper used in the permission step, repeated
|
||||
# because each YAML `run:` block is a fresh bash session.
|
||||
gh_retry() {
|
||||
local n=0 max=3
|
||||
while true; do
|
||||
if gh "$@"; then return 0; fi
|
||||
n=$((n+1))
|
||||
if [ "$n" -ge "$max" ]; then return 1; fi
|
||||
sleep $((n * 2))
|
||||
done
|
||||
}
|
||||
|
||||
# Fetch PR metadata. All fields here are server-controlled API
|
||||
# output, but we still allowlist before exporting so anything
|
||||
# weird short-circuits before $GITHUB_OUTPUT. Wrapped in
|
||||
# gh_retry so transient blips don't surface as "no autofix run
|
||||
# found" with a wrong remediation.
|
||||
if ! pr_json=$(gh_retry api "repos/${GH_REPO}/pulls/${PR}"); then
|
||||
echo "::error::PR metadata fetch failed after retries."
|
||||
echo "found_status=api-failed" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
head_sha=$(jq -r '.head.sha' <<< "${pr_json}")
|
||||
head_ref=$(jq -r '.head.ref' <<< "${pr_json}")
|
||||
head_repo=$(jq -r '.head.repo.full_name' <<< "${pr_json}")
|
||||
|
||||
[[ "${head_sha}" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Bad head_sha"; exit 1; }
|
||||
[[ "${head_ref}" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "::error::Bad head_ref"; exit 1; }
|
||||
[[ "${head_repo}" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "::error::Bad head_repo"; exit 1; }
|
||||
|
||||
# Find the latest successful pr-autofix.yml run for this head SHA.
|
||||
if ! runs_json=$(gh_retry api "repos/${GH_REPO}/actions/workflows/pr-autofix.yml/runs?head_sha=${head_sha}&per_page=10"); then
|
||||
echo "::error::Workflow run lookup failed after retries."
|
||||
echo "found_status=api-failed" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
run_id=$(jq -r '[.workflow_runs[] | select(.conclusion == "success")] | .[0].id // empty' <<< "${runs_json}")
|
||||
|
||||
if [ -n "${run_id}" ] && [[ "${run_id}" =~ ^[0-9]+$ ]]; then
|
||||
echo "found_status=success" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "found=true"
|
||||
echo "head_sha=${head_sha}"
|
||||
echo "head_ref=${head_ref}"
|
||||
echo "head_repo=${head_repo}"
|
||||
echo "run_id=${run_id}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# No successful run. Distinguish "still running" (producer in
|
||||
# flight after a recent push) from "never ran / all failed".
|
||||
# in_progress / queued / pending / waiting cover the GitHub
|
||||
# workflow-run lifecycle states that precede success/failure.
|
||||
in_progress=$(jq -r '[.workflow_runs[] | select(.status == "in_progress" or .status == "queued" or .status == "pending" or .status == "waiting")] | length' <<< "${runs_json}")
|
||||
if [ "${in_progress:-0}" -gt 0 ]; then
|
||||
echo "::warning::pr-autofix run is still in progress for head ${head_sha}."
|
||||
echo "found_status=in-progress" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::warning::No successful pr-autofix run found for head ${head_sha}."
|
||||
echo "found_status=not-found" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
# Existing `found` boolean is preserved so downstream gates
|
||||
# (`steps.locate.outputs.found == 'true'`) still work.
|
||||
echo "found=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Reply when locate did not yield a usable run
|
||||
if: steps.perm.outputs.allowed == 'true' && steps.locate.outputs.found != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
COMMENT_ID: ${{ github.event.comment.id }}
|
||||
PR: ${{ github.event.issue.number }}
|
||||
FOUND_STATUS: ${{ steps.locate.outputs.found_status }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
run_url="https://github.com/${GH_REPO}/actions/runs/${RUN_ID}"
|
||||
case "${FOUND_STATUS}" in
|
||||
in-progress)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="confused" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⏳ A pr-autofix run is still in progress for this PR's current head SHA. Wait for it to finish, then comment \`/autofix\` again. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
;;
|
||||
api-failed)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="confused" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⚠️ Couldn't reach the GitHub API to look up the autofix run (transient failure after retries). Please comment \`/autofix\` again. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
;;
|
||||
*)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="🤔 No successful autofix run found for this PR's current head SHA. Push a new commit to trigger one, then comment \`/autofix\` again." \
|
||||
>/dev/null
|
||||
;;
|
||||
esac
|
||||
exit 1
|
||||
|
||||
# Pinned to v8.0.1. Same SHA as pr-autofix-publish.yml.
|
||||
# `continue-on-error: true` lets the workflow proceed when the
|
||||
# artifact is expired or pruned (1-day retention). The apply
|
||||
# step distinguishes "patch file missing entirely" (artifact-
|
||||
# expired) from "patch file zero bytes" (genuinely empty patch).
|
||||
- name: Download autofix artifact
|
||||
if: steps.locate.outputs.found == 'true'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: autofix
|
||||
run-id: ${{ steps.locate.outputs.run_id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
path: autofix-in
|
||||
|
||||
# Pinned to v5.0.4. Verify SHA via:
|
||||
# gh api repos/actions/checkout/git/refs/tags/v5.0.4
|
||||
#
|
||||
# `persist-credentials: false` disables the default behavior where
|
||||
# actions/checkout writes the GITHUB_TOKEN into `.git/config` as an
|
||||
# extraheader. That default is convenient (subsequent git commands
|
||||
# auth automatically) but it means the token is sitting on disk in
|
||||
# the checkout directory — an `actions/upload-artifact` step on
|
||||
# this directory would leak the token. We don't upload, but
|
||||
# zizmor's `credential-persistence` lint flags it defensively.
|
||||
# Push auth is provided inline at push time via the URL.
|
||||
- name: Checkout PR head
|
||||
if: steps.locate.outputs.found == 'true'
|
||||
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.4
|
||||
with:
|
||||
repository: ${{ steps.locate.outputs.head_repo }}
|
||||
ref: ${{ steps.locate.outputs.head_sha }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
persist-credentials: false
|
||||
# Fetch full history so the push doesn't hit shallow-clone errors.
|
||||
fetch-depth: 0
|
||||
path: pr-checkout
|
||||
|
||||
- name: Apply patch and push
|
||||
id: apply
|
||||
if: steps.locate.outputs.found == 'true'
|
||||
env:
|
||||
HEAD_REF: ${{ steps.locate.outputs.head_ref }}
|
||||
HEAD_REPO: ${{ steps.locate.outputs.head_repo }}
|
||||
# The SHA we resolved earlier in `locate` — this is what the
|
||||
# remote ref MUST still equal at push time. If the contributor
|
||||
# force-pushed between resolve and now, the lease fails and
|
||||
# we surface that distinctly from a fork-without-maintainer
|
||||
# -edit push failure.
|
||||
HEAD_SHA: ${{ steps.locate.outputs.head_sha }}
|
||||
# Auth for the push only — never persisted to disk. Provided
|
||||
# via env to avoid interpolating into the shell command line.
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
shell: bash
|
||||
working-directory: pr-checkout
|
||||
run: |
|
||||
set -euo pipefail
|
||||
patch="../autofix-in/autofix.patch"
|
||||
|
||||
# Distinguish artifact-expired (file missing entirely, because
|
||||
# actions/download-artifact ran with continue-on-error and the
|
||||
# 1-day retention had elapsed) from genuinely empty patch
|
||||
# (file present, zero bytes, formatter found nothing).
|
||||
if [ ! -e "$patch" ]; then
|
||||
echo "::warning::Patch file does not exist — autofix artifact likely expired."
|
||||
echo "result=artifact-expired" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ ! -s "$patch" ]; then
|
||||
echo "::warning::Empty patch — nothing to apply."
|
||||
echo "result=empty-patch" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Sensitive-paths guard: refuse to apply patches that touch
|
||||
# `.github/` — workflow files, action definitions, CODEOWNERS,
|
||||
# dependabot config, etc. A malicious PR could ship a custom
|
||||
# prettier/ESLint config that reformats workflow YAML; the
|
||||
# producer would then capture those edits in autofix.patch,
|
||||
# and a maintainer running `/autofix` would push them under
|
||||
# `contents: write`. The default GITHUB_TOKEN lacks `workflows`
|
||||
# scope so the platform would reject workflow-file pushes
|
||||
# anyway, but that surfaces as a generic `push-failed` and
|
||||
# misleads users into enabling maintainer-edit. Reject early
|
||||
# with a specific reason. CODEOWNERS and dependabot.yml live
|
||||
# under .github/ but outside .github/workflows/ — the broader
|
||||
# match is intentional (they all govern trust boundaries).
|
||||
if grep -qE '^(diff --git|---|\+\+\+) [ab]?/?\.github/' "$patch"; then
|
||||
echo "::warning::Patch touches .github/ — refusing to apply (sensitive paths)."
|
||||
echo "result=sensitive-paths" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Re-entrancy guard: if HEAD itself is an autofix bot commit,
|
||||
# refuse to apply again. Without this, lint/formatter config
|
||||
# drift between runs could pump arbitrary apply commits into
|
||||
# the same PR if an automated agent watches the sticky and
|
||||
# re-fires `/autofix` on each new "fixes-available" surface.
|
||||
# The contributor can still get out by force-pushing a
|
||||
# human-authored commit to revert the autofix and re-trigger.
|
||||
head_author=$(git log -1 --format='%ae' HEAD)
|
||||
head_subject=$(git log -1 --format='%s' HEAD)
|
||||
if [ "${head_author}" = "41898282+github-actions[bot]@users.noreply.github.com" ] \
|
||||
&& [[ "${head_subject}" =~ ^chore\(autofix\) ]]; then
|
||||
echo "::warning::HEAD is an autofix bot commit — refusing to re-apply (loop guard)."
|
||||
echo "result=loop-prevented" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Idempotency probe: does the forward apply work?
|
||||
if git apply --check "$patch" 2>/dev/null; then
|
||||
echo "Patch applies cleanly — proceeding."
|
||||
elif git apply --check --reverse "$patch" 2>/dev/null; then
|
||||
# Reverse-check passes => the patch is already applied to
|
||||
# the current tree. Treat as success no-op.
|
||||
echo "Patch is already applied (reverse-check passed) — no-op."
|
||||
echo "result=already-applied" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
else
|
||||
echo "::error::Patch does not apply (stale or conflicting)."
|
||||
echo "result=stale" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Wrap the apply/commit phase so any non-zero exit sets a
|
||||
# meaningful `result=` instead of leaving it unset (which would
|
||||
# send the user to the `*` "unexpected state" arm with a
|
||||
# non-actionable confused-emoji reply).
|
||||
if ! {
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com" &&
|
||||
git config user.name "github-actions[bot]" &&
|
||||
git apply "$patch" &&
|
||||
git add -A &&
|
||||
git commit -m "chore(autofix): apply prettier + eslint fixes via /autofix command"
|
||||
}; then
|
||||
echo "::error::git apply / config / commit failed after idempotency probe passed."
|
||||
echo "result=apply-failed" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Push to the PR head branch with a lease against the resolved
|
||||
# SHA. The lease ensures the remote ref still points at HEAD_SHA
|
||||
# when the push lands — if the contributor force-pushed in the
|
||||
# window between resolve and now, the lease fails and we return
|
||||
# `lease-failed` (NOT `push-failed`, which would mislead users
|
||||
# into enabling maintainer-edit). For fork PRs, the push still
|
||||
# requires "Allow edits by maintainers" to be enabled.
|
||||
#
|
||||
# Auth is supplied inline via `-c http.<base>.extraheader` (NOT
|
||||
# via a `https://x-access-token:TOKEN@…` URL — those leak into
|
||||
# process listings and `git remote -v` output). The header is
|
||||
# set per-invocation; it never lands in `.git/config` on disk.
|
||||
# The token is base64-encoded for the Basic auth header per
|
||||
# GitHub's documented pattern for this scope.
|
||||
push_url="https://github.com/${HEAD_REPO}.git"
|
||||
auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 -w0)"
|
||||
# GitHub's secret-masker only masks the raw token, not its
|
||||
# base64-encoded form. Mask the encoded value so any subsequent
|
||||
# log line (set -x, GIT_TRACE, error spew) gets ***-redacted.
|
||||
echo "::add-mask::${auth_header}"
|
||||
push_stderr=$(mktemp)
|
||||
if git -c http.extraheader="${auth_header}" \
|
||||
push --force-with-lease="refs/heads/${HEAD_REF}:${HEAD_SHA}" \
|
||||
"${push_url}" "HEAD:${HEAD_REF}" 2>"$push_stderr"; then
|
||||
echo "result=applied" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
cat "$push_stderr" >&2
|
||||
# `--force-with-lease` reports "stale info" when the remote
|
||||
# ref has moved past the expected SHA. Other lease-failure
|
||||
# phrases git emits include "remote rejected" (server-side
|
||||
# reject), "non-fast-forward", and the literal flag name. Match
|
||||
# any of those to distinguish from auth/network/maintainer-
|
||||
# edit failures.
|
||||
if grep -qE "stale info|force-with-lease|rejected.*non-fast-forward|remote rejected|! \[rejected\]" "$push_stderr"; then
|
||||
echo "::error::git push lease failed — branch moved during apply."
|
||||
echo "result=lease-failed" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "::error::git push failed — likely fork without maintainer-edit enabled."
|
||||
echo "result=push-failed" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
- name: React and reply on outcome
|
||||
if: always() && steps.locate.outputs.found == 'true' && steps.apply.outcome != 'skipped'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
COMMENT_ID: ${{ github.event.comment.id }}
|
||||
PR: ${{ github.event.issue.number }}
|
||||
RESULT: ${{ steps.apply.outputs.result }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
run_url="https://github.com/${GH_REPO}/actions/runs/${RUN_ID}"
|
||||
|
||||
case "${RESULT}" in
|
||||
applied)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="+1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="✅ Applied autofix and pushed a commit. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
;;
|
||||
already-applied)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="+1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="✅ Autofix is already applied — no changes needed." \
|
||||
>/dev/null
|
||||
;;
|
||||
empty-patch)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="+1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="✅ No autofix to apply — formatter found nothing." \
|
||||
>/dev/null
|
||||
;;
|
||||
artifact-expired)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="confused" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⏳ The autofix artifact for this PR's head SHA has expired (1-day retention). Push a new commit to regenerate it, then comment \`/autofix\` again. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
loop-prevented)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="confused" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="🔁 Refusing to re-apply autofix on top of an existing autofix commit. If formatter rules drifted and you genuinely need another pass, push a human-authored commit (or revert the existing autofix commit) before commenting \`/autofix\` again. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
sensitive-paths)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="🛑 Refusing to apply: the autofix patch touches files under \`.github/\` (workflow / CODEOWNERS / dependabot config). Apply formatter changes to those files manually in a regular commit so they get human review. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
stale)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⚠️ The autofix patch is stale or conflicts with the current head — push a new commit to regenerate, then comment \`/autofix\` again. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
apply-failed)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⚠️ Autofix applied cleanly in the dry run, but \`git apply\` / \`git commit\` failed when actually landing the patch. This usually means a race with concurrent edits or a corrupt patch. See logs: ${run_url}" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
push-failed)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⚠️ Couldn't push the autofix commit. If this is a fork PR, please tick **Allow edits by maintainers** in the PR sidebar, then comment \`/autofix\` again. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
lease-failed)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="-1" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="⚠️ The PR head moved while autofix was applying — a new commit landed in the window between resolve and push. Comment \`/autofix\` again to retry against the latest head. ([apply run](${run_url}))" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \
|
||||
-f content="confused" >/dev/null
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="❓ Autofix run finished in an unexpected state (\`${RESULT:-unknown}\`). See logs: ${run_url}" \
|
||||
>/dev/null
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
316
.github/workflows/pr-autofix-publish.yml
vendored
Normal file
316
.github/workflows/pr-autofix-publish.yml
vendored
Normal file
|
|
@ -0,0 +1,316 @@
|
|||
name: PR Autofix (publish)
|
||||
|
||||
# TRUSTED HALF of the autofix pipeline.
|
||||
#
|
||||
# Triggered by `pr-autofix.yml` completing on a PR (including fork PRs).
|
||||
# Downloads the diff artifact produced by the untrusted job, verifies
|
||||
# its claimed PR identity against the workflow_run authority, then
|
||||
# posts (or edits) a single sticky summary comment plus a
|
||||
# `gitnexus/autofix` Check Run. This job NEVER checks out fork code —
|
||||
# it only consumes the diff (data) and calls the GitHub API. That
|
||||
# isolation is what makes it safe to run under `pull-requests: write`
|
||||
# on fork-triggered events.
|
||||
#
|
||||
# The sticky comment is the contributor signal: heading
|
||||
# "## :sparkles: PR Autofix" in the PR's top-level comments, with a
|
||||
# fenced `gitnexus-autofix` JSON block carrying machine-readable state
|
||||
# for AI agents. Contributors apply the patch by commenting `/autofix`
|
||||
# on the PR — handled by the separate `pr-autofix-apply.yml` workflow.
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ['PR Autofix']
|
||||
types: [completed]
|
||||
|
||||
concurrency:
|
||||
# Key on PR identity, NOT workflow_run.id — workflow_run.id is per-run
|
||||
# unique, which would defeat serialization and let two parallel
|
||||
# publishes both POST a sticky summary comment. CONTRIBUTING.md
|
||||
# § GitHub Actions — Concurrency Convention names this anti-pattern
|
||||
# explicitly. For fork PRs, `pull_requests[]` is empty in the
|
||||
# workflow_run payload, so we fall back to head-repo + head-branch.
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: publish-autofix
|
||||
if: >-
|
||||
github.event.workflow_run.event == 'pull_request'
|
||||
&& github.event.workflow_run.conclusion == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
pull-requests: write
|
||||
# Required by actions/download-artifact to fetch artifacts produced
|
||||
# by a different workflow run.
|
||||
actions: read
|
||||
# Required to create the `gitnexus/autofix` Check Run that reports
|
||||
# the outcome (clean / fixes-available) to the PR's Checks tab.
|
||||
# Branch protection or agents can grep the conclusion + output
|
||||
# title without parsing the sticky comment.
|
||||
checks: write
|
||||
steps:
|
||||
# Pinned to v8.0.1. Verify SHA via:
|
||||
# gh api repos/actions/download-artifact/git/refs/tags/v8.0.1
|
||||
- name: Download autofix artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: autofix
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
path: autofix-in
|
||||
|
||||
- name: Read and validate metadata
|
||||
id: meta
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test -f autofix-in/metadata.json
|
||||
jq . autofix-in/metadata.json
|
||||
|
||||
# The artifact comes from the untrusted half running fork code.
|
||||
# Every field is allowlist-validated before it can flow into
|
||||
# $GITHUB_OUTPUT. A newline in head_ref would otherwise let a
|
||||
# malicious branch name inject a second `pr_number=N` line and
|
||||
# redirect this job's reviewdog suggestions / sticky summary
|
||||
# comment onto a victim PR under github-actions[bot] with
|
||||
# pull-requests: write.
|
||||
assert_field() {
|
||||
local key="$1" pattern="$2" value
|
||||
value=$(jq -r ".${key} // empty" autofix-in/metadata.json)
|
||||
if [ -z "$value" ] || ! [[ "$value" =~ $pattern ]]; then
|
||||
echo "::error::metadata.${key} failed allowlist (got: $(printf '%q' "$value"))"
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
SCHEMA=$(assert_field schema '^gitnexus\.pr-autofix/v[0-9]+$')
|
||||
PR_NUMBER=$(assert_field pr_number '^[0-9]+$')
|
||||
HEAD_SHA=$(assert_field head_sha '^[0-9a-f]{40}$')
|
||||
HEAD_REF=$(assert_field head_ref '^[A-Za-z0-9._/-]+$')
|
||||
HEAD_REPO=$(assert_field head_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$')
|
||||
BASE_REPO=$(assert_field base_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$')
|
||||
CHANGED=$(assert_field changed_lines '^[0-9]+$')
|
||||
|
||||
# Defence-in-depth: refuse to act if the artifact claims to
|
||||
# belong to a different repo than the one that triggered us.
|
||||
if [ "$BASE_REPO" != "${GITHUB_REPOSITORY}" ]; then
|
||||
echo "::error::Artifact base_repo does not match \$GITHUB_REPOSITORY — refusing to publish."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
{
|
||||
echo "schema=${SCHEMA}"
|
||||
echo "pr_number=${PR_NUMBER}"
|
||||
echo "head_sha=${HEAD_SHA}"
|
||||
echo "head_ref=${HEAD_REF}"
|
||||
echo "head_repo=${HEAD_REPO}"
|
||||
echo "base_repo=${BASE_REPO}"
|
||||
echo "changed_lines=${CHANGED}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Cross-verify the artifact's claimed identity against the
|
||||
# GitHub-controlled workflow_run event. The previous step's
|
||||
# allowlist only proves the fields are well-formed — not that
|
||||
# they refer to the PR/SHA that actually triggered this run.
|
||||
# A fork-controlled `npm run lint:fix` could plausibly mutate
|
||||
# metadata.json to reference another PR or SHA, redirecting our
|
||||
# write-scoped sticky/check-run onto an attacker-chosen target.
|
||||
#
|
||||
# Authority sources are all server-controlled GitHub event fields:
|
||||
# - workflow_run.head_sha
|
||||
# - workflow_run.head_repository.full_name
|
||||
# - workflow_run.pull_requests[].number (within-repo PRs only;
|
||||
# empty array on fork PRs — fall back to commits/{sha}/pulls)
|
||||
#
|
||||
# Mismatch => fail loud BEFORE any sticky/check-run side effect.
|
||||
- name: Verify metadata against workflow_run authority
|
||||
id: verify
|
||||
if: steps.meta.outputs.changed_lines != '0'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
META_PR_NUMBER: ${{ steps.meta.outputs.pr_number }}
|
||||
META_HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
META_HEAD_REPO: ${{ steps.meta.outputs.head_repo }}
|
||||
WF_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
WF_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }}
|
||||
WF_PR_NUMBERS: ${{ toJSON(github.event.workflow_run.pull_requests.*.number) }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# 1) head_sha must match exactly. workflow_run.head_sha is the
|
||||
# commit GitHub actually ran the producer against — definitive.
|
||||
if [ "${META_HEAD_SHA}" != "${WF_HEAD_SHA}" ]; then
|
||||
echo "::error::Artifact head_sha (${META_HEAD_SHA}) does not match workflow_run.head_sha (${WF_HEAD_SHA}) — refusing to publish."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 2) head_repo must match exactly. Same authority anchor.
|
||||
if [ "${META_HEAD_REPO}" != "${WF_HEAD_REPO}" ]; then
|
||||
echo "::error::Artifact head_repo (${META_HEAD_REPO}) does not match workflow_run.head_repository (${WF_HEAD_REPO}) — refusing to publish."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3) pr_number must reference an open PR with this head SHA.
|
||||
# Within-repo PRs: workflow_run.pull_requests[] is populated.
|
||||
# Fork PRs: that array is empty by GitHub design — fall back
|
||||
# to the REST commit-to-PRs lookup. Fail closed if the lookup
|
||||
# finds no matching open PR (avoids attacker-forged PR ids).
|
||||
allowed_numbers=$(jq -c '.' <<< "${WF_PR_NUMBERS}")
|
||||
if [ "${allowed_numbers}" = "[]" ]; then
|
||||
echo "workflow_run.pull_requests is empty (fork PR) — falling back to commits/{sha}/pulls."
|
||||
allowed_numbers=$(gh api "repos/${GH_REPO}/commits/${WF_HEAD_SHA}/pulls" \
|
||||
--jq '[.[] | select(.state == "open") | .number]' 2>/dev/null || echo "[]")
|
||||
if [ "${allowed_numbers}" = "[]" ]; then
|
||||
echo "::error::No open PR found for head ${WF_HEAD_SHA} via commits/{sha}/pulls — refusing to publish."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! jq -e --argjson n "${META_PR_NUMBER}" 'index($n) != null' <<< "${allowed_numbers}" >/dev/null; then
|
||||
echo "::error::Artifact pr_number (${META_PR_NUMBER}) is not in the authoritative PR list (${allowed_numbers}) — refusing to publish."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Verified: metadata identity matches workflow_run authority (PR=${META_PR_NUMBER}, head_sha=${META_HEAD_SHA}, head_repo=${META_HEAD_REPO})."
|
||||
|
||||
- name: Upsert sticky summary comment
|
||||
# Only post when ci-quality found something fixable (= the
|
||||
# autofix patch is non-empty). When prettier/eslint are clean
|
||||
# the patch is zero bytes and the sticky comment is pure noise,
|
||||
# so we skip it.
|
||||
if: >-
|
||||
always()
|
||||
&& steps.meta.outputs.pr_number != ''
|
||||
&& steps.meta.outputs.changed_lines != '0'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
PR: ${{ steps.meta.outputs.pr_number }}
|
||||
CHANGED: ${{ steps.meta.outputs.changed_lines }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Stable heading + marker — agents grep for these exact strings.
|
||||
marker="<!-- gitnexus:pr-autofix-summary -->"
|
||||
heading="## :sparkles: PR Autofix"
|
||||
|
||||
# Single state. The /autofix slash command works for any diff
|
||||
# size — there's no 3K cap and no no-overlap dead-end because
|
||||
# the apply workflow uses `git apply` + push, not the GitHub
|
||||
# review-comment API.
|
||||
ui_state="fixes-available"
|
||||
prose="Found fixable formatting / unused-import issues across **${CHANGED}** changed lines. **Comment \`/autofix\` on this PR to apply them**, or run \`npm run lint:fix && npm run format\` locally."
|
||||
|
||||
# Machine-readable JSON block — agents parse this instead of
|
||||
# regexing English. Fenced code-block info string is
|
||||
# `gitnexus-autofix` so agents can locate it without ambiguity.
|
||||
# Schema bumped from v1 -> v2: adds `apply_command`. The v1
|
||||
# field set is preserved as a superset, but the `state` enum
|
||||
# is redefined (v1: suggestions-posted | skipped-too-large |
|
||||
# diff-no-overlap; v2: fixes-available). v1 readers checking
|
||||
# `schema == 'gitnexus.pr-autofix/v1'` see an unfamiliar version
|
||||
# and fall back to prose, which is the intended migration path.
|
||||
json=$(jq -n -c \
|
||||
--arg state "${ui_state}" \
|
||||
--argjson pr_number "${PR}" \
|
||||
--argjson changed_lines "${CHANGED}" \
|
||||
--arg head_sha "${HEAD_SHA}" \
|
||||
--arg run_id "${RUN_ID}" \
|
||||
'{schema:"gitnexus.pr-autofix/v2", state:$state, pr_number:$pr_number, changed_lines:$changed_lines, head_sha:$head_sha, run_id:$run_id, apply_command:"/autofix"}')
|
||||
|
||||
# Multi-line quoted string instead of a column-0 heredoc — YAML's
|
||||
# `run: |` block ends as soon as a content line dedents below the
|
||||
# block's first-line indent, which would mis-parse the workflow.
|
||||
body="${marker}
|
||||
${heading}
|
||||
|
||||
${prose}
|
||||
|
||||
\`\`\`gitnexus-autofix
|
||||
${json}
|
||||
\`\`\`"
|
||||
# Strip the leading 10-space indent that the YAML block requires
|
||||
# so the rendered comment body starts at column 0.
|
||||
body="$(printf '%s\n' "$body" | sed 's/^ //')"
|
||||
|
||||
# Small retry wrapper for transient 5xx / rate-limit responses
|
||||
# on the GitHub REST API. Three tries with linear backoff. We
|
||||
# only retry GET (idempotent) and PATCH on a known comment id
|
||||
# (idempotent). POST is NOT wrapped — retrying a comment-create
|
||||
# would create duplicates if the first attempt actually landed.
|
||||
gh_retry() {
|
||||
local n=0 max=3
|
||||
while true; do
|
||||
if gh "$@"; then return 0; fi
|
||||
n=$((n+1))
|
||||
if [ "$n" -ge "$max" ]; then return 1; fi
|
||||
sleep $((n * 2))
|
||||
done
|
||||
}
|
||||
|
||||
# Find existing bot comment by the marker and edit-in-place; else create.
|
||||
# CRITICAL: filter by `.user.login == "github-actions[bot]"`. A regular
|
||||
# user posting a comment containing the marker would otherwise be the
|
||||
# `head -n1` match; PATCH on someone else's comment 403s, `set -e`
|
||||
# aborts, and the bot is permanently DoS'd for that PR.
|
||||
existing=$(gh_retry api "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
--paginate --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"${marker}\"))) | .id" \
|
||||
| head -n1 || true)
|
||||
|
||||
if [ -n "${existing}" ]; then
|
||||
gh_retry api -X PATCH "repos/${GH_REPO}/issues/comments/${existing}" \
|
||||
-f body="${body}" >/dev/null
|
||||
echo "Updated comment ${existing}."
|
||||
else
|
||||
gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \
|
||||
-f body="${body}" >/dev/null
|
||||
echo "Created summary comment."
|
||||
fi
|
||||
|
||||
- name: Emit gitnexus/autofix Check Run
|
||||
# Stable check name `gitnexus/autofix` so PR-watching agents can
|
||||
# `gh pr checks <pr>` and read the conclusion + title without
|
||||
# parsing the sticky comment. Two outcomes:
|
||||
# clean → conclusion: success
|
||||
# fixes-available → conclusion: neutral
|
||||
# `neutral` does not block branch-protection required-checks but
|
||||
# is visually distinct from a green pass.
|
||||
if: always() && steps.meta.outputs.head_sha != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
HEAD_SHA: ${{ steps.meta.outputs.head_sha }}
|
||||
CHANGED: ${{ steps.meta.outputs.changed_lines }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [ "${CHANGED}" = "0" ]; then
|
||||
conclusion="success"
|
||||
title="Formatting clean"
|
||||
summary="Prettier and ESLint --fix produced no changes."
|
||||
else
|
||||
conclusion="neutral"
|
||||
title="Autofix available — comment /autofix to apply"
|
||||
summary="Comment \`/autofix\` on this PR to apply formatter + unused-import fixes (works at any diff size). Or run \`npm run lint:fix && npm run format\` locally."
|
||||
fi
|
||||
|
||||
gh api -X POST "repos/${GH_REPO}/check-runs" \
|
||||
-f name="gitnexus/autofix" \
|
||||
-f head_sha="${HEAD_SHA}" \
|
||||
-f status="completed" \
|
||||
-f conclusion="${conclusion}" \
|
||||
-f "output[title]=${title}" \
|
||||
-f "output[summary]=${summary}" \
|
||||
>/dev/null
|
||||
echo "Posted check-run gitnexus/autofix=${conclusion} (${title})"
|
||||
146
.github/workflows/pr-autofix.yml
vendored
Normal file
146
.github/workflows/pr-autofix.yml
vendored
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
name: PR Autofix
|
||||
|
||||
# UNTRUSTED HALF of the autofix pipeline.
|
||||
#
|
||||
# Runs `npm run lint:fix` + `npm run format` against the PR head
|
||||
# (including fork heads) and uploads the resulting diff as an artifact.
|
||||
# This job has NO privileged token and CANNOT post to the PR. The trusted
|
||||
# `pr-autofix-publish.yml` workflow downloads the artifact via
|
||||
# `workflow_run` and posts a sticky summary comment + Check Run.
|
||||
# Contributors apply the patch by commenting `/autofix` on the PR —
|
||||
# handled by the separate `pr-autofix-apply.yml` ChatOps workflow.
|
||||
#
|
||||
# Why the split:
|
||||
# ESLint loads plugins from fork-controlled `node_modules`, so running
|
||||
# it in a job with `pull-requests: write` would let a malicious fork PR
|
||||
# ship a poisoned eslint plugin and execute arbitrary code under that
|
||||
# token. By keeping fork code execution in this job (token: read-only)
|
||||
# and posting from a separate trusted job that never touches fork
|
||||
# code, we get the autofix UX for fork PRs without the supply-chain
|
||||
# hole. (See autofix.ci for the same pattern.)
|
||||
#
|
||||
# Removes unused imports via `eslint-plugin-unused-imports`, already in
|
||||
# devDependencies and wired into the `lint` config.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened]
|
||||
# Skip lockfile / generated-file PRs entirely — `action-suggester`
|
||||
# cannot post on diffs > ~3k lines (GitHub returns 406) and these
|
||||
# paths produce massive diffs no human wants suggested back inline.
|
||||
paths-ignore:
|
||||
- '**/package-lock.json'
|
||||
- '**/*.snap'
|
||||
- '**/dist/**'
|
||||
- '**/node_modules/**'
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
|
||||
# Don't cancel in-flight runs; the publish workflow may already be
|
||||
# downloading the artifact and a cancelled untrusted run produces no
|
||||
# signal at all (worse DX than waiting).
|
||||
cancel-in-progress: false
|
||||
|
||||
# This workflow runs untrusted fork code. Top-level deny-all and NO
|
||||
# job-level grants — the job can only read its own checkout.
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
autofix:
|
||||
name: autofix
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# PR head commit (not the synthetic merge ref) — we need the
|
||||
# exact tree the contributor pushed so suggestions line up.
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 22
|
||||
cache: npm
|
||||
cache-dependency-path: package-lock.json
|
||||
|
||||
# `--ignore-scripts` blocks pre/postinstall lifecycle hooks. ESLint
|
||||
# plugins still load from node_modules (that is the actual escape
|
||||
# hatch on a typical fork), but this job has no token to abuse —
|
||||
# which is the whole point of the split.
|
||||
- run: npm ci --ignore-scripts
|
||||
|
||||
- name: ESLint --fix (removes unused imports)
|
||||
run: npm run lint:fix
|
||||
# Lint errors that --fix can't auto-resolve must not block the
|
||||
# diff artifact — partial fixes are still useful as suggestions.
|
||||
continue-on-error: true
|
||||
|
||||
- name: Prettier --write
|
||||
run: npm run format
|
||||
continue-on-error: true
|
||||
|
||||
- name: Capture diff and metadata
|
||||
id: capture
|
||||
# Pass GitHub-context values via env: rather than `${{ }}`
|
||||
# interpolated directly into the bash body. `head.ref` and
|
||||
# `head.repo.full_name` are fork-controlled strings; expanding
|
||||
# them into shell source is the canonical template-injection
|
||||
# vector zizmor flags. Even though this job has `permissions: {}`,
|
||||
# routing through env: makes it impossible for a future scope
|
||||
# grant to turn into RCE. Inside bash, reference as `$HEAD_REF`
|
||||
# etc. — the values are then plain strings, not code.
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
BASE_REPO: ${{ github.repository }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p autofix-out
|
||||
|
||||
# Produce a unified diff of the working tree vs. the PR head.
|
||||
# Empty diff => nothing to suggest; the publish job short-circuits.
|
||||
git diff --no-color > autofix-out/autofix.patch
|
||||
|
||||
# NOTE: `changed_lines` is the line-count of the patch file,
|
||||
# (hunk headers + context lines + added/removed). Surfaced in
|
||||
# the sticky comment so contributors and AI agents have a
|
||||
# quick size hint before invoking `/autofix`.
|
||||
changed_lines=$(wc -l < autofix-out/autofix.patch | tr -d ' ')
|
||||
echo "changed_lines=${changed_lines}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Carry PR identity over to the trusted job. workflow_run
|
||||
# context is base-repo-only, so the publish job needs these
|
||||
# to call the GitHub PR API on the right resource.
|
||||
# CONTRACT: keep this schema in sync with pr-autofix-publish.yml's
|
||||
# `assert_field` validators and the agent-facing JSON block in
|
||||
# the sticky comment. Bump `schema` when changing field names.
|
||||
jq -n \
|
||||
--arg schema 'gitnexus.pr-autofix/v1' \
|
||||
--argjson pr_number "${PR_NUMBER}" \
|
||||
--arg head_sha "${HEAD_SHA}" \
|
||||
--arg head_ref "${HEAD_REF}" \
|
||||
--arg head_repo "${HEAD_REPO}" \
|
||||
--arg base_repo "${BASE_REPO}" \
|
||||
--argjson changed_lines "${changed_lines}" \
|
||||
'{schema:$schema, pr_number:$pr_number, head_sha:$head_sha, head_ref:$head_ref, head_repo:$head_repo, base_repo:$base_repo, changed_lines:$changed_lines}' \
|
||||
> autofix-out/metadata.json
|
||||
|
||||
echo "--- metadata ---"
|
||||
cat autofix-out/metadata.json
|
||||
echo "--- diff (head) ---"
|
||||
head -c 2000 autofix-out/autofix.patch || true
|
||||
|
||||
# Pinned to v7.0.1. Verify SHA via:
|
||||
# gh api repos/actions/upload-artifact/git/refs/tags/v7.0.1
|
||||
- name: Upload autofix artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: autofix
|
||||
path: autofix-out/
|
||||
retention-days: 1
|
||||
if-no-files-found: error
|
||||
2
.github/workflows/publish.yml
vendored
2
.github/workflows/publish.yml
vendored
|
|
@ -35,7 +35,7 @@ jobs:
|
|||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 22
|
||||
registry-url: https://registry.npmjs.org
|
||||
# Hermetic install for the published artifact — no cache carry-over
|
||||
# from non-tag contexts. setup-node v5+ caches by default when a
|
||||
|
|
|
|||
18
.github/workflows/release-candidate.yml
vendored
18
.github/workflows/release-candidate.yml
vendored
|
|
@ -149,7 +149,7 @@ jobs:
|
|||
|
||||
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
||||
with:
|
||||
node-version: 20
|
||||
node-version: 22
|
||||
registry-url: https://registry.npmjs.org
|
||||
# Hermetic install — release-candidate produces shipped artifacts.
|
||||
# setup-node v5+ caches by default when a packageManager field is
|
||||
|
|
@ -294,9 +294,11 @@ jobs:
|
|||
fi
|
||||
fi
|
||||
|
||||
echo "base=$BASE" >> "$GITHUB_OUTPUT"
|
||||
echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT"
|
||||
echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "base=$BASE"
|
||||
echo "rc_n=$NEXT_N"
|
||||
echo "rc_version=$RC_VERSION"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Apply rc version in-CI
|
||||
shell: bash
|
||||
|
|
@ -354,9 +356,11 @@ jobs:
|
|||
# remote ref, the push fails and we stop before npm publish.
|
||||
git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER"
|
||||
|
||||
echo "vtag=$VTAG" >> "$GITHUB_OUTPUT"
|
||||
echo "marker=$MARKER" >> "$GITHUB_OUTPUT"
|
||||
echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "vtag=$VTAG"
|
||||
echo "marker=$MARKER"
|
||||
echo "release_sha=$RELEASE_SHA"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Publish to npm (rc dist-tag)
|
||||
run: npm publish --provenance --access public --tag rc
|
||||
|
|
|
|||
14
.github/workflows/trivy.yml
vendored
14
.github/workflows/trivy.yml
vendored
|
|
@ -1,13 +1,19 @@
|
|||
name: Trivy Image Scan
|
||||
|
||||
# Builds Dockerfile.cli and Dockerfile.web, then scans the resulting images
|
||||
# for OS-package and language-package CVEs at HIGH/CRITICAL severity.
|
||||
# for OS-package and language-package CVEs at MEDIUM+ severity.
|
||||
# Findings upload to the Security tab; record-only (does not block merges).
|
||||
#
|
||||
# NOT triggered on PRs — image builds are slow and base-image CVE churn
|
||||
# shouldn't gate feature delivery.
|
||||
# Trigger on Dockerfile changes in PRs so base-image/npm-layer remediation can
|
||||
# be verified before merge without running image scans on every PR.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'Dockerfile.cli'
|
||||
- 'Dockerfile.web'
|
||||
- 'gitnexus/Dockerfile.test'
|
||||
- '.github/workflows/trivy.yml'
|
||||
push:
|
||||
branches: [main]
|
||||
schedule:
|
||||
|
|
@ -61,7 +67,7 @@ jobs:
|
|||
image-ref: scan-target:${{ matrix.image.name }}
|
||||
format: sarif
|
||||
output: trivy-${{ matrix.image.name }}.sarif
|
||||
severity: HIGH,CRITICAL
|
||||
severity: MEDIUM,HIGH,CRITICAL
|
||||
# Hides CVEs with no available fix in the base image.
|
||||
ignore-unfixed: true
|
||||
exit-code: '0'
|
||||
|
|
|
|||
34
.github/workflows/workflow-lint.yml
vendored
34
.github/workflows/workflow-lint.yml
vendored
|
|
@ -1,10 +1,13 @@
|
|||
name: Workflow Lint (zizmor)
|
||||
name: Workflow Lint
|
||||
|
||||
# Lints .github/workflows/** for known GitHub Actions security misconfigurations:
|
||||
# unpinned Actions, dangerous ${{ ... }} interpolation in run: blocks,
|
||||
# missing per-job permissions:, etc.
|
||||
# Lints .github/workflows/** for both:
|
||||
# - actionlint: YAML syntax, expression typing, shellcheck inside `run:`
|
||||
# blocks, unknown contexts, deprecated runner labels.
|
||||
# - zizmor: security misconfigurations — unpinned actions, dangerous
|
||||
# `${{ }}` interpolation, missing per-job permissions, etc.
|
||||
#
|
||||
# Scoped to PRs that touch .github/** only — keeps off the typical PR critical path.
|
||||
# Scoped to PRs that touch .github/** only — keeps off the typical PR
|
||||
# critical path.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
|
|
@ -17,6 +20,27 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
actionlint:
|
||||
name: actionlint
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# Pinned to v2.1.2. Verify SHA via:
|
||||
# gh api repos/raven-actions/actionlint/git/refs/tags/v2.1.2
|
||||
# The action wraps the upstream `rhysd/actionlint` binary and emits
|
||||
# GitHub-annotation-formatted findings on PRs.
|
||||
- name: Run actionlint
|
||||
uses: raven-actions/actionlint@205b530c5d9fa8f44ae9ed59f341a0db994aa6f8 # v2.1.2
|
||||
with:
|
||||
fail-on-error: true
|
||||
|
||||
zizmor:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
|
|
|||
9
.github/zizmor.yml
vendored
9
.github/zizmor.yml
vendored
|
|
@ -14,6 +14,15 @@ rules:
|
|||
# no checkout of fork code occurs. Header comment in the file documents.
|
||||
- ci-report.yml
|
||||
|
||||
# workflow_run is the trusted half of the autofix pipeline. The
|
||||
# untrusted half (pr-autofix.yml) runs fork code with permissions:{}
|
||||
# and produces only a diff artifact (data, not executable code). The
|
||||
# publish job consumes the artifact, allowlist-validates every field
|
||||
# of metadata.json before exporting to $GITHUB_OUTPUT, never checks
|
||||
# out fork code, and never executes anything fork-controlled. Header
|
||||
# comment in the file documents the split.
|
||||
- pr-autofix-publish.yml
|
||||
|
||||
# pull_request_target needed by claude-code-action to access secrets
|
||||
# and post review comments on fork PRs. Mitigated by: PR checkouts pin
|
||||
# the fork's HEAD SHA (not the branch ref) to prevent TOCTOU races,
|
||||
|
|
|
|||
|
|
@ -30,17 +30,17 @@ Format: `<type>[(scope)][!]: <subject>`
|
|||
|
||||
Allowed types and the release-notes section each one lands in (defined in `.github/release.yml`):
|
||||
|
||||
| Type | Label applied | Release-notes section |
|
||||
|------|---------------|-----------------------|
|
||||
| `feat` | `enhancement` | 🚀 Features |
|
||||
| `fix` | `bug` | 🐛 Bug Fixes |
|
||||
| `perf` | `performance` | 🏎️ Performance |
|
||||
| `refactor` | `refactor` | 🔄 Refactoring |
|
||||
| `test` | `test` | 🧪 Tests |
|
||||
| `ci` | `ci` | 👷 CI/CD |
|
||||
| `build` / `deps` | `dependencies` | 📦 Dependencies |
|
||||
| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) |
|
||||
| `chore` / `revert` | `chore` | (excluded from release notes) |
|
||||
| Type | Label applied | Release-notes section |
|
||||
| ------------------ | --------------- | ------------------------------------------------------------ |
|
||||
| `feat` | `enhancement` | 🚀 Features |
|
||||
| `fix` | `bug` | 🐛 Bug Fixes |
|
||||
| `perf` | `performance` | 🏎️ Performance |
|
||||
| `refactor` | `refactor` | 🔄 Refactoring |
|
||||
| `test` | `test` | 🧪 Tests |
|
||||
| `ci` | `ci` | 👷 CI/CD |
|
||||
| `build` / `deps` | `dependencies` | 📦 Dependencies |
|
||||
| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) |
|
||||
| `chore` / `revert` | `chore` | (excluded from release notes) |
|
||||
|
||||
Append `!` to the type (e.g. `feat(api)!: drop /v1 endpoint`) or include `BREAKING CHANGE:` in the PR body to flag a breaking change — the labeler then adds the `breaking` label and the 💥 Breaking Changes section is rendered first.
|
||||
|
||||
|
|
@ -81,17 +81,17 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency:
|
|||
- **Merge queue (`merge_group`)**: when this event is added, use `${{ github.workflow }}-${{ github.event.merge_group.head_ref }}` with `cancel-in-progress: false` (every queue entry is a distinct ref; never cancel).
|
||||
- **`cancel-in-progress` policy:**
|
||||
|
||||
| Event | `cancel-in-progress` | Why |
|
||||
|-------|----------------------|-----|
|
||||
| `pull_request` CI run | `true` | New push supersedes old run |
|
||||
| `push` to `main` | `false` | Every main commit gets validated |
|
||||
| Tag push (`v*` publish) | `false` | Never cancel mid-publish |
|
||||
| `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish |
|
||||
| `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional |
|
||||
| `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race |
|
||||
| Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR |
|
||||
| PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins |
|
||||
| Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes |
|
||||
| Event | `cancel-in-progress` | Why |
|
||||
| ---------------------------------------- | -------------------- | -------------------------------- |
|
||||
| `pull_request` CI run | `true` | New push supersedes old run |
|
||||
| `push` to `main` | `false` | Every main commit gets validated |
|
||||
| Tag push (`v*` publish) | `false` | Never cancel mid-publish |
|
||||
| `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish |
|
||||
| `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional |
|
||||
| `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race |
|
||||
| Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR |
|
||||
| PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins |
|
||||
| Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes |
|
||||
|
||||
- For workflows that serve multiple events at once (e.g. `ci.yml` handles `pull_request`, `push`, and `workflow_call`), make `cancel-in-progress` event-aware:
|
||||
|
||||
|
|
@ -103,6 +103,41 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency:
|
|||
|
||||
- When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape.
|
||||
|
||||
## CI automation contracts
|
||||
|
||||
Two workflows produce machine-readable signals on every PR. Coding agents and humans alike can rely on the names and shapes below — change them with intent.
|
||||
|
||||
### `gitnexus/autofix`
|
||||
|
||||
`pr-autofix.yml` (untrusted) + `pr-autofix-publish.yml` (trusted) run `prettier --write` and `eslint --fix` against the PR head and surface a single ChatOps button on the PR. Three signals are emitted:
|
||||
|
||||
| Surface | Where | Notes |
|
||||
| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- |
|
||||
| Sticky PR comment | Top-level comment with the HTML marker `<!-- gitnexus:pr-autofix-summary -->` and heading `## :sparkles: PR Autofix`. Only posted when there is something to fix; clean PRs stay silent. | Edit-in-place via marker; one comment per PR. |
|
||||
| Fenced JSON block | Inside the sticky, fenced as `gitnexus-autofix`. Schema `gitnexus.pr-autofix/v2` with fields `state` (`fixes-available`), `pr_number`, `head_sha`, `changed_lines`, `run_id`, and `apply_command` (literal `/autofix`). | Parseable signal — preferred over regexing prose. v1 fields preserved as a superset. |
|
||||
| Check Run | Stable name `gitnexus/autofix` on the PR head SHA. Conclusion: `success` (clean) or `neutral` (`fixes-available`). The neutral title is `Autofix available — comment /autofix to apply`. | Surfaced under PR Checks; readable via `gh pr checks <pr>`. |
|
||||
|
||||
To detect outcome from an agent: `gh pr checks <pr> --json name,conclusion,output | jq '.[] | select(.name == "gitnexus/autofix")'`.
|
||||
|
||||
Forks are supported. The untrusted half runs fork code with `permissions: {}` and ships the diff as an artifact; the trusted publish job consumes only the diff (data, not code) and posts the comment + check run.
|
||||
|
||||
#### Applying autofix
|
||||
|
||||
Comment `/autofix` on the PR (whole-line, no arguments). The `pr-autofix-apply.yml` workflow:
|
||||
|
||||
1. Validates the comment body matches `^/autofix\s*$` exactly. Quoted or inline mentions are silently ignored.
|
||||
2. Validates the commenter has `admin`, `write`, or `maintain` permission on the repo, OR is the PR author. Other commenters get a 👎 reaction and a refusal reply.
|
||||
3. Locates the most recent successful `pr-autofix.yml` run for the PR's current head SHA, downloads its `autofix` artifact, applies the patch, and pushes a `chore(autofix): ...` commit back to the PR head branch.
|
||||
4. Reacts ✅ on success, 👎 on stale-patch / push-failure, and posts a short reply with the apply-run URL in either case.
|
||||
|
||||
The apply workflow runs from the default branch's copy of the file regardless of where the comment originates — that's the trust anchor. There is no diff-size cap (the apply workflow uses `git apply` + push, not the GitHub review-comment API).
|
||||
|
||||
For fork PRs, the push succeeds only when the contributor has **Allow edits by maintainers** enabled on the PR (the default). When they have disabled it, the workflow fails loud with a 👎 reaction and an explanation comment.
|
||||
|
||||
Re-invoking `/autofix` after a successful apply is a safe no-op — the workflow detects the already-applied state via `git apply --check --reverse` and reacts ✅ without pushing.
|
||||
|
||||
**Sensitive paths.** The apply workflow refuses any patch that touches `.github/` (workflow files, CODEOWNERS, dependabot config). A malicious PR could ship a custom prettier or ESLint config that reformats workflow YAML; if accepted, those edits would be pushed under `contents: write` without human review. Apply formatter changes to files under `.github/` manually in a normal commit so they get the same review every other workflow change gets.
|
||||
|
||||
## AI-assisted contributions
|
||||
|
||||
If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes.
|
||||
|
|
@ -164,8 +199,7 @@ Two publish workflows ship `gitnexus` to npm:
|
|||
the Docker build.
|
||||
- Manually run `docker build` + `docker push` locally and sign with Cosign
|
||||
against the same digest.
|
||||
- Delete `rc/<HEAD_SHA>` and `v<RC>` tags, then redispatch with `force:
|
||||
true` to re-run the full RC pipeline (cuts a new RC number).
|
||||
- Delete `rc/<HEAD_SHA>` and `v<RC>` tags, then redispatch with `force: true` to re-run the full RC pipeline (cuts a new RC number).
|
||||
|
||||
The rc workflow never moves `latest`. To verify after a change, inspect dist-tags:
|
||||
|
||||
|
|
|
|||
|
|
@ -1,24 +1,32 @@
|
|||
ARG BUILDPLATFORM
|
||||
ARG TARGETPLATFORM
|
||||
# Pinned npm version used to replace the bundled npm in the upstream Node
|
||||
# image. Bumping requires a coordinated update in Dockerfile.web and
|
||||
# gitnexus/Dockerfile.test so all images bootstrap the same npm.
|
||||
ARG NPM_VERSION=11.14.1
|
||||
|
||||
# ── Builder ────────────────────────────────────────────────────────────
|
||||
# -- Builder -----------------------------------------------------------
|
||||
# Native modules (tree-sitter-*, onnxruntime-node, node-gyp builds for
|
||||
# tree-sitter-proto / tree-sitter-swift) require python3 + a C/C++ toolchain.
|
||||
FROM node:22-trixie-slim AS builder
|
||||
# node:22-bookworm-slim
|
||||
FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder
|
||||
ARG NPM_VERSION
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION}
|
||||
|
||||
# Toolchain for node-gyp / native builds.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends python3 make g++ git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Build gitnexus-shared first — gitnexus depends on it as a workspace.
|
||||
# Build gitnexus-shared first - gitnexus depends on it as a workspace.
|
||||
COPY gitnexus-shared/package.json gitnexus-shared/package-lock.json ./gitnexus-shared/
|
||||
RUN npm ci --prefix gitnexus-shared
|
||||
COPY gitnexus-shared ./gitnexus-shared
|
||||
RUN rm -f gitnexus-shared/tsconfig.tsbuildinfo
|
||||
RUN npm run build --prefix gitnexus-shared
|
||||
|
||||
# Copy the full gitnexus package before installing — `npm ci` triggers
|
||||
# Copy the full gitnexus package before installing - `npm ci` triggers
|
||||
# `postinstall` (patches tree-sitter-swift, builds the vendored
|
||||
# tree-sitter-proto) and `prepare` (compiles TypeScript via scripts/build.js),
|
||||
# both of which need the source tree.
|
||||
|
|
@ -28,11 +36,15 @@ RUN npm ci --prefix gitnexus
|
|||
# Drop dev dependencies for a smaller runtime layer.
|
||||
RUN npm prune --omit=dev --prefix gitnexus
|
||||
|
||||
# ── Runtime ────────────────────────────────────────────────────────────
|
||||
FROM node:22-trixie-slim AS runtime
|
||||
# -- Runtime -----------------------------------------------------------
|
||||
# node:22-bookworm-slim
|
||||
FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime
|
||||
|
||||
# curl for the healthcheck; git so `gitnexus` can clone repos at runtime.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/*
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf /usr/local/lib/node_modules/npm \
|
||||
&& rm -rf /usr/local/lib/node_modules/corepack \
|
||||
&& rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
|
@ -47,7 +59,7 @@ COPY --from=builder --chown=node:node /app/gitnexus/vendor ./gitnexus/vendor
|
|||
|
||||
USER node
|
||||
|
||||
# The web UI defaults to http://localhost:4747 — keep that contract.
|
||||
# The web UI defaults to http://localhost:4747 - keep that contract.
|
||||
ENV GITNEXUS_HOME=/data/gitnexus \
|
||||
NODE_ENV=production \
|
||||
PORT=4747
|
||||
|
|
|
|||
|
|
@ -1,10 +1,17 @@
|
|||
ARG BUILDPLATFORM
|
||||
ARG TARGETPLATFORM
|
||||
# Pinned npm version — keep in sync with Dockerfile.cli and
|
||||
# gitnexus/Dockerfile.test.
|
||||
ARG NPM_VERSION=11.14.1
|
||||
|
||||
FROM --platform=$BUILDPLATFORM node:22-alpine AS builder
|
||||
# node:22-bookworm-slim
|
||||
FROM --platform=$BUILDPLATFORM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder
|
||||
ARG NPM_VERSION
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION}
|
||||
|
||||
COPY gitnexus-shared/package.json gitnexus-shared/package-lock.json ./gitnexus-shared/
|
||||
RUN npm ci --prefix gitnexus-shared
|
||||
|
||||
|
|
@ -19,9 +26,13 @@ RUN npm ci --prefix gitnexus-web
|
|||
COPY gitnexus-web ./gitnexus-web
|
||||
RUN npm run build --prefix gitnexus-web
|
||||
|
||||
FROM node:22-alpine AS runtime
|
||||
# node:22-bookworm-slim
|
||||
FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime
|
||||
|
||||
RUN apk add --no-cache curl
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* \
|
||||
&& rm -rf /usr/local/lib/node_modules/npm \
|
||||
&& rm -rf /usr/local/lib/node_modules/corepack \
|
||||
&& rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
|
|
|
|||
|
|
@ -214,6 +214,7 @@ gitnexus clean --all --force # Delete all indexes
|
|||
gitnexus wiki [path] # Generate repository wiki from knowledge graph
|
||||
gitnexus wiki --model <model> # Wiki with custom LLM model (default: gpt-4o-mini)
|
||||
gitnexus wiki --base-url <url> # Wiki with custom LLM API base URL
|
||||
gitnexus publish # Notify the understand-quickly registry (opt-in, see below)
|
||||
|
||||
# Repository groups (multi-repo / monorepo service tracking)
|
||||
gitnexus group create <name> # Create a repository group
|
||||
|
|
@ -228,6 +229,12 @@ gitnexus group status <name> # Check staleness of repos in a group
|
|||
|
||||
If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget.
|
||||
|
||||
#### Publishing to understand-quickly (opt-in)
|
||||
|
||||
[`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) is a public registry of code-knowledge graphs that lists `gitnexus@1` as a first-class format. After registering your repo once (`npx @understand-quickly/cli add` or the [wizard](https://looptech-ai.github.io/understand-quickly/add.html)), `gitnexus publish` fires a single `repository_dispatch` event so the registry resyncs your entry on demand instead of waiting for the nightly job.
|
||||
|
||||
It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained GitHub PAT with `Repository dispatches: write` on the registry repo. Nothing else happens; no graph file is uploaded. See the [protocol spec](https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md) for the full contract.
|
||||
|
||||
### What Your AI Agent Gets
|
||||
|
||||
**16 tools** exposed via MCP (11 per-repo + 5 group):
|
||||
|
|
|
|||
|
|
@ -10,6 +10,10 @@
|
|||
".": {
|
||||
"types": "./dist/index.d.ts",
|
||||
"default": "./dist/index.js"
|
||||
},
|
||||
"./test-helpers": {
|
||||
"types": "./dist/test-helpers.d.ts",
|
||||
"default": "./dist/test-helpers.js"
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
|
|
|
|||
|
|
@ -143,6 +143,34 @@ export type { ScopeTree } from './scope-resolution/scope-tree.js';
|
|||
export { buildPositionIndex } from './scope-resolution/position-index.js';
|
||||
export type { PositionIndex } from './scope-resolution/position-index.js';
|
||||
|
||||
// Resilient fetch primitives — bounded retries + per-process circuit breaker.
|
||||
// Test-only helpers (`__resetBreakerRegistry__`, `classifyOutcome`) are
|
||||
// reachable via the separate `gitnexus-shared/test-helpers` subpath; do
|
||||
// NOT add them here. Production consumers must not call them.
|
||||
export { withRetry, computeBackoffMs } from './integrations/retry.js';
|
||||
export type { RetryOptions, RetryDecision } from './integrations/retry.js';
|
||||
export { CircuitBreaker, CircuitOpenError, getBreaker } from './integrations/circuit-breaker.js';
|
||||
export type { CircuitBreakerOptions } from './integrations/circuit-breaker.js';
|
||||
export {
|
||||
resilientFetch,
|
||||
ResilientFetchExhaustedError,
|
||||
RETRY_AFTER_CAP_MS,
|
||||
parseRetryAfter,
|
||||
} from './integrations/resilient-fetch.js';
|
||||
export type { ResilientFetchOptions } from './integrations/resilient-fetch.js';
|
||||
|
||||
// Understand-Quickly registry integration (opt-in)
|
||||
export {
|
||||
UNDERSTAND_QUICKLY_DISPATCH_URL,
|
||||
UNDERSTAND_QUICKLY_EVENT_TYPE,
|
||||
UNDERSTAND_QUICKLY_TOKEN_ENV,
|
||||
buildUqDispatchPayload,
|
||||
isValidOwnerRepo,
|
||||
parseOwnerRepoFromRemote,
|
||||
stripGitSuffix,
|
||||
} from './integrations/understand-quickly.js';
|
||||
export type { UqDispatchPayload } from './integrations/understand-quickly.js';
|
||||
|
||||
// Shadow-mode diff + aggregation (RFC §6.3; Ring 2 SHARED #918)
|
||||
export { diffResolutions } from './scope-resolution/shadow/diff.js';
|
||||
export type {
|
||||
|
|
|
|||
273
gitnexus-shared/src/integrations/circuit-breaker.ts
Normal file
273
gitnexus-shared/src/integrations/circuit-breaker.ts
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
/**
|
||||
* Per-process circuit breaker.
|
||||
*
|
||||
* Closed -> Open transition fires after `failureThreshold` consecutive
|
||||
* failures. While Open, `check` throws `CircuitOpenError` until
|
||||
* `cooldownMs` has elapsed since the breaker tripped. The first call
|
||||
* after the cooldown enters Half-Open and consumes the *probe permit*:
|
||||
* a recorded success returns to Closed; a recorded failure flips back
|
||||
* to Open with a fresh timestamp.
|
||||
*
|
||||
* Half-open admits exactly one in-flight probe at a time. Concurrent
|
||||
* callers attempting `check()` while a probe is outstanding receive
|
||||
* `CircuitOpenError` with `retryAfterMs = halfOpenRetryAfterMs` (default
|
||||
* 1000ms; configurable). This prevents the recovery-time thundering
|
||||
* herd that defeats the breaker's "fail fast" promise.
|
||||
*
|
||||
* Outcome reporting splits permit-release from state-resolution:
|
||||
* - `recordSuccess` — releases the probe permit, resets the failure
|
||||
* counter, transitions to Closed. Reserved for true 2xx/3xx outcomes.
|
||||
* - `recordFailure` — releases the probe permit, increments the
|
||||
* consecutive-failure counter, transitions to Open with a fresh
|
||||
* `openedAt` (when called from Half-Open or when the threshold
|
||||
* trips from Closed).
|
||||
* - `recordNeutral` — releases the probe permit, BUT leaves state and
|
||||
* counter untouched. Used for outcomes that are neither evidence of
|
||||
* backend health nor evidence of backend failure (caller-driven
|
||||
* cancellation, local timeout, terminal 4xx client errors). Critical
|
||||
* design point: if `recordNeutral` did not release the permit, a
|
||||
* single `TimeoutError` from per-attempt `AbortSignal.timeout` would
|
||||
* route through `recordNeutral` and permanently park the breaker in
|
||||
* half-open until process restart. Releasing the permit while leaving
|
||||
* state half-open keeps the "neutral doesn't claim health" semantic
|
||||
* without creating that wedge.
|
||||
*
|
||||
* Pairing invariant: every successful `check()` MUST be paired with
|
||||
* exactly one `record*()` on every code path including throws. Direct
|
||||
* consumers should wrap the protected operation in `try/finally`:
|
||||
*
|
||||
* breaker.check();
|
||||
* try {
|
||||
* const result = await operation();
|
||||
* breaker.recordSuccess();
|
||||
* return result;
|
||||
* } catch (err) {
|
||||
* // classify err and call recordFailure / recordNeutral / etc.
|
||||
* throw err;
|
||||
* }
|
||||
*
|
||||
* `resilientFetch`'s catch-all on `fetchImpl` already satisfies this
|
||||
* for that consumer.
|
||||
*
|
||||
* Atomicity model: the half-open gate relies on JavaScript event-loop
|
||||
* single-threadedness within a synchronous `check()` body. There is no
|
||||
* `await` inside `check()`; concurrent callers serialize on microtask
|
||||
* order, and exactly one observes `probeInFlight === false`. Do not
|
||||
* introduce `await` inside `check()` without revisiting the gate. If
|
||||
* this code is ever ported to a runtime with shared-memory threads
|
||||
* (Node `worker_threads` with `SharedArrayBuffer`, Web Workers with
|
||||
* shared registries), the boolean must become an atomic CAS — Resilience4j
|
||||
* and Hystrix use atomic permits *because* they run in JVM thread pools.
|
||||
*
|
||||
* Runtime-agnostic: depends only on a `now()` clock and standard JS —
|
||||
* no Node-only imports. Tests inject `now` to advance the clock
|
||||
* deterministically without `vi.useFakeTimers()`.
|
||||
*/
|
||||
|
||||
export class CircuitOpenError extends Error {
|
||||
override readonly name = 'CircuitOpenError';
|
||||
/** Approximate wait time before the breaker may transition to Half-Open
|
||||
* (or before the in-flight probe is expected to resolve). */
|
||||
readonly retryAfterMs: number;
|
||||
|
||||
constructor(retryAfterMs: number, key?: string) {
|
||||
super(
|
||||
key
|
||||
? `Circuit '${key}' is open; retry in ${Math.ceil(retryAfterMs / 1000)}s`
|
||||
: `Circuit is open; retry in ${Math.ceil(retryAfterMs / 1000)}s`,
|
||||
);
|
||||
this.retryAfterMs = retryAfterMs;
|
||||
}
|
||||
}
|
||||
|
||||
export interface CircuitBreakerOptions {
|
||||
/** Consecutive failures required to trip Closed -> Open. */
|
||||
failureThreshold?: number;
|
||||
/** Milliseconds Open before the next call may probe (Half-Open). */
|
||||
cooldownMs?: number;
|
||||
/**
|
||||
* Milliseconds to suggest in `CircuitOpenError.retryAfterMs` when the
|
||||
* breaker is Half-Open with the probe permit consumed. Default 1000ms.
|
||||
* Consumers with long-running protected ops (LLM streaming, large
|
||||
* uploads) should raise this — the cooldown clock is no longer the
|
||||
* right answer because cooldown has elapsed. Returning 0 invites
|
||||
* retry storms; returning the full cooldown misleads about wait.
|
||||
*/
|
||||
halfOpenRetryAfterMs?: number;
|
||||
/** Optional key for error messages and registry lookups. */
|
||||
key?: string;
|
||||
/** Clock override — defaults to `Date.now`. Tests inject deterministic time. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
type State = 'closed' | 'open' | 'half-open';
|
||||
|
||||
export class CircuitBreaker {
|
||||
private readonly failureThreshold: number;
|
||||
private readonly cooldownMs: number;
|
||||
private readonly halfOpenRetryAfterMs: number;
|
||||
private readonly key: string | undefined;
|
||||
private readonly now: () => number;
|
||||
|
||||
private state: State = 'closed';
|
||||
private consecutiveFailures = 0;
|
||||
private openedAt: number | null = null;
|
||||
/**
|
||||
* True between a successful `check()` and the next `record*()` call
|
||||
* during Half-Open. Gates concurrent callers from stampeding a still-
|
||||
* recovering dependency. Boolean rather than counter — single-permit
|
||||
* is the conservative end of the Hystrix/Resilience4j spectrum.
|
||||
*/
|
||||
private probeInFlight = false;
|
||||
|
||||
constructor(opts: CircuitBreakerOptions = {}) {
|
||||
this.failureThreshold = opts.failureThreshold ?? 3;
|
||||
this.cooldownMs = opts.cooldownMs ?? 30_000;
|
||||
this.halfOpenRetryAfterMs = opts.halfOpenRetryAfterMs ?? 1_000;
|
||||
this.key = opts.key;
|
||||
this.now = opts.now ?? (() => Date.now());
|
||||
}
|
||||
|
||||
/**
|
||||
* Throw `CircuitOpenError` if the breaker won't admit this call.
|
||||
* Otherwise consume the half-open probe permit (if applicable) and
|
||||
* return so the caller can attempt the protected work.
|
||||
*
|
||||
* Three rejection paths:
|
||||
* 1. Open and still in cooldown → throws with `retryAfterMs` =
|
||||
* remaining cooldown.
|
||||
* 2. Open with cooldown elapsed AND a probe is already in flight
|
||||
* (race: another caller transitioned to half-open and grabbed
|
||||
* the permit on a microtask before us) → throws with
|
||||
* `halfOpenRetryAfterMs`.
|
||||
* 3. Half-Open with probe in flight → throws with `halfOpenRetryAfterMs`.
|
||||
*
|
||||
* **Pairing invariant**: every successful return from `check()` MUST
|
||||
* be paired with exactly one `recordSuccess` / `recordFailure` /
|
||||
* `recordNeutral` on every code path including thrown exceptions.
|
||||
* Failing to pair leaves the probe permit consumed forever and
|
||||
* wedges the breaker. See file-header JSDoc for the canonical
|
||||
* try/finally pattern.
|
||||
*/
|
||||
check(): void {
|
||||
if (this.state === 'open' && this.openedAt !== null) {
|
||||
const elapsed = this.now() - this.openedAt;
|
||||
if (elapsed < this.cooldownMs) {
|
||||
throw new CircuitOpenError(this.cooldownMs - elapsed, this.key);
|
||||
}
|
||||
// Cooldown elapsed — transition to Half-Open. The very next
|
||||
// `probeInFlight` check below decides whether THIS caller gets
|
||||
// the permit or hits the gate.
|
||||
this.state = 'half-open';
|
||||
}
|
||||
|
||||
if (this.state === 'half-open') {
|
||||
if (this.probeInFlight) {
|
||||
throw new CircuitOpenError(this.halfOpenRetryAfterMs, this.key);
|
||||
}
|
||||
this.probeInFlight = true;
|
||||
}
|
||||
// Closed state falls through silently.
|
||||
}
|
||||
|
||||
recordSuccess(): void {
|
||||
this.probeInFlight = false;
|
||||
this.consecutiveFailures = 0;
|
||||
this.state = 'closed';
|
||||
this.openedAt = null;
|
||||
}
|
||||
|
||||
recordFailure(): void {
|
||||
this.probeInFlight = false;
|
||||
this.consecutiveFailures += 1;
|
||||
if (this.state === 'half-open' || this.consecutiveFailures >= this.failureThreshold) {
|
||||
this.state = 'open';
|
||||
this.openedAt = this.now();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases the probe permit BUT leaves state and counter untouched.
|
||||
* Use when an attempt produced a response or error that should not
|
||||
* influence breaker health in either direction — caller-driven aborts,
|
||||
* local AbortSignal timeouts, terminal 4xx client errors.
|
||||
*
|
||||
* Why permit-release-without-state-resolution: if `recordNeutral` did
|
||||
* not clear `probeInFlight`, a single `TimeoutError` from per-attempt
|
||||
* `AbortSignal.timeout` (which routes through neutral classification)
|
||||
* would permanently park the breaker in half-open. Since timeouts are
|
||||
* an *expected* outcome under flaky-dependency conditions, the cited
|
||||
* "per-attempt timeout bounds the stuck state" mitigation would itself
|
||||
* be the trigger for a permanent wedge. Releasing the permit closes
|
||||
* that loop while keeping the "neutral doesn't claim dependency
|
||||
* health" semantic.
|
||||
*
|
||||
* Calling `recordSuccess` for these would erase legitimate prior
|
||||
* failure signal; calling `recordFailure` would trip the breaker for
|
||||
* outcomes the backend isn't responsible for.
|
||||
*/
|
||||
recordNeutral(): void {
|
||||
this.probeInFlight = false;
|
||||
// State and consecutiveFailures are preserved by design.
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure read — no state mutation, no permit accounting. Returns the
|
||||
* *would-be* state at the current instant: 'half-open' if the breaker
|
||||
* is open with cooldown elapsed (regardless of whether a probe is in
|
||||
* flight), 'open' if open and still in cooldown, 'closed' otherwise.
|
||||
*
|
||||
* Inspection-only; safe to call from tests without consuming a probe
|
||||
* permit. The implicit Open -> Half-Open transition that mutates
|
||||
* `state` lives in `check()` only.
|
||||
*/
|
||||
getState(): State {
|
||||
if (this.state === 'open' && this.openedAt !== null) {
|
||||
const elapsed = this.now() - this.openedAt;
|
||||
if (elapsed >= this.cooldownMs) return 'half-open';
|
||||
}
|
||||
return this.state;
|
||||
}
|
||||
getConsecutiveFailures(): number {
|
||||
return this.consecutiveFailures;
|
||||
}
|
||||
/** Inspection-only test accessor for the half-open probe permit. */
|
||||
isProbeInFlight(): boolean {
|
||||
return this.probeInFlight;
|
||||
}
|
||||
/** Timestamp (ms since epoch) when the breaker last transitioned to Open,
|
||||
* or `null` if it's currently Closed. Useful for computing remaining
|
||||
* cooldown without consuming a probe permit via `check()`. */
|
||||
getOpenedAt(): number | null {
|
||||
return this.openedAt;
|
||||
}
|
||||
/** Configured cooldown duration in milliseconds. */
|
||||
getCooldownMs(): number {
|
||||
return this.cooldownMs;
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Per-process registry ────────────────────────────────────────────
|
||||
//
|
||||
// Single shared map keyed on caller-chosen strings. Used by
|
||||
// `resilient-fetch.ts` so multiple call sites targeting the same logical
|
||||
// endpoint share breaker state. Per-process only — not persisted.
|
||||
|
||||
const registry = new Map<string, CircuitBreaker>();
|
||||
|
||||
export function getBreaker(key: string, opts?: CircuitBreakerOptions): CircuitBreaker {
|
||||
let breaker = registry.get(key);
|
||||
if (!breaker) {
|
||||
breaker = new CircuitBreaker({ ...opts, key });
|
||||
registry.set(key, breaker);
|
||||
}
|
||||
return breaker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Test-only: clear all registered breakers. Tests must call this in
|
||||
* `beforeEach` to prevent breaker state from leaking across test cases.
|
||||
*/
|
||||
export function __resetBreakerRegistry__(): void {
|
||||
registry.clear();
|
||||
}
|
||||
279
gitnexus-shared/src/integrations/resilient-fetch.ts
Normal file
279
gitnexus-shared/src/integrations/resilient-fetch.ts
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
/**
|
||||
* `resilientFetch` — fetch wrapped in retry + circuit breaker, with
|
||||
* GitHub-flavoured retry classification baked in (Retry-After parsing,
|
||||
* 401/403/404/422 treated as terminal client errors).
|
||||
*
|
||||
* Designed for the `gitnexus publish` GitHub `repository_dispatch`
|
||||
* call, but the classification rules apply to any GitHub REST endpoint.
|
||||
* Runtime-agnostic — no Node-only imports.
|
||||
*/
|
||||
|
||||
import {
|
||||
CircuitBreaker,
|
||||
CircuitOpenError,
|
||||
getBreaker,
|
||||
type CircuitBreakerOptions,
|
||||
} from './circuit-breaker.js';
|
||||
import { computeBackoffMs, type RetryOptions } from './retry.js';
|
||||
|
||||
export { CircuitOpenError };
|
||||
|
||||
export interface ResilientFetchOptions {
|
||||
/** Optional fetch implementation override. Defaults to `globalThis.fetch`. */
|
||||
fetchImpl?: typeof fetch;
|
||||
/**
|
||||
* Logical key for the breaker. Defaults to `<host><pathname>` of the
|
||||
* request URL — call sites targeting the same endpoint share breaker
|
||||
* state regardless of query-string differences.
|
||||
*/
|
||||
breakerKey?: string;
|
||||
/** Per-call breaker override. Used for tests and one-off configuration. */
|
||||
breaker?: CircuitBreaker;
|
||||
/** Tuning knobs for the breaker registered under `breakerKey`. */
|
||||
breakerOptions?: CircuitBreakerOptions;
|
||||
/** Tuning knobs for the retry helper. */
|
||||
retry?: Partial<Pick<RetryOptions, 'maxAttempts' | 'baseDelayMs' | 'capDelayMs'>> & {
|
||||
sleep?: RetryOptions['sleep'];
|
||||
random?: RetryOptions['random'];
|
||||
};
|
||||
/** Clock override propagated into Retry-After HTTP-date math and breaker. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Cap on any single Retry-After wait — protects CLI from a buggy registry. */
|
||||
export const RETRY_AFTER_CAP_MS = 30_000;
|
||||
|
||||
const DEFAULT_RETRY = {
|
||||
maxAttempts: 3,
|
||||
baseDelayMs: 500,
|
||||
capDelayMs: 5_000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Parse a `Retry-After` header value into milliseconds.
|
||||
* Accepts either a delta-seconds integer (`"30"`) or an HTTP-date.
|
||||
* Returns null on parse failure or negative deltas.
|
||||
*/
|
||||
export function parseRetryAfter(value: string | null, now: () => number = Date.now): number | null {
|
||||
if (!value) return null;
|
||||
const trimmed = value.trim();
|
||||
if (trimmed === '') return null;
|
||||
|
||||
if (/^[0-9]+$/.test(trimmed)) {
|
||||
const seconds = parseInt(trimmed, 10);
|
||||
if (Number.isNaN(seconds) || seconds < 0) return null;
|
||||
return seconds * 1000;
|
||||
}
|
||||
|
||||
const target = Date.parse(trimmed);
|
||||
if (Number.isNaN(target)) return null;
|
||||
const delta = target - now();
|
||||
return delta >= 0 ? delta : 0;
|
||||
}
|
||||
|
||||
/** Internal: outcome classification used by the resilientFetch loop. */
|
||||
type Outcome =
|
||||
| { kind: 'success'; resp: Response }
|
||||
| { kind: 'terminal-client'; resp: Response } // 4xx other than 429: no retry, breaker neutral
|
||||
| { kind: 'retryable-status'; resp: Response; afterMs: number | undefined } // 5xx, 429
|
||||
| { kind: 'terminal-network'; err: unknown } // TimeoutError or AbortError: no retry, breaker neutral
|
||||
| { kind: 'retryable-network'; err: unknown }; // DNS, ECONNRESET, etc.
|
||||
|
||||
/** Exported for unit tests. */
|
||||
export function classifyOutcome(
|
||||
result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response },
|
||||
now: () => number,
|
||||
): Outcome {
|
||||
if (result.kind === 'error') {
|
||||
// Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`)
|
||||
// and caller-driven aborts (`AbortController.abort()` → `AbortError`)
|
||||
// are terminal: retrying against an already-aborted signal would
|
||||
// fail again immediately, and neither outcome reflects backend
|
||||
// health. They route through the breaker's neutral path.
|
||||
if (
|
||||
result.err instanceof DOMException &&
|
||||
(result.err.name === 'TimeoutError' || result.err.name === 'AbortError')
|
||||
) {
|
||||
return { kind: 'terminal-network', err: result.err };
|
||||
}
|
||||
return { kind: 'retryable-network', err: result.err };
|
||||
}
|
||||
const resp = result.resp;
|
||||
if (resp.status >= 200 && resp.status < 400) return { kind: 'success', resp };
|
||||
if (resp.status === 429) {
|
||||
// `resp.headers` is always present on a real `Response`, but tests
|
||||
// sometimes stub `fetch` with a plain `{ ok, status }` object. Be
|
||||
// defensive — a missing `Retry-After` falls through to exponential
|
||||
// backoff, which is the correct behaviour anyway.
|
||||
const retryAfterHeader =
|
||||
typeof resp.headers?.get === 'function' ? resp.headers.get('Retry-After') : null;
|
||||
const parsed = parseRetryAfter(retryAfterHeader, now);
|
||||
return {
|
||||
kind: 'retryable-status',
|
||||
resp,
|
||||
afterMs: parsed !== null ? Math.min(parsed, RETRY_AFTER_CAP_MS) : undefined,
|
||||
};
|
||||
}
|
||||
if (resp.status >= 500) return { kind: 'retryable-status', resp, afterMs: undefined };
|
||||
return { kind: 'terminal-client', resp };
|
||||
}
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
function defaultBreakerKey(input: string | URL): string {
|
||||
try {
|
||||
const url = typeof input === 'string' ? new URL(input) : input;
|
||||
return `${url.host}${url.pathname}`;
|
||||
} catch {
|
||||
return String(input);
|
||||
}
|
||||
}
|
||||
|
||||
/** Final error thrown when retries are exhausted on a 5xx / 429. */
|
||||
export class ResilientFetchExhaustedError extends Error {
|
||||
override readonly name = 'ResilientFetchExhaustedError';
|
||||
constructor(public readonly response: Response) {
|
||||
super(`Request failed after retries (HTTP ${response.status})`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap `fetch` with bounded retries and a per-process circuit breaker.
|
||||
*
|
||||
* Semantics:
|
||||
* - 5xx and 429 responses are retried; 429 honors `Retry-After` (capped).
|
||||
* - Network throws are retried unless they are `TimeoutError` DOMExceptions.
|
||||
* - Timeouts and 4xx (other than 429) are returned/thrown without retry
|
||||
* AND without incrementing the breaker — they reflect caller config
|
||||
* or local network state, not registry health.
|
||||
* - Each `fetch` call carries the caller-supplied `signal` (e.g. an
|
||||
* `AbortSignal.timeout()`) — that timeout bounds each individual
|
||||
* attempt, not the whole retry sequence.
|
||||
* - When the breaker is open, throws `CircuitOpenError` synchronously
|
||||
* without invoking `fetch`.
|
||||
* - When retries are exhausted on a 5xx / 429, throws
|
||||
* `ResilientFetchExhaustedError` carrying the last response.
|
||||
*
|
||||
* Cumulative wall-clock budget:
|
||||
* maxAttempts × (per-attempt-timeout + capDelayMs)
|
||||
* With defaults (3, 500ms base, 5000ms cap) and a typical 15s per-attempt
|
||||
* timeout from the caller's signal, worst case is ~3 × (15s + 5s) = 60s.
|
||||
* Callers that want a tighter total bound should reduce `maxAttempts` or
|
||||
* wrap `resilientFetch` in their own outer `AbortSignal.timeout()`.
|
||||
*/
|
||||
export async function resilientFetch(
|
||||
input: string | URL,
|
||||
init: RequestInit | undefined,
|
||||
opts: ResilientFetchOptions = {},
|
||||
): Promise<Response> {
|
||||
const fetchImpl = opts.fetchImpl ?? globalThis.fetch;
|
||||
const now = opts.now ?? (() => Date.now());
|
||||
const breaker =
|
||||
opts.breaker ?? getBreaker(opts.breakerKey ?? defaultBreakerKey(input), opts.breakerOptions);
|
||||
|
||||
const retryConfig = {
|
||||
maxAttempts: opts.retry?.maxAttempts ?? DEFAULT_RETRY.maxAttempts,
|
||||
baseDelayMs: opts.retry?.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs,
|
||||
capDelayMs: opts.retry?.capDelayMs ?? DEFAULT_RETRY.capDelayMs,
|
||||
};
|
||||
const sleep = opts.retry?.sleep ?? defaultSleep;
|
||||
const random = opts.retry?.random ?? Math.random;
|
||||
|
||||
// Fail fast on an open breaker, before invoking fetch.
|
||||
breaker.check();
|
||||
|
||||
for (let attempt = 0; attempt < retryConfig.maxAttempts; attempt++) {
|
||||
let result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response };
|
||||
try {
|
||||
// CodeQL js/server-side-request-forgery — flagged because `input`
|
||||
// is caller-supplied. Suppressed: every concrete caller passes
|
||||
// either a hardcoded URL constant (UNDERSTAND_QUICKLY_DISPATCH_URL,
|
||||
// OpenRouter base URL) or a value derived from configuration
|
||||
// (env vars, saved settings, the local backend URL). User-input
|
||||
// request fields (e.g. PR title, repo name) never flow into
|
||||
// `input`. Validating URL shape here would push false-positive
|
||||
// rejection onto every caller — wrong layer for the check.
|
||||
// lgtm[js/server-side-request-forgery]
|
||||
// codeql[js/server-side-request-forgery]
|
||||
const resp = await fetchImpl(input, init);
|
||||
result = { kind: 'response', resp };
|
||||
} catch (err) {
|
||||
result = { kind: 'error', err };
|
||||
}
|
||||
|
||||
const outcome = classifyOutcome(result, now);
|
||||
|
||||
switch (outcome.kind) {
|
||||
case 'success':
|
||||
breaker.recordSuccess();
|
||||
return outcome.resp;
|
||||
|
||||
case 'terminal-client':
|
||||
// 4xx: do not count as breaker failure (the server is healthy
|
||||
// and rejecting our request — auth, scope, or routing). But
|
||||
// also do NOT call recordSuccess: a 401 sandwiched between
|
||||
// 5xx responses would otherwise erase the running outage
|
||||
// signal. The breaker's neutral path leaves state untouched.
|
||||
breaker.recordNeutral();
|
||||
return outcome.resp;
|
||||
|
||||
case 'terminal-network':
|
||||
// Either `AbortSignal.timeout()` fired locally OR an external
|
||||
// caller cancelled the request via AbortController. The server
|
||||
// never had a chance to answer; this reflects the user's
|
||||
// network or an explicit cancel, not registry health. Don't
|
||||
// punish the breaker AND don't reset its outage signal.
|
||||
breaker.recordNeutral();
|
||||
throw outcome.err;
|
||||
|
||||
case 'retryable-status':
|
||||
if (attempt + 1 >= retryConfig.maxAttempts) {
|
||||
breaker.recordFailure();
|
||||
throw new ResilientFetchExhaustedError(outcome.resp);
|
||||
}
|
||||
await sleep(
|
||||
computeBackoffMs(
|
||||
attempt,
|
||||
retryConfig.baseDelayMs,
|
||||
retryConfig.capDelayMs,
|
||||
outcome.afterMs,
|
||||
random,
|
||||
),
|
||||
);
|
||||
break;
|
||||
|
||||
case 'retryable-network':
|
||||
if (attempt + 1 >= retryConfig.maxAttempts) {
|
||||
breaker.recordFailure();
|
||||
throw outcome.err;
|
||||
}
|
||||
await sleep(
|
||||
computeBackoffMs(
|
||||
attempt,
|
||||
retryConfig.baseDelayMs,
|
||||
retryConfig.capDelayMs,
|
||||
undefined,
|
||||
random,
|
||||
),
|
||||
);
|
||||
break;
|
||||
|
||||
default: {
|
||||
// Exhaustiveness guard. If a sixth `Outcome` kind is added in
|
||||
// future, TypeScript will refuse to assign it to `never` and
|
||||
// this line forces the maintainer to add an explicit arm
|
||||
// rather than silently fall through to retry/no-retry behaviour.
|
||||
const _exhaustive: never = outcome;
|
||||
throw new Error(`resilientFetch: unhandled outcome ${JSON.stringify(_exhaustive)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unreachable: every iteration of the loop either returns (success
|
||||
// / terminal-client) or throws (terminal-network / retry exhaustion).
|
||||
// The throw is here purely so TypeScript's control-flow analysis sees
|
||||
// the function never falls off the end without producing `Promise<Response>`.
|
||||
/* c8 ignore next 2 */
|
||||
throw new Error('resilientFetch: retry loop terminated unexpectedly');
|
||||
}
|
||||
105
gitnexus-shared/src/integrations/retry.ts
Normal file
105
gitnexus-shared/src/integrations/retry.ts
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
/**
|
||||
* Bounded retry helper with full-jitter exponential backoff.
|
||||
*
|
||||
* Runtime-agnostic: depends only on `setTimeout`, `Math.random`, and the
|
||||
* Promise machinery — no Node-only imports. Safe to consume from CLI,
|
||||
* server, or browser callers.
|
||||
*
|
||||
* Pattern reference: gitnexus/src/core/embeddings/http-client.ts. This
|
||||
* helper is the upgraded form: classification is caller-supplied (so
|
||||
* 4xx-vs-5xx-vs-timeout decisions live with the protocol that knows
|
||||
* them), backoff is exponential with full jitter, and an optional
|
||||
* `afterMs` lets callers honor `Retry-After` headers.
|
||||
*/
|
||||
|
||||
export interface RetryOptions {
|
||||
/** Initial delay before the first retry attempt, in milliseconds. */
|
||||
baseDelayMs: number;
|
||||
/** Upper bound on any single delay, in milliseconds. */
|
||||
capDelayMs: number;
|
||||
/** Total attempts including the first call. Must be >= 1. */
|
||||
maxAttempts: number;
|
||||
/**
|
||||
* Decide whether to retry after a thrown error.
|
||||
* Return `{retry:false}` to terminate immediately and rethrow.
|
||||
* Return `{retry:true}` to retry with exponential-backoff jitter.
|
||||
* Return `{retry:true, afterMs}` to wait at least `afterMs` (still
|
||||
* subject to `capDelayMs`) — used by callers parsing `Retry-After`.
|
||||
*/
|
||||
isRetryable: (err: unknown, attempt: number) => RetryDecision;
|
||||
/** Sleep override — defaults to `setTimeout`. Tests inject fake timers. */
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
/** Random override — defaults to `Math.random`. Tests inject seeded values. */
|
||||
random?: () => number;
|
||||
}
|
||||
|
||||
export type RetryDecision = { retry: false } | { retry: true; afterMs?: number };
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> =>
|
||||
new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Compute the delay before the next retry attempt.
|
||||
*
|
||||
* - When the caller specifies `afterMs` (e.g., from `Retry-After`), use
|
||||
* `min(afterMs, capDelayMs)` so a misbehaving server can't pin the
|
||||
* client for an arbitrarily long wait.
|
||||
* - Otherwise compute full-jitter exponential backoff:
|
||||
* `random() * min(cap, base * 2^attempt)`. Full jitter (rather than
|
||||
* "equal jitter") avoids retry-storm thundering herd, per AWS
|
||||
* guidance on backoff strategies.
|
||||
*/
|
||||
export function computeBackoffMs(
|
||||
attempt: number,
|
||||
baseDelayMs: number,
|
||||
capDelayMs: number,
|
||||
afterMs: number | undefined,
|
||||
random: () => number,
|
||||
): number {
|
||||
if (afterMs !== undefined) {
|
||||
return Math.min(Math.max(0, afterMs), capDelayMs);
|
||||
}
|
||||
const exponential = baseDelayMs * Math.pow(2, attempt);
|
||||
const upper = Math.min(capDelayMs, exponential);
|
||||
return Math.floor(random() * upper);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute `fn` with bounded retries.
|
||||
*
|
||||
* The classification of "retryable" is the caller's responsibility — see
|
||||
* `resilient-fetch.ts` for the GitHub-dispatch-specific rules. This
|
||||
* helper is the mechanical retry loop only.
|
||||
*/
|
||||
export async function withRetry<T>(
|
||||
fn: (attempt: number) => Promise<T>,
|
||||
opts: RetryOptions,
|
||||
): Promise<T> {
|
||||
if (opts.maxAttempts < 1) {
|
||||
throw new Error(`withRetry: maxAttempts must be >= 1, got ${opts.maxAttempts}`);
|
||||
}
|
||||
const sleep = opts.sleep ?? defaultSleep;
|
||||
const random = opts.random ?? Math.random;
|
||||
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < opts.maxAttempts; attempt++) {
|
||||
try {
|
||||
return await fn(attempt);
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
const decision = opts.isRetryable(err, attempt);
|
||||
if (!decision.retry) throw err;
|
||||
// Don't sleep after the final attempt.
|
||||
if (attempt + 1 >= opts.maxAttempts) break;
|
||||
const delayMs = computeBackoffMs(
|
||||
attempt,
|
||||
opts.baseDelayMs,
|
||||
opts.capDelayMs,
|
||||
decision.afterMs,
|
||||
random,
|
||||
);
|
||||
if (delayMs > 0) await sleep(delayMs);
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
151
gitnexus-shared/src/integrations/understand-quickly.ts
Normal file
151
gitnexus-shared/src/integrations/understand-quickly.ts
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
/**
|
||||
* Understand-Quickly registry integration helpers.
|
||||
*
|
||||
* Pure, runtime-agnostic logic for opting in to publishing a GitNexus
|
||||
* index to the [`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly)
|
||||
* registry. Lives in `gitnexus-shared` so both the Node CLI and any
|
||||
* future browser-side surface can construct identical dispatch payloads.
|
||||
*
|
||||
* Network I/O lives in the CLI command (`gitnexus/src/cli/publish.ts`)
|
||||
* to keep this module free of Node-only imports — see the comment at
|
||||
* the top of `gitnexus-shared/src/graph/types.ts`.
|
||||
*
|
||||
* The protocol contract (single dispatch event, no graph upload) is
|
||||
* documented at:
|
||||
* https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md
|
||||
*/
|
||||
|
||||
/**
|
||||
* URL of the registry repo's repository_dispatch endpoint. Hardcoded
|
||||
* because the registry is the canonical home for this integration —
|
||||
* users who want a private registry can fork and patch.
|
||||
*/
|
||||
export const UNDERSTAND_QUICKLY_DISPATCH_URL =
|
||||
'https://api.github.com/repos/looptech-ai/understand-quickly/dispatches';
|
||||
|
||||
/**
|
||||
* Event type the registry's sync workflow listens for.
|
||||
* See `looptech-ai/understand-quickly/.github/workflows/sync.yml`.
|
||||
*/
|
||||
export const UNDERSTAND_QUICKLY_EVENT_TYPE = 'sync-entry';
|
||||
|
||||
/** Environment variable that gates the dispatch. */
|
||||
export const UNDERSTAND_QUICKLY_TOKEN_ENV = 'UNDERSTAND_QUICKLY_TOKEN';
|
||||
|
||||
export interface UqDispatchPayload {
|
||||
event_type: typeof UNDERSTAND_QUICKLY_EVENT_TYPE;
|
||||
client_payload: {
|
||||
/** `<owner>/<repo>` shape — must match the registered entry. */
|
||||
id: string;
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the JSON body for the `repository_dispatch` ping. Pure — no
|
||||
* env reads, no network. Validates that `id` looks like `owner/repo`
|
||||
* (one slash, no whitespace, both halves non-empty) so a misconfigured
|
||||
* caller fails loudly before the round-trip.
|
||||
*/
|
||||
export function buildUqDispatchPayload(id: string): UqDispatchPayload {
|
||||
if (!isValidOwnerRepo(id)) {
|
||||
throw new Error(
|
||||
`[understand-quickly] expected id of the form "owner/repo", got "${id}". ` +
|
||||
`The registry uses this string to look up your entry in registry.json — ` +
|
||||
`it must match the GitHub owner/repo of the source code, not a local path.`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
event_type: UNDERSTAND_QUICKLY_EVENT_TYPE,
|
||||
client_payload: { id },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* `owner/repo` validation. Conservative on purpose: GitHub's actual
|
||||
* naming rules are looser, but we want to catch local paths
|
||||
* (`/Users/...`), bare slugs (`my-repo`), and accidental whitespace.
|
||||
*
|
||||
* Matches GitHub's published slug rules:
|
||||
* owner: starts with alnum, then alnum/hyphen only, must end with
|
||||
* alnum (no trailing hyphen — GitHub rejects this at account
|
||||
* creation, so a `my-org-/repo` input would otherwise pass us
|
||||
* and 422 from GitHub). No underscore, no dot. Length cap 39.
|
||||
* repo: any of alnum/dot/hyphen/underscore. Length cap 100.
|
||||
*/
|
||||
export function isValidOwnerRepo(id: string): boolean {
|
||||
return /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?\/[A-Za-z0-9._-]{1,100}$/.test(id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip a single trailing `.git` (case-insensitive) and any trailing
|
||||
* slashes from a URL-ish string. Bounded linear: each character is
|
||||
* visited at most twice, no backtracking.
|
||||
*
|
||||
* Replaces `s.replace(/\.git\/*$/i, '').replace(/\/+$/, '')` which
|
||||
* CodeQL's polynomial-regex check (codeql/js/polynomial-redos) flags as
|
||||
* a worst-case O(n²) on adversarial input like "////.../x".
|
||||
*/
|
||||
export function stripGitSuffix(input: string): string {
|
||||
let end = input.length;
|
||||
// Trim trailing '/'.
|
||||
while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--;
|
||||
// Drop one trailing '.git' (case-insensitive).
|
||||
if (end >= 4) {
|
||||
const tail = input.slice(end - 4, end).toLowerCase();
|
||||
if (tail === '.git') end -= 4;
|
||||
}
|
||||
// Trim trailing '/' that may have sat between '.git' and the rest.
|
||||
while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--;
|
||||
return input.slice(0, end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse `owner/repo` out of a git remote URL. Mirrors the heuristic in
|
||||
* `gitnexus/src/storage/git.ts:parseRepoNameFromUrl` but keeps both
|
||||
* halves so we can build a registry id. Returns `null` on shapes we
|
||||
* don't recognise.
|
||||
*
|
||||
* Examples:
|
||||
* git@github.com:looptech-ai/understand-quickly.git
|
||||
* https://github.com/looptech-ai/understand-quickly
|
||||
* ssh://git@github.com/looptech-ai/understand-quickly.git
|
||||
*/
|
||||
export function parseOwnerRepoFromRemote(url: string | null | undefined): string | null {
|
||||
if (!url) return null;
|
||||
const trimmed = url.trim();
|
||||
if (!trimmed) return null;
|
||||
// Strip a trailing `.git` (case-insensitive) and any trailing slashes
|
||||
// so https://h/o/r and https://h/o/r.git collapse to the same id.
|
||||
// Bounded-linear helper avoids the polynomial-regex CodeQL alert.
|
||||
const stripped = stripGitSuffix(trimmed);
|
||||
|
||||
// SCP-form SSH (`git@host:owner/repo`). Capture host so we can reject
|
||||
// non-GitHub remotes — a GitLab origin like
|
||||
// `https://gitlab.example.com/group/sub/project.git` would otherwise
|
||||
// silently dispatch the wrong id (LOW 9).
|
||||
const ssh = stripped.match(/^[^@]+@([^:]+):([^/]+)\/([^/]+)$/);
|
||||
if (ssh) {
|
||||
const host = ssh[1].toLowerCase();
|
||||
if (host !== 'github.com' && host !== 'www.github.com') return null;
|
||||
return `${ssh[2]}/${ssh[3]}`;
|
||||
}
|
||||
|
||||
// URL forms (https://, ssh://, git://, file://) — last two path segments.
|
||||
const url2 = stripped.match(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/([^/]+)\/(.+)$/);
|
||||
if (url2) {
|
||||
// Strip optional `userinfo@` (e.g. `ssh://git@github.com/...`).
|
||||
const authority = url2[1];
|
||||
const atIdx = authority.lastIndexOf('@');
|
||||
const hostAndPort = atIdx >= 0 ? authority.slice(atIdx + 1) : authority;
|
||||
// Strip `:port` suffix if present.
|
||||
const colonIdx = hostAndPort.indexOf(':');
|
||||
const host = (colonIdx >= 0 ? hostAndPort.slice(0, colonIdx) : hostAndPort).toLowerCase();
|
||||
if (host !== 'github.com' && host !== 'www.github.com') return null;
|
||||
const segments = url2[2].split('/').filter(Boolean);
|
||||
if (segments.length >= 2) {
|
||||
const [owner, repo] = segments.slice(-2);
|
||||
return `${owner}/${repo}`;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
13
gitnexus-shared/src/test-helpers.ts
Normal file
13
gitnexus-shared/src/test-helpers.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/**
|
||||
* Test-only helpers.
|
||||
*
|
||||
* Symbols here are reachable from `gitnexus-shared/test-helpers` so test
|
||||
* suites can reset shared registries or exercise internal classifiers,
|
||||
* but they are deliberately NOT re-exported from the main `gitnexus-shared`
|
||||
* barrel. Production consumers should never import this module — calling
|
||||
* `__resetBreakerRegistry__()` from a tool implementation would silently
|
||||
* nuke every circuit breaker process-wide.
|
||||
*/
|
||||
|
||||
export { __resetBreakerRegistry__ } from './integrations/circuit-breaker.js';
|
||||
export { classifyOutcome } from './integrations/resilient-fetch.js';
|
||||
24
gitnexus-web/package-lock.json
generated
24
gitnexus-web/package-lock.json
generated
|
|
@ -8,7 +8,7 @@
|
|||
"name": "gitnexus",
|
||||
"version": "0.0.0",
|
||||
"dependencies": {
|
||||
"@langchain/anthropic": "^1.3.28",
|
||||
"@langchain/anthropic": "^1.3.29",
|
||||
"@langchain/core": "^1.1.44",
|
||||
"@langchain/google-genai": "^2.1.28",
|
||||
"@langchain/langgraph": "^1.2.9",
|
||||
|
|
@ -95,9 +95,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@anthropic-ai/sdk": {
|
||||
"version": "0.90.0",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.90.0.tgz",
|
||||
"integrity": "sha512-MzZtPabJF1b0FTDl6Z6H5ljphPwACLGP13lu8MTiB8jXaW/YXlpOp+Po2cVou3MPM5+f5toyLnul9whKCy7fBg==",
|
||||
"version": "0.91.1",
|
||||
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz",
|
||||
"integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"json-schema-to-ts": "^3.1.1"
|
||||
|
|
@ -1396,25 +1396,25 @@
|
|||
}
|
||||
},
|
||||
"node_modules/@langchain/anthropic": {
|
||||
"version": "1.3.28",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.28.tgz",
|
||||
"integrity": "sha512-gOF8oXJL8xDdYes2KXNI9vFm/9TldBBBHOjuCdt27kganVaQKzLvTw5kV6R4mjbnFagV5CWteNH7APLZYCpdwg==",
|
||||
"version": "1.3.29",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.29.tgz",
|
||||
"integrity": "sha512-ep1qBIcV07bajsg3fDqMd39rYwoRLOEK/6lk+MCxlm1YB5SRoKKJAZANrblQ/4RYhZJnxf95c6BSQu8VoNbVAQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "^0.90.0",
|
||||
"@anthropic-ai/sdk": "^0.91.1",
|
||||
"zod": "^3.25.76 || ^4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@langchain/core": "^1.1.42"
|
||||
"@langchain/core": "^1.1.45"
|
||||
}
|
||||
},
|
||||
"node_modules/@langchain/core": {
|
||||
"version": "1.1.44",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.44.tgz",
|
||||
"integrity": "sha512-RePW1IjGCHr9ua2vcby3aE8mOOz3EnwDZxMEGbNDT91kf14eqkJqxDXvaZFviGdcN9DTrxM5RPQNAHmwSm4tbg==",
|
||||
"version": "1.1.45",
|
||||
"resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.45.tgz",
|
||||
"integrity": "sha512-Y/wvuglLTMKJahkl4QD9dBIdF/z/CxZJWdTfHJF/q2jtlJtoFf6Mb5JpGxZfsi3mBY6NSG941FSLTcqhCKrhBA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@cfworker/json-schema": "^4.0.2",
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"gitnexus-shared": "file:../gitnexus-shared",
|
||||
"@langchain/anthropic": "^1.3.28",
|
||||
"@langchain/anthropic": "^1.3.29",
|
||||
"@langchain/core": "^1.1.44",
|
||||
"@langchain/google-genai": "^2.1.28",
|
||||
"@langchain/langgraph": "^1.2.9",
|
||||
|
|
|
|||
|
|
@ -277,8 +277,10 @@ const extractInstanceName = (endpoint: string): string => {
|
|||
try {
|
||||
const url = new URL(endpoint);
|
||||
const hostname = url.hostname;
|
||||
// Extract the first part before .openai.azure.com
|
||||
const match = hostname.match(/^([^.]+)\.openai\.azure\.com/);
|
||||
// Extract the first part before .openai.azure.com. The trailing `$`
|
||||
// anchor is required (CodeQL js/regex/missing-regexp-anchor): without
|
||||
// it `evil.openai.azure.com.attacker.tld` would match.
|
||||
const match = hostname.match(/^([^.]+)\.openai\.azure\.com$/);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import {
|
|||
ProviderConfig,
|
||||
} from './types';
|
||||
import { DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OLLAMA_BASE_URL } from '../../config/ui-constants';
|
||||
import { resilientFetch } from 'gitnexus-shared';
|
||||
|
||||
const STORAGE_KEY = 'gitnexus-llm-settings';
|
||||
|
||||
|
|
@ -407,7 +408,10 @@ export const getAvailableModels = (provider: LLMProvider): string[] => {
|
|||
*/
|
||||
export const fetchOpenRouterModels = async (): Promise<Array<{ id: string; name: string }>> => {
|
||||
try {
|
||||
const response = await fetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`);
|
||||
const response = await resilientFetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`, undefined, {
|
||||
breakerKey: 'openrouter-models',
|
||||
retry: { maxAttempts: 2, baseDelayMs: 500, capDelayMs: 2_000 },
|
||||
});
|
||||
if (!response.ok) throw new Error('Failed to fetch models');
|
||||
const data = await response.json();
|
||||
return data.data.map((model: any) => ({
|
||||
|
|
|
|||
|
|
@ -278,8 +278,11 @@ export const createGraphRAGTools = (backend: GraphRAGBackend) => {
|
|||
const val = row[col];
|
||||
if (val === null || val === undefined) return '';
|
||||
if (typeof val === 'object') return JSON.stringify(val);
|
||||
// Truncate long values and escape pipe characters
|
||||
const str = String(val).replace(/\|/g, '\\|');
|
||||
// Truncate long values and escape pipe characters. Escape
|
||||
// backslashes FIRST so the subsequent pipe escape isn't
|
||||
// unescaped by a trailing backslash (CodeQL
|
||||
// js/incomplete-sanitization).
|
||||
const str = String(val).replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
|
||||
return str.length > 60 ? str.slice(0, 57) + '...' : str;
|
||||
});
|
||||
return `| ${values.join(' | ')} |`;
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
*/
|
||||
|
||||
import type { GraphNode, GraphRelationship } from 'gitnexus-shared';
|
||||
import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
|
@ -204,8 +205,32 @@ export function streamSSE<T = unknown>(url: string, handlers: SSEHandlers<T>): A
|
|||
|
||||
let _backendUrl = 'http://localhost:4747';
|
||||
|
||||
/**
|
||||
* Validate that a backend URL is a safe http:// or https:// origin before
|
||||
* storing it as the fetch target base (CodeQL js/client-side-request-forgery).
|
||||
*
|
||||
* Throws if the URL uses a non-HTTP scheme (e.g. javascript:, data:, file://).
|
||||
* All other well-formed http/https URLs are accepted — the client intentionally
|
||||
* supports connecting to remote GitNexus servers, not just localhost.
|
||||
*/
|
||||
export function validateBackendUrl(url: string): void {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
// Do not echo raw input — it may contain credentials.
|
||||
throw new Error('Invalid backend URL: must be a well-formed http:// or https:// URL');
|
||||
}
|
||||
if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
|
||||
// Use parsed.protocol only (scheme), not the full URL, to avoid leaking credentials.
|
||||
throw new Error(`Backend URL must use http:// or https:// (got ${parsed.protocol})`);
|
||||
}
|
||||
}
|
||||
|
||||
export const setBackendUrl = (url: string): void => {
|
||||
_backendUrl = url.replace(/\/$/, '');
|
||||
const trimmed = url.replace(/\/$/, '');
|
||||
validateBackendUrl(trimmed);
|
||||
_backendUrl = trimmed;
|
||||
};
|
||||
|
||||
export const getBackendUrl = (): string => _backendUrl;
|
||||
|
|
@ -237,29 +262,91 @@ export function normalizeServerUrl(input: string): string {
|
|||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const PROBE_TIMEOUT_MS = 2_000;
|
||||
|
||||
/** Idempotent HTTP methods. Other verbs (POST, PATCH, PUT, DELETE) get
|
||||
* a single-attempt retry budget by default to avoid duplicate side
|
||||
* effects on retry — a POST that 5xx'd may have already executed
|
||||
* server-side. Callers that have idempotency keys or otherwise know
|
||||
* their mutation is safe to retry can opt in via `forceRetry`. */
|
||||
const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
|
||||
|
||||
const fetchWithTimeout = async (
|
||||
url: string,
|
||||
init: RequestInit = {},
|
||||
timeoutMs: number = DEFAULT_TIMEOUT_MS,
|
||||
/**
|
||||
* Force a retry budget on non-idempotent methods. Default false.
|
||||
* Pass true only when the endpoint is known-idempotent (e.g. DELETE
|
||||
* of a known-deleted resource — second call is a 404 / no-op) AND
|
||||
* the duplicate-side-effect window is acceptable.
|
||||
*/
|
||||
forceRetry = false,
|
||||
): Promise<Response> => {
|
||||
const controller = new AbortController();
|
||||
// Merge external signal if provided
|
||||
// Merge the external caller signal (if any) with an
|
||||
// `AbortSignal.timeout()` so a timer-fired abort produces a
|
||||
// `DOMException` with `name === 'TimeoutError'` — which
|
||||
// `resilientFetch` correctly classifies as terminal-network (no
|
||||
// retry, no breaker hit). A manual `AbortController.abort()` would
|
||||
// produce `name === 'AbortError'` and route through the
|
||||
// retryable-network branch, which mis-penalizes the breaker for
|
||||
// user-side network slowness.
|
||||
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
||||
const externalSignal = init.signal;
|
||||
if (externalSignal) {
|
||||
externalSignal.addEventListener('abort', () => controller.abort());
|
||||
const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal;
|
||||
|
||||
const method = (init.method ?? 'GET').toUpperCase();
|
||||
const isIdempotent = IDEMPOTENT_METHODS.has(method);
|
||||
const maxAttempts = isIdempotent || forceRetry ? 2 : 1;
|
||||
|
||||
// Key the breaker by the current backend origin so switching backend
|
||||
// URLs (e.g. recovering from a flapping local server by pointing at
|
||||
// a different host) gives the new origin a fresh breaker state. A
|
||||
// single shared `'web-backend'` key would otherwise leave a user
|
||||
// locked out for the full cooldown after one bad host trips the
|
||||
// circuit. The malformed-URL fallback is defensive — `setBackendUrl`
|
||||
// normalizes input, so this branch shouldn't fire in practice.
|
||||
let breakerKey: string;
|
||||
try {
|
||||
breakerKey = `web-backend:${new URL(_backendUrl).origin}`;
|
||||
} catch {
|
||||
breakerKey = 'web-backend:invalid';
|
||||
}
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, { ...init, signal: controller.signal });
|
||||
// Bounded retries + 5xx/429 handling are delegated to resilientFetch.
|
||||
// Method-aware budget: idempotent verbs retry once on transient
|
||||
// backend failures; mutations (POST/PATCH/PUT/DELETE) default to
|
||||
// single-attempt to avoid duplicate side effects.
|
||||
const response = await resilientFetch(
|
||||
url,
|
||||
{ ...init, signal },
|
||||
{
|
||||
breakerKey,
|
||||
retry: { maxAttempts, baseDelayMs: 250, capDelayMs: 1500 },
|
||||
},
|
||||
);
|
||||
return response;
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
if (externalSignal?.aborted) {
|
||||
throw new BackendError('Request aborted', 0, 'network');
|
||||
}
|
||||
if (error instanceof CircuitOpenError) {
|
||||
throw new BackendError(
|
||||
`GitNexus backend at ${_backendUrl} is unhealthy; retry in ${Math.ceil(error.retryAfterMs / 1000)}s`,
|
||||
0,
|
||||
'network',
|
||||
);
|
||||
}
|
||||
if (error instanceof ResilientFetchExhaustedError) {
|
||||
// Fall through to caller — surface the raw response so assertOk
|
||||
// can craft the BackendError with the right code.
|
||||
return error.response;
|
||||
}
|
||||
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
||||
throw new BackendError(`Request to ${url} timed out after ${timeoutMs}ms`, 0, 'timeout');
|
||||
}
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
// External caller-driven cancellation — `timeoutSignal` would
|
||||
// have surfaced as TimeoutError above, so this branch covers
|
||||
// only the externally-aborted case.
|
||||
throw new BackendError('Request aborted', 0, 'network');
|
||||
}
|
||||
if (error instanceof TypeError) {
|
||||
throw new BackendError(
|
||||
`Network error reaching GitNexus backend at ${_backendUrl}: ${error.message}`,
|
||||
|
|
@ -268,8 +355,6 @@ const fetchWithTimeout = async (
|
|||
);
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
|
||||
|
|
|
|||
110
gitnexus-web/test/unit/backend-client-retry.test.ts
Normal file
110
gitnexus-web/test/unit/backend-client-retry.test.ts
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
/**
|
||||
* Method-aware retry budget + timeout-as-TimeoutError verification for
|
||||
* backend-client's `fetchWithTimeout`.
|
||||
*
|
||||
* Closes review findings on PR #1448:
|
||||
* - Non-idempotent POST/DELETE must NOT be retried by default —
|
||||
* a 5xx on `startAnalyze` could otherwise start a duplicate job.
|
||||
* - Timer-fired timeout must surface as `DOMException(name='TimeoutError')`,
|
||||
* not `AbortError`, so resilientFetch routes it through the
|
||||
* terminal-network branch (no retry, no breaker hit).
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { getBreaker } from 'gitnexus-shared';
|
||||
import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers';
|
||||
import { fetchRepos, setBackendUrl, startAnalyze } from '../../src/services/backend-client';
|
||||
|
||||
const BASE = 'http://localhost:4747';
|
||||
|
||||
describe('backend-client retry budget (method-aware)', () => {
|
||||
beforeEach(() => {
|
||||
__resetBreakerRegistry__();
|
||||
setBackendUrl(BASE);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it('GET retries once on transient 503 (idempotent verb)', async () => {
|
||||
let n = 0;
|
||||
const fetchMock = vi.fn(async () => {
|
||||
n += 1;
|
||||
if (n === 1) return new Response('boom', { status: 503 });
|
||||
return new Response('[]', {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const repos = await fetchRepos();
|
||||
expect(repos).toEqual([]);
|
||||
// 1 retry budget on idempotent GET → 2 total fetch calls.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('POST does NOT retry on 503 by default (non-idempotent verb)', async () => {
|
||||
const fetchMock = vi.fn(async () => new Response('boom', { status: 503 }));
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(startAnalyze({ path: '/tmp/repo' })).rejects.toBeTruthy();
|
||||
// Single attempt — never duplicates a job-start POST.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('switching backend URL after a circuit opens reaches a fresh breaker (U3)', async () => {
|
||||
// Pre-open the breaker for host-A by directly recording 3 failures.
|
||||
setBackendUrl('http://host-a.test:4747');
|
||||
const aKey = 'web-backend:http://host-a.test:4747';
|
||||
const breakerA = getBreaker(aKey);
|
||||
breakerA.recordFailure();
|
||||
breakerA.recordFailure();
|
||||
breakerA.recordFailure();
|
||||
expect(breakerA.getState()).toBe('open');
|
||||
|
||||
// Switch to host-B and make a request — must succeed against the
|
||||
// new origin without tripping the host-A circuit. Under the old
|
||||
// single-key behaviour the call would throw CircuitOpenError.
|
||||
setBackendUrl('http://host-b.test:4747');
|
||||
const fetchMock = vi.fn(
|
||||
async () =>
|
||||
new Response('[]', {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
const repos = await fetchRepos();
|
||||
expect(repos).toEqual([]);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Host-A's breaker is still open in cooldown.
|
||||
expect(breakerA.getState()).toBe('open');
|
||||
// Host-B has its own (fresh) breaker.
|
||||
const bKey = 'web-backend:http://host-b.test:4747';
|
||||
expect(getBreaker(bKey).getState()).toBe('closed');
|
||||
expect(getBreaker(bKey).getConsecutiveFailures()).toBe(0);
|
||||
});
|
||||
|
||||
it('breaker not incremented when timeout fires (TimeoutError, not AbortError)', async () => {
|
||||
// Reject directly with a TimeoutError DOMException, mimicking what
|
||||
// `fetch` produces when its `AbortSignal.timeout()`-wired signal
|
||||
// fires. The real-fetch path goes signal.reason → reject(reason);
|
||||
// we shortcut that here so the test doesn't have to wait the
|
||||
// 30-second default timeout.
|
||||
const fetchMock = vi.fn(async () => {
|
||||
throw new DOMException('aborted by timeout', 'TimeoutError');
|
||||
});
|
||||
vi.stubGlobal('fetch', fetchMock);
|
||||
|
||||
await expect(fetchRepos()).rejects.toMatchObject({ code: 'timeout' });
|
||||
|
||||
// The breaker must not have been penalized for a local timeout.
|
||||
expect(getBreaker(`web-backend:${BASE}`).getConsecutiveFailures()).toBe(0);
|
||||
// Timeout is terminal — no retry attempted.
|
||||
expect(fetchMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,11 @@
|
|||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { fetchGraph, normalizeServerUrl, setBackendUrl } from '../../src/services/backend-client';
|
||||
import {
|
||||
fetchGraph,
|
||||
getBackendUrl,
|
||||
normalizeServerUrl,
|
||||
setBackendUrl,
|
||||
validateBackendUrl,
|
||||
} from '../../src/services/backend-client';
|
||||
|
||||
describe('normalizeServerUrl', () => {
|
||||
it('adds http:// to localhost', () => {
|
||||
|
|
@ -165,3 +171,61 @@ describe('fetchGraph', () => {
|
|||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('validateBackendUrl', () => {
|
||||
it('allows http:// URLs', () => {
|
||||
expect(() => validateBackendUrl('http://localhost:4747')).not.toThrow();
|
||||
expect(() => validateBackendUrl('http://127.0.0.1:4747')).not.toThrow();
|
||||
});
|
||||
|
||||
it('allows https:// URLs', () => {
|
||||
expect(() => validateBackendUrl('https://gitnexus.example.com')).not.toThrow();
|
||||
expect(() => validateBackendUrl('https://my-server.internal:4747')).not.toThrow();
|
||||
});
|
||||
|
||||
it('rejects non-http schemes', () => {
|
||||
expect(() => validateBackendUrl('javascript:alert(1)')).toThrow('must use http:// or https://');
|
||||
expect(() => validateBackendUrl('file:///etc/passwd')).toThrow('must use http:// or https://');
|
||||
expect(() => validateBackendUrl('data:text/plain,evil')).toThrow(
|
||||
'must use http:// or https://',
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects malformed URLs', () => {
|
||||
expect(() => validateBackendUrl('not-a-url')).toThrow('Invalid backend URL');
|
||||
});
|
||||
|
||||
it('does not include the raw URL in error messages (credential hygiene)', () => {
|
||||
const urlWithCreds = 'javascript:alert("sk-secret")';
|
||||
let msg = '';
|
||||
try {
|
||||
validateBackendUrl(urlWithCreds);
|
||||
} catch (e) {
|
||||
msg = (e as Error).message;
|
||||
}
|
||||
expect(msg).not.toContain('sk-secret');
|
||||
expect(msg).not.toContain(urlWithCreds);
|
||||
});
|
||||
});
|
||||
|
||||
describe('setBackendUrl', () => {
|
||||
it('accepts valid http URLs', () => {
|
||||
expect(() => setBackendUrl('http://localhost:4747')).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts valid https URLs', () => {
|
||||
expect(() => setBackendUrl('https://my-server.example.com')).not.toThrow();
|
||||
});
|
||||
|
||||
it('rejects non-http/https schemes', () => {
|
||||
expect(() => setBackendUrl('javascript:alert(1)')).toThrow('must use http:// or https://');
|
||||
expect(() => setBackendUrl('file:///etc/passwd')).toThrow('must use http:// or https://');
|
||||
});
|
||||
|
||||
it('does not mutate _backendUrl when validation fails', () => {
|
||||
setBackendUrl('http://localhost:4747');
|
||||
expect(() => setBackendUrl('javascript:alert(1)')).toThrow();
|
||||
// State must be preserved — validation must happen before the assignment
|
||||
expect(getBackendUrl()).toBe('http://localhost:4747');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,15 @@
|
|||
FROM node:20-bookworm
|
||||
# Pinned npm version — keep in sync with the root Dockerfile.cli and
|
||||
# Dockerfile.web.
|
||||
ARG NPM_VERSION=11.14.1
|
||||
|
||||
# node:22-bookworm-slim
|
||||
FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e
|
||||
ARG NPM_VERSION
|
||||
WORKDIR /app
|
||||
RUN apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/*
|
||||
RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} \
|
||||
&& apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update \
|
||||
&& apt-get install -y python3 make g++ \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
COPY . .
|
||||
RUN npm ci --ignore-scripts \
|
||||
&& npm rebuild tree-sitter-swift 2>&1 \
|
||||
|
|
|
|||
32
gitnexus/package-lock.json
generated
32
gitnexus/package-lock.json
generated
|
|
@ -2065,9 +2065,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "25.6.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.1.tgz",
|
||||
"integrity": "sha512-coJCN8O1q4AGyyqCAUSP06P+SrMTu18BkEj3NVAK07q6QUneD2wzj3CLv9+yP+BMeZQlMvneXqqvDe3w+xcq7g==",
|
||||
"version": "25.6.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.2.tgz",
|
||||
"integrity": "sha512-sokuT28dxf9JT5Kady1fsXOvI4HVpjZa95NKT5y9PNTIrs2AsobR4GFAA90ZG8M+nxVRLysCXsVj6eGC7Vbrlw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.19.0"
|
||||
|
|
@ -3110,9 +3110,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/fast-uri": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.0.tgz",
|
||||
"integrity": "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA==",
|
||||
"version": "3.1.2",
|
||||
"resolved": "https://registry.npmjs.org/fast-uri/-/fast-uri-3.1.2.tgz",
|
||||
"integrity": "sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "github",
|
||||
|
|
@ -3476,9 +3476,9 @@
|
|||
"license": "MIT"
|
||||
},
|
||||
"node_modules/hono": {
|
||||
"version": "4.12.16",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
|
||||
"integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
|
||||
"version": "4.12.18",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.12.18.tgz",
|
||||
"integrity": "sha512-RWzP96k/yv0PQfyXnWjs6zot20TqfpfsNXhOnev8d1InAxubW93L11/oNUc3tQqn2G0bSdAOBpX+2uDFHV7kdQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=16.9.0"
|
||||
|
|
@ -4282,15 +4282,15 @@
|
|||
}
|
||||
},
|
||||
"node_modules/onnxruntime-common": {
|
||||
"version": "1.25.1",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.25.1.tgz",
|
||||
"integrity": "sha512-kKvYQFdos4LWJqhZ+nmKu3NT8NXzw8I5x9fNUKe1rNKcPfNKnYXUtW7JBpcKFsvLtrJashRgVYSbFap4cHxvNg==",
|
||||
"version": "1.26.0",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.26.0.tgz",
|
||||
"integrity": "sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/onnxruntime-node": {
|
||||
"version": "1.25.1",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.25.1.tgz",
|
||||
"integrity": "sha512-N0M58CGTiTsLkPpx9bxmRFi24GT6r67Qei/GrBEIiDyntcYdXU5vQZp112ypydG9vEKRFgbgUYQJnEi+jll8dg==",
|
||||
"version": "1.26.0",
|
||||
"resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.26.0.tgz",
|
||||
"integrity": "sha512-OHl6PiOEOqxaLHL0N9eFrbzS7IGmu3BtJNH3RTEnRAheCIkfc3gjcjl4sGcjp9C22ZC9YTquDOxSdT/stBQ6BQ==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"os": [
|
||||
|
|
@ -4301,7 +4301,7 @@
|
|||
"dependencies": {
|
||||
"adm-zip": "^0.5.16",
|
||||
"global-agent": "^4.1.3",
|
||||
"onnxruntime-common": "1.25.1"
|
||||
"onnxruntime-common": "1.26.0"
|
||||
}
|
||||
},
|
||||
"node_modules/onnxruntime-web": {
|
||||
|
|
|
|||
|
|
@ -116,6 +116,6 @@
|
|||
}
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
"node": ">=22.0.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import { warnMissingOptionalGrammars } from './optional-grammars.js';
|
|||
import { glob } from 'glob';
|
||||
import fs from 'fs/promises';
|
||||
import { cliError } from './cli-message.js';
|
||||
import { isHfDownloadFailure } from '../core/embeddings/hf-env.js';
|
||||
|
||||
// Capture stderr.write at module load BEFORE anything (LadybugDB native
|
||||
// init, progress bar, console redirection) can monkey-patch it. The
|
||||
|
|
@ -576,6 +577,26 @@ export const analyzeCommand = async (inputPath?: string, options?: AnalyzeOption
|
|||
return;
|
||||
}
|
||||
|
||||
// HF download failure — show clean guidance without the raw stack trace.
|
||||
// Checked before writeFatalToStderr so the user sees one focused message
|
||||
// rather than a stack-trace dump followed by a second remediation block.
|
||||
if (isHfDownloadFailure(msg) || msg.includes('Failed to download embedding model')) {
|
||||
cliError(
|
||||
` The embedding model could not be downloaded.\n` +
|
||||
` huggingface.co may be unreachable from your network\n` +
|
||||
` (e.g. behind a corporate proxy or a regional firewall).\n` +
|
||||
` Suggestions:\n` +
|
||||
` 1. Set HF_ENDPOINT to a mirror and retry:\n` +
|
||||
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
|
||||
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)\n` +
|
||||
` 2. Check your proxy / VPN settings.\n` +
|
||||
` 3. Once downloaded the model is cached — future runs work offline.\n`,
|
||||
{ recoveryHint: 'hf-endpoint-unreachable' },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// Bypass the redirected console.error and write the full stack to
|
||||
// the real stderr captured at module load. The redirected
|
||||
// console.error wraps every line with `\\x1b[2K\\r` (ANSI clear-line)
|
||||
|
|
|
|||
|
|
@ -160,6 +160,18 @@ program
|
|||
.description('Augment a search pattern with knowledge graph context (used by hooks)')
|
||||
.action(createLazyAction(() => import('./augment.js'), 'augmentCommand'));
|
||||
|
||||
program
|
||||
.command('publish [path]')
|
||||
.description(
|
||||
'Notify the understand-quickly registry that this repo has a fresh GitNexus index. ' +
|
||||
'Opt-in: requires UNDERSTAND_QUICKLY_TOKEN (fine-grained PAT with ' +
|
||||
'`Repository dispatches: write` on looptech-ai/understand-quickly). ' +
|
||||
'No-op without the token. See https://github.com/looptech-ai/understand-quickly.',
|
||||
)
|
||||
.option('--id <owner/repo>', 'Override the registry id (defaults to the origin remote)')
|
||||
.option('--skip-git', 'Treat cwd as the repo root and skip parent git-root discovery')
|
||||
.action(createLazyAction(() => import('./publish.js'), 'publishCommand'));
|
||||
|
||||
// ─── Direct Tool Commands (no MCP overhead) ────────────────────────
|
||||
// These invoke LocalBackend directly for use in eval, scripts, and CI.
|
||||
|
||||
|
|
|
|||
232
gitnexus/src/cli/publish.ts
Normal file
232
gitnexus/src/cli/publish.ts
Normal file
|
|
@ -0,0 +1,232 @@
|
|||
/**
|
||||
* `gitnexus publish` — opt-in ping to the understand-quickly registry.
|
||||
*
|
||||
* Fires a single `repository_dispatch` event at
|
||||
* `looptech-ai/understand-quickly` so the registry knows to refresh its
|
||||
* entry for the current repo. Does NOT upload anything: per the
|
||||
* understand-quickly protocol, the registry pulls the graph from a
|
||||
* raw-GitHub URL the user controls.
|
||||
*
|
||||
* https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md
|
||||
*
|
||||
* Defaults:
|
||||
* - Without `UNDERSTAND_QUICKLY_TOKEN` in the env, this is a no-op
|
||||
* (prints one informational line, exit 0). Same shape as the
|
||||
* `--publish` patterns in sibling tools.
|
||||
* - With the token, fires the dispatch and reports the response code.
|
||||
*
|
||||
* The `id` is derived from the repo's `origin` remote unless the caller
|
||||
* passes `--id <owner/repo>` explicitly. We deliberately do NOT auto-add
|
||||
* the repo to the registry — registration is one-time and uses the
|
||||
* `npx @understand-quickly/cli add` path documented in the protocol.
|
||||
*/
|
||||
|
||||
import path from 'path';
|
||||
import {
|
||||
UNDERSTAND_QUICKLY_DISPATCH_URL,
|
||||
UNDERSTAND_QUICKLY_TOKEN_ENV,
|
||||
buildUqDispatchPayload,
|
||||
isValidOwnerRepo,
|
||||
parseOwnerRepoFromRemote,
|
||||
} from 'gitnexus-shared';
|
||||
import { getGitRoot, getRemoteOriginUrl, getCurrentCommit } from '../storage/git.js';
|
||||
import { hasIndex } from '../storage/repo-manager.js';
|
||||
import { cliInfo, cliError } from './cli-message.js';
|
||||
|
||||
export interface PublishOptions {
|
||||
/** Override the auto-derived `owner/repo` id. */
|
||||
id?: string;
|
||||
/** Treat the cwd as the repo root (skip git-root walk). */
|
||||
skipGit?: boolean;
|
||||
}
|
||||
|
||||
const REGISTER_HINT =
|
||||
'Register your repo once with: npx @understand-quickly/cli add\n' +
|
||||
'Or use the wizard: https://looptech-ai.github.io/understand-quickly/add.html';
|
||||
|
||||
/**
|
||||
* Hard cap on the dispatch fetch to keep CI publish steps from stalling
|
||||
* for the OS TCP timeout (~2 min) when api.github.com is unreachable.
|
||||
* Matches the pattern used in `src/core/embeddings/http-client.ts`.
|
||||
*/
|
||||
const DISPATCH_TIMEOUT_MS = 15_000;
|
||||
|
||||
export const publishCommand = async (
|
||||
inputPath?: string,
|
||||
options: PublishOptions = {},
|
||||
): Promise<void> => {
|
||||
// ── 0. Token gate FIRST — guarantees true no-op without the token. ──
|
||||
// The README, CLI --help, and PR body all promise "exit 0 without
|
||||
// UNDERSTAND_QUICKLY_TOKEN". Doing the index/repo-root checks before
|
||||
// the token gate would make those promises false for users who haven't
|
||||
// run `gitnexus analyze` yet but want to verify the command is wired.
|
||||
const token = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV];
|
||||
if (!token) {
|
||||
cliInfo(
|
||||
`[understand-quickly] ${UNDERSTAND_QUICKLY_TOKEN_ENV} is not set — skipping dispatch.\n` +
|
||||
`Set it to a fine-grained PAT with "Repository dispatches: write" on ` +
|
||||
`looptech-ai/understand-quickly to enable instant resync.\n` +
|
||||
`(Without the token, the registry's nightly sync still picks up your entry.)`,
|
||||
{ skipped: 'no-token' },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 1. Resolve the repo root (same precedence as `analyze`) ──────────
|
||||
let repoPath: string;
|
||||
if (inputPath) {
|
||||
repoPath = path.resolve(inputPath);
|
||||
} else if (options.skipGit) {
|
||||
repoPath = path.resolve(process.cwd());
|
||||
} else {
|
||||
const gitRoot = getGitRoot(process.cwd());
|
||||
if (!gitRoot) {
|
||||
cliError(
|
||||
'[understand-quickly] not inside a git repository.\n' +
|
||||
'Run from a repo, or pass --skip-git to publish from the current directory.',
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
repoPath = gitRoot;
|
||||
}
|
||||
|
||||
// ── 2. Confirm a GitNexus index exists ───────────────────────────────
|
||||
// Publishing without an index is almost always a mistake — the
|
||||
// registry's nightly sync would fetch a stale or missing graph file
|
||||
// and mark the entry `missing`. Refuse loudly with a fix-it hint.
|
||||
if (!(await hasIndex(repoPath))) {
|
||||
cliError(
|
||||
`[understand-quickly] no GitNexus index found at ${repoPath}/.gitnexus.\n` +
|
||||
'Run `gitnexus analyze` first, then re-run `gitnexus publish`.',
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 3. Derive the registry id ─────────────────────────────────────────
|
||||
const id =
|
||||
options.id ?? parseOwnerRepoFromRemote(getRemoteOriginUrl(repoPath) ?? undefined) ?? null;
|
||||
if (!id || !isValidOwnerRepo(id)) {
|
||||
cliError(
|
||||
`[understand-quickly] could not derive a registry id from this repo.\n` +
|
||||
`Pass --id <owner/repo> explicitly (e.g. --id looptech-ai/${path.basename(repoPath)}).\n` +
|
||||
REGISTER_HINT,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// ── 4. Fire the dispatch ─────────────────────────────────────────────
|
||||
const payload = buildUqDispatchPayload(id);
|
||||
let response: Response;
|
||||
try {
|
||||
response = await fetch(UNDERSTAND_QUICKLY_DISPATCH_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Accept: 'application/vnd.github+json',
|
||||
Authorization: `Bearer ${token}`,
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
'Content-Type': 'application/json',
|
||||
'User-Agent': 'gitnexus-cli',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
signal: AbortSignal.timeout(DISPATCH_TIMEOUT_MS),
|
||||
});
|
||||
} catch (err) {
|
||||
// `AbortSignal.timeout()` throws a `DOMException` with `name ===
|
||||
// 'TimeoutError'` on Node 18.14+ (and on browsers/Bun). It is NOT
|
||||
// a plain `AbortError`. Match the pattern used in
|
||||
// gitnexus/src/core/embeddings/http-client.ts so the user sees the
|
||||
// targeted "timed out" message instead of a generic "operation
|
||||
// was aborted".
|
||||
const isTimeout = err instanceof DOMException && err.name === 'TimeoutError';
|
||||
if (isTimeout) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch timed out after ${DISPATCH_TIMEOUT_MS}ms. ` +
|
||||
`Check network access to api.github.com and retry.`,
|
||||
{ id },
|
||||
);
|
||||
} else {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
cliError(`[understand-quickly] dispatch network error: ${msg}`, { id });
|
||||
}
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// GitHub returns 204 on success. Distinct branches for 401/403/404/422
|
||||
// so users debug without checking the docs.
|
||||
if (response.status === 204) {
|
||||
await response.body?.cancel().catch(() => {});
|
||||
// `getCurrentCommit` is only meaningful in the success path — moving
|
||||
// it inside this branch removes a wasted child-process spawn on every
|
||||
// error response (LOW 7).
|
||||
const commit = getCurrentCommit(repoPath);
|
||||
cliInfo(
|
||||
`[understand-quickly] dispatched sync-entry for ${id}` +
|
||||
(commit ? ` @ ${commit.slice(0, 7)}` : '') +
|
||||
'.\n' +
|
||||
`Note: a 204 only confirms GitHub accepted the dispatch. Whether the ` +
|
||||
`registry workflow finds an entry for "${id}" is logged at ` +
|
||||
`https://github.com/looptech-ai/understand-quickly/actions/workflows/sync.yml`,
|
||||
{ id, commit, status: response.status },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 401) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 401 — the ${UNDERSTAND_QUICKLY_TOKEN_ENV} value is invalid or expired.\n` +
|
||||
`Regenerate a fine-grained PAT at https://github.com/settings/personal-access-tokens ` +
|
||||
`with Repository access scoped to looptech-ai/understand-quickly and the ` +
|
||||
`"Repository dispatches: write" permission, then retry.`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 403) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 403 — the token authenticated but ` +
|
||||
`lacks the "Repository dispatches: write" permission on ` +
|
||||
`looptech-ai/understand-quickly. Edit the PAT scopes and retry.`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 404) {
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 404 — the token cannot reach ` +
|
||||
`looptech-ai/understand-quickly. Verify the PAT has Repository access to ` +
|
||||
`that exact repo (not just your own org).`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.status === 422) {
|
||||
// Malformed event_type / client_payload — a code bug in this CLI,
|
||||
// not a user mistake. Surface so we get bug reports.
|
||||
const body422 = await response.text().catch(() => '');
|
||||
cliError(
|
||||
`[understand-quickly] dispatch returned 422 (this is a CLI bug; please report).\n` +
|
||||
`Body: ${body422 || '(empty)'}`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
// 5xx and anything else → bubble the body so the user has something to act on.
|
||||
const body = await response.text().catch(() => '');
|
||||
cliError(
|
||||
`[understand-quickly] dispatch failed with HTTP ${response.status}: ${body || '(empty body)'}`,
|
||||
{ id, status: response.status },
|
||||
);
|
||||
process.exitCode = 1;
|
||||
};
|
||||
|
|
@ -365,7 +365,12 @@ async function installClaudeCodeHooks(result: SetupResult): Promise<void> {
|
|||
}
|
||||
|
||||
const hookPath = path.join(destHooksDir, 'gitnexus-hook.cjs').replace(/\\/g, '/');
|
||||
const hookCmd = `node "${hookPath.replace(/"/g, '\\"')}"`;
|
||||
// Escape backslashes FIRST, then quotes (CodeQL js/incomplete-sanitization).
|
||||
// The previous shape `replace(/"/g, '\\"')` alone would let `path\with"quote`
|
||||
// become `path\with\"quote`, where the trailing `\` before `"` could
|
||||
// unescape the quote inside the surrounding double-quoted shell context.
|
||||
const escapedHookPath = hookPath.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
||||
const hookCmd = `node "${escapedHookPath}"`;
|
||||
|
||||
// Check which hook events need entries (idempotent: skip if already registered)
|
||||
const parsed = await (async () => {
|
||||
|
|
|
|||
|
|
@ -602,6 +602,38 @@ function hasGhCLI(): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict Gist URL predicate. Rejects:
|
||||
* - any URL that does not parse (URL constructor throws)
|
||||
* - schemes other than https (drops `http:`, `file:`, `gist:`-style spoofs)
|
||||
* - hostnames that are not exactly `gist.github.com` (drops substring spoofs
|
||||
* like `https://evil.com/?u=gist.github.com` and userinfo-prefixed shapes
|
||||
* like `https://[email protected]/...` — note that URL.hostname
|
||||
* strips userinfo, so the equality check rejects the userinfo-prefixed
|
||||
* spoof if the actual host differs from gist.github.com)
|
||||
* - any URL containing userinfo (`username[:password]@`), which the URL
|
||||
* parser exposes via `.username` / `.password`. Defense-in-depth: even
|
||||
* when hostname matches, a credential-bearing URL is suspect and not
|
||||
* produced by `gh gist create`.
|
||||
*
|
||||
* Closes the substring-bypass class CodeQL `js/incomplete-url-substring-
|
||||
* sanitization` flags.
|
||||
*/
|
||||
function isGistUrl(line: string): boolean {
|
||||
const trimmed = line.trim();
|
||||
try {
|
||||
const u = new URL(trimmed);
|
||||
return (
|
||||
u.protocol === 'https:' &&
|
||||
u.hostname === 'gist.github.com' &&
|
||||
u.username === '' &&
|
||||
u.password === ''
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
|
||||
try {
|
||||
const output = execFileSync(
|
||||
|
|
@ -610,13 +642,14 @@ function publishGist(htmlPath: string): { url: string; rawUrl: string } | null {
|
|||
{ encoding: 'utf-8', stdio: ['pipe', 'pipe', 'pipe'] },
|
||||
).trim();
|
||||
|
||||
// gh gist create prints the gist URL as the last line
|
||||
const lines = output.split('\n');
|
||||
const gistUrl = lines.find((l) => l.includes('gist.github.com')) || lines[lines.length - 1];
|
||||
// `gh gist create` prints the gist URL as a line in the output. Find the
|
||||
// first parseable Gist URL — if no line is a valid Gist URL, fail closed
|
||||
// (do NOT fall back to lines[last]: a non-Gist last line would propagate
|
||||
// through the regex below and produce a malformed `rawUrl`).
|
||||
const gistUrl = output.split('\n').find(isGistUrl);
|
||||
if (!gistUrl) return null;
|
||||
|
||||
if (!gistUrl || !gistUrl.includes('gist.github.com')) return null;
|
||||
|
||||
// Build a raw viewer URL via gist.githack.com
|
||||
// Build a raw viewer URL via gist.githack.com.
|
||||
// gist URL format: https://gist.github.com/{user}/{id}
|
||||
const match = gistUrl.match(/gist\.github\.com\/([^/]+)\/([a-f0-9]+)/);
|
||||
let rawUrl = gistUrl;
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ const DEFAULT_IGNORE_LIST = new Set([
|
|||
'bower_components',
|
||||
'jspm_packages',
|
||||
'vendor', // PHP/Go
|
||||
'third_party', // C/C++ (Google-style vendored dependencies)
|
||||
'3rdparty', // C/C++ (alternate spelling, also Qt convention)
|
||||
// 'packages' removed - commonly used for monorepo source code (lerna, pnpm, yarn workspaces)
|
||||
'venv',
|
||||
'.venv',
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ export async function augment(pattern: string, cwd?: string): Promise<string> {
|
|||
}
|
||||
|
||||
// Step 1: BM25 search (fast, no embeddings)
|
||||
const bm25Results = await searchFTSFromLbug(pattern, 10, repoId);
|
||||
const { results: bm25Results } = await searchFTSFromLbug(pattern, 10, repoId);
|
||||
|
||||
if (bm25Results.length === 0) return '';
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,12 @@ if (!process.env.ORT_LOG_LEVEL) {
|
|||
process.env.ORT_LOG_LEVEL = '3';
|
||||
}
|
||||
|
||||
import { pipeline, env, type FeatureExtractionPipeline } from '@huggingface/transformers';
|
||||
import {
|
||||
pipeline,
|
||||
env,
|
||||
type FeatureExtractionPipeline,
|
||||
type ProgressInfo,
|
||||
} from '@huggingface/transformers';
|
||||
import { existsSync } from 'fs';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { join, dirname } from 'path';
|
||||
|
|
@ -22,7 +27,7 @@ import { createRequire } from 'module';
|
|||
import { DEFAULT_EMBEDDING_CONFIG, type EmbeddingConfig, type ModelProgress } from './types.js';
|
||||
import { isHttpMode, getHttpDimensions, httpEmbed } from './http-client.js';
|
||||
import { resolveEmbeddingConfig } from './config.js';
|
||||
import { applyHfEnvOverrides } from './hf-env.js';
|
||||
import { applyHfEnvOverrides, isHfDownloadFailure, withHfDownloadRetry } from './hf-env.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
/**
|
||||
|
|
@ -171,13 +176,18 @@ export const initEmbedder = async (
|
|||
}
|
||||
|
||||
const progressCallback = onProgress
|
||||
? (data: any) => {
|
||||
? (data: ProgressInfo) => {
|
||||
const progress: ModelProgress = {
|
||||
status: data.status || 'progress',
|
||||
file: data.file,
|
||||
progress: data.progress,
|
||||
loaded: data.loaded,
|
||||
total: data.total,
|
||||
// Map the `progress_total` aggregate event (not in ModelProgress.status)
|
||||
// back to 'progress' so callers don't need to handle it separately.
|
||||
status:
|
||||
data.status === 'progress_total'
|
||||
? 'progress'
|
||||
: ((data.status as ModelProgress['status']) ?? 'progress'),
|
||||
file: 'file' in data ? data.file : undefined,
|
||||
progress: 'progress' in data ? data.progress : undefined,
|
||||
loaded: 'loaded' in data ? data.loaded : undefined,
|
||||
total: 'total' in data ? data.total : undefined,
|
||||
};
|
||||
onProgress(progress);
|
||||
}
|
||||
|
|
@ -202,17 +212,29 @@ export const initEmbedder = async (
|
|||
logger.info('🔧 Using WASM backend (slower)...');
|
||||
}
|
||||
|
||||
embedderInstance = await (pipeline as any)('feature-extraction', finalConfig.modelId, {
|
||||
device: device,
|
||||
dtype: 'fp32',
|
||||
progress_callback: progressCallback,
|
||||
session_options: {
|
||||
logSeverityLevel: 3,
|
||||
intraOpNumThreads: finalConfig.threads,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
embedderInstance = await withHfDownloadRetry(
|
||||
() =>
|
||||
pipeline('feature-extraction', finalConfig.modelId, {
|
||||
device: device,
|
||||
dtype: 'fp32',
|
||||
progress_callback: progressCallback,
|
||||
session_options: {
|
||||
logSeverityLevel: 3,
|
||||
intraOpNumThreads: finalConfig.threads,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
},
|
||||
}),
|
||||
{
|
||||
onRetry: isDev
|
||||
? (attempt, max, err) =>
|
||||
logger.warn(
|
||||
{ attempt, max, err: err.message },
|
||||
`⚠️ Model download network error (attempt ${attempt}/${max}), retrying…`,
|
||||
)
|
||||
: undefined,
|
||||
},
|
||||
});
|
||||
);
|
||||
currentDevice = device;
|
||||
|
||||
if (isDev) {
|
||||
|
|
@ -228,6 +250,20 @@ export const initEmbedder = async (
|
|||
|
||||
return embedderInstance!;
|
||||
} catch (deviceError) {
|
||||
// Network errors and circuit-open errors are not device-specific —
|
||||
// they will fail the same way on every device. Rethrow immediately
|
||||
// with actionable HF_ENDPOINT guidance rather than silently falling
|
||||
// back to the next device.
|
||||
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
|
||||
if (isHfDownloadFailure(errMsg)) {
|
||||
const endpointHint = process.env.HF_ENDPOINT
|
||||
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
|
||||
: `huggingface.co may be unreachable from your network.\n` +
|
||||
` Set HF_ENDPOINT to a mirror and retry:\n` +
|
||||
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
|
||||
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
|
||||
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
|
||||
}
|
||||
if (isDev && (device === 'cuda' || device === 'dml')) {
|
||||
const gpuType = device === 'dml' ? 'DirectML' : 'CUDA';
|
||||
logger.info(`⚠️ ${gpuType} not available, falling back to CPU...`);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,27 @@
|
|||
import os from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import { CircuitBreaker, withRetry } from 'gitnexus-shared';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Download resilience defaults
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Per-attempt timeout for the full model download (5 minutes). */
|
||||
export const HF_DOWNLOAD_TIMEOUT_MS = 5 * 60 * 1_000;
|
||||
/** Maximum total download attempts (1 initial + N-1 retries). */
|
||||
export const HF_MAX_ATTEMPTS = 3;
|
||||
/** Initial delay between retry attempts; doubles on each subsequent retry. */
|
||||
export const HF_BASE_DELAY_MS = 2_000;
|
||||
/** Number of consecutive failures required to open the circuit. */
|
||||
export const CB_FAILURE_THRESHOLD = 3;
|
||||
/** How long the circuit stays open before transitioning to half-open. */
|
||||
export const CB_RESET_TIMEOUT_MS = 60_000;
|
||||
/** Upper bound clamped on the env-override per-attempt timeout (30 minutes). */
|
||||
export const HF_MAX_TIMEOUT_MS = 30 * 60 * 1_000;
|
||||
/** Upper bound clamped on the env-override attempt count. */
|
||||
export const HF_MAX_ATTEMPTS_CAP = 10;
|
||||
|
||||
/**
|
||||
* @internal Exported only for unit tests and the two embedder entry points
|
||||
* (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Not part of the
|
||||
|
|
@ -60,3 +81,234 @@ export function applyHfEnvOverrides(env: HfEnvSubset): void {
|
|||
env.remoteHost = endpoint.endsWith('/') ? endpoint : endpoint + '/';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal Exported for unit tests and the two embedder entry points.
|
||||
*
|
||||
* Returns true when an error message indicates a network-level fetch failure
|
||||
* during HuggingFace model download (e.g. `TypeError: fetch failed`,
|
||||
* `ECONNREFUSED`, `ENOTFOUND`, `ETIMEDOUT`, `ECONNRESET`).
|
||||
*
|
||||
* These errors are not device-specific and cannot be fixed by falling back to
|
||||
* a different ONNX device — the caller should rethrow immediately with
|
||||
* guidance about `HF_ENDPOINT`.
|
||||
*/
|
||||
export function isNetworkFetchError(message: string): boolean {
|
||||
return (
|
||||
message.includes('fetch failed') ||
|
||||
message.includes('ECONNREFUSED') ||
|
||||
message.includes('ENOTFOUND') ||
|
||||
message.includes('ETIMEDOUT') ||
|
||||
message.includes('ECONNRESET')
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Circuit breaker
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */
|
||||
export const CIRCUIT_OPEN_TAG = 'hf-circuit-open';
|
||||
|
||||
/**
|
||||
* Module-level singleton shared by both embedder entry points
|
||||
* (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Per-process
|
||||
* only — not persisted across restarts. Backed by the shared
|
||||
* `CircuitBreaker` from `gitnexus-shared` (same state machine, same
|
||||
* semantics, plus the single-permit half-open gate that prevents
|
||||
* recovery-time stampedes).
|
||||
*/
|
||||
export const hfDownloadCircuit = new CircuitBreaker({
|
||||
failureThreshold: CB_FAILURE_THRESHOLD,
|
||||
cooldownMs: CB_RESET_TIMEOUT_MS,
|
||||
key: 'hf-download',
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Retry + timeout wrapper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** @internal Returns true for errors that should abort without retry (circuit-open). */
|
||||
export function isHfCircuitOpenError(message: string): boolean {
|
||||
return message.includes(CIRCUIT_OPEN_TAG);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true for any HuggingFace download failure that warrants showing the
|
||||
* `HF_ENDPOINT` remediation hint: either a raw network error or a
|
||||
* circuit-open rejection (which itself was caused by repeated network errors).
|
||||
*/
|
||||
export function isHfDownloadFailure(message: string): boolean {
|
||||
return isNetworkFetchError(message) || isHfCircuitOpenError(message);
|
||||
}
|
||||
|
||||
/** @internal Wraps `fn` in a hard time-limit. The timeout error contains
|
||||
* `ETIMEDOUT` so that `isNetworkFetchError` classifies it correctly.
|
||||
*/
|
||||
export function withDownloadTimeout<T>(fn: () => Promise<T>, timeoutMs: number): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
new Error(
|
||||
`ETIMEDOUT: model download timed out after ${Math.round(timeoutMs / 1000)}s — ` +
|
||||
`check your network speed or set HF_ENDPOINT to a faster mirror`,
|
||||
),
|
||||
),
|
||||
timeoutMs,
|
||||
);
|
||||
fn().then(
|
||||
(v) => {
|
||||
clearTimeout(timer);
|
||||
resolve(v);
|
||||
},
|
||||
(e) => {
|
||||
clearTimeout(timer);
|
||||
reject(e);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export interface HfRetryOptions {
|
||||
/** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */
|
||||
maxAttempts?: number;
|
||||
/** Delay before the first retry; doubles on each subsequent attempt (default: `HF_BASE_DELAY_MS`). */
|
||||
baseDelayMs?: number;
|
||||
/** Per-attempt wall-clock timeout in ms (default: `HF_DOWNLOAD_TIMEOUT_MS`). */
|
||||
timeoutMs?: number;
|
||||
/**
|
||||
* Circuit-breaker instance to use. Defaults to the module-level
|
||||
* `hfDownloadCircuit` singleton. Pass a fresh instance in tests.
|
||||
*/
|
||||
circuit?: CircuitBreaker;
|
||||
/**
|
||||
* Optional callback invoked before each retry (not the initial attempt).
|
||||
* @param attempt - 1-based retry number
|
||||
* @param max - total allowed attempts
|
||||
* @param error - the error that triggered the retry
|
||||
*/
|
||||
onRetry?: (attempt: number, max: number, error: Error) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retry wrapper for HuggingFace model downloads with per-attempt timeout and
|
||||
* circuit-breaker protection.
|
||||
*
|
||||
* Behaviour:
|
||||
* - If the circuit is **open**, fails immediately with a `CIRCUIT_OPEN_TAG`
|
||||
* message (so `isHfDownloadFailure` still returns true and the caller can
|
||||
* show `HF_ENDPOINT` guidance).
|
||||
* - Each attempt is wrapped in `withDownloadTimeout`.
|
||||
* - On a network-level error (`isNetworkFetchError`) the attempt is retried
|
||||
* with exponential back-off; non-network errors (e.g. ONNX device failure)
|
||||
* are rethrown immediately without retry.
|
||||
* - Every network failure is recorded on the circuit breaker; a success resets
|
||||
* it.
|
||||
* - After all attempts are exhausted, the last network error is rethrown
|
||||
* so the existing `isNetworkFetchError` / `isHfDownloadFailure` guards in
|
||||
* the calling code still fire.
|
||||
*/
|
||||
export async function withHfDownloadRetry<T>(
|
||||
fn: () => Promise<T>,
|
||||
options: HfRetryOptions = {},
|
||||
): Promise<T> {
|
||||
// Resolve effective values — explicit options take precedence over env vars,
|
||||
// which take precedence over built-in defaults. This lets users lower the
|
||||
// per-attempt timeout without rebuilding (e.g.
|
||||
// HF_DOWNLOAD_TIMEOUT_MS=60000 npx gitnexus analyze --embeddings
|
||||
// reduces the worst-case wait from 15 minutes to ~3 minutes).
|
||||
//
|
||||
// Upper bounds are clamped to prevent accidental runaway configuration:
|
||||
// - timeoutMs is capped at HF_MAX_TIMEOUT_MS (30 min)
|
||||
// - maxAttempts is floored (fractional values → integer) and capped at
|
||||
// HF_MAX_ATTEMPTS_CAP (10). Values ≤ 0, NaN, or Infinity fall back to
|
||||
// the built-in defaults.
|
||||
const envTimeout = Number(process.env.HF_DOWNLOAD_TIMEOUT_MS);
|
||||
const envMaxAttempts = Number(process.env.HF_MAX_ATTEMPTS);
|
||||
const resolvedTimeout =
|
||||
Number.isFinite(envTimeout) && envTimeout > 0
|
||||
? Math.min(envTimeout, HF_MAX_TIMEOUT_MS)
|
||||
: HF_DOWNLOAD_TIMEOUT_MS;
|
||||
const resolvedMaxAttempts =
|
||||
Number.isFinite(envMaxAttempts) && envMaxAttempts > 0
|
||||
? Math.min(Math.floor(envMaxAttempts), HF_MAX_ATTEMPTS_CAP)
|
||||
: HF_MAX_ATTEMPTS;
|
||||
const {
|
||||
maxAttempts = resolvedMaxAttempts,
|
||||
baseDelayMs = HF_BASE_DELAY_MS,
|
||||
timeoutMs = resolvedTimeout,
|
||||
circuit = hfDownloadCircuit,
|
||||
onRetry,
|
||||
} = options;
|
||||
if (circuit.getState() === 'open') {
|
||||
// Compute remaining cooldown without consuming a probe permit.
|
||||
const openedAt = circuit.getOpenedAt();
|
||||
const secsUntilReset =
|
||||
openedAt !== null ? Math.ceil((circuit.getCooldownMs() - (Date.now() - openedAt)) / 1000) : 0;
|
||||
throw new Error(
|
||||
`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` +
|
||||
(secsUntilReset > 0 ? ` — will reset in ~${secsUntilReset}s` : ''),
|
||||
);
|
||||
}
|
||||
|
||||
// Retry budget delegated to `withRetry` from gitnexus-shared. The
|
||||
// HF-specific bits — per-attempt timeout, network-vs-non-network
|
||||
// classification, circuit-breaker recording, onRetry callback — wire
|
||||
// through the `isRetryable` callback. `circuitTripped` is the
|
||||
// sentinel that lets us replace the final thrown error with a
|
||||
// CIRCUIT_OPEN_TAG message when the breaker tripped mid-loop.
|
||||
let circuitTripped = false;
|
||||
|
||||
try {
|
||||
return await withRetry(
|
||||
async () => {
|
||||
const result = await withDownloadTimeout(fn, timeoutMs);
|
||||
circuit.recordSuccess();
|
||||
return result;
|
||||
},
|
||||
{
|
||||
maxAttempts,
|
||||
baseDelayMs,
|
||||
// Disable the cap to match the bespoke pure-exponential
|
||||
// progression. With the default `HF_MAX_ATTEMPTS_CAP = 10` and
|
||||
// `baseDelayMs = 2000`, the largest possible delay is
|
||||
// `2000 * 2^9 = ~17 minutes` — bounded enough not to need a cap.
|
||||
capDelayMs: Number.MAX_SAFE_INTEGER,
|
||||
isRetryable: (err, attempt) => {
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
if (!isNetworkFetchError(error.message)) {
|
||||
// Non-network error (e.g. CUDA unavailable) — propagate
|
||||
// without retry. Use recordNeutral so the breaker's existing
|
||||
// failure-count progress isn't reset by a non-network failure
|
||||
// that says nothing about the CDN's health.
|
||||
circuit.recordNeutral();
|
||||
return { retry: false };
|
||||
}
|
||||
circuit.recordFailure();
|
||||
if (circuit.getState() === 'open') {
|
||||
// Circuit just tripped — fail fast, no more retries.
|
||||
circuitTripped = true;
|
||||
return { retry: false };
|
||||
}
|
||||
// Mirror the bespoke onRetry contract: fire only when there's
|
||||
// actually a next attempt.
|
||||
if (attempt + 1 < maxAttempts) {
|
||||
onRetry?.(attempt + 1, maxAttempts, error);
|
||||
}
|
||||
return { retry: true };
|
||||
},
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
if (circuitTripped) {
|
||||
throw new Error(
|
||||
`${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${CB_FAILURE_THRESHOLD} consecutive failures`,
|
||||
);
|
||||
}
|
||||
// All retries exhausted — rethrow the last network error so
|
||||
// isNetworkFetchError patterns in the calling code still match and
|
||||
// surface HF_ENDPOINT guidance.
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,13 +3,22 @@
|
|||
*
|
||||
* Shared fetch+retry logic for OpenAI-compatible /v1/embeddings endpoints.
|
||||
* Imported by both the core embedder (batch) and MCP embedder (query).
|
||||
*
|
||||
* Network resilience is delegated to `resilientFetch` from
|
||||
* `gitnexus-shared` — bounded retries with exponential-backoff jitter,
|
||||
* `Retry-After` honored on 429, and an in-process circuit breaker that
|
||||
* fails fast on a flapping endpoint. Per-attempt timeout is enforced
|
||||
* via `AbortSignal.timeout` on the underlying fetch.
|
||||
*/
|
||||
|
||||
import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared';
|
||||
|
||||
const HTTP_TIMEOUT_MS = 30_000;
|
||||
const HTTP_MAX_RETRIES = 2;
|
||||
const HTTP_RETRY_BACKOFF_MS = 1_000;
|
||||
const HTTP_BATCH_SIZE = 64;
|
||||
const DEFAULT_DIMS = 384;
|
||||
const HTTP_BREAKER_KEY = 'embeddings-http';
|
||||
|
||||
interface HttpConfig {
|
||||
baseUrl: string;
|
||||
|
|
@ -90,46 +99,51 @@ const httpEmbedBatch = async (
|
|||
model: string,
|
||||
apiKey: string,
|
||||
batchIndex = 0,
|
||||
attempt = 0,
|
||||
): Promise<EmbeddingItem[]> => {
|
||||
let resp: Response;
|
||||
try {
|
||||
resp = await fetch(url, {
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
resp = await resilientFetch(
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
signal: AbortSignal.timeout(HTTP_TIMEOUT_MS),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({ input: batch, model }),
|
||||
},
|
||||
body: JSON.stringify({ input: batch, model }),
|
||||
});
|
||||
{
|
||||
breakerKey: HTTP_BREAKER_KEY,
|
||||
retry: { maxAttempts: HTTP_MAX_RETRIES + 1, baseDelayMs: HTTP_RETRY_BACKOFF_MS },
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
// Timeouts should not be retried — the server is unresponsive.
|
||||
// AbortSignal.timeout() throws DOMException with name 'TimeoutError'.
|
||||
const isTimeout = err instanceof DOMException && err.name === 'TimeoutError';
|
||||
if (isTimeout) {
|
||||
if (err instanceof CircuitOpenError) {
|
||||
throw new Error(
|
||||
`Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`,
|
||||
);
|
||||
}
|
||||
if (err instanceof DOMException && err.name === 'TimeoutError') {
|
||||
throw new Error(
|
||||
`Embedding request timed out after ${HTTP_TIMEOUT_MS}ms (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
}
|
||||
// DNS, connection errors — retry with backoff
|
||||
if (attempt < HTTP_MAX_RETRIES) {
|
||||
const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1);
|
||||
if (err instanceof ResilientFetchExhaustedError) {
|
||||
throw new Error(
|
||||
`Embedding endpoint returned ${err.response.status} (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
}
|
||||
const reason = err instanceof Error ? err.message : String(err);
|
||||
throw new Error(`Embedding request failed (${safeUrl(url)}, batch ${batchIndex}): ${reason}`);
|
||||
}
|
||||
|
||||
if (!resp.ok) {
|
||||
const status = resp.status;
|
||||
if ((status === 429 || status >= 500) && attempt < HTTP_MAX_RETRIES) {
|
||||
const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1);
|
||||
await new Promise((r) => setTimeout(r, delay));
|
||||
return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1);
|
||||
}
|
||||
throw new Error(`Embedding endpoint returned ${status} (${safeUrl(url)}, batch ${batchIndex})`);
|
||||
// resilientFetch already retried 5xx/429; any non-OK response here is
|
||||
// a terminal client error (4xx other than 429).
|
||||
throw new Error(
|
||||
`Embedding endpoint returned ${resp.status} (${safeUrl(url)}, batch ${batchIndex})`,
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await resp.json()) as { data: EmbeddingItem[] };
|
||||
|
|
|
|||
|
|
@ -3,11 +3,14 @@
|
|||
* Lives in core/ so application code does not depend on the MCP package layer.
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { execFile, execFileSync } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import path from 'path';
|
||||
import { readRegistry, type RegistryEntry, type CwdMatch } from '../storage/repo-manager.js';
|
||||
import { findGitRootByDotGit, getCurrentCommit, getRemoteUrl } from '../storage/git.js';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
export interface StalenessInfo {
|
||||
isStale: boolean;
|
||||
commitsBehind: number;
|
||||
|
|
@ -41,6 +44,39 @@ export function checkStaleness(repoPath: string, lastCommit: string): StalenessI
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Async variant of {@link checkStaleness} — spawns git as a child process
|
||||
* instead of blocking the event loop. Used by `listRepos()` to check many
|
||||
* repos in parallel (issue #1363: 200 repos × sync spawn ≈ 50 s).
|
||||
*/
|
||||
export async function checkStalenessAsync(
|
||||
repoPath: string,
|
||||
lastCommit: string,
|
||||
): Promise<StalenessInfo> {
|
||||
try {
|
||||
// Note: promisified execFile captures stdout/stderr by default (no stdio option needed,
|
||||
// unlike the sync variant which requires explicit stdio: ['pipe','pipe','pipe']).
|
||||
const { stdout } = await execFileAsync('git', ['rev-list', '--count', `${lastCommit}..HEAD`], {
|
||||
cwd: repoPath,
|
||||
encoding: 'utf-8',
|
||||
});
|
||||
|
||||
const commitsBehind = parseInt(stdout.trim(), 10) || 0;
|
||||
|
||||
if (commitsBehind > 0) {
|
||||
return {
|
||||
isStale: true,
|
||||
commitsBehind,
|
||||
hint: `⚠️ Index is ${commitsBehind} commit${commitsBehind > 1 ? 's' : ''} behind HEAD. Run analyze tool to update.`,
|
||||
};
|
||||
}
|
||||
|
||||
return { isStale: false, commitsBehind: 0 };
|
||||
} catch {
|
||||
return { isStale: false, commitsBehind: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a sibling-clone HEAD against an indexed `lastCommit`. Returns
|
||||
* `undefined` when the indexed commit is not reachable from the sibling
|
||||
|
|
|
|||
|
|
@ -44,26 +44,6 @@ async function removeLbugFile(basePath: string): Promise<void> {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all stale `bridge.lbug.tmp.*` files (and their sidecars) from a
|
||||
* group directory. With randomBytes-based temp names, a crashed writeBridge
|
||||
* leaves behind a uniquely-named tmp file that no future run will target by
|
||||
* name — so we glob for the prefix and clean up everything matching.
|
||||
*/
|
||||
async function cleanStaleBridgeTmpFiles(groupDir: string): Promise<void> {
|
||||
try {
|
||||
const entries = await fsp.readdir(groupDir);
|
||||
const staleBases = entries.filter(
|
||||
(e) => e.startsWith('bridge.lbug.tmp.') && !LBUG_SIDECAR_SUFFIXES.some((s) => e.endsWith(s)),
|
||||
);
|
||||
for (const name of staleBases) {
|
||||
await removeLbugFile(path.join(groupDir, name));
|
||||
}
|
||||
} catch {
|
||||
/* best-effort: directory may not exist yet */
|
||||
}
|
||||
}
|
||||
|
||||
export function contractNodeId(
|
||||
repo: string,
|
||||
contractId: string,
|
||||
|
|
@ -299,8 +279,24 @@ export async function retryRename(src: string, dst: string, attempts = 3): Promi
|
|||
|
||||
export async function writeBridgeMeta(groupDir: string, meta: BridgeMeta): Promise<void> {
|
||||
const target = path.join(groupDir, 'meta.json');
|
||||
// Unpredictable suffix + O_EXCL via `'wx'` flag closes the symlink/
|
||||
// pre-create attack window. The third argument `0o600` is the
|
||||
// user-only mode mask — CodeQL's `js/insecure-temporary-file` query
|
||||
// sources its verdict from the `mode` argument, NOT from `flags`:
|
||||
// its `isSecureMode(mode)` predicate requires the low 6 bits to be
|
||||
// zero (no group/world bits). Without an explicit mode the file is
|
||||
// created with the process umask (typically 0o644 = group/world
|
||||
// readable), which the query treats as the actual vulnerability.
|
||||
// Both `'wx'` (runtime O_EXCL) AND `0o600` (CodeQL-credited mode)
|
||||
// are needed: one closes the symlink race, the other closes the
|
||||
// permissions exposure.
|
||||
const tmp = `${target}.tmp.${randomBytes(8).toString('hex')}`;
|
||||
await fsp.writeFile(tmp, JSON.stringify(meta, null, 2), 'utf-8');
|
||||
const handle = await fsp.open(tmp, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(JSON.stringify(meta, null, 2), 'utf-8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
// Use retryRename for consistency with writeBridge's atomic swap — on
|
||||
// Windows a concurrent reader can cause EBUSY/EPERM even on a tiny
|
||||
// meta.json, and we don't want meta write to be less robust than the
|
||||
|
|
@ -369,7 +365,19 @@ export async function writeBridge(
|
|||
const crossLinks = dedupeCrossLinks(input.crossLinks);
|
||||
|
||||
const finalPath = path.join(groupDir, 'bridge.lbug');
|
||||
const tmpPath = path.join(groupDir, `bridge.lbug.tmp.${randomBytes(8).toString('hex')}`);
|
||||
// Stage the temp database inside a unique mkdtemp directory rather than
|
||||
// a fixed `bridge.lbug.tmp` name. The previous shape was flagged by
|
||||
// CodeQL js/insecure-temporary-file as a predictable path: a co-located
|
||||
// attacker (or a parallel writeBridge call into the same group) could
|
||||
// pre-create or symlink that path before this writer opens it. mkdtemp
|
||||
// returns a directory whose suffix is filled with cryptographically
|
||||
// random bytes, so the staging path is unguessable AND collision-free
|
||||
// across parallel callers. We anchor the staging directory inside
|
||||
// `groupDir` so the subsequent rename of `bridge.lbug` (and its
|
||||
// `.wal` / `.shadow` sidecars) into place stays on the same filesystem
|
||||
// and remains atomic — moving across `os.tmpdir()` could trip EXDEV.
|
||||
const stagingDir = await fsp.mkdtemp(path.join(groupDir, 'bridge-tmp-'));
|
||||
const tmpPath = path.join(stagingDir, 'bridge.lbug');
|
||||
const bakPath = path.join(groupDir, 'bridge.lbug.bak');
|
||||
|
||||
const report: WriteBridgeReport = {
|
||||
|
|
@ -389,43 +397,42 @@ export async function writeBridge(
|
|||
}
|
||||
};
|
||||
|
||||
// Clean up stale tmp files left behind by previously crashed writeBridge
|
||||
// runs. With randomBytes-based names each run picks a unique path, so
|
||||
// the old fixed-name `removeLbugFile(tmpPath)` was a no-op — stale
|
||||
// artifacts accumulated. The glob-based helper finds *all* leftover
|
||||
// `bridge.lbug.tmp.*` entries and removes them (including sidecars).
|
||||
await cleanStaleBridgeTmpFiles(groupDir);
|
||||
// The mkdtemp staging directory above is freshly created with a unique
|
||||
// random suffix, so there are no leftover `bridge.lbug.tmp` / `.wal` /
|
||||
// `.shadow` sidecars from a previous crashed run to clean up here — the
|
||||
// directory is empty by construction.
|
||||
|
||||
// 1. Create temp DB, insert all data.
|
||||
//
|
||||
// Everything after `openBridgeDb` must run inside a try/finally so that
|
||||
// if ANY step before the explicit `closeBridgeDb` throws — schema
|
||||
// creation, a contract insert loop that rethrows, a snapshot write, the
|
||||
// cross-link loop, or anything else — the handle is still released. A
|
||||
// leaked handle holds the native LadybugDB file lock on tmpPath, which
|
||||
// (a) leaks a FD and (b) prevents the next writeBridge call from
|
||||
// reusing the same tmp slot.
|
||||
const handle = await openBridgeDb(tmpPath);
|
||||
let handleClosed = false;
|
||||
try {
|
||||
await ensureBridgeSchema(handle);
|
||||
// 1. Create temp DB, insert all data.
|
||||
//
|
||||
// Everything after `openBridgeDb` must run inside a try/finally so that
|
||||
// if ANY step before the explicit `closeBridgeDb` throws — schema
|
||||
// creation, a contract insert loop that rethrows, a snapshot write, the
|
||||
// cross-link loop, or anything else — the handle is still released. A
|
||||
// leaked handle holds the native LadybugDB file lock on tmpPath, which
|
||||
// (a) leaks a FD and (b) prevents the next writeBridge call from
|
||||
// reusing the same tmp slot.
|
||||
const handle = await openBridgeDb(tmpPath);
|
||||
let handleClosed = false;
|
||||
try {
|
||||
await ensureBridgeSchema(handle);
|
||||
|
||||
// Build the lookup index incrementally as contracts are inserted, so
|
||||
// failed inserts are never in the index (and therefore never resolved
|
||||
// by the cross-link loop below). This replaces a previous N+1 query
|
||||
// pattern where each link made up to 6 DB round-trips to find its
|
||||
// endpoints — see ContractLookupIndex.
|
||||
const lookupIndex = createContractLookupIndex();
|
||||
// Build the lookup index incrementally as contracts are inserted, so
|
||||
// failed inserts are never in the index (and therefore never resolved
|
||||
// by the cross-link loop below). This replaces a previous N+1 query
|
||||
// pattern where each link made up to 6 DB round-trips to find its
|
||||
// endpoints — see ContractLookupIndex.
|
||||
const lookupIndex = createContractLookupIndex();
|
||||
|
||||
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
|
||||
// that can't be serialized). The whole sync must not fail because one
|
||||
// contract is broken.
|
||||
for (const c of contracts) {
|
||||
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (n:Contract {
|
||||
// Insert contracts — tolerate individual failures (e.g., a corrupt meta
|
||||
// that can't be serialized). The whole sync must not fail because one
|
||||
// contract is broken.
|
||||
for (const c of contracts) {
|
||||
const id = contractNodeId(c.repo, c.contractId, c.role, c.symbolRef.filePath);
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (n:Contract {
|
||||
id: $id,
|
||||
contractId: $contractId,
|
||||
type: $type,
|
||||
|
|
@ -438,91 +445,91 @@ export async function writeBridge(
|
|||
confidence: $confidence,
|
||||
meta: $meta
|
||||
})`,
|
||||
{
|
||||
id,
|
||||
contractId: c.contractId,
|
||||
type: c.type,
|
||||
role: c.role,
|
||||
repo: c.repo,
|
||||
service: c.service ?? '',
|
||||
symbolUid: c.symbolUid,
|
||||
filePath: c.symbolRef.filePath,
|
||||
symbolName: c.symbolName,
|
||||
confidence: c.confidence,
|
||||
meta: JSON.stringify(c.meta),
|
||||
},
|
||||
);
|
||||
report.contractsInserted++;
|
||||
// Only index on successful insert — the cross-link loop must never
|
||||
// resolve to a row that isn't actually in the DB.
|
||||
indexContract(lookupIndex, c, id);
|
||||
} catch (err) {
|
||||
report.contractsFailed++;
|
||||
recordError('contract', id, err);
|
||||
{
|
||||
id,
|
||||
contractId: c.contractId,
|
||||
type: c.type,
|
||||
role: c.role,
|
||||
repo: c.repo,
|
||||
service: c.service ?? '',
|
||||
symbolUid: c.symbolUid,
|
||||
filePath: c.symbolRef.filePath,
|
||||
symbolName: c.symbolName,
|
||||
confidence: c.confidence,
|
||||
meta: JSON.stringify(c.meta),
|
||||
},
|
||||
);
|
||||
report.contractsInserted++;
|
||||
// Only index on successful insert — the cross-link loop must never
|
||||
// resolve to a row that isn't actually in the DB.
|
||||
indexContract(lookupIndex, c, id);
|
||||
} catch (err) {
|
||||
report.contractsFailed++;
|
||||
recordError('contract', id, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Insert repo snapshots
|
||||
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (s:RepoSnapshot {
|
||||
// Insert repo snapshots
|
||||
for (const [repoId, snap] of Object.entries(input.repoSnapshots)) {
|
||||
try {
|
||||
await queryBridge(
|
||||
handle,
|
||||
`CREATE (s:RepoSnapshot {
|
||||
id: $id,
|
||||
indexedAt: $indexedAt,
|
||||
lastCommit: $lastCommit
|
||||
})`,
|
||||
{
|
||||
id: repoId,
|
||||
indexedAt: snap.indexedAt,
|
||||
lastCommit: snap.lastCommit,
|
||||
},
|
||||
);
|
||||
report.snapshotsInserted++;
|
||||
} catch (err) {
|
||||
report.snapshotsFailed++;
|
||||
recordError('snapshot', repoId, err);
|
||||
}
|
||||
}
|
||||
|
||||
// Insert cross-links (tolerating missing nodes).
|
||||
//
|
||||
// `findContractNode` consults the in-memory lookup index built above,
|
||||
// not the DB — that's an O(1) pure-function lookup per endpoint instead
|
||||
// of the previous 2-3 DB queries. For M cross-links, the previous code
|
||||
// issued up to 6M round-trips; this version issues zero.
|
||||
//
|
||||
// `link.contractId` may differ between the consumer and provider sides
|
||||
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
|
||||
// `grpc::Service/Method`) — that's why we resolve each endpoint
|
||||
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
|
||||
// tuple rather than matching on contractId.
|
||||
for (const link of crossLinks) {
|
||||
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
|
||||
try {
|
||||
const fromId = findContractNode(
|
||||
lookupIndex,
|
||||
link.from.repo,
|
||||
'consumer',
|
||||
link.from.symbolUid,
|
||||
link.from.symbolRef.filePath,
|
||||
link.from.symbolRef.name,
|
||||
);
|
||||
const toId = findContractNode(
|
||||
lookupIndex,
|
||||
link.to.repo,
|
||||
'provider',
|
||||
link.to.symbolUid,
|
||||
link.to.symbolRef.filePath,
|
||||
link.to.symbolRef.name,
|
||||
);
|
||||
if (!fromId || !toId) {
|
||||
report.linksDroppedMissingNode++;
|
||||
continue;
|
||||
{
|
||||
id: repoId,
|
||||
indexedAt: snap.indexedAt,
|
||||
lastCommit: snap.lastCommit,
|
||||
},
|
||||
);
|
||||
report.snapshotsInserted++;
|
||||
} catch (err) {
|
||||
report.snapshotsFailed++;
|
||||
recordError('snapshot', repoId, err);
|
||||
}
|
||||
await queryBridge(
|
||||
handle,
|
||||
`
|
||||
}
|
||||
|
||||
// Insert cross-links (tolerating missing nodes).
|
||||
//
|
||||
// `findContractNode` consults the in-memory lookup index built above,
|
||||
// not the DB — that's an O(1) pure-function lookup per endpoint instead
|
||||
// of the previous 2-3 DB queries. For M cross-links, the previous code
|
||||
// issued up to 6M round-trips; this version issues zero.
|
||||
//
|
||||
// `link.contractId` may differ between the consumer and provider sides
|
||||
// (e.g. wildcard consumer `grpc::Service/*` → method-level provider
|
||||
// `grpc::Service/Method`) — that's why we resolve each endpoint
|
||||
// independently via its own `(repo, role, symbolUid, filePath, symbolName)`
|
||||
// tuple rather than matching on contractId.
|
||||
for (const link of crossLinks) {
|
||||
const linkId = `${link.from.repo}::${link.contractId}->${link.to.repo}::${link.contractId}`;
|
||||
try {
|
||||
const fromId = findContractNode(
|
||||
lookupIndex,
|
||||
link.from.repo,
|
||||
'consumer',
|
||||
link.from.symbolUid,
|
||||
link.from.symbolRef.filePath,
|
||||
link.from.symbolRef.name,
|
||||
);
|
||||
const toId = findContractNode(
|
||||
lookupIndex,
|
||||
link.to.repo,
|
||||
'provider',
|
||||
link.to.symbolUid,
|
||||
link.to.symbolRef.filePath,
|
||||
link.to.symbolRef.name,
|
||||
);
|
||||
if (!fromId || !toId) {
|
||||
report.linksDroppedMissingNode++;
|
||||
continue;
|
||||
}
|
||||
await queryBridge(
|
||||
handle,
|
||||
`
|
||||
MATCH (a:Contract), (b:Contract)
|
||||
WHERE a.id = $fromId AND b.id = $toId
|
||||
CREATE (a)-[:ContractLink {
|
||||
|
|
@ -533,83 +540,93 @@ export async function writeBridge(
|
|||
toRepo: $toRepo
|
||||
}]->(b)
|
||||
`,
|
||||
{
|
||||
fromId,
|
||||
toId,
|
||||
matchType: link.matchType,
|
||||
confidence: link.confidence,
|
||||
contractId: link.contractId,
|
||||
fromRepo: link.from.repo,
|
||||
toRepo: link.to.repo,
|
||||
},
|
||||
);
|
||||
report.linksInserted++;
|
||||
} catch (err) {
|
||||
report.linksFailed++;
|
||||
recordError('link', linkId, err);
|
||||
{
|
||||
fromId,
|
||||
toId,
|
||||
matchType: link.matchType,
|
||||
confidence: link.confidence,
|
||||
contractId: link.contractId,
|
||||
fromRepo: link.from.repo,
|
||||
toRepo: link.to.repo,
|
||||
},
|
||||
);
|
||||
report.linksInserted++;
|
||||
} catch (err) {
|
||||
report.linksFailed++;
|
||||
recordError('link', linkId, err);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Close temp DB (happy path). The finally block also calls
|
||||
// closeBridgeDb if we threw above; `handleClosed` prevents a
|
||||
// double-close on the native handle.
|
||||
await closeBridgeDb(handle);
|
||||
handleClosed = true;
|
||||
} finally {
|
||||
if (!handleClosed) {
|
||||
await closeBridgeDb(handle).catch(() => {
|
||||
/* ignore: cleanup path, best effort */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Close temp DB (happy path). The finally block also calls
|
||||
// closeBridgeDb if we threw above; `handleClosed` prevents a
|
||||
// double-close on the native handle.
|
||||
await closeBridgeDb(handle);
|
||||
handleClosed = true;
|
||||
} finally {
|
||||
if (!handleClosed) {
|
||||
await closeBridgeDb(handle).catch(() => {
|
||||
/* ignore: cleanup path, best effort */
|
||||
});
|
||||
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
|
||||
//
|
||||
// The current database file (with its `.wal` / `.shadow` sidecars) is
|
||||
// moved aside, then the freshly built tmp database takes its place.
|
||||
// We move the sidecars together with the main file so the open below
|
||||
// and any external readers see a consistent set; orphan sidecars from
|
||||
// the tmp namespace are then removed because LadybugDB looks for them
|
||||
// under the renamed-to base name and would reject mismatching IDs.
|
||||
try {
|
||||
await fsp.access(finalPath);
|
||||
await retryRename(finalPath, bakPath);
|
||||
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
|
||||
try {
|
||||
await fsp.access(`${finalPath}${suffix}`);
|
||||
await retryRename(`${finalPath}${suffix}`, `${bakPath}${suffix}`);
|
||||
} catch {
|
||||
/* sidecar absent — nothing to move */
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* no existing db */
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Atomic swap: old→.bak, tmp→final, rm .bak
|
||||
//
|
||||
// The current database file (with its `.wal` / `.shadow` sidecars) is
|
||||
// moved aside, then the freshly built tmp database takes its place.
|
||||
// We move the sidecars together with the main file so the open below
|
||||
// and any external readers see a consistent set; orphan sidecars from
|
||||
// the tmp namespace are then removed because LadybugDB looks for them
|
||||
// under the renamed-to base name and would reject mismatching IDs.
|
||||
try {
|
||||
await fsp.access(finalPath);
|
||||
await retryRename(finalPath, bakPath);
|
||||
await retryRename(tmpPath, finalPath);
|
||||
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
|
||||
// Rename — not delete — so the WAL (which may carry uncommitted-at-
|
||||
// close-time pages on a graceful close, depending on
|
||||
// `autoCheckpoint` / `checkpointThreshold`) and the `.shadow`
|
||||
// checkpoint snapshot stay paired with the database file under its
|
||||
// final name. LadybugDB 0.16.0's database-id check rejects an open
|
||||
// when the sidecars belong to a different base name.
|
||||
try {
|
||||
await fsp.access(`${finalPath}${suffix}`);
|
||||
await retryRename(`${finalPath}${suffix}`, `${bakPath}${suffix}`);
|
||||
await fsp.access(`${tmpPath}${suffix}`);
|
||||
await retryRename(`${tmpPath}${suffix}`, `${finalPath}${suffix}`);
|
||||
} catch {
|
||||
/* sidecar absent — nothing to move */
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* no existing db */
|
||||
}
|
||||
await retryRename(tmpPath, finalPath);
|
||||
for (const suffix of LBUG_SIDECAR_SUFFIXES) {
|
||||
// Rename — not delete — so the WAL (which may carry uncommitted-at-
|
||||
// close-time pages on a graceful close, depending on
|
||||
// `autoCheckpoint` / `checkpointThreshold`) and the `.shadow`
|
||||
// checkpoint snapshot stay paired with the database file under its
|
||||
// final name. LadybugDB 0.16.0's database-id check rejects an open
|
||||
// when the sidecars belong to a different base name.
|
||||
try {
|
||||
await fsp.access(`${tmpPath}${suffix}`);
|
||||
await retryRename(`${tmpPath}${suffix}`, `${finalPath}${suffix}`);
|
||||
} catch {
|
||||
/* sidecar absent — nothing to move */
|
||||
}
|
||||
}
|
||||
await removeLbugFile(bakPath);
|
||||
await removeLbugFile(bakPath);
|
||||
|
||||
// 4. Write meta.json
|
||||
await writeBridgeMeta(groupDir, {
|
||||
version: BRIDGE_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
missingRepos: input.missingRepos,
|
||||
});
|
||||
// 4. Write meta.json
|
||||
await writeBridgeMeta(groupDir, {
|
||||
version: BRIDGE_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
missingRepos: input.missingRepos,
|
||||
});
|
||||
|
||||
return report;
|
||||
return report;
|
||||
} finally {
|
||||
// Always remove the mkdtemp staging directory. On the happy path the
|
||||
// main file and sidecars have been renamed out of it, so it's empty;
|
||||
// on any error path it may still contain a partial database — either
|
||||
// way `recursive: true, force: true` removes it without surfacing
|
||||
// "directory not empty" or ENOENT.
|
||||
await fsp.rm(stagingDir, { recursive: true, force: true }).catch(() => {
|
||||
/* best-effort cleanup */
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
|
@ -705,8 +722,15 @@ export async function openBridgeDbReadOnly(groupDir: string): Promise<BridgeHand
|
|||
await new Promise((r) => setTimeout(r, delay));
|
||||
}
|
||||
}
|
||||
// Strip CRLF from user-controlled strings before logging to close
|
||||
// CodeQL js/log-injection. Pino's NDJSON serialization already
|
||||
// JSON-escapes all values, but we sanitize here as a defence-in-depth
|
||||
// measure so CodeQL can see the taint flow is broken.
|
||||
const safeGroupDir = String(groupDir).replace(/[\r\n]/g, ' ');
|
||||
const safeErrMsg =
|
||||
lastErr instanceof Error ? String(lastErr.message).replace(/[\r\n]/g, ' ') : undefined;
|
||||
bridgeLogger.debug(
|
||||
{ groupDir, err: lastErr, attempts: LBUG_OPEN_RETRY_ATTEMPTS },
|
||||
{ groupDir: safeGroupDir, errMsg: safeErrMsg, attempts: LBUG_OPEN_RETRY_ATTEMPTS },
|
||||
'openBridgeDbReadOnly gave up',
|
||||
);
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -4,9 +4,26 @@ import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from
|
|||
const _require = createRequire(import.meta.url);
|
||||
const yaml = _require('js-yaml') as typeof import('js-yaml');
|
||||
|
||||
const VALID_CONTRACT_TYPES: ContractType[] = ['http', 'grpc', 'thrift', 'topic', 'lib', 'custom'];
|
||||
const VALID_CONTRACT_TYPES: ContractType[] = [
|
||||
'http',
|
||||
'grpc',
|
||||
'thrift',
|
||||
'topic',
|
||||
'lib',
|
||||
'custom',
|
||||
'include',
|
||||
];
|
||||
const VALID_ROLES: ContractRole[] = ['provider', 'consumer'];
|
||||
|
||||
// Defaults matter for backward compatibility: any group.yaml that omits a
|
||||
// `detect.<type>` key inherits its value from this constant. Adding a new
|
||||
// extractor that defaults to `true` silently changes the behavior of every
|
||||
// existing group on the next sync. New extractors must default to `false`
|
||||
// (opt-in) so operators consciously enable them via group.yaml.
|
||||
//
|
||||
// `includes`: opt-in. The C/C++ IncludeExtractor (PR #1156) ships disabled by
|
||||
// default; enable with `detect.includes: true` for groups containing C/C++
|
||||
// repos that need cross-repo header tracking.
|
||||
const DEFAULT_DETECT = {
|
||||
http: true,
|
||||
grpc: true,
|
||||
|
|
@ -14,6 +31,7 @@ const DEFAULT_DETECT = {
|
|||
topics: true,
|
||||
shared_libs: true,
|
||||
embedding_fallback: true,
|
||||
includes: false,
|
||||
workspace_deps: false,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -91,6 +91,25 @@ function clampCrossDepth(raw: unknown): { depth: number; warning?: string } {
|
|||
return { depth: d };
|
||||
}
|
||||
|
||||
/**
|
||||
* Clamp the impact timeout to a sane bounded range. Callers can feed this
|
||||
* via tool params, so an unclamped value lets a single request hold a
|
||||
* timer slot for an arbitrarily long duration (CodeQL js/resource-
|
||||
* exhaustion). 100ms lower bound preserves test-suite scenarios that
|
||||
* exercise tight timeouts; 5min upper bound is well above any legitimate
|
||||
* single-impact compute. Applied at the validate boundary so the
|
||||
* downstream `deadline` (Date.now() + timeoutMs) and the local-leg
|
||||
* `setTimeout` see the same clamped value — earlier shapes had a 1hr
|
||||
* outer cap and a 5min inner clamp that disagreed.
|
||||
*/
|
||||
export const IMPACT_TIMEOUT_MIN_MS = 100;
|
||||
export const IMPACT_TIMEOUT_MAX_MS = 5 * 60 * 1_000;
|
||||
|
||||
export function clampTimeout(timeoutMs: number): number {
|
||||
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) return IMPACT_TIMEOUT_MIN_MS;
|
||||
return Math.min(IMPACT_TIMEOUT_MAX_MS, Math.max(IMPACT_TIMEOUT_MIN_MS, Math.trunc(timeoutMs)));
|
||||
}
|
||||
|
||||
export function validateGroupImpactParams(params: Record<string, unknown>):
|
||||
| {
|
||||
ok: true;
|
||||
|
|
@ -143,13 +162,19 @@ export function validateGroupImpactParams(params: Record<string, unknown>):
|
|||
const service = normalizeServicePrefix(params.service);
|
||||
const subgroup = typeof params.subgroup === 'string' ? params.subgroup : undefined;
|
||||
|
||||
let timeoutMs =
|
||||
// Clamp at the validate boundary so the downstream `deadline` (line
|
||||
// ~366) and `safeLocalImpact`'s `setTimeout` both see a single
|
||||
// bounded value. Without this, the outer deadline budgeted Phase-2
|
||||
// cross-repo fanout up to 1hr while only the inner setTimeout was
|
||||
// capped to 5min — the two halves of CodeQL #184's mitigation
|
||||
// disagreed.
|
||||
const rawTimeoutMs =
|
||||
typeof params.timeoutMs === 'number' && params.timeoutMs > 0
|
||||
? params.timeoutMs
|
||||
: typeof params.timeout === 'number' && params.timeout > 0
|
||||
? params.timeout
|
||||
: DEFAULT_LOCAL_IMPACT_TIMEOUT_MS;
|
||||
if (timeoutMs > 3_600_000) timeoutMs = 3_600_000;
|
||||
const timeoutMs = clampTimeout(rawTimeoutMs);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
|
|
@ -191,12 +216,13 @@ async function safeLocalImpact(
|
|||
impactParams: Parameters<GroupToolPort['impact']>[1],
|
||||
timeoutMs: number,
|
||||
): Promise<{ value: unknown; timedOut: boolean }> {
|
||||
const safeTimeoutMs = clampTimeout(timeoutMs);
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const impactP = port.impact(repo, impactParams).catch((err) => ({
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
}));
|
||||
const timeoutP = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(() => resolve('timeout'), timeoutMs);
|
||||
timer = setTimeout(() => resolve('timeout'), safeTimeoutMs);
|
||||
});
|
||||
const won = await Promise.race([
|
||||
impactP.then((v) => ({ tag: 'impact' as const, v })),
|
||||
|
|
@ -212,6 +238,65 @@ async function safeLocalImpact(
|
|||
return { value: won.v, timedOut: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Race a single Phase-2 `impactByUid` call against a remaining-budget
|
||||
* timer. The Codex adversarial review on PR #1331 surfaced that the
|
||||
* fanout loop only checked `Date.now() > deadline` *between* neighbor
|
||||
* calls — once `await port.impactByUid(...)` was reached, a hung
|
||||
* neighbor could pin the request indefinitely, and slow neighbors
|
||||
* could compound past the 5-min `IMPACT_TIMEOUT_MAX_MS` cap.
|
||||
*
|
||||
* This helper wraps each call: a `setTimeout(remainingMs)` aborts an
|
||||
* `AbortController` whose signal is forwarded to `impactByUid`, and a
|
||||
* `Promise.race` resolves to `{ timedOut: true }` when the timer
|
||||
* fires before the call completes. Implementors that ignore the
|
||||
* signal (current local backend) still see their await resolved by
|
||||
* the race; full cooperative cancellation inside the BFS is a future
|
||||
* follow-up. On rejection, the value is `null` (matching the
|
||||
* fanout's existing `if (fan == null)` truncation contract).
|
||||
*
|
||||
* Exported for direct unit testing — the helper IS the load-bearing
|
||||
* mitigation surface, so the U3 regression test pins it directly
|
||||
* rather than driving the full `runGroupImpact` path.
|
||||
*/
|
||||
export async function safeNeighborImpact(
|
||||
port: GroupToolPort,
|
||||
repoId: string,
|
||||
uid: string,
|
||||
direction: string,
|
||||
opts: {
|
||||
maxDepth: number;
|
||||
relationTypes: string[];
|
||||
minConfidence: number;
|
||||
includeTests: boolean;
|
||||
},
|
||||
remainingMs: number,
|
||||
): Promise<{ value: unknown; timedOut: boolean }> {
|
||||
const controller = new AbortController();
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const callP = port
|
||||
.impactByUid(repoId, uid, direction, { ...opts, signal: controller.signal })
|
||||
.catch(() => null);
|
||||
const timeoutP = new Promise<'timeout'>((resolve) => {
|
||||
timer = setTimeout(
|
||||
() => {
|
||||
controller.abort();
|
||||
resolve('timeout');
|
||||
},
|
||||
Math.max(0, remainingMs),
|
||||
);
|
||||
});
|
||||
const won = await Promise.race([
|
||||
callP.then((v) => ({ tag: 'impact' as const, v })),
|
||||
timeoutP.then(() => ({ tag: 'timeout' as const })),
|
||||
]);
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
if (won.tag === 'timeout') {
|
||||
return { value: null, timedOut: true };
|
||||
}
|
||||
return { value: won.v, timedOut: false };
|
||||
}
|
||||
|
||||
export function collectImpactSymbolUids(
|
||||
local: unknown,
|
||||
servicePrefix: string | undefined,
|
||||
|
|
@ -476,7 +561,8 @@ export async function runGroupImpact(
|
|||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
|
||||
if (Date.now() > deadline) {
|
||||
const remainingMs = deadline - Date.now();
|
||||
if (remainingMs <= 0) {
|
||||
truncatedRepos.push(n.neighborRepo);
|
||||
continue;
|
||||
}
|
||||
|
|
@ -492,13 +578,25 @@ export async function runGroupImpact(
|
|||
continue;
|
||||
}
|
||||
|
||||
const fan = await deps.port.impactByUid(neighborHandle.id, n.neighborUid, direction, {
|
||||
maxDepth,
|
||||
relationTypes: relationTypes ?? [],
|
||||
minConfidence,
|
||||
includeTests,
|
||||
});
|
||||
if (fan == null) {
|
||||
// Phase-2 hardening: race each impactByUid against a per-call
|
||||
// timeout derived from the remaining budget. Without this wrap a
|
||||
// single hung neighbor would pin the request past the clamped
|
||||
// timeout, which Codex's adversarial review on PR #1331 flagged
|
||||
// as the still-open half of CodeQL #184 / js/resource-exhaustion.
|
||||
const { value: fan, timedOut: neighborTimedOut } = await safeNeighborImpact(
|
||||
deps.port,
|
||||
neighborHandle.id,
|
||||
n.neighborUid,
|
||||
direction,
|
||||
{
|
||||
maxDepth,
|
||||
relationTypes: relationTypes ?? [],
|
||||
minConfidence,
|
||||
includeTests,
|
||||
},
|
||||
remainingMs,
|
||||
);
|
||||
if (neighborTimedOut || fan == null) {
|
||||
truncatedRepos.push(n.neighborRepo);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
610
gitnexus/src/core/group/extractors/include-extractor.ts
Normal file
610
gitnexus/src/core/group/extractors/include-extractor.ts
Normal file
|
|
@ -0,0 +1,610 @@
|
|||
import * as path from 'node:path';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import { glob } from 'glob';
|
||||
import Parser from 'tree-sitter';
|
||||
import C from 'tree-sitter-c';
|
||||
import Cpp from 'tree-sitter-cpp';
|
||||
import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js';
|
||||
import type { ExtractedContract, RepoHandle } from '../types.js';
|
||||
import { readSafe } from './fs-utils.js';
|
||||
import { buildSuffixIndex, type SuffixIndex } from '../../ingestion/import-resolvers/utils.js';
|
||||
import { createIgnoreFilter } from '../../../config/ignore-service.js';
|
||||
import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js';
|
||||
import { logger } from '../../logger.js';
|
||||
|
||||
/**
|
||||
* Cross-repo C/C++ `#include` dependency extractor.
|
||||
*
|
||||
* **Provider side:** registers every `.h/.hpp/.hxx/.hh` file in the repo
|
||||
* as a provider contract with `include::<relative-path>`.
|
||||
*
|
||||
* **Consumer side:** parses all C/C++ source/header files for `#include "…"`
|
||||
* directives, attempts suffix-based resolution against the repo's own file
|
||||
* list (reusing the same algorithm as the single-repo ingestion pipeline),
|
||||
* and emits unresolved include paths as consumer contracts.
|
||||
*
|
||||
* Matching: a consumer's `include::map/base/dice_map_view.h` in repo A
|
||||
* matches a provider's `include::map/base/dice_map_view.h` in repo B via
|
||||
* exact contract-id equality in `runExactMatch`.
|
||||
*/
|
||||
|
||||
// ---------- constants ----------
|
||||
|
||||
const HEADER_EXTENSIONS = new Set(['.h', '.hpp', '.hxx', '.hh']);
|
||||
|
||||
// Source = headers (provider-eligible) ∪ implementation files (.c/.cpp/.cc/.cxx).
|
||||
// Spread keeps the subset relationship explicit so a future contributor adding
|
||||
// a new header extension to HEADER_EXTENSIONS does not have to remember to
|
||||
// also add it here.
|
||||
const SOURCE_EXTENSIONS = new Set<string>([...HEADER_EXTENSIONS, '.c', '.cpp', '.cc', '.cxx']);
|
||||
|
||||
const INCLUDE_QUERY_SRC = '(preproc_include path: (_) @import.source) @import';
|
||||
|
||||
/**
|
||||
* Well-known C/C++ standard library headers that can appear in `#include "…"`
|
||||
* form (some projects use quotes for system headers).
|
||||
*/
|
||||
const SYSTEM_HEADERS = new Set([
|
||||
// C standard
|
||||
'assert.h',
|
||||
'complex.h',
|
||||
'ctype.h',
|
||||
'errno.h',
|
||||
'fenv.h',
|
||||
'float.h',
|
||||
'inttypes.h',
|
||||
'iso646.h',
|
||||
'limits.h',
|
||||
'locale.h',
|
||||
'math.h',
|
||||
'setjmp.h',
|
||||
'signal.h',
|
||||
'stdalign.h',
|
||||
'stdarg.h',
|
||||
'stdatomic.h',
|
||||
'stdbool.h',
|
||||
'stddef.h',
|
||||
'stdint.h',
|
||||
'stdio.h',
|
||||
'stdlib.h',
|
||||
'stdnoreturn.h',
|
||||
'string.h',
|
||||
'tgmath.h',
|
||||
'threads.h',
|
||||
'time.h',
|
||||
'uchar.h',
|
||||
'wchar.h',
|
||||
'wctype.h',
|
||||
// C++ standard (extensionless)
|
||||
'algorithm',
|
||||
'any',
|
||||
'array',
|
||||
'atomic',
|
||||
'barrier',
|
||||
'bit',
|
||||
'bitset',
|
||||
'cassert',
|
||||
'cctype',
|
||||
'cerrno',
|
||||
'cfenv',
|
||||
'cfloat',
|
||||
'charconv',
|
||||
'chrono',
|
||||
'cinttypes',
|
||||
'climits',
|
||||
'clocale',
|
||||
'cmath',
|
||||
'codecvt',
|
||||
'compare',
|
||||
'complex',
|
||||
'concepts',
|
||||
'condition_variable',
|
||||
'coroutine',
|
||||
'csetjmp',
|
||||
'csignal',
|
||||
'cstdarg',
|
||||
'cstddef',
|
||||
'cstdint',
|
||||
'cstdio',
|
||||
'cstdlib',
|
||||
'cstring',
|
||||
'ctime',
|
||||
'cuchar',
|
||||
'cwchar',
|
||||
'cwctype',
|
||||
'deque',
|
||||
'exception',
|
||||
'execution',
|
||||
'expected',
|
||||
'filesystem',
|
||||
'format',
|
||||
'forward_list',
|
||||
'fstream',
|
||||
'functional',
|
||||
'future',
|
||||
'generator',
|
||||
'initializer_list',
|
||||
'iomanip',
|
||||
'ios',
|
||||
'iosfwd',
|
||||
'iostream',
|
||||
'istream',
|
||||
'iterator',
|
||||
'latch',
|
||||
'limits',
|
||||
'list',
|
||||
'locale',
|
||||
'map',
|
||||
'mdspan',
|
||||
'memory',
|
||||
'memory_resource',
|
||||
'mutex',
|
||||
'new',
|
||||
'numbers',
|
||||
'numeric',
|
||||
'optional',
|
||||
'ostream',
|
||||
'print',
|
||||
'queue',
|
||||
'random',
|
||||
'ranges',
|
||||
'ratio',
|
||||
'regex',
|
||||
'scoped_allocator',
|
||||
'semaphore',
|
||||
'set',
|
||||
'shared_mutex',
|
||||
'source_location',
|
||||
'span',
|
||||
'spanstream',
|
||||
'sstream',
|
||||
'stack',
|
||||
'stacktrace',
|
||||
'stdexcept',
|
||||
'stdfloat',
|
||||
'stop_token',
|
||||
'streambuf',
|
||||
'string',
|
||||
'string_view',
|
||||
'strstream',
|
||||
'syncstream',
|
||||
'system_error',
|
||||
'thread',
|
||||
'tuple',
|
||||
'type_traits',
|
||||
'typeindex',
|
||||
'typeinfo',
|
||||
'unordered_map',
|
||||
'unordered_set',
|
||||
'utility',
|
||||
'valarray',
|
||||
'variant',
|
||||
'vector',
|
||||
'version',
|
||||
]);
|
||||
|
||||
/** Path prefixes that indicate system/kernel headers. */
|
||||
const SYSTEM_PATH_PREFIXES = [
|
||||
'sys/',
|
||||
'net/',
|
||||
'netinet/',
|
||||
'arpa/',
|
||||
'linux/',
|
||||
'asm/',
|
||||
'bits/',
|
||||
'gnu/',
|
||||
'mach/',
|
||||
'machine/',
|
||||
'xlocale/',
|
||||
];
|
||||
|
||||
/** Regex fallback for files that exceed tree-sitter's 32 KB parse limit. */
|
||||
const INCLUDE_REGEX = /^[ \t]*#\s*include\s*"([^"]+)"/gm;
|
||||
|
||||
// ---------- helpers ----------
|
||||
|
||||
/**
|
||||
* Normalize an include path to a canonical lowercase forward-slash form.
|
||||
*
|
||||
* IMPORTANT — case-folding caveat (PR #1156 review finding #3):
|
||||
* Header paths are lowercased so consumer `#include "Foo/Bar.h"` and
|
||||
* provider file `Foo/Bar.h` normalize to the same contract-id. This is
|
||||
* the right trade-off on case-insensitive filesystems (macOS, Windows)
|
||||
* but on case-sensitive Linux filesystems two distinct headers `Foo.h`
|
||||
* and `foo.h` in the same repo will collide onto the same provider
|
||||
* contract-id; only one survives `dedupe()`. The gain (reliable
|
||||
* cross-platform matching) outweighs the cost (extremely rare header
|
||||
* casing collisions inside a single repo).
|
||||
*/
|
||||
function normalizeIncludePath(raw: string): string {
|
||||
return raw.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase();
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip C/C++ block comments from a source blob. Used only by the
|
||||
* regex-fallback path to avoid emitting consumer contracts for
|
||||
* commented-out #include directives. Line comments (`// …`) cannot hide
|
||||
* #include directives because the regex anchors on start-of-line.
|
||||
* See PR #1156 review finding #5.
|
||||
*/
|
||||
function stripBlockComments(src: string): string {
|
||||
return src.replace(/\/\*[\s\S]*?\*\//g, '');
|
||||
}
|
||||
|
||||
function isAngleBracketInclude(rawNodeText: string): boolean {
|
||||
const trimmed = rawNodeText.trim();
|
||||
return trimmed.startsWith('<') && trimmed.endsWith('>');
|
||||
}
|
||||
|
||||
function isSystemHeader(cleanedPath: string): boolean {
|
||||
// Check well-known standard headers
|
||||
if (SYSTEM_HEADERS.has(cleanedPath)) return true;
|
||||
// Check system path prefixes
|
||||
const lower = cleanedPath.toLowerCase();
|
||||
return SYSTEM_PATH_PREFIXES.some((prefix) => lower.startsWith(prefix));
|
||||
}
|
||||
|
||||
function isHeaderFile(filePath: string): boolean {
|
||||
return HEADER_EXTENSIONS.has(path.extname(filePath).toLowerCase());
|
||||
}
|
||||
|
||||
function getLanguageForFile(filePath: string): unknown | null {
|
||||
const ext = path.extname(filePath).toLowerCase();
|
||||
switch (ext) {
|
||||
case '.c':
|
||||
case '.h':
|
||||
return C;
|
||||
case '.cpp':
|
||||
case '.cc':
|
||||
case '.cxx':
|
||||
case '.hpp':
|
||||
case '.hxx':
|
||||
case '.hh':
|
||||
return Cpp;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check whether an include path resolves to a file inside the local repo.
|
||||
*
|
||||
* Uses *exact full-path* matching on the suffix index — we never accept a
|
||||
* truncated suffix match. For `#include "foo/bar.h"` this checks:
|
||||
* (a) a file whose path ends with the full `foo/bar.h`
|
||||
* (b) if the include omitted the extension, a file whose path ends with
|
||||
* the include + one of the C/C++ header extensions
|
||||
*
|
||||
* Returns `true` when a local file matches — caller should suppress the
|
||||
* cross-repo consumer contract.
|
||||
*
|
||||
* See PR #1156 review finding #4 (suffixResolve ambiguity).
|
||||
*/
|
||||
function isLocalInclude(cleaned: string, suffixIndex: SuffixIndex): boolean {
|
||||
const candidates = [cleaned];
|
||||
if (!/\.[a-zA-Z0-9]+$/.test(cleaned)) {
|
||||
for (const ext of ['.h', '.hpp', '.hxx', '.hh']) candidates.push(cleaned + ext);
|
||||
}
|
||||
for (const c of candidates) {
|
||||
if (suffixIndex.get(c) || suffixIndex.getInsensitive(c)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// ---------- main class ----------
|
||||
|
||||
export class IncludeExtractor implements ContractExtractor {
|
||||
type = 'include' as const;
|
||||
|
||||
/**
|
||||
* Always returns `true`. NOT called by `sync.ts`, which gates extraction via
|
||||
* `config.detect.includes` instead (see `sync.ts:174`). Kept solely to satisfy
|
||||
* the `ContractExtractor` interface so the type stays uniform across extractors.
|
||||
*/
|
||||
async canExtract(_repo: RepoHandle): Promise<boolean> {
|
||||
return true;
|
||||
}
|
||||
|
||||
async extract(
|
||||
dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
_repo: RepoHandle,
|
||||
): Promise<ExtractedContract[]> {
|
||||
// 1. Build the local file list using the same discovery as ingestion
|
||||
// (createIgnoreFilter + getMaxFileSizeBytes). This guarantees the
|
||||
// universe of provider/consumer paths matches the universe of File
|
||||
// nodes in the LadybugDB graph — so no cross-link points at a UID
|
||||
// that group impact cannot fan out to.
|
||||
// (PR #1156 Codex follow-up: discovery aligned with ingestion.)
|
||||
const allFiles = await this.discoverIndexableFiles(repoPath);
|
||||
const normalizedFiles = allFiles.map((f) => f.replace(/\\/g, '/'));
|
||||
const suffixIndex = buildSuffixIndex(normalizedFiles, allFiles);
|
||||
|
||||
// 2. Provider: register all header files
|
||||
const providers = await this.extractProviders(dbExecutor, repoPath, allFiles);
|
||||
|
||||
// 3. Consumer: filter the shared discovery list for source extensions
|
||||
// and parse #include directives in those files.
|
||||
const sourceFiles = allFiles.filter((f) =>
|
||||
SOURCE_EXTENSIONS.has(path.extname(f).toLowerCase()),
|
||||
);
|
||||
const consumers = await this.extractConsumers(repoPath, sourceFiles, suffixIndex);
|
||||
|
||||
return this.dedupe([...providers, ...consumers]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover repo-relative file paths using exactly the same rules the
|
||||
* ingestion pipeline uses (`walkRepositoryPaths` in
|
||||
* `gitnexus/src/core/ingestion/filesystem-walker.ts`):
|
||||
* - `createIgnoreFilter` honors `.gitignore`, `.gitnexusignore`, the
|
||||
* hardcoded ignore list, and `.gitnexusignore` last-match-wins
|
||||
* negation.
|
||||
* - `getMaxFileSizeBytes()` drops files larger than the cap so we
|
||||
* never emit `File:<rel>` UIDs for files ingestion would skip.
|
||||
*
|
||||
* Uses sequential stat — there is no `READ_CONCURRENCY` batching here
|
||||
* because group sync runs at startup-time, not the ingestion hot path,
|
||||
* and parallelism gains are not worth the import-graph weight.
|
||||
*
|
||||
* MAINTENANCE: if `walkRepositoryPaths` changes its glob options, ignore
|
||||
* filter shape, or size-cap logic, mirror those changes here. The two
|
||||
* implementations exist because the consumers need different return
|
||||
* shapes (string[] vs ScannedFile[]) and different concurrency, but
|
||||
* they MUST agree on which files are reachable — that is what makes
|
||||
* `File:<rel>` UIDs in cross-links correspond to graph File nodes.
|
||||
*/
|
||||
private async discoverIndexableFiles(repoPath: string): Promise<string[]> {
|
||||
const ignoreFilter = await createIgnoreFilter(repoPath);
|
||||
const maxFileSizeBytes = getMaxFileSizeBytes();
|
||||
|
||||
const candidates = await glob('**/*', {
|
||||
cwd: repoPath,
|
||||
nodir: true,
|
||||
dot: false,
|
||||
ignore: ignoreFilter,
|
||||
});
|
||||
|
||||
const survivors: string[] = [];
|
||||
for (const rel of candidates) {
|
||||
try {
|
||||
const stat = await fs.stat(path.join(repoPath, rel));
|
||||
if (stat.size > maxFileSizeBytes) continue;
|
||||
survivors.push(rel);
|
||||
} catch (err) {
|
||||
// ENOENT is the documented benign race (glob enumerated a file
|
||||
// that was deleted before we stat'd it — same race
|
||||
// walkRepositoryPaths absorbs via Promise.allSettled). Anything
|
||||
// else (EACCES, EMFILE, EIO) deserves a warning so an operator
|
||||
// can spot a permission/resource problem instead of silently
|
||||
// shipping fewer contracts than expected.
|
||||
const code = (err as NodeJS.ErrnoException | undefined)?.code;
|
||||
if (code !== 'ENOENT') {
|
||||
logger.warn(
|
||||
{ err: (err as Error).message, file: rel, repoPath },
|
||||
'⚠️ IncludeExtractor: stat failed during discovery; skipping file',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
return survivors;
|
||||
}
|
||||
|
||||
// ---------- provider extraction ----------
|
||||
|
||||
private async extractProviders(
|
||||
dbExecutor: CypherExecutor | null,
|
||||
repoPath: string,
|
||||
allFiles: string[],
|
||||
): Promise<ExtractedContract[]> {
|
||||
// Strategy A: graph-assisted
|
||||
if (dbExecutor) {
|
||||
const graphProviders = await this.extractProvidersGraph(dbExecutor, repoPath);
|
||||
if (graphProviders.length > 0) return graphProviders;
|
||||
}
|
||||
// Strategy B: filesystem fallback
|
||||
return this.extractProvidersFallback(repoPath, allFiles);
|
||||
}
|
||||
|
||||
private async extractProvidersGraph(
|
||||
db: CypherExecutor,
|
||||
repoPath: string,
|
||||
): Promise<ExtractedContract[]> {
|
||||
try {
|
||||
const rows = await db(
|
||||
`MATCH (f:File)
|
||||
WHERE f.filePath =~ '.*\\\\.(h|hpp|hxx|hh)$'
|
||||
RETURN f.filePath AS filePath, f.id AS fileId`,
|
||||
);
|
||||
// gitnexus analyze stores absolute paths in the File.filePath column.
|
||||
// Provider contract IDs MUST be repo-relative — otherwise the consumer
|
||||
// emits `include::map/base/view.h` and the provider emits
|
||||
// `include::/abs/path/to/repo/map/base/view.h`, which never match
|
||||
// through runExactMatch and the cross-link silently disappears.
|
||||
// (PR #1156 follow-up review: graph provider absolute-path bug.)
|
||||
const normalizedRepoPath = path.resolve(repoPath);
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const r of rows) {
|
||||
if (typeof r.filePath !== 'string' || !r.filePath) continue;
|
||||
const absolute = r.filePath as string;
|
||||
const rel = path.relative(normalizedRepoPath, absolute);
|
||||
// Skip rows that resolve outside the repo (e.g., system headers
|
||||
// somehow indexed, or stale absolute paths from a different machine).
|
||||
// path.relative returns a `..`-prefixed path or an absolute path
|
||||
// when the target is outside the base — both are wrong for our IDs.
|
||||
if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) continue;
|
||||
const normalizedRel = rel.replace(/\\/g, '/');
|
||||
out.push({
|
||||
contractId: `include::${normalizeIncludePath(normalizedRel)}`,
|
||||
type: 'include' as const,
|
||||
role: 'provider' as const,
|
||||
symbolUid: String(r.fileId ?? ''),
|
||||
symbolRef: { filePath: normalizedRel, name: path.basename(normalizedRel) },
|
||||
symbolName: path.basename(normalizedRel),
|
||||
confidence: 1.0,
|
||||
meta: { source: 'graph' },
|
||||
});
|
||||
}
|
||||
return out;
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private extractProvidersFallback(_repoPath: string, allFiles: string[]): ExtractedContract[] {
|
||||
return allFiles
|
||||
.filter((f) => isHeaderFile(f))
|
||||
.map((f) => {
|
||||
const filePath = f.replace(/\\/g, '/');
|
||||
return {
|
||||
contractId: `include::${normalizeIncludePath(filePath)}`,
|
||||
type: 'include' as const,
|
||||
role: 'provider' as const,
|
||||
symbolUid: `File:${filePath}`,
|
||||
symbolRef: { filePath, name: path.basename(filePath) },
|
||||
symbolName: path.basename(filePath),
|
||||
confidence: 0.95,
|
||||
meta: { source: 'filesystem' },
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ---------- consumer extraction ----------
|
||||
|
||||
private async extractConsumers(
|
||||
repoPath: string,
|
||||
sourceFiles: string[],
|
||||
suffixIndex: SuffixIndex,
|
||||
): Promise<ExtractedContract[]> {
|
||||
const parser = new Parser();
|
||||
const out: ExtractedContract[] = [];
|
||||
// Compile the include query once per grammar to avoid re-compilation per file
|
||||
const queryCache = new Map<unknown, Parser.Query>();
|
||||
|
||||
for (const rel of sourceFiles) {
|
||||
const lang = getLanguageForFile(rel);
|
||||
if (!lang) continue;
|
||||
|
||||
const content = readSafe(repoPath, rel);
|
||||
if (!content) continue;
|
||||
|
||||
let query = queryCache.get(lang);
|
||||
if (!query) {
|
||||
try {
|
||||
query = new Parser.Query(lang, INCLUDE_QUERY_SRC);
|
||||
queryCache.set(lang, query);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Collect raw include paths: tree-sitter first, regex fallback for large files.
|
||||
// `extractionSource` is stamped on each emitted consumer contract so
|
||||
// regex-fallback contracts stay auditable post-hoc (PR #1156 review finding #6).
|
||||
let rawIncludes: string[];
|
||||
let extractionSource: 'tree_sitter' | 'regex_fallback';
|
||||
try {
|
||||
parser.setLanguage(lang);
|
||||
const tree = parser.parse(content);
|
||||
let matches: Parser.QueryMatch[];
|
||||
try {
|
||||
matches = query.matches(tree.rootNode);
|
||||
} catch {
|
||||
matches = [];
|
||||
}
|
||||
rawIncludes = [];
|
||||
extractionSource = 'tree_sitter';
|
||||
for (const match of matches) {
|
||||
const sourceNode = match.captures.find((c) => c.name === 'import.source');
|
||||
if (!sourceNode) continue;
|
||||
const rawText = sourceNode.node.text;
|
||||
if (isAngleBracketInclude(rawText)) continue;
|
||||
const cleaned = rawText.replace(/['"<>]/g, '');
|
||||
if (cleaned && cleaned.length <= 2048) rawIncludes.push(cleaned);
|
||||
}
|
||||
} catch {
|
||||
// tree-sitter failed (e.g. file > 32 KB) — fall back to regex.
|
||||
// Strip block comments first so we don't emit a consumer contract
|
||||
// for a commented-out #include (PR #1156 review finding #5).
|
||||
rawIncludes = [];
|
||||
extractionSource = 'regex_fallback';
|
||||
const scanTarget = stripBlockComments(content);
|
||||
INCLUDE_REGEX.lastIndex = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = INCLUDE_REGEX.exec(scanTarget)) !== null) {
|
||||
if (m[1] && m[1].length <= 2048) rawIncludes.push(m[1]);
|
||||
}
|
||||
}
|
||||
|
||||
for (const cleaned of rawIncludes) {
|
||||
// Filter: skip known system headers and system path prefixes
|
||||
if (isSystemHeader(cleaned)) continue;
|
||||
|
||||
// Skip relative-up includes: `#include "../include/foo.h"` is
|
||||
// almost always an intra-repo reference. The suffix index is built
|
||||
// from repo-relative paths, so isLocalInclude can never match
|
||||
// `../foo.h`, and emitting it as a consumer contract just pollutes
|
||||
// the registry with an entry no provider can ever satisfy.
|
||||
// (PR #1156 follow-up review: `../` relative includes produce
|
||||
// spurious consumer contracts.)
|
||||
if (cleaned.startsWith('../') || cleaned.startsWith('..\\')) continue;
|
||||
|
||||
// Skip macro-style includes: `#include PLATFORM_HEADER` parses as an
|
||||
// identifier under tree-sitter's `(_) @import.source` wildcard. The
|
||||
// identifier text passes the strip/clean step unchanged, so without
|
||||
// this guard we would emit `include::platform_header` as a consumer
|
||||
// contract — and no provider in any repo will ever expose a contract
|
||||
// for a macro identifier (no file is named `PLATFORM_HEADER`). The
|
||||
// contract would sit permanently orphaned in the registry. Real
|
||||
// header references always contain a path separator (`/`, `\`) or an
|
||||
// extension dot (`foo.h`), so an absent both is a reliable signal we
|
||||
// are looking at a macro identifier. (PR #1156 follow-up review:
|
||||
// macro includes emit orphaned consumer contracts.)
|
||||
if (!/[./\\]/.test(cleaned)) continue;
|
||||
|
||||
// Local resolution (PR #1156 review finding #4): only accept an
|
||||
// exact-suffix match on the *full* include path. The generic
|
||||
// suffixResolve() iterates all truncated suffixes, which would
|
||||
// silently suppress a cross-repo `#include "map/base/view.h"`
|
||||
// when the local repo has any `internal/view.h` — a realistic
|
||||
// false-negative in large C++ codebases. Here we only resolve
|
||||
// locally if a file path ends with the complete include string
|
||||
// (optionally re-appending one of the C/C++ header extensions
|
||||
// when the include already omits it).
|
||||
if (isLocalInclude(cleaned, suffixIndex)) continue;
|
||||
|
||||
// Unresolved: emit as consumer contract
|
||||
const normalizedRel = rel.replace(/\\/g, '/');
|
||||
out.push({
|
||||
contractId: `include::${normalizeIncludePath(cleaned)}`,
|
||||
type: 'include' as const,
|
||||
role: 'consumer' as const,
|
||||
symbolUid: `File:${normalizedRel}`,
|
||||
symbolRef: { filePath: normalizedRel, name: cleaned },
|
||||
symbolName: cleaned,
|
||||
confidence: 0.85,
|
||||
meta: {
|
||||
source: extractionSource,
|
||||
includePath: cleaned,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- deduplication ----------
|
||||
|
||||
private dedupe(items: ExtractedContract[]): ExtractedContract[] {
|
||||
const seen = new Set<string>();
|
||||
const out: ExtractedContract[] = [];
|
||||
for (const c of items) {
|
||||
const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`;
|
||||
if (seen.has(k)) continue;
|
||||
seen.add(k);
|
||||
out.push(c);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
|
@ -274,6 +274,14 @@ export class ManifestExtractor {
|
|||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else if (link.type === 'include') {
|
||||
rows = await executor(
|
||||
`MATCH (f:File) WHERE f.filePath = $contract
|
||||
RETURN f.id AS uid, f.name AS name, f.filePath AS filePath
|
||||
ORDER BY f.filePath ASC
|
||||
LIMIT 1`,
|
||||
{ contract: link.contract },
|
||||
);
|
||||
} else if (link.type === 'custom') {
|
||||
// Workspace extractors produce qualified contracts like "mathlex::Expression".
|
||||
// Graph nodes store the unqualified symbol name ("Expression"), so strip
|
||||
|
|
@ -358,6 +366,8 @@ export class ManifestExtractor {
|
|||
return `lib::${contract}`;
|
||||
case 'custom':
|
||||
return `custom::${contract}`;
|
||||
case 'include':
|
||||
return `include::${contract}`;
|
||||
default: {
|
||||
const _exhaustive: never = type;
|
||||
throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`);
|
||||
|
|
|
|||
|
|
@ -31,6 +31,32 @@ interface ImportedSymbol {
|
|||
filePath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Linear-time `[package].name = "..."` lookup. The previous regex
|
||||
* `^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"` had a nested
|
||||
* lazy quantifier on `\n` that CodeQL js/redos flagged as exponential
|
||||
* on inputs like `[package]\n` + many bare `\n`. We walk lines
|
||||
* explicitly: scan from the first `[package]` header until we hit the
|
||||
* next `[...]` section header, looking for the `name = "..."` line.
|
||||
* O(n) with the line count.
|
||||
*
|
||||
* Exported so the U8 ReDoS regression test can drive the production
|
||||
* line-walk directly with adversarial fixtures (multi-line strings,
|
||||
* trailing sections, etc.) instead of duplicating it inline.
|
||||
*/
|
||||
export function parseCargoPackageName(content: string): string | null {
|
||||
const lines = content.split('\n');
|
||||
const packageStart = lines.findIndex((l) => l.trim() === '[package]');
|
||||
if (packageStart < 0) return null;
|
||||
for (let i = packageStart + 1; i < lines.length; i++) {
|
||||
const line = lines[i].trimStart();
|
||||
if (line.startsWith('[')) break; // hit the next section header
|
||||
const m = /^name\s*=\s*"([^"]+)"/.exec(line);
|
||||
if (m) return m[1];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Cargo.toml to extract the crate name and workspace dependency
|
||||
* names. Uses simple line-based parsing — no TOML library needed for
|
||||
|
|
@ -47,12 +73,9 @@ async function parseCrateManifest(
|
|||
return null;
|
||||
}
|
||||
|
||||
let name = '';
|
||||
const name = parseCargoPackageName(content) ?? '';
|
||||
const workspaceDeps: string[] = [];
|
||||
|
||||
const nameMatch = content.match(/^\[package\]\s*\n(?:[^\[]*?\n)*?name\s*=\s*"([^"]+)"/m);
|
||||
if (nameMatch) name = nameMatch[1];
|
||||
|
||||
// Match dependencies that use workspace = true, which indicates they
|
||||
// are workspace-internal deps:
|
||||
// dep_name = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -217,6 +217,10 @@ export async function buildThriftContext(repoPath: string): Promise<ThriftContex
|
|||
cwd: repoPath,
|
||||
absolute: false,
|
||||
nodir: true,
|
||||
// TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter
|
||||
// (the canonical ingestion ignore filter, like include-extractor.ts now uses).
|
||||
// New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g.
|
||||
// third_party, 3rdparty added in commit a9936a9b) silently do not apply here.
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
});
|
||||
const namespacesByThrift = new Map<string, string>();
|
||||
|
|
@ -290,6 +294,10 @@ export class ThriftExtractor implements ContractExtractor {
|
|||
cwd: repoPath,
|
||||
absolute: false,
|
||||
nodir: true,
|
||||
// TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter
|
||||
// (the canonical ingestion ignore filter, like include-extractor.ts now uses).
|
||||
// New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g.
|
||||
// third_party, 3rdparty added in commit a9936a9b) silently do not apply here.
|
||||
ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'],
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -107,6 +107,8 @@ export function normalizeContractId(id: string): string {
|
|||
return `topic::${rest.trim().toLowerCase()}`;
|
||||
case 'lib':
|
||||
return `lib::${rest.toLowerCase()}`;
|
||||
case 'include':
|
||||
return `include::${rest.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase()}`;
|
||||
default:
|
||||
return id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,6 +65,15 @@ export interface GroupToolPort {
|
|||
relationTypes: string[];
|
||||
minConfidence: number;
|
||||
includeTests: boolean;
|
||||
// Optional cancellation signal. Callers (notably the cross-impact
|
||||
// Phase-2 fanout) wrap this call in a Promise.race against a
|
||||
// setTimeout-driven AbortController so a single hung neighbor
|
||||
// cannot exceed the request's clamped timeout budget. Implementors
|
||||
// may honor the signal cooperatively or simply let the caller's
|
||||
// race resolve the await — the latter is sufficient for the
|
||||
// resource-exhaustion mitigation. When the signal is absent or
|
||||
// already aborted at call time, behavior is unchanged.
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<unknown | null>;
|
||||
context(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,16 @@ import * as path from 'node:path';
|
|||
import * as os from 'node:os';
|
||||
import { randomBytes } from 'node:crypto';
|
||||
import type { ContractRegistry } from './types.js';
|
||||
import { retryRename } from './bridge-db.js';
|
||||
|
||||
/**
|
||||
* Build an unpredictable suffix for atomic-write tmp files. Replaces the
|
||||
* previous `Date.now()` pattern which CodeQL flagged as
|
||||
* js/insecure-temporary-file: a guessable suffix in a writable directory
|
||||
* lets a co-located attacker pre-create or symlink the tmp path before the
|
||||
* write lands.
|
||||
*/
|
||||
const tmpSuffix = (): string => randomBytes(8).toString('hex');
|
||||
|
||||
const CONTRACTS_FILE = 'contracts.json';
|
||||
|
||||
|
|
@ -35,10 +45,28 @@ export async function writeContractRegistry(
|
|||
registry: ContractRegistry,
|
||||
): Promise<void> {
|
||||
const targetPath = path.join(groupDir, CONTRACTS_FILE);
|
||||
const tmpPath = `${targetPath}.tmp.${randomBytes(8).toString('hex')}`;
|
||||
const tmpPath = `${targetPath}.tmp.${tmpSuffix()}`;
|
||||
|
||||
await fsp.writeFile(tmpPath, JSON.stringify(registry, null, 2), 'utf-8');
|
||||
await fsp.rename(tmpPath, targetPath);
|
||||
// O_EXCL via `'wx'` flag + explicit `0o600` mode — closes both halves
|
||||
// of the CodeQL js/insecure-temporary-file finding: `'wx'` rejects a
|
||||
// pre-planted symlink at the path, and `0o600` (user-only) prevents
|
||||
// the file from being created group/world readable while it briefly
|
||||
// contains contract data en route to the rename. The query's
|
||||
// `isSecureMode` predicate inspects ONLY the mode argument, not the
|
||||
// flags, so the explicit mode is what credits the fix.
|
||||
const handle = await fsp.open(tmpPath, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(JSON.stringify(registry, null, 2), 'utf-8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
// retryRename absorbs the documented Windows EPERM/EBUSY/EACCES race that
|
||||
// fires when AV scanners or another concurrent rename briefly hold the
|
||||
// destination handle between rename calls. Same helper bridge-db.ts uses
|
||||
// (lines 304, 583, 587, 595, 605, 677) for the bridge.lbug atomic swap —
|
||||
// single source of truth for the Windows-rename pattern across the group
|
||||
// package.
|
||||
await retryRename(tmpPath, targetPath);
|
||||
}
|
||||
|
||||
export async function readContractRegistry(groupDir: string): Promise<ContractRegistry | null> {
|
||||
|
|
@ -107,6 +135,38 @@ matching:
|
|||
# exclude_links_paths: [/ping, /health, /healthcheck]
|
||||
# exclude_links_param_only_paths: false
|
||||
`;
|
||||
await fsp.writeFile(path.join(groupDir, 'group.yaml'), template, 'utf-8');
|
||||
// Always write group.yaml with O_EXCL via `fsp.open(..., 'wx')` —
|
||||
// refuses to follow a pre-planted symlink at the target path, closing
|
||||
// the TOCTOU window between the existence check (line ~98) and the
|
||||
// write that CodeQL js/insecure-temporary-file flags. Under
|
||||
// `force=true` we unlink the existing file first (best-effort, no-op
|
||||
// when absent) so the subsequent O_EXCL open succeeds AND the same
|
||||
// symlink-rejection guarantee holds — this is strictly safer than
|
||||
// the previous `flag: force ? 'w' : 'wx'` shape, which silently
|
||||
// followed symlinks under force. CodeQL's rule does not recognize
|
||||
// the `writeFile(path, content, { flag: 'wx' })` shape as O_EXCL;
|
||||
// the explicit open() handle below is what credits the mitigation.
|
||||
const yamlPath = path.join(groupDir, 'group.yaml');
|
||||
if (force) {
|
||||
try {
|
||||
await fsp.unlink(yamlPath);
|
||||
} catch (err) {
|
||||
// ENOENT (file absent) is expected on first run; rethrow anything
|
||||
// else so we don't silently mask permission/EBUSY failures.
|
||||
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
|
||||
}
|
||||
}
|
||||
// `'wx'` rejects a pre-planted symlink at the path; `0o600` is
|
||||
// user-only (no group/world bits) — gitnexus storage is per-user
|
||||
// (`~/.gitnexus/...`), so any "other user wants to read this" case is
|
||||
// a misconfiguration, not a feature. Keeping the file user-only also
|
||||
// satisfies CodeQL's `isSecureMode` predicate (low 6 bits == 0) and
|
||||
// closes the js/insecure-temporary-file alert at this site.
|
||||
const handle = await fsp.open(yamlPath, 'wx', 0o600);
|
||||
try {
|
||||
await handle.writeFile(template, 'utf-8');
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return groupDir;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,12 +8,14 @@ import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
|
|||
import { GrpcExtractor } from './extractors/grpc-extractor.js';
|
||||
import { ThriftExtractor } from './extractors/thrift-extractor.js';
|
||||
import { TopicExtractor } from './extractors/topic-extractor.js';
|
||||
import { IncludeExtractor } from './extractors/include-extractor.js';
|
||||
import { ManifestExtractor } from './extractors/manifest-extractor.js';
|
||||
import { discoverWorkspaceLinks } from './extractors/workspace-extractor.js';
|
||||
import { buildProviderIndex, runExactMatch, runWildcardMatch } from './matching.js';
|
||||
import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
|
||||
import type { CypherExecutor } from './contract-extractor.js';
|
||||
import { writeContractRegistry } from './storage.js';
|
||||
import { writeBridge } from './bridge-db.js';
|
||||
import type { ContractRegistry } from './types.js';
|
||||
|
||||
import { logger } from '../logger.js';
|
||||
|
|
@ -100,6 +102,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
|||
const grpcEx = new GrpcExtractor();
|
||||
const thriftEx = new ThriftExtractor();
|
||||
const topicEx = new TopicExtractor();
|
||||
const includeEx = new IncludeExtractor();
|
||||
dbExecutors = new Map<string, CypherExecutor>();
|
||||
const openPoolIds: string[] = [];
|
||||
|
||||
|
|
@ -168,6 +171,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
|||
}
|
||||
}
|
||||
|
||||
if (config.detect.includes) {
|
||||
const extracted = await includeEx.extract(executor, handle.repoPath, handle);
|
||||
for (const c of extracted) {
|
||||
autoContracts.push({
|
||||
...c,
|
||||
repo: groupPath,
|
||||
service: assignService(c.symbolRef.filePath, boundaries),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const metaPath = path.join(handle.storagePath, 'meta.json');
|
||||
try {
|
||||
const raw = await fs.readFile(metaPath, 'utf-8');
|
||||
|
|
@ -270,6 +284,28 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis
|
|||
|
||||
if (opts?.groupDir && !opts.skipWrite) {
|
||||
await writeContractRegistry(opts.groupDir, registry);
|
||||
// writeBridge failure (disk full, schema error, permission denied) must
|
||||
// not mask the registry — contracts.json was just written successfully
|
||||
// and is the canonical source of truth. A stale or absent bridge
|
||||
// degrades impact queries to empty results, which is recoverable on
|
||||
// the next sync. Surface the failure as a warning so operators can
|
||||
// act, but do not propagate it.
|
||||
// (PR #1156 follow-up review: writeBridge error in sync.ts propagates
|
||||
// uncaught.)
|
||||
try {
|
||||
await writeBridge(opts.groupDir, {
|
||||
contracts: allContracts,
|
||||
crossLinks,
|
||||
repoSnapshots,
|
||||
missingRepos,
|
||||
});
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
logger.warn(
|
||||
{ err: msg, groupDir: opts.groupDir },
|
||||
'⚠️ writeBridge failed; contracts.json is intact but bridge.lbug is stale. Re-run `gitnexus group sync` to retry.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom';
|
||||
export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include';
|
||||
export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding';
|
||||
export type ContractRole = 'provider' | 'consumer';
|
||||
|
||||
|
|
@ -28,6 +28,7 @@ export interface DetectConfig {
|
|||
topics: boolean;
|
||||
shared_libs: boolean;
|
||||
embedding_fallback: boolean;
|
||||
includes: boolean;
|
||||
workspace_deps: boolean;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -769,9 +769,10 @@ export const processCalls = async (
|
|||
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
continue;
|
||||
|
|
@ -3280,9 +3281,10 @@ export const extractFetchCallsFromFiles = async (
|
|||
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -369,9 +369,20 @@ const RE_USE_AFTER =
|
|||
/\bUSE\s+(?:AFTER\s+)?(?:STANDARD\s+)?(?:EXCEPTION|ERROR)\s+ON\s+([A-Z][A-Z0-9-]+|INPUT|OUTPUT|I-O|EXTEND)\b/i;
|
||||
|
||||
// SET statement (condition, index)
|
||||
const RE_SET_TO_TRUE = /\bSET\s+((?:[A-Z][A-Z0-9-]+(?:\s+OF\s+[A-Z][A-Z0-9-]+)?\s+)+)TO\s+TRUE\b/i;
|
||||
const RE_SET_INDEX =
|
||||
/\bSET\s+((?:[A-Z][A-Z0-9-]+\s+)+)(TO|UP\s+BY|DOWN\s+BY)\s+(\d+|[A-Z][A-Z0-9-]+)/i;
|
||||
//
|
||||
// Catastrophic-backtracking note (CodeQL js/redos): the previous shape
|
||||
// `((?:[A-Z][A-Z0-9-]+(?:\s+OF\s+[A-Z][A-Z0-9-]+)?\s+)+)TO\s+TRUE`
|
||||
// nested `\s+` quantifiers across alternations and was exponential on
|
||||
// inputs like "SET a OF a OF a ... TO TRUE". Replaced with a lazy
|
||||
// dot-match bounded by the explicit `\s+TO\s+TRUE` suffix — `.+?` is
|
||||
// O(n) with the trailing anchor, and the captured group is parsed
|
||||
// downstream the same way as before.
|
||||
// Exported so the U8 ReDoS regression test can pin the exact production
|
||||
// pattern. Direct import is the only way to ensure the test's
|
||||
// pathological-input timing assertion exercises the production regex
|
||||
// instead of an inline copy that drifts.
|
||||
export const RE_SET_TO_TRUE = /\bSET\s+(.+?)\s+TO\s+TRUE\b/i;
|
||||
export const RE_SET_INDEX = /\bSET\s+(.+?)\s+(TO|UP\s+BY|DOWN\s+BY)\s+(\d+|[A-Z][A-Z0-9-]+)/i;
|
||||
|
||||
// INITIALIZE statement — data reset (captures targets before REPLACING/WITH clause)
|
||||
const RE_INITIALIZE = /\bINITIALIZE\s+([\s\S]*?)(?=\bREPLACING\b|\bWITH\b|\.\s*$|$)/i;
|
||||
|
|
|
|||
265
gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts
Normal file
265
gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts
Normal file
|
|
@ -0,0 +1,265 @@
|
|||
/**
|
||||
* Unreal Engine reflection-macro preprocessor for C++ source.
|
||||
*
|
||||
* Tree-sitter does not expand C preprocessor macros, so Unreal's reflection
|
||||
* markers (`UCLASS(...)`, `UFUNCTION(...)`, `MODULENAME_API`, ...) are parsed
|
||||
* verbatim. The result is mis-parsed declarations: in `class BRAWLUI_API
|
||||
* UMyClass : public UObject`, tree-sitter-cpp captures `BRAWLUI_API` as the
|
||||
* class name and the rest of the declaration becomes structurally wrong.
|
||||
*
|
||||
* This module elides those macros from the source text BEFORE tree-sitter
|
||||
* parses it. Replacement is **length-preserving** (each elided byte becomes
|
||||
* a space, newlines preserved) so byte offsets and line/column positions
|
||||
* tree-sitter reports remain identical to the original file. Symbol
|
||||
* locations in the graph stay accurate.
|
||||
*
|
||||
* A cheap detection guard short-circuits files that don't look like UE
|
||||
* sources, so non-UE C++ codebases pay no cost.
|
||||
*
|
||||
* Pure function — no tree-sitter dependency, safe for worker threads.
|
||||
*/
|
||||
/**
|
||||
* Strong UE markers — reflection macros that only Unreal Engine projects use.
|
||||
* Presence of one of these is sufficient evidence that the file is a UE source
|
||||
* and that `MODULENAME_API` tokens in it are intended as export macros.
|
||||
*
|
||||
* Importantly, `_API` tokens are NOT in this guard — `REST_API`, `HTTP_API`,
|
||||
* `MY_LIB_API` and similar identifiers appear in plenty of non-UE C++ codebases
|
||||
* as constants/enums/parameter names. We must not erase them just because the
|
||||
* file mentions an `_API` token.
|
||||
*/
|
||||
const HAS_UE_HINT =
|
||||
/\b(?:UCLASS|UFUNCTION|UPROPERTY|USTRUCT|UENUM|UINTERFACE|GENERATED_BODY|GENERATED_[A-Z_]+_BODY|UE_DEPRECATED|DECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE)/;
|
||||
|
||||
const SIMPLE_MACROS_NO_ARGS: readonly string[] = [
|
||||
'GENERATED_BODY',
|
||||
'GENERATED_UCLASS_BODY',
|
||||
'GENERATED_USTRUCT_BODY',
|
||||
'GENERATED_UINTERFACE_BODY',
|
||||
'GENERATED_IINTERFACE_BODY',
|
||||
'DECLARE_CLASS',
|
||||
'GENERATED_BODY_LEGACY',
|
||||
];
|
||||
|
||||
const PARENTHESIZED_MACROS: readonly string[] = [
|
||||
'UCLASS',
|
||||
'UFUNCTION',
|
||||
'UPROPERTY',
|
||||
'USTRUCT',
|
||||
'UENUM',
|
||||
'UINTERFACE',
|
||||
'UMETA',
|
||||
'UE_DEPRECATED',
|
||||
];
|
||||
|
||||
const DELEGATE_MACRO_RE =
|
||||
/\bDECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE(?:_(?:RetVal_OneParam|RetVal_TwoParams|RetVal_ThreeParams|RetVal_FourParams|RetVal_FiveParams|RetVal_SixParams|RetVal_SevenParams|RetVal_EightParams|RetVal_NineParams|RetVal|OneParam|TwoParams|ThreeParams|FourParams|FiveParams|SixParams|SevenParams|EightParams|NineParams|TenParams))?(?=\s*\()/g;
|
||||
|
||||
/**
|
||||
* Module export tokens like `BRAWLUI_API`, `ENGINE_API`, `COREUOBJECT_API`.
|
||||
* Pattern: ALL_CAPS identifier ending in `_API`. The leading word boundary
|
||||
* (`\b`) prevents matching mid-identifier.
|
||||
*/
|
||||
const API_MACRO_RE = /\b[A-Z][A-Z0-9_]*_API\b/g;
|
||||
|
||||
/** Replace `[start, end)` of `chars` with spaces, preserving newlines. */
|
||||
function eraseRange(chars: string[], start: number, end: number): void {
|
||||
for (let i = start; i < end; i++) {
|
||||
if (chars[i] !== '\n' && chars[i] !== '\r') {
|
||||
chars[i] = ' ';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the matching close paren for an opening paren at index `openIdx`.
|
||||
* Returns the index of `)` (inclusive end), or -1 if unbalanced.
|
||||
*
|
||||
* Handles nested parens and string/char literals so commas/parens inside
|
||||
* strings don't throw off the match. Does not attempt to handle raw string
|
||||
* literals (`R"(...)"`); UE reflection-macro arguments do not use them in
|
||||
* practice.
|
||||
*/
|
||||
function findMatchingParen(source: string, openIdx: number): number {
|
||||
if (source.charCodeAt(openIdx) !== 0x28) return -1;
|
||||
let depth = 1;
|
||||
let i = openIdx + 1;
|
||||
const len = source.length;
|
||||
while (i < len && depth > 0) {
|
||||
const ch = source.charCodeAt(i);
|
||||
// String literal
|
||||
if (ch === 0x22) {
|
||||
i++;
|
||||
while (i < len) {
|
||||
const c = source.charCodeAt(i);
|
||||
if (c === 0x5c) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === 0x22) {
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Char literal
|
||||
if (ch === 0x27) {
|
||||
i++;
|
||||
while (i < len) {
|
||||
const c = source.charCodeAt(i);
|
||||
if (c === 0x5c) {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (c === 0x27) {
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
// Line comment
|
||||
if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2f) {
|
||||
while (i < len && source.charCodeAt(i) !== 0x0a) i++;
|
||||
continue;
|
||||
}
|
||||
// Block comment
|
||||
if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2a) {
|
||||
i += 2;
|
||||
while (i < len) {
|
||||
if (source.charCodeAt(i) === 0x2a && source.charCodeAt(i + 1) === 0x2f) {
|
||||
i += 2;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (ch === 0x28) depth++;
|
||||
else if (ch === 0x29) {
|
||||
depth--;
|
||||
if (depth === 0) return i;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/** Match a whole-word identifier at `idx`. Returns the byte after the identifier, or -1 on miss. */
|
||||
function matchIdentifierAt(source: string, idx: number, name: string): number {
|
||||
if (idx > 0) {
|
||||
const prev = source.charCodeAt(idx - 1);
|
||||
if (
|
||||
(prev >= 0x30 && prev <= 0x39) ||
|
||||
(prev >= 0x41 && prev <= 0x5a) ||
|
||||
(prev >= 0x61 && prev <= 0x7a) ||
|
||||
prev === 0x5f
|
||||
) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
for (let k = 0; k < name.length; k++) {
|
||||
if (source.charCodeAt(idx + k) !== name.charCodeAt(k)) return -1;
|
||||
}
|
||||
const after = idx + name.length;
|
||||
if (after < source.length) {
|
||||
const next = source.charCodeAt(after);
|
||||
if (
|
||||
(next >= 0x30 && next <= 0x39) ||
|
||||
(next >= 0x41 && next <= 0x5a) ||
|
||||
(next >= 0x61 && next <= 0x7a) ||
|
||||
next === 0x5f
|
||||
) {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
return after;
|
||||
}
|
||||
|
||||
/** Skip ASCII whitespace forward from `idx`. Returns the next non-whitespace byte index. */
|
||||
function skipWhitespace(source: string, idx: number): number {
|
||||
const len = source.length;
|
||||
while (idx < len) {
|
||||
const ch = source.charCodeAt(idx);
|
||||
if (ch === 0x20 || ch === 0x09 || ch === 0x0a || ch === 0x0d) {
|
||||
idx++;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return idx;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip Unreal Engine reflection macros from C++ source, length-preserving.
|
||||
*
|
||||
* Returns the original string unchanged if no strong UE marker is detected,
|
||||
* so non-UE C++ files (including ones that contain `*_API`-suffixed
|
||||
* identifiers like `REST_API` or `HTTP_API`) incur only a single regex test.
|
||||
*
|
||||
* The `_filePath` parameter is part of the `LanguageProvider.preprocessSource`
|
||||
* contract but is unused — UE detection is purely content-based. Accepted and
|
||||
* ignored here so the function matches the hook signature exactly.
|
||||
*/
|
||||
export function stripUeMacros(source: string, _filePath?: string): string {
|
||||
if (!HAS_UE_HINT.test(source)) return source;
|
||||
|
||||
const chars: string[] = source.split('');
|
||||
|
||||
for (const macro of PARENTHESIZED_MACROS) {
|
||||
let searchFrom = 0;
|
||||
while (true) {
|
||||
const hit = source.indexOf(macro, searchFrom);
|
||||
if (hit < 0) break;
|
||||
searchFrom = hit + 1;
|
||||
const after = matchIdentifierAt(source, hit, macro);
|
||||
if (after < 0) continue;
|
||||
const parenIdx = skipWhitespace(source, after);
|
||||
if (source.charCodeAt(parenIdx) !== 0x28) continue;
|
||||
const close = findMatchingParen(source, parenIdx);
|
||||
if (close < 0) continue;
|
||||
eraseRange(chars, hit, close + 1);
|
||||
}
|
||||
}
|
||||
|
||||
for (const macro of SIMPLE_MACROS_NO_ARGS) {
|
||||
let searchFrom = 0;
|
||||
while (true) {
|
||||
const hit = source.indexOf(macro, searchFrom);
|
||||
if (hit < 0) break;
|
||||
searchFrom = hit + 1;
|
||||
const after = matchIdentifierAt(source, hit, macro);
|
||||
if (after < 0) continue;
|
||||
const parenIdx = skipWhitespace(source, after);
|
||||
if (source.charCodeAt(parenIdx) === 0x28) {
|
||||
const close = findMatchingParen(source, parenIdx);
|
||||
if (close < 0) continue;
|
||||
eraseRange(chars, hit, close + 1);
|
||||
} else {
|
||||
eraseRange(chars, hit, after);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const re of [DELEGATE_MACRO_RE, API_MACRO_RE]) {
|
||||
re.lastIndex = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = re.exec(source)) !== null) {
|
||||
const start = match.index;
|
||||
let end = start + match[0].length;
|
||||
if (re === DELEGATE_MACRO_RE) {
|
||||
const parenIdx = skipWhitespace(source, end);
|
||||
if (source.charCodeAt(parenIdx) === 0x28) {
|
||||
const close = findMatchingParen(source, parenIdx);
|
||||
if (close >= 0) end = close + 1;
|
||||
}
|
||||
}
|
||||
eraseRange(chars, start, end);
|
||||
}
|
||||
}
|
||||
|
||||
return chars.join('');
|
||||
}
|
||||
|
|
@ -9,6 +9,11 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js';
|
|||
|
||||
const CSHARP_VIS = new Set<FieldVisibility>(['public', 'private', 'protected', 'internal']);
|
||||
|
||||
const extractCsharpDeclaredType = (typeNode: SyntaxNode): string | undefined => {
|
||||
if (typeNode.type === 'generic_name') return typeNode.text.trim();
|
||||
return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim();
|
||||
};
|
||||
|
||||
/**
|
||||
* C# field extraction config.
|
||||
*
|
||||
|
|
@ -53,17 +58,17 @@ export const csharpConfig: FieldExtractionConfig = {
|
|||
const child = node.namedChild(i);
|
||||
if (child?.type === 'variable_declaration') {
|
||||
const typeNode = child.childForFieldName('type');
|
||||
if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim();
|
||||
if (typeNode) return extractCsharpDeclaredType(typeNode);
|
||||
// fallback: first child that is a type
|
||||
const first = child.firstNamedChild;
|
||||
if (first && first.type !== 'variable_declarator') {
|
||||
return extractSimpleTypeName(first) ?? first.text?.trim();
|
||||
return extractCsharpDeclaredType(first);
|
||||
}
|
||||
}
|
||||
}
|
||||
// property_declaration: type is first named child
|
||||
const typeNode = node.childForFieldName('type');
|
||||
if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim();
|
||||
if (typeNode) return extractCsharpDeclaredType(typeNode);
|
||||
return undefined;
|
||||
},
|
||||
|
||||
|
|
|
|||
|
|
@ -219,9 +219,13 @@ export const processHeritage = async (
|
|||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
// Use larger bufferSize for files > 32KB
|
||||
// Per-language source preprocessor (length-preserving, e.g. UE macro
|
||||
// stripping for C++). MUST mirror parsing-processor on cache miss so
|
||||
// re-parses see the same input as the cached AST.
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
// Skip files that can't be parsed
|
||||
|
|
@ -413,9 +417,10 @@ export async function extractExtractedHeritageFromFiles(
|
|||
|
||||
let tree = astCache.get(file.path);
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -305,9 +305,10 @@ export const processImports = async (
|
|||
let wasReparsed = false;
|
||||
|
||||
if (!tree) {
|
||||
const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content;
|
||||
try {
|
||||
tree = parser.parse(file.content, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(file.content),
|
||||
tree = parser.parse(parseContent, undefined, {
|
||||
bufferSize: getTreeSitterBufferSize(parseContent),
|
||||
});
|
||||
} catch (parseError) {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -116,6 +116,39 @@ interface LanguageProviderConfig {
|
|||
* Required for tree-sitter languages; empty string for standalone processors. */
|
||||
readonly treeSitterQueries: string;
|
||||
|
||||
/**
|
||||
* Optional source-text transform that runs **before** tree-sitter parses the file.
|
||||
*
|
||||
* Used to elide language constructs that confuse the grammar without affecting
|
||||
* source-position fidelity — e.g., Unreal Engine reflection macros (`UCLASS`,
|
||||
* `UFUNCTION`, `MODULENAME_API`) in C++ headers that prevent the parser from
|
||||
* recognising class/function names correctly.
|
||||
*
|
||||
* **Length / position preservation:** the returned string MUST have the same
|
||||
* JavaScript `.length` as the input AND preserve every newline (`\n`/`\r`)
|
||||
* position byte-for-byte. Implementations replace elided characters with
|
||||
* ASCII spaces while leaving newlines untouched. With this contract:
|
||||
*
|
||||
* - tree-sitter's reported `startPosition.row`/`startPosition.column`
|
||||
* match the original file exactly (line/column come from newline counts)
|
||||
* - `startIndex`/`endIndex` byte offsets match the original file exactly
|
||||
* **when the elided range is pure ASCII** (UTF-16 `.length` equals UTF-8
|
||||
* byte length only for ASCII).
|
||||
*
|
||||
* Implementations targeting languages where elided ranges may contain
|
||||
* non-ASCII content must therefore preserve byte length, not just `.length`,
|
||||
* if downstream code uses `startIndex` to slice the original UTF-8 bytes.
|
||||
* The current C++ UE-macro preprocessor relies on the practical fact that
|
||||
* UE reflection macros and module-export tokens are ASCII-only.
|
||||
*
|
||||
* Must be a pure function — same input always yields the same output. Called
|
||||
* once per file, on every code path that re-parses (parsing-processor, import
|
||||
* processor, heritage processor, call processor, parse worker).
|
||||
*
|
||||
* Default: undefined (no preprocessing — `file.content` is parsed verbatim).
|
||||
*/
|
||||
readonly preprocessSource?: (sourceText: string, filePath: string) => string;
|
||||
|
||||
// ── Core (required) ───────────────────────────────────────────────
|
||||
/** Type extraction: declarations, initializers, for-loop bindings */
|
||||
readonly typeConfig: LanguageTypeConfig;
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ import { cVariableConfig, cppVariableConfig } from '../variable-extractors/confi
|
|||
import { createCallExtractor } from '../call-extractors/generic.js';
|
||||
import { cCallConfig, cppCallConfig } from '../call-extractors/configs/c-cpp.js';
|
||||
import { createHeritageExtractor } from '../heritage-extractors/generic.js';
|
||||
import { stripUeMacros } from '../cpp-ue-preprocessor.js';
|
||||
|
||||
const C_BUILT_INS: ReadonlySet<string> = new Set([
|
||||
'printf',
|
||||
|
|
@ -410,6 +411,7 @@ export const cppProvider = defineLanguage({
|
|||
},
|
||||
] satisfies AstFrameworkPatternConfig[],
|
||||
treeSitterQueries: CPP_QUERIES,
|
||||
preprocessSource: stripUeMacros,
|
||||
typeConfig: cCppConfig,
|
||||
exportChecker: cCppExportChecker,
|
||||
importResolver: createImportResolver(cppImportConfig),
|
||||
|
|
|
|||
|
|
@ -42,6 +42,39 @@ const FUNCTION_NODE_TYPES = [
|
|||
'local_function_statement',
|
||||
] as const;
|
||||
|
||||
const BUILTIN_TYPE_NAMES = new Set([
|
||||
'bool',
|
||||
'byte',
|
||||
'char',
|
||||
'decimal',
|
||||
'double',
|
||||
'float',
|
||||
'int',
|
||||
'long',
|
||||
'object',
|
||||
'sbyte',
|
||||
'short',
|
||||
'string',
|
||||
'uint',
|
||||
'ulong',
|
||||
'ushort',
|
||||
'void',
|
||||
]);
|
||||
|
||||
function shouldEmitReadMember(memberNode: SyntaxNode): boolean {
|
||||
const parent = memberNode.parent;
|
||||
if (parent === null) return true;
|
||||
|
||||
switch (parent.type) {
|
||||
case 'invocation_expression':
|
||||
return parent.childForFieldName('function')?.id !== memberNode.id;
|
||||
case 'assignment_expression':
|
||||
return parent.childForFieldName('left')?.id !== memberNode.id;
|
||||
default:
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
export function emitCsharpScopeCaptures(
|
||||
sourceText: string,
|
||||
_filePath: string,
|
||||
|
|
@ -94,6 +127,14 @@ export function emitCsharpScopeCaptures(
|
|||
continue;
|
||||
}
|
||||
|
||||
if (grouped['@reference.read.member'] !== undefined) {
|
||||
const anchor = grouped['@reference.read.member'];
|
||||
const memberNode = findNodeAtRange(tree.rootNode, anchor.range, 'member_access_expression');
|
||||
if (memberNode === null || !shouldEmitReadMember(memberNode)) {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// Synthesize `this` / `base` receiver type-bindings on every
|
||||
// instance method-like. Tree-sitter can't cleanly express "the
|
||||
// implicit receiver of a non-static member of a class/struct/
|
||||
|
|
@ -209,9 +250,63 @@ export function emitCsharpScopeCaptures(
|
|||
}
|
||||
}
|
||||
|
||||
out.push(...synthesizeGenericTypeArgumentReferences(tree.rootNode));
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
function synthesizeGenericTypeArgumentReferences(root: SyntaxNode): CaptureMatch[] {
|
||||
const out: CaptureMatch[] = [];
|
||||
// Treat all generic type arguments as static type references, including
|
||||
// declaration signatures and call-site generic instantiations.
|
||||
visit(root, (node) => {
|
||||
if (node.type !== 'generic_name') return;
|
||||
const args = findNamedChild(node, 'type_argument_list');
|
||||
if (args === null) return;
|
||||
|
||||
for (const arg of args.namedChildren) {
|
||||
if (arg === null) continue;
|
||||
const nameNode = terminalTypeNameNode(arg);
|
||||
if (nameNode === null) continue;
|
||||
if (BUILTIN_TYPE_NAMES.has(nameNode.text)) continue;
|
||||
out.push({
|
||||
'@reference.type': nodeToCapture('@reference.type', nameNode),
|
||||
'@reference.name': nodeToCapture('@reference.name', nameNode),
|
||||
});
|
||||
}
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function terminalTypeNameNode(node: SyntaxNode): SyntaxNode | null {
|
||||
switch (node.type) {
|
||||
case 'identifier':
|
||||
return node;
|
||||
case 'nullable_type':
|
||||
return node.firstNamedChild === null ? null : terminalTypeNameNode(node.firstNamedChild);
|
||||
case 'qualified_name':
|
||||
return node.lastNamedChild;
|
||||
case 'generic_name':
|
||||
return node.childForFieldName('name') ?? node.firstNamedChild;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function findNamedChild(node: SyntaxNode, type: string): SyntaxNode | null {
|
||||
for (const child of node.namedChildren) {
|
||||
if (child !== null && child.type === type) return child;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function visit(node: SyntaxNode, cb: (node: SyntaxNode) => void): void {
|
||||
cb(node);
|
||||
for (const child of node.namedChildren) {
|
||||
if (child !== null) visit(child, cb);
|
||||
}
|
||||
}
|
||||
|
||||
/** C# 12 primary constructor: `class X(a, b) { }` / `record X(a, b)`.
|
||||
* The parameters are a bare `parameter_list` named child of the type
|
||||
* declaration (no `constructor_declaration` node). Emit a synthetic
|
||||
|
|
|
|||
|
|
@ -499,6 +499,12 @@ const CSHARP_SCOPE_QUERY = `
|
|||
left: (member_access_expression
|
||||
expression: "base" @reference.receiver
|
||||
name: (identifier) @reference.name)) @reference.write.member
|
||||
|
||||
;; References — field/property reads: \`obj.Name\`
|
||||
;; Emit-side filtering drops call targets and assignment left-hand sides.
|
||||
(member_access_expression
|
||||
expression: (_) @reference.receiver
|
||||
name: (identifier) @reference.name) @reference.read.member
|
||||
`;
|
||||
|
||||
let _parser: Parser | null = null;
|
||||
|
|
|
|||
|
|
@ -371,6 +371,11 @@ const processParsingSequential = async (
|
|||
isVueSetup = extracted.isSetup;
|
||||
}
|
||||
|
||||
// Per-language source-text transform (e.g., UE macro stripping for C++).
|
||||
// Length-preserving — see LanguageProvider.preprocessSource contract.
|
||||
parseContent =
|
||||
getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent;
|
||||
|
||||
try {
|
||||
await loadLanguage(language, file.path);
|
||||
} catch {
|
||||
|
|
|
|||
|
|
@ -23,7 +23,24 @@ interface ScriptBlock {
|
|||
lang: string;
|
||||
}
|
||||
|
||||
const SCRIPT_RE = /<script(\s[^>]*)?>([^]*?)<\/script>/g;
|
||||
// Closing-tag pattern accepts:
|
||||
// - whitespace before `>` — `</script >`, `</script\t\n>`
|
||||
// - attribute-like junk after `script` — `</script foo="bar">`,
|
||||
// `</script\t\n bar>`
|
||||
// - any case — `</SCRIPT>`, `</Script>`
|
||||
//
|
||||
// HTML5 parses `</script foo>` as a valid close tag (attributes on
|
||||
// close tags are ignored by the parser but still terminate the script
|
||||
// block). A strict `<\/script\s*>` would miss those forms and let a
|
||||
// crafted Vue file hide content from this extractor — exactly the
|
||||
// CodeQL `js/bad-tag-filter` failure mode (the published test cases
|
||||
// it checks include `</script foo="bar">` and `</script\t\n bar>`).
|
||||
//
|
||||
// `[^>]*` after `</script` accepts everything up to the next `>`,
|
||||
// matching the HTML parser's actual close-tag behaviour. The `i` flag
|
||||
// covers the case axis. PR #1330 CI surfaced both the case and
|
||||
// attribute axes; this expression closes both at once.
|
||||
const SCRIPT_RE = /<script(\s[^>]*)?>([^]*?)<\/script[^>]*>/gi;
|
||||
const TEMPLATE_COMPONENT_RE = /<([A-Z][A-Za-z0-9]+)/g;
|
||||
// Greedy: matches from the first <template> to the *last* </template>.
|
||||
// This is intentional — nested <template v-slot:...> tags are valid Vue
|
||||
|
|
|
|||
|
|
@ -1407,6 +1407,11 @@ const processFileGroup = (
|
|||
isVueSetup = extracted.isSetup;
|
||||
}
|
||||
|
||||
// Per-language source-text transform (e.g., UE macro stripping for C++).
|
||||
// Length-preserving — see LanguageProvider.preprocessSource contract.
|
||||
parseContent =
|
||||
getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent;
|
||||
|
||||
clearCaches(); // Reset memoization before each new file
|
||||
|
||||
let tree;
|
||||
|
|
|
|||
|
|
@ -301,11 +301,15 @@ export const streamAllCSVsToDisk = async (
|
|||
'Template',
|
||||
'Module',
|
||||
] as const;
|
||||
const propertyHeader = 'id,name,filePath,startLine,endLine,content,description,declaredType';
|
||||
const multiLangWriters = new Map<string, BufferedCSVWriter>();
|
||||
for (const t of MULTI_LANG_TYPES) {
|
||||
multiLangWriters.set(
|
||||
t,
|
||||
new BufferedCSVWriter(path.join(csvDir, `${t.toLowerCase()}.csv`), multiLangHeader),
|
||||
new BufferedCSVWriter(
|
||||
path.join(csvDir, `${t.toLowerCase()}.csv`),
|
||||
t === 'Property' ? propertyHeader : multiLangHeader,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -478,6 +482,9 @@ export const streamAllCSVsToDisk = async (
|
|||
escapeCSVNumber(node.properties.endLine, -1),
|
||||
escapeCSVField(content),
|
||||
escapeCSVField(node.properties.description || ''),
|
||||
...(node.label === 'Property'
|
||||
? [escapeCSVField(node.properties.declaredType || '')]
|
||||
: []),
|
||||
].join(','),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ import type { CachedEmbedding } from '../embeddings/types.js';
|
|||
import { extensionManager, type ExtensionEnsureOptions } from './extension-loader.js';
|
||||
import {
|
||||
closeLbugConnection,
|
||||
isDbBusyError,
|
||||
isOpenRetryExhausted,
|
||||
openLbugConnection,
|
||||
waitForWindowsHandleRelease,
|
||||
type LbugConnectionHandle,
|
||||
} from './lbug-config.js';
|
||||
import { isVectorExtensionSupportedByPlatform } from '../platform/capabilities.js';
|
||||
|
|
@ -185,21 +188,6 @@ const DB_LOCK_RETRY_ATTEMPTS = 3;
|
|||
/** Base back-off in ms between BUSY retries (multiplied by attempt number). */
|
||||
const DB_LOCK_RETRY_DELAY_MS = 500;
|
||||
|
||||
/**
|
||||
* Return true when the error message indicates that another process holds
|
||||
* an exclusive lock on the LadybugDB file (e.g. `gitnexus analyze` or
|
||||
* `gitnexus serve` running at the same time).
|
||||
*/
|
||||
export const isDbBusyError = (err: unknown): boolean => {
|
||||
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
||||
return (
|
||||
msg.includes('busy') ||
|
||||
msg.includes('lock') ||
|
||||
msg.includes('already in use') ||
|
||||
msg.includes('could not set lock')
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
* Return true when the error message indicates a write was attempted against
|
||||
* a read-only LadybugDB connection. The MCP query pool opens DBs read-only,
|
||||
|
|
@ -252,7 +240,11 @@ export const withLbugDb = async <T>(dbPath: string, operation: () => Promise<T>)
|
|||
});
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
if (!isDbBusyError(err) || attempt === DB_LOCK_RETRY_ATTEMPTS) {
|
||||
// Skip outer retry when the inner open-retry already exhausted: the
|
||||
// ~1.5s open-time budget was just spent, repeating the full reset+
|
||||
// reopen cycle would only add 4-5s of tail latency without changing
|
||||
// the outcome (both layers consult the same isDbBusyError matcher).
|
||||
if (!isDbBusyError(err) || isOpenRetryExhausted(err) || attempt === DB_LOCK_RETRY_ATTEMPTS) {
|
||||
throw err;
|
||||
}
|
||||
// Close stale connection inside the session lock to prevent race conditions
|
||||
|
|
@ -330,7 +322,16 @@ const doInitLbug = async (dbPath: string) => {
|
|||
await conn.query(schemaQuery);
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
if (!msg.includes('already exists')) {
|
||||
// Suppression list:
|
||||
// - "already exists": expected idempotent re-create on existing DBs
|
||||
// - "could not set lock on file": LadybugDB v0.16.1 emits this on
|
||||
// Windows when CREATE NODE TABLE runs against a path that was
|
||||
// just opened (the WAL handle from a fresh Database briefly
|
||||
// contests the table's first-write lock). The table is created
|
||||
// anyway and any genuine cross-process lock contention surfaces
|
||||
// on the next operation via withLbugDb's retry. Logging it here
|
||||
// would just be noise in CI.
|
||||
if (!msg.includes('already exists') && !isDbBusyError(err)) {
|
||||
logger.warn(`⚠️ Schema creation warning: ${msg.slice(0, 120)}`);
|
||||
}
|
||||
}
|
||||
|
|
@ -607,6 +608,9 @@ const getCopyQuery = (table: NodeTableName, filePath: string): string => {
|
|||
if (table === 'Method') {
|
||||
return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description, parameterCount, returnType) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
if (table === 'Property') {
|
||||
return `COPY ${t}(id, name, filePath, startLine, endLine, content, description, declaredType) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
}
|
||||
// TypeScript/JS code element tables have isExported; multi-language tables do not
|
||||
if (TABLES_WITH_EXPORTED.has(table)) {
|
||||
return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
||||
|
|
@ -658,6 +662,11 @@ export const insertNodeToLbug = async (
|
|||
? `, description: ${escapeValue(properties.description)}`
|
||||
: '';
|
||||
query = `CREATE (n:${t} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, isExported: ${!!properties.isExported}, content: ${escapeValue(properties.content || '')}${descPart}})`;
|
||||
} else if (label === 'Property') {
|
||||
const descPart = properties.description
|
||||
? `, description: ${escapeValue(properties.description)}`
|
||||
: '';
|
||||
query = `CREATE (n:${t} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, content: ${escapeValue(properties.content || '')}${descPart}, declaredType: ${escapeValue(properties.declaredType || '')}})`;
|
||||
} else {
|
||||
// Multi-language tables (Struct, Impl, Trait, Macro, etc.) — no isExported
|
||||
const descPart = properties.description
|
||||
|
|
@ -736,6 +745,11 @@ export const batchInsertNodesToLbug = async (
|
|||
? `, n.description = ${escapeValue(properties.description)}`
|
||||
: '';
|
||||
query = `MERGE (n:${t} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.isExported = ${!!properties.isExported}, n.content = ${escapeValue(properties.content || '')}${descPart}`;
|
||||
} else if (label === 'Property') {
|
||||
const descPart = properties.description
|
||||
? `, n.description = ${escapeValue(properties.description)}`
|
||||
: '';
|
||||
query = `MERGE (n:${t} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.content = ${escapeValue(properties.content || '')}${descPart}, n.declaredType = ${escapeValue(properties.declaredType || '')}`;
|
||||
} else {
|
||||
const descPart = properties.description
|
||||
? `, n.description = ${escapeValue(properties.description)}`
|
||||
|
|
@ -1064,6 +1078,9 @@ export const flushWAL = async (): Promise<void> => {
|
|||
*/
|
||||
export const safeClose = async (): Promise<void> => {
|
||||
await flushWAL();
|
||||
// Capture before close — currentDbPath stays set so the Windows post-close
|
||||
// probe below knows which file to wait on.
|
||||
const closingDbPath = currentDbPath;
|
||||
if (conn) {
|
||||
try {
|
||||
// eslint-disable-next-line no-restricted-syntax -- sole authorised close site
|
||||
|
|
@ -1082,6 +1099,24 @@ export const safeClose = async (): Promise<void> => {
|
|||
}
|
||||
db = null;
|
||||
}
|
||||
// Windows: libuv reports `db.close()` resolved before the kernel has
|
||||
// released the file handle. A subsequent `new Database(samePath)` in
|
||||
// the same process can race the release. The probe (lbug-config.ts)
|
||||
// forces any residual lock to surface as EBUSY/EPERM/EACCES so the
|
||||
// open-time retry absorbs the lag.
|
||||
if (process.platform === 'win32' && closingDbPath) {
|
||||
const released = await waitForWindowsHandleRelease(closingDbPath);
|
||||
if (!released) {
|
||||
// Probe exhausted with a lock code still in flight. The next
|
||||
// openLbugConnection will absorb whatever residual lag remains, but
|
||||
// a chronic warning helps operators spot AV interference (Windows
|
||||
// Defender holding the file far past the 250ms budget).
|
||||
logger.warn(
|
||||
{ dbPath: closingDbPath },
|
||||
'⚠️ LadybugDB file handle still locked after close (Windows). If this repeats, check antivirus/Defender exclusions for the GitNexus storage directory.',
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const closeLbug = async (): Promise<void> => {
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
import fs from 'fs/promises';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import type lbug from '@ladybugdb/core';
|
||||
|
||||
/**
|
||||
|
|
@ -42,10 +45,23 @@ export const LBUG_MAX_DB_SIZE: number = (() => {
|
|||
return 16 * 1024 * 1024 * 1024;
|
||||
})();
|
||||
|
||||
/** Matches WAL corruption errors from the LadybugDB engine. */
|
||||
const WAL_CORRUPTION_RE = /corrupt(ed)?\s+wal|invalid\s+wal\s+record|wal.*corrupt|checksum.*wal/i;
|
||||
|
||||
export const WAL_RECOVERY_SUGGESTION =
|
||||
'WAL corruption detected. Run `gitnexus analyze` to rebuild the index.';
|
||||
|
||||
export function isWalCorruptionError(err: unknown): boolean {
|
||||
if (!err) return false;
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
return WAL_CORRUPTION_RE.test(msg);
|
||||
}
|
||||
|
||||
type LbugModule = typeof lbug;
|
||||
|
||||
export interface LbugDatabaseOptions {
|
||||
readOnly?: boolean;
|
||||
throwOnWalReplayFailure?: boolean;
|
||||
}
|
||||
|
||||
export interface LbugConnectionHandle {
|
||||
|
|
@ -53,20 +69,200 @@ export interface LbugConnectionHandle {
|
|||
conn: lbug.Connection;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true when the error message indicates that a LadybugDB file lock
|
||||
* could not be acquired — either at construction time
|
||||
* (`new lbug.Database(...)` raises from `local_file_system.cpp`) or during
|
||||
* a query (another writer holds the exclusive lock).
|
||||
*
|
||||
* Lives here (not in `lbug-adapter.ts`) so both the construction-time
|
||||
* retry (`openWithLockRetry` in this file) and the query-time retry
|
||||
* (`withLbugDb` in `lbug-adapter.ts`) consult the same matcher. Callers
|
||||
* import directly from this module — no re-export to keep in sync.
|
||||
*/
|
||||
export const isDbBusyError = (err: unknown): boolean => {
|
||||
const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
|
||||
// `lock` already subsumes `could not set lock`; the broader term is kept
|
||||
// because graph-DB transient errors include "deadlock", "lock contention",
|
||||
// and the LadybugDB native module's "could not set lock on file" — all of
|
||||
// which deserve a retry. If a non-transient lock-shaped error ever
|
||||
// surfaces (e.g., "lock file missing" during recovery), tighten this
|
||||
// matcher rather than raising the retry budget.
|
||||
return msg.includes('busy') || msg.includes('lock') || msg.includes('already in use');
|
||||
};
|
||||
|
||||
export function createLbugDatabase(
|
||||
lbugModule: LbugModule,
|
||||
databasePath: string,
|
||||
options: LbugDatabaseOptions = {},
|
||||
): lbug.Database {
|
||||
return new lbugModule.Database(
|
||||
// .d.ts declares fewer args than the native constructor accepts.
|
||||
return new (lbugModule.Database as any)(
|
||||
databasePath,
|
||||
0,
|
||||
false,
|
||||
0, // bufferManagerSize
|
||||
false, // enableCompression (pinned for v0.16.0)
|
||||
options.readOnly ?? false,
|
||||
LBUG_MAX_DB_SIZE,
|
||||
);
|
||||
true, // autoCheckpoint
|
||||
-1, // checkpointThreshold
|
||||
options.throwOnWalReplayFailure ?? true,
|
||||
true, // enableChecksums
|
||||
) as lbug.Database;
|
||||
}
|
||||
|
||||
// ─── Lock-busy retry tuning knobs ───────────────────────────────────────────
|
||||
//
|
||||
// All four GitNexus retry pairs that touch native LadybugDB locks live with
|
||||
// a comment cross-reference here so an SRE tuning Windows flakes finds them
|
||||
// in one grep:
|
||||
//
|
||||
// 1. OPEN_LOCK_RETRY_ATTEMPTS / OPEN_LOCK_RETRY_DELAY_MS (this file)
|
||||
// → `new lbug.Database()` constructor lock failures
|
||||
// 2. HANDLE_RELEASE_PROBE_ATTEMPTS / HANDLE_RELEASE_PROBE_DELAY_MS (this file)
|
||||
// → post-close fs.open probe to absorb Windows handle-release lag
|
||||
// 3. DB_LOCK_RETRY_ATTEMPTS / DB_LOCK_RETRY_DELAY_MS (lbug-adapter.ts withLbugDb)
|
||||
// → query-time busy/lock retry around already-open connections
|
||||
//
|
||||
// `new lbug.Database()` calls into the native module which performs an
|
||||
// OS-level exclusive lock on `<dbPath>`. On Windows that lock can fail
|
||||
// for reasons specific to the OS (Defender briefly opens new files,
|
||||
// libuv handle release lags the JS-side close). 5 attempts × 100ms
|
||||
// linear back-off (max sleep 100+200+300+400 = 1s, plus 5 ctor RTTs
|
||||
// of 10–50ms each = ~1.0–1.2s worst case) clears the typical
|
||||
// AV-scanner hold without masking real cross-process conflicts.
|
||||
//
|
||||
// Source: https://github.com/LadybugDB/ladybug/blob/v0.16.1/src/common/file_system/local_file_system.cpp#L126
|
||||
const OPEN_LOCK_RETRY_ATTEMPTS = 5;
|
||||
const OPEN_LOCK_RETRY_DELAY_MS = 100;
|
||||
|
||||
const HANDLE_RELEASE_PROBE_ATTEMPTS = 5;
|
||||
const HANDLE_RELEASE_PROBE_DELAY_MS = 50;
|
||||
const HANDLE_RELEASE_LOCK_CODES = new Set(['EBUSY', 'EPERM', 'EACCES']);
|
||||
|
||||
/**
|
||||
* Test-fixture directory prefixes recognized by `isTestFixturePath`.
|
||||
*
|
||||
* IMPORTANT: this list must stay in sync with the prefixes passed to
|
||||
* `createTempDir` in `gitnexus/test/helpers/test-db.ts` and the prefixes
|
||||
* used by `withTestLbugDB` (`gitnexus/test/helpers/test-indexed-db.ts`).
|
||||
* If you add a new test that passes a custom prefix to `createTempDir`,
|
||||
* add it here too — otherwise the stale-sidecar sweep silently won't
|
||||
* fire for that fixture and CI flakes return.
|
||||
*
|
||||
* The default `createTempDir('gitnexus-test-')` and the lbug variant
|
||||
* `'gitnexus-lbug-'` cover today's call sites.
|
||||
*/
|
||||
const TEST_FIXTURE_PREFIXES = ['gitnexus-lbug-', 'gitnexus-test-'];
|
||||
|
||||
/**
|
||||
* Marker symbol attached to lock errors after `openWithLockRetry` exhausts
|
||||
* its budget. `withLbugDb`'s outer query-time retry consults this so it
|
||||
* does not re-retry a path that just spent up to ~1.5s in the open-time
|
||||
* loop — preventing 6s tail latencies (3× outer × 5× inner attempts).
|
||||
*
|
||||
* The symbol is internal to GitNexus; consumers should treat the underlying
|
||||
* error message as the user-visible signal.
|
||||
*/
|
||||
export const LBUG_OPEN_RETRY_EXHAUSTED = Symbol.for('gitnexus.lbug.openRetryExhausted');
|
||||
|
||||
export const isOpenRetryExhausted = (err: unknown): boolean => {
|
||||
if (err === null || err === undefined || typeof err !== 'object') return false;
|
||||
return (err as { [LBUG_OPEN_RETRY_EXHAUSTED]?: boolean })[LBUG_OPEN_RETRY_EXHAUSTED] === true;
|
||||
};
|
||||
|
||||
const tagOpenRetryExhausted = (err: unknown): unknown => {
|
||||
if (err && typeof err === 'object') {
|
||||
(err as { [LBUG_OPEN_RETRY_EXHAUSTED]?: boolean })[LBUG_OPEN_RETRY_EXHAUSTED] = true;
|
||||
}
|
||||
return err;
|
||||
};
|
||||
|
||||
/**
|
||||
* True when `dbPath` resolves to a recognized test fixture under the OS
|
||||
* temp directory. Used to gate the stale-sidecar sweep so production
|
||||
* paths never have their `.wal` / `.lock` files deleted.
|
||||
*
|
||||
* Defensive shape:
|
||||
* - `path.resolve` normalizes `..` segments before the prefix check, so
|
||||
* `<tmp>/gitnexus-lbug-x/../../etc/passwd` is rejected.
|
||||
* - The tmpRoot check trims any trailing separator returned by some
|
||||
* Windows TMP configurations (`C:\Users\X\Temp\`) so the startsWith
|
||||
* comparison stays correct.
|
||||
* - Only the IMMEDIATE parent directory is matched against the prefix
|
||||
* list. An ancestor walk would let a tmpdir whose own basename starts
|
||||
* with `gitnexus-lbug-` accept arbitrary nested paths under it.
|
||||
*/
|
||||
const isTestFixturePath = (dbPath: string): boolean => {
|
||||
const tmpRoot = os.tmpdir().replace(new RegExp(`${path.sep === '\\' ? '\\\\' : path.sep}+$`), '');
|
||||
const resolved = path.resolve(dbPath);
|
||||
if (!resolved.startsWith(tmpRoot + path.sep) && resolved !== tmpRoot) return false;
|
||||
const parentBase = path.basename(path.dirname(resolved));
|
||||
return TEST_FIXTURE_PREFIXES.some((p) => parentBase.startsWith(p));
|
||||
};
|
||||
|
||||
/** Exported only for direct unit testing — production callers use `openWithLockRetry`. */
|
||||
export const _isTestFixturePathForTest = isTestFixturePath;
|
||||
|
||||
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Attempt to remove stale `.wal` / `.lock` sidecars that a previous aborted
|
||||
* test run may have left behind. Best-effort: ENOENT is normal, anything
|
||||
* else is swallowed so the caller's retry can surface the original error.
|
||||
*/
|
||||
const sweepStaleSidecars = async (dbPath: string): Promise<void> => {
|
||||
for (const suffix of ['.wal', '.lock']) {
|
||||
try {
|
||||
await fs.unlink(dbPath + suffix);
|
||||
} catch {
|
||||
/* missing sidecar or permission error — let the open retry surface it */
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Run `construct` with bounded retries when `new lbug.Database(...)` throws
|
||||
* a busy/lock error. The original (loop-captured) error is preferred over
|
||||
* any post-sweep error so triage sees the real LadybugDB lock message.
|
||||
* On exhaustion the rethrown error is tagged via
|
||||
* `LBUG_OPEN_RETRY_EXHAUSTED` so the outer query-time retry in
|
||||
* `withLbugDb` skips re-retrying a freshly-exhausted path.
|
||||
*/
|
||||
const openWithLockRetry = async (
|
||||
construct: () => lbug.Database,
|
||||
dbPath: string,
|
||||
): Promise<lbug.Database> => {
|
||||
let originalLockError: unknown;
|
||||
for (let attempt = 1; attempt <= OPEN_LOCK_RETRY_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
return construct();
|
||||
} catch (err) {
|
||||
if (!isDbBusyError(err)) throw err;
|
||||
originalLockError = err;
|
||||
if (attempt === OPEN_LOCK_RETRY_ATTEMPTS) break;
|
||||
await sleep(OPEN_LOCK_RETRY_DELAY_MS * attempt);
|
||||
}
|
||||
}
|
||||
|
||||
// Final defense: only for recognized test fixtures, sweep stale sidecars
|
||||
// (a prior aborted test run can leave a `.wal` lock that survives the
|
||||
// tmp dir cleanup). Production paths never reach this branch — the guard
|
||||
// requires the immediate parent dir to match a test prefix AND the
|
||||
// resolved path to live under the OS temp directory.
|
||||
if (isTestFixturePath(dbPath)) {
|
||||
await sweepStaleSidecars(dbPath);
|
||||
try {
|
||||
return construct();
|
||||
} catch {
|
||||
// Intentionally do NOT overwrite originalLockError. The user-actionable
|
||||
// signal is "we exhausted lock retries" — a different error from the
|
||||
// post-sweep attempt is less useful than the lock failure that drove
|
||||
// the sweep in the first place.
|
||||
}
|
||||
}
|
||||
throw tagOpenRetryExhausted(originalLockError);
|
||||
};
|
||||
|
||||
export async function openLbugConnection(
|
||||
lbugModule: LbugModule,
|
||||
databasePath: string,
|
||||
|
|
@ -74,7 +270,10 @@ export async function openLbugConnection(
|
|||
): Promise<LbugConnectionHandle> {
|
||||
let db: lbug.Database | undefined;
|
||||
try {
|
||||
db = createLbugDatabase(lbugModule, databasePath, options);
|
||||
db = await openWithLockRetry(
|
||||
() => createLbugDatabase(lbugModule, databasePath, options),
|
||||
databasePath,
|
||||
);
|
||||
return { db, conn: new lbugModule.Connection(db) };
|
||||
} catch (err) {
|
||||
if (db) await db.close().catch(() => {});
|
||||
|
|
@ -86,3 +285,60 @@ export async function closeLbugConnection(handle: LbugConnectionHandle): Promise
|
|||
await handle.conn.close().catch(() => {});
|
||||
await handle.db.close().catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe `dbPath` AND its `.wal` sidecar after `db.close()` so any
|
||||
* residual native file handle surfaces as EBUSY/EPERM/EACCES and the
|
||||
* bounded retry absorbs the release lag. Windows-only — Linux/macOS do
|
||||
* not exhibit this race.
|
||||
*
|
||||
* Both files matter. Empirically, on rapid open→close→reopen cycles the
|
||||
* main `dbPath` handle releases first; the `.wal` handle from the
|
||||
* previous Database lingers and the new Database's first write (CREATE
|
||||
* NODE TABLE during schema init) fails with "Could not set lock on
|
||||
* file". Probing both makes safeClose actually return when the kernel
|
||||
* is fully done with the path.
|
||||
*
|
||||
* Returns `true` when both probes succeeded (or skipped on non-lock
|
||||
* errors / missing files). Returns `false` when either probe exhausted
|
||||
* its budget with a lock code still in flight.
|
||||
*
|
||||
* Defensive shape:
|
||||
* - Opens read+write (`'r+'`) so the probe actually surfaces exclusive
|
||||
* locks held by the previous Database. A read-only probe (`'r'`) is
|
||||
* insufficient — Windows will grant read access while the previous
|
||||
* handle's exclusive write lock is still in flight, which lets
|
||||
* `safeClose` return before the next CREATE NODE TABLE can lock the
|
||||
* file.
|
||||
* - `try/finally` around `handle.close()` guarantees no fd leak even
|
||||
* if close itself throws.
|
||||
*/
|
||||
export const waitForWindowsHandleRelease = async (dbPath: string): Promise<boolean> => {
|
||||
const mainReleased = await probeSinglePath(dbPath);
|
||||
const walReleased = await probeSinglePath(dbPath + '.wal');
|
||||
return mainReleased && walReleased;
|
||||
};
|
||||
|
||||
const probeSinglePath = async (filePath: string): Promise<boolean> => {
|
||||
for (let attempt = 1; attempt <= HANDLE_RELEASE_PROBE_ATTEMPTS; attempt++) {
|
||||
let handle: fs.FileHandle | undefined;
|
||||
try {
|
||||
handle = await fs.open(filePath, 'r+');
|
||||
return true;
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException | undefined)?.code;
|
||||
if (!code || !HANDLE_RELEASE_LOCK_CODES.has(code)) return true; // ENOENT / unrelated → not our problem
|
||||
if (attempt === HANDLE_RELEASE_PROBE_ATTEMPTS) return false;
|
||||
await sleep(HANDLE_RELEASE_PROBE_DELAY_MS * attempt);
|
||||
} finally {
|
||||
if (handle) {
|
||||
try {
|
||||
await handle.close();
|
||||
} catch {
|
||||
/* swallow — caller cannot do anything useful with a probe-close failure */
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
import fs from 'fs/promises';
|
||||
import lbug from '@ladybugdb/core';
|
||||
import { loadFTSExtension } from './lbug-adapter.js';
|
||||
import { createLbugDatabase } from './lbug-config.js';
|
||||
import { createLbugDatabase, isWalCorruptionError } from './lbug-config.js';
|
||||
|
||||
/** Per-repo pool: one Database, many Connections */
|
||||
interface PoolEntry {
|
||||
|
|
@ -97,7 +97,7 @@ let idleTimer: ReturnType<typeof setInterval> | null = null;
|
|||
// @ladybugdb/core), corrupting stdout in the pre-sentinel window. Routing
|
||||
// through the leaf breaks that chain.
|
||||
export { realStdoutWrite, realStderrWrite, setActiveStdoutWrite } from '../../mcp/stdio-capture.js';
|
||||
import { getActiveStdoutWrite } from '../../mcp/stdio-capture.js';
|
||||
import { getActiveStdoutWrite, realStderrWrite } from '../../mcp/stdio-capture.js';
|
||||
|
||||
let stdoutSilenceCount = 0;
|
||||
/** True while pre-warming connections — prevents watchdog from prematurely restoring stdout */
|
||||
|
|
@ -263,6 +263,46 @@ const WAITER_TIMEOUT_MS = 15_000;
|
|||
const LOCK_RETRY_ATTEMPTS = 3;
|
||||
const LOCK_RETRY_DELAY_MS = 2000;
|
||||
|
||||
async function openReadOnlyDatabase(dbPath: string): Promise<lbug.Database> {
|
||||
let db: lbug.Database | undefined;
|
||||
silenceStdout();
|
||||
try {
|
||||
db = createLbugDatabase(lbug, dbPath, {
|
||||
readOnly: true,
|
||||
throwOnWalReplayFailure: false,
|
||||
});
|
||||
await db.init();
|
||||
return db;
|
||||
} catch (err) {
|
||||
if (db) await db.close().catch(() => {});
|
||||
throw err;
|
||||
} finally {
|
||||
restoreStdout();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Quarantine the .wal file and retry opening the database.
|
||||
* Used when the initial open fails with a WAL corruption error.
|
||||
*/
|
||||
async function tryQuarantineAndReopen(dbPath: string, repoId: string): Promise<lbug.Database> {
|
||||
const walPath = dbPath + '.wal';
|
||||
const quarantineName = `${walPath}.corrupt.${Date.now()}-${Math.random().toString(36).slice(2)}`;
|
||||
try {
|
||||
await fs.rename(walPath, quarantineName);
|
||||
} catch {
|
||||
throw new Error(
|
||||
`LadybugDB WAL corruption detected for ${repoId}. ` +
|
||||
`Run \`gitnexus analyze\` to rebuild the index. (quarantine failed)`,
|
||||
);
|
||||
}
|
||||
realStderrWrite(
|
||||
`GitNexus: LadybugDB WAL quarantined for ${repoId}; graph may be stale. ` +
|
||||
`Run \`gitnexus analyze\` to rebuild the index.\n`,
|
||||
);
|
||||
return await openReadOnlyDatabase(dbPath);
|
||||
}
|
||||
|
||||
/** Deduplicates concurrent initLbug calls for the same repoId */
|
||||
const initPromises = new Map<string, Promise<void>>();
|
||||
|
||||
|
|
@ -319,16 +359,29 @@ async function doInitLbug(repoId: string, dbPath: string): Promise<void> {
|
|||
// avoids lock conflicts when `gitnexus analyze` is writing.
|
||||
let lastError: Error | null = null;
|
||||
for (let attempt = 1; attempt <= LOCK_RETRY_ATTEMPTS; attempt++) {
|
||||
silenceStdout();
|
||||
try {
|
||||
const db = createLbugDatabase(lbug, dbPath, { readOnly: true });
|
||||
restoreStdout();
|
||||
const db = await openReadOnlyDatabase(dbPath);
|
||||
shared = { db, refCount: 0, ftsLoaded: false };
|
||||
dbCache.set(dbPath, shared);
|
||||
break;
|
||||
} catch (err: any) {
|
||||
restoreStdout();
|
||||
lastError = err instanceof Error ? err : new Error(String(err));
|
||||
|
||||
if (isWalCorruptionError(lastError)) {
|
||||
try {
|
||||
const db = await tryQuarantineAndReopen(dbPath, repoId);
|
||||
shared = { db, refCount: 0, ftsLoaded: false };
|
||||
dbCache.set(dbPath, shared);
|
||||
break;
|
||||
} catch (retryErr) {
|
||||
throw new Error(
|
||||
`LadybugDB WAL corruption detected for ${repoId}. ` +
|
||||
`Run \`gitnexus analyze\` to rebuild the index. ` +
|
||||
`(${retryErr instanceof Error ? retryErr.message : String(retryErr)})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const isLockError =
|
||||
lastError.message.includes('Could not set lock') || lastError.message.includes('lock');
|
||||
if (!isLockError || attempt === LOCK_RETRY_ATTEMPTS) break;
|
||||
|
|
|
|||
|
|
@ -167,7 +167,18 @@ export const TYPE_ALIAS_SCHEMA = CODE_ELEMENT_BASE('TypeAlias');
|
|||
export const CONST_SCHEMA = CODE_ELEMENT_BASE('Const');
|
||||
export const STATIC_SCHEMA = CODE_ELEMENT_BASE('Static');
|
||||
export const VARIABLE_SCHEMA = CODE_ELEMENT_BASE('Variable');
|
||||
export const PROPERTY_SCHEMA = CODE_ELEMENT_BASE('Property');
|
||||
export const PROPERTY_SCHEMA = `
|
||||
CREATE NODE TABLE \`Property\` (
|
||||
id STRING,
|
||||
name STRING,
|
||||
filePath STRING,
|
||||
startLine INT64,
|
||||
endLine INT64,
|
||||
content STRING,
|
||||
description STRING,
|
||||
declaredType STRING,
|
||||
PRIMARY KEY (id)
|
||||
)`;
|
||||
export const RECORD_SCHEMA = CODE_ELEMENT_BASE('Record');
|
||||
export const DELEGATE_SCHEMA = CODE_ELEMENT_BASE('Delegate');
|
||||
export const ANNOTATION_SCHEMA = CODE_ELEMENT_BASE('Annotation');
|
||||
|
|
|
|||
|
|
@ -15,9 +15,16 @@ export interface BM25SearchResult {
|
|||
nodeIds?: string[];
|
||||
}
|
||||
|
||||
export interface FTSSearchResponse {
|
||||
results: BM25SearchResult[];
|
||||
/** True when at least one FTS index query succeeded (index exists). */
|
||||
ftsAvailable: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single FTS query via a custom executor (for MCP connection pool).
|
||||
* Returns the same shape as core queryFTS (from LadybugDB adapter).
|
||||
* Returns `null` when the query fails (e.g. FTS index does not exist) so the
|
||||
* caller can distinguish "zero matches" from "index missing".
|
||||
*/
|
||||
async function queryFTSViaExecutor(
|
||||
executor: (cypher: string) => Promise<any[]>,
|
||||
|
|
@ -25,7 +32,7 @@ async function queryFTSViaExecutor(
|
|||
indexName: string,
|
||||
query: string,
|
||||
limit: number,
|
||||
): Promise<Array<{ filePath: string; score: number; nodeId: string }>> {
|
||||
): Promise<Array<{ filePath: string; score: number; nodeId: string }> | null> {
|
||||
// Escape single quotes and backslashes to prevent Cypher injection
|
||||
const escapedQuery = query.replace(/\\/g, '\\\\').replace(/'/g, "''");
|
||||
const cypher = `
|
||||
|
|
@ -46,7 +53,7 @@ async function queryFTSViaExecutor(
|
|||
};
|
||||
});
|
||||
} catch {
|
||||
return [];
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -65,8 +72,9 @@ export const searchFTSFromLbug = async (
|
|||
query: string,
|
||||
limit: number = 20,
|
||||
repoId?: string,
|
||||
): Promise<BM25SearchResult[]> => {
|
||||
): Promise<FTSSearchResponse> => {
|
||||
const resultsByIndex: any[][] = [];
|
||||
let queriesSucceeded = 0;
|
||||
|
||||
if (repoId) {
|
||||
// Use MCP connection pool via dynamic import
|
||||
|
|
@ -77,15 +85,27 @@ export const searchFTSFromLbug = async (
|
|||
const executor = (cypher: string) => executeQuery(repoId, cypher);
|
||||
|
||||
for (const { table, indexName } of FTS_INDEXES) {
|
||||
resultsByIndex.push(await queryFTSViaExecutor(executor, table, indexName, query, limit));
|
||||
const result = await queryFTSViaExecutor(executor, table, indexName, query, limit);
|
||||
if (result !== null) {
|
||||
queriesSucceeded++;
|
||||
resultsByIndex.push(result);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Use core lbug adapter (CLI / pipeline context) — also sequential for safety.
|
||||
for (const { table, indexName } of FTS_INDEXES) {
|
||||
resultsByIndex.push(await queryFTS(table, indexName, query, limit, false).catch(() => []));
|
||||
try {
|
||||
const result = await queryFTS(table, indexName, query, limit, false);
|
||||
queriesSucceeded++;
|
||||
resultsByIndex.push(result);
|
||||
} catch {
|
||||
// FTS index may not exist — count as failed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ftsAvailable = queriesSucceeded > 0;
|
||||
|
||||
// Collect all node scores per filePath to track which nodes actually matched
|
||||
const fileNodeScores = new Map<string, Array<{ score: number; nodeId: string }>>();
|
||||
|
||||
|
|
@ -116,10 +136,13 @@ export const searchFTSFromLbug = async (
|
|||
.sort((a, b) => b.score - a.score)
|
||||
.slice(0, limit);
|
||||
|
||||
return sorted.map((r, index) => ({
|
||||
filePath: r.filePath,
|
||||
score: r.score,
|
||||
rank: index + 1,
|
||||
nodeIds: r.nodeIds,
|
||||
}));
|
||||
return {
|
||||
results: sorted.map((r, index) => ({
|
||||
filePath: r.filePath,
|
||||
score: r.score,
|
||||
rank: index + 1,
|
||||
nodeIds: r.nodeIds,
|
||||
})),
|
||||
ftsAvailable,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -113,12 +113,13 @@ export const mergeWithRRF = (
|
|||
};
|
||||
|
||||
/**
|
||||
* Check if hybrid search is available
|
||||
* LadybugDB FTS is always available once the database is initialized.
|
||||
* Semantic search is optional - hybrid works with just FTS if embeddings aren't ready.
|
||||
* Check if hybrid search is available.
|
||||
* FTS indexes may be missing on read-only MCP connections (see #1403);
|
||||
* callers should inspect `ftsAvailable` from searchFTSFromLbug for
|
||||
* per-query availability. This helper is a coarse gate only.
|
||||
*/
|
||||
export const isHybridSearchReady = (): boolean => {
|
||||
return true; // FTS is always available via LadybugDB when DB is open
|
||||
return true; // FTS is attempted on every query; ftsAvailable signals actual availability
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -160,7 +161,7 @@ export const hybridSearch = async (
|
|||
) => Promise<SemanticSearchResult[]>,
|
||||
): Promise<HybridSearchResult[]> => {
|
||||
// Use LadybugDB FTS for always-fresh BM25 results
|
||||
const bm25Results = await searchFTSFromLbug(query, limit);
|
||||
const { results: bm25Results } = await searchFTSFromLbug(query, limit);
|
||||
const semanticResults = await semanticSearch(executeQuery, query, limit);
|
||||
return mergeWithRRF(bm25Results, semanticResults, limit);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { logger } from '../logger.js';
|
||||
import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared';
|
||||
/**
|
||||
* LLM Client for Wiki Generation
|
||||
*
|
||||
|
|
@ -76,6 +77,49 @@ export function estimateTokens(text: string): number {
|
|||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that a base URL supplied for LLM API calls is a safe HTTP/HTTPS
|
||||
* endpoint (CWE-918 / CodeQL js/http-to-file-access).
|
||||
*
|
||||
* Allowed:
|
||||
* - https:// with any hostname (public LLM APIs, Azure, OpenRouter, …)
|
||||
* - http:// restricted to localhost / 127.0.0.1 (local servers: Ollama, LiteLLM, …)
|
||||
*
|
||||
* Rejected:
|
||||
* - file://, data:, javascript:, and any other non-HTTP scheme
|
||||
* - http:// aimed at non-loopback hosts (avoids SSRF against internal networks)
|
||||
*
|
||||
* Throws with a descriptive message on validation failure so callers surface a
|
||||
* clear error rather than an opaque network error.
|
||||
*/
|
||||
export function validateLLMBaseUrl(baseUrl: string): void {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(baseUrl);
|
||||
} catch {
|
||||
// Do not include the raw input in the message — it may contain credentials.
|
||||
throw new Error('Invalid LLM base URL: must be a well-formed http:// or https:// URL');
|
||||
}
|
||||
|
||||
if (!['https:', 'http:'].includes(parsed.protocol)) {
|
||||
// Use parsed.protocol only (scheme), not the full URL, to avoid leaking credentials.
|
||||
throw new Error(`LLM base URL must use http:// or https:// (got ${parsed.protocol})`);
|
||||
}
|
||||
|
||||
if (parsed.protocol === 'http:') {
|
||||
// Node's URL parser preserves IPv6 brackets in hostname (e.g. "[::1]"),
|
||||
// so strip them before comparing to bare address literals.
|
||||
const host = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
||||
if (host !== 'localhost' && host !== '127.0.0.1' && host !== '::1') {
|
||||
// Use parsed.origin (scheme+host+port, no credentials) instead of the full URL.
|
||||
throw new Error(
|
||||
`Insecure http:// LLM base URLs are only allowed for localhost/127.0.0.1. ` +
|
||||
`Use https:// for remote endpoints (got ${parsed.origin})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if the given base URL is an Azure OpenAI endpoint.
|
||||
* Uses proper hostname matching to avoid spoofed URLs like
|
||||
|
|
@ -86,8 +130,10 @@ export function isAzureProvider(baseUrl: string): boolean {
|
|||
const { hostname } = new URL(baseUrl);
|
||||
return hostname.endsWith('.openai.azure.com') || hostname.endsWith('.services.ai.azure.com');
|
||||
} catch {
|
||||
// If URL is malformed, fall back to substring check
|
||||
return baseUrl.includes('.openai.azure.com') || baseUrl.includes('.services.ai.azure.com');
|
||||
// Malformed URL — refuse to call this Azure rather than fall back to a
|
||||
// substring check, which is bypassable by `https://evil.com/?u=.openai.azure.com`
|
||||
// (CodeQL js/incomplete-url-substring-sanitization).
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -125,6 +171,9 @@ export async function callLLM(
|
|||
systemPrompt?: string,
|
||||
options?: CallLLMOptions,
|
||||
): Promise<LLMResponse> {
|
||||
// Validate base URL before any fetch (CodeQL js/http-to-file-access)
|
||||
validateLLMBaseUrl(config.baseUrl);
|
||||
|
||||
const messages: Array<{ role: string; content: string }> = [];
|
||||
if (systemPrompt) {
|
||||
messages.push({ role: 'system', content: systemPrompt });
|
||||
|
|
@ -168,86 +217,85 @@ export async function callLLM(
|
|||
? { 'api-key': config.apiKey }
|
||||
: { Authorization: `Bearer ${config.apiKey}` };
|
||||
|
||||
const MAX_RETRIES = 3;
|
||||
let lastError: Error | null = null;
|
||||
|
||||
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
// Network resilience (bounded retries with exponential-backoff jitter,
|
||||
// 5xx + 429 + Retry-After handling, in-process circuit breaker on the
|
||||
// LLM endpoint) is delegated to resilientFetch. Provider-specific
|
||||
// error parsing (Azure content filter, empty-content checks) stays
|
||||
// here since it requires response-body inspection.
|
||||
let response: Response;
|
||||
try {
|
||||
response = await resilientFetch(
|
||||
url,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...authHeaders,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => 'unknown error');
|
||||
|
||||
// Azure content filter — surface a clear message instead of a generic API error
|
||||
if (
|
||||
azure &&
|
||||
response.status === 400 &&
|
||||
(errorText.includes('content_filter') ||
|
||||
errorText.includes('ResponsibleAIPolicyViolation'))
|
||||
) {
|
||||
throw new Error(
|
||||
`Azure content filter blocked this request. The prompt triggered content policy. Details: ${errorText.slice(0, 300)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Rate limit — wait with exponential backoff and retry
|
||||
if (response.status === 429 && attempt < MAX_RETRIES - 1) {
|
||||
const retryAfter = parseInt(response.headers.get('retry-after') || '0', 10);
|
||||
const delay = retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 3000;
|
||||
await sleep(delay);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Server error — retry with backoff
|
||||
if (response.status >= 500 && attempt < MAX_RETRIES - 1) {
|
||||
await sleep((attempt + 1) * 2000);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`);
|
||||
}
|
||||
|
||||
// Streaming path
|
||||
if (useStream && response.body) {
|
||||
return await readSSEStream(response.body, options!.onChunk!);
|
||||
}
|
||||
|
||||
// Non-streaming path
|
||||
const json = (await response.json()) as any;
|
||||
const choice = json.choices?.[0];
|
||||
if (!choice?.message?.content) {
|
||||
throw new Error('LLM returned empty response');
|
||||
}
|
||||
|
||||
return {
|
||||
content: choice.message.content,
|
||||
promptTokens: json.usage?.prompt_tokens,
|
||||
completionTokens: json.usage?.completion_tokens,
|
||||
};
|
||||
} catch (err: any) {
|
||||
lastError = err;
|
||||
|
||||
// Network error — retry with backoff
|
||||
if (
|
||||
attempt < MAX_RETRIES - 1 &&
|
||||
(err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.message?.includes('fetch'))
|
||||
) {
|
||||
await sleep((attempt + 1) * 3000);
|
||||
continue;
|
||||
}
|
||||
|
||||
throw err;
|
||||
// Per-attempt timeout. Without this each retry can hang
|
||||
// indefinitely on a frozen TCP connection — the per-call
|
||||
// signal is the only timeout `resilientFetch` honors;
|
||||
// `capDelayMs` only bounds the *backoff* between attempts.
|
||||
// 60s matches typical LLM completion budgets.
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
},
|
||||
{
|
||||
breakerKey: `wiki-llm-${new URL(url).host}`,
|
||||
retry: { maxAttempts: 3, baseDelayMs: 2_000, capDelayMs: 30_000 },
|
||||
},
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof CircuitOpenError) {
|
||||
throw new Error(
|
||||
`LLM endpoint circuit open: retry in ${Math.ceil(err.retryAfterMs / 1000)}s. ${err.message}`,
|
||||
);
|
||||
}
|
||||
if (err instanceof ResilientFetchExhaustedError) {
|
||||
const errorText = await err.response.text().catch(() => 'unknown error');
|
||||
throw new Error(
|
||||
`LLM API error (${err.response.status} after retries): ${errorText.slice(0, 500)}`,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
throw lastError || new Error('LLM call failed after retries');
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => 'unknown error');
|
||||
|
||||
// Azure content filter — surface a clear message instead of a generic API error.
|
||||
if (
|
||||
azure &&
|
||||
response.status === 400 &&
|
||||
(errorText.includes('content_filter') || errorText.includes('ResponsibleAIPolicyViolation'))
|
||||
) {
|
||||
throw new Error(
|
||||
`Azure content filter blocked this request. The prompt triggered content policy. Details: ${errorText.slice(0, 300)}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Any other non-OK response here is a terminal 4xx — resilientFetch
|
||||
// already retried 5xx/429 to exhaustion and would have thrown above.
|
||||
throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`);
|
||||
}
|
||||
|
||||
// Streaming path
|
||||
if (useStream && response.body) {
|
||||
return await readSSEStream(response.body, options!.onChunk!);
|
||||
}
|
||||
|
||||
// Non-streaming path
|
||||
const json = (await response.json()) as any;
|
||||
const choice = json.choices?.[0];
|
||||
if (!choice?.message?.content) {
|
||||
throw new Error('LLM returned empty response');
|
||||
}
|
||||
|
||||
return {
|
||||
content: choice.message.content,
|
||||
promptTokens: json.usage?.prompt_tokens,
|
||||
completionTokens: json.usage?.completion_tokens,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -310,7 +358,3 @@ async function readSSEStream(
|
|||
|
||||
return { content };
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,11 @@ import {
|
|||
httpEmbedQuery,
|
||||
} from '../../core/embeddings/http-client.js';
|
||||
import { resolveEmbeddingConfig } from '../../core/embeddings/config.js';
|
||||
import { applyHfEnvOverrides } from '../../core/embeddings/hf-env.js';
|
||||
import {
|
||||
applyHfEnvOverrides,
|
||||
isHfDownloadFailure,
|
||||
withHfDownloadRetry,
|
||||
} from '../../core/embeddings/hf-env.js';
|
||||
import { silenceStdout, restoreStdout, realStderrWrite } from '../../core/lbug/pool-adapter.js';
|
||||
|
||||
import { logger } from '../../core/logger.js';
|
||||
|
|
@ -69,23 +73,39 @@ export const initEmbedder = async (): Promise<FeatureExtractionPipeline> => {
|
|||
silenceStdout();
|
||||
process.stderr.write = (() => true) as any;
|
||||
try {
|
||||
embedderInstance = await (pipeline as any)('feature-extraction', MODEL_ID, {
|
||||
device: device,
|
||||
dtype: 'fp32',
|
||||
session_options: {
|
||||
logSeverityLevel: 3,
|
||||
intraOpNumThreads: embeddingConfig.threads,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
},
|
||||
});
|
||||
embedderInstance = await withHfDownloadRetry(() =>
|
||||
pipeline('feature-extraction', MODEL_ID, {
|
||||
device: device,
|
||||
dtype: 'fp32',
|
||||
session_options: {
|
||||
logSeverityLevel: 3,
|
||||
intraOpNumThreads: embeddingConfig.threads,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
},
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
restoreStdout();
|
||||
process.stderr.write = realStderrWrite;
|
||||
}
|
||||
logger.info({ device }, 'GitNexus: Embedding model loaded');
|
||||
return embedderInstance!;
|
||||
} catch {
|
||||
} catch (deviceError) {
|
||||
// Network errors and circuit-open errors are not device-specific —
|
||||
// they will fail the same way on every device. Rethrow immediately
|
||||
// with actionable HF_ENDPOINT guidance rather than silently falling
|
||||
// back to the next device.
|
||||
const errMsg = deviceError instanceof Error ? deviceError.message : String(deviceError);
|
||||
if (isHfDownloadFailure(errMsg)) {
|
||||
const endpointHint = process.env.HF_ENDPOINT
|
||||
? `The configured endpoint (${process.env.HF_ENDPOINT}) may be unreachable.`
|
||||
: `huggingface.co may be unreachable from your network.\n` +
|
||||
` Set HF_ENDPOINT to a mirror and retry:\n` +
|
||||
` HF_ENDPOINT=https://hf-mirror.com npx gitnexus analyze --embeddings\n` +
|
||||
` (Windows: set HF_ENDPOINT=https://hf-mirror.com && npx gitnexus analyze --embeddings)`;
|
||||
throw new Error(`Failed to download embedding model: ${errMsg}\n ${endpointHint}`);
|
||||
}
|
||||
if (device === 'cpu') throw new Error('Failed to load embedding model');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import {
|
|||
isLbugReady,
|
||||
isWriteQuery,
|
||||
} from '../../core/lbug/pool-adapter.js';
|
||||
import { isWalCorruptionError, WAL_RECOVERY_SUGGESTION } from '../../core/lbug/lbug-config.js';
|
||||
export { isWriteQuery };
|
||||
// Embedding imports are lazy (dynamic import) to avoid loading onnxruntime-node
|
||||
// at MCP server startup — crashes on unsupported Node ABI versions (#89)
|
||||
|
|
@ -40,7 +41,7 @@ import {
|
|||
isVectorExtensionSupportedByPlatform,
|
||||
} from '../../core/platform/capabilities.js';
|
||||
import { PhaseTimer } from '../../core/search/phase-timer.js';
|
||||
import { checkStaleness, checkCwdMatch } from '../../core/git-staleness.js';
|
||||
import { checkStalenessAsync, checkCwdMatch } from '../../core/git-staleness.js';
|
||||
import { logger } from '../../core/logger.js';
|
||||
// AI context generation is CLI-only (gitnexus analyze)
|
||||
// import { generateAIContextFiles } from '../../cli/ai-context.js';
|
||||
|
|
@ -554,8 +555,15 @@ export class LocalBackend {
|
|||
byRemote.set(h.remoteUrl, list);
|
||||
}
|
||||
|
||||
return handles.map((h) => {
|
||||
const stale = checkStaleness(h.repoPath, h.lastCommit);
|
||||
// Check staleness for all repos in parallel instead of sequentially.
|
||||
// Each check spawns an async `git rev-list` — with 200 repos the sync
|
||||
// variant took ~50 s; parallel async brings it under a second (#1363).
|
||||
const stalenessResults = await Promise.all(
|
||||
handles.map((h) => checkStalenessAsync(h.repoPath, h.lastCommit)),
|
||||
);
|
||||
|
||||
return handles.map((h, i) => {
|
||||
const stale = stalenessResults[i];
|
||||
const selfNorm = norm(h.repoPath);
|
||||
const siblings = h.remoteUrl
|
||||
? (byRemote.get(h.remoteUrl) ?? []).filter((e) => norm(e.repoPath) !== selfNorm)
|
||||
|
|
@ -971,7 +979,7 @@ export class LocalBackend {
|
|||
timing,
|
||||
...(!ftsUsed && {
|
||||
warning:
|
||||
'FTS extension unavailable - keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.',
|
||||
'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.',
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
|
@ -985,9 +993,9 @@ export class LocalBackend {
|
|||
limit: number,
|
||||
): Promise<{ results: any[]; ftsUsed: boolean }> {
|
||||
const { searchFTSFromLbug } = await import('../../core/search/bm25-index.js');
|
||||
let bm25Results;
|
||||
let ftsResponse;
|
||||
try {
|
||||
bm25Results = await searchFTSFromLbug(query, limit, repo.id);
|
||||
ftsResponse = await searchFTSFromLbug(query, limit, repo.id);
|
||||
} catch (err: any) {
|
||||
logger.error(
|
||||
{ err: err.message },
|
||||
|
|
@ -996,7 +1004,8 @@ export class LocalBackend {
|
|||
return { results: [], ftsUsed: false };
|
||||
}
|
||||
|
||||
const ftsUsed = bm25Results.length === 0 || bm25Results[0]?.ftsUsed !== false;
|
||||
const bm25Results = ftsResponse.results;
|
||||
const ftsUsed = ftsResponse.ftsAvailable;
|
||||
|
||||
const results: any[] = [];
|
||||
|
||||
|
|
@ -1218,7 +1227,14 @@ export class LocalBackend {
|
|||
const result = await executeQuery(repo.id, params.query);
|
||||
return result;
|
||||
} catch (err: any) {
|
||||
return { error: err.message || 'Query failed' };
|
||||
const msg = err.message || 'Query failed';
|
||||
if (isWalCorruptionError(err)) {
|
||||
return {
|
||||
error: msg,
|
||||
recoverySuggestion: WAL_RECOVERY_SUGGESTION,
|
||||
};
|
||||
}
|
||||
return { error: msg };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1672,6 +1688,30 @@ export class LocalBackend {
|
|||
kind?: string;
|
||||
include_content?: boolean;
|
||||
},
|
||||
): Promise<any> {
|
||||
try {
|
||||
return await this._contextImpl(repo, params);
|
||||
} catch (err: any) {
|
||||
const msg = (err instanceof Error ? err.message : String(err)) || 'Context query failed';
|
||||
if (isWalCorruptionError(err)) {
|
||||
return {
|
||||
error: msg,
|
||||
recoverySuggestion: WAL_RECOVERY_SUGGESTION,
|
||||
};
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private async _contextImpl(
|
||||
repo: RepoHandle,
|
||||
params: {
|
||||
name?: string;
|
||||
uid?: string;
|
||||
file_path?: string;
|
||||
kind?: string;
|
||||
include_content?: boolean;
|
||||
},
|
||||
): Promise<any> {
|
||||
await this.ensureInitialized(repo.id);
|
||||
|
||||
|
|
@ -1716,12 +1756,13 @@ export class LocalBackend {
|
|||
repo.id,
|
||||
`
|
||||
MATCH (caller)-[r:CodeRelation]->(n {id: $symId})
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES']
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES']
|
||||
RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind
|
||||
LIMIT 30
|
||||
`,
|
||||
{ symId },
|
||||
);
|
||||
let typedPropertyRows: any[] = [];
|
||||
|
||||
// Fix #480: Class/Interface nodes have no direct CALLS/IMPORTS edges —
|
||||
// those point to Constructor and File nodes respectively. Fetch those
|
||||
|
|
@ -1755,23 +1796,24 @@ export class LocalBackend {
|
|||
|
||||
if (isClassLike) {
|
||||
try {
|
||||
// Run both incoming-ref queries in parallel — they are independent.
|
||||
const [ctorIncoming, fileIncoming] = await Promise.all([
|
||||
executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
// Run incoming-ref queries in parallel — they are independent.
|
||||
const [ctorIncoming, fileIncoming, typedPropertyIncoming, typedProperties] =
|
||||
await Promise.all([
|
||||
executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
MATCH (n)-[hm:CodeRelation]->(ctor:Constructor)
|
||||
WHERE n.id = $symId AND hm.type = 'HAS_METHOD'
|
||||
MATCH (caller)-[r:CodeRelation]->(ctor)
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'ACCESSES']
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'ACCESSES']
|
||||
RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind
|
||||
LIMIT 30
|
||||
`,
|
||||
{ symId },
|
||||
),
|
||||
executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
{ symId },
|
||||
),
|
||||
executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
MATCH (f:File)-[rel:CodeRelation]->(n)
|
||||
WHERE n.id = $symId AND rel.type = 'DEFINES'
|
||||
MATCH (caller)-[r:CodeRelation]->(f)
|
||||
|
|
@ -1779,9 +1821,45 @@ export class LocalBackend {
|
|||
RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind
|
||||
LIMIT 30
|
||||
`,
|
||||
{ symId },
|
||||
),
|
||||
]);
|
||||
{ symId },
|
||||
),
|
||||
executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
MATCH (p:\`Property\`)
|
||||
WHERE p.declaredType = $name
|
||||
OR p.declaredType STARTS WITH $genericPrefix
|
||||
OR p.declaredType CONTAINS $genericArg
|
||||
MATCH (caller)-[r:CodeRelation]->(p)
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'ACCESSES']
|
||||
RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind
|
||||
LIMIT 30
|
||||
`,
|
||||
{
|
||||
name: sym.name,
|
||||
genericPrefix: `${sym.name}<`,
|
||||
genericArg: `<${sym.name}>`,
|
||||
},
|
||||
),
|
||||
executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
MATCH (p:\`Property\`)
|
||||
WHERE p.declaredType = $name
|
||||
OR p.declaredType STARTS WITH $genericPrefix
|
||||
OR p.declaredType CONTAINS $genericArg
|
||||
RETURN p.id AS uid, p.name AS name, p.filePath AS filePath, labels(p)[0] AS kind,
|
||||
p.declaredType AS declaredType
|
||||
LIMIT 30
|
||||
`,
|
||||
{
|
||||
name: sym.name,
|
||||
genericPrefix: `${sym.name}<`,
|
||||
genericArg: `<${sym.name}>`,
|
||||
},
|
||||
),
|
||||
]);
|
||||
typedPropertyRows = typedProperties;
|
||||
|
||||
// Deduplicate by (relType, uid) — a caller can have multiple relation
|
||||
// types to the same target (e.g. both IMPORTS and CALLS), and each
|
||||
|
|
@ -1789,7 +1867,7 @@ export class LocalBackend {
|
|||
const seenKeys = new Set(
|
||||
incomingRows.map((r: any) => `${r.relType || r[0]}:${r.uid || r[1]}`),
|
||||
);
|
||||
for (const r of [...ctorIncoming, ...fileIncoming]) {
|
||||
for (const r of [...ctorIncoming, ...fileIncoming, ...typedPropertyIncoming]) {
|
||||
const key = `${r.relType || r[0]}:${r.uid || r[1]}`;
|
||||
if (!seenKeys.has(key)) {
|
||||
seenKeys.add(key);
|
||||
|
|
@ -1806,7 +1884,7 @@ export class LocalBackend {
|
|||
repo.id,
|
||||
`
|
||||
MATCH (n {id: $symId})-[r:CodeRelation]->(target)
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES']
|
||||
WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES']
|
||||
RETURN r.type AS relType, target.id AS uid, target.name AS name, target.filePath AS filePath, labels(target)[0] AS kind
|
||||
LIMIT 30
|
||||
`,
|
||||
|
|
@ -1895,6 +1973,17 @@ export class LocalBackend {
|
|||
},
|
||||
incoming: categorize(incomingRows),
|
||||
outgoing: categorize(outgoingRows),
|
||||
...(typedPropertyRows.length > 0
|
||||
? {
|
||||
typed_properties: typedPropertyRows.map((r: any) => ({
|
||||
uid: r.uid || r[0],
|
||||
name: r.name || r[1],
|
||||
filePath: r.filePath || r[2],
|
||||
kind: r.kind || r[3],
|
||||
declaredType: r.declaredType || r[4],
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
processes: processRows.map((r: any) => ({
|
||||
id: r.pid || r[0],
|
||||
name: r.label || r[1],
|
||||
|
|
@ -2433,6 +2522,7 @@ export class LocalBackend {
|
|||
impactedCount: 0,
|
||||
risk: 'UNKNOWN',
|
||||
suggestion: 'The graph query failed — try gitnexus context <symbol> as a fallback',
|
||||
...(isWalCorruptionError(err) ? { recoverySuggestion: WAL_RECOVERY_SUGGESTION } : {}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -2459,6 +2549,7 @@ export class LocalBackend {
|
|||
const mappedRelTypes = params.relationTypes?.flatMap((t: string) =>
|
||||
t === 'OVERRIDES' ? ['OVERRIDES', 'METHOD_OVERRIDES'] : [t],
|
||||
);
|
||||
const hasExplicitRelationTypes = mappedRelTypes !== undefined && mappedRelTypes.length > 0;
|
||||
const rawRelTypes =
|
||||
mappedRelTypes && mappedRelTypes.length > 0
|
||||
? mappedRelTypes.filter((t: string) => VALID_RELATION_TYPES.has(t))
|
||||
|
|
@ -2467,6 +2558,7 @@ export class LocalBackend {
|
|||
'IMPORTS',
|
||||
'EXTENDS',
|
||||
'IMPLEMENTS',
|
||||
'USES',
|
||||
'METHOD_OVERRIDES',
|
||||
'OVERRIDES',
|
||||
'METHOD_IMPLEMENTS',
|
||||
|
|
@ -2479,6 +2571,7 @@ export class LocalBackend {
|
|||
'IMPORTS',
|
||||
'EXTENDS',
|
||||
'IMPLEMENTS',
|
||||
'USES',
|
||||
'METHOD_OVERRIDES',
|
||||
'OVERRIDES',
|
||||
'METHOD_IMPLEMENTS',
|
||||
|
|
@ -2538,9 +2631,16 @@ export class LocalBackend {
|
|||
};
|
||||
const symType = outcome.resolvedLabel || outcome.symbol.type || '';
|
||||
|
||||
const effectiveRelationTypes =
|
||||
(symType === 'Class' || symType === 'Interface') &&
|
||||
!hasExplicitRelationTypes &&
|
||||
!relationTypes.includes('ACCESSES')
|
||||
? [...relationTypes, 'ACCESSES']
|
||||
: relationTypes;
|
||||
|
||||
return this._runImpactBFS(repo, sym, symType, direction, {
|
||||
maxDepth,
|
||||
relationTypes,
|
||||
relationTypes: effectiveRelationTypes,
|
||||
includeTests,
|
||||
minConfidence,
|
||||
});
|
||||
|
|
@ -2619,6 +2719,30 @@ export class LocalBackend {
|
|||
frontier.push(rid);
|
||||
}
|
||||
}
|
||||
|
||||
const typedPropertyRows = await executeParameterized(
|
||||
repo.id,
|
||||
`
|
||||
MATCH (p:\`Property\`)
|
||||
WHERE p.declaredType = $name
|
||||
OR p.declaredType STARTS WITH $genericPrefix
|
||||
OR p.declaredType CONTAINS $genericArg
|
||||
RETURN p.id AS id, p.name AS name, labels(p)[0] AS type, p.filePath AS filePath
|
||||
`,
|
||||
{
|
||||
name: sym.name,
|
||||
genericPrefix: `${sym.name}<`,
|
||||
genericArg: `<${sym.name}>`,
|
||||
},
|
||||
);
|
||||
|
||||
for (const r of typedPropertyRows) {
|
||||
const rid = r.id || r[0];
|
||||
if (rid && !visited.has(rid)) {
|
||||
visited.add(rid);
|
||||
frontier.push(rid);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logQueryError('impact:class-node-expansion', e);
|
||||
}
|
||||
|
|
@ -2984,8 +3108,14 @@ export class LocalBackend {
|
|||
relationTypes: string[];
|
||||
minConfidence: number;
|
||||
includeTests: boolean;
|
||||
signal?: AbortSignal;
|
||||
},
|
||||
): Promise<any | null> {
|
||||
// Honor an already-aborted signal at the entry boundary as a fast
|
||||
// path. Cooperative cancellation inside _runImpactBFS is out of
|
||||
// scope — the caller's Promise.race against the same signal
|
||||
// resolves the await regardless of how long this body runs.
|
||||
if (opts.signal?.aborted) return null;
|
||||
try {
|
||||
await this.refreshRepos();
|
||||
await this.ensureInitialized(repoId);
|
||||
|
|
|
|||
|
|
@ -755,8 +755,13 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
|
||||
if (isMatch && ['queued', 'cloning', 'analyzing'].includes(job.status)) {
|
||||
if (process.env.DEBUG) {
|
||||
console.log(
|
||||
`[debug] resolveRepo waiting for active job ${job.id} (${normalizedName})...`,
|
||||
// Sanitize user-controlled values to prevent log injection (CodeQL js/log-injection).
|
||||
logger.debug(
|
||||
{
|
||||
jobId: String(job.id).replace(/[\r\n]/g, ' '),
|
||||
repoName: String(normalizedName).replace(/[\r\n]/g, ' '),
|
||||
},
|
||||
'[debug] resolveRepo waiting for active job',
|
||||
);
|
||||
}
|
||||
for (let wait = 0; wait < HOLD_QUEUE_TIMEOUT_SECS; wait++) {
|
||||
|
|
@ -780,7 +785,11 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
// (e.g. registry file not yet flushed after clone completes).
|
||||
if (!found && normalizedName && !isRetry) {
|
||||
if (process.env.DEBUG) {
|
||||
console.log(`[debug] resolveRepo 404 for "${normalizedName}". Triggering deep init...`);
|
||||
// Sanitize user-controlled values to prevent log injection (CodeQL js/log-injection).
|
||||
logger.debug(
|
||||
{ repoName: String(normalizedName).replace(/[\r\n]/g, ' ') },
|
||||
'[debug] resolveRepo 404, triggering deep init',
|
||||
);
|
||||
}
|
||||
await backend.init();
|
||||
return await resolveRepo(normalizedName, true, req);
|
||||
|
|
@ -1071,11 +1080,12 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
|
||||
const results = await withLbugDb(lbugPath, async () => {
|
||||
let searchResults: any[];
|
||||
let ftsAvailable: boolean | undefined;
|
||||
|
||||
if (mode === 'semantic') {
|
||||
const { isEmbedderReady } = await import('../core/embeddings/embedder.js');
|
||||
if (!isEmbedderReady()) {
|
||||
return [] as any[];
|
||||
return { searchResults: [] as any[], ftsAvailable: undefined };
|
||||
}
|
||||
const { semanticSearch: semSearch } =
|
||||
await import('../core/embeddings/embedding-pipeline.js');
|
||||
|
|
@ -1088,8 +1098,9 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
sources: ['semantic'],
|
||||
}));
|
||||
} else if (mode === 'bm25') {
|
||||
searchResults = await searchFTSFromLbug(query, limit);
|
||||
searchResults = searchResults.map((r: any, i: number) => ({
|
||||
const ftsResponse = await searchFTSFromLbug(query, limit);
|
||||
ftsAvailable = ftsResponse.ftsAvailable;
|
||||
searchResults = ftsResponse.results.map((r: any, i: number) => ({
|
||||
...r,
|
||||
rank: i + 1,
|
||||
sources: ['bm25'],
|
||||
|
|
@ -1102,11 +1113,13 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
await import('../core/embeddings/embedding-pipeline.js');
|
||||
searchResults = await hybridSearch(query, limit, executeQuery, semSearch);
|
||||
} else {
|
||||
searchResults = await searchFTSFromLbug(query, limit);
|
||||
const ftsResponse = await searchFTSFromLbug(query, limit);
|
||||
ftsAvailable = ftsResponse.ftsAvailable;
|
||||
searchResults = ftsResponse.results;
|
||||
}
|
||||
}
|
||||
|
||||
if (!enrich) return searchResults;
|
||||
if (!enrich) return { searchResults, ftsAvailable };
|
||||
|
||||
// Server-side enrichment: add connections, cluster, processes per result
|
||||
// Uses parameterized queries to prevent Cypher injection via nodeId
|
||||
|
|
@ -1188,9 +1201,14 @@ export const createServer = async (port: number, host: string = '127.0.0.1') =>
|
|||
}),
|
||||
);
|
||||
|
||||
return enriched;
|
||||
return { searchResults: enriched, ftsAvailable };
|
||||
});
|
||||
res.json({ results });
|
||||
const response: any = { results: results.searchResults ?? results };
|
||||
if (results.ftsAvailable === false) {
|
||||
response.warning =
|
||||
'FTS indexes missing — keyword search degraded. Run: gitnexus analyze --force to rebuild indexes.';
|
||||
}
|
||||
res.json(response);
|
||||
} catch (err: any) {
|
||||
res.status(500).json({ error: err.message || 'Search failed' });
|
||||
}
|
||||
|
|
|
|||
18
gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs
vendored
Normal file
18
gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs
vendored
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
namespace App;
|
||||
|
||||
public class USER_INFO
|
||||
{
|
||||
public string? USER_ID { get; set; }
|
||||
}
|
||||
|
||||
public interface IEntityTypeConfiguration<T>
|
||||
{
|
||||
}
|
||||
|
||||
public class UserInfoConfiguration : IEntityTypeConfiguration<USER_INFO>
|
||||
{
|
||||
public Task<List<USER_INFO>> Load(List<USER_INFO> users)
|
||||
{
|
||||
return Task.FromResult(users);
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,13 @@ export async function cleanupTempDir(tmpDir: string): Promise<void> {
|
|||
/**
|
||||
* Create a temporary directory for LadybugDB tests.
|
||||
* Returns the path and a cleanup function.
|
||||
*
|
||||
* IMPORTANT: when adding a new test that passes a custom `prefix`, also add
|
||||
* the prefix to `TEST_FIXTURE_PREFIXES` in
|
||||
* `gitnexus/src/core/lbug/lbug-config.ts`. The stale-sidecar sweep relies
|
||||
* on the prefix list to recognize test fixtures; an unknown prefix means
|
||||
* the sweep silently won't fire for that fixture and Windows CI flakes
|
||||
* return.
|
||||
*/
|
||||
export async function createTempDir(prefix: string = 'gitnexus-test-'): Promise<TestDBHandle> {
|
||||
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), prefix));
|
||||
|
|
|
|||
77
gitnexus/test/integration/context-typed-property.test.ts
Normal file
77
gitnexus/test/integration/context-typed-property.test.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Integration test: context() expands Class symbols through typed properties.
|
||||
*
|
||||
* Reproduces EF-style usage where code reads a DbContext property
|
||||
* (`db.USER_INFO`) whose source type is `DbSet<USER_INFO>`. The direct
|
||||
* graph edge is Method -> Property, not Method -> Class, so context() must
|
||||
* use the same typed-property bridge that impact() uses.
|
||||
*/
|
||||
import { beforeAll, describe, expect, it, vi } from 'vitest';
|
||||
import { LocalBackend } from '../../src/mcp/local/local-backend.js';
|
||||
import { listRegisteredRepos } from '../../src/storage/repo-manager.js';
|
||||
import { withTestLbugDB } from '../helpers/test-indexed-db.js';
|
||||
|
||||
vi.mock('../../src/storage/repo-manager.js', () => ({
|
||||
listRegisteredRepos: vi.fn().mockResolvedValue([]),
|
||||
cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }),
|
||||
findSiblingClones: vi.fn().mockResolvedValue([]),
|
||||
}));
|
||||
|
||||
const SEED = [
|
||||
`CREATE (c:Class {id:'Class:Models/USER_INFO.cs:USER_INFO', name:'USER_INFO', filePath:'Models/USER_INFO.cs', startLine:1, endLine:5, content:'public class USER_INFO {}', description:''})`,
|
||||
`CREATE (p:\`Property\` {id:'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO', name:'USER_INFO', filePath:'Data/UserDbContext.cs', startLine:10, endLine:10, content:'public DbSet<USER_INFO> USER_INFO { get; set; }', description:'', declaredType:'DbSet<USER_INFO>'})`,
|
||||
`CREATE (m:Method {id:'Method:Services/UserService.cs:UserService.GetUserInfo#1', name:'GetUserInfo', filePath:'Services/UserService.cs', startLine:20, endLine:30, isExported:false, content:'db.USER_INFO.FirstOrDefault();', description:'', parameterCount:1, returnType:'USER_INFO'})`,
|
||||
`MATCH (m:Method {id:'Method:Services/UserService.cs:UserService.GetUserInfo#1'}), (p:\`Property\` {id:'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO'}) CREATE (m)-[:CodeRelation {type:'ACCESSES', confidence:1.0, reason:'read', step:1}]->(p)`,
|
||||
];
|
||||
|
||||
withTestLbugDB(
|
||||
'context-typed-property',
|
||||
(handle) => {
|
||||
let backend: LocalBackend;
|
||||
|
||||
beforeAll(async () => {
|
||||
backend = (handle as any)._backend;
|
||||
});
|
||||
|
||||
describe('context() typed-property expansion', () => {
|
||||
it('surfaces property callers and explains the typed property bridge', async () => {
|
||||
const result = await backend.callTool('context', {
|
||||
uid: 'Class:Models/USER_INFO.cs:USER_INFO',
|
||||
});
|
||||
|
||||
expect(result.status).toBe('found');
|
||||
expect(result.symbol.kind).toBe('Class');
|
||||
|
||||
const accesses = result.incoming.accesses || [];
|
||||
expect(accesses.map((r: any) => r.name)).toContain('GetUserInfo');
|
||||
|
||||
expect(result.typed_properties).toEqual([
|
||||
expect.objectContaining({
|
||||
uid: 'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO',
|
||||
name: 'USER_INFO',
|
||||
declaredType: 'DbSet<USER_INFO>',
|
||||
}),
|
||||
]);
|
||||
});
|
||||
});
|
||||
},
|
||||
{
|
||||
seed: SEED,
|
||||
poolAdapter: true,
|
||||
afterSetup: async (handle) => {
|
||||
vi.mocked(listRegisteredRepos).mockResolvedValue([
|
||||
{
|
||||
name: 'test-repo',
|
||||
path: '/test/repo',
|
||||
storagePath: handle.tmpHandle.dbPath,
|
||||
indexedAt: new Date().toISOString(),
|
||||
lastCommit: 'abc123',
|
||||
stats: { files: 3, nodes: 3, communities: 0, processes: 0 },
|
||||
},
|
||||
]);
|
||||
const backend = new LocalBackend();
|
||||
await backend.init();
|
||||
(handle as any)._backend = backend;
|
||||
},
|
||||
},
|
||||
);
|
||||
195
gitnexus/test/integration/group/include-extractor-sync.test.ts
Normal file
195
gitnexus/test/integration/group/include-extractor-sync.test.ts
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
/**
|
||||
* Integration test: IncludeExtractor output → group matching → bridge DB.
|
||||
*
|
||||
* Covers PR #1156 review finding #7: verifies that the full runtime path
|
||||
* (IncludeExtractor → StoredContract → runExactMatch → CrossLinks → writeBridge)
|
||||
* stays wired up. A regression in either normalizeContractId or the include
|
||||
* branch of ManifestExtractor.resolveSymbol would produce 0 cross-links and
|
||||
* fail this test.
|
||||
*/
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { parseGroupConfig } from '../../../src/core/group/config-parser.js';
|
||||
import { syncGroup } from '../../../src/core/group/sync.js';
|
||||
import type { StoredContract } from '../../../src/core/group/types.js';
|
||||
import { IncludeExtractor } from '../../../src/core/group/extractors/include-extractor.js';
|
||||
import { normalizeContractId } from '../../../src/core/group/matching.js';
|
||||
|
||||
const GROUP_YAML = [
|
||||
'version: 1',
|
||||
'name: include-test-group',
|
||||
'description: "IncludeExtractor integration test"',
|
||||
'',
|
||||
'repos:',
|
||||
' app/provider: include-provider',
|
||||
' app/consumer: include-consumer',
|
||||
'',
|
||||
'links: []',
|
||||
'packages: {}',
|
||||
'',
|
||||
'detect:',
|
||||
' http: false',
|
||||
' grpc: false',
|
||||
' topics: false',
|
||||
' shared_libs: false',
|
||||
' includes: true',
|
||||
' embedding_fallback: false',
|
||||
'',
|
||||
'matching:',
|
||||
' bm25_threshold: 0.7',
|
||||
' embedding_threshold: 0.65',
|
||||
' max_candidates_per_step: 3',
|
||||
].join('\n');
|
||||
|
||||
describe('IncludeExtractor → syncGroup integration (finding #7)', () => {
|
||||
it('produces a CrossLink when provider and consumer emit the same include contract-id', async () => {
|
||||
const config = parseGroupConfig(GROUP_YAML);
|
||||
|
||||
// Mock the IncludeExtractor output directly — a header provider in one
|
||||
// repo and a quoted #include consumer in the other, both normalized to
|
||||
// the same include::map/base/view.h contract-id.
|
||||
const mockContracts: StoredContract[] = [
|
||||
{
|
||||
contractId: 'include::map/base/view.h',
|
||||
type: 'include',
|
||||
role: 'provider',
|
||||
symbolUid: 'File:map/base/view.h',
|
||||
symbolRef: { filePath: 'map/base/view.h', name: 'view.h' },
|
||||
symbolName: 'view.h',
|
||||
confidence: 0.95,
|
||||
meta: { source: 'filesystem' },
|
||||
repo: 'app/provider',
|
||||
},
|
||||
{
|
||||
contractId: 'include::map/base/view.h',
|
||||
type: 'include',
|
||||
role: 'consumer',
|
||||
symbolUid: 'File:src/controller.cpp',
|
||||
symbolRef: { filePath: 'src/controller.cpp', name: 'map/base/view.h' },
|
||||
symbolName: 'map/base/view.h',
|
||||
confidence: 0.85,
|
||||
meta: { source: 'tree_sitter', includePath: 'map/base/view.h' },
|
||||
repo: 'app/consumer',
|
||||
},
|
||||
];
|
||||
|
||||
const result = await syncGroup(config, {
|
||||
extractorOverride: async () => mockContracts,
|
||||
skipWrite: true,
|
||||
});
|
||||
|
||||
const includeLinks = result.crossLinks.filter((l) => l.type === 'include');
|
||||
expect(includeLinks.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const link = includeLinks[0];
|
||||
expect(link.contractId).toBe('include::map/base/view.h');
|
||||
expect(link.matchType).toBe('exact');
|
||||
expect(link.from.repo).toBe('app/consumer');
|
||||
expect(link.to.repo).toBe('app/provider');
|
||||
});
|
||||
|
||||
it('normalizes mixed-case / backslash include paths to the same contract-id end-to-end', async () => {
|
||||
const config = parseGroupConfig(GROUP_YAML);
|
||||
|
||||
// Provider writes the canonical form; consumer's include has mixed case
|
||||
// and a backslash. After normalizeContractId they must still match.
|
||||
const providerId = 'include::map/base/view.h';
|
||||
const rawConsumerId = 'include::Map\\Base\\View.h';
|
||||
|
||||
// Sanity — normalizeContractId must collapse them.
|
||||
expect(normalizeContractId(rawConsumerId)).toBe(providerId);
|
||||
|
||||
const mockContracts: StoredContract[] = [
|
||||
{
|
||||
contractId: providerId,
|
||||
type: 'include',
|
||||
role: 'provider',
|
||||
symbolUid: 'File:map/base/view.h',
|
||||
symbolRef: { filePath: 'map/base/view.h', name: 'view.h' },
|
||||
symbolName: 'view.h',
|
||||
confidence: 0.95,
|
||||
meta: { source: 'filesystem' },
|
||||
repo: 'app/provider',
|
||||
},
|
||||
{
|
||||
contractId: rawConsumerId,
|
||||
type: 'include',
|
||||
role: 'consumer',
|
||||
symbolUid: 'File:src/controller.cpp',
|
||||
symbolRef: { filePath: 'src/controller.cpp', name: 'Map/Base/View.h' },
|
||||
symbolName: 'Map/Base/View.h',
|
||||
confidence: 0.85,
|
||||
meta: { source: 'tree_sitter', includePath: 'Map\\Base\\View.h' },
|
||||
repo: 'app/consumer',
|
||||
},
|
||||
];
|
||||
|
||||
const result = await syncGroup(config, {
|
||||
extractorOverride: async () => mockContracts,
|
||||
skipWrite: true,
|
||||
});
|
||||
|
||||
const includeLinks = result.crossLinks.filter((l) => l.type === 'include');
|
||||
expect(includeLinks.length).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
it('round-trip: extractor output from two real temp repos produces matching contract-ids', async () => {
|
||||
// Drives the extractor directly (no `syncGroup`) against two on-disk
|
||||
// fixture repos, then hands the StoredContract-shaped output to
|
||||
// syncGroup via extractorOverride. This exercises the real extraction
|
||||
// code + the matching pipeline together.
|
||||
const providerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-int-provider-'));
|
||||
const consumerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-int-consumer-'));
|
||||
try {
|
||||
fs.mkdirSync(path.join(providerDir, 'shared/api'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(providerDir, 'shared/api/client.h'),
|
||||
'#pragma once\nstruct Client {};',
|
||||
);
|
||||
fs.mkdirSync(path.join(consumerDir, 'src'), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(consumerDir, 'src/main.cpp'),
|
||||
'#include "shared/api/client.h"\nint main(){return 0;}',
|
||||
);
|
||||
|
||||
const extractor = new IncludeExtractor();
|
||||
const providerOutput = await extractor.extract(null, providerDir, {
|
||||
id: 'provider',
|
||||
path: 'app/provider',
|
||||
repoPath: providerDir,
|
||||
storagePath: path.join(providerDir, '.gitnexus'),
|
||||
});
|
||||
const consumerOutput = await extractor.extract(null, consumerDir, {
|
||||
id: 'consumer',
|
||||
path: 'app/consumer',
|
||||
repoPath: consumerDir,
|
||||
storagePath: path.join(consumerDir, '.gitnexus'),
|
||||
});
|
||||
|
||||
const stored: StoredContract[] = [
|
||||
...providerOutput
|
||||
.filter((c) => c.role === 'provider')
|
||||
.map((c) => ({ ...c, repo: 'app/provider' })),
|
||||
...consumerOutput
|
||||
.filter((c) => c.role === 'consumer')
|
||||
.map((c) => ({ ...c, repo: 'app/consumer' })),
|
||||
];
|
||||
|
||||
const config = parseGroupConfig(GROUP_YAML);
|
||||
const result = await syncGroup(config, {
|
||||
extractorOverride: async () => stored,
|
||||
skipWrite: true,
|
||||
});
|
||||
|
||||
const includeLinks = result.crossLinks.filter((l) => l.type === 'include');
|
||||
expect(includeLinks.length).toBeGreaterThanOrEqual(1);
|
||||
expect(includeLinks[0].contractId).toBe('include::shared/api/client.h');
|
||||
expect(includeLinks[0].matchType).toBe('exact');
|
||||
} finally {
|
||||
fs.rmSync(providerDir, { recursive: true, force: true });
|
||||
fs.rmSync(consumerDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
41
gitnexus/test/integration/lbug-close-handle-release.test.ts
Normal file
41
gitnexus/test/integration/lbug-close-handle-release.test.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
/**
|
||||
* Integration test: safeClose's Windows post-close handle-release wait.
|
||||
*
|
||||
* On Windows, libuv reports `db.close()` resolved before the kernel has
|
||||
* released the file handle. A subsequent open of the same path can then
|
||||
* race the release and surface "Could not set lock on file". `safeClose`
|
||||
* probes the file with `fs.open` to force the residual lock to surface,
|
||||
* absorbed by the open-time retry in `lbug-config.ts`.
|
||||
*/
|
||||
import path from 'path';
|
||||
import { describe, it } from 'vitest';
|
||||
import { createTempDir } from '../helpers/test-db.js';
|
||||
|
||||
describe('safeClose — close + reopen does not surface lock errors', () => {
|
||||
it('survives 10 sequential open/close/reopen cycles on the same path', async () => {
|
||||
const tmp = await createTempDir('gitnexus-lbug-close-cycle-');
|
||||
const dbPath = path.join(tmp.dbPath, 'lbug');
|
||||
try {
|
||||
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
for (let i = 0; i < 10; i++) {
|
||||
await adapter.initLbug(dbPath);
|
||||
await adapter.closeLbug();
|
||||
}
|
||||
} finally {
|
||||
await tmp.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('safeClose is idempotent — calling twice in a row does not throw', async () => {
|
||||
const tmp = await createTempDir('gitnexus-lbug-idempotent-');
|
||||
const dbPath = path.join(tmp.dbPath, 'lbug');
|
||||
try {
|
||||
const adapter = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
await adapter.initLbug(dbPath);
|
||||
await adapter.closeLbug();
|
||||
await adapter.closeLbug();
|
||||
} finally {
|
||||
await tmp.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -14,7 +14,7 @@ import { withTestLbugDB } from '../helpers/test-indexed-db.js';
|
|||
|
||||
// Pure-function tests — no DB needed, but grouped here for cohesion
|
||||
// with the retry logic they guard.
|
||||
import { isDbBusyError } from '../../src/core/lbug/lbug-adapter.js';
|
||||
import { isDbBusyError } from '../../src/core/lbug/lbug-config.js';
|
||||
|
||||
describe('isDbBusyError', () => {
|
||||
it('returns true for "busy" errors (case-insensitive)', () => {
|
||||
|
|
@ -46,6 +46,18 @@ describe('isDbBusyError', () => {
|
|||
expect(isDbBusyError(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
// Documented behavior for lock-shaped strings: the matcher is intentionally
|
||||
// broad because in graph-DB contexts these are all transient. If LadybugDB
|
||||
// ever surfaces a non-transient lock-shaped error (e.g., a recovery-time
|
||||
// "lock file missing"), tighten the matcher and add a negative test here
|
||||
// rather than raising the retry budget.
|
||||
it('treats other lock-shaped errors as transient (current intentional behavior)', () => {
|
||||
expect(isDbBusyError(new Error('deadlock detected'))).toBe(true);
|
||||
expect(isDbBusyError(new Error('unlock failed'))).toBe(true);
|
||||
expect(isDbBusyError(new Error('lock contention'))).toBe(true);
|
||||
expect(isDbBusyError(new Error('Could not open lock file'))).toBe(true);
|
||||
});
|
||||
|
||||
it('handles non-Error values gracefully', () => {
|
||||
expect(isDbBusyError('BUSY error')).toBe(true);
|
||||
expect(isDbBusyError(42)).toBe(false);
|
||||
|
|
|
|||
310
gitnexus/test/integration/lbug-open-retry.test.ts
Normal file
310
gitnexus/test/integration/lbug-open-retry.test.ts
Normal file
|
|
@ -0,0 +1,310 @@
|
|||
/**
|
||||
* Integration tests: open-time lock-busy retry in `lbug-config.ts`.
|
||||
*
|
||||
* The lock IO exception raised by `local_file_system.cpp` happens
|
||||
* synchronously inside `new lbug.Database(...)`, before any query is
|
||||
* issued — so `withLbugDb`'s query-time retry cannot see it. These tests
|
||||
* exercise the construction-time retry wrapper directly by stubbing the
|
||||
* `Database` constructor.
|
||||
*
|
||||
* See: docs/plans/2026-05-08-002-fix-windows-lbug-lock-ci-flakes-plan.md
|
||||
*/
|
||||
import fs from 'fs/promises';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import {
|
||||
_isTestFixturePathForTest as isTestFixturePath,
|
||||
isDbBusyError,
|
||||
isOpenRetryExhausted,
|
||||
openLbugConnection,
|
||||
waitForWindowsHandleRelease,
|
||||
} from '../../src/core/lbug/lbug-config.js';
|
||||
|
||||
// ─── Minimal stub of the `lbug` module surface used by openLbugConnection ──
|
||||
|
||||
interface StubModuleControl {
|
||||
/** Errors thrown by sequential `new Database(...)` calls. `null` = success. */
|
||||
databaseThrows: Array<Error | null>;
|
||||
/** Number of times the `Database` constructor was invoked. */
|
||||
databaseCallCount: number;
|
||||
/** Number of times `db.close()` was called. */
|
||||
closeCallCount: number;
|
||||
}
|
||||
|
||||
const makeStubLbug = (control: StubModuleControl) => {
|
||||
class FakeDatabase {
|
||||
constructor(_path: string, ..._rest: unknown[]) {
|
||||
control.databaseCallCount++;
|
||||
const next = control.databaseThrows.shift();
|
||||
if (next instanceof Error) throw next;
|
||||
}
|
||||
async close(): Promise<void> {
|
||||
control.closeCallCount++;
|
||||
}
|
||||
}
|
||||
class FakeConnection {
|
||||
constructor(_db: FakeDatabase) {}
|
||||
async close(): Promise<void> {}
|
||||
}
|
||||
return { Database: FakeDatabase, Connection: FakeConnection } as any;
|
||||
};
|
||||
|
||||
describe('isDbBusyError', () => {
|
||||
it('matches the documented Windows lock-error wording', () => {
|
||||
expect(isDbBusyError(new Error('Could not set lock on file foo.lbug'))).toBe(true);
|
||||
expect(isDbBusyError(new Error('database is locked'))).toBe(true);
|
||||
});
|
||||
it('does not match unrelated errors', () => {
|
||||
expect(isDbBusyError(new Error('Cypher syntax error'))).toBe(false);
|
||||
expect(isDbBusyError(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('openLbugConnection — open-time lock-busy retry', () => {
|
||||
it('returns a handle when the constructor succeeds on the first try', async () => {
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [null],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
const handle = await openLbugConnection(stub, '/some/path/lbug');
|
||||
expect(handle.db).toBeDefined();
|
||||
expect(handle.conn).toBeDefined();
|
||||
expect(control.databaseCallCount).toBe(1);
|
||||
});
|
||||
|
||||
it('retries on busy/lock errors and succeeds on a later attempt', async () => {
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [new Error('Could not set lock on file'), null],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
const handle = await openLbugConnection(stub, '/some/path/lbug');
|
||||
expect(handle.db).toBeDefined();
|
||||
expect(control.databaseCallCount).toBe(2);
|
||||
});
|
||||
|
||||
it('exhausts the retry budget and rethrows the last error preserving its message', async () => {
|
||||
const lockErr = new Error('Could not set lock on file foo.lbug');
|
||||
const control: StubModuleControl = {
|
||||
// 5 attempts + production paths get no sweep retry, so 5 throws total.
|
||||
databaseThrows: [lockErr, lockErr, lockErr, lockErr, lockErr],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
await expect(openLbugConnection(stub, '/var/data/non-test/lbug')).rejects.toThrow(
|
||||
'Could not set lock on file foo.lbug',
|
||||
);
|
||||
expect(control.databaseCallCount).toBe(5);
|
||||
});
|
||||
|
||||
it('tags the exhausted error so withLbugDb skips its outer retry', async () => {
|
||||
const lockErr = new Error('Could not set lock on file');
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [lockErr, lockErr, lockErr, lockErr, lockErr],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
let caught: unknown;
|
||||
try {
|
||||
await openLbugConnection(stub, '/var/data/non-test/lbug');
|
||||
} catch (err) {
|
||||
caught = err;
|
||||
}
|
||||
expect(caught).toBeDefined();
|
||||
expect(isOpenRetryExhausted(caught)).toBe(true);
|
||||
expect(isOpenRetryExhausted(new Error('plain error'))).toBe(false);
|
||||
expect(isOpenRetryExhausted(null)).toBe(false);
|
||||
expect(isOpenRetryExhausted(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it('does not retry non-busy errors', async () => {
|
||||
const syntaxErr = new Error('Cypher syntax error');
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [syntaxErr],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
await expect(openLbugConnection(stub, '/some/path/lbug')).rejects.toThrow(
|
||||
'Cypher syntax error',
|
||||
);
|
||||
expect(control.databaseCallCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('openLbugConnection — stale-sidecar sweep (test fixtures only)', () => {
|
||||
let fixtureDir: string;
|
||||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-lbug-sweep-'));
|
||||
dbPath = path.join(fixtureDir, 'lbug');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(fixtureDir, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
it('sweeps stale .wal/.lock for a recognized test fixture path and retries once', async () => {
|
||||
await fs.writeFile(dbPath + '.wal', 'stale');
|
||||
await fs.writeFile(dbPath + '.lock', 'stale');
|
||||
|
||||
const lockErr = new Error('Could not set lock on file');
|
||||
const control: StubModuleControl = {
|
||||
// 5 retries throw, then sweep + 1 final attempt succeeds (6 total).
|
||||
databaseThrows: [lockErr, lockErr, lockErr, lockErr, lockErr, null],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
|
||||
const handle = await openLbugConnection(stub, dbPath);
|
||||
expect(handle.db).toBeDefined();
|
||||
expect(control.databaseCallCount).toBe(6);
|
||||
|
||||
// Sidecars removed by the sweep
|
||||
await expect(fs.access(dbPath + '.wal')).rejects.toThrow();
|
||||
await expect(fs.access(dbPath + '.lock')).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('does not sweep production paths even if they share the prefix', async () => {
|
||||
// A non-tmp dir that *starts* with the prefix must still be rejected.
|
||||
const lockErr = new Error('Could not set lock on file');
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [lockErr, lockErr, lockErr, lockErr, lockErr],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
|
||||
// Path is outside os.tmpdir() so the predicate must reject it.
|
||||
await expect(openLbugConnection(stub, '/var/data/gitnexus-lbug-fake/lbug')).rejects.toThrow(
|
||||
'Could not set lock on file',
|
||||
);
|
||||
expect(control.databaseCallCount).toBe(5); // no sweep retry
|
||||
});
|
||||
|
||||
it('handles missing sidecars gracefully (ENOENT swallowed, retry runs)', async () => {
|
||||
// No .wal or .lock pre-created — sweep ENOENTs both, then succeeds.
|
||||
const lockErr = new Error('Could not set lock on file');
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [lockErr, lockErr, lockErr, lockErr, lockErr, null],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
|
||||
const handle = await openLbugConnection(stub, dbPath);
|
||||
expect(handle.db).toBeDefined();
|
||||
expect(control.databaseCallCount).toBe(6);
|
||||
});
|
||||
|
||||
it('sweep retry that throws a different error preserves the original lock error', async () => {
|
||||
// 5 lock errors, then sweep fires, then post-sweep throws an unrelated
|
||||
// error. The user-actionable signal is "lock retries exhausted" — the
|
||||
// post-sweep error must NOT shadow the original lock message.
|
||||
const lockErr = new Error('Could not set lock on file foo.lbug');
|
||||
const unrelatedErr = new Error('Schema validation error during open');
|
||||
const control: StubModuleControl = {
|
||||
databaseThrows: [lockErr, lockErr, lockErr, lockErr, lockErr, unrelatedErr],
|
||||
databaseCallCount: 0,
|
||||
closeCallCount: 0,
|
||||
};
|
||||
const stub = makeStubLbug(control);
|
||||
|
||||
let caught: Error | undefined;
|
||||
try {
|
||||
await openLbugConnection(stub, dbPath);
|
||||
} catch (err) {
|
||||
caught = err as Error;
|
||||
}
|
||||
expect(caught?.message).toBe('Could not set lock on file foo.lbug');
|
||||
expect(control.databaseCallCount).toBe(6); // sweep retry did fire
|
||||
});
|
||||
});
|
||||
|
||||
describe('isTestFixturePath — production-safety guard', () => {
|
||||
it('accepts a fixture under os.tmpdir with a recognized prefix on the immediate parent', () => {
|
||||
const tmp = os.tmpdir();
|
||||
expect(isTestFixturePath(path.join(tmp, 'gitnexus-lbug-XXX', 'lbug'))).toBe(true);
|
||||
expect(isTestFixturePath(path.join(tmp, 'gitnexus-test-YYY', 'lbug'))).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects production paths even with a matching prefix', () => {
|
||||
expect(isTestFixturePath('/var/data/gitnexus-lbug-fake/lbug')).toBe(false);
|
||||
expect(isTestFixturePath('/home/user/gitnexus-test-foo/lbug')).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects path traversal attempts that resolve outside tmpdir', () => {
|
||||
const tmp = os.tmpdir();
|
||||
const traversal = path.join(tmp, 'gitnexus-lbug-x', '..', '..', 'etc', 'passwd');
|
||||
expect(isTestFixturePath(traversal)).toBe(false);
|
||||
});
|
||||
|
||||
it('rejects when the immediate parent does not match even if a deeper ancestor does', () => {
|
||||
// Tightening: ancestor walk would have allowed nested paths under
|
||||
// `<tmp>/gitnexus-lbug-x/inner/lbug` to satisfy the predicate. We
|
||||
// require the immediate parent to match.
|
||||
const tmp = os.tmpdir();
|
||||
expect(isTestFixturePath(path.join(tmp, 'gitnexus-lbug-x', 'inner', 'lbug'))).toBe(false);
|
||||
});
|
||||
|
||||
it('handles tmpdir trailing-separator gracefully', () => {
|
||||
// Some Windows TMP configs return a trailing separator; the predicate
|
||||
// strips it before the prefix check so fixtures still match.
|
||||
const tmp = os.tmpdir();
|
||||
const fixture = path.join(tmp, 'gitnexus-lbug-trailing', 'lbug');
|
||||
// Whether or not os.tmpdir() itself has a trailing separator,
|
||||
// the predicate must accept legit fixtures.
|
||||
expect(isTestFixturePath(fixture)).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects unrelated prefixes in tmpdir', () => {
|
||||
const tmp = os.tmpdir();
|
||||
expect(isTestFixturePath(path.join(tmp, 'random-dir', 'lbug'))).toBe(false);
|
||||
expect(isTestFixturePath(path.join(tmp, 'malicious', 'lbug'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('waitForWindowsHandleRelease', () => {
|
||||
let fixtureDir: string;
|
||||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
fixtureDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-lbug-probe-'));
|
||||
dbPath = path.join(fixtureDir, 'lbug');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(fixtureDir, { recursive: true, force: true }).catch(() => {});
|
||||
});
|
||||
|
||||
it('returns true when the file exists and is openable', async () => {
|
||||
await fs.writeFile(dbPath, 'fake-db-content');
|
||||
const released = await waitForWindowsHandleRelease(dbPath);
|
||||
expect(released).toBe(true);
|
||||
});
|
||||
|
||||
it('returns true when the file does not exist (ENOENT is non-lock)', async () => {
|
||||
// No fs.writeFile — path does not exist. Probe should bail to true,
|
||||
// not retry, since ENOENT is not a lock code.
|
||||
const released = await waitForWindowsHandleRelease(dbPath);
|
||||
expect(released).toBe(true);
|
||||
});
|
||||
|
||||
it('does not leak the file handle when close succeeds', async () => {
|
||||
// Smoke test: 50 sequential probes with a real file. If close were
|
||||
// skipped, fd usage would climb. We rely on test process not OOMing
|
||||
// as the simplest indicator; fd table caps catch egregious leaks.
|
||||
await fs.writeFile(dbPath, 'fake-db-content');
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await waitForWindowsHandleRelease(dbPath);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -1440,6 +1440,24 @@ describe('Write access tracking (C#)', () => {
|
|||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Generic type references: IEntityTypeConfiguration<USER_INFO>, List<USER_INFO>
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('C# generic type-reference tracking', () => {
|
||||
let result: PipelineResult;
|
||||
|
||||
beforeAll(async () => {
|
||||
result = await runPipelineFromRepo(path.join(FIXTURES, 'csharp-generic-type-refs'), () => {});
|
||||
}, 60000);
|
||||
|
||||
it('emits USES edges for generic type arguments', () => {
|
||||
const uses = getRelationships(result, 'USES').filter((e) => e.target === 'USER_INFO');
|
||||
expect(edgeSet(uses)).toContain('UserInfoConfiguration → USER_INFO');
|
||||
expect(edgeSet(uses)).toContain('Load → USER_INFO');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Call-result variable binding (Phase 9): var user = GetUser(); user.Save()
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -11,6 +11,9 @@ import type { GraphRelationship } from 'gitnexus-shared';
|
|||
const LEGACY_RESOLVER_PARITY_EXPECTED_FAILURES: Readonly<Record<string, ReadonlySet<string>>> = {
|
||||
csharp: new Set([
|
||||
'emits the using-import edge App/Program.cs -> Models/User.cs through the scope-resolution path',
|
||||
// Generic type-argument USES edges are emitted by the registry-primary
|
||||
// resolver only; the legacy DAG path does not synthesize these references.
|
||||
'emits USES edges for generic type arguments',
|
||||
]),
|
||||
go: new Set([
|
||||
// The legacy DAG path does not resolve method calls when the method is
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ withTestLbugDB(
|
|||
(_handle) => {
|
||||
describe('searchFTSFromLbug — core adapter (no repoId)', () => {
|
||||
it('returns ranked results for a matching query', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 10);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 10);
|
||||
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ withTestLbugDB(
|
|||
});
|
||||
|
||||
it('results are ordered by descending score', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 10);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 10);
|
||||
|
||||
for (let i = 1; i < results.length; i++) {
|
||||
expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
|
||||
|
|
@ -48,7 +48,7 @@ withTestLbugDB(
|
|||
});
|
||||
|
||||
it('auth-related files rank higher than unrelated files', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 10);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 10);
|
||||
const filePaths = results.map((r) => r.filePath);
|
||||
|
||||
expect(filePaths).toContain('src/auth.ts');
|
||||
|
|
@ -61,7 +61,7 @@ withTestLbugDB(
|
|||
});
|
||||
|
||||
it('merges scores from multiple node types for the same filePath', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 20);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 20);
|
||||
|
||||
const authResult = results.find((r) => r.filePath === 'src/auth.ts');
|
||||
expect(authResult).toBeDefined();
|
||||
|
|
@ -73,12 +73,12 @@ withTestLbugDB(
|
|||
});
|
||||
|
||||
it('respects limit parameter', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 2);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 2);
|
||||
expect(results.length).toBeLessThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('returns empty array for a non-matching query', async () => {
|
||||
const results = await searchFTSFromLbug('xyzzyplughtwisty', 10);
|
||||
const { results } = await searchFTSFromLbug('xyzzyplughtwisty', 10);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
@ -87,32 +87,32 @@ withTestLbugDB(
|
|||
|
||||
describe('unhappy paths', () => {
|
||||
it('returns empty array for empty query string', async () => {
|
||||
const results = await searchFTSFromLbug('', 10);
|
||||
const { results } = await searchFTSFromLbug('', 10);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array for whitespace-only query', async () => {
|
||||
const results = await searchFTSFromLbug(' ', 10);
|
||||
const { results } = await searchFTSFromLbug(' ', 10);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles special characters in query gracefully', async () => {
|
||||
const results = await searchFTSFromLbug('user* OR auth+', 10);
|
||||
const { results } = await searchFTSFromLbug('user* OR auth+', 10);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
it('handles limit of 0', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 0);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 0);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles negative limit gracefully', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', -1);
|
||||
const { results } = await searchFTSFromLbug('user authentication', -1);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
it('handles very large limit', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 100000);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 100000);
|
||||
expect(results.length).toBeLessThanOrEqual(100000);
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ withTestLbugDB(
|
|||
(handle) => {
|
||||
describe('searchFTSFromLbug — MCP pool adapter (with repoId)', () => {
|
||||
it('returns ranked results via pool adapter', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 10, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 10, handle.repoId);
|
||||
|
||||
expect(results.length).toBeGreaterThan(0);
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ withTestLbugDB(
|
|||
});
|
||||
|
||||
it('results are ordered by descending score via pool adapter', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 10, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 10, handle.repoId);
|
||||
|
||||
for (let i = 1; i < results.length; i++) {
|
||||
expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
|
||||
|
|
@ -43,12 +43,12 @@ withTestLbugDB(
|
|||
});
|
||||
|
||||
it('returns empty array for non-matching query via pool adapter', async () => {
|
||||
const results = await searchFTSFromLbug('xyzzyplughtwisty', 10, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('xyzzyplughtwisty', 10, handle.repoId);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('respects limit parameter via pool adapter', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 1, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 1, handle.repoId);
|
||||
expect(results.length).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -57,22 +57,22 @@ withTestLbugDB(
|
|||
|
||||
describe('unhappy paths', () => {
|
||||
it('returns empty array for empty query via pool', async () => {
|
||||
const results = await searchFTSFromLbug('', 10, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('', 10, handle.repoId);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('returns empty array for whitespace-only query via pool', async () => {
|
||||
const results = await searchFTSFromLbug(' ', 10, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug(' ', 10, handle.repoId);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
|
||||
it('handles special characters in query via pool', async () => {
|
||||
const results = await searchFTSFromLbug('user* OR auth+', 10, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('user* OR auth+', 10, handle.repoId);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
it('handles limit of 0 via pool', async () => {
|
||||
const results = await searchFTSFromLbug('user authentication', 0, handle.repoId);
|
||||
const { results } = await searchFTSFromLbug('user authentication', 0, handle.repoId);
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -42,20 +42,24 @@ describe('BM25 search', () => {
|
|||
});
|
||||
|
||||
describe('searchFTSFromLbug', () => {
|
||||
it('returns empty array when LadybugDB is not initialized', async () => {
|
||||
// Without LadybugDB init, search should return empty (not crash)
|
||||
const results = await searchFTSFromLbug('test query');
|
||||
it('returns empty results when LadybugDB is not initialized', async () => {
|
||||
// Simulate an uninitialized DB: queryFTS throws instead of returning rows
|
||||
const { queryFTS } = await import('../../src/core/lbug/lbug-adapter.js');
|
||||
vi.mocked(queryFTS).mockRejectedValue(new Error('DB not initialized'));
|
||||
|
||||
const { results, ftsAvailable } = await searchFTSFromLbug('test query');
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
expect(results).toHaveLength(0);
|
||||
expect(ftsAvailable).toBe(false);
|
||||
});
|
||||
|
||||
it('handles empty query', async () => {
|
||||
const results = await searchFTSFromLbug('');
|
||||
const { results } = await searchFTSFromLbug('');
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
|
||||
it('accepts custom limit parameter', async () => {
|
||||
const results = await searchFTSFromLbug('test', 5);
|
||||
const { results } = await searchFTSFromLbug('test', 5);
|
||||
expect(Array.isArray(results)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
|
@ -105,7 +109,7 @@ describe('BM25 search', () => {
|
|||
.mockResolvedValueOnce([]) // Method
|
||||
.mockResolvedValueOnce([]); // Interface
|
||||
|
||||
const results = await searchFTSFromLbug('queryset');
|
||||
const { results } = await searchFTSFromLbug('queryset');
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].filePath).toBe('src/views.py');
|
||||
|
|
@ -127,7 +131,7 @@ describe('BM25 search', () => {
|
|||
.mockResolvedValueOnce([]) // Method
|
||||
.mockResolvedValueOnce([]); // Interface
|
||||
|
||||
const results = await searchFTSFromLbug('model');
|
||||
const { results } = await searchFTSFromLbug('model');
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].score).toBe(8); // 5+3
|
||||
|
|
@ -147,7 +151,7 @@ describe('BM25 search', () => {
|
|||
.mockResolvedValueOnce([]) // Method
|
||||
.mockResolvedValueOnce([]); // Interface
|
||||
|
||||
const results = await searchFTSFromLbug('util');
|
||||
const { results } = await searchFTSFromLbug('util');
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0].nodeIds).toEqual([]);
|
||||
|
|
@ -171,7 +175,7 @@ describe('BM25 search', () => {
|
|||
.mockResolvedValueOnce([]) // Method
|
||||
.mockResolvedValueOnce([]); // Interface
|
||||
|
||||
const results = await searchFTSFromLbug('auth');
|
||||
const { results } = await searchFTSFromLbug('auth');
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
// All 3 hits (scores 9+7+4=20) — each from a different table, all top-3
|
||||
|
|
@ -192,7 +196,7 @@ describe('BM25 search', () => {
|
|||
.mockResolvedValueOnce([]) // Method
|
||||
.mockResolvedValueOnce([]); // Interface
|
||||
|
||||
const results = await searchFTSFromLbug('fn');
|
||||
const { results } = await searchFTSFromLbug('fn');
|
||||
|
||||
expect(results[0].filePath).toBe('src/high.py');
|
||||
expect(results[1].filePath).toBe('src/low.py');
|
||||
|
|
@ -220,7 +224,7 @@ describe('BM25 search', () => {
|
|||
return [];
|
||||
});
|
||||
|
||||
const results = await searchFTSFromLbug('login', 5, REPO);
|
||||
const { results } = await searchFTSFromLbug('login', 5, REPO);
|
||||
|
||||
expect(results).toEqual([
|
||||
{ filePath: 'src/auth.ts', score: 8, rank: 1, nodeIds: ['func:login'] },
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ vi.mock('../../src/storage/repo-manager.js', () => ({
|
|||
// tests don't shell out to git.
|
||||
vi.mock('../../src/core/git-staleness.js', () => ({
|
||||
checkStaleness: vi.fn().mockReturnValue({ isStale: false, commitsBehind: 0 }),
|
||||
checkStalenessAsync: vi.fn().mockResolvedValue({ isStale: false, commitsBehind: 0 }),
|
||||
checkCwdMatch: vi.fn().mockResolvedValue({ match: 'none' }),
|
||||
}));
|
||||
|
||||
|
|
@ -61,7 +62,7 @@ vi.mock('../../src/core/platform/capabilities.js', async (importOriginal) => {
|
|||
|
||||
// Also mock the search modules to avoid loading onnxruntime
|
||||
vi.mock('../../src/core/search/bm25-index.js', () => ({
|
||||
searchFTSFromLbug: vi.fn().mockResolvedValue([]),
|
||||
searchFTSFromLbug: vi.fn().mockResolvedValue({ results: [], ftsAvailable: true }),
|
||||
}));
|
||||
|
||||
vi.mock('../../src/mcp/core/embedder.js', () => ({
|
||||
|
|
@ -194,6 +195,27 @@ describe('LocalBackend.callTool', () => {
|
|||
expect(result).toHaveProperty('definitions');
|
||||
});
|
||||
|
||||
it('includes FTS-unavailable warning when ftsAvailable is false (#1403)', async () => {
|
||||
const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js');
|
||||
vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ results: [], ftsAvailable: false });
|
||||
(executeParameterized as any).mockResolvedValue([]);
|
||||
|
||||
const result = await backend.callTool('query', { query: 'ProcessActivity' });
|
||||
|
||||
expect(result).toHaveProperty('warning');
|
||||
expect((result as any).warning).toMatch(/gitnexus analyze --force/);
|
||||
});
|
||||
|
||||
it('does not include warning when ftsAvailable is true with zero results', async () => {
|
||||
const { searchFTSFromLbug } = await import('../../src/core/search/bm25-index.js');
|
||||
vi.mocked(searchFTSFromLbug).mockResolvedValueOnce({ results: [], ftsAvailable: true });
|
||||
(executeParameterized as any).mockResolvedValue([]);
|
||||
|
||||
const result = await backend.callTool('query', { query: 'nonexistent' });
|
||||
|
||||
expect(result).not.toHaveProperty('warning');
|
||||
});
|
||||
|
||||
it('skips vector index query when VECTOR is unsupported by the platform', async () => {
|
||||
const cap = _captureLogger();
|
||||
platformMocks.isVectorExtensionSupportedByPlatform.mockReturnValue(false);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ vi.mock('../../src/cli/mcp.js', () => ({
|
|||
vi.mock('../../src/cli/setup.js', () => ({
|
||||
setupCommand: vi.fn(),
|
||||
}));
|
||||
vi.mock('../../src/cli/publish.js', () => ({
|
||||
publishCommand: vi.fn(),
|
||||
}));
|
||||
|
||||
describe('CLI commands', () => {
|
||||
describe('version', () => {
|
||||
|
|
@ -84,4 +87,11 @@ describe('CLI commands', () => {
|
|||
expect(typeof setupCommand).toBe('function');
|
||||
});
|
||||
});
|
||||
|
||||
describe('publishCommand', () => {
|
||||
it('is a function', async () => {
|
||||
const { publishCommand } = await import('../../src/cli/publish.js');
|
||||
expect(typeof publishCommand).toBe('function');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -63,4 +63,17 @@ describe('CLI help surface', () => {
|
|||
expect(result.stdout).toContain('--model <model>');
|
||||
expect(result.stdout).toContain('--gist');
|
||||
});
|
||||
|
||||
it('publish help names the registry, the token env var, and the opt-out behaviour', () => {
|
||||
const result = runHelp('publish');
|
||||
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout).toContain('--id <owner/repo>');
|
||||
expect(result.stdout).toContain('--skip-git');
|
||||
// Discoverability contract: a contributor scanning `--help` must see
|
||||
// (a) which registry this dispatches to, and (b) the env var that
|
||||
// gates the opt-in. Both are part of the no-token contract.
|
||||
expect(result.stdout).toContain('understand-quickly');
|
||||
expect(result.stdout).toContain('UNDERSTAND_QUICKLY_TOKEN');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
272
gitnexus/test/unit/cpp-ue-preprocessor.test.ts
Normal file
272
gitnexus/test/unit/cpp-ue-preprocessor.test.ts
Normal file
|
|
@ -0,0 +1,272 @@
|
|||
import { describe, it, expect } from 'vitest';
|
||||
import Parser from 'tree-sitter';
|
||||
import CPP from 'tree-sitter-cpp';
|
||||
import { stripUeMacros } from '../../src/core/ingestion/cpp-ue-preprocessor.js';
|
||||
|
||||
describe('stripUeMacros — detection guard', () => {
|
||||
it('returns input unchanged when no UE markers are present', () => {
|
||||
const src = `class Plain {\npublic:\n int Get() const;\n};`;
|
||||
expect(stripUeMacros(src)).toBe(src);
|
||||
});
|
||||
|
||||
it('returns input unchanged for STL-style code', () => {
|
||||
const src = `#include <vector>\nstd::vector<int> v;`;
|
||||
expect(stripUeMacros(src)).toBe(src);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripUeMacros — length preservation', () => {
|
||||
const ueSamples: string[] = [
|
||||
`UCLASS()\nclass BRAWLUI_API UMyClass : public UObject { GENERATED_BODY() public: UFUNCTION() void Run(); };`,
|
||||
`UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Combat") int32 Health;`,
|
||||
`USTRUCT(BlueprintType)\nstruct ENGINE_API FMyData { GENERATED_BODY() float Value; };`,
|
||||
`DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMyDelegate, int32, A, FString, B);`,
|
||||
`UE_DEPRECATED(5.0, "Use NewThing instead") void OldThing();`,
|
||||
];
|
||||
|
||||
for (const src of ueSamples) {
|
||||
it(`preserves byte length: ${src.slice(0, 40).replace(/\n/g, '\\n')}…`, () => {
|
||||
const out = stripUeMacros(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
});
|
||||
|
||||
it(`preserves newline positions: ${src.slice(0, 40).replace(/\n/g, '\\n')}…`, () => {
|
||||
const out = stripUeMacros(src);
|
||||
const inputNewlines: number[] = [];
|
||||
const outputNewlines: number[] = [];
|
||||
for (let i = 0; i < src.length; i++) {
|
||||
if (src.charCodeAt(i) === 0x0a) inputNewlines.push(i);
|
||||
if (out.charCodeAt(i) === 0x0a) outputNewlines.push(i);
|
||||
}
|
||||
expect(outputNewlines).toEqual(inputNewlines);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
describe('stripUeMacros — macro removal', () => {
|
||||
it('elides UCLASS(...) with arguments', () => {
|
||||
const src = `UCLASS(BlueprintType, Category="Foo")\nclass UFoo {};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UCLASS');
|
||||
expect(out).not.toContain('BlueprintType');
|
||||
expect(out).toContain('class UFoo {};');
|
||||
});
|
||||
|
||||
it('elides UCLASS() with empty parens', () => {
|
||||
const src = `UCLASS()\nclass UBar {};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UCLASS');
|
||||
expect(out).toContain('class UBar {};');
|
||||
});
|
||||
|
||||
it('elides MODULE_API export macros (BRAWLUI_API style) when paired with a UE marker', () => {
|
||||
const src = `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject {};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('BRAWLUI_API');
|
||||
expect(out).toContain('class');
|
||||
expect(out).toContain('UMyClass');
|
||||
expect(out).toContain('public UObject');
|
||||
});
|
||||
|
||||
it('elides multiple distinct *_API tokens in same file when UE marker is present', () => {
|
||||
const src = `UCLASS()\nclass CORE_API A {};\nUCLASS()\nclass UMG_API B : public A {};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('CORE_API');
|
||||
expect(out).not.toContain('UMG_API');
|
||||
expect(out).toContain('class');
|
||||
expect(out).toContain('A {};');
|
||||
});
|
||||
|
||||
it('elides GENERATED_BODY() inside class body', () => {
|
||||
const src = `class UThing { GENERATED_BODY() public: void Foo(); };`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('GENERATED_BODY');
|
||||
expect(out).toContain('public:');
|
||||
expect(out).toContain('void Foo();');
|
||||
});
|
||||
|
||||
it('elides UFUNCTION(...) before method declarations', () => {
|
||||
const src = `class X { UFUNCTION(BlueprintCallable, Server, Reliable) void DoThing(); };`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UFUNCTION');
|
||||
expect(out).not.toContain('BlueprintCallable');
|
||||
expect(out).toContain('void DoThing();');
|
||||
});
|
||||
|
||||
it('elides UPROPERTY(...) before field declarations', () => {
|
||||
const src = `class X { UPROPERTY(EditAnywhere) int32 Health; };`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UPROPERTY');
|
||||
expect(out).not.toContain('EditAnywhere');
|
||||
expect(out).toContain('int32 Health;');
|
||||
});
|
||||
|
||||
it('elides DECLARE_DYNAMIC_MULTICAST_DELEGATE_*Params(...)', () => {
|
||||
const src = `DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMyDelegate, int32, Value);\nclass X {};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('DECLARE_DYNAMIC_MULTICAST_DELEGATE');
|
||||
expect(out).not.toContain('FMyDelegate');
|
||||
expect(out).toContain('class X {};');
|
||||
});
|
||||
|
||||
it('elides UE_DEPRECATED(...) before function declarations', () => {
|
||||
const src = `UE_DEPRECATED(5.1, "Reason") void Old();`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UE_DEPRECATED');
|
||||
expect(out).not.toContain('5.1');
|
||||
expect(out).toContain('void Old();');
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripUeMacros — non-UE files left alone', () => {
|
||||
it('does NOT strip standalone *_API identifiers when no UE marker is present', () => {
|
||||
const src = `enum class Status { REST_API = 1, HTTP_API = 2, MY_LIB_API = 3 };\nvoid handle(REST_API status);`;
|
||||
expect(stripUeMacros(src)).toBe(src);
|
||||
});
|
||||
|
||||
it('does NOT strip _API tokens in a file that only mentions DECLARE_DELEGATE-like macros from non-UE codebases', () => {
|
||||
const src = `// Custom delegate framework, not UE\n#define DECLARE_HANDLER(x) void x()\nDECLARE_HANDLER(MyHandler);\nint REST_API = 0;`;
|
||||
expect(stripUeMacros(src)).toBe(src);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripUeMacros — non-ASCII content preservation', () => {
|
||||
it('leaves non-ASCII content outside elided ranges intact and at the same .length offset', () => {
|
||||
const src = `// Comment with non-ASCII: café résumé naïve\nUCLASS()\nclass UMyClass : public UObject\n{\n GENERATED_BODY()\n // Trailing: 日本語 αβγ\n};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out.length).toBe(src.length);
|
||||
expect(out).toContain('café résumé naïve');
|
||||
expect(out).toContain('日本語 αβγ');
|
||||
expect(out).toContain('class UMyClass : public UObject');
|
||||
expect(out).not.toContain('UCLASS');
|
||||
expect(out).not.toContain('GENERATED_BODY');
|
||||
});
|
||||
|
||||
it('preserves newline positions when the file contains non-ASCII characters', () => {
|
||||
const src = `// café\nUPROPERTY()\nint32 Health;\n// résumé\nUFUNCTION()\nvoid Run();`;
|
||||
const out = stripUeMacros(src);
|
||||
const inputNewlines: number[] = [];
|
||||
const outputNewlines: number[] = [];
|
||||
for (let i = 0; i < src.length; i++) {
|
||||
if (src.charCodeAt(i) === 0x0a) inputNewlines.push(i);
|
||||
if (out.charCodeAt(i) === 0x0a) outputNewlines.push(i);
|
||||
}
|
||||
expect(outputNewlines).toEqual(inputNewlines);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripUeMacros — false-positive guards', () => {
|
||||
it('does NOT strip identifiers that merely contain UCLASS as a substring', () => {
|
||||
const src = `void NotUCLASSAtAll(); int MyUCLASS = 0;`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).toBe(src);
|
||||
});
|
||||
|
||||
it('does NOT strip _API substrings inside larger identifiers', () => {
|
||||
const src = `class MY_APIName {};\nint not_my_API_thing = 0;`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).toContain('MY_APIName');
|
||||
expect(out).toContain('not_my_API_thing');
|
||||
});
|
||||
|
||||
it('does not eat parens balanced inside string literals', () => {
|
||||
const src = `UFUNCTION(meta=(DisplayName="Foo (Bar)")) void Z();`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UFUNCTION');
|
||||
expect(out).not.toContain('DisplayName');
|
||||
expect(out).toContain('void Z();');
|
||||
});
|
||||
|
||||
it('handles UCLASS with deeply nested parens in arguments', () => {
|
||||
const src = `UCLASS(meta=(Categories=("A.B", "C.D")), Within=Foo) class UDeep {};`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).not.toContain('UCLASS');
|
||||
expect(out).not.toContain('Categories');
|
||||
expect(out).toContain('class UDeep {};');
|
||||
});
|
||||
|
||||
it('leaves Qt macros alone (only UE markers stripped)', () => {
|
||||
const src = `class QFoo { Q_OBJECT public: void Bar(); };`;
|
||||
const out = stripUeMacros(src);
|
||||
expect(out).toContain('Q_OBJECT');
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripUeMacros — class-name extraction sanity', () => {
|
||||
it('after stripping, "class UMyClass" appears immediately after "class "', () => {
|
||||
const src = `UCLASS(BlueprintType)\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n};`;
|
||||
const out = stripUeMacros(src);
|
||||
const classIdx = out.indexOf('class ');
|
||||
expect(classIdx).toBeGreaterThanOrEqual(0);
|
||||
const tail = out.slice(classIdx + 'class '.length).trimStart();
|
||||
expect(tail.startsWith('UMyClass')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('stripUeMacros — tree-sitter extraction (end-to-end)', () => {
|
||||
/**
|
||||
* Walk the parse tree and return the captured class name(s). Works against
|
||||
* the actual tree-sitter-cpp grammar so this is a true integration check
|
||||
* for the core PR claim: the indexer now sees `UMyClass`, not `BRAWLUI_API`.
|
||||
*/
|
||||
function extractClassNames(source: string): string[] {
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(CPP as unknown as Parser.Language);
|
||||
const tree = parser.parse(source);
|
||||
const names: string[] = [];
|
||||
const stack: Parser.SyntaxNode[] = [tree.rootNode];
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop()!;
|
||||
if (node.type === 'class_specifier' || node.type === 'struct_specifier') {
|
||||
const nameNode = node.childForFieldName('name');
|
||||
if (nameNode) names.push(nameNode.text);
|
||||
}
|
||||
for (let i = node.namedChildCount - 1; i >= 0; i--) {
|
||||
const child = node.namedChild(i);
|
||||
if (child) stack.push(child);
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
it('tree-sitter-cpp captures UMyClass as the class name (not BRAWLUI_API)', () => {
|
||||
const src = `UCLASS(BlueprintType)\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n public:\n UFUNCTION()\n void Run();\n};`;
|
||||
const out = stripUeMacros(src);
|
||||
const names = extractClassNames(out);
|
||||
expect(names).toContain('UMyClass');
|
||||
expect(names).not.toContain('BRAWLUI_API');
|
||||
});
|
||||
|
||||
it('tree-sitter-cpp captures struct name correctly through USTRUCT + MODULE_API', () => {
|
||||
const src = `USTRUCT(BlueprintType)\nstruct ENGINE_API FMyData : public FBase\n{\n GENERATED_BODY()\n float Value;\n};`;
|
||||
const out = stripUeMacros(src);
|
||||
const names = extractClassNames(out);
|
||||
expect(names).toContain('FMyData');
|
||||
expect(names).not.toContain('ENGINE_API');
|
||||
});
|
||||
|
||||
it('tree-sitter-cpp source positions are preserved across stripping (line numbers match)', () => {
|
||||
const src = `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n public:\n void Run();\n};`;
|
||||
const out = stripUeMacros(src);
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(CPP as unknown as Parser.Language);
|
||||
const tree = parser.parse(out);
|
||||
const stack: Parser.SyntaxNode[] = [tree.rootNode];
|
||||
let runLine: number | undefined;
|
||||
while (stack.length > 0) {
|
||||
const node = stack.pop()!;
|
||||
if (node.type === 'function_declarator') {
|
||||
const declarator = node.childForFieldName('declarator');
|
||||
if (declarator?.text === 'Run') {
|
||||
runLine = node.startPosition.row;
|
||||
break;
|
||||
}
|
||||
}
|
||||
for (let i = node.namedChildCount - 1; i >= 0; i--) {
|
||||
const child = node.namedChild(i);
|
||||
if (child) stack.push(child);
|
||||
}
|
||||
}
|
||||
expect(runLine).toBe(5); // 0-indexed: "void Run();" is on line 6 (index 5)
|
||||
});
|
||||
});
|
||||
90
gitnexus/test/unit/group/bridge-storage-tempfile.test.ts
Normal file
90
gitnexus/test/unit/group/bridge-storage-tempfile.test.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
/**
|
||||
* Regression tests for U6 — closes CodeQL js/insecure-temporary-file
|
||||
* (#191/#192/#193) and js/log-injection (#188) in core/group.
|
||||
*
|
||||
* The fixes replace `Date.now()` suffix tmp files with crypto.randomBytes
|
||||
* suffixes + open the tmp file with `flag: 'wx'` (O_EXCL). These tests
|
||||
* pin both behaviors so a future refactor that drops either signal
|
||||
* regenerates the CodeQL alert AND fails a test.
|
||||
*/
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs/promises';
|
||||
import os from 'node:os';
|
||||
import { writeContractRegistry, createGroupDir } from '../../../src/core/group/storage.js';
|
||||
import { writeBridgeMeta } from '../../../src/core/group/bridge-db.js';
|
||||
import type { ContractRegistry } from '../../../src/core/group/types.js';
|
||||
|
||||
/**
|
||||
* Build a minimal `ContractRegistry` literal with overridable fields.
|
||||
* Replaces the `as never` cast that bypassed the type entirely — keeps
|
||||
* the test free of unrelated boilerplate while still type-checking the
|
||||
* fields under test.
|
||||
*/
|
||||
function makeRegistry(overrides: Partial<ContractRegistry> = {}): ContractRegistry {
|
||||
return {
|
||||
version: 1,
|
||||
generatedAt: '2026-05-07T00:00:00Z',
|
||||
repoSnapshots: {},
|
||||
missingRepos: [],
|
||||
contracts: [],
|
||||
crossLinks: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
let tmpRoot: string;
|
||||
let groupDir: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'gitnexus-u6-'));
|
||||
groupDir = path.join(tmpRoot, 'fixture-group');
|
||||
await fs.mkdir(groupDir, { recursive: true });
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await fs.rm(tmpRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('writeContractRegistry — tempfile hardening', () => {
|
||||
it('back-to-back writes within the same ms do not collide on the tmp path', async () => {
|
||||
// The previous `${path}.tmp.${Date.now()}` shape collided when two writers
|
||||
// landed in the same millisecond. crypto.randomBytes makes the suffix
|
||||
// essentially-unique. Sequential writes here pin the unique-suffix
|
||||
// property without depending on Windows-specific concurrent-rename
|
||||
// behavior (which has its own pre-existing retry pattern in the
|
||||
// sibling `writeBridge` function and is out of scope for this test).
|
||||
await writeContractRegistry(groupDir, makeRegistry({ version: 1 }));
|
||||
await writeContractRegistry(groupDir, makeRegistry({ version: 2 }));
|
||||
const written = await fs.readFile(path.join(groupDir, 'contracts.json'), 'utf-8');
|
||||
const parsed = JSON.parse(written);
|
||||
expect(parsed.version).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('writeBridgeMeta — tempfile hardening', () => {
|
||||
it('back-to-back writes do not collide on the tmp path', async () => {
|
||||
await writeBridgeMeta(groupDir, { version: 1, generatedAt: 'a', missingRepos: [] });
|
||||
await writeBridgeMeta(groupDir, { version: 2, generatedAt: 'b', missingRepos: [] });
|
||||
const meta = JSON.parse(await fs.readFile(path.join(groupDir, 'meta.json'), 'utf-8'));
|
||||
expect(meta.version).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('createGroupDir — exclusive-create on group.yaml', () => {
|
||||
it('refuses to overwrite an existing group without force', async () => {
|
||||
const gnxDir = path.join(tmpRoot, 'gnx-existing');
|
||||
await createGroupDir(gnxDir, 'mygroup');
|
||||
// Second call without force should throw — same behavior as before this
|
||||
// commit, but now backed by O_EXCL at the writeFile level rather than
|
||||
// only the up-front existence check (closes the TOCTOU CodeQL flagged).
|
||||
await expect(createGroupDir(gnxDir, 'mygroup')).rejects.toThrow(/already exists/);
|
||||
});
|
||||
|
||||
it('overwrites with force=true', async () => {
|
||||
const gnxDir = path.join(tmpRoot, 'gnx-force');
|
||||
await createGroupDir(gnxDir, 'mygroup');
|
||||
// Should succeed without throwing.
|
||||
await expect(createGroupDir(gnxDir, 'mygroup', true)).resolves.toBeTruthy();
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue