diff --git a/.github/actions/setup-gitnexus-web/action.yml b/.github/actions/setup-gitnexus-web/action.yml index 86331e36d..8f895423a 100644 --- a/.github/actions/setup-gitnexus-web/action.yml +++ b/.github/actions/setup-gitnexus-web/action.yml @@ -1,5 +1,5 @@ name: Setup GitNexus Web -description: Setup Node.js 20.19+ (vite 7 floor), build gitnexus-shared, install web dependencies +description: Setup Node.js 22, build gitnexus-shared, install web dependencies runs: using: composite @@ -7,9 +7,7 @@ runs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: # Vite 7 requires Node ^20.19.0 || >=22.12.0 (require(esm) support). - # Pin explicitly so we don't depend on the floating "20" alias resolving - # to a high enough patch version on every runner image. - node-version: '20.19.0' + node-version: 22 cache: npm cache-dependency-path: gitnexus-web/package-lock.json diff --git a/.github/actions/setup-gitnexus/action.yml b/.github/actions/setup-gitnexus/action.yml index e946f1040..b9b4acb7e 100644 --- a/.github/actions/setup-gitnexus/action.yml +++ b/.github/actions/setup-gitnexus/action.yml @@ -1,5 +1,5 @@ name: Setup GitNexus -description: Setup Node.js 20, install dependencies, and optionally build +description: Setup Node.js 22, install dependencies, and optionally build inputs: build: @@ -12,7 +12,7 @@ runs: steps: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: gitnexus/package-lock.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f3530e4d3..c99b666eb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,8 @@ updates: directory: / schedule: interval: weekly + cooldown: + default-days: 7 open-pull-requests-limit: 5 commit-message: prefix: chore @@ -15,6 +17,36 @@ updates: - dependencies - ci + # Keep pinned Docker base-image digests current for the root Dockerfiles. + - package-ecosystem: docker + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + - ci + + # Keep the nested test-image Docker base digest current as well. + - package-ecosystem: docker + directory: /gitnexus + schedule: + interval: weekly + cooldown: + default-days: 7 + open-pull-requests-limit: 5 + commit-message: + prefix: chore(deps) + include: scope + labels: + - dependencies + - ci + # Gitnexus npm deps — tree-sitter grammars checked daily so we catch # new releases that unblock the tree-sitter 0.25 upgrade ASAP. Grammars # are grouped so lockstep bumps produce a single PR. The tree-sitter @@ -25,6 +57,11 @@ updates: directory: /gitnexus schedule: interval: daily + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 open-pull-requests-limit: 10 commit-message: prefix: chore(deps) @@ -54,6 +91,11 @@ updates: directory: /gitnexus-web schedule: interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 open-pull-requests-limit: 5 commit-message: prefix: chore(deps) @@ -67,6 +109,11 @@ updates: directory: /gitnexus-shared schedule: interval: weekly + cooldown: + default-days: 7 + semver-major-days: 30 + semver-minor-days: 7 + semver-patch-days: 3 open-pull-requests-limit: 5 commit-message: prefix: chore(deps) diff --git a/.github/workflows/ci-quality.yml b/.github/workflows/ci-quality.yml index 36a936c82..cc334fcaa 100644 --- a/.github/workflows/ci-quality.yml +++ b/.github/workflows/ci-quality.yml @@ -11,7 +11,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: package-lock.json - run: npm ci @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 cache: npm cache-dependency-path: package-lock.json - run: npm ci diff --git a/.github/workflows/ci-report.yml b/.github/workflows/ci-report.yml index 03908bb2b..03c933ad0 100644 --- a/.github/workflows/ci-report.yml +++ b/.github/workflows/ci-report.yml @@ -95,31 +95,33 @@ jobs: # Validate PR number is a positive integer (artifact comes from # untrusted fork code, so treat contents defensively). - PR_NUM=$(cat "$DIR/pr_number" | tr -d '[:space:]') + PR_NUM=$(tr -d '[:space:]' < "$DIR/pr_number") if ! [[ "$PR_NUM" =~ ^[0-9]+$ ]]; then echo "skip=true" >> "$GITHUB_OUTPUT" echo "::error::Invalid PR number in artifact: '$PR_NUM'" exit 0 fi - echo "skip=false" >> "$GITHUB_OUTPUT" - echo "pr_number=$PR_NUM" >> "$GITHUB_OUTPUT" # Validate job-result strings against known GitHub Actions values. # Artifact contents come from the PR workflow (potentially untrusted # fork code), so we whitelist to prevent newline injection into # GITHUB_OUTPUT. validate_result() { local val - val=$(cat "$1" | tr -d '[:space:]') + val=$(tr -d '[:space:]' < "$1") case "$val" in success|failure|cancelled|skipped) echo "$val" ;; *) echo "unknown" ;; esac } - echo "quality=$(validate_result "$DIR/quality_result")" >> "$GITHUB_OUTPUT" - echo "tests=$(validate_result "$DIR/tests_result")" >> "$GITHUB_OUTPUT" - echo "e2e=$(validate_result "$DIR/e2e_result")" >> "$GITHUB_OUTPUT" + { + echo "skip=false" + echo "pr_number=$PR_NUM" + echo "quality=$(validate_result "$DIR/quality_result")" + echo "tests=$(validate_result "$DIR/tests_result")" + echo "e2e=$(validate_result "$DIR/e2e_result")" + } >> "$GITHUB_OUTPUT" - name: Checkout (for vitest config) if: steps.meta.outputs.skip != 'true' @@ -279,14 +281,17 @@ jobs: fi } - read CLI_T CLI_P CLI_F CLI_S CLI_SU CLI_D <<< "$(sum_results "$RESULTS_FILE")" - read WEB_T WEB_P WEB_F WEB_S WEB_SU WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")" + # `_` placeholder for the suite-count column — positional + # readability for sum_results' 6-field output, but the value + # isn't surfaced in the report (suites are tracked per-test + # framework, not as a top-line metric). + read -r CLI_T CLI_P CLI_F CLI_S _ CLI_D <<< "$(sum_results "$RESULTS_FILE")" + read -r WEB_T WEB_P WEB_F WEB_S _ WEB_D <<< "$(sum_results "$WEB_RESULTS_FILE")" TOTAL=$((CLI_T + WEB_T)) PASSED=$((CLI_P + WEB_P)) FAILED=$((CLI_F + WEB_F)) SKIPPED=$((CLI_S + WEB_S)) - SUITES=$((CLI_SU + WEB_SU)) DURATION=$((CLI_D > WEB_D ? CLI_D : WEB_D)) # ── Status helpers ── diff --git a/.github/workflows/pr-autofix-apply.yml b/.github/workflows/pr-autofix-apply.yml new file mode 100644 index 000000000..b2ec8495f --- /dev/null +++ b/.github/workflows/pr-autofix-apply.yml @@ -0,0 +1,595 @@ +name: PR Autofix (apply) + +# CHATOPS HALF of the autofix pipeline. +# +# Triggered when a contributor comments `/autofix` on a PR. Validates +# permission, locates the most recent successful `pr-autofix.yml` +# artifact for the PR's current head SHA, applies the patch to the PR +# head, and pushes a commit back to the PR branch. +# +# This workflow runs from the default branch's copy of the file +# regardless of where the comment originates -- that's the trust +# anchor. Comment body and author login are untrusted; both flow +# through env vars and pattern-matched, never interpolated into shell. +# +# Fork PR support: `git push` with the GITHUB_TOKEN succeeds against +# fork branches only when the contributor enabled "Allow edits by +# maintainers" on the PR (the default). When they disabled it, we +# fail loud with a 👎 reaction and an explanation comment. + +on: + issue_comment: + types: [created] + +concurrency: + # Per-PR scope. issue_comment events expose `github.event.issue.number` + # for both PR and Issue comments; the `pull_request != null` guard on + # the job ensures we only run on PRs, so this number is the PR number. + # cancel-in-progress: false — a second `/autofix` should wait for the + # first to finish (idempotency check on the second invocation handles + # the no-op case). + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }} + cancel-in-progress: false + +permissions: {} + +jobs: + apply: + name: apply-autofix + # Pre-filter at the workflow level so non-PR comments and unrelated + # comments don't even spawn a runner. The job-level body re-check + # below (Step 1) is the strict gate. + if: >- + github.event.issue.pull_request != null + && startsWith(github.event.comment.body, '/autofix') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + # React on the triggering comment + post reply comments. + pull-requests: write + # Push the apply commit to the PR head branch. + contents: write + # Required by actions/download-artifact to fetch artifacts produced + # by a different workflow run. + actions: read + steps: + - name: Validate comment body precisely + id: body + env: + BODY: ${{ github.event.comment.body }} + shell: bash + run: | + set -euo pipefail + # Whole-line, case-sensitive match: `^/autofix\s*$`. The + # workflow-level startsWith guard is coarse — `please don't + # /autofix this code` would pass that filter but fail this one. + # We exit silently (no reaction) on body mismatch so quoted + # text in unrelated discussions doesn't get a visible response. + if [[ ! "${BODY}" =~ ^/autofix[[:space:]]*$ ]]; then + echo "Body did not match strict /autofix regex — exiting silently." + echo "match=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "match=true" >> "$GITHUB_OUTPUT" + + - name: Validate commenter permission + id: perm + if: steps.body.outputs.match == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENTER: ${{ github.event.comment.user.login }} + PR_AUTHOR: ${{ github.event.issue.user.login }} + shell: bash + run: | + set -euo pipefail + + # Retry wrapper for transient 5xx / 429 / network blips. + # Mirrors the helper in pr-autofix-publish.yml. Used on + # idempotent GETs only; reactions/comment-POSTs are NOT + # wrapped (retrying a POST would dupe the resource). + gh_retry() { + local n=0 max=3 + while true; do + if gh "$@"; then return 0; fi + n=$((n+1)) + if [ "$n" -ge "$max" ]; then return 1; fi + sleep $((n * 2)) + done + } + + # Allowlist the commenter login before it flows into a URL. + # GitHub usernames: alphanumeric + dashes, max 39 chars. + if ! [[ "${COMMENTER}" =~ ^[A-Za-z0-9-]{1,39}$ ]]; then + echo "::error::Invalid commenter login format: $(printf '%q' "${COMMENTER}")" + echo "allowed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Self-comparison: PR author can always /autofix their own PR. + if [ "${COMMENTER}" = "${PR_AUTHOR}" ]; then + echo "Commenter is PR author — granting access." + echo "allowed=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Repo permission lookup. admin/write/maintain are sufficient. + # Distinguish API failure (5xx, 429, network) from genuine + # permission denial (404 = not a collaborator). Conflating them + # would silently refuse a legitimate maintainer with a public + # 👎 every time GitHub blips. gh_retry handles transient blips; + # the stderr-grep distinguishes 404 from persistent failure. + perm_stderr=$(mktemp) + if permission=$(gh_retry api "repos/${GH_REPO}/collaborators/${COMMENTER}/permission" \ + --jq '.permission' 2>"$perm_stderr"); then + echo "Commenter permission: ${permission}" + case "${permission}" in + admin|write|maintain) + echo "allowed=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "allowed=false" >> "$GITHUB_OUTPUT" + ;; + esac + else + err=$(cat "$perm_stderr") + echo "Permission lookup stderr: ${err}" >&2 + # 404 (not a collaborator) is a genuine deny. + # Anything else is a transient API/network failure. + if grep -qE "HTTP 404|Not Found" "$perm_stderr"; then + echo "allowed=false" >> "$GITHUB_OUTPUT" + else + echo "::error::Permission lookup failed transiently — refusing to act." + echo "allowed=api-failed" >> "$GITHUB_OUTPUT" + fi + fi + + - name: React 😕 on transient permission-API failure + if: steps.body.outputs.match == 'true' && steps.perm.outputs.allowed == 'api-failed' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Couldn't verify your repo permission (transient GitHub API failure). Please comment \`/autofix\` again. ([apply run](https://github.com/${GH_REPO}/actions/runs/${RUN_ID}))" \ + >/dev/null + exit 1 + + - name: React 👎 on permission denial + if: steps.body.outputs.match == 'true' && steps.perm.outputs.allowed == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + shell: bash + run: | + set -euo pipefail + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🚫 \`/autofix\` is restricted to users with write access or the PR author. Comment ignored." \ + >/dev/null + # Hard exit so the rest of the job is skipped. + exit 1 + + - name: React 👀 to acknowledge + if: steps.perm.outputs.allowed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + shell: bash + run: | + set -euo pipefail + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="eyes" >/dev/null + + - name: Resolve PR head and locate autofix run + id: locate + if: steps.perm.outputs.allowed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + shell: bash + run: | + set -euo pipefail + + # Same retry wrapper used in the permission step, repeated + # because each YAML `run:` block is a fresh bash session. + gh_retry() { + local n=0 max=3 + while true; do + if gh "$@"; then return 0; fi + n=$((n+1)) + if [ "$n" -ge "$max" ]; then return 1; fi + sleep $((n * 2)) + done + } + + # Fetch PR metadata. All fields here are server-controlled API + # output, but we still allowlist before exporting so anything + # weird short-circuits before $GITHUB_OUTPUT. Wrapped in + # gh_retry so transient blips don't surface as "no autofix run + # found" with a wrong remediation. + if ! pr_json=$(gh_retry api "repos/${GH_REPO}/pulls/${PR}"); then + echo "::error::PR metadata fetch failed after retries." + echo "found_status=api-failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + head_sha=$(jq -r '.head.sha' <<< "${pr_json}") + head_ref=$(jq -r '.head.ref' <<< "${pr_json}") + head_repo=$(jq -r '.head.repo.full_name' <<< "${pr_json}") + + [[ "${head_sha}" =~ ^[0-9a-f]{40}$ ]] || { echo "::error::Bad head_sha"; exit 1; } + [[ "${head_ref}" =~ ^[A-Za-z0-9._/-]+$ ]] || { echo "::error::Bad head_ref"; exit 1; } + [[ "${head_repo}" =~ ^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$ ]] || { echo "::error::Bad head_repo"; exit 1; } + + # Find the latest successful pr-autofix.yml run for this head SHA. + if ! runs_json=$(gh_retry api "repos/${GH_REPO}/actions/workflows/pr-autofix.yml/runs?head_sha=${head_sha}&per_page=10"); then + echo "::error::Workflow run lookup failed after retries." + echo "found_status=api-failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + + run_id=$(jq -r '[.workflow_runs[] | select(.conclusion == "success")] | .[0].id // empty' <<< "${runs_json}") + + if [ -n "${run_id}" ] && [[ "${run_id}" =~ ^[0-9]+$ ]]; then + echo "found_status=success" >> "$GITHUB_OUTPUT" + { + echo "found=true" + echo "head_sha=${head_sha}" + echo "head_ref=${head_ref}" + echo "head_repo=${head_repo}" + echo "run_id=${run_id}" + } >> "$GITHUB_OUTPUT" + exit 0 + fi + + # No successful run. Distinguish "still running" (producer in + # flight after a recent push) from "never ran / all failed". + # in_progress / queued / pending / waiting cover the GitHub + # workflow-run lifecycle states that precede success/failure. + in_progress=$(jq -r '[.workflow_runs[] | select(.status == "in_progress" or .status == "queued" or .status == "pending" or .status == "waiting")] | length' <<< "${runs_json}") + if [ "${in_progress:-0}" -gt 0 ]; then + echo "::warning::pr-autofix run is still in progress for head ${head_sha}." + echo "found_status=in-progress" >> "$GITHUB_OUTPUT" + else + echo "::warning::No successful pr-autofix run found for head ${head_sha}." + echo "found_status=not-found" >> "$GITHUB_OUTPUT" + fi + # Existing `found` boolean is preserved so downstream gates + # (`steps.locate.outputs.found == 'true'`) still work. + echo "found=false" >> "$GITHUB_OUTPUT" + + - name: Reply when locate did not yield a usable run + if: steps.perm.outputs.allowed == 'true' && steps.locate.outputs.found != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + FOUND_STATUS: ${{ steps.locate.outputs.found_status }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + run_url="https://github.com/${GH_REPO}/actions/runs/${RUN_ID}" + case "${FOUND_STATUS}" in + in-progress) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⏳ A pr-autofix run is still in progress for this PR's current head SHA. Wait for it to finish, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + ;; + api-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Couldn't reach the GitHub API to look up the autofix run (transient failure after retries). Please comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + ;; + *) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🤔 No successful autofix run found for this PR's current head SHA. Push a new commit to trigger one, then comment \`/autofix\` again." \ + >/dev/null + ;; + esac + exit 1 + + # Pinned to v8.0.1. Same SHA as pr-autofix-publish.yml. + # `continue-on-error: true` lets the workflow proceed when the + # artifact is expired or pruned (1-day retention). The apply + # step distinguishes "patch file missing entirely" (artifact- + # expired) from "patch file zero bytes" (genuinely empty patch). + - name: Download autofix artifact + if: steps.locate.outputs.found == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + continue-on-error: true + with: + name: autofix + run-id: ${{ steps.locate.outputs.run_id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: autofix-in + + # Pinned to v5.0.4. Verify SHA via: + # gh api repos/actions/checkout/git/refs/tags/v5.0.4 + # + # `persist-credentials: false` disables the default behavior where + # actions/checkout writes the GITHUB_TOKEN into `.git/config` as an + # extraheader. That default is convenient (subsequent git commands + # auth automatically) but it means the token is sitting on disk in + # the checkout directory — an `actions/upload-artifact` step on + # this directory would leak the token. We don't upload, but + # zizmor's `credential-persistence` lint flags it defensively. + # Push auth is provided inline at push time via the URL. + - name: Checkout PR head + if: steps.locate.outputs.found == 'true' + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.4 + with: + repository: ${{ steps.locate.outputs.head_repo }} + ref: ${{ steps.locate.outputs.head_sha }} + token: ${{ secrets.GITHUB_TOKEN }} + persist-credentials: false + # Fetch full history so the push doesn't hit shallow-clone errors. + fetch-depth: 0 + path: pr-checkout + + - name: Apply patch and push + id: apply + if: steps.locate.outputs.found == 'true' + env: + HEAD_REF: ${{ steps.locate.outputs.head_ref }} + HEAD_REPO: ${{ steps.locate.outputs.head_repo }} + # The SHA we resolved earlier in `locate` — this is what the + # remote ref MUST still equal at push time. If the contributor + # force-pushed between resolve and now, the lease fails and + # we surface that distinctly from a fork-without-maintainer + # -edit push failure. + HEAD_SHA: ${{ steps.locate.outputs.head_sha }} + # Auth for the push only — never persisted to disk. Provided + # via env to avoid interpolating into the shell command line. + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: bash + working-directory: pr-checkout + run: | + set -euo pipefail + patch="../autofix-in/autofix.patch" + + # Distinguish artifact-expired (file missing entirely, because + # actions/download-artifact ran with continue-on-error and the + # 1-day retention had elapsed) from genuinely empty patch + # (file present, zero bytes, formatter found nothing). + if [ ! -e "$patch" ]; then + echo "::warning::Patch file does not exist — autofix artifact likely expired." + echo "result=artifact-expired" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ ! -s "$patch" ]; then + echo "::warning::Empty patch — nothing to apply." + echo "result=empty-patch" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Sensitive-paths guard: refuse to apply patches that touch + # `.github/` — workflow files, action definitions, CODEOWNERS, + # dependabot config, etc. A malicious PR could ship a custom + # prettier/ESLint config that reformats workflow YAML; the + # producer would then capture those edits in autofix.patch, + # and a maintainer running `/autofix` would push them under + # `contents: write`. The default GITHUB_TOKEN lacks `workflows` + # scope so the platform would reject workflow-file pushes + # anyway, but that surfaces as a generic `push-failed` and + # misleads users into enabling maintainer-edit. Reject early + # with a specific reason. CODEOWNERS and dependabot.yml live + # under .github/ but outside .github/workflows/ — the broader + # match is intentional (they all govern trust boundaries). + if grep -qE '^(diff --git|---|\+\+\+) [ab]?/?\.github/' "$patch"; then + echo "::warning::Patch touches .github/ — refusing to apply (sensitive paths)." + echo "result=sensitive-paths" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Re-entrancy guard: if HEAD itself is an autofix bot commit, + # refuse to apply again. Without this, lint/formatter config + # drift between runs could pump arbitrary apply commits into + # the same PR if an automated agent watches the sticky and + # re-fires `/autofix` on each new "fixes-available" surface. + # The contributor can still get out by force-pushing a + # human-authored commit to revert the autofix and re-trigger. + head_author=$(git log -1 --format='%ae' HEAD) + head_subject=$(git log -1 --format='%s' HEAD) + if [ "${head_author}" = "41898282+github-actions[bot]@users.noreply.github.com" ] \ + && [[ "${head_subject}" =~ ^chore\(autofix\) ]]; then + echo "::warning::HEAD is an autofix bot commit — refusing to re-apply (loop guard)." + echo "result=loop-prevented" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Idempotency probe: does the forward apply work? + if git apply --check "$patch" 2>/dev/null; then + echo "Patch applies cleanly — proceeding." + elif git apply --check --reverse "$patch" 2>/dev/null; then + # Reverse-check passes => the patch is already applied to + # the current tree. Treat as success no-op. + echo "Patch is already applied (reverse-check passed) — no-op." + echo "result=already-applied" >> "$GITHUB_OUTPUT" + exit 0 + else + echo "::error::Patch does not apply (stale or conflicting)." + echo "result=stale" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Wrap the apply/commit phase so any non-zero exit sets a + # meaningful `result=` instead of leaving it unset (which would + # send the user to the `*` "unexpected state" arm with a + # non-actionable confused-emoji reply). + if ! { + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" && + git config user.name "github-actions[bot]" && + git apply "$patch" && + git add -A && + git commit -m "chore(autofix): apply prettier + eslint fixes via /autofix command" + }; then + echo "::error::git apply / config / commit failed after idempotency probe passed." + echo "result=apply-failed" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Push to the PR head branch with a lease against the resolved + # SHA. The lease ensures the remote ref still points at HEAD_SHA + # when the push lands — if the contributor force-pushed in the + # window between resolve and now, the lease fails and we return + # `lease-failed` (NOT `push-failed`, which would mislead users + # into enabling maintainer-edit). For fork PRs, the push still + # requires "Allow edits by maintainers" to be enabled. + # + # Auth is supplied inline via `-c http..extraheader` (NOT + # via a `https://x-access-token:TOKEN@…` URL — those leak into + # process listings and `git remote -v` output). The header is + # set per-invocation; it never lands in `.git/config` on disk. + # The token is base64-encoded for the Basic auth header per + # GitHub's documented pattern for this scope. + push_url="https://github.com/${HEAD_REPO}.git" + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 -w0)" + # GitHub's secret-masker only masks the raw token, not its + # base64-encoded form. Mask the encoded value so any subsequent + # log line (set -x, GIT_TRACE, error spew) gets ***-redacted. + echo "::add-mask::${auth_header}" + push_stderr=$(mktemp) + if git -c http.extraheader="${auth_header}" \ + push --force-with-lease="refs/heads/${HEAD_REF}:${HEAD_SHA}" \ + "${push_url}" "HEAD:${HEAD_REF}" 2>"$push_stderr"; then + echo "result=applied" >> "$GITHUB_OUTPUT" + else + cat "$push_stderr" >&2 + # `--force-with-lease` reports "stale info" when the remote + # ref has moved past the expected SHA. Other lease-failure + # phrases git emits include "remote rejected" (server-side + # reject), "non-fast-forward", and the literal flag name. Match + # any of those to distinguish from auth/network/maintainer- + # edit failures. + if grep -qE "stale info|force-with-lease|rejected.*non-fast-forward|remote rejected|! \[rejected\]" "$push_stderr"; then + echo "::error::git push lease failed — branch moved during apply." + echo "result=lease-failed" >> "$GITHUB_OUTPUT" + else + echo "::error::git push failed — likely fork without maintainer-edit enabled." + echo "result=push-failed" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + - name: React and reply on outcome + if: always() && steps.locate.outputs.found == 'true' && steps.apply.outcome != 'skipped' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + COMMENT_ID: ${{ github.event.comment.id }} + PR: ${{ github.event.issue.number }} + RESULT: ${{ steps.apply.outputs.result }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + run_url="https://github.com/${GH_REPO}/actions/runs/${RUN_ID}" + + case "${RESULT}" in + applied) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="+1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="✅ Applied autofix and pushed a commit. ([apply run](${run_url}))" \ + >/dev/null + ;; + already-applied) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="+1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="✅ Autofix is already applied — no changes needed." \ + >/dev/null + ;; + empty-patch) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="+1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="✅ No autofix to apply — formatter found nothing." \ + >/dev/null + ;; + artifact-expired) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⏳ The autofix artifact for this PR's head SHA has expired (1-day retention). Push a new commit to regenerate it, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + loop-prevented) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🔁 Refusing to re-apply autofix on top of an existing autofix commit. If formatter rules drifted and you genuinely need another pass, push a human-authored commit (or revert the existing autofix commit) before commenting \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + sensitive-paths) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="🛑 Refusing to apply: the autofix patch touches files under \`.github/\` (workflow / CODEOWNERS / dependabot config). Apply formatter changes to those files manually in a regular commit so they get human review. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + stale) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ The autofix patch is stale or conflicts with the current head — push a new commit to regenerate, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + apply-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Autofix applied cleanly in the dry run, but \`git apply\` / \`git commit\` failed when actually landing the patch. This usually means a race with concurrent edits or a corrupt patch. See logs: ${run_url}" \ + >/dev/null + exit 1 + ;; + push-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ Couldn't push the autofix commit. If this is a fork PR, please tick **Allow edits by maintainers** in the PR sidebar, then comment \`/autofix\` again. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + lease-failed) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="-1" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="⚠️ The PR head moved while autofix was applying — a new commit landed in the window between resolve and push. Comment \`/autofix\` again to retry against the latest head. ([apply run](${run_url}))" \ + >/dev/null + exit 1 + ;; + *) + gh api -X POST "repos/${GH_REPO}/issues/comments/${COMMENT_ID}/reactions" \ + -f content="confused" >/dev/null + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="❓ Autofix run finished in an unexpected state (\`${RESULT:-unknown}\`). See logs: ${run_url}" \ + >/dev/null + exit 1 + ;; + esac diff --git a/.github/workflows/pr-autofix-publish.yml b/.github/workflows/pr-autofix-publish.yml new file mode 100644 index 000000000..08ad1d60f --- /dev/null +++ b/.github/workflows/pr-autofix-publish.yml @@ -0,0 +1,316 @@ +name: PR Autofix (publish) + +# TRUSTED HALF of the autofix pipeline. +# +# Triggered by `pr-autofix.yml` completing on a PR (including fork PRs). +# Downloads the diff artifact produced by the untrusted job, verifies +# its claimed PR identity against the workflow_run authority, then +# posts (or edits) a single sticky summary comment plus a +# `gitnexus/autofix` Check Run. This job NEVER checks out fork code — +# it only consumes the diff (data) and calls the GitHub API. That +# isolation is what makes it safe to run under `pull-requests: write` +# on fork-triggered events. +# +# The sticky comment is the contributor signal: heading +# "## :sparkles: PR Autofix" in the PR's top-level comments, with a +# fenced `gitnexus-autofix` JSON block carrying machine-readable state +# for AI agents. Contributors apply the patch by commenting `/autofix` +# on the PR — handled by the separate `pr-autofix-apply.yml` workflow. + +on: + workflow_run: + workflows: ['PR Autofix'] + types: [completed] + +concurrency: + # Key on PR identity, NOT workflow_run.id — workflow_run.id is per-run + # unique, which would defeat serialization and let two parallel + # publishes both POST a sticky summary comment. CONTRIBUTING.md + # § GitHub Actions — Concurrency Convention names this anti-pattern + # explicitly. For fork PRs, `pull_requests[]` is empty in the + # workflow_run payload, so we fall back to head-repo + head-branch. + group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || format('{0}/{1}', github.event.workflow_run.head_repository.full_name, github.event.workflow_run.head_branch) }} + cancel-in-progress: false + +permissions: {} + +jobs: + publish: + name: publish-autofix + if: >- + github.event.workflow_run.event == 'pull_request' + && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + pull-requests: write + # Required by actions/download-artifact to fetch artifacts produced + # by a different workflow run. + actions: read + # Required to create the `gitnexus/autofix` Check Run that reports + # the outcome (clean / fixes-available) to the PR's Checks tab. + # Branch protection or agents can grep the conclusion + output + # title without parsing the sticky comment. + checks: write + steps: + # Pinned to v8.0.1. Verify SHA via: + # gh api repos/actions/download-artifact/git/refs/tags/v8.0.1 + - name: Download autofix artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: autofix + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + path: autofix-in + + - name: Read and validate metadata + id: meta + shell: bash + run: | + set -euo pipefail + test -f autofix-in/metadata.json + jq . autofix-in/metadata.json + + # The artifact comes from the untrusted half running fork code. + # Every field is allowlist-validated before it can flow into + # $GITHUB_OUTPUT. A newline in head_ref would otherwise let a + # malicious branch name inject a second `pr_number=N` line and + # redirect this job's reviewdog suggestions / sticky summary + # comment onto a victim PR under github-actions[bot] with + # pull-requests: write. + assert_field() { + local key="$1" pattern="$2" value + value=$(jq -r ".${key} // empty" autofix-in/metadata.json) + if [ -z "$value" ] || ! [[ "$value" =~ $pattern ]]; then + echo "::error::metadata.${key} failed allowlist (got: $(printf '%q' "$value"))" + exit 1 + fi + printf '%s' "$value" + } + + SCHEMA=$(assert_field schema '^gitnexus\.pr-autofix/v[0-9]+$') + PR_NUMBER=$(assert_field pr_number '^[0-9]+$') + HEAD_SHA=$(assert_field head_sha '^[0-9a-f]{40}$') + HEAD_REF=$(assert_field head_ref '^[A-Za-z0-9._/-]+$') + HEAD_REPO=$(assert_field head_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') + BASE_REPO=$(assert_field base_repo '^[A-Za-z0-9._-]+/[A-Za-z0-9._-]+$') + CHANGED=$(assert_field changed_lines '^[0-9]+$') + + # Defence-in-depth: refuse to act if the artifact claims to + # belong to a different repo than the one that triggered us. + if [ "$BASE_REPO" != "${GITHUB_REPOSITORY}" ]; then + echo "::error::Artifact base_repo does not match \$GITHUB_REPOSITORY — refusing to publish." + exit 1 + fi + + { + echo "schema=${SCHEMA}" + echo "pr_number=${PR_NUMBER}" + echo "head_sha=${HEAD_SHA}" + echo "head_ref=${HEAD_REF}" + echo "head_repo=${HEAD_REPO}" + echo "base_repo=${BASE_REPO}" + echo "changed_lines=${CHANGED}" + } >> "$GITHUB_OUTPUT" + + # Cross-verify the artifact's claimed identity against the + # GitHub-controlled workflow_run event. The previous step's + # allowlist only proves the fields are well-formed — not that + # they refer to the PR/SHA that actually triggered this run. + # A fork-controlled `npm run lint:fix` could plausibly mutate + # metadata.json to reference another PR or SHA, redirecting our + # write-scoped sticky/check-run onto an attacker-chosen target. + # + # Authority sources are all server-controlled GitHub event fields: + # - workflow_run.head_sha + # - workflow_run.head_repository.full_name + # - workflow_run.pull_requests[].number (within-repo PRs only; + # empty array on fork PRs — fall back to commits/{sha}/pulls) + # + # Mismatch => fail loud BEFORE any sticky/check-run side effect. + - name: Verify metadata against workflow_run authority + id: verify + if: steps.meta.outputs.changed_lines != '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + META_PR_NUMBER: ${{ steps.meta.outputs.pr_number }} + META_HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + META_HEAD_REPO: ${{ steps.meta.outputs.head_repo }} + WF_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + WF_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + WF_PR_NUMBERS: ${{ toJSON(github.event.workflow_run.pull_requests.*.number) }} + shell: bash + run: | + set -euo pipefail + + # 1) head_sha must match exactly. workflow_run.head_sha is the + # commit GitHub actually ran the producer against — definitive. + if [ "${META_HEAD_SHA}" != "${WF_HEAD_SHA}" ]; then + echo "::error::Artifact head_sha (${META_HEAD_SHA}) does not match workflow_run.head_sha (${WF_HEAD_SHA}) — refusing to publish." + exit 1 + fi + + # 2) head_repo must match exactly. Same authority anchor. + if [ "${META_HEAD_REPO}" != "${WF_HEAD_REPO}" ]; then + echo "::error::Artifact head_repo (${META_HEAD_REPO}) does not match workflow_run.head_repository (${WF_HEAD_REPO}) — refusing to publish." + exit 1 + fi + + # 3) pr_number must reference an open PR with this head SHA. + # Within-repo PRs: workflow_run.pull_requests[] is populated. + # Fork PRs: that array is empty by GitHub design — fall back + # to the REST commit-to-PRs lookup. Fail closed if the lookup + # finds no matching open PR (avoids attacker-forged PR ids). + allowed_numbers=$(jq -c '.' <<< "${WF_PR_NUMBERS}") + if [ "${allowed_numbers}" = "[]" ]; then + echo "workflow_run.pull_requests is empty (fork PR) — falling back to commits/{sha}/pulls." + allowed_numbers=$(gh api "repos/${GH_REPO}/commits/${WF_HEAD_SHA}/pulls" \ + --jq '[.[] | select(.state == "open") | .number]' 2>/dev/null || echo "[]") + if [ "${allowed_numbers}" = "[]" ]; then + echo "::error::No open PR found for head ${WF_HEAD_SHA} via commits/{sha}/pulls — refusing to publish." + exit 1 + fi + fi + + if ! jq -e --argjson n "${META_PR_NUMBER}" 'index($n) != null' <<< "${allowed_numbers}" >/dev/null; then + echo "::error::Artifact pr_number (${META_PR_NUMBER}) is not in the authoritative PR list (${allowed_numbers}) — refusing to publish." + exit 1 + fi + + echo "Verified: metadata identity matches workflow_run authority (PR=${META_PR_NUMBER}, head_sha=${META_HEAD_SHA}, head_repo=${META_HEAD_REPO})." + + - name: Upsert sticky summary comment + # Only post when ci-quality found something fixable (= the + # autofix patch is non-empty). When prettier/eslint are clean + # the patch is zero bytes and the sticky comment is pure noise, + # so we skip it. + if: >- + always() + && steps.meta.outputs.pr_number != '' + && steps.meta.outputs.changed_lines != '0' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + PR: ${{ steps.meta.outputs.pr_number }} + CHANGED: ${{ steps.meta.outputs.changed_lines }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + RUN_ID: ${{ github.run_id }} + shell: bash + run: | + set -euo pipefail + + # Stable heading + marker — agents grep for these exact strings. + marker="" + heading="## :sparkles: PR Autofix" + + # Single state. The /autofix slash command works for any diff + # size — there's no 3K cap and no no-overlap dead-end because + # the apply workflow uses `git apply` + push, not the GitHub + # review-comment API. + ui_state="fixes-available" + prose="Found fixable formatting / unused-import issues across **${CHANGED}** changed lines. **Comment \`/autofix\` on this PR to apply them**, or run \`npm run lint:fix && npm run format\` locally." + + # Machine-readable JSON block — agents parse this instead of + # regexing English. Fenced code-block info string is + # `gitnexus-autofix` so agents can locate it without ambiguity. + # Schema bumped from v1 -> v2: adds `apply_command`. The v1 + # field set is preserved as a superset, but the `state` enum + # is redefined (v1: suggestions-posted | skipped-too-large | + # diff-no-overlap; v2: fixes-available). v1 readers checking + # `schema == 'gitnexus.pr-autofix/v1'` see an unfamiliar version + # and fall back to prose, which is the intended migration path. + json=$(jq -n -c \ + --arg state "${ui_state}" \ + --argjson pr_number "${PR}" \ + --argjson changed_lines "${CHANGED}" \ + --arg head_sha "${HEAD_SHA}" \ + --arg run_id "${RUN_ID}" \ + '{schema:"gitnexus.pr-autofix/v2", state:$state, pr_number:$pr_number, changed_lines:$changed_lines, head_sha:$head_sha, run_id:$run_id, apply_command:"/autofix"}') + + # Multi-line quoted string instead of a column-0 heredoc — YAML's + # `run: |` block ends as soon as a content line dedents below the + # block's first-line indent, which would mis-parse the workflow. + body="${marker} + ${heading} + + ${prose} + + \`\`\`gitnexus-autofix + ${json} + \`\`\`" + # Strip the leading 10-space indent that the YAML block requires + # so the rendered comment body starts at column 0. + body="$(printf '%s\n' "$body" | sed 's/^ //')" + + # Small retry wrapper for transient 5xx / rate-limit responses + # on the GitHub REST API. Three tries with linear backoff. We + # only retry GET (idempotent) and PATCH on a known comment id + # (idempotent). POST is NOT wrapped — retrying a comment-create + # would create duplicates if the first attempt actually landed. + gh_retry() { + local n=0 max=3 + while true; do + if gh "$@"; then return 0; fi + n=$((n+1)) + if [ "$n" -ge "$max" ]; then return 1; fi + sleep $((n * 2)) + done + } + + # Find existing bot comment by the marker and edit-in-place; else create. + # CRITICAL: filter by `.user.login == "github-actions[bot]"`. A regular + # user posting a comment containing the marker would otherwise be the + # `head -n1` match; PATCH on someone else's comment 403s, `set -e` + # aborts, and the bot is permanently DoS'd for that PR. + existing=$(gh_retry api "repos/${GH_REPO}/issues/${PR}/comments" \ + --paginate --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | contains(\"${marker}\"))) | .id" \ + | head -n1 || true) + + if [ -n "${existing}" ]; then + gh_retry api -X PATCH "repos/${GH_REPO}/issues/comments/${existing}" \ + -f body="${body}" >/dev/null + echo "Updated comment ${existing}." + else + gh api -X POST "repos/${GH_REPO}/issues/${PR}/comments" \ + -f body="${body}" >/dev/null + echo "Created summary comment." + fi + + - name: Emit gitnexus/autofix Check Run + # Stable check name `gitnexus/autofix` so PR-watching agents can + # `gh pr checks ` and read the conclusion + title without + # parsing the sticky comment. Two outcomes: + # clean → conclusion: success + # fixes-available → conclusion: neutral + # `neutral` does not block branch-protection required-checks but + # is visually distinct from a green pass. + if: always() && steps.meta.outputs.head_sha != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD_SHA: ${{ steps.meta.outputs.head_sha }} + CHANGED: ${{ steps.meta.outputs.changed_lines }} + shell: bash + run: | + set -euo pipefail + + if [ "${CHANGED}" = "0" ]; then + conclusion="success" + title="Formatting clean" + summary="Prettier and ESLint --fix produced no changes." + else + conclusion="neutral" + title="Autofix available — comment /autofix to apply" + summary="Comment \`/autofix\` on this PR to apply formatter + unused-import fixes (works at any diff size). Or run \`npm run lint:fix && npm run format\` locally." + fi + + gh api -X POST "repos/${GH_REPO}/check-runs" \ + -f name="gitnexus/autofix" \ + -f head_sha="${HEAD_SHA}" \ + -f status="completed" \ + -f conclusion="${conclusion}" \ + -f "output[title]=${title}" \ + -f "output[summary]=${summary}" \ + >/dev/null + echo "Posted check-run gitnexus/autofix=${conclusion} (${title})" diff --git a/.github/workflows/pr-autofix.yml b/.github/workflows/pr-autofix.yml new file mode 100644 index 000000000..04eb2468d --- /dev/null +++ b/.github/workflows/pr-autofix.yml @@ -0,0 +1,146 @@ +name: PR Autofix + +# UNTRUSTED HALF of the autofix pipeline. +# +# Runs `npm run lint:fix` + `npm run format` against the PR head +# (including fork heads) and uploads the resulting diff as an artifact. +# This job has NO privileged token and CANNOT post to the PR. The trusted +# `pr-autofix-publish.yml` workflow downloads the artifact via +# `workflow_run` and posts a sticky summary comment + Check Run. +# Contributors apply the patch by commenting `/autofix` on the PR — +# handled by the separate `pr-autofix-apply.yml` ChatOps workflow. +# +# Why the split: +# ESLint loads plugins from fork-controlled `node_modules`, so running +# it in a job with `pull-requests: write` would let a malicious fork PR +# ship a poisoned eslint plugin and execute arbitrary code under that +# token. By keeping fork code execution in this job (token: read-only) +# and posting from a separate trusted job that never touches fork +# code, we get the autofix UX for fork PRs without the supply-chain +# hole. (See autofix.ci for the same pattern.) +# +# Removes unused imports via `eslint-plugin-unused-imports`, already in +# devDependencies and wired into the `lint` config. + +on: + pull_request: + types: [opened, synchronize, reopened] + # Skip lockfile / generated-file PRs entirely — `action-suggester` + # cannot post on diffs > ~3k lines (GitHub returns 406) and these + # paths produce massive diffs no human wants suggested back inline. + paths-ignore: + - '**/package-lock.json' + - '**/*.snap' + - '**/dist/**' + - '**/node_modules/**' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + # Don't cancel in-flight runs; the publish workflow may already be + # downloading the artifact and a cancelled untrusted run produces no + # signal at all (worse DX than waiting). + cancel-in-progress: false + +# This workflow runs untrusted fork code. Top-level deny-all and NO +# job-level grants — the job can only read its own checkout. +permissions: {} + +jobs: + autofix: + name: autofix + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + # PR head commit (not the synthetic merge ref) — we need the + # exact tree the contributor pushed so suggestions line up. + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + cache: npm + cache-dependency-path: package-lock.json + + # `--ignore-scripts` blocks pre/postinstall lifecycle hooks. ESLint + # plugins still load from node_modules (that is the actual escape + # hatch on a typical fork), but this job has no token to abuse — + # which is the whole point of the split. + - run: npm ci --ignore-scripts + + - name: ESLint --fix (removes unused imports) + run: npm run lint:fix + # Lint errors that --fix can't auto-resolve must not block the + # diff artifact — partial fixes are still useful as suggestions. + continue-on-error: true + + - name: Prettier --write + run: npm run format + continue-on-error: true + + - name: Capture diff and metadata + id: capture + # Pass GitHub-context values via env: rather than `${{ }}` + # interpolated directly into the bash body. `head.ref` and + # `head.repo.full_name` are fork-controlled strings; expanding + # them into shell source is the canonical template-injection + # vector zizmor flags. Even though this job has `permissions: {}`, + # routing through env: makes it impossible for a future scope + # grant to turn into RCE. Inside bash, reference as `$HEAD_REF` + # etc. — the values are then plain strings, not code. + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + BASE_REPO: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + mkdir -p autofix-out + + # Produce a unified diff of the working tree vs. the PR head. + # Empty diff => nothing to suggest; the publish job short-circuits. + git diff --no-color > autofix-out/autofix.patch + + # NOTE: `changed_lines` is the line-count of the patch file, + # (hunk headers + context lines + added/removed). Surfaced in + # the sticky comment so contributors and AI agents have a + # quick size hint before invoking `/autofix`. + changed_lines=$(wc -l < autofix-out/autofix.patch | tr -d ' ') + echo "changed_lines=${changed_lines}" >> "$GITHUB_OUTPUT" + + # Carry PR identity over to the trusted job. workflow_run + # context is base-repo-only, so the publish job needs these + # to call the GitHub PR API on the right resource. + # CONTRACT: keep this schema in sync with pr-autofix-publish.yml's + # `assert_field` validators and the agent-facing JSON block in + # the sticky comment. Bump `schema` when changing field names. + jq -n \ + --arg schema 'gitnexus.pr-autofix/v1' \ + --argjson pr_number "${PR_NUMBER}" \ + --arg head_sha "${HEAD_SHA}" \ + --arg head_ref "${HEAD_REF}" \ + --arg head_repo "${HEAD_REPO}" \ + --arg base_repo "${BASE_REPO}" \ + --argjson changed_lines "${changed_lines}" \ + '{schema:$schema, pr_number:$pr_number, head_sha:$head_sha, head_ref:$head_ref, head_repo:$head_repo, base_repo:$base_repo, changed_lines:$changed_lines}' \ + > autofix-out/metadata.json + + echo "--- metadata ---" + cat autofix-out/metadata.json + echo "--- diff (head) ---" + head -c 2000 autofix-out/autofix.patch || true + + # Pinned to v7.0.1. Verify SHA via: + # gh api repos/actions/upload-artifact/git/refs/tags/v7.0.1 + - name: Upload autofix artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: autofix + path: autofix-out/ + retention-days: 1 + if-no-files-found: error diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d372c163a..d5af63205 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,7 +35,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 registry-url: https://registry.npmjs.org # Hermetic install for the published artifact — no cache carry-over # from non-tag contexts. setup-node v5+ caches by default when a diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml index 9ab00c0bd..ff75bca26 100644 --- a/.github/workflows/release-candidate.yml +++ b/.github/workflows/release-candidate.yml @@ -149,7 +149,7 @@ jobs: - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 20 + node-version: 22 registry-url: https://registry.npmjs.org # Hermetic install — release-candidate produces shipped artifacts. # setup-node v5+ caches by default when a packageManager field is @@ -294,9 +294,11 @@ jobs: fi fi - echo "base=$BASE" >> "$GITHUB_OUTPUT" - echo "rc_n=$NEXT_N" >> "$GITHUB_OUTPUT" - echo "rc_version=$RC_VERSION" >> "$GITHUB_OUTPUT" + { + echo "base=$BASE" + echo "rc_n=$NEXT_N" + echo "rc_version=$RC_VERSION" + } >> "$GITHUB_OUTPUT" - name: Apply rc version in-CI shell: bash @@ -354,9 +356,11 @@ jobs: # remote ref, the push fails and we stop before npm publish. git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" - echo "vtag=$VTAG" >> "$GITHUB_OUTPUT" - echo "marker=$MARKER" >> "$GITHUB_OUTPUT" - echo "release_sha=$RELEASE_SHA" >> "$GITHUB_OUTPUT" + { + echo "vtag=$VTAG" + echo "marker=$MARKER" + echo "release_sha=$RELEASE_SHA" + } >> "$GITHUB_OUTPUT" - name: Publish to npm (rc dist-tag) run: npm publish --provenance --access public --tag rc diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index f476ee7bc..f7fba75e9 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -1,13 +1,19 @@ name: Trivy Image Scan # Builds Dockerfile.cli and Dockerfile.web, then scans the resulting images -# for OS-package and language-package CVEs at HIGH/CRITICAL severity. +# for OS-package and language-package CVEs at MEDIUM+ severity. # Findings upload to the Security tab; record-only (does not block merges). # -# NOT triggered on PRs — image builds are slow and base-image CVE churn -# shouldn't gate feature delivery. +# Trigger on Dockerfile changes in PRs so base-image/npm-layer remediation can +# be verified before merge without running image scans on every PR. on: + pull_request: + paths: + - 'Dockerfile.cli' + - 'Dockerfile.web' + - 'gitnexus/Dockerfile.test' + - '.github/workflows/trivy.yml' push: branches: [main] schedule: @@ -61,7 +67,7 @@ jobs: image-ref: scan-target:${{ matrix.image.name }} format: sarif output: trivy-${{ matrix.image.name }}.sarif - severity: HIGH,CRITICAL + severity: MEDIUM,HIGH,CRITICAL # Hides CVEs with no available fix in the base image. ignore-unfixed: true exit-code: '0' diff --git a/.github/workflows/workflow-lint.yml b/.github/workflows/workflow-lint.yml index 97f388d28..7b38b8ddd 100644 --- a/.github/workflows/workflow-lint.yml +++ b/.github/workflows/workflow-lint.yml @@ -1,10 +1,13 @@ -name: Workflow Lint (zizmor) +name: Workflow Lint -# Lints .github/workflows/** for known GitHub Actions security misconfigurations: -# unpinned Actions, dangerous ${{ ... }} interpolation in run: blocks, -# missing per-job permissions:, etc. +# Lints .github/workflows/** for both: +# - actionlint: YAML syntax, expression typing, shellcheck inside `run:` +# blocks, unknown contexts, deprecated runner labels. +# - zizmor: security misconfigurations — unpinned actions, dangerous +# `${{ }}` interpolation, missing per-job permissions, etc. # -# Scoped to PRs that touch .github/** only — keeps off the typical PR critical path. +# Scoped to PRs that touch .github/** only — keeps off the typical PR +# critical path. on: pull_request: @@ -17,6 +20,27 @@ concurrency: cancel-in-progress: true jobs: + actionlint: + name: actionlint + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # Pinned to v2.1.2. Verify SHA via: + # gh api repos/raven-actions/actionlint/git/refs/tags/v2.1.2 + # The action wraps the upstream `rhysd/actionlint` binary and emits + # GitHub-annotation-formatted findings on PRs. + - name: Run actionlint + uses: raven-actions/actionlint@205b530c5d9fa8f44ae9ed59f341a0db994aa6f8 # v2.1.2 + with: + fail-on-error: true + zizmor: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/zizmor.yml b/.github/zizmor.yml index c534679c1..b2f89e3ba 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -14,6 +14,15 @@ rules: # no checkout of fork code occurs. Header comment in the file documents. - ci-report.yml + # workflow_run is the trusted half of the autofix pipeline. The + # untrusted half (pr-autofix.yml) runs fork code with permissions:{} + # and produces only a diff artifact (data, not executable code). The + # publish job consumes the artifact, allowlist-validates every field + # of metadata.json before exporting to $GITHUB_OUTPUT, never checks + # out fork code, and never executes anything fork-controlled. Header + # comment in the file documents the split. + - pr-autofix-publish.yml + # pull_request_target needed by claude-code-action to access secrets # and post review comments on fork PRs. Mitigated by: PR checkouts pin # the fork's HEAD SHA (not the branch ref) to prevent TOCTOU races, diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0ed2aeba5..d4f6b12b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,17 +30,17 @@ Format: `[(scope)][!]: ` Allowed types and the release-notes section each one lands in (defined in `.github/release.yml`): -| Type | Label applied | Release-notes section | -|------|---------------|-----------------------| -| `feat` | `enhancement` | 🚀 Features | -| `fix` | `bug` | 🐛 Bug Fixes | -| `perf` | `performance` | 🏎️ Performance | -| `refactor` | `refactor` | 🔄 Refactoring | -| `test` | `test` | 🧪 Tests | -| `ci` | `ci` | 👷 CI/CD | -| `build` / `deps` | `dependencies` | 📦 Dependencies | -| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) | -| `chore` / `revert` | `chore` | (excluded from release notes) | +| Type | Label applied | Release-notes section | +| ------------------ | --------------- | ------------------------------------------------------------ | +| `feat` | `enhancement` | 🚀 Features | +| `fix` | `bug` | 🐛 Bug Fixes | +| `perf` | `performance` | 🏎️ Performance | +| `refactor` | `refactor` | 🔄 Refactoring | +| `test` | `test` | 🧪 Tests | +| `ci` | `ci` | 👷 CI/CD | +| `build` / `deps` | `dependencies` | 📦 Dependencies | +| `docs` | `documentation` | (grouped under Other Changes unless a Docs section is added) | +| `chore` / `revert` | `chore` | (excluded from release notes) | Append `!` to the type (e.g. `feat(api)!: drop /v1 endpoint`) or include `BREAKING CHANGE:` in the PR body to flag a breaking change — the labeler then adds the `breaking` label and the 💥 Breaking Changes section is rendered first. @@ -81,17 +81,17 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency: - **Merge queue (`merge_group`)**: when this event is added, use `${{ github.workflow }}-${{ github.event.merge_group.head_ref }}` with `cancel-in-progress: false` (every queue entry is a distinct ref; never cancel). - **`cancel-in-progress` policy:** - | Event | `cancel-in-progress` | Why | - |-------|----------------------|-----| - | `pull_request` CI run | `true` | New push supersedes old run | - | `push` to `main` | `false` | Every main commit gets validated | - | Tag push (`v*` publish) | `false` | Never cancel mid-publish | - | `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish | - | `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional | - | `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race | - | Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR | - | PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins | - | Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes | + | Event | `cancel-in-progress` | Why | + | ---------------------------------------- | -------------------- | -------------------------------- | + | `pull_request` CI run | `true` | New push supersedes old run | + | `push` to `main` | `false` | Every main commit gets validated | + | Tag push (`v*` publish) | `false` | Never cancel mid-publish | + | `push` to `main` for release-candidate | `false` | Never cancel mid-RC publish | + | `workflow_dispatch` (release/publish) | `false` | Manual runs are intentional | + | `workflow_run` (sticky-comment reports) | `false` | Serialize, don't race | + | Per-PR bot workflows (`@claude`, review) | `false` | Serialize comments per PR | + | PR-meta re-checks (pr-description-check) | `true` | Cheap, latest wins | + | Single-slot utilities (triage sweep) | `true` | Latest dispatch supersedes | - For workflows that serve multiple events at once (e.g. `ci.yml` handles `pull_request`, `push`, and `workflow_call`), make `cancel-in-progress` event-aware: @@ -103,6 +103,41 @@ Every workflow under `.github/workflows/` MUST declare a top-level `concurrency: - When adding a new workflow, copy the concurrency block from an existing workflow of the same event shape. +## CI automation contracts + +Two workflows produce machine-readable signals on every PR. Coding agents and humans alike can rely on the names and shapes below — change them with intent. + +### `gitnexus/autofix` + +`pr-autofix.yml` (untrusted) + `pr-autofix-publish.yml` (trusted) run `prettier --write` and `eslint --fix` against the PR head and surface a single ChatOps button on the PR. Three signals are emitted: + +| Surface | Where | Notes | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| Sticky PR comment | Top-level comment with the HTML marker `` and heading `## :sparkles: PR Autofix`. Only posted when there is something to fix; clean PRs stay silent. | Edit-in-place via marker; one comment per PR. | +| Fenced JSON block | Inside the sticky, fenced as `gitnexus-autofix`. Schema `gitnexus.pr-autofix/v2` with fields `state` (`fixes-available`), `pr_number`, `head_sha`, `changed_lines`, `run_id`, and `apply_command` (literal `/autofix`). | Parseable signal — preferred over regexing prose. v1 fields preserved as a superset. | +| Check Run | Stable name `gitnexus/autofix` on the PR head SHA. Conclusion: `success` (clean) or `neutral` (`fixes-available`). The neutral title is `Autofix available — comment /autofix to apply`. | Surfaced under PR Checks; readable via `gh pr checks `. | + +To detect outcome from an agent: `gh pr checks --json name,conclusion,output | jq '.[] | select(.name == "gitnexus/autofix")'`. + +Forks are supported. The untrusted half runs fork code with `permissions: {}` and ships the diff as an artifact; the trusted publish job consumes only the diff (data, not code) and posts the comment + check run. + +#### Applying autofix + +Comment `/autofix` on the PR (whole-line, no arguments). The `pr-autofix-apply.yml` workflow: + +1. Validates the comment body matches `^/autofix\s*$` exactly. Quoted or inline mentions are silently ignored. +2. Validates the commenter has `admin`, `write`, or `maintain` permission on the repo, OR is the PR author. Other commenters get a 👎 reaction and a refusal reply. +3. Locates the most recent successful `pr-autofix.yml` run for the PR's current head SHA, downloads its `autofix` artifact, applies the patch, and pushes a `chore(autofix): ...` commit back to the PR head branch. +4. Reacts ✅ on success, 👎 on stale-patch / push-failure, and posts a short reply with the apply-run URL in either case. + +The apply workflow runs from the default branch's copy of the file regardless of where the comment originates — that's the trust anchor. There is no diff-size cap (the apply workflow uses `git apply` + push, not the GitHub review-comment API). + +For fork PRs, the push succeeds only when the contributor has **Allow edits by maintainers** enabled on the PR (the default). When they have disabled it, the workflow fails loud with a 👎 reaction and an explanation comment. + +Re-invoking `/autofix` after a successful apply is a safe no-op — the workflow detects the already-applied state via `git apply --check --reverse` and reacts ✅ without pushing. + +**Sensitive paths.** The apply workflow refuses any patch that touches `.github/` (workflow files, CODEOWNERS, dependabot config). A malicious PR could ship a custom prettier or ESLint config that reformats workflow YAML; if accepted, those edits would be pushed under `contents: write` without human review. Apply formatter changes to files under `.github/` manually in a normal commit so they get the same review every other workflow change gets. + ## AI-assisted contributions If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUDE.md`) and avoid drive-by refactors unrelated to the issue. Prefer incremental, test-backed changes. @@ -164,8 +199,7 @@ Two publish workflows ship `gitnexus` to npm: the Docker build. - Manually run `docker build` + `docker push` locally and sign with Cosign against the same digest. - - Delete `rc/` and `v` tags, then redispatch with `force: - true` to re-run the full RC pipeline (cuts a new RC number). + - Delete `rc/` and `v` tags, then redispatch with `force: true` to re-run the full RC pipeline (cuts a new RC number). The rc workflow never moves `latest`. To verify after a change, inspect dist-tags: diff --git a/Dockerfile.cli b/Dockerfile.cli index c45292e06..925f295f0 100644 --- a/Dockerfile.cli +++ b/Dockerfile.cli @@ -1,24 +1,32 @@ ARG BUILDPLATFORM ARG TARGETPLATFORM +# Pinned npm version used to replace the bundled npm in the upstream Node +# image. Bumping requires a coordinated update in Dockerfile.web and +# gitnexus/Dockerfile.test so all images bootstrap the same npm. +ARG NPM_VERSION=11.14.1 -# ── Builder ──────────────────────────────────────────────────────────── +# -- Builder ----------------------------------------------------------- # Native modules (tree-sitter-*, onnxruntime-node, node-gyp builds for # tree-sitter-proto / tree-sitter-swift) require python3 + a C/C++ toolchain. -FROM node:22-trixie-slim AS builder +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder +ARG NPM_VERSION WORKDIR /app +RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} + # Toolchain for node-gyp / native builds. RUN apt-get update && apt-get install -y --no-install-recommends python3 make g++ git && rm -rf /var/lib/apt/lists/* -# Build gitnexus-shared first — gitnexus depends on it as a workspace. +# Build gitnexus-shared first - gitnexus depends on it as a workspace. COPY gitnexus-shared/package.json gitnexus-shared/package-lock.json ./gitnexus-shared/ RUN npm ci --prefix gitnexus-shared COPY gitnexus-shared ./gitnexus-shared RUN rm -f gitnexus-shared/tsconfig.tsbuildinfo RUN npm run build --prefix gitnexus-shared -# Copy the full gitnexus package before installing — `npm ci` triggers +# Copy the full gitnexus package before installing - `npm ci` triggers # `postinstall` (patches tree-sitter-swift, builds the vendored # tree-sitter-proto) and `prepare` (compiles TypeScript via scripts/build.js), # both of which need the source tree. @@ -28,11 +36,15 @@ RUN npm ci --prefix gitnexus # Drop dev dependencies for a smaller runtime layer. RUN npm prune --omit=dev --prefix gitnexus -# ── Runtime ──────────────────────────────────────────────────────────── -FROM node:22-trixie-slim AS runtime +# -- Runtime ----------------------------------------------------------- +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime # curl for the healthcheck; git so `gitnexus` can clone repos at runtime. -RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends curl git && rm -rf /var/lib/apt/lists/* \ + && rm -rf /usr/local/lib/node_modules/npm \ + && rm -rf /usr/local/lib/node_modules/corepack \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack WORKDIR /app @@ -47,7 +59,7 @@ COPY --from=builder --chown=node:node /app/gitnexus/vendor ./gitnexus/vendor USER node -# The web UI defaults to http://localhost:4747 — keep that contract. +# The web UI defaults to http://localhost:4747 - keep that contract. ENV GITNEXUS_HOME=/data/gitnexus \ NODE_ENV=production \ PORT=4747 diff --git a/Dockerfile.web b/Dockerfile.web index 7d20e09ce..b102a700e 100644 --- a/Dockerfile.web +++ b/Dockerfile.web @@ -1,10 +1,17 @@ ARG BUILDPLATFORM ARG TARGETPLATFORM +# Pinned npm version — keep in sync with Dockerfile.cli and +# gitnexus/Dockerfile.test. +ARG NPM_VERSION=11.14.1 -FROM --platform=$BUILDPLATFORM node:22-alpine AS builder +# node:22-bookworm-slim +FROM --platform=$BUILDPLATFORM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS builder +ARG NPM_VERSION WORKDIR /app +RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} + COPY gitnexus-shared/package.json gitnexus-shared/package-lock.json ./gitnexus-shared/ RUN npm ci --prefix gitnexus-shared @@ -19,9 +26,13 @@ RUN npm ci --prefix gitnexus-web COPY gitnexus-web ./gitnexus-web RUN npm run build --prefix gitnexus-web -FROM node:22-alpine AS runtime +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e AS runtime -RUN apk add --no-cache curl +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* \ + && rm -rf /usr/local/lib/node_modules/npm \ + && rm -rf /usr/local/lib/node_modules/corepack \ + && rm -f /usr/local/bin/npm /usr/local/bin/npx /usr/local/bin/corepack WORKDIR /app diff --git a/README.md b/README.md index f5f3c5a88..6beadb63d 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ gitnexus clean --all --force # Delete all indexes gitnexus wiki [path] # Generate repository wiki from knowledge graph gitnexus wiki --model # Wiki with custom LLM model (default: gpt-4o-mini) gitnexus wiki --base-url # Wiki with custom LLM API base URL +gitnexus publish # Notify the understand-quickly registry (opt-in, see below) # Repository groups (multi-repo / monorepo service tracking) gitnexus group create # Create a repository group @@ -228,6 +229,12 @@ gitnexus group status # Check staleness of repos in a group If `analyze` reports a worker parse timeout on a large or unusual repository, it keeps running and falls back safely. To give slow worker jobs more time, use `gitnexus analyze --worker-timeout 60` or set `GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=60000`. For very large files, `GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES` controls the worker job byte budget. +#### Publishing to understand-quickly (opt-in) + +[`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) is a public registry of code-knowledge graphs that lists `gitnexus@1` as a first-class format. After registering your repo once (`npx @understand-quickly/cli add` or the [wizard](https://looptech-ai.github.io/understand-quickly/add.html)), `gitnexus publish` fires a single `repository_dispatch` event so the registry resyncs your entry on demand instead of waiting for the nightly job. + +It is opt-in and a no-op without `UNDERSTAND_QUICKLY_TOKEN` — a fine-grained GitHub PAT with `Repository dispatches: write` on the registry repo. Nothing else happens; no graph file is uploaded. See the [protocol spec](https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md) for the full contract. + ### What Your AI Agent Gets **16 tools** exposed via MCP (11 per-repo + 5 group): diff --git a/gitnexus-shared/package.json b/gitnexus-shared/package.json index 7c1e6847a..0a5d7a2db 100644 --- a/gitnexus-shared/package.json +++ b/gitnexus-shared/package.json @@ -10,6 +10,10 @@ ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" + }, + "./test-helpers": { + "types": "./dist/test-helpers.d.ts", + "default": "./dist/test-helpers.js" } }, "scripts": { diff --git a/gitnexus-shared/src/index.ts b/gitnexus-shared/src/index.ts index ea66c3855..3c82658f1 100644 --- a/gitnexus-shared/src/index.ts +++ b/gitnexus-shared/src/index.ts @@ -143,6 +143,34 @@ export type { ScopeTree } from './scope-resolution/scope-tree.js'; export { buildPositionIndex } from './scope-resolution/position-index.js'; export type { PositionIndex } from './scope-resolution/position-index.js'; +// Resilient fetch primitives — bounded retries + per-process circuit breaker. +// Test-only helpers (`__resetBreakerRegistry__`, `classifyOutcome`) are +// reachable via the separate `gitnexus-shared/test-helpers` subpath; do +// NOT add them here. Production consumers must not call them. +export { withRetry, computeBackoffMs } from './integrations/retry.js'; +export type { RetryOptions, RetryDecision } from './integrations/retry.js'; +export { CircuitBreaker, CircuitOpenError, getBreaker } from './integrations/circuit-breaker.js'; +export type { CircuitBreakerOptions } from './integrations/circuit-breaker.js'; +export { + resilientFetch, + ResilientFetchExhaustedError, + RETRY_AFTER_CAP_MS, + parseRetryAfter, +} from './integrations/resilient-fetch.js'; +export type { ResilientFetchOptions } from './integrations/resilient-fetch.js'; + +// Understand-Quickly registry integration (opt-in) +export { + UNDERSTAND_QUICKLY_DISPATCH_URL, + UNDERSTAND_QUICKLY_EVENT_TYPE, + UNDERSTAND_QUICKLY_TOKEN_ENV, + buildUqDispatchPayload, + isValidOwnerRepo, + parseOwnerRepoFromRemote, + stripGitSuffix, +} from './integrations/understand-quickly.js'; +export type { UqDispatchPayload } from './integrations/understand-quickly.js'; + // Shadow-mode diff + aggregation (RFC §6.3; Ring 2 SHARED #918) export { diffResolutions } from './scope-resolution/shadow/diff.js'; export type { diff --git a/gitnexus-shared/src/integrations/circuit-breaker.ts b/gitnexus-shared/src/integrations/circuit-breaker.ts new file mode 100644 index 000000000..29782a8fb --- /dev/null +++ b/gitnexus-shared/src/integrations/circuit-breaker.ts @@ -0,0 +1,273 @@ +/** + * Per-process circuit breaker. + * + * Closed -> Open transition fires after `failureThreshold` consecutive + * failures. While Open, `check` throws `CircuitOpenError` until + * `cooldownMs` has elapsed since the breaker tripped. The first call + * after the cooldown enters Half-Open and consumes the *probe permit*: + * a recorded success returns to Closed; a recorded failure flips back + * to Open with a fresh timestamp. + * + * Half-open admits exactly one in-flight probe at a time. Concurrent + * callers attempting `check()` while a probe is outstanding receive + * `CircuitOpenError` with `retryAfterMs = halfOpenRetryAfterMs` (default + * 1000ms; configurable). This prevents the recovery-time thundering + * herd that defeats the breaker's "fail fast" promise. + * + * Outcome reporting splits permit-release from state-resolution: + * - `recordSuccess` — releases the probe permit, resets the failure + * counter, transitions to Closed. Reserved for true 2xx/3xx outcomes. + * - `recordFailure` — releases the probe permit, increments the + * consecutive-failure counter, transitions to Open with a fresh + * `openedAt` (when called from Half-Open or when the threshold + * trips from Closed). + * - `recordNeutral` — releases the probe permit, BUT leaves state and + * counter untouched. Used for outcomes that are neither evidence of + * backend health nor evidence of backend failure (caller-driven + * cancellation, local timeout, terminal 4xx client errors). Critical + * design point: if `recordNeutral` did not release the permit, a + * single `TimeoutError` from per-attempt `AbortSignal.timeout` would + * route through `recordNeutral` and permanently park the breaker in + * half-open until process restart. Releasing the permit while leaving + * state half-open keeps the "neutral doesn't claim health" semantic + * without creating that wedge. + * + * Pairing invariant: every successful `check()` MUST be paired with + * exactly one `record*()` on every code path including throws. Direct + * consumers should wrap the protected operation in `try/finally`: + * + * breaker.check(); + * try { + * const result = await operation(); + * breaker.recordSuccess(); + * return result; + * } catch (err) { + * // classify err and call recordFailure / recordNeutral / etc. + * throw err; + * } + * + * `resilientFetch`'s catch-all on `fetchImpl` already satisfies this + * for that consumer. + * + * Atomicity model: the half-open gate relies on JavaScript event-loop + * single-threadedness within a synchronous `check()` body. There is no + * `await` inside `check()`; concurrent callers serialize on microtask + * order, and exactly one observes `probeInFlight === false`. Do not + * introduce `await` inside `check()` without revisiting the gate. If + * this code is ever ported to a runtime with shared-memory threads + * (Node `worker_threads` with `SharedArrayBuffer`, Web Workers with + * shared registries), the boolean must become an atomic CAS — Resilience4j + * and Hystrix use atomic permits *because* they run in JVM thread pools. + * + * Runtime-agnostic: depends only on a `now()` clock and standard JS — + * no Node-only imports. Tests inject `now` to advance the clock + * deterministically without `vi.useFakeTimers()`. + */ + +export class CircuitOpenError extends Error { + override readonly name = 'CircuitOpenError'; + /** Approximate wait time before the breaker may transition to Half-Open + * (or before the in-flight probe is expected to resolve). */ + readonly retryAfterMs: number; + + constructor(retryAfterMs: number, key?: string) { + super( + key + ? `Circuit '${key}' is open; retry in ${Math.ceil(retryAfterMs / 1000)}s` + : `Circuit is open; retry in ${Math.ceil(retryAfterMs / 1000)}s`, + ); + this.retryAfterMs = retryAfterMs; + } +} + +export interface CircuitBreakerOptions { + /** Consecutive failures required to trip Closed -> Open. */ + failureThreshold?: number; + /** Milliseconds Open before the next call may probe (Half-Open). */ + cooldownMs?: number; + /** + * Milliseconds to suggest in `CircuitOpenError.retryAfterMs` when the + * breaker is Half-Open with the probe permit consumed. Default 1000ms. + * Consumers with long-running protected ops (LLM streaming, large + * uploads) should raise this — the cooldown clock is no longer the + * right answer because cooldown has elapsed. Returning 0 invites + * retry storms; returning the full cooldown misleads about wait. + */ + halfOpenRetryAfterMs?: number; + /** Optional key for error messages and registry lookups. */ + key?: string; + /** Clock override — defaults to `Date.now`. Tests inject deterministic time. */ + now?: () => number; +} + +type State = 'closed' | 'open' | 'half-open'; + +export class CircuitBreaker { + private readonly failureThreshold: number; + private readonly cooldownMs: number; + private readonly halfOpenRetryAfterMs: number; + private readonly key: string | undefined; + private readonly now: () => number; + + private state: State = 'closed'; + private consecutiveFailures = 0; + private openedAt: number | null = null; + /** + * True between a successful `check()` and the next `record*()` call + * during Half-Open. Gates concurrent callers from stampeding a still- + * recovering dependency. Boolean rather than counter — single-permit + * is the conservative end of the Hystrix/Resilience4j spectrum. + */ + private probeInFlight = false; + + constructor(opts: CircuitBreakerOptions = {}) { + this.failureThreshold = opts.failureThreshold ?? 3; + this.cooldownMs = opts.cooldownMs ?? 30_000; + this.halfOpenRetryAfterMs = opts.halfOpenRetryAfterMs ?? 1_000; + this.key = opts.key; + this.now = opts.now ?? (() => Date.now()); + } + + /** + * Throw `CircuitOpenError` if the breaker won't admit this call. + * Otherwise consume the half-open probe permit (if applicable) and + * return so the caller can attempt the protected work. + * + * Three rejection paths: + * 1. Open and still in cooldown → throws with `retryAfterMs` = + * remaining cooldown. + * 2. Open with cooldown elapsed AND a probe is already in flight + * (race: another caller transitioned to half-open and grabbed + * the permit on a microtask before us) → throws with + * `halfOpenRetryAfterMs`. + * 3. Half-Open with probe in flight → throws with `halfOpenRetryAfterMs`. + * + * **Pairing invariant**: every successful return from `check()` MUST + * be paired with exactly one `recordSuccess` / `recordFailure` / + * `recordNeutral` on every code path including thrown exceptions. + * Failing to pair leaves the probe permit consumed forever and + * wedges the breaker. See file-header JSDoc for the canonical + * try/finally pattern. + */ + check(): void { + if (this.state === 'open' && this.openedAt !== null) { + const elapsed = this.now() - this.openedAt; + if (elapsed < this.cooldownMs) { + throw new CircuitOpenError(this.cooldownMs - elapsed, this.key); + } + // Cooldown elapsed — transition to Half-Open. The very next + // `probeInFlight` check below decides whether THIS caller gets + // the permit or hits the gate. + this.state = 'half-open'; + } + + if (this.state === 'half-open') { + if (this.probeInFlight) { + throw new CircuitOpenError(this.halfOpenRetryAfterMs, this.key); + } + this.probeInFlight = true; + } + // Closed state falls through silently. + } + + recordSuccess(): void { + this.probeInFlight = false; + this.consecutiveFailures = 0; + this.state = 'closed'; + this.openedAt = null; + } + + recordFailure(): void { + this.probeInFlight = false; + this.consecutiveFailures += 1; + if (this.state === 'half-open' || this.consecutiveFailures >= this.failureThreshold) { + this.state = 'open'; + this.openedAt = this.now(); + } + } + + /** + * Releases the probe permit BUT leaves state and counter untouched. + * Use when an attempt produced a response or error that should not + * influence breaker health in either direction — caller-driven aborts, + * local AbortSignal timeouts, terminal 4xx client errors. + * + * Why permit-release-without-state-resolution: if `recordNeutral` did + * not clear `probeInFlight`, a single `TimeoutError` from per-attempt + * `AbortSignal.timeout` (which routes through neutral classification) + * would permanently park the breaker in half-open. Since timeouts are + * an *expected* outcome under flaky-dependency conditions, the cited + * "per-attempt timeout bounds the stuck state" mitigation would itself + * be the trigger for a permanent wedge. Releasing the permit closes + * that loop while keeping the "neutral doesn't claim dependency + * health" semantic. + * + * Calling `recordSuccess` for these would erase legitimate prior + * failure signal; calling `recordFailure` would trip the breaker for + * outcomes the backend isn't responsible for. + */ + recordNeutral(): void { + this.probeInFlight = false; + // State and consecutiveFailures are preserved by design. + } + + /** + * Pure read — no state mutation, no permit accounting. Returns the + * *would-be* state at the current instant: 'half-open' if the breaker + * is open with cooldown elapsed (regardless of whether a probe is in + * flight), 'open' if open and still in cooldown, 'closed' otherwise. + * + * Inspection-only; safe to call from tests without consuming a probe + * permit. The implicit Open -> Half-Open transition that mutates + * `state` lives in `check()` only. + */ + getState(): State { + if (this.state === 'open' && this.openedAt !== null) { + const elapsed = this.now() - this.openedAt; + if (elapsed >= this.cooldownMs) return 'half-open'; + } + return this.state; + } + getConsecutiveFailures(): number { + return this.consecutiveFailures; + } + /** Inspection-only test accessor for the half-open probe permit. */ + isProbeInFlight(): boolean { + return this.probeInFlight; + } + /** Timestamp (ms since epoch) when the breaker last transitioned to Open, + * or `null` if it's currently Closed. Useful for computing remaining + * cooldown without consuming a probe permit via `check()`. */ + getOpenedAt(): number | null { + return this.openedAt; + } + /** Configured cooldown duration in milliseconds. */ + getCooldownMs(): number { + return this.cooldownMs; + } +} + +// ─── Per-process registry ──────────────────────────────────────────── +// +// Single shared map keyed on caller-chosen strings. Used by +// `resilient-fetch.ts` so multiple call sites targeting the same logical +// endpoint share breaker state. Per-process only — not persisted. + +const registry = new Map(); + +export function getBreaker(key: string, opts?: CircuitBreakerOptions): CircuitBreaker { + let breaker = registry.get(key); + if (!breaker) { + breaker = new CircuitBreaker({ ...opts, key }); + registry.set(key, breaker); + } + return breaker; +} + +/** + * Test-only: clear all registered breakers. Tests must call this in + * `beforeEach` to prevent breaker state from leaking across test cases. + */ +export function __resetBreakerRegistry__(): void { + registry.clear(); +} diff --git a/gitnexus-shared/src/integrations/resilient-fetch.ts b/gitnexus-shared/src/integrations/resilient-fetch.ts new file mode 100644 index 000000000..c91b9db3a --- /dev/null +++ b/gitnexus-shared/src/integrations/resilient-fetch.ts @@ -0,0 +1,279 @@ +/** + * `resilientFetch` — fetch wrapped in retry + circuit breaker, with + * GitHub-flavoured retry classification baked in (Retry-After parsing, + * 401/403/404/422 treated as terminal client errors). + * + * Designed for the `gitnexus publish` GitHub `repository_dispatch` + * call, but the classification rules apply to any GitHub REST endpoint. + * Runtime-agnostic — no Node-only imports. + */ + +import { + CircuitBreaker, + CircuitOpenError, + getBreaker, + type CircuitBreakerOptions, +} from './circuit-breaker.js'; +import { computeBackoffMs, type RetryOptions } from './retry.js'; + +export { CircuitOpenError }; + +export interface ResilientFetchOptions { + /** Optional fetch implementation override. Defaults to `globalThis.fetch`. */ + fetchImpl?: typeof fetch; + /** + * Logical key for the breaker. Defaults to `` of the + * request URL — call sites targeting the same endpoint share breaker + * state regardless of query-string differences. + */ + breakerKey?: string; + /** Per-call breaker override. Used for tests and one-off configuration. */ + breaker?: CircuitBreaker; + /** Tuning knobs for the breaker registered under `breakerKey`. */ + breakerOptions?: CircuitBreakerOptions; + /** Tuning knobs for the retry helper. */ + retry?: Partial> & { + sleep?: RetryOptions['sleep']; + random?: RetryOptions['random']; + }; + /** Clock override propagated into Retry-After HTTP-date math and breaker. */ + now?: () => number; +} + +/** Cap on any single Retry-After wait — protects CLI from a buggy registry. */ +export const RETRY_AFTER_CAP_MS = 30_000; + +const DEFAULT_RETRY = { + maxAttempts: 3, + baseDelayMs: 500, + capDelayMs: 5_000, +}; + +/** + * Parse a `Retry-After` header value into milliseconds. + * Accepts either a delta-seconds integer (`"30"`) or an HTTP-date. + * Returns null on parse failure or negative deltas. + */ +export function parseRetryAfter(value: string | null, now: () => number = Date.now): number | null { + if (!value) return null; + const trimmed = value.trim(); + if (trimmed === '') return null; + + if (/^[0-9]+$/.test(trimmed)) { + const seconds = parseInt(trimmed, 10); + if (Number.isNaN(seconds) || seconds < 0) return null; + return seconds * 1000; + } + + const target = Date.parse(trimmed); + if (Number.isNaN(target)) return null; + const delta = target - now(); + return delta >= 0 ? delta : 0; +} + +/** Internal: outcome classification used by the resilientFetch loop. */ +type Outcome = + | { kind: 'success'; resp: Response } + | { kind: 'terminal-client'; resp: Response } // 4xx other than 429: no retry, breaker neutral + | { kind: 'retryable-status'; resp: Response; afterMs: number | undefined } // 5xx, 429 + | { kind: 'terminal-network'; err: unknown } // TimeoutError or AbortError: no retry, breaker neutral + | { kind: 'retryable-network'; err: unknown }; // DNS, ECONNRESET, etc. + +/** Exported for unit tests. */ +export function classifyOutcome( + result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response }, + now: () => number, +): Outcome { + if (result.kind === 'error') { + // Both timer-fired aborts (`AbortSignal.timeout()` → `TimeoutError`) + // and caller-driven aborts (`AbortController.abort()` → `AbortError`) + // are terminal: retrying against an already-aborted signal would + // fail again immediately, and neither outcome reflects backend + // health. They route through the breaker's neutral path. + if ( + result.err instanceof DOMException && + (result.err.name === 'TimeoutError' || result.err.name === 'AbortError') + ) { + return { kind: 'terminal-network', err: result.err }; + } + return { kind: 'retryable-network', err: result.err }; + } + const resp = result.resp; + if (resp.status >= 200 && resp.status < 400) return { kind: 'success', resp }; + if (resp.status === 429) { + // `resp.headers` is always present on a real `Response`, but tests + // sometimes stub `fetch` with a plain `{ ok, status }` object. Be + // defensive — a missing `Retry-After` falls through to exponential + // backoff, which is the correct behaviour anyway. + const retryAfterHeader = + typeof resp.headers?.get === 'function' ? resp.headers.get('Retry-After') : null; + const parsed = parseRetryAfter(retryAfterHeader, now); + return { + kind: 'retryable-status', + resp, + afterMs: parsed !== null ? Math.min(parsed, RETRY_AFTER_CAP_MS) : undefined, + }; + } + if (resp.status >= 500) return { kind: 'retryable-status', resp, afterMs: undefined }; + return { kind: 'terminal-client', resp }; +} + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +function defaultBreakerKey(input: string | URL): string { + try { + const url = typeof input === 'string' ? new URL(input) : input; + return `${url.host}${url.pathname}`; + } catch { + return String(input); + } +} + +/** Final error thrown when retries are exhausted on a 5xx / 429. */ +export class ResilientFetchExhaustedError extends Error { + override readonly name = 'ResilientFetchExhaustedError'; + constructor(public readonly response: Response) { + super(`Request failed after retries (HTTP ${response.status})`); + } +} + +/** + * Wrap `fetch` with bounded retries and a per-process circuit breaker. + * + * Semantics: + * - 5xx and 429 responses are retried; 429 honors `Retry-After` (capped). + * - Network throws are retried unless they are `TimeoutError` DOMExceptions. + * - Timeouts and 4xx (other than 429) are returned/thrown without retry + * AND without incrementing the breaker — they reflect caller config + * or local network state, not registry health. + * - Each `fetch` call carries the caller-supplied `signal` (e.g. an + * `AbortSignal.timeout()`) — that timeout bounds each individual + * attempt, not the whole retry sequence. + * - When the breaker is open, throws `CircuitOpenError` synchronously + * without invoking `fetch`. + * - When retries are exhausted on a 5xx / 429, throws + * `ResilientFetchExhaustedError` carrying the last response. + * + * Cumulative wall-clock budget: + * maxAttempts × (per-attempt-timeout + capDelayMs) + * With defaults (3, 500ms base, 5000ms cap) and a typical 15s per-attempt + * timeout from the caller's signal, worst case is ~3 × (15s + 5s) = 60s. + * Callers that want a tighter total bound should reduce `maxAttempts` or + * wrap `resilientFetch` in their own outer `AbortSignal.timeout()`. + */ +export async function resilientFetch( + input: string | URL, + init: RequestInit | undefined, + opts: ResilientFetchOptions = {}, +): Promise { + const fetchImpl = opts.fetchImpl ?? globalThis.fetch; + const now = opts.now ?? (() => Date.now()); + const breaker = + opts.breaker ?? getBreaker(opts.breakerKey ?? defaultBreakerKey(input), opts.breakerOptions); + + const retryConfig = { + maxAttempts: opts.retry?.maxAttempts ?? DEFAULT_RETRY.maxAttempts, + baseDelayMs: opts.retry?.baseDelayMs ?? DEFAULT_RETRY.baseDelayMs, + capDelayMs: opts.retry?.capDelayMs ?? DEFAULT_RETRY.capDelayMs, + }; + const sleep = opts.retry?.sleep ?? defaultSleep; + const random = opts.retry?.random ?? Math.random; + + // Fail fast on an open breaker, before invoking fetch. + breaker.check(); + + for (let attempt = 0; attempt < retryConfig.maxAttempts; attempt++) { + let result: { kind: 'error'; err: unknown } | { kind: 'response'; resp: Response }; + try { + // CodeQL js/server-side-request-forgery — flagged because `input` + // is caller-supplied. Suppressed: every concrete caller passes + // either a hardcoded URL constant (UNDERSTAND_QUICKLY_DISPATCH_URL, + // OpenRouter base URL) or a value derived from configuration + // (env vars, saved settings, the local backend URL). User-input + // request fields (e.g. PR title, repo name) never flow into + // `input`. Validating URL shape here would push false-positive + // rejection onto every caller — wrong layer for the check. + // lgtm[js/server-side-request-forgery] + // codeql[js/server-side-request-forgery] + const resp = await fetchImpl(input, init); + result = { kind: 'response', resp }; + } catch (err) { + result = { kind: 'error', err }; + } + + const outcome = classifyOutcome(result, now); + + switch (outcome.kind) { + case 'success': + breaker.recordSuccess(); + return outcome.resp; + + case 'terminal-client': + // 4xx: do not count as breaker failure (the server is healthy + // and rejecting our request — auth, scope, or routing). But + // also do NOT call recordSuccess: a 401 sandwiched between + // 5xx responses would otherwise erase the running outage + // signal. The breaker's neutral path leaves state untouched. + breaker.recordNeutral(); + return outcome.resp; + + case 'terminal-network': + // Either `AbortSignal.timeout()` fired locally OR an external + // caller cancelled the request via AbortController. The server + // never had a chance to answer; this reflects the user's + // network or an explicit cancel, not registry health. Don't + // punish the breaker AND don't reset its outage signal. + breaker.recordNeutral(); + throw outcome.err; + + case 'retryable-status': + if (attempt + 1 >= retryConfig.maxAttempts) { + breaker.recordFailure(); + throw new ResilientFetchExhaustedError(outcome.resp); + } + await sleep( + computeBackoffMs( + attempt, + retryConfig.baseDelayMs, + retryConfig.capDelayMs, + outcome.afterMs, + random, + ), + ); + break; + + case 'retryable-network': + if (attempt + 1 >= retryConfig.maxAttempts) { + breaker.recordFailure(); + throw outcome.err; + } + await sleep( + computeBackoffMs( + attempt, + retryConfig.baseDelayMs, + retryConfig.capDelayMs, + undefined, + random, + ), + ); + break; + + default: { + // Exhaustiveness guard. If a sixth `Outcome` kind is added in + // future, TypeScript will refuse to assign it to `never` and + // this line forces the maintainer to add an explicit arm + // rather than silently fall through to retry/no-retry behaviour. + const _exhaustive: never = outcome; + throw new Error(`resilientFetch: unhandled outcome ${JSON.stringify(_exhaustive)}`); + } + } + } + + // Unreachable: every iteration of the loop either returns (success + // / terminal-client) or throws (terminal-network / retry exhaustion). + // The throw is here purely so TypeScript's control-flow analysis sees + // the function never falls off the end without producing `Promise`. + /* c8 ignore next 2 */ + throw new Error('resilientFetch: retry loop terminated unexpectedly'); +} diff --git a/gitnexus-shared/src/integrations/retry.ts b/gitnexus-shared/src/integrations/retry.ts new file mode 100644 index 000000000..774ca2542 --- /dev/null +++ b/gitnexus-shared/src/integrations/retry.ts @@ -0,0 +1,105 @@ +/** + * Bounded retry helper with full-jitter exponential backoff. + * + * Runtime-agnostic: depends only on `setTimeout`, `Math.random`, and the + * Promise machinery — no Node-only imports. Safe to consume from CLI, + * server, or browser callers. + * + * Pattern reference: gitnexus/src/core/embeddings/http-client.ts. This + * helper is the upgraded form: classification is caller-supplied (so + * 4xx-vs-5xx-vs-timeout decisions live with the protocol that knows + * them), backoff is exponential with full jitter, and an optional + * `afterMs` lets callers honor `Retry-After` headers. + */ + +export interface RetryOptions { + /** Initial delay before the first retry attempt, in milliseconds. */ + baseDelayMs: number; + /** Upper bound on any single delay, in milliseconds. */ + capDelayMs: number; + /** Total attempts including the first call. Must be >= 1. */ + maxAttempts: number; + /** + * Decide whether to retry after a thrown error. + * Return `{retry:false}` to terminate immediately and rethrow. + * Return `{retry:true}` to retry with exponential-backoff jitter. + * Return `{retry:true, afterMs}` to wait at least `afterMs` (still + * subject to `capDelayMs`) — used by callers parsing `Retry-After`. + */ + isRetryable: (err: unknown, attempt: number) => RetryDecision; + /** Sleep override — defaults to `setTimeout`. Tests inject fake timers. */ + sleep?: (ms: number) => Promise; + /** Random override — defaults to `Math.random`. Tests inject seeded values. */ + random?: () => number; +} + +export type RetryDecision = { retry: false } | { retry: true; afterMs?: number }; + +const defaultSleep = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Compute the delay before the next retry attempt. + * + * - When the caller specifies `afterMs` (e.g., from `Retry-After`), use + * `min(afterMs, capDelayMs)` so a misbehaving server can't pin the + * client for an arbitrarily long wait. + * - Otherwise compute full-jitter exponential backoff: + * `random() * min(cap, base * 2^attempt)`. Full jitter (rather than + * "equal jitter") avoids retry-storm thundering herd, per AWS + * guidance on backoff strategies. + */ +export function computeBackoffMs( + attempt: number, + baseDelayMs: number, + capDelayMs: number, + afterMs: number | undefined, + random: () => number, +): number { + if (afterMs !== undefined) { + return Math.min(Math.max(0, afterMs), capDelayMs); + } + const exponential = baseDelayMs * Math.pow(2, attempt); + const upper = Math.min(capDelayMs, exponential); + return Math.floor(random() * upper); +} + +/** + * Execute `fn` with bounded retries. + * + * The classification of "retryable" is the caller's responsibility — see + * `resilient-fetch.ts` for the GitHub-dispatch-specific rules. This + * helper is the mechanical retry loop only. + */ +export async function withRetry( + fn: (attempt: number) => Promise, + opts: RetryOptions, +): Promise { + if (opts.maxAttempts < 1) { + throw new Error(`withRetry: maxAttempts must be >= 1, got ${opts.maxAttempts}`); + } + const sleep = opts.sleep ?? defaultSleep; + const random = opts.random ?? Math.random; + + let lastError: unknown; + for (let attempt = 0; attempt < opts.maxAttempts; attempt++) { + try { + return await fn(attempt); + } catch (err) { + lastError = err; + const decision = opts.isRetryable(err, attempt); + if (!decision.retry) throw err; + // Don't sleep after the final attempt. + if (attempt + 1 >= opts.maxAttempts) break; + const delayMs = computeBackoffMs( + attempt, + opts.baseDelayMs, + opts.capDelayMs, + decision.afterMs, + random, + ); + if (delayMs > 0) await sleep(delayMs); + } + } + throw lastError; +} diff --git a/gitnexus-shared/src/integrations/understand-quickly.ts b/gitnexus-shared/src/integrations/understand-quickly.ts new file mode 100644 index 000000000..f30e7461b --- /dev/null +++ b/gitnexus-shared/src/integrations/understand-quickly.ts @@ -0,0 +1,151 @@ +/** + * Understand-Quickly registry integration helpers. + * + * Pure, runtime-agnostic logic for opting in to publishing a GitNexus + * index to the [`looptech-ai/understand-quickly`](https://github.com/looptech-ai/understand-quickly) + * registry. Lives in `gitnexus-shared` so both the Node CLI and any + * future browser-side surface can construct identical dispatch payloads. + * + * Network I/O lives in the CLI command (`gitnexus/src/cli/publish.ts`) + * to keep this module free of Node-only imports — see the comment at + * the top of `gitnexus-shared/src/graph/types.ts`. + * + * The protocol contract (single dispatch event, no graph upload) is + * documented at: + * https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md + */ + +/** + * URL of the registry repo's repository_dispatch endpoint. Hardcoded + * because the registry is the canonical home for this integration — + * users who want a private registry can fork and patch. + */ +export const UNDERSTAND_QUICKLY_DISPATCH_URL = + 'https://api.github.com/repos/looptech-ai/understand-quickly/dispatches'; + +/** + * Event type the registry's sync workflow listens for. + * See `looptech-ai/understand-quickly/.github/workflows/sync.yml`. + */ +export const UNDERSTAND_QUICKLY_EVENT_TYPE = 'sync-entry'; + +/** Environment variable that gates the dispatch. */ +export const UNDERSTAND_QUICKLY_TOKEN_ENV = 'UNDERSTAND_QUICKLY_TOKEN'; + +export interface UqDispatchPayload { + event_type: typeof UNDERSTAND_QUICKLY_EVENT_TYPE; + client_payload: { + /** `/` shape — must match the registered entry. */ + id: string; + }; +} + +/** + * Build the JSON body for the `repository_dispatch` ping. Pure — no + * env reads, no network. Validates that `id` looks like `owner/repo` + * (one slash, no whitespace, both halves non-empty) so a misconfigured + * caller fails loudly before the round-trip. + */ +export function buildUqDispatchPayload(id: string): UqDispatchPayload { + if (!isValidOwnerRepo(id)) { + throw new Error( + `[understand-quickly] expected id of the form "owner/repo", got "${id}". ` + + `The registry uses this string to look up your entry in registry.json — ` + + `it must match the GitHub owner/repo of the source code, not a local path.`, + ); + } + return { + event_type: UNDERSTAND_QUICKLY_EVENT_TYPE, + client_payload: { id }, + }; +} + +/** + * `owner/repo` validation. Conservative on purpose: GitHub's actual + * naming rules are looser, but we want to catch local paths + * (`/Users/...`), bare slugs (`my-repo`), and accidental whitespace. + * + * Matches GitHub's published slug rules: + * owner: starts with alnum, then alnum/hyphen only, must end with + * alnum (no trailing hyphen — GitHub rejects this at account + * creation, so a `my-org-/repo` input would otherwise pass us + * and 422 from GitHub). No underscore, no dot. Length cap 39. + * repo: any of alnum/dot/hyphen/underscore. Length cap 100. + */ +export function isValidOwnerRepo(id: string): boolean { + return /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,37}[A-Za-z0-9])?\/[A-Za-z0-9._-]{1,100}$/.test(id); +} + +/** + * Strip a single trailing `.git` (case-insensitive) and any trailing + * slashes from a URL-ish string. Bounded linear: each character is + * visited at most twice, no backtracking. + * + * Replaces `s.replace(/\.git\/*$/i, '').replace(/\/+$/, '')` which + * CodeQL's polynomial-regex check (codeql/js/polynomial-redos) flags as + * a worst-case O(n²) on adversarial input like "////.../x". + */ +export function stripGitSuffix(input: string): string { + let end = input.length; + // Trim trailing '/'. + while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--; + // Drop one trailing '.git' (case-insensitive). + if (end >= 4) { + const tail = input.slice(end - 4, end).toLowerCase(); + if (tail === '.git') end -= 4; + } + // Trim trailing '/' that may have sat between '.git' and the rest. + while (end > 0 && input.charCodeAt(end - 1) === 0x2f) end--; + return input.slice(0, end); +} + +/** + * Parse `owner/repo` out of a git remote URL. Mirrors the heuristic in + * `gitnexus/src/storage/git.ts:parseRepoNameFromUrl` but keeps both + * halves so we can build a registry id. Returns `null` on shapes we + * don't recognise. + * + * Examples: + * git@github.com:looptech-ai/understand-quickly.git + * https://github.com/looptech-ai/understand-quickly + * ssh://git@github.com/looptech-ai/understand-quickly.git + */ +export function parseOwnerRepoFromRemote(url: string | null | undefined): string | null { + if (!url) return null; + const trimmed = url.trim(); + if (!trimmed) return null; + // Strip a trailing `.git` (case-insensitive) and any trailing slashes + // so https://h/o/r and https://h/o/r.git collapse to the same id. + // Bounded-linear helper avoids the polynomial-regex CodeQL alert. + const stripped = stripGitSuffix(trimmed); + + // SCP-form SSH (`git@host:owner/repo`). Capture host so we can reject + // non-GitHub remotes — a GitLab origin like + // `https://gitlab.example.com/group/sub/project.git` would otherwise + // silently dispatch the wrong id (LOW 9). + const ssh = stripped.match(/^[^@]+@([^:]+):([^/]+)\/([^/]+)$/); + if (ssh) { + const host = ssh[1].toLowerCase(); + if (host !== 'github.com' && host !== 'www.github.com') return null; + return `${ssh[2]}/${ssh[3]}`; + } + + // URL forms (https://, ssh://, git://, file://) — last two path segments. + const url2 = stripped.match(/^[a-zA-Z][a-zA-Z0-9+.-]*:\/\/([^/]+)\/(.+)$/); + if (url2) { + // Strip optional `userinfo@` (e.g. `ssh://git@github.com/...`). + const authority = url2[1]; + const atIdx = authority.lastIndexOf('@'); + const hostAndPort = atIdx >= 0 ? authority.slice(atIdx + 1) : authority; + // Strip `:port` suffix if present. + const colonIdx = hostAndPort.indexOf(':'); + const host = (colonIdx >= 0 ? hostAndPort.slice(0, colonIdx) : hostAndPort).toLowerCase(); + if (host !== 'github.com' && host !== 'www.github.com') return null; + const segments = url2[2].split('/').filter(Boolean); + if (segments.length >= 2) { + const [owner, repo] = segments.slice(-2); + return `${owner}/${repo}`; + } + } + return null; +} diff --git a/gitnexus-shared/src/test-helpers.ts b/gitnexus-shared/src/test-helpers.ts new file mode 100644 index 000000000..a92441878 --- /dev/null +++ b/gitnexus-shared/src/test-helpers.ts @@ -0,0 +1,13 @@ +/** + * Test-only helpers. + * + * Symbols here are reachable from `gitnexus-shared/test-helpers` so test + * suites can reset shared registries or exercise internal classifiers, + * but they are deliberately NOT re-exported from the main `gitnexus-shared` + * barrel. Production consumers should never import this module — calling + * `__resetBreakerRegistry__()` from a tool implementation would silently + * nuke every circuit breaker process-wide. + */ + +export { __resetBreakerRegistry__ } from './integrations/circuit-breaker.js'; +export { classifyOutcome } from './integrations/resilient-fetch.js'; diff --git a/gitnexus-web/package-lock.json b/gitnexus-web/package-lock.json index 7fcc0d299..3a560ca5a 100644 --- a/gitnexus-web/package-lock.json +++ b/gitnexus-web/package-lock.json @@ -8,7 +8,7 @@ "name": "gitnexus", "version": "0.0.0", "dependencies": { - "@langchain/anthropic": "^1.3.28", + "@langchain/anthropic": "^1.3.29", "@langchain/core": "^1.1.44", "@langchain/google-genai": "^2.1.28", "@langchain/langgraph": "^1.2.9", @@ -95,9 +95,9 @@ } }, "node_modules/@anthropic-ai/sdk": { - "version": "0.90.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.90.0.tgz", - "integrity": "sha512-MzZtPabJF1b0FTDl6Z6H5ljphPwACLGP13lu8MTiB8jXaW/YXlpOp+Po2cVou3MPM5+f5toyLnul9whKCy7fBg==", + "version": "0.91.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.91.1.tgz", + "integrity": "sha512-LAmu761tSN9r66ixvmciswUj/ZC+1Q4iAfpedTfSVLeswRwnY3n2Nb6Tsk+cLPP28aLOPWeMgIuTuCcMC6W/iw==", "license": "MIT", "dependencies": { "json-schema-to-ts": "^3.1.1" @@ -1396,25 +1396,25 @@ } }, "node_modules/@langchain/anthropic": { - "version": "1.3.28", - "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.28.tgz", - "integrity": "sha512-gOF8oXJL8xDdYes2KXNI9vFm/9TldBBBHOjuCdt27kganVaQKzLvTw5kV6R4mjbnFagV5CWteNH7APLZYCpdwg==", + "version": "1.3.29", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.29.tgz", + "integrity": "sha512-ep1qBIcV07bajsg3fDqMd39rYwoRLOEK/6lk+MCxlm1YB5SRoKKJAZANrblQ/4RYhZJnxf95c6BSQu8VoNbVAQ==", "license": "MIT", "dependencies": { - "@anthropic-ai/sdk": "^0.90.0", + "@anthropic-ai/sdk": "^0.91.1", "zod": "^3.25.76 || ^4" }, "engines": { "node": ">=20" }, "peerDependencies": { - "@langchain/core": "^1.1.42" + "@langchain/core": "^1.1.45" } }, "node_modules/@langchain/core": { - "version": "1.1.44", - "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.44.tgz", - "integrity": "sha512-RePW1IjGCHr9ua2vcby3aE8mOOz3EnwDZxMEGbNDT91kf14eqkJqxDXvaZFviGdcN9DTrxM5RPQNAHmwSm4tbg==", + "version": "1.1.45", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.1.45.tgz", + "integrity": "sha512-Y/wvuglLTMKJahkl4QD9dBIdF/z/CxZJWdTfHJF/q2jtlJtoFf6Mb5JpGxZfsi3mBY6NSG941FSLTcqhCKrhBA==", "license": "MIT", "dependencies": { "@cfworker/json-schema": "^4.0.2", diff --git a/gitnexus-web/package.json b/gitnexus-web/package.json index 81d7fc5fa..51d7b0520 100644 --- a/gitnexus-web/package.json +++ b/gitnexus-web/package.json @@ -19,7 +19,7 @@ }, "dependencies": { "gitnexus-shared": "file:../gitnexus-shared", - "@langchain/anthropic": "^1.3.28", + "@langchain/anthropic": "^1.3.29", "@langchain/core": "^1.1.44", "@langchain/google-genai": "^2.1.28", "@langchain/langgraph": "^1.2.9", diff --git a/gitnexus-web/src/core/llm/settings-service.ts b/gitnexus-web/src/core/llm/settings-service.ts index 5e49cb7af..86330d2a5 100644 --- a/gitnexus-web/src/core/llm/settings-service.ts +++ b/gitnexus-web/src/core/llm/settings-service.ts @@ -20,6 +20,7 @@ import { ProviderConfig, } from './types'; import { DEFAULT_OPENROUTER_BASE_URL, DEFAULT_OLLAMA_BASE_URL } from '../../config/ui-constants'; +import { resilientFetch } from 'gitnexus-shared'; const STORAGE_KEY = 'gitnexus-llm-settings'; @@ -407,7 +408,10 @@ export const getAvailableModels = (provider: LLMProvider): string[] => { */ export const fetchOpenRouterModels = async (): Promise> => { try { - const response = await fetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`); + const response = await resilientFetch(`${DEFAULT_OPENROUTER_BASE_URL}/models`, undefined, { + breakerKey: 'openrouter-models', + retry: { maxAttempts: 2, baseDelayMs: 500, capDelayMs: 2_000 }, + }); if (!response.ok) throw new Error('Failed to fetch models'); const data = await response.json(); return data.data.map((model: any) => ({ diff --git a/gitnexus-web/src/services/backend-client.ts b/gitnexus-web/src/services/backend-client.ts index ec8c1a964..e887e3901 100644 --- a/gitnexus-web/src/services/backend-client.ts +++ b/gitnexus-web/src/services/backend-client.ts @@ -7,6 +7,7 @@ */ import type { GraphNode, GraphRelationship } from 'gitnexus-shared'; +import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; // ── Types ────────────────────────────────────────────────────────────────── @@ -204,8 +205,32 @@ export function streamSSE(url: string, handlers: SSEHandlers): A let _backendUrl = 'http://localhost:4747'; +/** + * Validate that a backend URL is a safe http:// or https:// origin before + * storing it as the fetch target base (CodeQL js/client-side-request-forgery). + * + * Throws if the URL uses a non-HTTP scheme (e.g. javascript:, data:, file://). + * All other well-formed http/https URLs are accepted — the client intentionally + * supports connecting to remote GitNexus servers, not just localhost. + */ +export function validateBackendUrl(url: string): void { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + // Do not echo raw input — it may contain credentials. + throw new Error('Invalid backend URL: must be a well-formed http:// or https:// URL'); + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + // Use parsed.protocol only (scheme), not the full URL, to avoid leaking credentials. + throw new Error(`Backend URL must use http:// or https:// (got ${parsed.protocol})`); + } +} + export const setBackendUrl = (url: string): void => { - _backendUrl = url.replace(/\/$/, ''); + const trimmed = url.replace(/\/$/, ''); + validateBackendUrl(trimmed); + _backendUrl = trimmed; }; export const getBackendUrl = (): string => _backendUrl; @@ -237,29 +262,91 @@ export function normalizeServerUrl(input: string): string { const DEFAULT_TIMEOUT_MS = 30_000; const PROBE_TIMEOUT_MS = 2_000; +/** Idempotent HTTP methods. Other verbs (POST, PATCH, PUT, DELETE) get + * a single-attempt retry budget by default to avoid duplicate side + * effects on retry — a POST that 5xx'd may have already executed + * server-side. Callers that have idempotency keys or otherwise know + * their mutation is safe to retry can opt in via `forceRetry`. */ +const IDEMPOTENT_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']); + const fetchWithTimeout = async ( url: string, init: RequestInit = {}, timeoutMs: number = DEFAULT_TIMEOUT_MS, + /** + * Force a retry budget on non-idempotent methods. Default false. + * Pass true only when the endpoint is known-idempotent (e.g. DELETE + * of a known-deleted resource — second call is a 404 / no-op) AND + * the duplicate-side-effect window is acceptable. + */ + forceRetry = false, ): Promise => { - const controller = new AbortController(); - // Merge external signal if provided + // Merge the external caller signal (if any) with an + // `AbortSignal.timeout()` so a timer-fired abort produces a + // `DOMException` with `name === 'TimeoutError'` — which + // `resilientFetch` correctly classifies as terminal-network (no + // retry, no breaker hit). A manual `AbortController.abort()` would + // produce `name === 'AbortError'` and route through the + // retryable-network branch, which mis-penalizes the breaker for + // user-side network slowness. + const timeoutSignal = AbortSignal.timeout(timeoutMs); const externalSignal = init.signal; - if (externalSignal) { - externalSignal.addEventListener('abort', () => controller.abort()); + const signal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal; + + const method = (init.method ?? 'GET').toUpperCase(); + const isIdempotent = IDEMPOTENT_METHODS.has(method); + const maxAttempts = isIdempotent || forceRetry ? 2 : 1; + + // Key the breaker by the current backend origin so switching backend + // URLs (e.g. recovering from a flapping local server by pointing at + // a different host) gives the new origin a fresh breaker state. A + // single shared `'web-backend'` key would otherwise leave a user + // locked out for the full cooldown after one bad host trips the + // circuit. The malformed-URL fallback is defensive — `setBackendUrl` + // normalizes input, so this branch shouldn't fire in practice. + let breakerKey: string; + try { + breakerKey = `web-backend:${new URL(_backendUrl).origin}`; + } catch { + breakerKey = 'web-backend:invalid'; } - const timer = setTimeout(() => controller.abort(), timeoutMs); try { - const response = await fetch(url, { ...init, signal: controller.signal }); + // Bounded retries + 5xx/429 handling are delegated to resilientFetch. + // Method-aware budget: idempotent verbs retry once on transient + // backend failures; mutations (POST/PATCH/PUT/DELETE) default to + // single-attempt to avoid duplicate side effects. + const response = await resilientFetch( + url, + { ...init, signal }, + { + breakerKey, + retry: { maxAttempts, baseDelayMs: 250, capDelayMs: 1500 }, + }, + ); return response; } catch (error: unknown) { - if (error instanceof DOMException && error.name === 'AbortError') { - if (externalSignal?.aborted) { - throw new BackendError('Request aborted', 0, 'network'); - } + if (error instanceof CircuitOpenError) { + throw new BackendError( + `GitNexus backend at ${_backendUrl} is unhealthy; retry in ${Math.ceil(error.retryAfterMs / 1000)}s`, + 0, + 'network', + ); + } + if (error instanceof ResilientFetchExhaustedError) { + // Fall through to caller — surface the raw response so assertOk + // can craft the BackendError with the right code. + return error.response; + } + if (error instanceof DOMException && error.name === 'TimeoutError') { throw new BackendError(`Request to ${url} timed out after ${timeoutMs}ms`, 0, 'timeout'); } + if (error instanceof DOMException && error.name === 'AbortError') { + // External caller-driven cancellation — `timeoutSignal` would + // have surfaced as TimeoutError above, so this branch covers + // only the externally-aborted case. + throw new BackendError('Request aborted', 0, 'network'); + } if (error instanceof TypeError) { throw new BackendError( `Network error reaching GitNexus backend at ${_backendUrl}: ${error.message}`, @@ -268,8 +355,6 @@ const fetchWithTimeout = async ( ); } throw error; - } finally { - clearTimeout(timer); } }; diff --git a/gitnexus-web/test/unit/backend-client-retry.test.ts b/gitnexus-web/test/unit/backend-client-retry.test.ts new file mode 100644 index 000000000..ea0fcf3a7 --- /dev/null +++ b/gitnexus-web/test/unit/backend-client-retry.test.ts @@ -0,0 +1,110 @@ +/** + * Method-aware retry budget + timeout-as-TimeoutError verification for + * backend-client's `fetchWithTimeout`. + * + * Closes review findings on PR #1448: + * - Non-idempotent POST/DELETE must NOT be retried by default — + * a 5xx on `startAnalyze` could otherwise start a duplicate job. + * - Timer-fired timeout must surface as `DOMException(name='TimeoutError')`, + * not `AbortError`, so resilientFetch routes it through the + * terminal-network branch (no retry, no breaker hit). + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { getBreaker } from 'gitnexus-shared'; +import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers'; +import { fetchRepos, setBackendUrl, startAnalyze } from '../../src/services/backend-client'; + +const BASE = 'http://localhost:4747'; + +describe('backend-client retry budget (method-aware)', () => { + beforeEach(() => { + __resetBreakerRegistry__(); + setBackendUrl(BASE); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('GET retries once on transient 503 (idempotent verb)', async () => { + let n = 0; + const fetchMock = vi.fn(async () => { + n += 1; + if (n === 1) return new Response('boom', { status: 503 }); + return new Response('[]', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }); + }); + vi.stubGlobal('fetch', fetchMock); + + const repos = await fetchRepos(); + expect(repos).toEqual([]); + // 1 retry budget on idempotent GET → 2 total fetch calls. + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it('POST does NOT retry on 503 by default (non-idempotent verb)', async () => { + const fetchMock = vi.fn(async () => new Response('boom', { status: 503 })); + vi.stubGlobal('fetch', fetchMock); + + await expect(startAnalyze({ path: '/tmp/repo' })).rejects.toBeTruthy(); + // Single attempt — never duplicates a job-start POST. + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it('switching backend URL after a circuit opens reaches a fresh breaker (U3)', async () => { + // Pre-open the breaker for host-A by directly recording 3 failures. + setBackendUrl('http://host-a.test:4747'); + const aKey = 'web-backend:http://host-a.test:4747'; + const breakerA = getBreaker(aKey); + breakerA.recordFailure(); + breakerA.recordFailure(); + breakerA.recordFailure(); + expect(breakerA.getState()).toBe('open'); + + // Switch to host-B and make a request — must succeed against the + // new origin without tripping the host-A circuit. Under the old + // single-key behaviour the call would throw CircuitOpenError. + setBackendUrl('http://host-b.test:4747'); + const fetchMock = vi.fn( + async () => + new Response('[]', { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }), + ); + vi.stubGlobal('fetch', fetchMock); + + const repos = await fetchRepos(); + expect(repos).toEqual([]); + expect(fetchMock).toHaveBeenCalledTimes(1); + + // Host-A's breaker is still open in cooldown. + expect(breakerA.getState()).toBe('open'); + // Host-B has its own (fresh) breaker. + const bKey = 'web-backend:http://host-b.test:4747'; + expect(getBreaker(bKey).getState()).toBe('closed'); + expect(getBreaker(bKey).getConsecutiveFailures()).toBe(0); + }); + + it('breaker not incremented when timeout fires (TimeoutError, not AbortError)', async () => { + // Reject directly with a TimeoutError DOMException, mimicking what + // `fetch` produces when its `AbortSignal.timeout()`-wired signal + // fires. The real-fetch path goes signal.reason → reject(reason); + // we shortcut that here so the test doesn't have to wait the + // 30-second default timeout. + const fetchMock = vi.fn(async () => { + throw new DOMException('aborted by timeout', 'TimeoutError'); + }); + vi.stubGlobal('fetch', fetchMock); + + await expect(fetchRepos()).rejects.toMatchObject({ code: 'timeout' }); + + // The breaker must not have been penalized for a local timeout. + expect(getBreaker(`web-backend:${BASE}`).getConsecutiveFailures()).toBe(0); + // Timeout is terminal — no retry attempted. + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/gitnexus-web/test/unit/server-connection.test.ts b/gitnexus-web/test/unit/server-connection.test.ts index f5ee43c53..e39b829a1 100644 --- a/gitnexus-web/test/unit/server-connection.test.ts +++ b/gitnexus-web/test/unit/server-connection.test.ts @@ -1,5 +1,11 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; -import { fetchGraph, normalizeServerUrl, setBackendUrl } from '../../src/services/backend-client'; +import { + fetchGraph, + getBackendUrl, + normalizeServerUrl, + setBackendUrl, + validateBackendUrl, +} from '../../src/services/backend-client'; describe('normalizeServerUrl', () => { it('adds http:// to localhost', () => { @@ -165,3 +171,61 @@ describe('fetchGraph', () => { }); }); }); + +describe('validateBackendUrl', () => { + it('allows http:// URLs', () => { + expect(() => validateBackendUrl('http://localhost:4747')).not.toThrow(); + expect(() => validateBackendUrl('http://127.0.0.1:4747')).not.toThrow(); + }); + + it('allows https:// URLs', () => { + expect(() => validateBackendUrl('https://gitnexus.example.com')).not.toThrow(); + expect(() => validateBackendUrl('https://my-server.internal:4747')).not.toThrow(); + }); + + it('rejects non-http schemes', () => { + expect(() => validateBackendUrl('javascript:alert(1)')).toThrow('must use http:// or https://'); + expect(() => validateBackendUrl('file:///etc/passwd')).toThrow('must use http:// or https://'); + expect(() => validateBackendUrl('data:text/plain,evil')).toThrow( + 'must use http:// or https://', + ); + }); + + it('rejects malformed URLs', () => { + expect(() => validateBackendUrl('not-a-url')).toThrow('Invalid backend URL'); + }); + + it('does not include the raw URL in error messages (credential hygiene)', () => { + const urlWithCreds = 'javascript:alert("sk-secret")'; + let msg = ''; + try { + validateBackendUrl(urlWithCreds); + } catch (e) { + msg = (e as Error).message; + } + expect(msg).not.toContain('sk-secret'); + expect(msg).not.toContain(urlWithCreds); + }); +}); + +describe('setBackendUrl', () => { + it('accepts valid http URLs', () => { + expect(() => setBackendUrl('http://localhost:4747')).not.toThrow(); + }); + + it('accepts valid https URLs', () => { + expect(() => setBackendUrl('https://my-server.example.com')).not.toThrow(); + }); + + it('rejects non-http/https schemes', () => { + expect(() => setBackendUrl('javascript:alert(1)')).toThrow('must use http:// or https://'); + expect(() => setBackendUrl('file:///etc/passwd')).toThrow('must use http:// or https://'); + }); + + it('does not mutate _backendUrl when validation fails', () => { + setBackendUrl('http://localhost:4747'); + expect(() => setBackendUrl('javascript:alert(1)')).toThrow(); + // State must be preserved — validation must happen before the assignment + expect(getBackendUrl()).toBe('http://localhost:4747'); + }); +}); diff --git a/gitnexus/Dockerfile.test b/gitnexus/Dockerfile.test index 0282129ff..37374a5f5 100644 --- a/gitnexus/Dockerfile.test +++ b/gitnexus/Dockerfile.test @@ -1,6 +1,15 @@ -FROM node:20-bookworm +# Pinned npm version — keep in sync with the root Dockerfile.cli and +# Dockerfile.web. +ARG NPM_VERSION=11.14.1 + +# node:22-bookworm-slim +FROM node:22-bookworm-slim@sha256:9f6d5975c7dca860947d3915877f85607946403fc55349f39b4bc3688448bb6e +ARG NPM_VERSION WORKDIR /app -RUN apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update && apt-get install -y python3 make g++ && rm -rf /var/lib/apt/lists/* +RUN npx --yes npm@${NPM_VERSION} install -g npm@${NPM_VERSION} \ + && apt-get -o Acquire::Check-Valid-Until=false -o Acquire::Check-Date=false update \ + && apt-get install -y python3 make g++ \ + && rm -rf /var/lib/apt/lists/* COPY . . RUN npm ci --ignore-scripts \ && npm rebuild tree-sitter-swift 2>&1 \ diff --git a/gitnexus/package.json b/gitnexus/package.json index 84f702762..810052502 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -116,6 +116,6 @@ } }, "engines": { - "node": ">=20.0.0" + "node": ">=22.0.0" } } diff --git a/gitnexus/src/cli/index.ts b/gitnexus/src/cli/index.ts index b89b40db0..e4a455d40 100644 --- a/gitnexus/src/cli/index.ts +++ b/gitnexus/src/cli/index.ts @@ -160,6 +160,18 @@ program .description('Augment a search pattern with knowledge graph context (used by hooks)') .action(createLazyAction(() => import('./augment.js'), 'augmentCommand')); +program + .command('publish [path]') + .description( + 'Notify the understand-quickly registry that this repo has a fresh GitNexus index. ' + + 'Opt-in: requires UNDERSTAND_QUICKLY_TOKEN (fine-grained PAT with ' + + '`Repository dispatches: write` on looptech-ai/understand-quickly). ' + + 'No-op without the token. See https://github.com/looptech-ai/understand-quickly.', + ) + .option('--id ', 'Override the registry id (defaults to the origin remote)') + .option('--skip-git', 'Treat cwd as the repo root and skip parent git-root discovery') + .action(createLazyAction(() => import('./publish.js'), 'publishCommand')); + // ─── Direct Tool Commands (no MCP overhead) ──────────────────────── // These invoke LocalBackend directly for use in eval, scripts, and CI. diff --git a/gitnexus/src/cli/publish.ts b/gitnexus/src/cli/publish.ts new file mode 100644 index 000000000..8aedc9c35 --- /dev/null +++ b/gitnexus/src/cli/publish.ts @@ -0,0 +1,232 @@ +/** + * `gitnexus publish` — opt-in ping to the understand-quickly registry. + * + * Fires a single `repository_dispatch` event at + * `looptech-ai/understand-quickly` so the registry knows to refresh its + * entry for the current repo. Does NOT upload anything: per the + * understand-quickly protocol, the registry pulls the graph from a + * raw-GitHub URL the user controls. + * + * https://github.com/looptech-ai/understand-quickly/blob/main/docs/integrations/protocol.md + * + * Defaults: + * - Without `UNDERSTAND_QUICKLY_TOKEN` in the env, this is a no-op + * (prints one informational line, exit 0). Same shape as the + * `--publish` patterns in sibling tools. + * - With the token, fires the dispatch and reports the response code. + * + * The `id` is derived from the repo's `origin` remote unless the caller + * passes `--id ` explicitly. We deliberately do NOT auto-add + * the repo to the registry — registration is one-time and uses the + * `npx @understand-quickly/cli add` path documented in the protocol. + */ + +import path from 'path'; +import { + UNDERSTAND_QUICKLY_DISPATCH_URL, + UNDERSTAND_QUICKLY_TOKEN_ENV, + buildUqDispatchPayload, + isValidOwnerRepo, + parseOwnerRepoFromRemote, +} from 'gitnexus-shared'; +import { getGitRoot, getRemoteOriginUrl, getCurrentCommit } from '../storage/git.js'; +import { hasIndex } from '../storage/repo-manager.js'; +import { cliInfo, cliError } from './cli-message.js'; + +export interface PublishOptions { + /** Override the auto-derived `owner/repo` id. */ + id?: string; + /** Treat the cwd as the repo root (skip git-root walk). */ + skipGit?: boolean; +} + +const REGISTER_HINT = + 'Register your repo once with: npx @understand-quickly/cli add\n' + + 'Or use the wizard: https://looptech-ai.github.io/understand-quickly/add.html'; + +/** + * Hard cap on the dispatch fetch to keep CI publish steps from stalling + * for the OS TCP timeout (~2 min) when api.github.com is unreachable. + * Matches the pattern used in `src/core/embeddings/http-client.ts`. + */ +const DISPATCH_TIMEOUT_MS = 15_000; + +export const publishCommand = async ( + inputPath?: string, + options: PublishOptions = {}, +): Promise => { + // ── 0. Token gate FIRST — guarantees true no-op without the token. ── + // The README, CLI --help, and PR body all promise "exit 0 without + // UNDERSTAND_QUICKLY_TOKEN". Doing the index/repo-root checks before + // the token gate would make those promises false for users who haven't + // run `gitnexus analyze` yet but want to verify the command is wired. + const token = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + if (!token) { + cliInfo( + `[understand-quickly] ${UNDERSTAND_QUICKLY_TOKEN_ENV} is not set — skipping dispatch.\n` + + `Set it to a fine-grained PAT with "Repository dispatches: write" on ` + + `looptech-ai/understand-quickly to enable instant resync.\n` + + `(Without the token, the registry's nightly sync still picks up your entry.)`, + { skipped: 'no-token' }, + ); + return; + } + + // ── 1. Resolve the repo root (same precedence as `analyze`) ────────── + let repoPath: string; + if (inputPath) { + repoPath = path.resolve(inputPath); + } else if (options.skipGit) { + repoPath = path.resolve(process.cwd()); + } else { + const gitRoot = getGitRoot(process.cwd()); + if (!gitRoot) { + cliError( + '[understand-quickly] not inside a git repository.\n' + + 'Run from a repo, or pass --skip-git to publish from the current directory.', + ); + process.exitCode = 1; + return; + } + repoPath = gitRoot; + } + + // ── 2. Confirm a GitNexus index exists ─────────────────────────────── + // Publishing without an index is almost always a mistake — the + // registry's nightly sync would fetch a stale or missing graph file + // and mark the entry `missing`. Refuse loudly with a fix-it hint. + if (!(await hasIndex(repoPath))) { + cliError( + `[understand-quickly] no GitNexus index found at ${repoPath}/.gitnexus.\n` + + 'Run `gitnexus analyze` first, then re-run `gitnexus publish`.', + ); + process.exitCode = 1; + return; + } + + // ── 3. Derive the registry id ───────────────────────────────────────── + const id = + options.id ?? parseOwnerRepoFromRemote(getRemoteOriginUrl(repoPath) ?? undefined) ?? null; + if (!id || !isValidOwnerRepo(id)) { + cliError( + `[understand-quickly] could not derive a registry id from this repo.\n` + + `Pass --id explicitly (e.g. --id looptech-ai/${path.basename(repoPath)}).\n` + + REGISTER_HINT, + ); + process.exitCode = 1; + return; + } + + // ── 4. Fire the dispatch ───────────────────────────────────────────── + const payload = buildUqDispatchPayload(id); + let response: Response; + try { + response = await fetch(UNDERSTAND_QUICKLY_DISPATCH_URL, { + method: 'POST', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + 'User-Agent': 'gitnexus-cli', + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(DISPATCH_TIMEOUT_MS), + }); + } catch (err) { + // `AbortSignal.timeout()` throws a `DOMException` with `name === + // 'TimeoutError'` on Node 18.14+ (and on browsers/Bun). It is NOT + // a plain `AbortError`. Match the pattern used in + // gitnexus/src/core/embeddings/http-client.ts so the user sees the + // targeted "timed out" message instead of a generic "operation + // was aborted". + const isTimeout = err instanceof DOMException && err.name === 'TimeoutError'; + if (isTimeout) { + cliError( + `[understand-quickly] dispatch timed out after ${DISPATCH_TIMEOUT_MS}ms. ` + + `Check network access to api.github.com and retry.`, + { id }, + ); + } else { + const msg = err instanceof Error ? err.message : String(err); + cliError(`[understand-quickly] dispatch network error: ${msg}`, { id }); + } + process.exitCode = 1; + return; + } + + // GitHub returns 204 on success. Distinct branches for 401/403/404/422 + // so users debug without checking the docs. + if (response.status === 204) { + await response.body?.cancel().catch(() => {}); + // `getCurrentCommit` is only meaningful in the success path — moving + // it inside this branch removes a wasted child-process spawn on every + // error response (LOW 7). + const commit = getCurrentCommit(repoPath); + cliInfo( + `[understand-quickly] dispatched sync-entry for ${id}` + + (commit ? ` @ ${commit.slice(0, 7)}` : '') + + '.\n' + + `Note: a 204 only confirms GitHub accepted the dispatch. Whether the ` + + `registry workflow finds an entry for "${id}" is logged at ` + + `https://github.com/looptech-ai/understand-quickly/actions/workflows/sync.yml`, + { id, commit, status: response.status }, + ); + return; + } + + if (response.status === 401) { + cliError( + `[understand-quickly] dispatch returned 401 — the ${UNDERSTAND_QUICKLY_TOKEN_ENV} value is invalid or expired.\n` + + `Regenerate a fine-grained PAT at https://github.com/settings/personal-access-tokens ` + + `with Repository access scoped to looptech-ai/understand-quickly and the ` + + `"Repository dispatches: write" permission, then retry.`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + if (response.status === 403) { + cliError( + `[understand-quickly] dispatch returned 403 — the token authenticated but ` + + `lacks the "Repository dispatches: write" permission on ` + + `looptech-ai/understand-quickly. Edit the PAT scopes and retry.`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + if (response.status === 404) { + cliError( + `[understand-quickly] dispatch returned 404 — the token cannot reach ` + + `looptech-ai/understand-quickly. Verify the PAT has Repository access to ` + + `that exact repo (not just your own org).`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + if (response.status === 422) { + // Malformed event_type / client_payload — a code bug in this CLI, + // not a user mistake. Surface so we get bug reports. + const body422 = await response.text().catch(() => ''); + cliError( + `[understand-quickly] dispatch returned 422 (this is a CLI bug; please report).\n` + + `Body: ${body422 || '(empty)'}`, + { id, status: response.status }, + ); + process.exitCode = 1; + return; + } + + // 5xx and anything else → bubble the body so the user has something to act on. + const body = await response.text().catch(() => ''); + cliError( + `[understand-quickly] dispatch failed with HTTP ${response.status}: ${body || '(empty body)'}`, + { id, status: response.status }, + ); + process.exitCode = 1; +}; diff --git a/gitnexus/src/config/ignore-service.ts b/gitnexus/src/config/ignore-service.ts index ce1fda913..2c3eebe3b 100644 --- a/gitnexus/src/config/ignore-service.ts +++ b/gitnexus/src/config/ignore-service.ts @@ -25,6 +25,8 @@ const DEFAULT_IGNORE_LIST = new Set([ 'bower_components', 'jspm_packages', 'vendor', // PHP/Go + 'third_party', // C/C++ (Google-style vendored dependencies) + '3rdparty', // C/C++ (alternate spelling, also Qt convention) // 'packages' removed - commonly used for monorepo source code (lerna, pnpm, yarn workspaces) 'venv', '.venv', diff --git a/gitnexus/src/core/embeddings/hf-env.ts b/gitnexus/src/core/embeddings/hf-env.ts index 95548fff2..5ae6d89ae 100644 --- a/gitnexus/src/core/embeddings/hf-env.ts +++ b/gitnexus/src/core/embeddings/hf-env.ts @@ -1,6 +1,8 @@ import os from 'node:os'; import { join } from 'node:path'; +import { CircuitBreaker, withRetry } from 'gitnexus-shared'; + // --------------------------------------------------------------------------- // Download resilience defaults // --------------------------------------------------------------------------- @@ -108,70 +110,19 @@ export function isNetworkFetchError(message: string): boolean { /** @internal Used by `withHfDownloadRetry` to mark a circuit-open rejection. */ export const CIRCUIT_OPEN_TAG = 'hf-circuit-open'; -/** Circuit-breaker states. */ -type CircuitState = 'closed' | 'open' | 'half-open'; - /** - * Circuit breaker for HuggingFace model downloads. - * - * After `failureThreshold` consecutive network failures the circuit opens and - * all subsequent calls to `withHfDownloadRetry` fail immediately without - * issuing any network requests. After `resetTimeoutMs` the circuit enters the - * half-open state and the next call is attempted — if it succeeds the circuit - * closes again; if it fails the circuit re-opens. - * - * Exported for unit-testing; production code should use the module-level - * `hfDownloadCircuit` singleton. + * Module-level singleton shared by both embedder entry points + * (`core/embeddings/embedder.ts` + `mcp/core/embedder.ts`). Per-process + * only — not persisted across restarts. Backed by the shared + * `CircuitBreaker` from `gitnexus-shared` (same state machine, same + * semantics, plus the single-permit half-open gate that prevents + * recovery-time stampedes). */ -export class HfDownloadCircuitBreaker { - private _state: CircuitState = 'closed'; - private _failures = 0; - /** Timestamp of the last recorded failure (ms since epoch). */ - lastFailureAt = 0; - - constructor( - readonly failureThreshold: number = CB_FAILURE_THRESHOLD, - readonly resetTimeoutMs: number = CB_RESET_TIMEOUT_MS, - ) {} - - /** Effective state, factoring in the reset-timeout transition. */ - get state(): CircuitState { - if (this._state === 'open' && Date.now() - this.lastFailureAt > this.resetTimeoutMs) { - this._state = 'half-open'; - } - return this._state; - } - - /** Returns true when the circuit is open and calls should be rejected. */ - isOpen(): boolean { - return this.state === 'open'; - } - - /** Record a successful call — resets the failure counter and closes the circuit. */ - recordSuccess(): void { - this._failures = 0; - this._state = 'closed'; - } - - /** Record a failed call — increments the counter and opens the circuit when the threshold is reached. */ - recordFailure(): void { - this._failures++; - this.lastFailureAt = Date.now(); - if (this._failures >= this.failureThreshold) { - this._state = 'open'; - } - } - - /** @internal Reset to initial state (used in tests). */ - reset(): void { - this._failures = 0; - this._state = 'closed'; - this.lastFailureAt = 0; - } -} - -/** Module-level singleton shared by both embedder entry points. */ -export const hfDownloadCircuit = new HfDownloadCircuitBreaker(); +export const hfDownloadCircuit = new CircuitBreaker({ + failureThreshold: CB_FAILURE_THRESHOLD, + cooldownMs: CB_RESET_TIMEOUT_MS, + key: 'hf-download', +}); // --------------------------------------------------------------------------- // Retry + timeout wrapper @@ -219,11 +170,6 @@ export function withDownloadTimeout(fn: () => Promise, timeoutMs: number): }); } -/** @internal Async sleep (exposed for testing). */ -export function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - export interface HfRetryOptions { /** Maximum total attempts including the initial one (default: `HF_MAX_ATTEMPTS`). */ maxAttempts?: number; @@ -235,7 +181,7 @@ export interface HfRetryOptions { * Circuit-breaker instance to use. Defaults to the module-level * `hfDownloadCircuit` singleton. Pass a fresh instance in tests. */ - circuit?: HfDownloadCircuitBreaker; + circuit?: CircuitBreaker; /** * Optional callback invoked before each retry (not the initial attempt). * @param attempt - 1-based retry number @@ -295,49 +241,74 @@ export async function withHfDownloadRetry( circuit = hfDownloadCircuit, onRetry, } = options; - if (circuit.isOpen()) { - const secsUntilReset = Math.ceil( - (circuit.resetTimeoutMs - (Date.now() - circuit.lastFailureAt)) / 1000, - ); + if (circuit.getState() === 'open') { + // Compute remaining cooldown without consuming a probe permit. + const openedAt = circuit.getOpenedAt(); + const secsUntilReset = + openedAt !== null ? Math.ceil((circuit.getCooldownMs() - (Date.now() - openedAt)) / 1000) : 0; throw new Error( `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit is open after repeated network failures` + (secsUntilReset > 0 ? ` — will reset in ~${secsUntilReset}s` : ''), ); } - let lastError: Error = new Error('unknown error'); + // Retry budget delegated to `withRetry` from gitnexus-shared. The + // HF-specific bits — per-attempt timeout, network-vs-non-network + // classification, circuit-breaker recording, onRetry callback — wire + // through the `isRetryable` callback. `circuitTripped` is the + // sentinel that lets us replace the final thrown error with a + // CIRCUIT_OPEN_TAG message when the breaker tripped mid-loop. + let circuitTripped = false; - for (let attempt = 0; attempt < maxAttempts; attempt++) { - try { - const result = await withDownloadTimeout(fn, timeoutMs); - circuit.recordSuccess(); - return result; - } catch (err) { - lastError = err instanceof Error ? err : new Error(String(err)); - - if (!isNetworkFetchError(lastError.message)) { - // Non-network error (e.g. CUDA unavailable) — propagate without retry - throw lastError; - } - - circuit.recordFailure(); - - if (circuit.isOpen()) { - // Circuit just tripped — fail fast, no more retries - throw new Error( - `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${circuit.failureThreshold} consecutive failures`, - ); - } - - if (attempt < maxAttempts - 1) { - const delay = baseDelayMs * Math.pow(2, attempt); - onRetry?.(attempt + 1, maxAttempts, lastError); - await sleep(delay); - } + try { + return await withRetry( + async () => { + const result = await withDownloadTimeout(fn, timeoutMs); + circuit.recordSuccess(); + return result; + }, + { + maxAttempts, + baseDelayMs, + // Disable the cap to match the bespoke pure-exponential + // progression. With the default `HF_MAX_ATTEMPTS_CAP = 10` and + // `baseDelayMs = 2000`, the largest possible delay is + // `2000 * 2^9 = ~17 minutes` — bounded enough not to need a cap. + capDelayMs: Number.MAX_SAFE_INTEGER, + isRetryable: (err, attempt) => { + const error = err instanceof Error ? err : new Error(String(err)); + if (!isNetworkFetchError(error.message)) { + // Non-network error (e.g. CUDA unavailable) — propagate + // without retry. Use recordNeutral so the breaker's existing + // failure-count progress isn't reset by a non-network failure + // that says nothing about the CDN's health. + circuit.recordNeutral(); + return { retry: false }; + } + circuit.recordFailure(); + if (circuit.getState() === 'open') { + // Circuit just tripped — fail fast, no more retries. + circuitTripped = true; + return { retry: false }; + } + // Mirror the bespoke onRetry contract: fire only when there's + // actually a next attempt. + if (attempt + 1 < maxAttempts) { + onRetry?.(attempt + 1, maxAttempts, error); + } + return { retry: true }; + }, + }, + ); + } catch (err) { + if (circuitTripped) { + throw new Error( + `${CIRCUIT_OPEN_TAG}: HuggingFace download circuit opened after ${CB_FAILURE_THRESHOLD} consecutive failures`, + ); } + // All retries exhausted — rethrow the last network error so + // isNetworkFetchError patterns in the calling code still match and + // surface HF_ENDPOINT guidance. + throw err; } - - // All retries exhausted — throw the last network error so isNetworkFetchError - // patterns in the calling code still match and surface HF_ENDPOINT guidance. - throw lastError; } diff --git a/gitnexus/src/core/embeddings/http-client.ts b/gitnexus/src/core/embeddings/http-client.ts index 85ad79111..e3fb06045 100644 --- a/gitnexus/src/core/embeddings/http-client.ts +++ b/gitnexus/src/core/embeddings/http-client.ts @@ -3,13 +3,22 @@ * * Shared fetch+retry logic for OpenAI-compatible /v1/embeddings endpoints. * Imported by both the core embedder (batch) and MCP embedder (query). + * + * Network resilience is delegated to `resilientFetch` from + * `gitnexus-shared` — bounded retries with exponential-backoff jitter, + * `Retry-After` honored on 429, and an in-process circuit breaker that + * fails fast on a flapping endpoint. Per-attempt timeout is enforced + * via `AbortSignal.timeout` on the underlying fetch. */ +import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; + const HTTP_TIMEOUT_MS = 30_000; const HTTP_MAX_RETRIES = 2; const HTTP_RETRY_BACKOFF_MS = 1_000; const HTTP_BATCH_SIZE = 64; const DEFAULT_DIMS = 384; +const HTTP_BREAKER_KEY = 'embeddings-http'; interface HttpConfig { baseUrl: string; @@ -90,46 +99,51 @@ const httpEmbedBatch = async ( model: string, apiKey: string, batchIndex = 0, - attempt = 0, ): Promise => { let resp: Response; try { - resp = await fetch(url, { - method: 'POST', - signal: AbortSignal.timeout(HTTP_TIMEOUT_MS), - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${apiKey}`, + resp = await resilientFetch( + url, + { + method: 'POST', + signal: AbortSignal.timeout(HTTP_TIMEOUT_MS), + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ input: batch, model }), }, - body: JSON.stringify({ input: batch, model }), - }); + { + breakerKey: HTTP_BREAKER_KEY, + retry: { maxAttempts: HTTP_MAX_RETRIES + 1, baseDelayMs: HTTP_RETRY_BACKOFF_MS }, + }, + ); } catch (err) { - // Timeouts should not be retried — the server is unresponsive. - // AbortSignal.timeout() throws DOMException with name 'TimeoutError'. - const isTimeout = err instanceof DOMException && err.name === 'TimeoutError'; - if (isTimeout) { + if (err instanceof CircuitOpenError) { + throw new Error( + `Embedding endpoint circuit open (${safeUrl(url)}, batch ${batchIndex}): retry in ${Math.ceil(err.retryAfterMs / 1000)}s`, + ); + } + if (err instanceof DOMException && err.name === 'TimeoutError') { throw new Error( `Embedding request timed out after ${HTTP_TIMEOUT_MS}ms (${safeUrl(url)}, batch ${batchIndex})`, ); } - // DNS, connection errors — retry with backoff - if (attempt < HTTP_MAX_RETRIES) { - const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1); - await new Promise((r) => setTimeout(r, delay)); - return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1); + if (err instanceof ResilientFetchExhaustedError) { + throw new Error( + `Embedding endpoint returned ${err.response.status} (${safeUrl(url)}, batch ${batchIndex})`, + ); } const reason = err instanceof Error ? err.message : String(err); throw new Error(`Embedding request failed (${safeUrl(url)}, batch ${batchIndex}): ${reason}`); } if (!resp.ok) { - const status = resp.status; - if ((status === 429 || status >= 500) && attempt < HTTP_MAX_RETRIES) { - const delay = HTTP_RETRY_BACKOFF_MS * (attempt + 1); - await new Promise((r) => setTimeout(r, delay)); - return httpEmbedBatch(url, batch, model, apiKey, batchIndex, attempt + 1); - } - throw new Error(`Embedding endpoint returned ${status} (${safeUrl(url)}, batch ${batchIndex})`); + // resilientFetch already retried 5xx/429; any non-OK response here is + // a terminal client error (4xx other than 429). + throw new Error( + `Embedding endpoint returned ${resp.status} (${safeUrl(url)}, batch ${batchIndex})`, + ); } const data = (await resp.json()) as { data: EmbeddingItem[] }; diff --git a/gitnexus/src/core/group/bridge-db.ts b/gitnexus/src/core/group/bridge-db.ts index ef6244b22..7f44253bf 100644 --- a/gitnexus/src/core/group/bridge-db.ts +++ b/gitnexus/src/core/group/bridge-db.ts @@ -722,13 +722,15 @@ export async function openBridgeDbReadOnly(groupDir: string): Promise setTimeout(r, delay)); } } - // Pino's NDJSON serialization is structurally injection-resistant - // (CodeQL js/log-injection): groupDir and err.message are JSON-escaped - // by the serializer, so no manual CRLF / U+2028 / ANSI sanitization is - // needed. Demoted to debug — only fires when the bridge truly gave up - // after retries, and operators only need it at debug verbosity. + // Strip CRLF from user-controlled strings before logging to close + // CodeQL js/log-injection. Pino's NDJSON serialization already + // JSON-escapes all values, but we sanitize here as a defence-in-depth + // measure so CodeQL can see the taint flow is broken. + const safeGroupDir = String(groupDir).replace(/[\r\n]/g, ' '); + const safeErrMsg = + lastErr instanceof Error ? String(lastErr.message).replace(/[\r\n]/g, ' ') : undefined; bridgeLogger.debug( - { groupDir, err: lastErr, attempts: LBUG_OPEN_RETRY_ATTEMPTS }, + { groupDir: safeGroupDir, errMsg: safeErrMsg, attempts: LBUG_OPEN_RETRY_ATTEMPTS }, 'openBridgeDbReadOnly gave up', ); return null; diff --git a/gitnexus/src/core/group/config-parser.ts b/gitnexus/src/core/group/config-parser.ts index 73a9021b9..29c868171 100644 --- a/gitnexus/src/core/group/config-parser.ts +++ b/gitnexus/src/core/group/config-parser.ts @@ -4,9 +4,26 @@ import type { GroupConfig, GroupManifestLink, ContractType, ContractRole } from const _require = createRequire(import.meta.url); const yaml = _require('js-yaml') as typeof import('js-yaml'); -const VALID_CONTRACT_TYPES: ContractType[] = ['http', 'grpc', 'thrift', 'topic', 'lib', 'custom']; +const VALID_CONTRACT_TYPES: ContractType[] = [ + 'http', + 'grpc', + 'thrift', + 'topic', + 'lib', + 'custom', + 'include', +]; const VALID_ROLES: ContractRole[] = ['provider', 'consumer']; +// Defaults matter for backward compatibility: any group.yaml that omits a +// `detect.` key inherits its value from this constant. Adding a new +// extractor that defaults to `true` silently changes the behavior of every +// existing group on the next sync. New extractors must default to `false` +// (opt-in) so operators consciously enable them via group.yaml. +// +// `includes`: opt-in. The C/C++ IncludeExtractor (PR #1156) ships disabled by +// default; enable with `detect.includes: true` for groups containing C/C++ +// repos that need cross-repo header tracking. const DEFAULT_DETECT = { http: true, grpc: true, @@ -14,6 +31,7 @@ const DEFAULT_DETECT = { topics: true, shared_libs: true, embedding_fallback: true, + includes: false, workspace_deps: false, }; diff --git a/gitnexus/src/core/group/extractors/include-extractor.ts b/gitnexus/src/core/group/extractors/include-extractor.ts new file mode 100644 index 000000000..7bbfd61ed --- /dev/null +++ b/gitnexus/src/core/group/extractors/include-extractor.ts @@ -0,0 +1,610 @@ +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import { glob } from 'glob'; +import Parser from 'tree-sitter'; +import C from 'tree-sitter-c'; +import Cpp from 'tree-sitter-cpp'; +import type { ContractExtractor, CypherExecutor } from '../contract-extractor.js'; +import type { ExtractedContract, RepoHandle } from '../types.js'; +import { readSafe } from './fs-utils.js'; +import { buildSuffixIndex, type SuffixIndex } from '../../ingestion/import-resolvers/utils.js'; +import { createIgnoreFilter } from '../../../config/ignore-service.js'; +import { getMaxFileSizeBytes } from '../../ingestion/utils/max-file-size.js'; +import { logger } from '../../logger.js'; + +/** + * Cross-repo C/C++ `#include` dependency extractor. + * + * **Provider side:** registers every `.h/.hpp/.hxx/.hh` file in the repo + * as a provider contract with `include::`. + * + * **Consumer side:** parses all C/C++ source/header files for `#include "…"` + * directives, attempts suffix-based resolution against the repo's own file + * list (reusing the same algorithm as the single-repo ingestion pipeline), + * and emits unresolved include paths as consumer contracts. + * + * Matching: a consumer's `include::map/base/dice_map_view.h` in repo A + * matches a provider's `include::map/base/dice_map_view.h` in repo B via + * exact contract-id equality in `runExactMatch`. + */ + +// ---------- constants ---------- + +const HEADER_EXTENSIONS = new Set(['.h', '.hpp', '.hxx', '.hh']); + +// Source = headers (provider-eligible) ∪ implementation files (.c/.cpp/.cc/.cxx). +// Spread keeps the subset relationship explicit so a future contributor adding +// a new header extension to HEADER_EXTENSIONS does not have to remember to +// also add it here. +const SOURCE_EXTENSIONS = new Set([...HEADER_EXTENSIONS, '.c', '.cpp', '.cc', '.cxx']); + +const INCLUDE_QUERY_SRC = '(preproc_include path: (_) @import.source) @import'; + +/** + * Well-known C/C++ standard library headers that can appear in `#include "…"` + * form (some projects use quotes for system headers). + */ +const SYSTEM_HEADERS = new Set([ + // C standard + 'assert.h', + 'complex.h', + 'ctype.h', + 'errno.h', + 'fenv.h', + 'float.h', + 'inttypes.h', + 'iso646.h', + 'limits.h', + 'locale.h', + 'math.h', + 'setjmp.h', + 'signal.h', + 'stdalign.h', + 'stdarg.h', + 'stdatomic.h', + 'stdbool.h', + 'stddef.h', + 'stdint.h', + 'stdio.h', + 'stdlib.h', + 'stdnoreturn.h', + 'string.h', + 'tgmath.h', + 'threads.h', + 'time.h', + 'uchar.h', + 'wchar.h', + 'wctype.h', + // C++ standard (extensionless) + 'algorithm', + 'any', + 'array', + 'atomic', + 'barrier', + 'bit', + 'bitset', + 'cassert', + 'cctype', + 'cerrno', + 'cfenv', + 'cfloat', + 'charconv', + 'chrono', + 'cinttypes', + 'climits', + 'clocale', + 'cmath', + 'codecvt', + 'compare', + 'complex', + 'concepts', + 'condition_variable', + 'coroutine', + 'csetjmp', + 'csignal', + 'cstdarg', + 'cstddef', + 'cstdint', + 'cstdio', + 'cstdlib', + 'cstring', + 'ctime', + 'cuchar', + 'cwchar', + 'cwctype', + 'deque', + 'exception', + 'execution', + 'expected', + 'filesystem', + 'format', + 'forward_list', + 'fstream', + 'functional', + 'future', + 'generator', + 'initializer_list', + 'iomanip', + 'ios', + 'iosfwd', + 'iostream', + 'istream', + 'iterator', + 'latch', + 'limits', + 'list', + 'locale', + 'map', + 'mdspan', + 'memory', + 'memory_resource', + 'mutex', + 'new', + 'numbers', + 'numeric', + 'optional', + 'ostream', + 'print', + 'queue', + 'random', + 'ranges', + 'ratio', + 'regex', + 'scoped_allocator', + 'semaphore', + 'set', + 'shared_mutex', + 'source_location', + 'span', + 'spanstream', + 'sstream', + 'stack', + 'stacktrace', + 'stdexcept', + 'stdfloat', + 'stop_token', + 'streambuf', + 'string', + 'string_view', + 'strstream', + 'syncstream', + 'system_error', + 'thread', + 'tuple', + 'type_traits', + 'typeindex', + 'typeinfo', + 'unordered_map', + 'unordered_set', + 'utility', + 'valarray', + 'variant', + 'vector', + 'version', +]); + +/** Path prefixes that indicate system/kernel headers. */ +const SYSTEM_PATH_PREFIXES = [ + 'sys/', + 'net/', + 'netinet/', + 'arpa/', + 'linux/', + 'asm/', + 'bits/', + 'gnu/', + 'mach/', + 'machine/', + 'xlocale/', +]; + +/** Regex fallback for files that exceed tree-sitter's 32 KB parse limit. */ +const INCLUDE_REGEX = /^[ \t]*#\s*include\s*"([^"]+)"/gm; + +// ---------- helpers ---------- + +/** + * Normalize an include path to a canonical lowercase forward-slash form. + * + * IMPORTANT — case-folding caveat (PR #1156 review finding #3): + * Header paths are lowercased so consumer `#include "Foo/Bar.h"` and + * provider file `Foo/Bar.h` normalize to the same contract-id. This is + * the right trade-off on case-insensitive filesystems (macOS, Windows) + * but on case-sensitive Linux filesystems two distinct headers `Foo.h` + * and `foo.h` in the same repo will collide onto the same provider + * contract-id; only one survives `dedupe()`. The gain (reliable + * cross-platform matching) outweighs the cost (extremely rare header + * casing collisions inside a single repo). + */ +function normalizeIncludePath(raw: string): string { + return raw.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase(); +} + +/** + * Strip C/C++ block comments from a source blob. Used only by the + * regex-fallback path to avoid emitting consumer contracts for + * commented-out #include directives. Line comments (`// …`) cannot hide + * #include directives because the regex anchors on start-of-line. + * See PR #1156 review finding #5. + */ +function stripBlockComments(src: string): string { + return src.replace(/\/\*[\s\S]*?\*\//g, ''); +} + +function isAngleBracketInclude(rawNodeText: string): boolean { + const trimmed = rawNodeText.trim(); + return trimmed.startsWith('<') && trimmed.endsWith('>'); +} + +function isSystemHeader(cleanedPath: string): boolean { + // Check well-known standard headers + if (SYSTEM_HEADERS.has(cleanedPath)) return true; + // Check system path prefixes + const lower = cleanedPath.toLowerCase(); + return SYSTEM_PATH_PREFIXES.some((prefix) => lower.startsWith(prefix)); +} + +function isHeaderFile(filePath: string): boolean { + return HEADER_EXTENSIONS.has(path.extname(filePath).toLowerCase()); +} + +function getLanguageForFile(filePath: string): unknown | null { + const ext = path.extname(filePath).toLowerCase(); + switch (ext) { + case '.c': + case '.h': + return C; + case '.cpp': + case '.cc': + case '.cxx': + case '.hpp': + case '.hxx': + case '.hh': + return Cpp; + default: + return null; + } +} + +/** + * Check whether an include path resolves to a file inside the local repo. + * + * Uses *exact full-path* matching on the suffix index — we never accept a + * truncated suffix match. For `#include "foo/bar.h"` this checks: + * (a) a file whose path ends with the full `foo/bar.h` + * (b) if the include omitted the extension, a file whose path ends with + * the include + one of the C/C++ header extensions + * + * Returns `true` when a local file matches — caller should suppress the + * cross-repo consumer contract. + * + * See PR #1156 review finding #4 (suffixResolve ambiguity). + */ +function isLocalInclude(cleaned: string, suffixIndex: SuffixIndex): boolean { + const candidates = [cleaned]; + if (!/\.[a-zA-Z0-9]+$/.test(cleaned)) { + for (const ext of ['.h', '.hpp', '.hxx', '.hh']) candidates.push(cleaned + ext); + } + for (const c of candidates) { + if (suffixIndex.get(c) || suffixIndex.getInsensitive(c)) return true; + } + return false; +} + +// ---------- main class ---------- + +export class IncludeExtractor implements ContractExtractor { + type = 'include' as const; + + /** + * Always returns `true`. NOT called by `sync.ts`, which gates extraction via + * `config.detect.includes` instead (see `sync.ts:174`). Kept solely to satisfy + * the `ContractExtractor` interface so the type stays uniform across extractors. + */ + async canExtract(_repo: RepoHandle): Promise { + return true; + } + + async extract( + dbExecutor: CypherExecutor | null, + repoPath: string, + _repo: RepoHandle, + ): Promise { + // 1. Build the local file list using the same discovery as ingestion + // (createIgnoreFilter + getMaxFileSizeBytes). This guarantees the + // universe of provider/consumer paths matches the universe of File + // nodes in the LadybugDB graph — so no cross-link points at a UID + // that group impact cannot fan out to. + // (PR #1156 Codex follow-up: discovery aligned with ingestion.) + const allFiles = await this.discoverIndexableFiles(repoPath); + const normalizedFiles = allFiles.map((f) => f.replace(/\\/g, '/')); + const suffixIndex = buildSuffixIndex(normalizedFiles, allFiles); + + // 2. Provider: register all header files + const providers = await this.extractProviders(dbExecutor, repoPath, allFiles); + + // 3. Consumer: filter the shared discovery list for source extensions + // and parse #include directives in those files. + const sourceFiles = allFiles.filter((f) => + SOURCE_EXTENSIONS.has(path.extname(f).toLowerCase()), + ); + const consumers = await this.extractConsumers(repoPath, sourceFiles, suffixIndex); + + return this.dedupe([...providers, ...consumers]); + } + + /** + * Discover repo-relative file paths using exactly the same rules the + * ingestion pipeline uses (`walkRepositoryPaths` in + * `gitnexus/src/core/ingestion/filesystem-walker.ts`): + * - `createIgnoreFilter` honors `.gitignore`, `.gitnexusignore`, the + * hardcoded ignore list, and `.gitnexusignore` last-match-wins + * negation. + * - `getMaxFileSizeBytes()` drops files larger than the cap so we + * never emit `File:` UIDs for files ingestion would skip. + * + * Uses sequential stat — there is no `READ_CONCURRENCY` batching here + * because group sync runs at startup-time, not the ingestion hot path, + * and parallelism gains are not worth the import-graph weight. + * + * MAINTENANCE: if `walkRepositoryPaths` changes its glob options, ignore + * filter shape, or size-cap logic, mirror those changes here. The two + * implementations exist because the consumers need different return + * shapes (string[] vs ScannedFile[]) and different concurrency, but + * they MUST agree on which files are reachable — that is what makes + * `File:` UIDs in cross-links correspond to graph File nodes. + */ + private async discoverIndexableFiles(repoPath: string): Promise { + const ignoreFilter = await createIgnoreFilter(repoPath); + const maxFileSizeBytes = getMaxFileSizeBytes(); + + const candidates = await glob('**/*', { + cwd: repoPath, + nodir: true, + dot: false, + ignore: ignoreFilter, + }); + + const survivors: string[] = []; + for (const rel of candidates) { + try { + const stat = await fs.stat(path.join(repoPath, rel)); + if (stat.size > maxFileSizeBytes) continue; + survivors.push(rel); + } catch (err) { + // ENOENT is the documented benign race (glob enumerated a file + // that was deleted before we stat'd it — same race + // walkRepositoryPaths absorbs via Promise.allSettled). Anything + // else (EACCES, EMFILE, EIO) deserves a warning so an operator + // can spot a permission/resource problem instead of silently + // shipping fewer contracts than expected. + const code = (err as NodeJS.ErrnoException | undefined)?.code; + if (code !== 'ENOENT') { + logger.warn( + { err: (err as Error).message, file: rel, repoPath }, + '⚠️ IncludeExtractor: stat failed during discovery; skipping file', + ); + } + } + } + return survivors; + } + + // ---------- provider extraction ---------- + + private async extractProviders( + dbExecutor: CypherExecutor | null, + repoPath: string, + allFiles: string[], + ): Promise { + // Strategy A: graph-assisted + if (dbExecutor) { + const graphProviders = await this.extractProvidersGraph(dbExecutor, repoPath); + if (graphProviders.length > 0) return graphProviders; + } + // Strategy B: filesystem fallback + return this.extractProvidersFallback(repoPath, allFiles); + } + + private async extractProvidersGraph( + db: CypherExecutor, + repoPath: string, + ): Promise { + try { + const rows = await db( + `MATCH (f:File) + WHERE f.filePath =~ '.*\\\\.(h|hpp|hxx|hh)$' + RETURN f.filePath AS filePath, f.id AS fileId`, + ); + // gitnexus analyze stores absolute paths in the File.filePath column. + // Provider contract IDs MUST be repo-relative — otherwise the consumer + // emits `include::map/base/view.h` and the provider emits + // `include::/abs/path/to/repo/map/base/view.h`, which never match + // through runExactMatch and the cross-link silently disappears. + // (PR #1156 follow-up review: graph provider absolute-path bug.) + const normalizedRepoPath = path.resolve(repoPath); + const out: ExtractedContract[] = []; + for (const r of rows) { + if (typeof r.filePath !== 'string' || !r.filePath) continue; + const absolute = r.filePath as string; + const rel = path.relative(normalizedRepoPath, absolute); + // Skip rows that resolve outside the repo (e.g., system headers + // somehow indexed, or stale absolute paths from a different machine). + // path.relative returns a `..`-prefixed path or an absolute path + // when the target is outside the base — both are wrong for our IDs. + if (!rel || rel.startsWith('..') || path.isAbsolute(rel)) continue; + const normalizedRel = rel.replace(/\\/g, '/'); + out.push({ + contractId: `include::${normalizeIncludePath(normalizedRel)}`, + type: 'include' as const, + role: 'provider' as const, + symbolUid: String(r.fileId ?? ''), + symbolRef: { filePath: normalizedRel, name: path.basename(normalizedRel) }, + symbolName: path.basename(normalizedRel), + confidence: 1.0, + meta: { source: 'graph' }, + }); + } + return out; + } catch { + return []; + } + } + + private extractProvidersFallback(_repoPath: string, allFiles: string[]): ExtractedContract[] { + return allFiles + .filter((f) => isHeaderFile(f)) + .map((f) => { + const filePath = f.replace(/\\/g, '/'); + return { + contractId: `include::${normalizeIncludePath(filePath)}`, + type: 'include' as const, + role: 'provider' as const, + symbolUid: `File:${filePath}`, + symbolRef: { filePath, name: path.basename(filePath) }, + symbolName: path.basename(filePath), + confidence: 0.95, + meta: { source: 'filesystem' }, + }; + }); + } + + // ---------- consumer extraction ---------- + + private async extractConsumers( + repoPath: string, + sourceFiles: string[], + suffixIndex: SuffixIndex, + ): Promise { + const parser = new Parser(); + const out: ExtractedContract[] = []; + // Compile the include query once per grammar to avoid re-compilation per file + const queryCache = new Map(); + + for (const rel of sourceFiles) { + const lang = getLanguageForFile(rel); + if (!lang) continue; + + const content = readSafe(repoPath, rel); + if (!content) continue; + + let query = queryCache.get(lang); + if (!query) { + try { + query = new Parser.Query(lang, INCLUDE_QUERY_SRC); + queryCache.set(lang, query); + } catch { + continue; + } + } + + // Collect raw include paths: tree-sitter first, regex fallback for large files. + // `extractionSource` is stamped on each emitted consumer contract so + // regex-fallback contracts stay auditable post-hoc (PR #1156 review finding #6). + let rawIncludes: string[]; + let extractionSource: 'tree_sitter' | 'regex_fallback'; + try { + parser.setLanguage(lang); + const tree = parser.parse(content); + let matches: Parser.QueryMatch[]; + try { + matches = query.matches(tree.rootNode); + } catch { + matches = []; + } + rawIncludes = []; + extractionSource = 'tree_sitter'; + for (const match of matches) { + const sourceNode = match.captures.find((c) => c.name === 'import.source'); + if (!sourceNode) continue; + const rawText = sourceNode.node.text; + if (isAngleBracketInclude(rawText)) continue; + const cleaned = rawText.replace(/['"<>]/g, ''); + if (cleaned && cleaned.length <= 2048) rawIncludes.push(cleaned); + } + } catch { + // tree-sitter failed (e.g. file > 32 KB) — fall back to regex. + // Strip block comments first so we don't emit a consumer contract + // for a commented-out #include (PR #1156 review finding #5). + rawIncludes = []; + extractionSource = 'regex_fallback'; + const scanTarget = stripBlockComments(content); + INCLUDE_REGEX.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = INCLUDE_REGEX.exec(scanTarget)) !== null) { + if (m[1] && m[1].length <= 2048) rawIncludes.push(m[1]); + } + } + + for (const cleaned of rawIncludes) { + // Filter: skip known system headers and system path prefixes + if (isSystemHeader(cleaned)) continue; + + // Skip relative-up includes: `#include "../include/foo.h"` is + // almost always an intra-repo reference. The suffix index is built + // from repo-relative paths, so isLocalInclude can never match + // `../foo.h`, and emitting it as a consumer contract just pollutes + // the registry with an entry no provider can ever satisfy. + // (PR #1156 follow-up review: `../` relative includes produce + // spurious consumer contracts.) + if (cleaned.startsWith('../') || cleaned.startsWith('..\\')) continue; + + // Skip macro-style includes: `#include PLATFORM_HEADER` parses as an + // identifier under tree-sitter's `(_) @import.source` wildcard. The + // identifier text passes the strip/clean step unchanged, so without + // this guard we would emit `include::platform_header` as a consumer + // contract — and no provider in any repo will ever expose a contract + // for a macro identifier (no file is named `PLATFORM_HEADER`). The + // contract would sit permanently orphaned in the registry. Real + // header references always contain a path separator (`/`, `\`) or an + // extension dot (`foo.h`), so an absent both is a reliable signal we + // are looking at a macro identifier. (PR #1156 follow-up review: + // macro includes emit orphaned consumer contracts.) + if (!/[./\\]/.test(cleaned)) continue; + + // Local resolution (PR #1156 review finding #4): only accept an + // exact-suffix match on the *full* include path. The generic + // suffixResolve() iterates all truncated suffixes, which would + // silently suppress a cross-repo `#include "map/base/view.h"` + // when the local repo has any `internal/view.h` — a realistic + // false-negative in large C++ codebases. Here we only resolve + // locally if a file path ends with the complete include string + // (optionally re-appending one of the C/C++ header extensions + // when the include already omits it). + if (isLocalInclude(cleaned, suffixIndex)) continue; + + // Unresolved: emit as consumer contract + const normalizedRel = rel.replace(/\\/g, '/'); + out.push({ + contractId: `include::${normalizeIncludePath(cleaned)}`, + type: 'include' as const, + role: 'consumer' as const, + symbolUid: `File:${normalizedRel}`, + symbolRef: { filePath: normalizedRel, name: cleaned }, + symbolName: cleaned, + confidence: 0.85, + meta: { + source: extractionSource, + includePath: cleaned, + }, + }); + } + } + + return out; + } + + // ---------- deduplication ---------- + + private dedupe(items: ExtractedContract[]): ExtractedContract[] { + const seen = new Set(); + const out: ExtractedContract[] = []; + for (const c of items) { + const k = `${c.contractId}|${c.role}|${c.symbolRef.filePath}`; + if (seen.has(k)) continue; + seen.add(k); + out.push(c); + } + return out; + } +} diff --git a/gitnexus/src/core/group/extractors/manifest-extractor.ts b/gitnexus/src/core/group/extractors/manifest-extractor.ts index 2af3db595..f4d0f77cf 100644 --- a/gitnexus/src/core/group/extractors/manifest-extractor.ts +++ b/gitnexus/src/core/group/extractors/manifest-extractor.ts @@ -274,6 +274,14 @@ export class ManifestExtractor { LIMIT 1`, { contract: link.contract }, ); + } else if (link.type === 'include') { + rows = await executor( + `MATCH (f:File) WHERE f.filePath = $contract + RETURN f.id AS uid, f.name AS name, f.filePath AS filePath + ORDER BY f.filePath ASC + LIMIT 1`, + { contract: link.contract }, + ); } else if (link.type === 'custom') { // Workspace extractors produce qualified contracts like "mathlex::Expression". // Graph nodes store the unqualified symbol name ("Expression"), so strip @@ -358,6 +366,8 @@ export class ManifestExtractor { return `lib::${contract}`; case 'custom': return `custom::${contract}`; + case 'include': + return `include::${contract}`; default: { const _exhaustive: never = type; throw new Error(`Unhandled ContractType: ${String(_exhaustive)}`); diff --git a/gitnexus/src/core/group/extractors/thrift-extractor.ts b/gitnexus/src/core/group/extractors/thrift-extractor.ts index cfd8fef02..709968790 100644 --- a/gitnexus/src/core/group/extractors/thrift-extractor.ts +++ b/gitnexus/src/core/group/extractors/thrift-extractor.ts @@ -217,6 +217,10 @@ export async function buildThriftContext(repoPath: string): Promise(); @@ -290,6 +294,10 @@ export class ThriftExtractor implements ContractExtractor { cwd: repoPath, absolute: false, nodir: true, + // TODO(#1156-followup): replace this hand-rolled list with createIgnoreFilter + // (the canonical ingestion ignore filter, like include-extractor.ts now uses). + // New entries to DEFAULT_IGNORE_LIST in src/config/ignore-service.ts (e.g. + // third_party, 3rdparty added in commit a9936a9b) silently do not apply here. ignore: ['**/node_modules/**', '**/.git/**', '**/vendor/**', '**/dist/**', '**/build/**'], }); diff --git a/gitnexus/src/core/group/matching.ts b/gitnexus/src/core/group/matching.ts index 3431f8ddf..0b27655c6 100644 --- a/gitnexus/src/core/group/matching.ts +++ b/gitnexus/src/core/group/matching.ts @@ -107,6 +107,8 @@ export function normalizeContractId(id: string): string { return `topic::${rest.trim().toLowerCase()}`; case 'lib': return `lib::${rest.toLowerCase()}`; + case 'include': + return `include::${rest.replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+/g, '/').toLowerCase()}`; default: return id; } diff --git a/gitnexus/src/core/group/storage.ts b/gitnexus/src/core/group/storage.ts index cc3dbfdc9..bc08fd7f9 100644 --- a/gitnexus/src/core/group/storage.ts +++ b/gitnexus/src/core/group/storage.ts @@ -4,6 +4,7 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { randomBytes } from 'node:crypto'; import type { ContractRegistry } from './types.js'; +import { retryRename } from './bridge-db.js'; /** * Build an unpredictable suffix for atomic-write tmp files. Replaces the @@ -59,7 +60,13 @@ export async function writeContractRegistry( } finally { await handle.close(); } - await fsp.rename(tmpPath, targetPath); + // retryRename absorbs the documented Windows EPERM/EBUSY/EACCES race that + // fires when AV scanners or another concurrent rename briefly hold the + // destination handle between rename calls. Same helper bridge-db.ts uses + // (lines 304, 583, 587, 595, 605, 677) for the bridge.lbug atomic swap — + // single source of truth for the Windows-rename pattern across the group + // package. + await retryRename(tmpPath, targetPath); } export async function readContractRegistry(groupDir: string): Promise { diff --git a/gitnexus/src/core/group/sync.ts b/gitnexus/src/core/group/sync.ts index 7ed065131..cd64fdf8c 100644 --- a/gitnexus/src/core/group/sync.ts +++ b/gitnexus/src/core/group/sync.ts @@ -8,12 +8,14 @@ import { HttpRouteExtractor } from './extractors/http-route-extractor.js'; import { GrpcExtractor } from './extractors/grpc-extractor.js'; import { ThriftExtractor } from './extractors/thrift-extractor.js'; import { TopicExtractor } from './extractors/topic-extractor.js'; +import { IncludeExtractor } from './extractors/include-extractor.js'; import { ManifestExtractor } from './extractors/manifest-extractor.js'; import { discoverWorkspaceLinks } from './extractors/workspace-extractor.js'; import { buildProviderIndex, runExactMatch, runWildcardMatch } from './matching.js'; import { detectServiceBoundaries, assignService } from './service-boundary-detector.js'; import type { CypherExecutor } from './contract-extractor.js'; import { writeContractRegistry } from './storage.js'; +import { writeBridge } from './bridge-db.js'; import type { ContractRegistry } from './types.js'; import { logger } from '../logger.js'; @@ -100,6 +102,7 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis const grpcEx = new GrpcExtractor(); const thriftEx = new ThriftExtractor(); const topicEx = new TopicExtractor(); + const includeEx = new IncludeExtractor(); dbExecutors = new Map(); const openPoolIds: string[] = []; @@ -168,6 +171,17 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis } } + if (config.detect.includes) { + const extracted = await includeEx.extract(executor, handle.repoPath, handle); + for (const c of extracted) { + autoContracts.push({ + ...c, + repo: groupPath, + service: assignService(c.symbolRef.filePath, boundaries), + }); + } + } + const metaPath = path.join(handle.storagePath, 'meta.json'); try { const raw = await fs.readFile(metaPath, 'utf-8'); @@ -270,6 +284,28 @@ export async function syncGroup(config: GroupConfig, opts?: SyncOptions): Promis if (opts?.groupDir && !opts.skipWrite) { await writeContractRegistry(opts.groupDir, registry); + // writeBridge failure (disk full, schema error, permission denied) must + // not mask the registry — contracts.json was just written successfully + // and is the canonical source of truth. A stale or absent bridge + // degrades impact queries to empty results, which is recoverable on + // the next sync. Surface the failure as a warning so operators can + // act, but do not propagate it. + // (PR #1156 follow-up review: writeBridge error in sync.ts propagates + // uncaught.) + try { + await writeBridge(opts.groupDir, { + contracts: allContracts, + crossLinks, + repoSnapshots, + missingRepos, + }); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + logger.warn( + { err: msg, groupDir: opts.groupDir }, + '⚠️ writeBridge failed; contracts.json is intact but bridge.lbug is stale. Re-run `gitnexus group sync` to retry.', + ); + } } return { diff --git a/gitnexus/src/core/group/types.ts b/gitnexus/src/core/group/types.ts index 7d0a14251..8e43ff78f 100644 --- a/gitnexus/src/core/group/types.ts +++ b/gitnexus/src/core/group/types.ts @@ -1,4 +1,4 @@ -export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom'; +export type ContractType = 'http' | 'grpc' | 'thrift' | 'topic' | 'lib' | 'custom' | 'include'; export type MatchType = 'exact' | 'manifest' | 'wildcard' | 'bm25' | 'embedding'; export type ContractRole = 'provider' | 'consumer'; @@ -28,6 +28,7 @@ export interface DetectConfig { topics: boolean; shared_libs: boolean; embedding_fallback: boolean; + includes: boolean; workspace_deps: boolean; } diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 6c59578b0..bf0206057 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -769,9 +769,10 @@ export const processCalls = async ( let tree = astCache.get(file.path); if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch (parseError) { continue; @@ -3280,9 +3281,10 @@ export const extractFetchCallsFromFiles = async ( let tree = astCache.get(file.path); if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch { continue; diff --git a/gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts b/gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts new file mode 100644 index 000000000..baa5d17ff --- /dev/null +++ b/gitnexus/src/core/ingestion/cpp-ue-preprocessor.ts @@ -0,0 +1,265 @@ +/** + * Unreal Engine reflection-macro preprocessor for C++ source. + * + * Tree-sitter does not expand C preprocessor macros, so Unreal's reflection + * markers (`UCLASS(...)`, `UFUNCTION(...)`, `MODULENAME_API`, ...) are parsed + * verbatim. The result is mis-parsed declarations: in `class BRAWLUI_API + * UMyClass : public UObject`, tree-sitter-cpp captures `BRAWLUI_API` as the + * class name and the rest of the declaration becomes structurally wrong. + * + * This module elides those macros from the source text BEFORE tree-sitter + * parses it. Replacement is **length-preserving** (each elided byte becomes + * a space, newlines preserved) so byte offsets and line/column positions + * tree-sitter reports remain identical to the original file. Symbol + * locations in the graph stay accurate. + * + * A cheap detection guard short-circuits files that don't look like UE + * sources, so non-UE C++ codebases pay no cost. + * + * Pure function — no tree-sitter dependency, safe for worker threads. + */ +/** + * Strong UE markers — reflection macros that only Unreal Engine projects use. + * Presence of one of these is sufficient evidence that the file is a UE source + * and that `MODULENAME_API` tokens in it are intended as export macros. + * + * Importantly, `_API` tokens are NOT in this guard — `REST_API`, `HTTP_API`, + * `MY_LIB_API` and similar identifiers appear in plenty of non-UE C++ codebases + * as constants/enums/parameter names. We must not erase them just because the + * file mentions an `_API` token. + */ +const HAS_UE_HINT = + /\b(?:UCLASS|UFUNCTION|UPROPERTY|USTRUCT|UENUM|UINTERFACE|GENERATED_BODY|GENERATED_[A-Z_]+_BODY|UE_DEPRECATED|DECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE)/; + +const SIMPLE_MACROS_NO_ARGS: readonly string[] = [ + 'GENERATED_BODY', + 'GENERATED_UCLASS_BODY', + 'GENERATED_USTRUCT_BODY', + 'GENERATED_UINTERFACE_BODY', + 'GENERATED_IINTERFACE_BODY', + 'DECLARE_CLASS', + 'GENERATED_BODY_LEGACY', +]; + +const PARENTHESIZED_MACROS: readonly string[] = [ + 'UCLASS', + 'UFUNCTION', + 'UPROPERTY', + 'USTRUCT', + 'UENUM', + 'UINTERFACE', + 'UMETA', + 'UE_DEPRECATED', +]; + +const DELEGATE_MACRO_RE = + /\bDECLARE_(?:DYNAMIC_)?(?:MULTICAST_)?DELEGATE(?:_(?:RetVal_OneParam|RetVal_TwoParams|RetVal_ThreeParams|RetVal_FourParams|RetVal_FiveParams|RetVal_SixParams|RetVal_SevenParams|RetVal_EightParams|RetVal_NineParams|RetVal|OneParam|TwoParams|ThreeParams|FourParams|FiveParams|SixParams|SevenParams|EightParams|NineParams|TenParams))?(?=\s*\()/g; + +/** + * Module export tokens like `BRAWLUI_API`, `ENGINE_API`, `COREUOBJECT_API`. + * Pattern: ALL_CAPS identifier ending in `_API`. The leading word boundary + * (`\b`) prevents matching mid-identifier. + */ +const API_MACRO_RE = /\b[A-Z][A-Z0-9_]*_API\b/g; + +/** Replace `[start, end)` of `chars` with spaces, preserving newlines. */ +function eraseRange(chars: string[], start: number, end: number): void { + for (let i = start; i < end; i++) { + if (chars[i] !== '\n' && chars[i] !== '\r') { + chars[i] = ' '; + } + } +} + +/** + * Find the matching close paren for an opening paren at index `openIdx`. + * Returns the index of `)` (inclusive end), or -1 if unbalanced. + * + * Handles nested parens and string/char literals so commas/parens inside + * strings don't throw off the match. Does not attempt to handle raw string + * literals (`R"(...)"`); UE reflection-macro arguments do not use them in + * practice. + */ +function findMatchingParen(source: string, openIdx: number): number { + if (source.charCodeAt(openIdx) !== 0x28) return -1; + let depth = 1; + let i = openIdx + 1; + const len = source.length; + while (i < len && depth > 0) { + const ch = source.charCodeAt(i); + // String literal + if (ch === 0x22) { + i++; + while (i < len) { + const c = source.charCodeAt(i); + if (c === 0x5c) { + i += 2; + continue; + } + if (c === 0x22) { + i++; + break; + } + i++; + } + continue; + } + // Char literal + if (ch === 0x27) { + i++; + while (i < len) { + const c = source.charCodeAt(i); + if (c === 0x5c) { + i += 2; + continue; + } + if (c === 0x27) { + i++; + break; + } + i++; + } + continue; + } + // Line comment + if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2f) { + while (i < len && source.charCodeAt(i) !== 0x0a) i++; + continue; + } + // Block comment + if (ch === 0x2f && source.charCodeAt(i + 1) === 0x2a) { + i += 2; + while (i < len) { + if (source.charCodeAt(i) === 0x2a && source.charCodeAt(i + 1) === 0x2f) { + i += 2; + break; + } + i++; + } + continue; + } + if (ch === 0x28) depth++; + else if (ch === 0x29) { + depth--; + if (depth === 0) return i; + } + i++; + } + return -1; +} + +/** Match a whole-word identifier at `idx`. Returns the byte after the identifier, or -1 on miss. */ +function matchIdentifierAt(source: string, idx: number, name: string): number { + if (idx > 0) { + const prev = source.charCodeAt(idx - 1); + if ( + (prev >= 0x30 && prev <= 0x39) || + (prev >= 0x41 && prev <= 0x5a) || + (prev >= 0x61 && prev <= 0x7a) || + prev === 0x5f + ) { + return -1; + } + } + for (let k = 0; k < name.length; k++) { + if (source.charCodeAt(idx + k) !== name.charCodeAt(k)) return -1; + } + const after = idx + name.length; + if (after < source.length) { + const next = source.charCodeAt(after); + if ( + (next >= 0x30 && next <= 0x39) || + (next >= 0x41 && next <= 0x5a) || + (next >= 0x61 && next <= 0x7a) || + next === 0x5f + ) { + return -1; + } + } + return after; +} + +/** Skip ASCII whitespace forward from `idx`. Returns the next non-whitespace byte index. */ +function skipWhitespace(source: string, idx: number): number { + const len = source.length; + while (idx < len) { + const ch = source.charCodeAt(idx); + if (ch === 0x20 || ch === 0x09 || ch === 0x0a || ch === 0x0d) { + idx++; + continue; + } + break; + } + return idx; +} + +/** + * Strip Unreal Engine reflection macros from C++ source, length-preserving. + * + * Returns the original string unchanged if no strong UE marker is detected, + * so non-UE C++ files (including ones that contain `*_API`-suffixed + * identifiers like `REST_API` or `HTTP_API`) incur only a single regex test. + * + * The `_filePath` parameter is part of the `LanguageProvider.preprocessSource` + * contract but is unused — UE detection is purely content-based. Accepted and + * ignored here so the function matches the hook signature exactly. + */ +export function stripUeMacros(source: string, _filePath?: string): string { + if (!HAS_UE_HINT.test(source)) return source; + + const chars: string[] = source.split(''); + + for (const macro of PARENTHESIZED_MACROS) { + let searchFrom = 0; + while (true) { + const hit = source.indexOf(macro, searchFrom); + if (hit < 0) break; + searchFrom = hit + 1; + const after = matchIdentifierAt(source, hit, macro); + if (after < 0) continue; + const parenIdx = skipWhitespace(source, after); + if (source.charCodeAt(parenIdx) !== 0x28) continue; + const close = findMatchingParen(source, parenIdx); + if (close < 0) continue; + eraseRange(chars, hit, close + 1); + } + } + + for (const macro of SIMPLE_MACROS_NO_ARGS) { + let searchFrom = 0; + while (true) { + const hit = source.indexOf(macro, searchFrom); + if (hit < 0) break; + searchFrom = hit + 1; + const after = matchIdentifierAt(source, hit, macro); + if (after < 0) continue; + const parenIdx = skipWhitespace(source, after); + if (source.charCodeAt(parenIdx) === 0x28) { + const close = findMatchingParen(source, parenIdx); + if (close < 0) continue; + eraseRange(chars, hit, close + 1); + } else { + eraseRange(chars, hit, after); + } + } + } + + for (const re of [DELEGATE_MACRO_RE, API_MACRO_RE]) { + re.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = re.exec(source)) !== null) { + const start = match.index; + let end = start + match[0].length; + if (re === DELEGATE_MACRO_RE) { + const parenIdx = skipWhitespace(source, end); + if (source.charCodeAt(parenIdx) === 0x28) { + const close = findMatchingParen(source, parenIdx); + if (close >= 0) end = close + 1; + } + } + eraseRange(chars, start, end); + } + } + + return chars.join(''); +} diff --git a/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts b/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts index 4d83a6cd7..75032a9d7 100644 --- a/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts +++ b/gitnexus/src/core/ingestion/field-extractors/configs/csharp.ts @@ -9,6 +9,11 @@ import type { SyntaxNode } from '../../utils/ast-helpers.js'; const CSHARP_VIS = new Set(['public', 'private', 'protected', 'internal']); +const extractCsharpDeclaredType = (typeNode: SyntaxNode): string | undefined => { + if (typeNode.type === 'generic_name') return typeNode.text.trim(); + return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim(); +}; + /** * C# field extraction config. * @@ -53,17 +58,17 @@ export const csharpConfig: FieldExtractionConfig = { const child = node.namedChild(i); if (child?.type === 'variable_declaration') { const typeNode = child.childForFieldName('type'); - if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim(); + if (typeNode) return extractCsharpDeclaredType(typeNode); // fallback: first child that is a type const first = child.firstNamedChild; if (first && first.type !== 'variable_declarator') { - return extractSimpleTypeName(first) ?? first.text?.trim(); + return extractCsharpDeclaredType(first); } } } // property_declaration: type is first named child const typeNode = node.childForFieldName('type'); - if (typeNode) return extractSimpleTypeName(typeNode) ?? typeNode.text?.trim(); + if (typeNode) return extractCsharpDeclaredType(typeNode); return undefined; }, diff --git a/gitnexus/src/core/ingestion/heritage-processor.ts b/gitnexus/src/core/ingestion/heritage-processor.ts index 2c973ad8e..f8628e651 100644 --- a/gitnexus/src/core/ingestion/heritage-processor.ts +++ b/gitnexus/src/core/ingestion/heritage-processor.ts @@ -219,9 +219,13 @@ export const processHeritage = async ( let tree = astCache.get(file.path); if (!tree) { // Use larger bufferSize for files > 32KB + // Per-language source preprocessor (length-preserving, e.g. UE macro + // stripping for C++). MUST mirror parsing-processor on cache miss so + // re-parses see the same input as the cached AST. + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch (parseError) { // Skip files that can't be parsed @@ -413,9 +417,10 @@ export async function extractExtractedHeritageFromFiles( let tree = astCache.get(file.path); if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch { continue; diff --git a/gitnexus/src/core/ingestion/import-processor.ts b/gitnexus/src/core/ingestion/import-processor.ts index 03cbed5b7..6f0b40b6f 100644 --- a/gitnexus/src/core/ingestion/import-processor.ts +++ b/gitnexus/src/core/ingestion/import-processor.ts @@ -305,9 +305,10 @@ export const processImports = async ( let wasReparsed = false; if (!tree) { + const parseContent = provider.preprocessSource?.(file.content, file.path) ?? file.content; try { - tree = parser.parse(file.content, undefined, { - bufferSize: getTreeSitterBufferSize(file.content), + tree = parser.parse(parseContent, undefined, { + bufferSize: getTreeSitterBufferSize(parseContent), }); } catch (parseError) { continue; diff --git a/gitnexus/src/core/ingestion/language-provider.ts b/gitnexus/src/core/ingestion/language-provider.ts index 12ce0839b..e139cf5f3 100644 --- a/gitnexus/src/core/ingestion/language-provider.ts +++ b/gitnexus/src/core/ingestion/language-provider.ts @@ -116,6 +116,39 @@ interface LanguageProviderConfig { * Required for tree-sitter languages; empty string for standalone processors. */ readonly treeSitterQueries: string; + /** + * Optional source-text transform that runs **before** tree-sitter parses the file. + * + * Used to elide language constructs that confuse the grammar without affecting + * source-position fidelity — e.g., Unreal Engine reflection macros (`UCLASS`, + * `UFUNCTION`, `MODULENAME_API`) in C++ headers that prevent the parser from + * recognising class/function names correctly. + * + * **Length / position preservation:** the returned string MUST have the same + * JavaScript `.length` as the input AND preserve every newline (`\n`/`\r`) + * position byte-for-byte. Implementations replace elided characters with + * ASCII spaces while leaving newlines untouched. With this contract: + * + * - tree-sitter's reported `startPosition.row`/`startPosition.column` + * match the original file exactly (line/column come from newline counts) + * - `startIndex`/`endIndex` byte offsets match the original file exactly + * **when the elided range is pure ASCII** (UTF-16 `.length` equals UTF-8 + * byte length only for ASCII). + * + * Implementations targeting languages where elided ranges may contain + * non-ASCII content must therefore preserve byte length, not just `.length`, + * if downstream code uses `startIndex` to slice the original UTF-8 bytes. + * The current C++ UE-macro preprocessor relies on the practical fact that + * UE reflection macros and module-export tokens are ASCII-only. + * + * Must be a pure function — same input always yields the same output. Called + * once per file, on every code path that re-parses (parsing-processor, import + * processor, heritage processor, call processor, parse worker). + * + * Default: undefined (no preprocessing — `file.content` is parsed verbatim). + */ + readonly preprocessSource?: (sourceText: string, filePath: string) => string; + // ── Core (required) ─────────────────────────────────────────────── /** Type extraction: declarations, initializers, for-loop bindings */ readonly typeConfig: LanguageTypeConfig; diff --git a/gitnexus/src/core/ingestion/languages/c-cpp.ts b/gitnexus/src/core/ingestion/languages/c-cpp.ts index a5b3e5729..693e8cec2 100644 --- a/gitnexus/src/core/ingestion/languages/c-cpp.ts +++ b/gitnexus/src/core/ingestion/languages/c-cpp.ts @@ -45,6 +45,7 @@ import { cVariableConfig, cppVariableConfig } from '../variable-extractors/confi import { createCallExtractor } from '../call-extractors/generic.js'; import { cCallConfig, cppCallConfig } from '../call-extractors/configs/c-cpp.js'; import { createHeritageExtractor } from '../heritage-extractors/generic.js'; +import { stripUeMacros } from '../cpp-ue-preprocessor.js'; const C_BUILT_INS: ReadonlySet = new Set([ 'printf', @@ -410,6 +411,7 @@ export const cppProvider = defineLanguage({ }, ] satisfies AstFrameworkPatternConfig[], treeSitterQueries: CPP_QUERIES, + preprocessSource: stripUeMacros, typeConfig: cCppConfig, exportChecker: cCppExportChecker, importResolver: createImportResolver(cppImportConfig), diff --git a/gitnexus/src/core/ingestion/languages/csharp/captures.ts b/gitnexus/src/core/ingestion/languages/csharp/captures.ts index 2f913a14e..e29552e78 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/captures.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/captures.ts @@ -42,6 +42,39 @@ const FUNCTION_NODE_TYPES = [ 'local_function_statement', ] as const; +const BUILTIN_TYPE_NAMES = new Set([ + 'bool', + 'byte', + 'char', + 'decimal', + 'double', + 'float', + 'int', + 'long', + 'object', + 'sbyte', + 'short', + 'string', + 'uint', + 'ulong', + 'ushort', + 'void', +]); + +function shouldEmitReadMember(memberNode: SyntaxNode): boolean { + const parent = memberNode.parent; + if (parent === null) return true; + + switch (parent.type) { + case 'invocation_expression': + return parent.childForFieldName('function')?.id !== memberNode.id; + case 'assignment_expression': + return parent.childForFieldName('left')?.id !== memberNode.id; + default: + return true; + } +} + export function emitCsharpScopeCaptures( sourceText: string, _filePath: string, @@ -94,6 +127,14 @@ export function emitCsharpScopeCaptures( continue; } + if (grouped['@reference.read.member'] !== undefined) { + const anchor = grouped['@reference.read.member']; + const memberNode = findNodeAtRange(tree.rootNode, anchor.range, 'member_access_expression'); + if (memberNode === null || !shouldEmitReadMember(memberNode)) { + continue; + } + } + // Synthesize `this` / `base` receiver type-bindings on every // instance method-like. Tree-sitter can't cleanly express "the // implicit receiver of a non-static member of a class/struct/ @@ -209,9 +250,63 @@ export function emitCsharpScopeCaptures( } } + out.push(...synthesizeGenericTypeArgumentReferences(tree.rootNode)); + return out; } +function synthesizeGenericTypeArgumentReferences(root: SyntaxNode): CaptureMatch[] { + const out: CaptureMatch[] = []; + // Treat all generic type arguments as static type references, including + // declaration signatures and call-site generic instantiations. + visit(root, (node) => { + if (node.type !== 'generic_name') return; + const args = findNamedChild(node, 'type_argument_list'); + if (args === null) return; + + for (const arg of args.namedChildren) { + if (arg === null) continue; + const nameNode = terminalTypeNameNode(arg); + if (nameNode === null) continue; + if (BUILTIN_TYPE_NAMES.has(nameNode.text)) continue; + out.push({ + '@reference.type': nodeToCapture('@reference.type', nameNode), + '@reference.name': nodeToCapture('@reference.name', nameNode), + }); + } + }); + return out; +} + +function terminalTypeNameNode(node: SyntaxNode): SyntaxNode | null { + switch (node.type) { + case 'identifier': + return node; + case 'nullable_type': + return node.firstNamedChild === null ? null : terminalTypeNameNode(node.firstNamedChild); + case 'qualified_name': + return node.lastNamedChild; + case 'generic_name': + return node.childForFieldName('name') ?? node.firstNamedChild; + default: + return null; + } +} + +function findNamedChild(node: SyntaxNode, type: string): SyntaxNode | null { + for (const child of node.namedChildren) { + if (child !== null && child.type === type) return child; + } + return null; +} + +function visit(node: SyntaxNode, cb: (node: SyntaxNode) => void): void { + cb(node); + for (const child of node.namedChildren) { + if (child !== null) visit(child, cb); + } +} + /** C# 12 primary constructor: `class X(a, b) { }` / `record X(a, b)`. * The parameters are a bare `parameter_list` named child of the type * declaration (no `constructor_declaration` node). Emit a synthetic diff --git a/gitnexus/src/core/ingestion/languages/csharp/query.ts b/gitnexus/src/core/ingestion/languages/csharp/query.ts index 11da2d8cd..f299aaafa 100644 --- a/gitnexus/src/core/ingestion/languages/csharp/query.ts +++ b/gitnexus/src/core/ingestion/languages/csharp/query.ts @@ -499,6 +499,12 @@ const CSHARP_SCOPE_QUERY = ` left: (member_access_expression expression: "base" @reference.receiver name: (identifier) @reference.name)) @reference.write.member + +;; References — field/property reads: \`obj.Name\` +;; Emit-side filtering drops call targets and assignment left-hand sides. +(member_access_expression + expression: (_) @reference.receiver + name: (identifier) @reference.name) @reference.read.member `; let _parser: Parser | null = null; diff --git a/gitnexus/src/core/ingestion/parsing-processor.ts b/gitnexus/src/core/ingestion/parsing-processor.ts index 8803ec023..98036fbe8 100644 --- a/gitnexus/src/core/ingestion/parsing-processor.ts +++ b/gitnexus/src/core/ingestion/parsing-processor.ts @@ -371,6 +371,11 @@ const processParsingSequential = async ( isVueSetup = extracted.isSetup; } + // Per-language source-text transform (e.g., UE macro stripping for C++). + // Length-preserving — see LanguageProvider.preprocessSource contract. + parseContent = + getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent; + try { await loadLanguage(language, file.path); } catch { diff --git a/gitnexus/src/core/ingestion/workers/parse-worker.ts b/gitnexus/src/core/ingestion/workers/parse-worker.ts index 5c6712562..4442b5a65 100644 --- a/gitnexus/src/core/ingestion/workers/parse-worker.ts +++ b/gitnexus/src/core/ingestion/workers/parse-worker.ts @@ -1407,6 +1407,11 @@ const processFileGroup = ( isVueSetup = extracted.isSetup; } + // Per-language source-text transform (e.g., UE macro stripping for C++). + // Length-preserving — see LanguageProvider.preprocessSource contract. + parseContent = + getProvider(language).preprocessSource?.(parseContent, file.path) ?? parseContent; + clearCaches(); // Reset memoization before each new file let tree; diff --git a/gitnexus/src/core/lbug/csv-generator.ts b/gitnexus/src/core/lbug/csv-generator.ts index 616a00cac..20df51ebd 100644 --- a/gitnexus/src/core/lbug/csv-generator.ts +++ b/gitnexus/src/core/lbug/csv-generator.ts @@ -301,11 +301,15 @@ export const streamAllCSVsToDisk = async ( 'Template', 'Module', ] as const; + const propertyHeader = 'id,name,filePath,startLine,endLine,content,description,declaredType'; const multiLangWriters = new Map(); for (const t of MULTI_LANG_TYPES) { multiLangWriters.set( t, - new BufferedCSVWriter(path.join(csvDir, `${t.toLowerCase()}.csv`), multiLangHeader), + new BufferedCSVWriter( + path.join(csvDir, `${t.toLowerCase()}.csv`), + t === 'Property' ? propertyHeader : multiLangHeader, + ), ); } @@ -478,6 +482,9 @@ export const streamAllCSVsToDisk = async ( escapeCSVNumber(node.properties.endLine, -1), escapeCSVField(content), escapeCSVField(node.properties.description || ''), + ...(node.label === 'Property' + ? [escapeCSVField(node.properties.declaredType || '')] + : []), ].join(','), ); } diff --git a/gitnexus/src/core/lbug/lbug-adapter.ts b/gitnexus/src/core/lbug/lbug-adapter.ts index fb4cf76de..cb88986ca 100644 --- a/gitnexus/src/core/lbug/lbug-adapter.ts +++ b/gitnexus/src/core/lbug/lbug-adapter.ts @@ -608,6 +608,9 @@ const getCopyQuery = (table: NodeTableName, filePath: string): string => { if (table === 'Method') { return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description, parameterCount, returnType) FROM "${filePath}" ${COPY_CSV_OPTS}`; } + if (table === 'Property') { + return `COPY ${t}(id, name, filePath, startLine, endLine, content, description, declaredType) FROM "${filePath}" ${COPY_CSV_OPTS}`; + } // TypeScript/JS code element tables have isExported; multi-language tables do not if (TABLES_WITH_EXPORTED.has(table)) { return `COPY ${t}(id, name, filePath, startLine, endLine, isExported, content, description) FROM "${filePath}" ${COPY_CSV_OPTS}`; @@ -659,6 +662,11 @@ export const insertNodeToLbug = async ( ? `, description: ${escapeValue(properties.description)}` : ''; query = `CREATE (n:${t} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, isExported: ${!!properties.isExported}, content: ${escapeValue(properties.content || '')}${descPart}})`; + } else if (label === 'Property') { + const descPart = properties.description + ? `, description: ${escapeValue(properties.description)}` + : ''; + query = `CREATE (n:${t} {id: ${escapeValue(properties.id)}, name: ${escapeValue(properties.name)}, filePath: ${escapeValue(properties.filePath)}, startLine: ${properties.startLine || 0}, endLine: ${properties.endLine || 0}, content: ${escapeValue(properties.content || '')}${descPart}, declaredType: ${escapeValue(properties.declaredType || '')}})`; } else { // Multi-language tables (Struct, Impl, Trait, Macro, etc.) — no isExported const descPart = properties.description @@ -737,6 +745,11 @@ export const batchInsertNodesToLbug = async ( ? `, n.description = ${escapeValue(properties.description)}` : ''; query = `MERGE (n:${t} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.isExported = ${!!properties.isExported}, n.content = ${escapeValue(properties.content || '')}${descPart}`; + } else if (label === 'Property') { + const descPart = properties.description + ? `, n.description = ${escapeValue(properties.description)}` + : ''; + query = `MERGE (n:${t} {id: ${escapeValue(properties.id)}}) SET n.name = ${escapeValue(properties.name)}, n.filePath = ${escapeValue(properties.filePath)}, n.startLine = ${properties.startLine || 0}, n.endLine = ${properties.endLine || 0}, n.content = ${escapeValue(properties.content || '')}${descPart}, n.declaredType = ${escapeValue(properties.declaredType || '')}`; } else { const descPart = properties.description ? `, n.description = ${escapeValue(properties.description)}` diff --git a/gitnexus/src/core/lbug/schema.ts b/gitnexus/src/core/lbug/schema.ts index 5579036f1..cd6838dd8 100644 --- a/gitnexus/src/core/lbug/schema.ts +++ b/gitnexus/src/core/lbug/schema.ts @@ -167,7 +167,18 @@ export const TYPE_ALIAS_SCHEMA = CODE_ELEMENT_BASE('TypeAlias'); export const CONST_SCHEMA = CODE_ELEMENT_BASE('Const'); export const STATIC_SCHEMA = CODE_ELEMENT_BASE('Static'); export const VARIABLE_SCHEMA = CODE_ELEMENT_BASE('Variable'); -export const PROPERTY_SCHEMA = CODE_ELEMENT_BASE('Property'); +export const PROPERTY_SCHEMA = ` +CREATE NODE TABLE \`Property\` ( + id STRING, + name STRING, + filePath STRING, + startLine INT64, + endLine INT64, + content STRING, + description STRING, + declaredType STRING, + PRIMARY KEY (id) +)`; export const RECORD_SCHEMA = CODE_ELEMENT_BASE('Record'); export const DELEGATE_SCHEMA = CODE_ELEMENT_BASE('Delegate'); export const ANNOTATION_SCHEMA = CODE_ELEMENT_BASE('Annotation'); diff --git a/gitnexus/src/core/wiki/llm-client.ts b/gitnexus/src/core/wiki/llm-client.ts index 172b8b00f..37fe7a9f2 100644 --- a/gitnexus/src/core/wiki/llm-client.ts +++ b/gitnexus/src/core/wiki/llm-client.ts @@ -1,4 +1,5 @@ import { logger } from '../logger.js'; +import { CircuitOpenError, ResilientFetchExhaustedError, resilientFetch } from 'gitnexus-shared'; /** * LLM Client for Wiki Generation * @@ -76,6 +77,49 @@ export function estimateTokens(text: string): number { return Math.ceil(text.length / 4); } +/** + * Validate that a base URL supplied for LLM API calls is a safe HTTP/HTTPS + * endpoint (CWE-918 / CodeQL js/http-to-file-access). + * + * Allowed: + * - https:// with any hostname (public LLM APIs, Azure, OpenRouter, …) + * - http:// restricted to localhost / 127.0.0.1 (local servers: Ollama, LiteLLM, …) + * + * Rejected: + * - file://, data:, javascript:, and any other non-HTTP scheme + * - http:// aimed at non-loopback hosts (avoids SSRF against internal networks) + * + * Throws with a descriptive message on validation failure so callers surface a + * clear error rather than an opaque network error. + */ +export function validateLLMBaseUrl(baseUrl: string): void { + let parsed: URL; + try { + parsed = new URL(baseUrl); + } catch { + // Do not include the raw input in the message — it may contain credentials. + throw new Error('Invalid LLM base URL: must be a well-formed http:// or https:// URL'); + } + + if (!['https:', 'http:'].includes(parsed.protocol)) { + // Use parsed.protocol only (scheme), not the full URL, to avoid leaking credentials. + throw new Error(`LLM base URL must use http:// or https:// (got ${parsed.protocol})`); + } + + if (parsed.protocol === 'http:') { + // Node's URL parser preserves IPv6 brackets in hostname (e.g. "[::1]"), + // so strip them before comparing to bare address literals. + const host = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, ''); + if (host !== 'localhost' && host !== '127.0.0.1' && host !== '::1') { + // Use parsed.origin (scheme+host+port, no credentials) instead of the full URL. + throw new Error( + `Insecure http:// LLM base URLs are only allowed for localhost/127.0.0.1. ` + + `Use https:// for remote endpoints (got ${parsed.origin})`, + ); + } + } +} + /** * Returns true if the given base URL is an Azure OpenAI endpoint. * Uses proper hostname matching to avoid spoofed URLs like @@ -127,6 +171,9 @@ export async function callLLM( systemPrompt?: string, options?: CallLLMOptions, ): Promise { + // Validate base URL before any fetch (CodeQL js/http-to-file-access) + validateLLMBaseUrl(config.baseUrl); + const messages: Array<{ role: string; content: string }> = []; if (systemPrompt) { messages.push({ role: 'system', content: systemPrompt }); @@ -170,86 +217,85 @@ export async function callLLM( ? { 'api-key': config.apiKey } : { Authorization: `Bearer ${config.apiKey}` }; - const MAX_RETRIES = 3; - let lastError: Error | null = null; - - for (let attempt = 0; attempt < MAX_RETRIES; attempt++) { - try { - const response = await fetch(url, { + // Network resilience (bounded retries with exponential-backoff jitter, + // 5xx + 429 + Retry-After handling, in-process circuit breaker on the + // LLM endpoint) is delegated to resilientFetch. Provider-specific + // error parsing (Azure content filter, empty-content checks) stays + // here since it requires response-body inspection. + let response: Response; + try { + response = await resilientFetch( + url, + { method: 'POST', headers: { 'Content-Type': 'application/json', ...authHeaders, }, body: JSON.stringify(body), - }); - - if (!response.ok) { - const errorText = await response.text().catch(() => 'unknown error'); - - // Azure content filter — surface a clear message instead of a generic API error - if ( - azure && - response.status === 400 && - (errorText.includes('content_filter') || - errorText.includes('ResponsibleAIPolicyViolation')) - ) { - throw new Error( - `Azure content filter blocked this request. The prompt triggered content policy. Details: ${errorText.slice(0, 300)}`, - ); - } - - // Rate limit — wait with exponential backoff and retry - if (response.status === 429 && attempt < MAX_RETRIES - 1) { - const retryAfter = parseInt(response.headers.get('retry-after') || '0', 10); - const delay = retryAfter > 0 ? retryAfter * 1000 : 2 ** attempt * 3000; - await sleep(delay); - continue; - } - - // Server error — retry with backoff - if (response.status >= 500 && attempt < MAX_RETRIES - 1) { - await sleep((attempt + 1) * 2000); - continue; - } - - throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`); - } - - // Streaming path - if (useStream && response.body) { - return await readSSEStream(response.body, options!.onChunk!); - } - - // Non-streaming path - const json = (await response.json()) as any; - const choice = json.choices?.[0]; - if (!choice?.message?.content) { - throw new Error('LLM returned empty response'); - } - - return { - content: choice.message.content, - promptTokens: json.usage?.prompt_tokens, - completionTokens: json.usage?.completion_tokens, - }; - } catch (err: any) { - lastError = err; - - // Network error — retry with backoff - if ( - attempt < MAX_RETRIES - 1 && - (err.code === 'ECONNREFUSED' || err.code === 'ETIMEDOUT' || err.message?.includes('fetch')) - ) { - await sleep((attempt + 1) * 3000); - continue; - } - - throw err; + // Per-attempt timeout. Without this each retry can hang + // indefinitely on a frozen TCP connection — the per-call + // signal is the only timeout `resilientFetch` honors; + // `capDelayMs` only bounds the *backoff* between attempts. + // 60s matches typical LLM completion budgets. + signal: AbortSignal.timeout(60_000), + }, + { + breakerKey: `wiki-llm-${new URL(url).host}`, + retry: { maxAttempts: 3, baseDelayMs: 2_000, capDelayMs: 30_000 }, + }, + ); + } catch (err) { + if (err instanceof CircuitOpenError) { + throw new Error( + `LLM endpoint circuit open: retry in ${Math.ceil(err.retryAfterMs / 1000)}s. ${err.message}`, + ); } + if (err instanceof ResilientFetchExhaustedError) { + const errorText = await err.response.text().catch(() => 'unknown error'); + throw new Error( + `LLM API error (${err.response.status} after retries): ${errorText.slice(0, 500)}`, + ); + } + throw err; } - throw lastError || new Error('LLM call failed after retries'); + if (!response.ok) { + const errorText = await response.text().catch(() => 'unknown error'); + + // Azure content filter — surface a clear message instead of a generic API error. + if ( + azure && + response.status === 400 && + (errorText.includes('content_filter') || errorText.includes('ResponsibleAIPolicyViolation')) + ) { + throw new Error( + `Azure content filter blocked this request. The prompt triggered content policy. Details: ${errorText.slice(0, 300)}`, + ); + } + + // Any other non-OK response here is a terminal 4xx — resilientFetch + // already retried 5xx/429 to exhaustion and would have thrown above. + throw new Error(`LLM API error (${response.status}): ${errorText.slice(0, 500)}`); + } + + // Streaming path + if (useStream && response.body) { + return await readSSEStream(response.body, options!.onChunk!); + } + + // Non-streaming path + const json = (await response.json()) as any; + const choice = json.choices?.[0]; + if (!choice?.message?.content) { + throw new Error('LLM returned empty response'); + } + + return { + content: choice.message.content, + promptTokens: json.usage?.prompt_tokens, + completionTokens: json.usage?.completion_tokens, + }; } /** @@ -312,7 +358,3 @@ async function readSSEStream( return { content }; } - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/gitnexus/src/mcp/local/local-backend.ts b/gitnexus/src/mcp/local/local-backend.ts index 6175af875..167c1db81 100644 --- a/gitnexus/src/mcp/local/local-backend.ts +++ b/gitnexus/src/mcp/local/local-backend.ts @@ -1756,12 +1756,13 @@ export class LocalBackend { repo.id, ` MATCH (caller)-[r:CodeRelation]->(n {id: $symId}) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind LIMIT 30 `, { symId }, ); + let typedPropertyRows: any[] = []; // Fix #480: Class/Interface nodes have no direct CALLS/IMPORTS edges — // those point to Constructor and File nodes respectively. Fetch those @@ -1795,23 +1796,24 @@ export class LocalBackend { if (isClassLike) { try { - // Run both incoming-ref queries in parallel — they are independent. - const [ctorIncoming, fileIncoming] = await Promise.all([ - executeParameterized( - repo.id, - ` + // Run incoming-ref queries in parallel — they are independent. + const [ctorIncoming, fileIncoming, typedPropertyIncoming, typedProperties] = + await Promise.all([ + executeParameterized( + repo.id, + ` MATCH (n)-[hm:CodeRelation]->(ctor:Constructor) WHERE n.id = $symId AND hm.type = 'HAS_METHOD' MATCH (caller)-[r:CodeRelation]->(ctor) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'ACCESSES'] RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind LIMIT 30 `, - { symId }, - ), - executeParameterized( - repo.id, - ` + { symId }, + ), + executeParameterized( + repo.id, + ` MATCH (f:File)-[rel:CodeRelation]->(n) WHERE n.id = $symId AND rel.type = 'DEFINES' MATCH (caller)-[r:CodeRelation]->(f) @@ -1819,9 +1821,45 @@ export class LocalBackend { RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind LIMIT 30 `, - { symId }, - ), - ]); + { symId }, + ), + executeParameterized( + repo.id, + ` + MATCH (p:\`Property\`) + WHERE p.declaredType = $name + OR p.declaredType STARTS WITH $genericPrefix + OR p.declaredType CONTAINS $genericArg + MATCH (caller)-[r:CodeRelation]->(p) + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'ACCESSES'] + RETURN r.type AS relType, caller.id AS uid, caller.name AS name, caller.filePath AS filePath, labels(caller)[0] AS kind + LIMIT 30 + `, + { + name: sym.name, + genericPrefix: `${sym.name}<`, + genericArg: `<${sym.name}>`, + }, + ), + executeParameterized( + repo.id, + ` + MATCH (p:\`Property\`) + WHERE p.declaredType = $name + OR p.declaredType STARTS WITH $genericPrefix + OR p.declaredType CONTAINS $genericArg + RETURN p.id AS uid, p.name AS name, p.filePath AS filePath, labels(p)[0] AS kind, + p.declaredType AS declaredType + LIMIT 30 + `, + { + name: sym.name, + genericPrefix: `${sym.name}<`, + genericArg: `<${sym.name}>`, + }, + ), + ]); + typedPropertyRows = typedProperties; // Deduplicate by (relType, uid) — a caller can have multiple relation // types to the same target (e.g. both IMPORTS and CALLS), and each @@ -1829,7 +1867,7 @@ export class LocalBackend { const seenKeys = new Set( incomingRows.map((r: any) => `${r.relType || r[0]}:${r.uid || r[1]}`), ); - for (const r of [...ctorIncoming, ...fileIncoming]) { + for (const r of [...ctorIncoming, ...fileIncoming, ...typedPropertyIncoming]) { const key = `${r.relType || r[0]}:${r.uid || r[1]}`; if (!seenKeys.has(key)) { seenKeys.add(key); @@ -1846,7 +1884,7 @@ export class LocalBackend { repo.id, ` MATCH (n {id: $symId})-[r:CodeRelation]->(target) - WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] + WHERE r.type IN ['CALLS', 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', 'USES', 'HAS_METHOD', 'HAS_PROPERTY', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', 'ACCESSES'] RETURN r.type AS relType, target.id AS uid, target.name AS name, target.filePath AS filePath, labels(target)[0] AS kind LIMIT 30 `, @@ -1935,6 +1973,17 @@ export class LocalBackend { }, incoming: categorize(incomingRows), outgoing: categorize(outgoingRows), + ...(typedPropertyRows.length > 0 + ? { + typed_properties: typedPropertyRows.map((r: any) => ({ + uid: r.uid || r[0], + name: r.name || r[1], + filePath: r.filePath || r[2], + kind: r.kind || r[3], + declaredType: r.declaredType || r[4], + })), + } + : {}), processes: processRows.map((r: any) => ({ id: r.pid || r[0], name: r.label || r[1], @@ -2500,6 +2549,7 @@ export class LocalBackend { const mappedRelTypes = params.relationTypes?.flatMap((t: string) => t === 'OVERRIDES' ? ['OVERRIDES', 'METHOD_OVERRIDES'] : [t], ); + const hasExplicitRelationTypes = mappedRelTypes !== undefined && mappedRelTypes.length > 0; const rawRelTypes = mappedRelTypes && mappedRelTypes.length > 0 ? mappedRelTypes.filter((t: string) => VALID_RELATION_TYPES.has(t)) @@ -2508,6 +2558,7 @@ export class LocalBackend { 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', + 'USES', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', @@ -2520,6 +2571,7 @@ export class LocalBackend { 'IMPORTS', 'EXTENDS', 'IMPLEMENTS', + 'USES', 'METHOD_OVERRIDES', 'OVERRIDES', 'METHOD_IMPLEMENTS', @@ -2579,9 +2631,16 @@ export class LocalBackend { }; const symType = outcome.resolvedLabel || outcome.symbol.type || ''; + const effectiveRelationTypes = + (symType === 'Class' || symType === 'Interface') && + !hasExplicitRelationTypes && + !relationTypes.includes('ACCESSES') + ? [...relationTypes, 'ACCESSES'] + : relationTypes; + return this._runImpactBFS(repo, sym, symType, direction, { maxDepth, - relationTypes, + relationTypes: effectiveRelationTypes, includeTests, minConfidence, }); @@ -2660,6 +2719,30 @@ export class LocalBackend { frontier.push(rid); } } + + const typedPropertyRows = await executeParameterized( + repo.id, + ` + MATCH (p:\`Property\`) + WHERE p.declaredType = $name + OR p.declaredType STARTS WITH $genericPrefix + OR p.declaredType CONTAINS $genericArg + RETURN p.id AS id, p.name AS name, labels(p)[0] AS type, p.filePath AS filePath + `, + { + name: sym.name, + genericPrefix: `${sym.name}<`, + genericArg: `<${sym.name}>`, + }, + ); + + for (const r of typedPropertyRows) { + const rid = r.id || r[0]; + if (rid && !visited.has(rid)) { + visited.add(rid); + frontier.push(rid); + } + } } catch (e) { logQueryError('impact:class-node-expansion', e); } diff --git a/gitnexus/src/server/api.ts b/gitnexus/src/server/api.ts index cc65daa3d..2d49fabc4 100644 --- a/gitnexus/src/server/api.ts +++ b/gitnexus/src/server/api.ts @@ -744,8 +744,13 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => if (isMatch && ['queued', 'cloning', 'analyzing'].includes(job.status)) { if (process.env.DEBUG) { - console.log( - `[debug] resolveRepo waiting for active job ${job.id} (${normalizedName})...`, + // Sanitize user-controlled values to prevent log injection (CodeQL js/log-injection). + logger.debug( + { + jobId: String(job.id).replace(/[\r\n]/g, ' '), + repoName: String(normalizedName).replace(/[\r\n]/g, ' '), + }, + '[debug] resolveRepo waiting for active job', ); } for (let wait = 0; wait < HOLD_QUEUE_TIMEOUT_SECS; wait++) { @@ -769,7 +774,11 @@ export const createServer = async (port: number, host: string = '127.0.0.1') => // (e.g. registry file not yet flushed after clone completes). if (!found && normalizedName && !isRetry) { if (process.env.DEBUG) { - console.log(`[debug] resolveRepo 404 for "${normalizedName}". Triggering deep init...`); + // Sanitize user-controlled values to prevent log injection (CodeQL js/log-injection). + logger.debug( + { repoName: String(normalizedName).replace(/[\r\n]/g, ' ') }, + '[debug] resolveRepo 404, triggering deep init', + ); } await backend.init(); return await resolveRepo(normalizedName, true, req); diff --git a/gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs b/gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs new file mode 100644 index 000000000..28674ecda --- /dev/null +++ b/gitnexus/test/fixtures/lang-resolution/csharp-generic-type-refs/Program.cs @@ -0,0 +1,18 @@ +namespace App; + +public class USER_INFO +{ + public string? USER_ID { get; set; } +} + +public interface IEntityTypeConfiguration +{ +} + +public class UserInfoConfiguration : IEntityTypeConfiguration +{ + public Task> Load(List users) + { + return Task.FromResult(users); + } +} diff --git a/gitnexus/test/integration/context-typed-property.test.ts b/gitnexus/test/integration/context-typed-property.test.ts new file mode 100644 index 000000000..aef442147 --- /dev/null +++ b/gitnexus/test/integration/context-typed-property.test.ts @@ -0,0 +1,77 @@ +/** + * Integration test: context() expands Class symbols through typed properties. + * + * Reproduces EF-style usage where code reads a DbContext property + * (`db.USER_INFO`) whose source type is `DbSet`. The direct + * graph edge is Method -> Property, not Method -> Class, so context() must + * use the same typed-property bridge that impact() uses. + */ +import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { LocalBackend } from '../../src/mcp/local/local-backend.js'; +import { listRegisteredRepos } from '../../src/storage/repo-manager.js'; +import { withTestLbugDB } from '../helpers/test-indexed-db.js'; + +vi.mock('../../src/storage/repo-manager.js', () => ({ + listRegisteredRepos: vi.fn().mockResolvedValue([]), + cleanupOldKuzuFiles: vi.fn().mockResolvedValue({ found: false, needsReindex: false }), + findSiblingClones: vi.fn().mockResolvedValue([]), +})); + +const SEED = [ + `CREATE (c:Class {id:'Class:Models/USER_INFO.cs:USER_INFO', name:'USER_INFO', filePath:'Models/USER_INFO.cs', startLine:1, endLine:5, content:'public class USER_INFO {}', description:''})`, + `CREATE (p:\`Property\` {id:'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO', name:'USER_INFO', filePath:'Data/UserDbContext.cs', startLine:10, endLine:10, content:'public DbSet USER_INFO { get; set; }', description:'', declaredType:'DbSet'})`, + `CREATE (m:Method {id:'Method:Services/UserService.cs:UserService.GetUserInfo#1', name:'GetUserInfo', filePath:'Services/UserService.cs', startLine:20, endLine:30, isExported:false, content:'db.USER_INFO.FirstOrDefault();', description:'', parameterCount:1, returnType:'USER_INFO'})`, + `MATCH (m:Method {id:'Method:Services/UserService.cs:UserService.GetUserInfo#1'}), (p:\`Property\` {id:'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO'}) CREATE (m)-[:CodeRelation {type:'ACCESSES', confidence:1.0, reason:'read', step:1}]->(p)`, +]; + +withTestLbugDB( + 'context-typed-property', + (handle) => { + let backend: LocalBackend; + + beforeAll(async () => { + backend = (handle as any)._backend; + }); + + describe('context() typed-property expansion', () => { + it('surfaces property callers and explains the typed property bridge', async () => { + const result = await backend.callTool('context', { + uid: 'Class:Models/USER_INFO.cs:USER_INFO', + }); + + expect(result.status).toBe('found'); + expect(result.symbol.kind).toBe('Class'); + + const accesses = result.incoming.accesses || []; + expect(accesses.map((r: any) => r.name)).toContain('GetUserInfo'); + + expect(result.typed_properties).toEqual([ + expect.objectContaining({ + uid: 'Property:Data/UserDbContext.cs:UserDbContext.USER_INFO', + name: 'USER_INFO', + declaredType: 'DbSet', + }), + ]); + }); + }); + }, + { + seed: SEED, + poolAdapter: true, + afterSetup: async (handle) => { + vi.mocked(listRegisteredRepos).mockResolvedValue([ + { + name: 'test-repo', + path: '/test/repo', + storagePath: handle.tmpHandle.dbPath, + indexedAt: new Date().toISOString(), + lastCommit: 'abc123', + stats: { files: 3, nodes: 3, communities: 0, processes: 0 }, + }, + ]); + const backend = new LocalBackend(); + await backend.init(); + (handle as any)._backend = backend; + }, + }, +); diff --git a/gitnexus/test/integration/group/include-extractor-sync.test.ts b/gitnexus/test/integration/group/include-extractor-sync.test.ts new file mode 100644 index 000000000..908664bdc --- /dev/null +++ b/gitnexus/test/integration/group/include-extractor-sync.test.ts @@ -0,0 +1,195 @@ +/** + * Integration test: IncludeExtractor output → group matching → bridge DB. + * + * Covers PR #1156 review finding #7: verifies that the full runtime path + * (IncludeExtractor → StoredContract → runExactMatch → CrossLinks → writeBridge) + * stays wired up. A regression in either normalizeContractId or the include + * branch of ManifestExtractor.resolveSymbol would produce 0 cross-links and + * fail this test. + */ +import { describe, it, expect } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { parseGroupConfig } from '../../../src/core/group/config-parser.js'; +import { syncGroup } from '../../../src/core/group/sync.js'; +import type { StoredContract } from '../../../src/core/group/types.js'; +import { IncludeExtractor } from '../../../src/core/group/extractors/include-extractor.js'; +import { normalizeContractId } from '../../../src/core/group/matching.js'; + +const GROUP_YAML = [ + 'version: 1', + 'name: include-test-group', + 'description: "IncludeExtractor integration test"', + '', + 'repos:', + ' app/provider: include-provider', + ' app/consumer: include-consumer', + '', + 'links: []', + 'packages: {}', + '', + 'detect:', + ' http: false', + ' grpc: false', + ' topics: false', + ' shared_libs: false', + ' includes: true', + ' embedding_fallback: false', + '', + 'matching:', + ' bm25_threshold: 0.7', + ' embedding_threshold: 0.65', + ' max_candidates_per_step: 3', +].join('\n'); + +describe('IncludeExtractor → syncGroup integration (finding #7)', () => { + it('produces a CrossLink when provider and consumer emit the same include contract-id', async () => { + const config = parseGroupConfig(GROUP_YAML); + + // Mock the IncludeExtractor output directly — a header provider in one + // repo and a quoted #include consumer in the other, both normalized to + // the same include::map/base/view.h contract-id. + const mockContracts: StoredContract[] = [ + { + contractId: 'include::map/base/view.h', + type: 'include', + role: 'provider', + symbolUid: 'File:map/base/view.h', + symbolRef: { filePath: 'map/base/view.h', name: 'view.h' }, + symbolName: 'view.h', + confidence: 0.95, + meta: { source: 'filesystem' }, + repo: 'app/provider', + }, + { + contractId: 'include::map/base/view.h', + type: 'include', + role: 'consumer', + symbolUid: 'File:src/controller.cpp', + symbolRef: { filePath: 'src/controller.cpp', name: 'map/base/view.h' }, + symbolName: 'map/base/view.h', + confidence: 0.85, + meta: { source: 'tree_sitter', includePath: 'map/base/view.h' }, + repo: 'app/consumer', + }, + ]; + + const result = await syncGroup(config, { + extractorOverride: async () => mockContracts, + skipWrite: true, + }); + + const includeLinks = result.crossLinks.filter((l) => l.type === 'include'); + expect(includeLinks.length).toBeGreaterThanOrEqual(1); + + const link = includeLinks[0]; + expect(link.contractId).toBe('include::map/base/view.h'); + expect(link.matchType).toBe('exact'); + expect(link.from.repo).toBe('app/consumer'); + expect(link.to.repo).toBe('app/provider'); + }); + + it('normalizes mixed-case / backslash include paths to the same contract-id end-to-end', async () => { + const config = parseGroupConfig(GROUP_YAML); + + // Provider writes the canonical form; consumer's include has mixed case + // and a backslash. After normalizeContractId they must still match. + const providerId = 'include::map/base/view.h'; + const rawConsumerId = 'include::Map\\Base\\View.h'; + + // Sanity — normalizeContractId must collapse them. + expect(normalizeContractId(rawConsumerId)).toBe(providerId); + + const mockContracts: StoredContract[] = [ + { + contractId: providerId, + type: 'include', + role: 'provider', + symbolUid: 'File:map/base/view.h', + symbolRef: { filePath: 'map/base/view.h', name: 'view.h' }, + symbolName: 'view.h', + confidence: 0.95, + meta: { source: 'filesystem' }, + repo: 'app/provider', + }, + { + contractId: rawConsumerId, + type: 'include', + role: 'consumer', + symbolUid: 'File:src/controller.cpp', + symbolRef: { filePath: 'src/controller.cpp', name: 'Map/Base/View.h' }, + symbolName: 'Map/Base/View.h', + confidence: 0.85, + meta: { source: 'tree_sitter', includePath: 'Map\\Base\\View.h' }, + repo: 'app/consumer', + }, + ]; + + const result = await syncGroup(config, { + extractorOverride: async () => mockContracts, + skipWrite: true, + }); + + const includeLinks = result.crossLinks.filter((l) => l.type === 'include'); + expect(includeLinks.length).toBeGreaterThanOrEqual(1); + }); + + it('round-trip: extractor output from two real temp repos produces matching contract-ids', async () => { + // Drives the extractor directly (no `syncGroup`) against two on-disk + // fixture repos, then hands the StoredContract-shaped output to + // syncGroup via extractorOverride. This exercises the real extraction + // code + the matching pipeline together. + const providerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-int-provider-')); + const consumerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-int-consumer-')); + try { + fs.mkdirSync(path.join(providerDir, 'shared/api'), { recursive: true }); + fs.writeFileSync( + path.join(providerDir, 'shared/api/client.h'), + '#pragma once\nstruct Client {};', + ); + fs.mkdirSync(path.join(consumerDir, 'src'), { recursive: true }); + fs.writeFileSync( + path.join(consumerDir, 'src/main.cpp'), + '#include "shared/api/client.h"\nint main(){return 0;}', + ); + + const extractor = new IncludeExtractor(); + const providerOutput = await extractor.extract(null, providerDir, { + id: 'provider', + path: 'app/provider', + repoPath: providerDir, + storagePath: path.join(providerDir, '.gitnexus'), + }); + const consumerOutput = await extractor.extract(null, consumerDir, { + id: 'consumer', + path: 'app/consumer', + repoPath: consumerDir, + storagePath: path.join(consumerDir, '.gitnexus'), + }); + + const stored: StoredContract[] = [ + ...providerOutput + .filter((c) => c.role === 'provider') + .map((c) => ({ ...c, repo: 'app/provider' })), + ...consumerOutput + .filter((c) => c.role === 'consumer') + .map((c) => ({ ...c, repo: 'app/consumer' })), + ]; + + const config = parseGroupConfig(GROUP_YAML); + const result = await syncGroup(config, { + extractorOverride: async () => stored, + skipWrite: true, + }); + + const includeLinks = result.crossLinks.filter((l) => l.type === 'include'); + expect(includeLinks.length).toBeGreaterThanOrEqual(1); + expect(includeLinks[0].contractId).toBe('include::shared/api/client.h'); + expect(includeLinks[0].matchType).toBe('exact'); + } finally { + fs.rmSync(providerDir, { recursive: true, force: true }); + fs.rmSync(consumerDir, { recursive: true, force: true }); + } + }); +}); diff --git a/gitnexus/test/integration/resolvers/csharp.test.ts b/gitnexus/test/integration/resolvers/csharp.test.ts index 613ff7ec2..073b2da16 100644 --- a/gitnexus/test/integration/resolvers/csharp.test.ts +++ b/gitnexus/test/integration/resolvers/csharp.test.ts @@ -1440,6 +1440,24 @@ describe('Write access tracking (C#)', () => { }); }); +// --------------------------------------------------------------------------- +// Generic type references: IEntityTypeConfiguration, List +// --------------------------------------------------------------------------- + +describe('C# generic type-reference tracking', () => { + let result: PipelineResult; + + beforeAll(async () => { + result = await runPipelineFromRepo(path.join(FIXTURES, 'csharp-generic-type-refs'), () => {}); + }, 60000); + + it('emits USES edges for generic type arguments', () => { + const uses = getRelationships(result, 'USES').filter((e) => e.target === 'USER_INFO'); + expect(edgeSet(uses)).toContain('UserInfoConfiguration → USER_INFO'); + expect(edgeSet(uses)).toContain('Load → USER_INFO'); + }); +}); + // --------------------------------------------------------------------------- // Call-result variable binding (Phase 9): var user = GetUser(); user.Save() // --------------------------------------------------------------------------- diff --git a/gitnexus/test/integration/resolvers/helpers.ts b/gitnexus/test/integration/resolvers/helpers.ts index 5368c1715..e92526b19 100644 --- a/gitnexus/test/integration/resolvers/helpers.ts +++ b/gitnexus/test/integration/resolvers/helpers.ts @@ -11,6 +11,9 @@ import type { GraphRelationship } from 'gitnexus-shared'; const LEGACY_RESOLVER_PARITY_EXPECTED_FAILURES: Readonly>> = { csharp: new Set([ 'emits the using-import edge App/Program.cs -> Models/User.cs through the scope-resolution path', + // Generic type-argument USES edges are emitted by the registry-primary + // resolver only; the legacy DAG path does not synthesize these references. + 'emits USES edges for generic type arguments', ]), go: new Set([ // The legacy DAG path does not resolve method calls when the method is diff --git a/gitnexus/test/unit/cli-commands.test.ts b/gitnexus/test/unit/cli-commands.test.ts index a059a37bc..a42afb25c 100644 --- a/gitnexus/test/unit/cli-commands.test.ts +++ b/gitnexus/test/unit/cli-commands.test.ts @@ -10,6 +10,9 @@ vi.mock('../../src/cli/mcp.js', () => ({ vi.mock('../../src/cli/setup.js', () => ({ setupCommand: vi.fn(), })); +vi.mock('../../src/cli/publish.js', () => ({ + publishCommand: vi.fn(), +})); describe('CLI commands', () => { describe('version', () => { @@ -84,4 +87,11 @@ describe('CLI commands', () => { expect(typeof setupCommand).toBe('function'); }); }); + + describe('publishCommand', () => { + it('is a function', async () => { + const { publishCommand } = await import('../../src/cli/publish.js'); + expect(typeof publishCommand).toBe('function'); + }); + }); }); diff --git a/gitnexus/test/unit/cli-index-help.test.ts b/gitnexus/test/unit/cli-index-help.test.ts index 59109c8d9..889a7473f 100644 --- a/gitnexus/test/unit/cli-index-help.test.ts +++ b/gitnexus/test/unit/cli-index-help.test.ts @@ -63,4 +63,17 @@ describe('CLI help surface', () => { expect(result.stdout).toContain('--model '); expect(result.stdout).toContain('--gist'); }); + + it('publish help names the registry, the token env var, and the opt-out behaviour', () => { + const result = runHelp('publish'); + + expect(result.status).toBe(0); + expect(result.stdout).toContain('--id '); + expect(result.stdout).toContain('--skip-git'); + // Discoverability contract: a contributor scanning `--help` must see + // (a) which registry this dispatches to, and (b) the env var that + // gates the opt-in. Both are part of the no-token contract. + expect(result.stdout).toContain('understand-quickly'); + expect(result.stdout).toContain('UNDERSTAND_QUICKLY_TOKEN'); + }); }); diff --git a/gitnexus/test/unit/cpp-ue-preprocessor.test.ts b/gitnexus/test/unit/cpp-ue-preprocessor.test.ts new file mode 100644 index 000000000..087c5ba2a --- /dev/null +++ b/gitnexus/test/unit/cpp-ue-preprocessor.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect } from 'vitest'; +import Parser from 'tree-sitter'; +import CPP from 'tree-sitter-cpp'; +import { stripUeMacros } from '../../src/core/ingestion/cpp-ue-preprocessor.js'; + +describe('stripUeMacros — detection guard', () => { + it('returns input unchanged when no UE markers are present', () => { + const src = `class Plain {\npublic:\n int Get() const;\n};`; + expect(stripUeMacros(src)).toBe(src); + }); + + it('returns input unchanged for STL-style code', () => { + const src = `#include \nstd::vector v;`; + expect(stripUeMacros(src)).toBe(src); + }); +}); + +describe('stripUeMacros — length preservation', () => { + const ueSamples: string[] = [ + `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject { GENERATED_BODY() public: UFUNCTION() void Run(); };`, + `UPROPERTY(EditAnywhere, BlueprintReadOnly, Category = "Combat") int32 Health;`, + `USTRUCT(BlueprintType)\nstruct ENGINE_API FMyData { GENERATED_BODY() float Value; };`, + `DECLARE_DYNAMIC_MULTICAST_DELEGATE_TwoParams(FMyDelegate, int32, A, FString, B);`, + `UE_DEPRECATED(5.0, "Use NewThing instead") void OldThing();`, + ]; + + for (const src of ueSamples) { + it(`preserves byte length: ${src.slice(0, 40).replace(/\n/g, '\\n')}…`, () => { + const out = stripUeMacros(src); + expect(out.length).toBe(src.length); + }); + + it(`preserves newline positions: ${src.slice(0, 40).replace(/\n/g, '\\n')}…`, () => { + const out = stripUeMacros(src); + const inputNewlines: number[] = []; + const outputNewlines: number[] = []; + for (let i = 0; i < src.length; i++) { + if (src.charCodeAt(i) === 0x0a) inputNewlines.push(i); + if (out.charCodeAt(i) === 0x0a) outputNewlines.push(i); + } + expect(outputNewlines).toEqual(inputNewlines); + }); + } +}); + +describe('stripUeMacros — macro removal', () => { + it('elides UCLASS(...) with arguments', () => { + const src = `UCLASS(BlueprintType, Category="Foo")\nclass UFoo {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('UCLASS'); + expect(out).not.toContain('BlueprintType'); + expect(out).toContain('class UFoo {};'); + }); + + it('elides UCLASS() with empty parens', () => { + const src = `UCLASS()\nclass UBar {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('UCLASS'); + expect(out).toContain('class UBar {};'); + }); + + it('elides MODULE_API export macros (BRAWLUI_API style) when paired with a UE marker', () => { + const src = `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('BRAWLUI_API'); + expect(out).toContain('class'); + expect(out).toContain('UMyClass'); + expect(out).toContain('public UObject'); + }); + + it('elides multiple distinct *_API tokens in same file when UE marker is present', () => { + const src = `UCLASS()\nclass CORE_API A {};\nUCLASS()\nclass UMG_API B : public A {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('CORE_API'); + expect(out).not.toContain('UMG_API'); + expect(out).toContain('class'); + expect(out).toContain('A {};'); + }); + + it('elides GENERATED_BODY() inside class body', () => { + const src = `class UThing { GENERATED_BODY() public: void Foo(); };`; + const out = stripUeMacros(src); + expect(out).not.toContain('GENERATED_BODY'); + expect(out).toContain('public:'); + expect(out).toContain('void Foo();'); + }); + + it('elides UFUNCTION(...) before method declarations', () => { + const src = `class X { UFUNCTION(BlueprintCallable, Server, Reliable) void DoThing(); };`; + const out = stripUeMacros(src); + expect(out).not.toContain('UFUNCTION'); + expect(out).not.toContain('BlueprintCallable'); + expect(out).toContain('void DoThing();'); + }); + + it('elides UPROPERTY(...) before field declarations', () => { + const src = `class X { UPROPERTY(EditAnywhere) int32 Health; };`; + const out = stripUeMacros(src); + expect(out).not.toContain('UPROPERTY'); + expect(out).not.toContain('EditAnywhere'); + expect(out).toContain('int32 Health;'); + }); + + it('elides DECLARE_DYNAMIC_MULTICAST_DELEGATE_*Params(...)', () => { + const src = `DECLARE_DYNAMIC_MULTICAST_DELEGATE_OneParam(FMyDelegate, int32, Value);\nclass X {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('DECLARE_DYNAMIC_MULTICAST_DELEGATE'); + expect(out).not.toContain('FMyDelegate'); + expect(out).toContain('class X {};'); + }); + + it('elides UE_DEPRECATED(...) before function declarations', () => { + const src = `UE_DEPRECATED(5.1, "Reason") void Old();`; + const out = stripUeMacros(src); + expect(out).not.toContain('UE_DEPRECATED'); + expect(out).not.toContain('5.1'); + expect(out).toContain('void Old();'); + }); +}); + +describe('stripUeMacros — non-UE files left alone', () => { + it('does NOT strip standalone *_API identifiers when no UE marker is present', () => { + const src = `enum class Status { REST_API = 1, HTTP_API = 2, MY_LIB_API = 3 };\nvoid handle(REST_API status);`; + expect(stripUeMacros(src)).toBe(src); + }); + + it('does NOT strip _API tokens in a file that only mentions DECLARE_DELEGATE-like macros from non-UE codebases', () => { + const src = `// Custom delegate framework, not UE\n#define DECLARE_HANDLER(x) void x()\nDECLARE_HANDLER(MyHandler);\nint REST_API = 0;`; + expect(stripUeMacros(src)).toBe(src); + }); +}); + +describe('stripUeMacros — non-ASCII content preservation', () => { + it('leaves non-ASCII content outside elided ranges intact and at the same .length offset', () => { + const src = `// Comment with non-ASCII: café résumé naïve\nUCLASS()\nclass UMyClass : public UObject\n{\n GENERATED_BODY()\n // Trailing: 日本語 αβγ\n};`; + const out = stripUeMacros(src); + expect(out.length).toBe(src.length); + expect(out).toContain('café résumé naïve'); + expect(out).toContain('日本語 αβγ'); + expect(out).toContain('class UMyClass : public UObject'); + expect(out).not.toContain('UCLASS'); + expect(out).not.toContain('GENERATED_BODY'); + }); + + it('preserves newline positions when the file contains non-ASCII characters', () => { + const src = `// café\nUPROPERTY()\nint32 Health;\n// résumé\nUFUNCTION()\nvoid Run();`; + const out = stripUeMacros(src); + const inputNewlines: number[] = []; + const outputNewlines: number[] = []; + for (let i = 0; i < src.length; i++) { + if (src.charCodeAt(i) === 0x0a) inputNewlines.push(i); + if (out.charCodeAt(i) === 0x0a) outputNewlines.push(i); + } + expect(outputNewlines).toEqual(inputNewlines); + }); +}); + +describe('stripUeMacros — false-positive guards', () => { + it('does NOT strip identifiers that merely contain UCLASS as a substring', () => { + const src = `void NotUCLASSAtAll(); int MyUCLASS = 0;`; + const out = stripUeMacros(src); + expect(out).toBe(src); + }); + + it('does NOT strip _API substrings inside larger identifiers', () => { + const src = `class MY_APIName {};\nint not_my_API_thing = 0;`; + const out = stripUeMacros(src); + expect(out).toContain('MY_APIName'); + expect(out).toContain('not_my_API_thing'); + }); + + it('does not eat parens balanced inside string literals', () => { + const src = `UFUNCTION(meta=(DisplayName="Foo (Bar)")) void Z();`; + const out = stripUeMacros(src); + expect(out).not.toContain('UFUNCTION'); + expect(out).not.toContain('DisplayName'); + expect(out).toContain('void Z();'); + }); + + it('handles UCLASS with deeply nested parens in arguments', () => { + const src = `UCLASS(meta=(Categories=("A.B", "C.D")), Within=Foo) class UDeep {};`; + const out = stripUeMacros(src); + expect(out).not.toContain('UCLASS'); + expect(out).not.toContain('Categories'); + expect(out).toContain('class UDeep {};'); + }); + + it('leaves Qt macros alone (only UE markers stripped)', () => { + const src = `class QFoo { Q_OBJECT public: void Bar(); };`; + const out = stripUeMacros(src); + expect(out).toContain('Q_OBJECT'); + }); +}); + +describe('stripUeMacros — class-name extraction sanity', () => { + it('after stripping, "class UMyClass" appears immediately after "class "', () => { + const src = `UCLASS(BlueprintType)\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n};`; + const out = stripUeMacros(src); + const classIdx = out.indexOf('class '); + expect(classIdx).toBeGreaterThanOrEqual(0); + const tail = out.slice(classIdx + 'class '.length).trimStart(); + expect(tail.startsWith('UMyClass')).toBe(true); + }); +}); + +describe('stripUeMacros — tree-sitter extraction (end-to-end)', () => { + /** + * Walk the parse tree and return the captured class name(s). Works against + * the actual tree-sitter-cpp grammar so this is a true integration check + * for the core PR claim: the indexer now sees `UMyClass`, not `BRAWLUI_API`. + */ + function extractClassNames(source: string): string[] { + const parser = new Parser(); + parser.setLanguage(CPP as unknown as Parser.Language); + const tree = parser.parse(source); + const names: string[] = []; + const stack: Parser.SyntaxNode[] = [tree.rootNode]; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.type === 'class_specifier' || node.type === 'struct_specifier') { + const nameNode = node.childForFieldName('name'); + if (nameNode) names.push(nameNode.text); + } + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child) stack.push(child); + } + } + return names; + } + + it('tree-sitter-cpp captures UMyClass as the class name (not BRAWLUI_API)', () => { + const src = `UCLASS(BlueprintType)\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n public:\n UFUNCTION()\n void Run();\n};`; + const out = stripUeMacros(src); + const names = extractClassNames(out); + expect(names).toContain('UMyClass'); + expect(names).not.toContain('BRAWLUI_API'); + }); + + it('tree-sitter-cpp captures struct name correctly through USTRUCT + MODULE_API', () => { + const src = `USTRUCT(BlueprintType)\nstruct ENGINE_API FMyData : public FBase\n{\n GENERATED_BODY()\n float Value;\n};`; + const out = stripUeMacros(src); + const names = extractClassNames(out); + expect(names).toContain('FMyData'); + expect(names).not.toContain('ENGINE_API'); + }); + + it('tree-sitter-cpp source positions are preserved across stripping (line numbers match)', () => { + const src = `UCLASS()\nclass BRAWLUI_API UMyClass : public UObject\n{\n GENERATED_BODY()\n public:\n void Run();\n};`; + const out = stripUeMacros(src); + const parser = new Parser(); + parser.setLanguage(CPP as unknown as Parser.Language); + const tree = parser.parse(out); + const stack: Parser.SyntaxNode[] = [tree.rootNode]; + let runLine: number | undefined; + while (stack.length > 0) { + const node = stack.pop()!; + if (node.type === 'function_declarator') { + const declarator = node.childForFieldName('declarator'); + if (declarator?.text === 'Run') { + runLine = node.startPosition.row; + break; + } + } + for (let i = node.namedChildCount - 1; i >= 0; i--) { + const child = node.namedChild(i); + if (child) stack.push(child); + } + } + expect(runLine).toBe(5); // 0-indexed: "void Run();" is on line 6 (index 5) + }); +}); diff --git a/gitnexus/test/unit/group/config-parser.test.ts b/gitnexus/test/unit/group/config-parser.test.ts index e1d3b540f..22bb2ad26 100644 --- a/gitnexus/test/unit/group/config-parser.test.ts +++ b/gitnexus/test/unit/group/config-parser.test.ts @@ -75,6 +75,62 @@ repos: expect(config.detect.thrift).toBe(true); }); + // PR #1156 Codex follow-up: include extraction is opt-in. Existing + // group.yaml files that do not declare `detect.includes` must not gain + // a wave of new include::* contracts on the next sync after upgrade. + describe('detect.includes opt-in default', () => { + it('defaults includes detection to false when detect block omits it', () => { + const minimal = ` +version: 1 +name: test +repos: + app: my-app +`; + const config = parseGroupConfig(minimal); + expect(config.detect.includes).toBe(false); + }); + + it('defaults includes detection to false when detect block is present but omits the key', () => { + const yaml = ` +version: 1 +name: test +repos: + app: my-app +detect: + http: true + grpc: false +`; + const config = parseGroupConfig(yaml); + expect(config.detect.includes).toBe(false); + }); + + it('honors explicit detect.includes: true (opt-in works)', () => { + const yaml = ` +version: 1 +name: test +repos: + app: my-app +detect: + includes: true +`; + const config = parseGroupConfig(yaml); + expect(config.detect.includes).toBe(true); + }); + + it('honors explicit detect.includes: false', () => { + const yaml = ` +version: 1 +name: test +repos: + app: my-app +detect: + includes: false +`; + const config = parseGroupConfig(yaml); + expect(config.detect.includes).toBe(false); + }); + }); + it('parses thrift manifest links', () => { const yaml = ` version: 1 diff --git a/gitnexus/test/unit/group/include-extractor.test.ts b/gitnexus/test/unit/group/include-extractor.test.ts new file mode 100644 index 000000000..3956cd5f2 --- /dev/null +++ b/gitnexus/test/unit/group/include-extractor.test.ts @@ -0,0 +1,563 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; +import { IncludeExtractor } from '../../../src/core/group/extractors/include-extractor.js'; +import type { RepoHandle } from '../../../src/core/group/types.js'; +import { normalizeContractId } from '../../../src/core/group/matching.js'; + +describe('IncludeExtractor', () => { + let tmpDir: string; + let extractor: IncludeExtractor; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-')); + extractor = new IncludeExtractor(); + }); + + afterEach(() => { + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + function writeFile(relPath: string, content: string): void { + const full = path.join(tmpDir, relPath); + fs.mkdirSync(path.dirname(full), { recursive: true }); + fs.writeFileSync(full, content); + } + + const makeRepo = (repoPath: string): RepoHandle => ({ + id: 'test-repo', + path: 'test/app', + repoPath, + storagePath: path.join(repoPath, '.gitnexus'), + }); + + // ---- Provider detection ---- + + describe('provider extraction', () => { + it('registers .h files as providers', async () => { + writeFile('map/base/view.h', '#pragma once\nclass View {};'); + writeFile('map/base/types.h', '#pragma once\nstruct Point {};'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers).toHaveLength(2); + const ids = providers.map((p) => p.contractId).sort(); + expect(ids).toEqual(['include::map/base/types.h', 'include::map/base/view.h']); + expect(providers[0].type).toBe('include'); + expect(providers[0].confidence).toBeGreaterThanOrEqual(0.95); + }); + + it('registers .hpp files as providers', async () => { + writeFile('utils/helper.hpp', '#pragma once\ntemplate T id(T x) { return x; }'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers).toHaveLength(1); + expect(providers[0].contractId).toBe('include::utils/helper.hpp'); + }); + + it('does not register .cpp files as providers', async () => { + writeFile('src/main.cpp', 'int main() { return 0; }'); + writeFile('src/utils.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers).toHaveLength(1); + expect(providers[0].contractId).toBe('include::src/utils.h'); + }); + }); + + // ---- Consumer detection ---- + + describe('consumer extraction', () => { + it('emits unresolved includes as consumers', async () => { + writeFile( + 'src/main.cpp', + `#include "map/base/view.h" +#include "map/base/types.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(2); + const ids = consumers.map((c) => c.contractId).sort(); + expect(ids).toEqual(['include::map/base/types.h', 'include::map/base/view.h']); + expect(consumers[0].type).toBe('include'); + expect(consumers[0].confidence).toBe(0.85); + }); + + it('skips locally resolved includes', async () => { + writeFile('map/base/view.h', '#pragma once\nclass View {};'); + writeFile( + 'src/main.cpp', + `#include "map/base/view.h" +#include "external/lib.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Only external/lib.h should be a consumer — map/base/view.h resolves locally + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::external/lib.h'); + }); + + it('skips angle-bracket includes', async () => { + writeFile( + 'src/main.cpp', + `#include +#include +#include "app/interface.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::app/interface.h'); + }); + + it('skips well-known system headers in quotes', async () => { + writeFile( + 'src/main.cpp', + `#include "stdio.h" +#include "stdlib.h" +#include "app/config.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::app/config.h'); + }); + + it('skips system path prefixes', async () => { + writeFile( + 'src/main.c', + `#include "sys/types.h" +#include "linux/input.h" +#include "mylib/types.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::mylib/types.h'); + }); + }); + + // ---- Cross-repo matching scenario ---- + + describe('cross-repo matching', () => { + it('provider and consumer produce matching contractIds', async () => { + // Simulate provider repo (header-only) + const providerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-provider-')); + const providerFile = path.join(providerDir, 'map/base/dice_map_view.h'); + fs.mkdirSync(path.dirname(providerFile), { recursive: true }); + fs.writeFileSync(providerFile, '#pragma once\nclass DiceMapView {};'); + + // Simulate consumer repo + const consumerDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-include-consumer-')); + const consumerFile = path.join(consumerDir, 'src/controller.cpp'); + fs.mkdirSync(path.dirname(consumerFile), { recursive: true }); + fs.writeFileSync(consumerFile, '#include "map/base/dice_map_view.h"\nvoid init() {}'); + + try { + const providerContracts = await extractor.extract(null, providerDir, makeRepo(providerDir)); + const consumerContracts = await extractor.extract(null, consumerDir, makeRepo(consumerDir)); + + const providers = providerContracts.filter((c) => c.role === 'provider'); + const consumers = consumerContracts.filter((c) => c.role === 'consumer'); + + expect(providers.length).toBeGreaterThanOrEqual(1); + expect(consumers.length).toBeGreaterThanOrEqual(1); + + const providerIds = new Set(providers.map((p) => normalizeContractId(p.contractId))); + const consumerIds = consumers.map((c) => normalizeContractId(c.contractId)); + + // The consumer's include path should match a provider's file path + expect(providerIds.has(consumerIds[0])).toBe(true); + } finally { + fs.rmSync(providerDir, { recursive: true, force: true }); + fs.rmSync(consumerDir, { recursive: true, force: true }); + } + }); + }); + + // ---- Review finding #4: suffixResolve ambiguity ---- + + describe('finding #4: suffix-ambiguity does not silently suppress cross-repo include', () => { + it('emits a cross-repo contract when the include path does not match any local file (even if a shorter suffix does)', async () => { + // local repo has `internal/api.h` but NOT `ext/api.h` + writeFile('internal/api.h', '#pragma once'); + writeFile( + 'src/main.cpp', + `#include "ext/api.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Previously suffixResolve would match `api.h` against `internal/api.h` + // and drop the cross-repo contract. After finding #4 fix, we only + // accept exact full-path matches — so `ext/api.h` must still be + // emitted as a consumer contract. + expect(consumers).toHaveLength(1); + expect(consumers[0].contractId).toBe('include::ext/api.h'); + }); + + it('still suppresses a local include when the FULL path matches', async () => { + writeFile('ext/api.h', '#pragma once'); + writeFile('src/main.cpp', '#include "ext/api.h"\nint main(){return 0;}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + + it('resolves locally when include omits extension and a matching .h exists', async () => { + writeFile('foo/bar.h', '#pragma once'); + writeFile('src/main.cpp', '#include "foo/bar"\nint main(){return 0;}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + }); + + // ---- Review finding #5: regex fallback must strip block comments ---- + + describe('finding #5: regex fallback ignores block-commented includes', () => { + it('does not emit a contract for an #include inside /* ... */', async () => { + // Force regex fallback by producing a file larger than tree-sitter's + // 32 KB hard cap. The include we care about lives inside a block + // comment that spans the file. + const filler = 'int dummy_' + 'x'.repeat(32) + ' = 0;\n'.repeat(1200); + const content = `/* + * Historical include, kept for reference only: + * #include "legacy/old-api.h" + */ +${filler} +#include "real/api.h" +int main(){return 0;}`; + writeFile('src/huge.cpp', content); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + const ids = consumers.map((c) => c.contractId); + + // The live include should appear; the commented-out one must NOT. + expect(ids).toContain('include::real/api.h'); + expect(ids).not.toContain('include::legacy/old-api.h'); + }); + }); + + // ---- Review finding #6: meta.source must reflect which extraction path ran ---- + + describe('finding #6: meta.source reflects extraction path', () => { + it('stamps `tree_sitter` on contracts produced via AST walking', async () => { + writeFile('src/main.cpp', '#include "app/small.h"\nint main(){return 0;}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(1); + expect((consumers[0].meta as { source?: string } | undefined)?.source).toBe('tree_sitter'); + }); + + it('meta.source is one of the two documented values (tree_sitter | regex_fallback)', async () => { + // Regex fallback is a defensive branch that only fires if + // parser.setLanguage() or parser.parse() throws. In practice + // tree-sitter-c/cpp handles realistic inputs, so we only assert + // the meta.source contract: it is always present and always one of + // the two documented values. This guards against future regressions + // that might hard-code the wrong string. + writeFile('src/main.cpp', '#include "ext/whatever.h"\nint main(){return 0;}'); + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumer = contracts.find((c) => c.role === 'consumer'); + expect(consumer).toBeDefined(); + const src = (consumer?.meta as { source?: string } | undefined)?.source; + expect(['tree_sitter', 'regex_fallback']).toContain(src); + }); + }); + + // ---- Review finding #3: provider id collision on case-sensitive FS ---- + + describe('finding #3: case-folding is documented and deterministic', () => { + it('collapses `Foo.h` and `foo.h` onto the same provider contract-id (documented trade-off)', async () => { + writeFile('Foo.h', '#pragma once\n// Capital Foo'); + // On case-insensitive filesystems (macOS default) the second writeFile + // will overwrite the first, so we only create this when distinct files + // can coexist (case-sensitive FS, e.g. Linux CI). + try { + fs.writeFileSync(path.join(tmpDir, 'foo.h'), '#pragma once\n// lowercase foo'); + } catch { + // Ignore — some FS won't allow both names to coexist. + } + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + const ids = providers.map((p) => p.contractId); + + // Both files (if they coexist) must normalize to the same id. + // dedupe() keeps only one; caller code must be aware of this. + expect(ids).toContain('include::foo.h'); + // Never see a mixed-case contract-id leak out. + expect(ids.every((id) => id === id.toLowerCase())).toBe(true); + }); + }); + + // ---- Deduplication ---- + + describe('deduplication', () => { + it('deduplicates same include from multiple source files', async () => { + writeFile('src/a.cpp', '#include "ext/api.h"\nvoid a() {}'); + writeFile('src/b.cpp', '#include "ext/api.h"\nvoid b() {}'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Both files include "ext/api.h" — each should produce a separate + // consumer contract (different symbolRef.filePath) + expect(consumers).toHaveLength(2); + const files = consumers.map((c) => c.symbolRef.filePath).sort(); + expect(files).toEqual(['src/a.cpp', 'src/b.cpp']); + }); + }); + + // ---- normalizeContractId ---- + + describe('normalizeContractId for include', () => { + it('lowercases the path', () => { + expect(normalizeContractId('include::Map/Base/Foo.h')).toBe('include::map/base/foo.h'); + }); + + it('normalizes backslashes', () => { + expect(normalizeContractId('include::map\\base\\foo.h')).toBe('include::map/base/foo.h'); + }); + + it('strips leading ./', () => { + expect(normalizeContractId('include::./foo.h')).toBe('include::foo.h'); + }); + + it('collapses consecutive slashes', () => { + expect(normalizeContractId('include::map//base///foo.h')).toBe('include::map/base/foo.h'); + }); + }); + + // ---- PR #1156 follow-up: `../` relative includes ---- + + describe('follow-up: `../` relative includes are skipped', () => { + it('does not emit a consumer contract for `#include "../foo.h"`', async () => { + // Producer: a header that exists locally but only via parent reference + writeFile('include/foo.h', '#pragma once'); + writeFile( + 'src/sub/main.cpp', + `#include "../../include/foo.h" +#include "real/cross_repo.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + // Only `real/cross_repo.h` should remain — the `..`-prefixed include + // is intra-repo noise that no provider can ever satisfy. + expect(consumers.map((c) => c.contractId)).toEqual(['include::real/cross_repo.h']); + }); + + it('skips backslash-form `..\\` for completeness', async () => { + writeFile( + 'src/main.cpp', + `#include "..\\\\sibling\\\\foo.h" +#include "remote/header.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + const ids = consumers.map((c) => c.contractId); + expect(ids).toContain('include::remote/header.h'); + expect(ids.some((id) => id.includes('..'))).toBe(false); + }); + }); + + // ---- PR #1156 follow-up: macro-style includes ---- + + describe('follow-up: macro-style #include emits no consumer contract', () => { + it('does not emit a consumer contract for `#include PLATFORM_HEADER` (no separator, no dot)', async () => { + // `#include PLATFORM_HEADER` parses under tree-sitter as an identifier + // node, slips past the existing system-header / `..` filters, and used + // to leak through as a permanently orphaned consumer contract because + // no file is ever named `PLATFORM_HEADER`. Verify the macro guard + // suppresses it while preserving the real cross-repo include. + writeFile( + 'src/main.cpp', + `#include PLATFORM_HEADER +#include "real/api.h" +int main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers.map((c) => c.contractId)).toEqual(['include::real/api.h']); + }); + + it('skips multiple macro identifiers in the same translation unit', async () => { + writeFile( + 'src/cfg.cpp', + `#include CONFIG_HEADER +#include PLATFORM_HEADER +#include ASSERT_H_ +int main(){return 0;}`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumers = contracts.filter((c) => c.role === 'consumer'); + + expect(consumers).toHaveLength(0); + }); + }); + + // ---- PR #1156 follow-up: graph provider absolute paths ---- + + describe('follow-up: extractProvidersGraph strips repo root from absolute paths', () => { + it('produces repo-relative contract IDs when the graph returns absolute paths', async () => { + writeFile('map/base/view.h', '#pragma once\nclass View {};'); + writeFile('utils/types.hpp', '#pragma once'); + + // Stub the Cypher executor to return absolute paths the way + // gitnexus analyze actually persists them. + const absolute1 = path.join(tmpDir, 'map/base/view.h'); + const absolute2 = path.join(tmpDir, 'utils/types.hpp'); + const stubDb = async () => [ + { filePath: absolute1, fileId: 'File:abs:1' }, + { filePath: absolute2, fileId: 'File:abs:2' }, + ]; + + const contracts = await extractor.extract(stubDb, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + const ids = providers.map((p) => p.contractId).sort(); + expect(ids).toEqual(['include::map/base/view.h', 'include::utils/types.hpp']); + expect(providers.every((p) => p.meta?.source === 'graph')).toBe(true); + }); + + it('drops graph rows whose path resolves outside the repo root', async () => { + writeFile('local/header.h', '#pragma once'); + const absoluteLocal = path.join(tmpDir, 'local/header.h'); + const stubDb = async () => [ + { filePath: absoluteLocal, fileId: 'File:1' }, + // Stale absolute path from a different machine — must be skipped. + { filePath: '/some/other/repo/foreign.h', fileId: 'File:2' }, + ]; + + const contracts = await extractor.extract(stubDb, tmpDir, makeRepo(tmpDir)); + const providers = contracts.filter((c) => c.role === 'provider'); + + expect(providers.map((p) => p.contractId)).toEqual(['include::local/header.h']); + }); + }); + + // ---- PR #1156 Codex follow-up: discovery aligned with ingestion ---- + + describe('follow-up: file discovery honors createIgnoreFilter and getMaxFileSizeBytes', () => { + it('does not emit a provider contract for a header excluded by .gitignore', async () => { + writeFile('.gitignore', 'vendor-headers/\n'); + writeFile('vendor-headers/blocked.h', '#pragma once'); + writeFile('src/wanted.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::src/wanted.h'); + expect(providerIds).not.toContain('include::vendor-headers/blocked.h'); + }); + + it('does not emit a provider contract for a header excluded by .gitnexusignore', async () => { + writeFile('.gitnexusignore', 'legacy/\n'); + writeFile('legacy/old.h', '#pragma once'); + writeFile('src/current.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::src/current.h'); + expect(providerIds).not.toContain('include::legacy/old.h'); + }); + + it('does not parse #include directives in a source file excluded by .gitignore', async () => { + // The ignored source file references a header that would otherwise be + // a cross-repo consumer. After alignment, the ignored file is invisible + // to the consumer scan — no consumer contract should appear. + writeFile('.gitignore', 'generated/\n'); + writeFile( + 'generated/auto.cpp', + `#include "remote/should_not_appear.h" +int auto_main() { return 0; }`, + ); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumerIds = contracts.filter((c) => c.role === 'consumer').map((c) => c.contractId); + + expect(consumerIds).not.toContain('include::remote/should_not_appear.h'); + }); + + it('skips a provider header whose size exceeds GITNEXUS_MAX_FILE_SIZE', async () => { + const previous = process.env.GITNEXUS_MAX_FILE_SIZE; + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; // 1 KB cap + try { + // 4 KB header — comfortably exceeds the cap. + const oversized = '#pragma once\n' + 'x'.repeat(4 * 1024); + writeFile('huge/big.h', oversized); + writeFile('small/tiny.h', '#pragma once'); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const providerIds = contracts.filter((c) => c.role === 'provider').map((p) => p.contractId); + + expect(providerIds).toContain('include::small/tiny.h'); + expect(providerIds).not.toContain('include::huge/big.h'); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_MAX_FILE_SIZE; + else process.env.GITNEXUS_MAX_FILE_SIZE = previous; + } + }); + + it('skips parsing #include directives in source files exceeding GITNEXUS_MAX_FILE_SIZE', async () => { + const previous = process.env.GITNEXUS_MAX_FILE_SIZE; + process.env.GITNEXUS_MAX_FILE_SIZE = '1'; + try { + const oversized = + '#include "remote/should_not_appear.h"\n' + + '// padding to push the file past 1 KB\n' + + 'x'.repeat(4 * 1024); + writeFile('big/main.cpp', oversized); + + const contracts = await extractor.extract(null, tmpDir, makeRepo(tmpDir)); + const consumerIds = contracts.filter((c) => c.role === 'consumer').map((c) => c.contractId); + + expect(consumerIds).not.toContain('include::remote/should_not_appear.h'); + } finally { + if (previous === undefined) delete process.env.GITNEXUS_MAX_FILE_SIZE; + else process.env.GITNEXUS_MAX_FILE_SIZE = previous; + } + }); + }); +}); diff --git a/gitnexus/test/unit/group/sync.test.ts b/gitnexus/test/unit/group/sync.test.ts index 88bc3af2d..9f14db231 100644 --- a/gitnexus/test/unit/group/sync.test.ts +++ b/gitnexus/test/unit/group/sync.test.ts @@ -3,6 +3,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import * as os from 'node:os'; import { syncGroup, stableRepoPoolId } from '../../../src/core/group/sync.js'; +import { cleanupTempDir } from '../../helpers/test-db.js'; import { _captureLogger } from '../../../src/core/logger.js'; import type { GroupConfig, @@ -583,6 +584,47 @@ service OrderService { } }); + it('does not extract include contracts during real sync when includes detection is disabled', async () => { + // PR #1156 Codex follow-up: ce-code-review T1 — verifies the gate at + // sync.ts:174 honors `detect.includes: false`. Mirrors the existing + // thrift-off pattern at sync.test.ts:545. + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-sync-includes-off-')); + const storageDir = path.join(tmpDir, '.gitnexus'); + fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); + fs.mkdirSync(storageDir, { recursive: true }); + fs.writeFileSync(path.join(tmpDir, 'src', 'view.h'), '#pragma once\nclass View {};'); + + const config = makeConfig({ 'app/cpp-lib': 'cpp-lib-repo' }); + config.detect.http = false; + config.detect.grpc = false; + config.detect.thrift = false; + config.detect.topics = false; + config.detect.includes = false; + + const poolAdapter = await import('../../../src/core/lbug/pool-adapter.js'); + const initSpy = vi.spyOn(poolAdapter, 'initLbug').mockResolvedValue(undefined); + const closeSpy = vi.spyOn(poolAdapter, 'closeLbug').mockResolvedValue(undefined); + + try { + const result = await syncGroup(config, { + resolveRepoHandle: async (_name, groupPath) => ({ + id: 'cpp-lib-repo', + path: groupPath, + repoPath: tmpDir, + storagePath: storageDir, + }), + skipWrite: true, + }); + + expect(result.missingRepos).toHaveLength(0); + expect(result.contracts.filter((c) => c.type === 'include')).toHaveLength(0); + } finally { + initSpy.mockRestore(); + closeSpy.mockRestore(); + await cleanupTempDir(tmpDir); + } + }); + it('dedupes duplicate wildcard cross-links during sync', async () => { const config = makeConfig({ 'app/provider': 'provider-repo', 'app/consumer': 'consumer-repo' }); const provider: StoredContract = { @@ -689,7 +731,12 @@ service OrderService { expect(registry.version).toBe(1); expect(registry.contracts).toHaveLength(0); } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); + // syncGroup now writes bridge.lbug + WAL/shadow sidecars when + // skipWrite is false. On Windows, LadybugDB's checkpoint thread can + // briefly outlive closeBridgeDb, holding a Win32 lock on the file. + // cleanupTempDir tolerates the documented Windows-native lock codes + // (EBUSY/EPERM/EACCES/ENOTEMPTY) with bounded retries. + await cleanupTempDir(tmpDir); } }); diff --git a/gitnexus/test/unit/hf-env.test.ts b/gitnexus/test/unit/hf-env.test.ts index fa8f23bd3..5999b34a2 100644 --- a/gitnexus/test/unit/hf-env.test.ts +++ b/gitnexus/test/unit/hf-env.test.ts @@ -1,12 +1,12 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; import os from 'node:os'; import { join } from 'node:path'; +import { CircuitBreaker } from 'gitnexus-shared'; import { applyHfEnvOverrides, isNetworkFetchError, isHfDownloadFailure, isHfCircuitOpenError, - HfDownloadCircuitBreaker, withDownloadTimeout, withHfDownloadRetry, CIRCUIT_OPEN_TAG, @@ -154,85 +154,13 @@ describe('isHfDownloadFailure', () => { }); }); -describe('HfDownloadCircuitBreaker', () => { - it('starts in closed state', () => { - const cb = new HfDownloadCircuitBreaker(); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - }); - - it('opens after reaching the failure threshold', () => { - const cb = new HfDownloadCircuitBreaker(3); - cb.recordFailure(); - cb.recordFailure(); - expect(cb.isOpen()).toBe(false); - cb.recordFailure(); // threshold reached - expect(cb.isOpen()).toBe(true); - expect(cb.state).toBe('open'); - }); - - it('closes on recordSuccess after being open', () => { - const cb = new HfDownloadCircuitBreaker(1); - cb.recordFailure(); - expect(cb.isOpen()).toBe(true); - cb.recordSuccess(); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - }); - - it('transitions to half-open after the reset timeout', () => { - vi.useFakeTimers(); - try { - const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); - cb.recordFailure(); - expect(cb.isOpen()).toBe(true); - vi.advanceTimersByTime(200); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('half-open'); - } finally { - vi.useRealTimers(); - } - }); - - it('reset() restores closed state', () => { - const cb = new HfDownloadCircuitBreaker(1); - cb.recordFailure(); - expect(cb.isOpen()).toBe(true); - cb.reset(); - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - }); - - it('re-opens when a failure is recorded in half-open state', () => { - vi.useFakeTimers(); - try { - const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); - cb.recordFailure(); // opens the circuit - vi.advanceTimersByTime(200); // advance past reset timeout - expect(cb.state).toBe('half-open'); // getter transitions _state to half-open - cb.recordFailure(); // failure in half-open → re-opens - expect(cb.isOpen()).toBe(true); - expect(cb.state).toBe('open'); - } finally { - vi.useRealTimers(); - } - }); - - it('closes the circuit when success is recorded in half-open state', () => { - vi.useFakeTimers(); - try { - const cb = new HfDownloadCircuitBreaker(1, 100 /* 100ms */); - cb.recordFailure(); // opens the circuit - vi.advanceTimersByTime(200); // advance past reset timeout - expect(cb.state).toBe('half-open'); - cb.recordSuccess(); // success in half-open → closes - expect(cb.isOpen()).toBe(false); - expect(cb.state).toBe('closed'); - } finally { - vi.useRealTimers(); - } - }); -}); +// CircuitBreaker state-machine tests live in +// `gitnexus/test/unit/integrations/circuit-breaker.test.ts` — that suite +// already covers the closed/open/half-open transitions, recordSuccess/ +// recordFailure semantics, half-open probe gating, and configurable +// thresholds. No need to duplicate here; this file's remaining tests +// focus on HF-specific composition (withHfDownloadRetry, env-var +// overrides, error classification). describe('withDownloadTimeout', () => { it('resolves when fn completes before the timeout', async () => { @@ -262,7 +190,7 @@ describe('withDownloadTimeout', () => { describe('withHfDownloadRetry', () => { it('returns the result on first success', async () => { const fn = vi.fn().mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(); + const cb = new CircuitBreaker(); const result = await withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 }); expect(result).toBe('ok'); expect(fn).toHaveBeenCalledTimes(1); @@ -270,7 +198,7 @@ describe('withHfDownloadRetry', () => { it('retries on network errors and succeeds on second attempt', async () => { const fn = vi.fn().mockRejectedValueOnce(new Error('fetch failed')).mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(); + const cb = new CircuitBreaker(); const result = await withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, @@ -282,7 +210,7 @@ describe('withHfDownloadRetry', () => { it('throws the last network error after all attempts are exhausted', async () => { const fn = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:443')); - const cb = new HfDownloadCircuitBreaker(99 /* high threshold */); + const cb = new CircuitBreaker({ failureThreshold: 99 }); await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0 }), ).rejects.toThrow('ECONNREFUSED'); @@ -291,7 +219,7 @@ describe('withHfDownloadRetry', () => { it('does not retry non-network errors', async () => { const fn = vi.fn().mockRejectedValue(new Error('Failed to initialize CUDA backend')); - const cb = new HfDownloadCircuitBreaker(); + const cb = new CircuitBreaker(); await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0 }), ).rejects.toThrow('Failed to initialize CUDA backend'); @@ -300,7 +228,7 @@ describe('withHfDownloadRetry', () => { it('fails immediately when the circuit is already open', async () => { const fn = vi.fn().mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(1); + const cb = new CircuitBreaker({ failureThreshold: 1 }); cb.recordFailure(); // open the circuit await expect(withHfDownloadRetry(fn, { circuit: cb })).rejects.toThrow(CIRCUIT_OPEN_TAG); expect(fn).not.toHaveBeenCalled(); @@ -308,12 +236,12 @@ describe('withHfDownloadRetry', () => { it('opens the circuit after failureThreshold failures and throws a circuit-open error', async () => { const fn = vi.fn().mockRejectedValue(new Error('ENOTFOUND huggingface.co')); - const cb = new HfDownloadCircuitBreaker(2 /* threshold */, 60_000); + const cb = new CircuitBreaker({ failureThreshold: 2, cooldownMs: 60_000 }); // First call: 2 attempts, threshold=2 → circuit opens on 2nd failure await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 2, baseDelayMs: 0 }), ).rejects.toThrow(CIRCUIT_OPEN_TAG); - expect(cb.isOpen()).toBe(true); + expect(cb.getState()).toBe('open'); }); it('calls onRetry with correct arguments on each retry', async () => { @@ -322,7 +250,7 @@ describe('withHfDownloadRetry', () => { .mockRejectedValueOnce(new Error('fetch failed')) .mockRejectedValueOnce(new Error('fetch failed')) .mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); const onRetry = vi.fn(); await withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 3, baseDelayMs: 0, onRetry }); expect(onRetry).toHaveBeenCalledTimes(2); @@ -342,11 +270,11 @@ describe('withHfDownloadRetry', () => { it('resets the circuit on success', async () => { const fn = vi.fn().mockResolvedValue('value'); - const cb = new HfDownloadCircuitBreaker(5); + const cb = new CircuitBreaker({ failureThreshold: 5 }); cb.recordFailure(); cb.recordFailure(); // 2 failures, circuit still closed await withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 }); - expect(cb.state).toBe('closed'); + expect(cb.getState()).toBe('closed'); }); }); @@ -371,7 +299,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=1 gives exactly 1 attempt', async () => { process.env.HF_MAX_ATTEMPTS = '1'; const fn = vi.fn().mockRejectedValue(new Error('ECONNREFUSED 127.0.0.1:443')); - const cb = new HfDownloadCircuitBreaker(99_999 /* high threshold */); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'ECONNREFUSED', ); @@ -381,7 +309,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=2 gives exactly 2 attempts', async () => { process.env.HF_MAX_ATTEMPTS = '2'; const fn = vi.fn().mockRejectedValue(new Error('ENOTFOUND huggingface.co')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'ENOTFOUND', ); @@ -391,7 +319,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=abc falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = 'abc'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -401,7 +329,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=0 falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = '0'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -411,7 +339,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=-1 falls back to the built-in default', async () => { process.env.HF_MAX_ATTEMPTS = '-1'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -421,7 +349,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS is clamped to HF_MAX_ATTEMPTS_CAP', async () => { process.env.HF_MAX_ATTEMPTS = '9999'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999 /* very high threshold */); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -431,7 +359,7 @@ describe('withHfDownloadRetry env overrides', () => { it('HF_MAX_ATTEMPTS=2.9 is floored to 2', async () => { process.env.HF_MAX_ATTEMPTS = '2.9'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99_999); + const cb = new CircuitBreaker({ failureThreshold: 99_999 }); await expect(withHfDownloadRetry(fn, { circuit: cb, baseDelayMs: 0 })).rejects.toThrow( 'fetch failed', ); @@ -443,7 +371,7 @@ describe('withHfDownloadRetry env overrides', () => { try { process.env.HF_DOWNLOAD_TIMEOUT_MS = '50'; const neverResolves = () => new Promise(() => {}); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); const promise = withHfDownloadRetry(neverResolves, { circuit: cb, maxAttempts: 1 }); vi.advanceTimersByTime(100); await expect(promise).rejects.toThrow('ETIMEDOUT'); @@ -458,7 +386,7 @@ describe('withHfDownloadRetry env overrides', () => { // we just verify that the env var rejection causes options.timeoutMs to be // the default constant (not -1) by confirming the resolved value is used. const fn = vi.fn().mockResolvedValue('ok'); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); // Provide explicit timeoutMs to avoid the default 5-minute wait const result = await withHfDownloadRetry(fn, { circuit: cb, timeoutMs: 100 }); expect(result).toBe('ok'); @@ -470,7 +398,7 @@ describe('withHfDownloadRetry env overrides', () => { // Set an env value exceeding the 30-minute cap process.env.HF_DOWNLOAD_TIMEOUT_MS = String(HF_MAX_TIMEOUT_MS + 60_000); const neverResolves = () => new Promise(() => {}); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); const promise = withHfDownloadRetry(neverResolves, { circuit: cb, maxAttempts: 1 }); // Advance just past the 30-minute cap vi.advanceTimersByTime(HF_MAX_TIMEOUT_MS + 1); @@ -483,7 +411,7 @@ describe('withHfDownloadRetry env overrides', () => { it('explicit options override env vars', async () => { process.env.HF_MAX_ATTEMPTS = '5'; const fn = vi.fn().mockRejectedValue(new Error('fetch failed')); - const cb = new HfDownloadCircuitBreaker(99); + const cb = new CircuitBreaker({ failureThreshold: 99 }); // explicit maxAttempts: 2 must win over HF_MAX_ATTEMPTS=5 await expect( withHfDownloadRetry(fn, { circuit: cb, maxAttempts: 2, baseDelayMs: 0 }), diff --git a/gitnexus/test/unit/ignore-service.test.ts b/gitnexus/test/unit/ignore-service.test.ts index b4e5cdca1..1e1908137 100644 --- a/gitnexus/test/unit/ignore-service.test.ts +++ b/gitnexus/test/unit/ignore-service.test.ts @@ -28,6 +28,8 @@ describe('shouldIgnorePath', () => { it.each([ 'node_modules', 'vendor', + 'third_party', + '3rdparty', 'venv', '.venv', '__pycache__', diff --git a/gitnexus/test/unit/integrations/circuit-breaker.test.ts b/gitnexus/test/unit/integrations/circuit-breaker.test.ts new file mode 100644 index 000000000..4f08cd4f8 --- /dev/null +++ b/gitnexus/test/unit/integrations/circuit-breaker.test.ts @@ -0,0 +1,395 @@ +import { describe, it, expect, beforeEach } from 'vitest'; +import { CircuitBreaker, CircuitOpenError, getBreaker } from 'gitnexus-shared'; +import { __resetBreakerRegistry__ } from 'gitnexus-shared/test-helpers'; + +describe('CircuitBreaker', () => { + beforeEach(() => __resetBreakerRegistry__()); + + function makeClock(start = 1_700_000_000_000) { + let t = start; + return { + now: () => t, + advance: (ms: number) => { + t += ms; + }, + }; + } + + it('runs through check/recordSuccess in closed state', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 3, cooldownMs: 30_000, now: clock.now }); + expect(b.getState()).toBe('closed'); + b.check(); // does not throw + b.recordSuccess(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(0); + }); + + it('stays closed below the failure threshold', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 3, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + b.recordFailure(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(2); + }); + + it('opens after failureThreshold consecutive failures and check throws', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 3, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + b.recordFailure(); + b.recordFailure(); + expect(b.getState()).toBe('open'); + expect(() => b.check()).toThrow(CircuitOpenError); + }); + + it('CircuitOpenError.retryAfterMs decreases as time advances', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(30_000); + + clock.advance(10_000); + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(20_000); + }); + + it('transitions Open -> Half-Open after cooldown elapses (via check)', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + expect(b.getState()).toBe('open'); + clock.advance(31_000); + b.check(); // should not throw + // After check, internal state is half-open (next call probes). + expect(b.getConsecutiveFailures()).toBe(1); // unchanged until next outcome + }); + + it('half-open + recordSuccess -> closed and counter reset', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + b.check(); + b.recordSuccess(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(0); + }); + + it('half-open + recordFailure -> open with fresh openedAt', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + const firstOpen = clock.now(); + clock.advance(31_000); // cooldown expired + b.check(); // half-open + b.recordFailure(); + // Open with fresh timestamp — full cooldown again. + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + expect(caught?.retryAfterMs).toBe(30_000); + // Sanity: not the original openedAt (would be negative remaining). + expect(clock.now()).toBeGreaterThan(firstOpen); + }); + + it('recordSuccess from closed state with prior partial failures resets counter', () => { + const b = new CircuitBreaker({ failureThreshold: 5 }); + b.recordFailure(); + b.recordFailure(); + expect(b.getConsecutiveFailures()).toBe(2); + b.recordSuccess(); + expect(b.getConsecutiveFailures()).toBe(0); + expect(b.getState()).toBe('closed'); + }); + + describe('recordNeutral (U1)', () => { + it('is a no-op from closed state with zero prior failures', () => { + const b = new CircuitBreaker({ failureThreshold: 3 }); + b.recordNeutral(); + expect(b.getState()).toBe('closed'); + expect(b.getConsecutiveFailures()).toBe(0); + }); + + it('preserves partial-failure progress (does not reset counter)', () => { + const b = new CircuitBreaker({ failureThreshold: 3 }); + b.recordFailure(); + b.recordFailure(); + b.recordNeutral(); + expect(b.getConsecutiveFailures()).toBe(2); + expect(b.getState()).toBe('closed'); + // Real third failure still trips the breaker — neutrals didn't + // erase the running count toward the threshold. + b.recordFailure(); + expect(b.getState()).toBe('open'); + }); + + it('does not reset openedAt or transition out of open state', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + expect(b.getState()).toBe('open'); + b.recordNeutral(); + // Still open; cooldown clock unchanged. + expect(() => b.check()).toThrow(CircuitOpenError); + }); + + it('leaves half-open state alone (next true outcome decides)', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + b.check(); // half-open + b.recordNeutral(); + // Still half-open; a subsequent recordFailure flips to open. + b.recordFailure(); + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + }); + + it('integration: 2 failures + 5 neutrals + 1 failure → opens on third real failure', () => { + const b = new CircuitBreaker({ failureThreshold: 3 }); + b.recordFailure(); + b.recordFailure(); + for (let i = 0; i < 5; i++) b.recordNeutral(); + expect(b.getConsecutiveFailures()).toBe(2); + expect(b.getState()).toBe('closed'); + b.recordFailure(); + expect(b.getState()).toBe('open'); + }); + }); + + describe('half-open probe permit gate (U1)', () => { + it('admits exactly one caller after cooldown; subsequent check() throws halfOpenRetryAfterMs', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: 30_000, + halfOpenRetryAfterMs: 1_000, + now: clock.now, + }); + b.recordFailure(); + clock.advance(31_000); + + // Caller A: gets the probe permit. + b.check(); + expect(b.isProbeInFlight()).toBe(true); + + // Caller B: blocked. + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + expect(caught?.retryAfterMs).toBe(1_000); + + // Caller A's recordSuccess clears the breaker. + b.recordSuccess(); + expect(b.isProbeInFlight()).toBe(false); + expect(b.getState()).toBe('closed'); + + // Caller C: succeeds in closed state. + b.check(); + expect(b.getState()).toBe('closed'); + }); + + it('recordFailure on probe re-opens with fresh cooldown (NOT halfOpenRetryAfterMs)', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: 30_000, + halfOpenRetryAfterMs: 1_000, + now: clock.now, + }); + b.recordFailure(); + clock.advance(31_000); + + b.check(); // A: probe + expect(() => b.check()).toThrow(CircuitOpenError); // B: blocked + + b.recordFailure(); // A reports failure → reopens with fresh openedAt + + // C: should see the fresh cooldown remaining, not the probe-in-flight 1s default. + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught).toBeInstanceOf(CircuitOpenError); + // Fresh openedAt = current clock; cooldown is 30s; retryAfter ≈ 30s. + expect(caught?.retryAfterMs).toBe(30_000); + }); + + it('recordNeutral releases the probe permit but leaves state half-open', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + b.check(); // A: probe + expect(b.isProbeInFlight()).toBe(true); + + b.recordNeutral(); // A: neutral — permit released, state untouched + expect(b.isProbeInFlight()).toBe(false); + expect(b.getState()).toBe('half-open'); + + // B: succeeds (becomes the new probe), no longer blocked. + b.check(); + expect(b.isProbeInFlight()).toBe(true); + + // B's recordSuccess clears the breaker. + b.recordSuccess(); + expect(b.getState()).toBe('closed'); + }); + + it('three sequential probes via neutrals: A → A.neutral → B → B.neutral → C', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + const initialFailures = b.getConsecutiveFailures(); + clock.advance(31_000); + + for (let i = 0; i < 3; i++) { + b.check(); + b.recordNeutral(); + } + // Counter unchanged; state still half-open; permit released. + expect(b.getConsecutiveFailures()).toBe(initialFailures); + expect(b.getState()).toBe('half-open'); + expect(b.isProbeInFlight()).toBe(false); + }); + + it('5 same-tick sequential callers: exactly one passes, the other 4 throw', () => { + // `check()` is synchronous — these calls execute on a single + // microtask in declaration order. The first mutates probeInFlight + // = true; the next four observe the mutation and throw. This + // tests mutation ordering, not true concurrency (the actual + // interleaved-async-microtask scenario lives in U2). + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + const results: Array<'pass' | 'throw'> = []; + for (let i = 0; i < 5; i++) { + try { + b.check(); + results.push('pass'); + } catch { + results.push('throw'); + } + } + expect(results.filter((r) => r === 'pass').length).toBe(1); + expect(results.filter((r) => r === 'throw').length).toBe(4); + }); + + it('probe permit consumed; clock advances another full cooldown without record*; still throws', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + b.check(); // probe permit consumed + clock.advance(60_000); // another full cooldown elapses, no record* + + // Half-open semantics: wait for an outcome, not a timer. The + // permit-consumed state doesn't auto-resolve on time. + expect(() => b.check()).toThrow(CircuitOpenError); + }); + + it('halfOpenRetryAfterMs default is 1000 when not configured', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + b.check(); + + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(1_000); + }); + + it('halfOpenRetryAfterMs is configurable for long-running protected ops', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: 30_000, + halfOpenRetryAfterMs: 10_000, // LLM-streaming-friendly + now: clock.now, + }); + b.recordFailure(); + clock.advance(31_000); + b.check(); + + let caught: CircuitOpenError | null = null; + try { + b.check(); + } catch (err) { + caught = err as CircuitOpenError; + } + expect(caught?.retryAfterMs).toBe(10_000); + }); + + it('getState() is a pure read — does not consume the probe permit', () => { + const clock = makeClock(); + const b = new CircuitBreaker({ failureThreshold: 1, cooldownMs: 30_000, now: clock.now }); + b.recordFailure(); + clock.advance(31_000); + + // Test calls getState() to inspect — must not consume the permit. + expect(b.getState()).toBe('half-open'); + expect(b.isProbeInFlight()).toBe(false); + // First check() still gets the permit. + b.check(); + expect(b.isProbeInFlight()).toBe(true); + }); + }); + + describe('getBreaker registry', () => { + it('returns the same instance for the same key', () => { + const a = getBreaker('endpoint-a'); + const b = getBreaker('endpoint-a'); + expect(a).toBe(b); + }); + + it('returns different instances for different keys', () => { + const a = getBreaker('endpoint-a'); + const b = getBreaker('endpoint-b'); + expect(a).not.toBe(b); + }); + + it('__resetBreakerRegistry__ clears all instances', () => { + const a = getBreaker('endpoint-a'); + __resetBreakerRegistry__(); + const a2 = getBreaker('endpoint-a'); + expect(a2).not.toBe(a); + }); + }); +}); diff --git a/gitnexus/test/unit/integrations/resilient-fetch.test.ts b/gitnexus/test/unit/integrations/resilient-fetch.test.ts new file mode 100644 index 000000000..05299c0ba --- /dev/null +++ b/gitnexus/test/unit/integrations/resilient-fetch.test.ts @@ -0,0 +1,558 @@ +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { + CircuitBreaker, + CircuitOpenError, + parseRetryAfter, + resilientFetch, + ResilientFetchExhaustedError, + RETRY_AFTER_CAP_MS, +} from 'gitnexus-shared'; +import { __resetBreakerRegistry__, classifyOutcome } from 'gitnexus-shared/test-helpers'; + +describe('parseRetryAfter', () => { + it('parses delta-seconds form', () => { + expect(parseRetryAfter('30')).toBe(30_000); + expect(parseRetryAfter('0')).toBe(0); + }); + it('returns null on negative or non-numeric garbage', () => { + expect(parseRetryAfter(null)).toBeNull(); + expect(parseRetryAfter('')).toBeNull(); + expect(parseRetryAfter(' ')).toBeNull(); + expect(parseRetryAfter('not-a-number')).toBeNull(); + }); + it('parses HTTP-date form against an injected clock', () => { + const now = () => Date.parse('Wed, 21 Oct 2025 07:28:00 GMT'); + expect(parseRetryAfter('Wed, 21 Oct 2025 07:28:30 GMT', now)).toBe(30_000); + }); + it('returns 0 (not negative) on past HTTP-date', () => { + const now = () => Date.parse('Wed, 21 Oct 2025 08:00:00 GMT'); + expect(parseRetryAfter('Wed, 21 Oct 2025 07:28:00 GMT', now)).toBe(0); + }); +}); + +describe('classifyOutcome', () => { + const now = () => 1_700_000_000_000; + + it('classifies 2xx as success', () => { + const resp = new Response(null, { status: 204 }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('success'); + }); + it('classifies 5xx as retryable-status without afterMs', () => { + const resp = new Response(null, { status: 503 }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('retryable-status'); + if (out.kind === 'retryable-status') expect(out.afterMs).toBeUndefined(); + }); + it('classifies 429 with Retry-After (capped) as retryable-status', () => { + const resp = new Response(null, { status: 429, headers: { 'Retry-After': '99999' } }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('retryable-status'); + if (out.kind === 'retryable-status') expect(out.afterMs).toBe(RETRY_AFTER_CAP_MS); + }); + it('classifies 429 from a header-less fetch mock without throwing', () => { + // Tests sometimes stub `fetch` with a plain `{ ok, status }` object + // (e.g. http-embedder.test.ts). Real `Response` always carries + // `Headers`, but the helper must not crash when the stub does not. + // Falls through to exponential-backoff retry like a 429 with no + // Retry-After header. + const resp = { ok: false, status: 429 } as unknown as Response; + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('retryable-status'); + if (out.kind === 'retryable-status') expect(out.afterMs).toBeUndefined(); + }); + it('classifies 401/403/404/422 as terminal-client', () => { + for (const status of [401, 403, 404, 422, 400]) { + const resp = new Response(null, { status }); + const out = classifyOutcome({ kind: 'response', resp }, now); + expect(out.kind).toBe('terminal-client'); + } + }); + it('classifies TimeoutError as terminal-network', () => { + const err = new DOMException('aborted', 'TimeoutError'); + const out = classifyOutcome({ kind: 'error', err }, now); + expect(out.kind).toBe('terminal-network'); + }); + it('classifies generic network throw as retryable-network', () => { + const err = new TypeError('fetch failed'); + const out = classifyOutcome({ kind: 'error', err }, now); + expect(out.kind).toBe('retryable-network'); + }); +}); + +describe('resilientFetch', () => { + const URL_STR = 'https://example.test/api/dispatch'; + + beforeEach(() => __resetBreakerRegistry__()); + + function jsonResp(status: number, headers?: Record): Response { + return new Response(null, { status, headers }); + } + + function makeBreaker(opts: Partial[0]> = {}) { + let t = 1_700_000_000_000; + const breaker = new CircuitBreaker({ + failureThreshold: 3, + cooldownMs: 30_000, + key: 'test', + now: () => t, + ...opts, + }); + return { breaker, advance: (ms: number) => (t += ms) }; + } + + it('204 returns immediately, no retries, breaker stays closed', async () => { + const fetchImpl = vi.fn(async () => jsonResp(204)); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }); + expect(resp.status).toBe(204); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + expect(breaker.getState()).toBe('closed'); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('one 503 then 204 → retried once, returns 204, breaker stays closed', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(503) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, random: () => 0.5, baseDelayMs: 100, capDelayMs: 1000 }, + }); + expect(resp.status).toBe(204); + expect(fetchImpl).toHaveBeenCalledTimes(2); + expect(sleep).toHaveBeenCalledTimes(1); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('429 with Retry-After honored (capped at RETRY_AFTER_CAP_MS)', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(429, { 'Retry-After': '1' }) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }); + expect(sleep).toHaveBeenCalledWith(1000); // 1s + }); + + it('429 with absurd Retry-After is capped to RETRY_AFTER_CAP_MS', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(429, { 'Retry-After': '99999' }) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, capDelayMs: 999_999 }, // ensure cap comes from RETRY_AFTER_CAP_MS, not retry config + }); + expect(sleep).toHaveBeenCalledWith(RETRY_AFTER_CAP_MS); + }); + + it('429 without Retry-After falls back to exponential-backoff delay', async () => { + let n = 0; + const fetchImpl = vi.fn(async () => { + n += 1; + return n === 1 ? jsonResp(429) : jsonResp(204); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, baseDelayMs: 100, capDelayMs: 1000, random: () => 0.5 }, + }); + // attempt 0: full-jitter upper = min(1000, 100*1) = 100; floor(0.5*100) = 50 + expect(sleep).toHaveBeenCalledWith(50); + }); + + it('401 returned as Response, no retry, breaker not incremented', async () => { + const fetchImpl = vi.fn(async () => jsonResp(401)); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }); + expect(resp.status).toBe(401); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('422 returned as Response, no retry', async () => { + const fetchImpl = vi.fn(async () => jsonResp(422)); + const { breaker } = makeBreaker(); + const resp = await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {} }, + }); + expect(resp.status).toBe(422); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + + it('TimeoutError rethrown immediately, no retry, breaker not incremented', async () => { + const fetchImpl = vi.fn(async () => { + throw new DOMException('aborted', 'TimeoutError'); + }); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep }, + }), + ).rejects.toThrow(DOMException); + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + expect(breaker.getConsecutiveFailures()).toBe(0); + }); + + it('three consecutive 503 throws ResilientFetchExhaustedError; breaker increments by 1', async () => { + const fetchImpl = vi.fn(async () => jsonResp(503)); + const sleep = vi.fn(async () => {}); + const { breaker } = makeBreaker(); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(ResilientFetchExhaustedError); + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect(breaker.getConsecutiveFailures()).toBe(1); + }); + + it('after three exhausted 503 batches, breaker opens and fails fast', async () => { + const fetchImpl = vi.fn(async () => jsonResp(503)); + const { breaker } = makeBreaker({ failureThreshold: 3, cooldownMs: 60_000 }); + for (let i = 0; i < 3; i++) { + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(ResilientFetchExhaustedError); + } + expect(breaker.getState()).toBe('open'); + // 4th call: breaker open, no fetch invoked. + const fetchCallsBefore = fetchImpl.mock.calls.length; + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(CircuitOpenError); + expect(fetchImpl.mock.calls.length).toBe(fetchCallsBefore); + }); + + it('retryable-network error retries, breaker counts only on exhaustion', async () => { + const fetchImpl = vi.fn(async () => { + throw new TypeError('fetch failed'); + }); + const { breaker } = makeBreaker(); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 3 }, + }), + ).rejects.toBeInstanceOf(TypeError); + expect(fetchImpl).toHaveBeenCalledTimes(3); + expect(breaker.getConsecutiveFailures()).toBe(1); + }); + + describe('U2: terminal outcomes route through recordNeutral', () => { + it('401 does not erase prior partial-failure progress on the breaker', async () => { + const { breaker } = makeBreaker(); + // Pre-seed the breaker with 2 failures (still closed; threshold 3). + breaker.recordFailure(); + breaker.recordFailure(); + expect(breaker.getConsecutiveFailures()).toBe(2); + + const fetchImpl = vi.fn(async () => jsonResp(401)); + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {} }, + }); + + // Counter MUST stay at 2 — under the old behaviour recordSuccess + // would have reset to 0 and the next 5xx batch would have started + // from scratch instead of tipping over the threshold. + expect(breaker.getConsecutiveFailures()).toBe(2); + expect(breaker.getState()).toBe('closed'); + }); + + it('TimeoutError does not erase prior partial-failure progress', async () => { + const { breaker } = makeBreaker(); + breaker.recordFailure(); + breaker.recordFailure(); + + const fetchImpl = vi.fn(async () => { + throw new DOMException('aborted by timeout', 'TimeoutError'); + }); + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {} }, + }), + ).rejects.toBeInstanceOf(DOMException); + + expect(breaker.getConsecutiveFailures()).toBe(2); + }); + + it('external AbortError is terminal: no retry, breaker untouched', async () => { + const { breaker } = makeBreaker(); + breaker.recordFailure(); + + const fetchImpl = vi.fn(async () => { + throw new DOMException('aborted by caller', 'AbortError'); + }); + const sleep = vi.fn(async () => {}); + + await expect( + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep, maxAttempts: 3 }, + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + // Counter unchanged — neither incremented (no failure) nor reset + // (no synthetic success). + expect(breaker.getConsecutiveFailures()).toBe(1); + }); + + it('interleaved 5xx + 401 + 5xx + 401 + 5xx opens breaker on third real failure', async () => { + const { breaker } = makeBreaker({ failureThreshold: 3 }); + const sequence = [503, 401, 503, 401, 503]; + let i = 0; + const fetchImpl = vi.fn(async () => jsonResp(sequence[i++])); + + // Each call uses maxAttempts:1 so each surfaces a single response + // (5xx → ResilientFetchExhaustedError; 4xx → returned Response). + const driveOne = () => + resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + + await expect(driveOne()).rejects.toBeInstanceOf(ResilientFetchExhaustedError); // 5xx fail #1 + await driveOne(); // 401 neutral + await expect(driveOne()).rejects.toBeInstanceOf(ResilientFetchExhaustedError); // 5xx fail #2 + await driveOne(); // 401 neutral + await expect(driveOne()).rejects.toBeInstanceOf(ResilientFetchExhaustedError); // 5xx fail #3 → opens + + expect(breaker.getState()).toBe('open'); + expect(fetchImpl).toHaveBeenCalledTimes(5); + }); + }); + + describe('half-open single-probe gating (U2)', () => { + /** Test helper: a fetch mock whose Response is controlled by the test. */ + function deferredFetch(): { + promise: Promise; + resolve: (resp: Response) => void; + reject: (err: unknown) => void; + } { + let resolve!: (resp: Response) => void; + let reject!: (err: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; + } + + /** Builds a clock-injected breaker pre-opened with cooldown elapsed. */ + function preOpenedBreaker(opts: { cooldownMs: number; halfOpenRetryAfterMs?: number }): { + breaker: CircuitBreaker; + advance: (ms: number) => void; + } { + let t = 1_700_000_000_000; + const breaker = new CircuitBreaker({ + failureThreshold: 1, + cooldownMs: opts.cooldownMs, + halfOpenRetryAfterMs: opts.halfOpenRetryAfterMs ?? 1_000, + key: 'test', + now: () => t, + }); + breaker.recordFailure(); + t += opts.cooldownMs + 1; // cooldown elapsed + return { breaker, advance: (ms) => (t += ms) }; + } + + it('happy: 3 concurrent calls — exactly 1 hits fetch, others throw CircuitOpenError', async () => { + const { breaker } = preOpenedBreaker({ cooldownMs: 10 }); + const deferred = deferredFetch(); + const fetchImpl = vi.fn(() => deferred.promise); + + // Synchronous portion of each `resilientFetch` runs eagerly up to + // the first await, so by the time r2/r3 are constructed the probe + // permit is already consumed by r1 and they reject synchronously. + const r1 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r2 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r3 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + + // Resolve the probe with 200; r1 should now settle. + deferred.resolve(new Response(null, { status: 200 })); + + const results = await Promise.allSettled([r1, r2, r3]); + + expect(results[0].status).toBe('fulfilled'); + if (results[0].status === 'fulfilled') { + expect(results[0].value.status).toBe(200); + } + expect(results[1].status).toBe('rejected'); + if (results[1].status === 'rejected') { + expect(results[1].reason).toBeInstanceOf(CircuitOpenError); + } + expect(results[2].status).toBe('rejected'); + if (results[2].status === 'rejected') { + expect(results[2].reason).toBeInstanceOf(CircuitOpenError); + } + + // Only ONE underlying fetch was invoked. + expect(fetchImpl).toHaveBeenCalledTimes(1); + // Breaker closed after the probe's success. + expect(breaker.getState()).toBe('closed'); + }); + + it('error: probe gets 503 — exhausted error; subsequent caller sees fresh full cooldown', async () => { + const { breaker } = preOpenedBreaker({ cooldownMs: 10_000, halfOpenRetryAfterMs: 1_000 }); + const deferred = deferredFetch(); + const fetchImpl = vi.fn(() => deferred.promise); + + const r1 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r2 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r3 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + + // Probe fails with 503 → exhausted (maxAttempts: 1) → recordFailure → reopen. + deferred.resolve(new Response(null, { status: 503 })); + + const results = await Promise.allSettled([r1, r2, r3]); + + expect(results[0].status).toBe('rejected'); + if (results[0].status === 'rejected') { + expect(results[0].reason).toBeInstanceOf(ResilientFetchExhaustedError); + } + expect(results[1].status).toBe('rejected'); + if (results[1].status === 'rejected') { + expect(results[1].reason).toBeInstanceOf(CircuitOpenError); + // Blocked-while-half-open used the halfOpenRetryAfterMs default. + expect((results[1].reason as CircuitOpenError).retryAfterMs).toBe(1_000); + } + + // Breaker has re-opened with a fresh openedAt. + expect(breaker.getState()).toBe('open'); + + // r4: should see the fresh full cooldown, NOT the probe-in-flight 1000ms. + let r4Caught: CircuitOpenError | null = null; + try { + await resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + } catch (err) { + r4Caught = err as CircuitOpenError; + } + expect(r4Caught).toBeInstanceOf(CircuitOpenError); + expect(r4Caught?.retryAfterMs).toBe(10_000); + }); + + it('cancellation: probe AbortError releases permit; next caller becomes new probe', async () => { + const { breaker } = preOpenedBreaker({ cooldownMs: 10_000 }); + const deferred1 = deferredFetch(); + const deferred2 = deferredFetch(); + let callIdx = 0; + const fetchImpl = vi.fn(() => (callIdx++ === 0 ? deferred1.promise : deferred2.promise)); + + // r1 admitted as the probe; r2 blocked while r1 still in flight. + const r1 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + const r2 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + await expect(r2).rejects.toBeInstanceOf(CircuitOpenError); + + // Cancel the probe — `AbortError` routes through terminal-network → + // `recordNeutral` → permit released, state stays half-open. + deferred1.reject(new DOMException('aborted by caller', 'AbortError')); + await expect(r1).rejects.toMatchObject({ name: 'AbortError' }); + + expect(breaker.isProbeInFlight()).toBe(false); + expect(breaker.getState()).toBe('half-open'); + + // r3: now succeeds and becomes the new probe. + const r3 = resilientFetch(URL_STR, undefined, { + fetchImpl: fetchImpl as unknown as typeof fetch, + breaker, + retry: { sleep: async () => {}, maxAttempts: 1 }, + }); + deferred2.resolve(new Response(null, { status: 200 })); + const r3Resp = await r3; + expect(r3Resp.status).toBe(200); + expect(breaker.getState()).toBe('closed'); + // Two fetches total: the cancelled probe + the recovery probe. + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/gitnexus/test/unit/integrations/retry.test.ts b/gitnexus/test/unit/integrations/retry.test.ts new file mode 100644 index 000000000..2dd4f4cf3 --- /dev/null +++ b/gitnexus/test/unit/integrations/retry.test.ts @@ -0,0 +1,128 @@ +import { describe, it, expect, vi } from 'vitest'; +import { computeBackoffMs, withRetry, type RetryOptions } from 'gitnexus-shared'; + +describe('computeBackoffMs', () => { + it('returns afterMs (capped) when caller supplies it', () => { + expect(computeBackoffMs(0, 500, 5000, 1500, () => 0.5)).toBe(1500); + expect(computeBackoffMs(0, 500, 5000, 99_999, () => 0.5)).toBe(5000); + expect(computeBackoffMs(0, 500, 5000, 0, () => 0.5)).toBe(0); + expect(computeBackoffMs(0, 500, 5000, -1, () => 0.5)).toBe(0); + }); + + it('full-jitter delay falls within [0, min(cap, base * 2^attempt)]', () => { + // attempt 0: upper = min(5000, 500 * 1) = 500 + expect(computeBackoffMs(0, 500, 5000, undefined, () => 0)).toBe(0); + expect(computeBackoffMs(0, 500, 5000, undefined, () => 0.999)).toBeLessThan(500); + // attempt 1: upper = min(5000, 500 * 2) = 1000 + expect(computeBackoffMs(1, 500, 5000, undefined, () => 0.5)).toBe(500); + // attempt 4: 500 * 16 = 8000, capped at 5000 + expect(computeBackoffMs(4, 500, 5000, undefined, () => 0.5)).toBe(2500); + expect(computeBackoffMs(4, 500, 5000, undefined, () => 0.999)).toBeLessThan(5000); + }); +}); + +describe('withRetry', () => { + function makeOpts(overrides: Partial = {}): RetryOptions { + return { + maxAttempts: 3, + baseDelayMs: 10, + capDelayMs: 100, + isRetryable: () => ({ retry: true }), + sleep: vi.fn(async () => {}), + random: () => 0.5, + ...overrides, + }; + } + + it('returns immediately when fn succeeds first try', async () => { + const sleep = vi.fn(async () => {}); + const fn = vi.fn(async () => 'ok'); + const result = await withRetry(fn, makeOpts({ sleep })); + expect(result).toBe('ok'); + expect(fn).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('retries when isRetryable returns retry:true and second call succeeds', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls === 1) throw new Error('boom'); + return 'ok'; + }; + const result = await withRetry(fn, makeOpts({ sleep })); + expect(result).toBe('ok'); + expect(calls).toBe(2); + expect(sleep).toHaveBeenCalledTimes(1); + }); + + it('honors afterMs returned by isRetryable', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls === 1) throw new Error('throttle'); + return 'ok'; + }; + await withRetry( + fn, + makeOpts({ + sleep, + isRetryable: () => ({ retry: true, afterMs: 1500 }), + capDelayMs: 5000, + }), + ); + expect(sleep).toHaveBeenCalledWith(1500); + }); + + it('caps afterMs at capDelayMs', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + if (calls === 1) throw new Error('throttle'); + return 'ok'; + }; + await withRetry( + fn, + makeOpts({ + sleep, + isRetryable: () => ({ retry: true, afterMs: 10_000 }), + capDelayMs: 3000, + }), + ); + expect(sleep).toHaveBeenCalledWith(3000); + }); + + it('rethrows immediately when isRetryable returns retry:false', async () => { + const sleep = vi.fn(async () => {}); + const fn = vi.fn(async () => { + throw new Error('terminal'); + }); + await expect( + withRetry(fn, makeOpts({ sleep, isRetryable: () => ({ retry: false }) })), + ).rejects.toThrow('terminal'); + expect(fn).toHaveBeenCalledTimes(1); + expect(sleep).not.toHaveBeenCalled(); + }); + + it('throws the last error when maxAttempts exhausted', async () => { + const sleep = vi.fn(async () => {}); + let calls = 0; + const fn = async () => { + calls += 1; + throw new Error(`boom-${calls}`); + }; + await expect(withRetry(fn, makeOpts({ sleep, maxAttempts: 3 }))).rejects.toThrow('boom-3'); + expect(calls).toBe(3); + // 3 attempts → 2 sleeps between them; final attempt does not sleep. + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it('rejects maxAttempts < 1', async () => { + await expect(withRetry(async () => 'ok', makeOpts({ maxAttempts: 0 }))).rejects.toThrow( + /maxAttempts must be >= 1/, + ); + }); +}); diff --git a/gitnexus/test/unit/publish.test.ts b/gitnexus/test/unit/publish.test.ts new file mode 100644 index 000000000..2ba767594 --- /dev/null +++ b/gitnexus/test/unit/publish.test.ts @@ -0,0 +1,316 @@ +import { afterEach, beforeEach, describe, expect, it, test, vi } from 'vitest'; +import fs from 'fs/promises'; +import os from 'os'; +import path from 'path'; +import { performance } from 'node:perf_hooks'; +import { + buildUqDispatchPayload, + isValidOwnerRepo, + parseOwnerRepoFromRemote, + stripGitSuffix, + UNDERSTAND_QUICKLY_TOKEN_ENV, +} from 'gitnexus-shared'; + +describe('understand-quickly helpers (gitnexus-shared)', () => { + describe('isValidOwnerRepo', () => { + it.each([ + ['looptech-ai/understand-quickly', true], + ['abhigyanpatwari/GitNexus', true], + // LOW 8: GitHub user/org slugs are alnum/hyphen only — no underscore. + ['Some_Org/Some.Repo-2', false], + ['', false], + ['just-a-name', false], + ['/Users/me/code/repo', false], + ['org/with spaces', false], + ['org//double', false], + // LOW 8 additions: + ['some_org/repo', false], // underscore in owner — invalid + ['-org/repo', false], // leading hyphen — invalid + ['org-/repo', false], // trailing hyphen — GitHub rejects at account creation; we mirror that here + ['org/repo_with_underscore', true], + ['org/.dotfile', true], // repos may start with dot + ])('returns %s for %j', (id, expected) => { + expect(isValidOwnerRepo(id as string)).toBe(expected); + }); + }); + + describe('stripGitSuffix (BLOCKER 1 — ReDoS-safe)', () => { + it.each([ + ['https://github.com/o/r.git', 'https://github.com/o/r'], + ['https://github.com/o/r.git/', 'https://github.com/o/r'], + ['https://github.com/o/r/', 'https://github.com/o/r'], + ['https://github.com/o/r', 'https://github.com/o/r'], + ['https://github.com/o/r.GIT', 'https://github.com/o/r'], + ['https://github.com/o/r//', 'https://github.com/o/r'], + ['', ''], + ['/', ''], + ])('strips %j -> %j', (input, expected) => { + expect(stripGitSuffix(input)).toBe(expected); + }); + + test('linear time on adversarial trailing slashes (regression for ReDoS)', () => { + const adversarial = 'https://github.com/o/r' + '/'.repeat(10_000); + const start = performance.now(); + const result = stripGitSuffix(adversarial); + const elapsed = performance.now() - start; + expect(result).toBe('https://github.com/o/r'); + expect(elapsed).toBeLessThan(50); // generous; should be sub-millisecond + }); + + test('parseOwnerRepoFromRemote terminates quickly on adversarial input', () => { + const adversarial = 'https://github.com/o/r.git' + '/'.repeat(10_000); + const start = performance.now(); + const result = parseOwnerRepoFromRemote(adversarial); + const elapsed = performance.now() - start; + expect(result).toBe('o/r'); + expect(elapsed).toBeLessThan(50); + }); + }); + + describe('parseOwnerRepoFromRemote', () => { + it.each([ + ['git@github.com:looptech-ai/understand-quickly.git', 'looptech-ai/understand-quickly'], + ['https://github.com/looptech-ai/understand-quickly', 'looptech-ai/understand-quickly'], + ['https://github.com/looptech-ai/understand-quickly.git', 'looptech-ai/understand-quickly'], + ['ssh://git@github.com/abhigyanpatwari/GitNexus.git', 'abhigyanpatwari/GitNexus'], + ])('parses %s -> %s', (url, expected) => { + expect(parseOwnerRepoFromRemote(url)).toBe(expected); + }); + + // LOW 9: non-GitHub remotes must be rejected — a wrong id is worse + // than no id, since the user can always pass --id explicitly. + it.each([ + ['https://gitlab.example.com/group/sub/project.git'], + ['git@gitlab.example.com:group/sub/project.git'], + ['https://bitbucket.org/team/repo.git'], + ])('returns null for non-GitHub host %j', (input) => { + expect(parseOwnerRepoFromRemote(input)).toBeNull(); + }); + + it.each([null, undefined, '', ' ', 'not-a-url', 'https://github.com/'])( + 'returns null for %j', + (input) => { + expect(parseOwnerRepoFromRemote(input as string | null | undefined)).toBeNull(); + }, + ); + }); + + describe('buildUqDispatchPayload', () => { + it('wraps the id in the registry-expected event shape', () => { + expect(buildUqDispatchPayload('looptech-ai/understand-quickly')).toEqual({ + event_type: 'sync-entry', + client_payload: { id: 'looptech-ai/understand-quickly' }, + }); + }); + + it('throws on a malformed id rather than building an invalid payload', () => { + expect(() => buildUqDispatchPayload('just-a-name')).toThrow(/owner\/repo/); + expect(() => buildUqDispatchPayload('/Users/me/repo')).toThrow(/owner\/repo/); + }); + }); +}); + +describe('publishCommand (no-token no-op)', () => { + let tempDir: string; + let originalToken: string | undefined; + let exitCodeBefore: number | undefined; + + beforeEach(async () => { + vi.resetModules(); + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publish-test-')); + // Simulate an existing index so hasIndex() returns true. + await fs.mkdir(path.join(tempDir, '.gitnexus'), { recursive: true }); + await fs.writeFile( + path.join(tempDir, '.gitnexus', 'meta.json'), + JSON.stringify({ repoPath: tempDir, lastCommit: '', indexedAt: '' }), + 'utf-8', + ); + originalToken = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + delete process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + exitCodeBefore = process.exitCode; + process.exitCode = 0; + }); + + afterEach(async () => { + if (originalToken !== undefined) { + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = originalToken; + } else { + delete process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + } + process.exitCode = exitCodeBefore; + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it('exits 0 without firing a network call when the token is unset', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(() => { + throw new Error('publishCommand should NOT call fetch when the token is missing'); + }); + + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { id: 'looptech-ai/understand-quickly', skipGit: true }); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(process.exitCode ?? 0).toBe(0); + fetchSpy.mockRestore(); + }); + + it('exits 0 with no token even when no index/repo exists (BLOCKER 2)', async () => { + // Per the README, CLI --help, and PR body: without a token, the + // command must be a no-op even if the repo lacks `.gitnexus/`. + const noIndexDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publish-noidx-')); + try { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockImplementation(() => { + throw new Error('publishCommand should NOT call fetch when the token is missing'); + }); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(noIndexDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(process.exitCode ?? 0).toBe(0); + fetchSpy.mockRestore(); + } finally { + await fs.rm(noIndexDir, { recursive: true, force: true }); + } + }); +}); + +describe('publishCommand response branches (MEDIUM 5)', () => { + let tempDir: string; + let originalToken: string | undefined; + let exitCodeBefore: number | undefined; + let fetchSpy: ReturnType; + + beforeEach(async () => { + vi.resetModules(); + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'gn-publish-resp-')); + await fs.mkdir(path.join(tempDir, '.gitnexus'), { recursive: true }); + await fs.writeFile( + path.join(tempDir, '.gitnexus', 'meta.json'), + JSON.stringify({ repoPath: tempDir, lastCommit: '', indexedAt: '' }), + 'utf-8', + ); + originalToken = process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = 'pat_test'; + exitCodeBefore = process.exitCode; + process.exitCode = 0; + fetchSpy = vi.spyOn(globalThis, 'fetch'); + }); + + afterEach(async () => { + if (originalToken !== undefined) { + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = originalToken; + } else { + delete process.env[UNDERSTAND_QUICKLY_TOKEN_ENV]; + } + process.exitCode = exitCodeBefore; + vi.restoreAllMocks(); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + function mockResponse(status: number, body = '') { + fetchSpy.mockResolvedValueOnce({ + status, + ok: status >= 200 && status < 300, + text: async () => body, + body: { cancel: async () => {} }, + headers: new Headers(), + } as unknown as Response); + } + + it('204 → exit 0 with success message', async () => { + mockResponse(204); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(process.exitCode ?? 0).toBe(0); + }); + + it('401 → exit 1 with PAT-invalid hint', async () => { + mockResponse(401, '{"message":"Bad credentials"}'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('403 → exit 1 with scope-missing hint', async () => { + mockResponse(403, '{"message":"Resource not accessible"}'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('404 → exit 1 with repo-access hint', async () => { + mockResponse(404, '{"message":"Not Found"}'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('5xx → exit 1 with raw body', async () => { + mockResponse(503, 'gateway timeout'); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('network throw → exit 1', async () => { + fetchSpy.mockRejectedValueOnce(new Error('ECONNRESET')); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + }); + + it('TimeoutError (HIGH 4 — fetch timeout) → exit 1 with timed-out message', async () => { + // `AbortSignal.timeout()` throws a real `DOMException` with + // `name === 'TimeoutError'`. Faking it as `Error{name:'AbortError'}` + // (the previous shape of this test) hid a mismatch in publish.ts — + // the catch branch only matched 'AbortError' and the user-facing + // "timed out" message never fired in production. + const abort = new DOMException('The operation was aborted due to timeout', 'TimeoutError'); + fetchSpy.mockRejectedValueOnce(abort); + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + expect(process.exitCode).toBe(1); + const written = errSpy.mock.calls.map((c) => String(c[0])).join(''); + expect(written).toMatch(/timed out/i); + errSpy.mockRestore(); + }); + + it('token never appears in any logged output', async () => { + process.env[UNDERSTAND_QUICKLY_TOKEN_ENV] = 'pat_secret_value'; + mockResponse(401, ''); + const errSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + const { publishCommand } = await import('../../src/cli/publish.js'); + await publishCommand(tempDir, { + id: 'looptech-ai/understand-quickly', + skipGit: true, + }); + const written = errSpy.mock.calls.map((c) => String(c[0])).join(''); + expect(written).not.toContain('pat_secret_value'); + errSpy.mockRestore(); + }); +}); diff --git a/gitnexus/test/unit/schema.test.ts b/gitnexus/test/unit/schema.test.ts index cab80984a..17db9bdf9 100644 --- a/gitnexus/test/unit/schema.test.ts +++ b/gitnexus/test/unit/schema.test.ts @@ -13,6 +13,7 @@ import { CLASS_SCHEMA, INTERFACE_SCHEMA, METHOD_SCHEMA, + PROPERTY_SCHEMA, CODE_ELEMENT_SCHEMA, COMMUNITY_SCHEMA, PROCESS_SCHEMA, @@ -117,6 +118,11 @@ describe('LadybugDB Schema', () => { expect(FUNCTION_SCHEMA).toContain('isExported BOOLEAN'); }); + it('Property schema preserves declaredType', () => { + expect(SCHEMA_QUERIES).toContain(PROPERTY_SCHEMA); + expect(PROPERTY_SCHEMA).toContain('declaredType STRING'); + }); + it('Community schema has heuristicLabel and cohesion', () => { expect(COMMUNITY_SCHEMA).toContain('heuristicLabel STRING'); expect(COMMUNITY_SCHEMA).toContain('cohesion DOUBLE'); diff --git a/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts b/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts index c8111d203..9fa9bf181 100644 --- a/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts +++ b/gitnexus/test/unit/scope-resolution/csharp/csharp-captures.test.ts @@ -422,4 +422,47 @@ describe('emitCsharpScopeCaptures — references', () => { expect(m!['@reference.receiver'].text).toBe('obj'); expect(m!['@reference.name'].text).toBe('Name'); }); + + it('captures member reads `obj.Name`', () => { + const m = findMatch('class A { void M(User obj) { var name = obj.Name; } }', (t) => + t.includes('@reference.read.member'), + ); + expect(m).toBeDefined(); + expect(m!['@reference.receiver'].text).toBe('obj'); + expect(m!['@reference.name'].text).toBe('Name'); + }); + + it('does not capture member calls as member reads', () => { + const matches = emitCsharpScopeCaptures( + 'class A { void M(User obj) { obj.Save(); } }', + 'test.cs', + ); + expect(matches.some((m) => '@reference.call.member' in m)).toBe(true); + expect(matches.some((m) => '@reference.read.member' in m)).toBe(false); + }); + + it('captures generic type arguments as type references', () => { + const matches = emitCsharpScopeCaptures( + 'class A : IEntityTypeConfiguration { public Task> Load(List users) => null!; }', + 'test.cs', + ); + const names = matches + .filter((m) => '@reference.type' in m) + .map((m) => m['@reference.name'].text); + + expect(names).toContain('USER_INFO'); + expect(names).not.toContain('string'); + }); + + it('captures call-site generic type arguments as type references', () => { + const matches = emitCsharpScopeCaptures( + 'class A { void M(IRepo repo) { repo.Get(); } }', + 'test.cs', + ); + const names = matches + .filter((m) => '@reference.type' in m) + .map((m) => m['@reference.name'].text); + + expect(names).toContain('USER_INFO'); + }); }); diff --git a/gitnexus/test/unit/skip-git-cli.test.ts b/gitnexus/test/unit/skip-git-cli.test.ts index e82bb7b13..069a9686f 100644 --- a/gitnexus/test/unit/skip-git-cli.test.ts +++ b/gitnexus/test/unit/skip-git-cli.test.ts @@ -5,9 +5,11 @@ import os from 'os'; import fs from 'fs'; describe('--skip-git CLI flag', () => { + const cliPath = path.resolve(__dirname, '../../dist/cli/index.js'); + it('Commander maps --skip-git to options.skipGit (not --no-git inversion)', () => { // Verify the CLI defines --skip-git and --skip-agents-md in analyze help. - const helpOutput = execSync('node dist/cli/index.js analyze --help', { + const helpOutput = execSync(`node "${cliPath}" analyze --help`, { cwd: path.resolve(__dirname, '../..'), encoding: 'utf8', timeout: 10000, @@ -37,8 +39,59 @@ describe('--skip-git CLI flag', () => { } }); + it('still respects .gitnexusignore when run with --skip-git', () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-skip-git-ignore-')); + const gitnexusHome = fs.mkdtempSync(path.join(os.tmpdir(), 'gn-skip-git-ignore-home-')); + fs.mkdirSync(path.join(tmpDir, 'src'), { recursive: true }); + fs.mkdirSync(path.join(tmpDir, 'customskip'), { recursive: true }); + fs.writeFileSync(path.join(tmpDir, '.gitnexusignore'), 'customskip/\n'); + fs.writeFileSync(path.join(tmpDir, 'src', 'keep.ts'), 'export function keep() { return 1; }\n'); + fs.writeFileSync( + path.join(tmpDir, 'customskip', 'leaked.ts'), + 'export function leaked() { return 42; }\n', + ); + + const env = { + ...process.env, + HOME: gitnexusHome, + GITNEXUS_HOME: gitnexusHome, + GITNEXUS_LBUG_EXTENSION_INSTALL: 'never', + }; + + try { + execSync(`node "${cliPath}" analyze "${tmpDir}" --skip-git --skip-agents-md`, { + encoding: 'utf8', + timeout: 60000, + env, + }); + + const keepContext = execSync( + `node "${cliPath}" context keep --repo "${path.basename(tmpDir)}"`, + { + encoding: 'utf8', + timeout: 60000, + env, + }, + ); + expect(keepContext).toContain('"status": "found"'); + expect(keepContext).toContain('"filePath": "src/keep.ts"'); + + const leakedContext = execSync( + `node "${cliPath}" context leaked --repo "${path.basename(tmpDir)}"`, + { + encoding: 'utf8', + timeout: 60000, + env, + }, + ); + expect(leakedContext).toContain(`"error": "Symbol 'leaked' not found"`); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + fs.rmSync(gitnexusHome, { recursive: true, force: true }); + } + }); + describe('--skip-git does not walk up to parent git repo (#1232)', () => { - const cliPath = path.resolve(__dirname, '../../dist/cli/index.js'); let parentDir: string; let gitnexusHome: string; diff --git a/gitnexus/test/unit/wiki-llm-client.test.ts b/gitnexus/test/unit/wiki-llm-client.test.ts index c9d429904..52b633566 100644 --- a/gitnexus/test/unit/wiki-llm-client.test.ts +++ b/gitnexus/test/unit/wiki-llm-client.test.ts @@ -5,6 +5,7 @@ import { isAzureProvider, isReasoningModel, buildRequestUrl, + validateLLMBaseUrl, } from '../../src/core/wiki/llm-client.js'; describe('isAzureProvider', () => { @@ -330,3 +331,88 @@ describe('readSSEStream — content_filter handling', () => { ).rejects.toThrow('content filter'); }); }); + +describe('validateLLMBaseUrl', () => { + it('allows https:// for any public host', () => { + expect(() => validateLLMBaseUrl('https://api.openai.com/v1')).not.toThrow(); + expect(() => validateLLMBaseUrl('https://openrouter.ai/api/v1')).not.toThrow(); + expect(() => validateLLMBaseUrl('https://myres.openai.azure.com/openai/v1')).not.toThrow(); + }); + + it('allows http:// for localhost', () => { + expect(() => validateLLMBaseUrl('http://localhost:11434/v1')).not.toThrow(); + expect(() => validateLLMBaseUrl('http://127.0.0.1:11434/v1')).not.toThrow(); + // IPv6 loopback — Node's URL parser preserves brackets in hostname: "[::1]" + expect(() => validateLLMBaseUrl('http://[::1]:11434/v1')).not.toThrow(); + }); + + it('allows http:// for LOCALHOST (uppercase) — lowercased before comparison', () => { + expect(() => validateLLMBaseUrl('http://LOCALHOST:11434/v1')).not.toThrow(); + }); + + it('rejects http:// for non-loopback hosts', () => { + expect(() => validateLLMBaseUrl('http://evil.example.com/v1')).toThrow('Insecure http://'); + expect(() => validateLLMBaseUrl('http://192.168.1.1/v1')).toThrow('Insecure http://'); + // Private IP ranges + expect(() => validateLLMBaseUrl('http://10.0.0.1/v1')).toThrow('Insecure http://'); + // AWS/GCP IMDS — should be blocked + expect(() => validateLLMBaseUrl('http://169.254.169.254/latest/meta-data')).toThrow( + 'Insecure http://', + ); + }); + + it('rejects http:// hostname-spoofing attempts', () => { + // Full-hostname comparison prevents prefix/suffix attacks + expect(() => validateLLMBaseUrl('http://localhost.evil.com/v1')).toThrow('Insecure http://'); + expect(() => validateLLMBaseUrl('http://127.0.0.1.evil.com/v1')).toThrow('Insecure http://'); + // Trailing dot — hostname 'localhost.' ≠ 'localhost' + expect(() => validateLLMBaseUrl('http://localhost./v1')).toThrow('Insecure http://'); + }); + + it('rejects http:// non-loopback IPv6 addresses', () => { + // Link-local IPv6 + expect(() => validateLLMBaseUrl('http://[fe80::1]/v1')).toThrow('Insecure http://'); + // IPv4-mapped IPv6 loopback — bracket-stripped to '::ffff:127.0.0.1' ≠ '::1' + expect(() => validateLLMBaseUrl('http://[::ffff:127.0.0.1]/v1')).toThrow('Insecure http://'); + }); + + it('rejects non-http schemes', () => { + expect(() => validateLLMBaseUrl('file:///etc/passwd')).toThrow('must use http:// or https://'); + expect(() => validateLLMBaseUrl('javascript:alert(1)')).toThrow('must use http:// or https://'); + expect(() => validateLLMBaseUrl('data:text/plain,evil')).toThrow( + 'must use http:// or https://', + ); + expect(() => validateLLMBaseUrl('ftp://example.com')).toThrow('must use http:// or https://'); + }); + + it('rejects malformed URLs', () => { + expect(() => validateLLMBaseUrl('not-a-url')).toThrow('Invalid LLM base URL'); + expect(() => validateLLMBaseUrl('')).toThrow('Invalid LLM base URL'); + }); + + it('does not include the raw URL in error messages (credential hygiene)', () => { + // Simulates a URL with an embedded API key + const urlWithCreds = 'http://192.168.1.1/v1?apikey=sk-secret'; + let msg = ''; + try { + validateLLMBaseUrl(urlWithCreds); + } catch (e) { + msg = (e as Error).message; + } + expect(msg).not.toContain('sk-secret'); + expect(msg).not.toContain(urlWithCreds); + }); + + it('callLLM rejects an invalid base URL before fetching', async () => { + const { callLLM } = await import('../../src/core/wiki/llm-client.js'); + await expect( + callLLM('prompt', { + apiKey: 'key', + baseUrl: 'file:///etc/passwd', + model: 'gpt-4o', + maxTokens: 100, + temperature: 0, + }), + ).rejects.toThrow('must use http:// or https://'); + }); +});