diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd07337b7..c28e49be9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,14 +11,14 @@ permissions: # Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". # Hardcoded `CI-` prefix (not `${{ github.workflow }}`) because this workflow is -# invoked as a reusable workflow from publish.yml and release-candidate.yml. In -# called-workflow context `github.workflow` evaluation is ambiguous across GitHub -# Actions versions, and a prefix that could resolve to the caller's name would -# share a concurrency group with the caller → deadlock. A literal prefix is -# immune. Direct `pull_request` invocations use `CI-`; invocations from a -# reusable-workflow caller fall into a per-run-unique group that never serializes -# with the caller. `push` to main is handled by release-candidate.yml, which -# calls this workflow once before publishing. +# invoked as a reusable workflow from publish.yml. In called-workflow context +# `github.workflow` evaluation is ambiguous across GitHub Actions versions, and a +# prefix that could resolve to the caller's name would share a concurrency group +# with the caller → deadlock. A literal prefix is immune. Direct `pull_request` +# invocations use `CI-`; invocations from a reusable-workflow caller fall +# into a per-run-unique group that never serializes with the caller. `push` to +# main is handled by publish.yml (RC mode), which calls this workflow once +# before publishing. concurrency: group: ${{ github.event_name == 'pull_request' && format('CI-{0}', github.ref) || format('CI-nested-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0cd526768..9c4ba0d8f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,6 +25,15 @@ on: a gitnexus/package.json whose version matches the tag. required: true type: string + # Explicit secret contract — callers pass these by name. Replaces the + # blanket `secrets: inherit` pattern (zizmor `secrets-inherit` audit). + # GHCR auth uses the implicit GITHUB_TOKEN; only Docker Hub credentials + # need to be passed through. + secrets: + DOCKERHUB_USERNAME: + required: true + DOCKERHUB_TOKEN: + required: true permissions: contents: read @@ -73,7 +82,7 @@ jobs: steps: # Only the workflow_call path requires a non-empty `inputs.tag` — callers - # (e.g. release-candidate.yml) must pass the RC tag explicitly. On direct + # (publish.yml in RC mode) must pass the RC tag explicitly. On direct # tag pushes the tag comes from `github.ref`, so `inputs.tag` is always # empty and validating it here would break every real release (#1064). # The downstream "Verify tag matches gitnexus/package.json version" step diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0694b5e4c..122130310 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,62 +1,421 @@ -name: Publish to npm +name: Publish + +# ───────────────────────────────────────────────────────────────────────────── +# Sole publisher for the `gitnexus` npm package, GitHub Releases, and Docker +# images. Replaces the former two-workflow design — see issue #1609 for the +# double-publish race this unification closes. +# +# Two release modes, both routed through this file: +# • Release candidate (rc) — triggered by push to `main` or workflow_dispatch. +# The RC path computes the next rc version, applies it in-CI, pushes a +# detached release commit with v-rc. + rc/ marker +# atomically, then publishes to npm with --tag rc and creates a GitHub +# prerelease. RC-only docker.yml invocation follows. +# • Stable — triggered by push of a v tag (no -rc.* +# suffix). Verifies package.json matches the tag, publishes to npm with +# --tag latest, creates a stable GitHub Release. No docker (RC-only). +# +# ⚠️ SELF-TRIGGER INVARIANT — DO NOT WEAKEN ⚠️ +# The `tags:` filter below uses a negative glob `'!v*-rc.*'` to prevent the +# workflow from re-triggering itself when the RC path pushes its own v-tag. +# Without this exclusion, every RC publish double-fires (the bug fixed by +# #1609). If a NEW prerelease channel is introduced (e.g. `-beta.N`, +# `-alpha.N`, `-next.N`), the negative-glob list MUST be extended in +# lock-step or self-trigger returns. The same invariant applies to the +# `Classify` step further below — its accepted-tag regex must align with +# the trigger filter's exclusion list. +# ───────────────────────────────────────────────────────────────────────────── on: push: + branches: [main] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE' tags: + # Negative-globbed exclusion of RC tags this workflow itself produces + # (see the SELF-TRIGGER INVARIANT in the header comment). - 'v*' - -# No workflow-level permissions — scoped per job below. + - '!v*-rc.*' + workflow_dispatch: + inputs: + bump: + description: >- + Cycle policy. 'auto' (default) continues the active rc cycle on + this branch if there is one, otherwise bumps patch from latest. + Choose 'patch' / 'minor' / 'major' to explicitly start or reset + an rc cycle. + required: false + default: 'auto' + type: choice + options: + - auto + - patch + - minor + - major + force: + description: 'Publish even when HEAD already has an rc marker' + required: false + default: 'false' + type: choice + options: + - 'false' + - 'true' +# Workflow-level deny-all; each job declares the minimum it needs. permissions: {} -# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". -# Tag refs are unique per release, so distinct tags run in parallel. Re-pushes of the -# same tag serialize. cancel-in-progress: false — never cancel a publish mid-flight. +# Distinct refs (refs/heads/main, refs/tags/v*) run in parallel. The +# release-PR-skip in rc-guard is the load-bearing invariant that prevents +# an RC main-push and a stable tag-push colliding on the same release commit. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: + # ── Phase 1: classify the triggering event into a release mode ───────────── + route: + name: Classify release event + runs-on: ubuntu-latest + timeout-minutes: 2 + permissions: + contents: read + outputs: + mode: ${{ steps.classify.outputs.mode }} + head_sha: ${{ steps.classify.outputs.head_sha }} + bump_input: ${{ inputs.bump }} + force_input: ${{ inputs.force }} + steps: + - name: Classify + id: classify + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + GH_REF: ${{ github.ref }} + GH_REF_NAME: ${{ github.ref_name }} + run: | + set -euo pipefail + + HEAD_SHA="${GITHUB_SHA}" + echo "head_sha=${HEAD_SHA}" >> "$GITHUB_OUTPUT" + + # Sanitize before logging (annotation-injection defense in depth). + REF_SAFE="${GH_REF//::/__}" + REF_NAME_SAFE="${GH_REF_NAME//::/__}" + echo "event=${EVENT_NAME} ref=${REF_SAFE} ref_name=${REF_NAME_SAFE}" + + MODE="" + case "${EVENT_NAME}" in + workflow_dispatch) + # Manual dispatch is only valid on main — that's the only ref + # where a real publish makes sense. + if [ "${GH_REF}" = "refs/heads/main" ]; then + MODE="rc" + else + echo "::error::workflow_dispatch is only permitted on refs/heads/main (got ${REF_SAFE})." + exit 1 + fi + ;; + push) + case "${GH_REF}" in + refs/heads/main) + MODE="rc" + ;; + refs/tags/v*) + # The trigger filter already excluded v*-rc.* tags. Anything + # reaching here is either a stable semver or a malformed v*. + TAG="${GH_REF#refs/tags/}" + if [[ "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + MODE="stable" + else + echo "::error::malformed v* tag rejected: ${REF_NAME_SAFE}" + echo "::error::stable tags must match ^v[0-9]+\\.[0-9]+\\.[0-9]+\$" + exit 1 + fi + ;; + *) + echo "::error::unexpected push ref ${REF_SAFE} reached publish workflow." + exit 1 + ;; + esac + ;; + *) + echo "::error::unsupported event ${EVENT_NAME}." + exit 1 + ;; + esac + + echo "mode=${MODE}" >> "$GITHUB_OUTPUT" + echo "Classified as mode=${MODE}" + + # ── Phase 2 (RC only): dedup marker + release-PR skip ────────────────────── + rc-guard: + name: RC guard (marker + release-PR skip) + needs: route + if: needs.route.outputs.mode == 'rc' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.decide.outputs.should_run }} + head_sha: ${{ steps.decide.outputs.head_sha }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + fetch-tags: true + # rc-guard reads only — no git pushes from this job. Skip the + # default extraheader credential persistence (artipacked audit). + persist-credentials: false + + - name: Decide + id: decide + shell: bash + env: + FORCE: ${{ inputs.force }} + BUMP_INPUT: ${{ inputs.bump }} + EVENT_NAME: ${{ github.event_name }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + HEAD_SHA=$(git rev-parse HEAD) + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + + if [ "$FORCE" = "true" ]; then + echo "Force flag set — running regardless of marker tag." + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # Explicit cycle reset on dispatch bypasses dedup. + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && [ -n "${BUMP_INPUT:-}" ] \ + && [ "${BUMP_INPUT:-auto}" != "auto" ]; then + echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup." + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + # ── Skip when the merge commit corresponds to a release ─────────── + # This skip is load-bearing: it prevents an RC build firing on the + # release-PR commit from racing the imminent stable-tag push on the + # same SHA. Two complementary checks: + # 1. HEAD subject matches `chore: release vX.Y.Z` (the canonical + # release-PR title). Anchored to require the bare title or the + # squash-merge `(#NNNN)` suffix exactly. Case-insensitive so + # `Chore: Release v1.2.3` (IDE auto-capitalization) still + # matches — prior commit-author conventions left the door open. + # 2. Squash-merged PR carries the `release` label. + # Either match suppresses the rc build — stable releases publish on + # the v-tag instead. + HEAD_SUBJECT="$(git log -1 --pretty=%s HEAD)" + # Sanitize GitHub-Actions annotation prefixes before logging — even + # though %s strips newlines, a crafted subject containing `::error::` + # could forge log annotations. + HEAD_SUBJECT_SAFE="${HEAD_SUBJECT//::/__}" + RELEASE_SUBJECT_RE='^chore:[[:space:]]*release[[:space:]]+v[0-9]+\.[0-9]+\.[0-9]+([[:space:]]+\(#[0-9]+\))?$' + shopt -s nocasematch + if [[ "$HEAD_SUBJECT" =~ $RELEASE_SUBJECT_RE ]]; then + shopt -u nocasematch + echo "HEAD commit subject matches a release commit — skipping rc." + echo " subject (sanitised): $HEAD_SUBJECT_SAFE" + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + shopt -u nocasematch + + # Squash-merge commits include `(#NNNN)` at the end of the subject. + if [[ "$HEAD_SUBJECT" =~ \(#([0-9]+)\)[[:space:]]*$ ]]; then + PR_NUM="${BASH_REMATCH[1]}" + echo "Detected squash-merge of PR #$PR_NUM — checking labels." + if LABELS_JSON="$(gh pr view "$PR_NUM" --repo "$REPO" --json labels 2>/dev/null)"; then + if printf '%s' "$LABELS_JSON" | jq -e '.labels[] | select(.name == "release")' >/dev/null; then + echo "PR #$PR_NUM has the 'release' label — skipping rc." + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "PR #$PR_NUM has no 'release' label — proceeding." + else + # Lookup failure is not fatal — fall through to dedup check. + echo "::warning::Could not read labels for PR #${PR_NUM} — falling through." + fi + fi + + # Dedup: is there already an rc/ marker pointing at HEAD? + MARKER="rc/${HEAD_SHA}" + if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then + echo "HEAD already has marker $MARKER — skipping." + echo "should_run=false" >> "$GITHUB_OUTPUT" + else + echo "No marker on HEAD — proceeding." + echo "should_run=true" >> "$GITHUB_OUTPUT" + fi + + # ── Phase 3: reusable CI gate ────────────────────────────────────────────── + # Runs for both rc (when guard says go) and stable. No `secrets:` passed — + # ci.yml and its entire reusable-workflow chain (ci-quality, ci-tests, + # ci-e2e, ci-scope-parity, ci-report) reference zero `secrets.*` values; + # passing any would be unused surface. GITHUB_TOKEN is implicit. ci: + needs: [route, rc-guard] + if: ${{ always() && (needs.route.outputs.mode == 'stable' || needs.rc-guard.outputs.should_run == 'true') }} uses: ./.github/workflows/ci.yml permissions: contents: read actions: read - # No pull-requests:write — `ci.yml`'s save-pr-meta job is gated on - # `github.event_name == 'pull_request'`, so it never runs during a - # tag-triggered publish. Least-privilege for release-critical paths. + # ── Phase 4: publish to npm + push refs (RC path) ────────────────────────── + # INVARIANT: `timeout-minutes` MUST stay below the App-token TTL (~60 min + # for actions/create-github-app-token installation tokens). The atomic + # tag-push step relies on the token minted at job start; if the job ever + # runs longer than the TTL, the push fails with an opaque 401. If you + # need to raise the timeout, re-mint the token immediately before the + # `Create and push rc tags` step instead. publish: - needs: ci + name: Publish to npm + needs: [route, rc-guard, ci] + if: ${{ always() && needs.ci.result == 'success' && (needs.route.outputs.mode == 'stable' || needs.rc-guard.outputs.should_run == 'true') }} runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 permissions: + # contents: write — RC path needs it for `git push --atomic` (v-tag + + # marker). Stable path runs in the same job and inherits the grant; it + # never invokes `git push`, so the elevated scope is unused there. + # id-token: write — npm provenance attestation. contents: write id-token: write + outputs: + # Two distinct step IDs feed this output; exactly one fires per run. + vtag: ${{ steps.rc-tags.outputs.vtag || steps.stable-vtag.outputs.vtag }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # ── Mint short-lived GitHub App token (RC only) ────────────────────── + # Industry direction (2025-2026): GitHub Apps with + # `actions/create-github-app-token` over long-lived PATs for + # workflow-touching tag pushes. Same fine-grained permission surface, + # ~1h expiry, not tied to a user seat, organizationally auditable. + # Replaces a prior fine-grained PAT. + # + # Required secrets (set in repo Settings → Secrets and variables → Actions): + # secrets.RELEASE_APP_ID — the App's numeric ID + # secrets.RELEASE_APP_PRIVATE_KEY — the App's PEM private key + # (The App ID is technically not sensitive — it's visible on the App's + # settings page — but storing it as a secret is harmless and avoids + # mixing storage classes for the same App.) + # The App must be installed on this repository with: + # - Contents: write (push the v-tag and rc marker) + # - Workflows: write (because the v-tag's tree may touch + # .github/workflows/**, which the default + # GITHUB_TOKEN cannot author) + # - Metadata: read (required for the `gh api /users/[bot]` + # bot-identity lookup in the tag-push step) + - name: Mint GitHub App token (RC) + if: needs.route.outputs.mode == 'rc' + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + # `client-id` is the renamed input that supersedes the deprecated + # `app-id` in v3.x. The action accepts the App's numeric ID or + # its Client ID under this name. We pass the numeric App ID, + # which the action resolves correctly. + client-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + # ── Separate checkout steps per mode ───────────────────────────────── + # Conditional `token:` expressions are footguns: empty string passed to + # actions/checkout fails opaquely, and `|| github.token` silently + # degrades a missing token to GITHUB_TOKEN, masking auth failures until + # the eventual `git push`. Two distinct steps make the auth contract + # explicit and fail loudly at checkout when the App token mint failed + # on the RC path. + - name: Checkout (RC) + if: needs.route.outputs.mode == 'rc' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + fetch-tags: true + # Short-lived GitHub App installation token. Required because the + # v-tag push lands at a SHA whose tree may touch + # `.github/workflows/**`, which the default GITHUB_TOKEN cannot + # author. + token: ${{ steps.app-token.outputs.token }} + # Do not persist the token in .git/config (artipacked audit). The + # RC tag push uses an inline `http.extraheader` at push time only; + # the credential never lands on disk. See the + # `Create and push rc tags` step below. + persist-credentials: false + + - name: Checkout (stable) + if: needs.route.outputs.mode == 'stable' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + # No `token:` — actions/checkout uses GITHUB_TOKEN by default. Stable + # path performs no git pushes; the default scope is sufficient. + with: + # No git pushes from the stable path either. Skip credential + # persistence (artipacked audit). + persist-credentials: false + + - name: Working-tree sanity + # Defense in depth (mirrors the vtag integrity gate, but on the input side): + # if a route-mode regression skipped both checkout `if:` gates, all + # downstream steps would run on a bare runner and produce confusing + # ENOENT errors. Fail loudly and early here instead. + shell: bash + run: | + if [ ! -f gitnexus/package.json ]; then + echo "::error::no working tree at gitnexus/package.json — route classification likely failed silently." + exit 1 + fi + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: 22 - registry-url: https://registry.npmjs.org - # Hermetic install for the published artifact — no cache carry-over - # from non-tag contexts. setup-node v5+ caches by default when a - # packageManager field is present in package.json, so the explicit - # opt-out is required to clear the zizmor cache-poisoning audit. - # ~30s slower per release; runs rarely. + # Node 24 ships with npm >= 11.5.x, which is the minimum that + # supports npm Trusted Publishing OIDC. Node 22 ships with npm + # 10.9.x (no OIDC) and `npm install -g npm@latest` to self-upgrade + # is fragile — it can crash the in-flight reify with + # `MODULE_NOT_FOUND` on `promise-retry` etc. Bumping the Node + # version is the clean fix; the package's `engines` field is + # `>=22.0.0` so consumer-side compatibility is unaffected (this + # Node version is only used during publish, not by package users). + node-version: 24 + # `registry-url:` is intentionally OMITTED. Under npm Trusted + # Publishing, OIDC only engages when no credential is configured. + # Setting `registry-url:` would make setup-node write + # `//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}` into the + # runner's .npmrc AND export NODE_AUTH_TOKEN from its `token:` + # input (default github.token). `npm publish` would then attempt + # GITHUB_TOKEN as the npm token, get rejected with 404, and OIDC + # would never be tried. See actions/setup-node#1440 and the GitHub + # Community discussion #176761 for the upstream bug and consensus + # workaround. + # + # Hermetic install for published artifacts — opt out of the v5+ + # default packageManager-based caching (clears the zizmor + # cache-poisoning audit). ~30s slower per release; runs rarely. package-manager-cache: false + - name: Build gitnexus-shared run: npm install && npm run build working-directory: gitnexus-shared - - run: npm ci + - name: Install gitnexus dependencies + run: npm ci working-directory: gitnexus - - name: Verify version consistency + # ── Stable-only: verify the tag and package.json agree ─────────────── + - name: Verify version consistency (stable) + if: needs.route.outputs.mode == 'stable' shell: bash + working-directory: gitnexus run: | + set -euo pipefail TAG_VERSION="${GITHUB_REF#refs/tags/v}" - if ! [[ "$TAG_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?$ ]]; then - echo "::error::Tag does not follow semver: v$TAG_VERSION" + # Stable mode REJECTS prerelease suffixes — those are filtered at + # trigger by the negative-glob filter, but defend at the bash layer too. + if ! [[ "$TAG_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::Stable tag must be ^v[0-9]+.[0-9]+.[0-9]+$ — got v$TAG_VERSION" exit 1 fi PKG_VERSION=$(node -p "require('./package.json').version") @@ -65,24 +424,376 @@ jobs: exit 1 fi echo "Version verified: $PKG_VERSION" - working-directory: gitnexus - - name: Build + # ── RC-only: compute the next rc version against the live registry ── + - name: Resolve rc version (rc) + id: rc-version + if: needs.route.outputs.mode == 'rc' + shell: bash + working-directory: gitnexus + env: + BUMP_INPUT: ${{ inputs.bump }} + EVENT_NAME: ${{ github.event_name }} + PKG_NAME: gitnexus + run: | + set -euo pipefail + + # 1. Current published `latest` — the floor for any new rc base. + # Only E404 ("never published") falls back to package.json; any + # other error (network, auth, malformed response) fails fast + # (retry-loud policy: never silently substitute on transient errors). + NPM_STDERR_LATEST="$(mktemp)" + if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then + : + else + if grep -qiE 'E404|not found' "$NPM_STDERR_LATEST"; then + CURRENT_LATEST="$(node -p "require('./package.json').version")" + echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST" + else + echo "::error::npm registry unreachable for 'view version':" >&2 + cat "$NPM_STDERR_LATEST" >&2 + rm -f "$NPM_STDERR_LATEST" + exit 1 + fi + fi + rm -f "$NPM_STDERR_LATEST" + CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}" + + # 2. Full version list — needed for the counter and active-cycle + # inference. Same E404-only fallback. + NPM_STDERR_VERSIONS="$(mktemp)" + if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then + : + else + if grep -qiE 'E404|not found' "$NPM_STDERR_VERSIONS"; then + VERSIONS_JSON='[]' + echo "No published versions for $PKG_NAME yet (E404)." + else + echo "::error::npm registry unreachable for 'view versions':" >&2 + cat "$NPM_STDERR_VERSIONS" >&2 + rm -f "$NPM_STDERR_VERSIONS" + exit 1 + fi + fi + rm -f "$NPM_STDERR_VERSIONS" + + # 3. Base selection. + # - workflow_dispatch + bump != auto → explicit cycle reset. + # - Otherwise (push, or dispatch with bump=auto) → continue the + # highest active rc base > latest if any; else patch from latest. + # Curated wrapper around `npx semver` — bare npx errors are noisy + # and don't distinguish registry-unreachable from invalid-bump-spec. + semver_bump() { + local kind="$1" current="$2" stderr_file out + stderr_file="$(mktemp)" + if out="$(npx --yes -p semver@7 semver -i "$kind" "$current" 2>"$stderr_file")"; then + rm -f "$stderr_file" + printf '%s' "$out" + return 0 + fi + echo "::error::semver bump failed (kind=${kind}, current=${current}):" >&2 + cat "$stderr_file" >&2 + rm -f "$stderr_file" + return 1 + } + + if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && [ -n "${BUMP_INPUT:-}" ] \ + && [ "${BUMP_INPUT:-auto}" != "auto" ]; then + BASE="$(semver_bump "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")" + echo "Explicit bump=$BUMP_INPUT → BASE=$BASE" + else + cat > /tmp/active_base.mjs <<'NODESCRIPT' + const latest = process.env.LATEST; + let v; + try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } + if (!Array.isArray(v)) v = [v]; + const parse = s => s.split(".").map(n => parseInt(n, 10)); + const gt = (a, b) => { + const [A, B] = [parse(a), parse(b)]; + for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i]; + return false; + }; + const bases = new Set(); + for (const s of v) { + const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s); + if (m && gt(m[1], latest)) bases.add(m[1]); + } + if (!bases.size) { process.stdout.write(""); process.exit(0); } + const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1); + process.stdout.write(sorted[sorted.length - 1]); + NODESCRIPT + ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)" + if [ -n "$ACTIVE_BASE" ]; then + BASE="$ACTIVE_BASE" + echo "Continuing active rc cycle → BASE=$BASE" + else + BASE="$(semver_bump patch "$CURRENT_LATEST_CLEAN")" + echo "No active rc cycle → patch bump from latest → BASE=$BASE" + fi + fi + + # 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1. + cat > /tmp/next_rc.mjs <<'NODESCRIPT' + const base = process.env.BASE; + const prefix = base + "-rc."; + let v; + try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } + if (!Array.isArray(v)) v = [v]; + const ns = v + .filter(s => typeof s === "string" && s.startsWith(prefix)) + .map(s => parseInt(s.slice(prefix.length), 10)) + .filter(n => Number.isInteger(n) && n >= 0); + process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1)); + NODESCRIPT + NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)" + RC_VERSION="${BASE}-rc.${NEXT_N}" + echo "Computed rc: $RC_VERSION" + + # 5. Defensive: if the exact version already exists on the registry + # (race with another run), abort before re-publishing. + NPM_STDERR_EXISTS="$(mktemp)" + if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then + rm -f "$NPM_STDERR_EXISTS" + echo "::error::Version $RC_VERSION already exists on npm — aborting." + exit 1 + else + if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then + rm -f "$NPM_STDERR_EXISTS" + # Version doesn't exist — safe to proceed. + else + echo "::error::npm registry unreachable for existence check:" >&2 + cat "$NPM_STDERR_EXISTS" >&2 + rm -f "$NPM_STDERR_EXISTS" + exit 1 + fi + fi + + { + echo "base=$BASE" + echo "rc_n=$NEXT_N" + echo "rc_version=$RC_VERSION" + } >> "$GITHUB_OUTPUT" + + - name: Apply rc version in-CI + if: needs.route.outputs.mode == 'rc' + shell: bash + working-directory: gitnexus + run: | + set -euo pipefail + npm version "${{ steps.rc-version.outputs.rc_version }}" \ + --no-git-tag-version --allow-same-version + + - name: Build gitnexus run: npm run build working-directory: gitnexus - name: Dry-run publish - run: npm publish --dry-run - working-directory: gitnexus - - - name: Publish to npm - run: npm publish --provenance --access public + # Cheap verification that the tarball assembles before the real publish. + shell: bash working-directory: gitnexus env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + NPM_TAG: ${{ needs.route.outputs.mode == 'rc' && 'rc' || 'latest' }} + run: npm publish --dry-run --tag "$NPM_TAG" - - name: Extract release notes from CHANGELOG + # ── Acquire the "rc lock" BEFORE publishing (idempotency anchor) ───── + # We create two refs and push atomically: + # v → annotated tag on a detached release commit whose + # tree contains the rewritten package.json, so the + # tag's source matches the npm tarball. + # rc/ → lightweight tag on HEAD; the guard's dedup key. + # Push fails → nothing published. Push succeeds, npm fails → marker + # blocks retries until manual cleanup (see Rollback Runbook in plan). + - name: Create and push rc tags + id: rc-tags + if: needs.route.outputs.mode == 'rc' + shell: bash + working-directory: gitnexus + env: + RC_VERSION: ${{ steps.rc-version.outputs.rc_version }} + HEAD_SHA: ${{ needs.rc-guard.outputs.head_sha }} + # Short-lived GitHub App token. Auth is supplied inline at push + # time via `http.extraheader` (per GitHub's documented + # x-access-token Basic pattern). It is NOT persisted in + # .git/config (artipacked audit) — checkout above ran with + # `persist-credentials: false`. + PUSH_TOKEN: ${{ steps.app-token.outputs.token }} + # App's slug from create-github-app-token (e.g. `gitnexus-release-bot`). + # Used to attribute the release commit to the App identity rather + # than the generic github-actions[bot]. The bot's numeric user-id + # is resolved at runtime via the GitHub API (the action does not + # expose it directly as of v3.2.0). + APP_SLUG: ${{ steps.app-token.outputs.app-slug }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -euo pipefail + VTAG="v${RC_VERSION}" + MARKER="rc/${HEAD_SHA}" + + # Resolve the App's bot user-id and construct the noreply email + # in the GitHub-canonical `+[bot]@users.noreply.github.com` + # shape. `[bot]` is part of the actual login on GitHub. + # + # The lookup is wrapped in a bounded retry because the first RC + # after App installation may hit propagation delay (404), and + # transient api.github.com 5xx during heavy org activity is a real + # failure class. Without retry, every transient blip aborts the + # entire release after CI has already succeeded. + BOT_LOGIN="${APP_SLUG}[bot]" + BOT_USER_ID="" + api_stderr="$(mktemp)" + for attempt in 1 2 3; do + if BOT_USER_ID="$(gh api "/users/${BOT_LOGIN}" --jq .id 2>"$api_stderr")" \ + && [[ "${BOT_USER_ID}" =~ ^[0-9]+$ ]]; then + break + fi + BOT_USER_ID="" + if [ "$attempt" -lt 3 ]; then + echo "::warning::bot user-id lookup attempt ${attempt} failed; retrying in $((attempt * 5))s" + sleep $((attempt * 5)) + fi + done + if ! [[ "${BOT_USER_ID}" =~ ^[0-9]+$ ]]; then + echo "::error::Could not resolve bot user-id for ${BOT_LOGIN} after 3 attempts." + echo "::error::gh api stderr:" + cat "$api_stderr" >&2 || true + echo "::error::Common causes: (a) newly-installed App — user record still propagating to /users/ (wait ~5min, redispatch with force=true); (b) App lacks Metadata: read permission; (c) transient api.github.com 5xx (redispatch)." + rm -f "$api_stderr" + exit 1 + fi + rm -f "$api_stderr" + git config user.name "${BOT_LOGIN}" + git config user.email "${BOT_USER_ID}+${BOT_LOGIN}@users.noreply.github.com" + + # Detached release commit with the version bump — main stays + # pristine, but the v-tag's tree matches the published package + # exactly (release-integrity). + git add package.json package-lock.json 2>/dev/null || git add package.json + git commit -m "release: ${VTAG}" --allow-empty + RELEASE_SHA="$(git rev-parse HEAD)" + echo "Detached release commit: $RELEASE_SHA" + + git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG" + git tag "$MARKER" "$HEAD_SHA" + + # Inline auth header. The base64-encoded form is masked as well + # as the raw token, because GitHub's secret-masker only masks the + # raw value — any subsequent `set -x` / GIT_TRACE line would + # otherwise expose the encoded credential. + # + # `set +x` wraps the compute+mask pair so that if an operator + # enables ACTIONS_STEP_DEBUG=true for triage (which turns on + # `set -x` globally), the assignment is NOT traced for the one + # line between compute and mask-registration. Without this wrap, + # debug mode would log `+ auth_header='Authorization: Basic '` + # exposing a still-valid (~1h) App token. + { set +x; } 2>/dev/null + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${PUSH_TOKEN}" | base64 -w0)" + echo "::add-mask::${auth_header}" + # Re-enable tracing only when explicitly requested via step-debug. + if [ "${ACTIONS_STEP_DEBUG:-false}" = "true" ]; then set -x; fi + + # Atomic push of both refs. If either would clobber an existing + # remote ref, the push fails and we stop before npm publish. + git -c http.extraheader="${auth_header}" \ + push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" + + { + echo "vtag=$VTAG" + echo "marker=$MARKER" + echo "release_sha=$RELEASE_SHA" + } >> "$GITHUB_OUTPUT" + + - name: Set vtag (stable) + id: stable-vtag + if: needs.route.outputs.mode == 'stable' + shell: bash + # github.ref_name flows in via env to avoid templating into the + # shell source (template-injection audit). Even though refs are + # constrained by git naming rules, the env-passthrough pattern + # makes injection structurally impossible. + env: + REF_NAME: ${{ github.ref_name }} + run: | + echo "vtag=${REF_NAME}" >> "$GITHUB_OUTPUT" + + # ── vtag integrity gate ────────────────────────────────────────────── + # Fail closed before any artifact-producing step (npm publish, Release, + # Docker) runs against an empty or mode-mismatched vtag. Prevents the + # silent "Release named main" / "Docker tagged from ref fallback" + # failure modes that the previous draft was vulnerable to. + - name: vtag integrity gate + id: vtag-gate + shell: bash + env: + MODE: ${{ needs.route.outputs.mode }} + VTAG: ${{ steps.rc-tags.outputs.vtag || steps.stable-vtag.outputs.vtag }} + run: | + set -euo pipefail + + if [ -z "$VTAG" ]; then + echo "::error::vtag is empty — refusing to create GitHub Release or trigger Docker." + exit 1 + fi + + case "$MODE" in + rc) + if ! [[ "$VTAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then + echo "::error::vtag '${VTAG}' does not match rc shape ^v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+$" + exit 1 + fi + ;; + stable) + if ! [[ "$VTAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::vtag '${VTAG}' does not match stable shape ^v[0-9]+.[0-9]+.[0-9]+$" + exit 1 + fi + ;; + *) + echo "::error::unknown mode '${MODE}' at vtag integrity gate." + exit 1 + ;; + esac + + echo "vtag verified: ${VTAG} (mode=${MODE})" + echo "vtag=${VTAG}" >> "$GITHUB_OUTPUT" + + # npm Trusted Publishing (GA'd 2025-07-31). OIDC authentication only + # engages when no npm credential is configured anywhere — the absence + # is the signal. Two upstream behaviors had to be neutralized for + # this to work: + # + # 1. setup-node's `registry-url:` is omitted (see the setup-node + # step above). With it, setup-node writes + # `//registry.npmjs.org/:_authToken=${NODE_AUTH_TOKEN}` into + # .npmrc and exports NODE_AUTH_TOKEN from `token:` (defaulting + # to github.token). npm publish then sends GITHUB_TOKEN as the + # bearer credential and the registry returns 404. OIDC is never + # tried because npm thinks it already has a credential. + # 2. The runner's bundled npm (10.9.x on Node 22) has no OIDC + # support; the upgrade step above pins it to >= 11.5.1. + # + # Provenance is auto-attached by the registry on trusted-publisher + # publishes — no --provenance flag needed. + # + # Prerequisite: register the package as a trusted publisher at + # https://www.npmjs.com/package/gitnexus/access (Publishing access → + # Trusted Publishers → GitHub Actions): + # Owner: abhigyanpatwari + # Repository: GitNexus + # Workflow: publish.yml + # Environment: (none) + - name: Publish to npm + shell: bash + working-directory: gitnexus + env: + NPM_TAG: ${{ needs.route.outputs.mode == 'rc' && 'rc' || 'latest' }} + run: npm publish --access public --tag "$NPM_TAG" + + # ── Stable-only: pull CHANGELOG body if present ────────────────────── + - name: Extract release notes from CHANGELOG (stable) id: changelog + if: needs.route.outputs.mode == 'stable' shell: bash run: | VERSION="${GITHUB_REF#refs/tags/v}" @@ -98,5 +809,90 @@ jobs: - name: Create GitHub Release uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 with: - body_path: ${{ steps.changelog.outputs.fallback == 'false' && '/tmp/release-notes.md' || '' }} - generate_release_notes: ${{ steps.changelog.outputs.fallback == 'true' }} + tag_name: ${{ steps.vtag-gate.outputs.vtag }} + name: >- + ${{ needs.route.outputs.mode == 'rc' + && format('Release Candidate {0}', steps.vtag-gate.outputs.vtag) + || steps.vtag-gate.outputs.vtag }} + prerelease: ${{ needs.route.outputs.mode == 'rc' }} + make_latest: ${{ needs.route.outputs.mode == 'stable' && 'true' || 'false' }} + # Stable: prefer CHANGELOG body, fall back to auto-generated. + # RC: always auto-generated + the prerelease body block below. + body_path: >- + ${{ needs.route.outputs.mode == 'stable' && steps.changelog.outputs.fallback == 'false' + && '/tmp/release-notes.md' || '' }} + generate_release_notes: >- + ${{ needs.route.outputs.mode == 'rc' + || steps.changelog.outputs.fallback == 'true' }} + body: >- + ${{ needs.route.outputs.mode == 'rc' && format( + 'Automated release candidate build from `main`.{0}{0}**npm:** `npm install gitnexus@rc`{0}**Version:** `{1}`{0}**Target base:** `{2}` (rc #{3}){0}**Source commit (main):** {4}{0}**Release commit (versioned tree):** {5}{0}{0}Release candidates are pre-stable builds intended for early testing. Stable releases remain on the `latest` dist-tag.', + '\n', + steps.rc-version.outputs.rc_version, + steps.rc-version.outputs.base, + steps.rc-version.outputs.rc_n, + needs.rc-guard.outputs.head_sha, + steps.rc-tags.outputs.release_sha + ) || '' }} + + # ── RC partial-failure cleanup ─────────────────────────────────────── + # If anything after the atomic tag-push step failed (npm publish + # blew up, GitHub Release call timed out, etc.), the v-tag and + # rc/ marker are already on origin. External consumers + # (Renovate, Dependabot, Releases RSS) can ingest a phantom tag for + # a version that was never published to npm. This step deletes them + # automatically so the operator's recovery is just "redispatch with + # force=true on the next commit", not a manual ref cleanup. + # + # Scoped strictly to RC + real (non-dry-run) + the rc-tags step + # actually produced a vtag (otherwise nothing to clean up). The + # App token is still valid (~1h TTL, job timeout 20min). + - name: Cleanup pushed tags on partial failure + if: ${{ failure() && needs.route.outputs.mode == 'rc' && steps.rc-tags.outputs.vtag != '' }} + shell: bash + working-directory: gitnexus + env: + VTAG: ${{ steps.rc-tags.outputs.vtag }} + MARKER: ${{ steps.rc-tags.outputs.marker }} + PUSH_TOKEN: ${{ steps.app-token.outputs.token }} + run: | + set -uo pipefail + echo "::warning::Publish step failed after tag push. Cleaning up remote refs to prevent phantom-version ingestion by downstream consumers." + + { set +x; } 2>/dev/null + auth_header="Authorization: Basic $(printf 'x-access-token:%s' "${PUSH_TOKEN}" | base64 -w0)" + echo "::add-mask::${auth_header}" + if [ "${ACTIONS_STEP_DEBUG:-false}" = "true" ]; then set -x; fi + + # Delete v-tag and marker. Each delete is best-effort — if one + # is already absent (atomic push partially rejected, or earlier + # cleanup ran), the other still gets attempted. + for ref in "refs/tags/${VTAG}" "refs/tags/${MARKER}"; do + if git -c http.extraheader="${auth_header}" push origin --delete "${ref}" 2>&1; then + echo "deleted origin ${ref}" + else + echo "::warning::could not delete origin ${ref} — may already be absent or protected. Manual cleanup may be required." + fi + done + + echo "::notice::Cleanup complete. To retry the release, redispatch the workflow with force=true on the same SHA, or push a new commit to main." + + # ── Phase 5 (RC only): Docker images ─────────────────────────────────────── + # R6: Docker remains RC-only. Stable Docker builds are explicitly deferred. + # Secrets are passed explicitly (not via `secrets: inherit`) so the + # callee's secret surface is auditable from the caller's source. + docker: + name: Build & Push RC Docker images + needs: [route, publish] + if: ${{ needs.route.outputs.mode == 'rc' && needs.publish.outputs.vtag != '' }} + uses: ./.github/workflows/docker.yml + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + permissions: + contents: read + packages: write + id-token: write + attestations: write + with: + tag: ${{ needs.publish.outputs.vtag }} diff --git a/.github/workflows/release-candidate.yml b/.github/workflows/release-candidate.yml deleted file mode 100644 index 2129e765a..000000000 --- a/.github/workflows/release-candidate.yml +++ /dev/null @@ -1,459 +0,0 @@ -name: Release Candidate - -on: - # Publish a release-candidate build whenever a merge/commit lands on main. - # Docs/README-only changes are filtered out so prose updates don't - # cut a release. - push: - branches: [main] - paths-ignore: - - '**.md' - - 'docs/**' - - 'LICENSE' - workflow_dispatch: - inputs: - bump: - description: >- - Cycle policy. 'auto' (default) continues the active rc cycle on - this branch if there is one, otherwise bumps patch from latest. - Choose 'patch' / 'minor' / 'major' to explicitly start or reset - an rc cycle. - required: false - default: 'auto' - type: choice - options: - - auto - - patch - - minor - - major - force: - description: 'Publish even when HEAD already has an rc marker' - required: false - default: 'false' - type: choice - options: - - 'false' - - 'true' - -# No workflow-level permissions — scoped per job below. -permissions: {} - -# Concurrency convention: see CONTRIBUTING.md → "GitHub Actions — Concurrency Convention". -# Serialize all runs on the same ref (push + workflow_dispatch) to prevent two publishes -# racing on the rc counter. cancel-in-progress: false — the earlier merge publishes first. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: false - -jobs: - # ── Skip when HEAD already has an rc marker (retry / duplicate dispatch) ── - # The marker is a lightweight tag `rc/` pushed *before* `npm - # publish`, so a failed publish leaves the marker in place and the guard - # refuses to re-publish. Recovery path after a partial failure: - # git push --delete origin rc/ v - # then redispatch with force=true. - guard: - name: Check if release candidate should run - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: read # read PR labels on the merge commit - outputs: - should_run: ${{ steps.decide.outputs.should_run }} - head_sha: ${{ steps.decide.outputs.head_sha }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - fetch-tags: true - - - name: Decide - id: decide - shell: bash - env: - FORCE: ${{ inputs.force }} - BUMP_INPUT: ${{ inputs.bump }} - EVENT_NAME: ${{ github.event_name }} - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - run: | - set -euo pipefail - HEAD_SHA=$(git rev-parse HEAD) - echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" - - if [ "$FORCE" = "true" ]; then - echo "Force flag set — running regardless of marker tag." - echo "should_run=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # An explicit cycle reset on dispatch (bump != auto) also bypasses - # the dedup guard — the maintainer is deliberately asking for a - # new rc from the same commit. - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && [ -n "${BUMP_INPUT:-}" ] \ - && [ "${BUMP_INPUT:-auto}" != "auto" ]; then - echo "Explicit bump=$BUMP_INPUT — bypassing marker dedup." - echo "should_run=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # ── Skip when the merge commit corresponds to a release ───────── - # Two complementary checks (belt-and-suspenders): - # 1. The HEAD commit subject matches `chore: release vX.Y.Z` - # (the canonical release-PR title in this repo). Anchored - # at both ends to require the bare title or the squash-merge - # `(#NNNN)` suffix exactly — rejects noisy variants like - # `chore: release v1.0.0 (something unrelated)`. - # 2. The squash-merged PR carries the `release` label. - # Either match suppresses the rc build — stable releases publish - # via publish.yml on the v-tag, so the rc cycle should pause for - # them rather than racing the npm publish. - HEAD_SUBJECT="$(git log -1 --pretty=%s HEAD)" - # Sanitise GitHub-Actions annotation prefixes before logging the - # raw subject — defence-in-depth so a hypothetical commit subject - # containing `::error::` or `::set-output::` cannot forge log - # annotations even though %s strips newlines. - HEAD_SUBJECT_SAFE="${HEAD_SUBJECT//::/__}" - RELEASE_SUBJECT_RE='^chore:[[:space:]]*release[[:space:]]+v[0-9]+\.[0-9]+\.[0-9]+([[:space:]]+\(#[0-9]+\))?$' - if [[ "$HEAD_SUBJECT" =~ $RELEASE_SUBJECT_RE ]]; then - echo "HEAD commit subject matches a release commit — skipping rc." - echo " subject (sanitised): $HEAD_SUBJECT_SAFE" - echo "should_run=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # Squash-merge commits include `(#NNNN)` at the end of the subject. - if [[ "$HEAD_SUBJECT" =~ \(#([0-9]+)\)[[:space:]]*$ ]]; then - PR_NUM="${BASH_REMATCH[1]}" - echo "Detected squash-merge of PR #$PR_NUM — checking labels." - if LABELS_JSON="$(gh pr view "$PR_NUM" --repo "$REPO" --json labels 2>/dev/null)"; then - if printf '%s' "$LABELS_JSON" | jq -e '.labels[] | select(.name == "release")' >/dev/null; then - echo "PR #$PR_NUM has the 'release' label — skipping rc." - echo "should_run=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - echo "PR #$PR_NUM has no 'release' label — proceeding." - else - # Lookup failure is not fatal — fall through to the dedup check - # so a transient GH API hiccup doesn't silently suppress rc builds. - echo "::warning::Could not read labels for PR #${PR_NUM} — falling through." - fi - fi - - # Dedup: is there already an rc/ marker pointing at HEAD? - MARKER="rc/${HEAD_SHA}" - if git rev-parse "refs/tags/$MARKER" >/dev/null 2>&1; then - echo "HEAD already has marker $MARKER — skipping." - echo "should_run=false" >> "$GITHUB_OUTPUT" - else - echo "No marker on HEAD — proceeding." - echo "should_run=true" >> "$GITHUB_OUTPUT" - fi - - # ── Reuse the stable CI workflow ───────────────────────────────────── - ci: - needs: guard - if: needs.guard.outputs.should_run == 'true' - uses: ./.github/workflows/ci.yml - permissions: - contents: read - secrets: inherit - - # ── Publish the rc build to npm + create GitHub prerelease ─────────── - publish: - name: Publish release candidate to npm - needs: [guard, ci] - if: needs.guard.outputs.should_run == 'true' - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - # The default GITHUB_TOKEN cannot be granted `workflows: write`, so - # tag pushes that reach a commit which modified `.github/workflows/**` - # are rejected with: "refusing to allow a GitHub App to create or - # update workflow ... without `workflows` permission". We pass a - # fine-grained PAT (RELEASE_PUSH_TOKEN, scoped to this repo with - # Contents: write + Workflows: write) to `actions/checkout` so that - # the subsequent `git push --atomic` of the v-tag and rc marker - # carries the PAT's identity. Job-level GITHUB_TOKEN keeps its - # scoped permissions for everything else (npm provenance, etc.). - contents: write # push rc tag + marker (via PAT) - id-token: write # npm provenance - outputs: - vtag: ${{ steps.reltag.outputs.vtag }} - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 0 - fetch-tags: true - # Use the PAT so `origin` is preauthed for `git push`. Without - # this the default GITHUB_TOKEN is wired into the remote, and a - # workflows-touching tag push is rejected — see the permissions - # block above. - token: ${{ secrets.RELEASE_PUSH_TOKEN }} - - - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 22 - registry-url: https://registry.npmjs.org - # Hermetic install — release-candidate produces shipped artifacts. - # setup-node v5+ caches by default when a packageManager field is - # present in package.json; explicit opt-out is required to clear - # the zizmor cache-poisoning audit. See cache-poisoning audit. - package-manager-cache: false - - - name: Build gitnexus-shared - run: npm install && npm run build - working-directory: gitnexus-shared - - - name: Install gitnexus dependencies - run: npm ci - working-directory: gitnexus - - - name: Resolve rc version - id: version - shell: bash - working-directory: gitnexus - env: - BUMP_INPUT: ${{ inputs.bump }} - EVENT_NAME: ${{ github.event_name }} - PKG_NAME: gitnexus - run: | - set -euo pipefail - - # 1. Current published `latest` — the floor for any new rc base. - # Only E404 ("never published") falls back to package.json; any - # other error (network, auth, malformed response) fails fast. - NPM_STDERR_LATEST="$(mktemp)" - if CURRENT_LATEST="$(npm view "$PKG_NAME" version 2>"$NPM_STDERR_LATEST")"; then - : - else - if grep -q 'E404' "$NPM_STDERR_LATEST"; then - CURRENT_LATEST="$(node -p "require('./package.json').version")" - echo "Package not on registry (E404) — seeding from package.json: $CURRENT_LATEST" - else - echo "::error::npm registry unreachable for 'view version':" >&2 - cat "$NPM_STDERR_LATEST" >&2 - rm -f "$NPM_STDERR_LATEST" - exit 1 - fi - fi - rm -f "$NPM_STDERR_LATEST" - CURRENT_LATEST_CLEAN="${CURRENT_LATEST%%-*}" - - # 2. Full version list — needed for the counter and for active-cycle - # inference. Same E404-only fallback. - NPM_STDERR_VERSIONS="$(mktemp)" - if VERSIONS_JSON="$(npm view "$PKG_NAME" versions --json 2>"$NPM_STDERR_VERSIONS")"; then - : - else - if grep -q 'E404' "$NPM_STDERR_VERSIONS"; then - VERSIONS_JSON='[]' - echo "No published versions for $PKG_NAME yet (E404)." - else - echo "::error::npm registry unreachable for 'view versions':" >&2 - cat "$NPM_STDERR_VERSIONS" >&2 - rm -f "$NPM_STDERR_VERSIONS" - exit 1 - fi - fi - rm -f "$NPM_STDERR_VERSIONS" - - # 3. Base selection. - # - workflow_dispatch + bump ∈ {patch,minor,major} → explicit cycle - # reset from latest. - # - Everything else (push, or dispatch with bump=auto) → continue - # the highest active rc base > latest if one exists; else - # default to patch from latest. - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && [ -n "${BUMP_INPUT:-}" ] \ - && [ "${BUMP_INPUT:-auto}" != "auto" ]; then - BASE="$(npx --yes -p semver@7 semver -i "$BUMP_INPUT" "$CURRENT_LATEST_CLEAN")" - echo "Explicit bump=$BUMP_INPUT → BASE=$BASE" - else - cat > /tmp/active_base.mjs <<'NODESCRIPT' - const latest = process.env.LATEST; - let v; - try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } - if (!Array.isArray(v)) v = [v]; - const parse = s => s.split(".").map(n => parseInt(n, 10)); - const gt = (a, b) => { - const [A, B] = [parse(a), parse(b)]; - for (let i = 0; i < 3; i++) if (A[i] !== B[i]) return A[i] > B[i]; - return false; - }; - const bases = new Set(); - for (const s of v) { - const m = /^(\d+\.\d+\.\d+)-rc\.\d+$/.exec(s); - if (m && gt(m[1], latest)) bases.add(m[1]); - } - if (!bases.size) { process.stdout.write(""); process.exit(0); } - const sorted = [...bases].sort((a, b) => gt(a, b) ? 1 : -1); - process.stdout.write(sorted[sorted.length - 1]); - NODESCRIPT - ACTIVE_BASE="$(LATEST="$CURRENT_LATEST_CLEAN" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/active_base.mjs)" - if [ -n "$ACTIVE_BASE" ]; then - BASE="$ACTIVE_BASE" - echo "Continuing active rc cycle → BASE=$BASE" - else - BASE="$(npx --yes -p semver@7 semver -i patch "$CURRENT_LATEST_CLEAN")" - echo "No active rc cycle → patch bump from latest → BASE=$BASE" - fi - fi - - # 4. Counter: 1 + max existing N for `${BASE}-rc.*`, else 1. - cat > /tmp/next_rc.mjs <<'NODESCRIPT' - const base = process.env.BASE; - const prefix = base + "-rc."; - let v; - try { v = JSON.parse(process.env.VERSIONS_JSON); } catch { v = []; } - if (!Array.isArray(v)) v = [v]; - const ns = v - .filter(s => typeof s === "string" && s.startsWith(prefix)) - .map(s => parseInt(s.slice(prefix.length), 10)) - .filter(n => Number.isInteger(n) && n >= 0); - process.stdout.write(String(ns.length ? Math.max(...ns) + 1 : 1)); - NODESCRIPT - NEXT_N="$(BASE="$BASE" VERSIONS_JSON="$VERSIONS_JSON" node /tmp/next_rc.mjs)" - RC_VERSION="${BASE}-rc.${NEXT_N}" - echo "Computed rc: $RC_VERSION" - - # 5. Defensive: if the exact version already exists on the registry - # (e.g., race with another run), abort before re-publishing. - # Same E404-only pattern used above — a transient network - # failure must fail loudly, not pretend the version is missing. - NPM_STDERR_EXISTS="$(mktemp)" - if npm view "$PKG_NAME@$RC_VERSION" version 2>"$NPM_STDERR_EXISTS" >/dev/null; then - rm -f "$NPM_STDERR_EXISTS" - echo "::error::Version $RC_VERSION already exists on npm — aborting." - exit 1 - else - if grep -qiE 'E404|not found' "$NPM_STDERR_EXISTS"; then - rm -f "$NPM_STDERR_EXISTS" - # Version doesn't exist — safe to proceed. - else - echo "::error::npm registry unreachable for existence check:" >&2 - cat "$NPM_STDERR_EXISTS" >&2 - rm -f "$NPM_STDERR_EXISTS" - exit 1 - fi - fi - - { - echo "base=$BASE" - echo "rc_n=$NEXT_N" - echo "rc_version=$RC_VERSION" - } >> "$GITHUB_OUTPUT" - - - name: Apply rc version in-CI - shell: bash - working-directory: gitnexus - run: | - set -euo pipefail - npm version "${{ steps.version.outputs.rc_version }}" \ - --no-git-tag-version --allow-same-version - - - name: Build gitnexus - run: npm run build - working-directory: gitnexus - - - name: Dry-run publish - run: npm publish --dry-run --tag rc - working-directory: gitnexus - - # ── Acquire the "rc lock" BEFORE publishing (fixes idempotency) ───── - # We create two tags and push them atomically: - # v → annotated tag on a detached release commit - # whose tree contains the rewritten package.json - # (so the tag's source matches the npm tarball) - # rc/ → lightweight tag on HEAD; the guard's dedup key - # If this push fails, nothing is published — safe. - # If this push succeeds but npm publish fails, the marker stays on - # the remote and blocks retries until an operator manually cleans up. - - name: Create and push rc tags - id: reltag - shell: bash - working-directory: gitnexus - env: - RC_VERSION: ${{ steps.version.outputs.rc_version }} - HEAD_SHA: ${{ needs.guard.outputs.head_sha }} - run: | - set -euo pipefail - VTAG="v${RC_VERSION}" - MARKER="rc/${HEAD_SHA}" - git config user.name 'github-actions[bot]' - git config user.email '41898282+github-actions[bot]@users.noreply.github.com' - - # Detached release commit with the version bump — keeps `main` - # pristine but gives the v-tag a tree that matches the published - # package contents exactly (fixes release-integrity gap). - git add package.json package-lock.json 2>/dev/null || git add package.json - git commit -m "release: ${VTAG}" --allow-empty - RELEASE_SHA="$(git rev-parse HEAD)" - echo "Detached release commit: $RELEASE_SHA" - - # Annotated release tag on the release commit. - git tag -a "$VTAG" "$RELEASE_SHA" -m "$VTAG" - # Lightweight marker on the user-visible HEAD for the guard. - git tag "$MARKER" "$HEAD_SHA" - - # Atomic push of both refs. If either would clobber an existing - # remote ref, the push fails and we stop before npm publish. - git push --atomic origin "refs/tags/$VTAG" "refs/tags/$MARKER" - - { - echo "vtag=$VTAG" - echo "marker=$MARKER" - echo "release_sha=$RELEASE_SHA" - } >> "$GITHUB_OUTPUT" - - - name: Publish to npm (rc dist-tag) - run: npm publish --provenance --access public --tag rc - working-directory: gitnexus - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Create GitHub prerelease - uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v2 - with: - tag_name: ${{ steps.reltag.outputs.vtag }} - name: Release Candidate ${{ steps.reltag.outputs.vtag }} - prerelease: true - make_latest: 'false' - generate_release_notes: true - body: | - Automated release candidate build from `main`. - - **npm:** `npm install gitnexus@rc` - **Version:** `${{ steps.version.outputs.rc_version }}` - **Target base:** `${{ steps.version.outputs.base }}` (rc #${{ steps.version.outputs.rc_n }}) - **Source commit (main):** ${{ needs.guard.outputs.head_sha }} - **Release commit (versioned tree):** ${{ steps.reltag.outputs.release_sha }} - - Release candidates are pre-stable builds intended for early testing. - Stable releases remain on the `latest` dist-tag. - - # ── Build & push RC Docker images ──────────────────────────────────── - # Calls docker.yml as a reusable workflow so that the build, signing, and - # attestation logic stays in one place. The publish job exposes `vtag` - # (e.g. `v1.2.3-rc.1`) as an output so we can pass it as the tag input. - # RC images are signed with Cosign keyless signing; the OIDC identity - # will be `docker.yml@refs/heads/main` (the caller's ref) rather than a - # tag ref — see README.md § Docker for the correct verify command for RCs. - docker: - name: Build & Push RC Docker images - needs: [guard, publish] - if: needs.guard.outputs.should_run == 'true' && needs.publish.outputs.vtag != '' - uses: ./.github/workflows/docker.yml - # Reusable workflows do not receive caller secrets unless inherited; without - # this, DOCKERHUB_* / GITHUB_TOKEN are empty in docker.yml → "Username and - # password required" on Docker Hub login (see same pattern on `ci:` above). - secrets: inherit - permissions: - contents: read - packages: write - id-token: write - attestations: write - with: - tag: ${{ needs.publish.outputs.vtag }} diff --git a/.github/zizmor.yml b/.github/zizmor.yml index b2f89e3ba..5d93b4cce 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -37,7 +37,8 @@ rules: - pr-labeler.yml # Note: cache-poisoning is NOT exempted. The two prior findings in - # publish.yml and release-candidate.yml were fixed structurally by - # dropping `cache: npm` from those workflows (matches the pattern used - # by PyO3/maturin for the same audit). See the commit that added this - # file for the rationale. + # publish.yml and the former release-candidate.yml were fixed structurally + # by dropping `cache: npm` from those workflows (matches the pattern used + # by PyO3/maturin for the same audit). After the publish-workflow + # unification (issue #1609), only publish.yml remains; the same + # cache-poisoning hardening applies there. diff --git a/AGENTS.md b/AGENTS.md index b9b9138b1..99fc68bd0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -155,9 +155,9 @@ npx gitnexus analyze --embeddings # also generate embeddings for new/changed npx gitnexus analyze --drop-embeddings # explicit opt-in to wipe existing embeddings ``` -`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** at `.gitnexus/parse-cache.json` for chunks whose file contents haven't changed since the last run. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). +`analyze` runs **incrementally by default**. The pipeline still parses every file every run (cross-file resolution requires it), but tree-sitter parsing is **served from a content-addressed cache** under `.gitnexus/parse-cache/` (per-chunk JSON shards plus `index.json`) for chunks whose file contents haven't changed since the last run. Older installs may still have a legacy single file `.gitnexus/parse-cache.json`, which is read for backward compatibility but no longer written. Only changed-file rows (and their importers) are rewritten in LadybugDB; unchanged-file rows are preserved. Output is byte-equivalent to a full rebuild. Pass `--force` to wipe and re-index from scratch (e.g., to recover from a corrupt index, or after upgrading GitNexus). -The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete `.gitnexus/parse-cache.json` at any time — it'll be rebuilt on the next analyze. +The parse cache key is **content-addressed and version-tagged**: it survives `--force` runs, and is automatically invalidated by a `gitnexus` package upgrade (so a new tree-sitter grammar doesn't silently replay stale parse output). Safe to delete the whole `.gitnexus/parse-cache/` directory (and remove any legacy `.gitnexus/parse-cache.json` if present) at any time — it'll be rebuilt on the next analyze. Check `.gitnexus/meta.json` `stats.embeddings` (0 = none). A plain `analyze` no longer drops existing vectors — pass `--drop-embeddings` to wipe. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d4f6b12b2..e048d4f2f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -144,16 +144,18 @@ If you use coding agents, follow project context files (e.g. `AGENTS.md`, `CLAUD ## Releases -Two publish workflows ship `gitnexus` to npm: +One workflow ships `gitnexus` to npm — `.github/workflows/publish.yml`. It +routes between two modes based on the triggering event: -- **Stable** (`.github/workflows/publish.yml`) — triggered by pushing any `v*` - tag. Publishes to the `latest` dist-tag with a changelog-backed GitHub - release. Maintainers are expected to tag from `main` as a convention; the - workflow itself does not enforce branch reachability. -- **Release Candidate** (`.github/workflows/release-candidate.yml`) — runs on - every push to `main` (typically a merged PR) plus manual dispatch. Docs-only - changes are skipped via `paths-ignore`. Publishes to the `rc` dist-tag with - version `X.Y.Z-rc.N` and a GitHub prerelease, where: +- **Stable mode** — triggered by pushing any `v` tag (no `-rc.*` + suffix; RC tags are excluded at trigger via a negative glob). Publishes to + the `latest` dist-tag with a changelog-backed GitHub release. Maintainers + are expected to tag from `main` as a convention; the workflow itself does + not enforce branch reachability. No Docker build (RC-only). +- **Release-candidate mode** — runs on every push to `main` (typically a + merged PR) plus manual `workflow_dispatch`. Docs-only changes are skipped + via `paths-ignore`. Publishes to the `rc` dist-tag with version + `X.Y.Z-rc.N` and a GitHub prerelease, where: - `X.Y.Z` is selected automatically. On push (and on dispatch with `bump: auto`, the default) the workflow **continues the active rc cycle**: if the registry already has `X.Y.Z-rc.*` versions with `X.Y.Z` > current @@ -170,36 +172,64 @@ Two publish workflows ship `gitnexus` to npm: caller's ref — see README.md § Docker for the verify command). Idempotency: the workflow pushes an `rc/` marker tag and a - `v` release tag **atomically, before** calling `npm publish`. The guard - refuses to re-run once the marker exists, so a post-publish failure will - not mint a duplicate rc for the same commit. The `v` tag points at a - detached release commit whose `package.json` matches the npm tarball - exactly (traceable releases). Recovery after a partial failure: + `v` release tag **atomically, before** calling `npm publish`. The + RC guard refuses to re-run once the marker exists, so a post-publish + failure will not mint a duplicate rc for the same commit. The `v` + tag points at a detached release commit whose `package.json` matches + the npm tarball exactly (traceable releases). The RC tag is excluded + from this workflow's `push: tags:` filter, so it does **not** re-trigger + publishing — preventing the double-publish failure mode tracked in #1609. + Recovery after a partial failure: the workflow's `if: failure()` cleanup + step in the `publish` job auto-deletes the v-tag and marker on most + post-publish failures, so the typical retry is just: + + ```bash + gh workflow run publish.yml --ref main -f force=true + # or push a new commit to main, which will cut a fresh RC + ``` + + If auto-cleanup didn't run (e.g. the cleanup step itself failed, or the + failure happened in the route/rc-guard phase before the marker was + pushed), manual cleanup is: ```bash git push --delete origin rc/ v - # then redispatch the workflow with force: true + # then redispatch with force: true ``` + **Release-PR-skip subject pattern.** The rc-guard job recognizes a + squash-merged release commit by matching the commit subject against + `^chore: release vX.Y.Z` (optionally followed by ` (#NNNN)` for the + squash-merge PR-number suffix). Match is case-insensitive — `Chore: Release v1.2.3` + works too. PRs that should suppress the RC build must either use this + subject shape, or carry the `release` label so the label-based fallback + fires. Other release-style subjects (`chore(release): v1.2.3`, + `release: v1.2.3`) will NOT trigger the skip — please name the release + PR exactly `chore: release vX.Y.Z` to keep the dedup deterministic. + **Docker-only partial failure:** if `publish` succeeds (npm tarball + tags are live) but the `docker` job subsequently fails (e.g. GHCR flakiness), the npm RC is already published and the `rc/` marker is in place. - Re-running `release-candidate.yml` with `force: true` will abort at the - "Version already exists on npm" guard. To recover without cutting a new RC: + Recovery without cutting a new RC: ```bash - # 1. Manually trigger only the docker workflow, passing the existing RC tag: - gh workflow run docker.yml --ref main -f tag=v - # (requires a workflow_dispatch trigger on docker.yml — see note below) + # Re-run only the failed docker job from the original workflow run: + gh run rerun --failed ``` - Because `docker.yml` intentionally has no `workflow_dispatch` (images are - tag-driven by design), the practical recovery options are: - - Wait for the next commit on `main`, which will cut a new RC that includes - the Docker build. - - Manually run `docker build` + `docker push` locally and sign with Cosign - against the same digest. - - Delete `rc/` and `v` tags, then redispatch with `force: true` to re-run the full RC pipeline (cuts a new RC number). + Find the run ID via `gh run list --workflow=publish.yml --branch main`. + `docker.yml` intentionally has no `workflow_dispatch` trigger (images are + tag-driven by design), so the gh-run-rerun path is the supported recovery. + + **GitHub Release transient failure** (npm publish succeeded, Release step + failed): the npm artifact is live but no GitHub Release page exists. + Recover by either re-running the failed job (`gh run rerun --failed`), + or creating the Release manually: + + ```bash + gh release create v --prerelease --generate-notes # RC + gh release create v --notes-file gitnexus/CHANGELOG.md # stable + ``` The rc workflow never moves `latest`. To verify after a change, inspect dist-tags: diff --git a/GUARDRAILS.md b/GUARDRAILS.md index c09f0319a..2e2b9db41 100644 --- a/GUARDRAILS.md +++ b/GUARDRAILS.md @@ -36,7 +36,7 @@ Format: **Trigger → Instruction → Reason**. Append new Signs when the same m ### Index seems corrupt or "incremental" is misbehaving - **Trigger:** `analyze` produces unexpected results, or `meta.json.incrementalInProgress` is set, or the index is in a half-state after a crash. -- **Do:** `npx gitnexus analyze --force` to rebuild from scratch. The dirty-flag check forces this automatically when a previous incremental run didn't complete cleanly, but `--force` is the manual escape hatch. Safe to delete `.gitnexus/parse-cache.json` at any time — content-addressed, will be regenerated. +- **Do:** `npx gitnexus analyze --force` to rebuild from scratch. The dirty-flag check forces this automatically when a previous incremental run didn't complete cleanly, but `--force` is the manual escape hatch. Safe to delete the `.gitnexus/parse-cache/` directory (and any legacy `.gitnexus/parse-cache.json`) at any time — content-addressed, will be regenerated. - **Why:** Incremental writeback is selective DB row replacement; if the on-disk state is inconsistent for any reason, a full rebuild is the cheapest path back to a known-good index. ### Embeddings vanished after analyze diff --git a/README.md b/README.md index 96b38a8eb..f99564ffc 100644 --- a/README.md +++ b/README.md @@ -429,7 +429,7 @@ The Docker images are version-locked to the npm package: Both registries receive the same digest from a single build step, so you can pull from either and the signature verifies identically. - Release-candidate images (e.g. `:1.7.0-rc.1`) are published alongside each - RC npm release. They are built by `release-candidate.yml` calling `docker.yml` + RC npm release. They are built by `publish.yml` calling `docker.yml` as a reusable workflow after the RC tag is created and pushed. - `:latest` is auto-promoted only from non-prerelease tags by the Docker metadata action, so it always points at a real, npm-published version. @@ -462,7 +462,7 @@ registries because both sets of tags were signed at the same digest in one workflow run. **Release candidates** — signed from `refs/heads/main` (the caller's ref when -`release-candidate.yml` invokes `docker.yml` as a reusable workflow): +`publish.yml` invokes `docker.yml` as a reusable workflow): ```bash cosign verify ghcr.io/abhigyanpatwari/gitnexus:1.7.0-rc.1 \ diff --git a/gitnexus/src/core/ingestion/call-processor.ts b/gitnexus/src/core/ingestion/call-processor.ts index 35a59dab4..1b5a234b4 100644 --- a/gitnexus/src/core/ingestion/call-processor.ts +++ b/gitnexus/src/core/ingestion/call-processor.ts @@ -766,6 +766,15 @@ export const processCalls = async ( importedRawReturnTypesMap?: ReadonlyMap>, heritageMap?: HeritageMap, bindingAccumulator?: BindingAccumulator, + /** + * Optional cache for compiled `Parser.Query` objects keyed by language name. + * When provided, compiled queries are reused across calls instead of being + * re-compiled from the query string for every file. Callers that invoke + * `processCalls` many times with single-file batches (e.g. the cross-file + * propagation phase) should pass a long-lived map here to avoid O(N) + * query recompilation overhead. + */ + compiledQueryCache?: Map, ): Promise => { const parser = await loadParser(); const collectedHeritage: ExtractedHeritage[] = []; @@ -843,7 +852,11 @@ export const processCalls = async ( let matches; try { const lang = parser.getLanguage(); - const query = new Parser.Query(lang, queryStr); + let query = compiledQueryCache?.get(language); + if (!query) { + query = new Parser.Query(lang, queryStr); + compiledQueryCache?.set(language, query); + } matches = query.matches(tree.rootNode); } catch (queryError) { logger.warn({ queryError }, `Query error for ${file.path}:`); diff --git a/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts b/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts index 5c014ed73..6c10ccb11 100644 --- a/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts +++ b/gitnexus/src/core/ingestion/pipeline-phases/cross-file-impl.ts @@ -16,12 +16,18 @@ import { } from '../call-processor.js'; import type { createResolutionContext } from '../model/resolution-context.js'; import { createASTCache } from '../ast-cache.js'; -import { type PipelineProgress, getLanguageFromFilename } from 'gitnexus-shared'; +import { + type PipelineProgress, + getLanguageFromFilename, + type SupportedLanguages, +} from 'gitnexus-shared'; import { readFileContents } from '../filesystem-walker.js'; import { isLanguageAvailable } from '../../tree-sitter/parser-loader.js'; +import { isRegistryPrimary } from '../registry-primary-flag.js'; import { topologicalLevelSort } from '../utils/graph-sort.js'; import type { KnowledgeGraph } from '../../graph/types.js'; import { isDev } from '../utils/env.js'; +import type Parser from 'tree-sitter'; import { logger } from '../../logger.js'; /** Max AST trees to keep in LRU cache for cross-file binding propagation. */ @@ -114,6 +120,36 @@ export async function runCrossFileBindingPropagation( let crossFileResolved = 0; const crossFileStart = Date.now(); const astCache = createASTCache(AST_CACHE_CAP); + // Compiled query objects keyed by language name. Shared across all processCalls + // invocations in this phase so the same tree-sitter query string is only + // compiled once per language instead of once per file (O(1) vs O(N)). + const compiledQueryCache = new Map(); + + // Snapshot total topological candidates for progress math. We walk the + // levels once more here (fast — no I/O) so we can report meaningful + // percentages rather than a frozen display. + let totalCandidates = 0; + for (const level of levels) { + for (const filePath of level) { + if (totalCandidates >= MAX_CROSS_FILE_REPROCESS) break; + const imports = ctx.namedImportMap.get(filePath); + if (!imports) continue; + if (!allPathSet.has(filePath)) continue; + const lang = getLanguageFromFilename(filePath); + if (!lang || !isLanguageAvailable(lang)) continue; + // Registry-primary languages have their call resolution handled by the + // scope-resolution pipeline — processCalls skips them immediately. Skip + // here too so we avoid the I/O cost (readFileContents) and map-building + // overhead for files that would be no-ops anyway. + if (isRegistryPrimary(lang)) continue; + totalCandidates++; + } + if (totalCandidates >= MAX_CROSS_FILE_REPROCESS) break; + } + const cappedTotal = Math.min(totalCandidates, MAX_CROSS_FILE_REPROCESS); + + /** Emit a progress event every PROGRESS_INTERVAL files so the UI stays alive. */ + const PROGRESS_INTERVAL = 25; for (const level of levels) { const levelCandidates: { @@ -151,6 +187,10 @@ export async function runCrossFileBindingPropagation( const lang = getLanguageFromFilename(filePath); if (!lang || !isLanguageAvailable(lang)) continue; + // Registry-primary languages have their call resolution handled by the + // scope-resolution pipeline — processCalls skips them immediately. Skip + // here to avoid readFileContents I/O and map-building for no-op files. + if (isRegistryPrimary(lang)) continue; levelCandidates.push({ filePath, seeded, importedReturns, importedRawReturns }); } @@ -188,8 +228,24 @@ export async function runCrossFileBindingPropagation( bindings.size > 0 ? bindings : undefined, importedReturnTypesMap.size > 0 ? importedReturnTypesMap : undefined, importedRawReturnTypesMap.size > 0 ? importedRawReturnTypesMap : undefined, + undefined, + undefined, + compiledQueryCache, ); crossFileResolved++; + + // Emit progress every PROGRESS_INTERVAL files so the UI shows real + // movement instead of a frozen display (cross-file can take minutes + // on large repos with many cross-file imports). + if (crossFileResolved % PROGRESS_INTERVAL === 0 || crossFileResolved === cappedTotal) { + const pct = cappedTotal > 0 ? Math.round((crossFileResolved / cappedTotal) * 8) : 0; + onProgress({ + phase: 'parsing', + percent: 82 + pct, + message: `Cross-file type propagation (${crossFileResolved}/${cappedTotal} files)...`, + stats: { filesProcessed: crossFileResolved, totalFiles, nodesCreated: graph.nodeCount }, + }); + } } if (crossFileResolved >= MAX_CROSS_FILE_REPROCESS) { diff --git a/gitnexus/src/storage/parse-cache.ts b/gitnexus/src/storage/parse-cache.ts index a1abf76fa..680ca1ab2 100644 --- a/gitnexus/src/storage/parse-cache.ts +++ b/gitnexus/src/storage/parse-cache.ts @@ -70,15 +70,28 @@ const GITNEXUS_PKG_VERSION = (() => { })(); export const PARSE_CACHE_VERSION = `${SCHEMA_BUMP}+${GITNEXUS_PKG_VERSION}`; -const CACHE_FILENAME = 'parse-cache.json'; +const LEGACY_CACHE_FILENAME = 'parse-cache.json'; +const CACHE_DIRNAME = 'parse-cache'; +const CACHE_INDEX_FILENAME = 'index.json'; -/** On-disk shape. */ +/** Keys on disk always come from `computeChunkHash` — 64-char lowercase hex. */ +const CHUNK_CACHE_KEY_HEX_RE = /^[a-f0-9]{64}$/; + +const isValidChunkCacheKey = (chunkHash: string): boolean => CHUNK_CACHE_KEY_HEX_RE.test(chunkHash); + +/** On-disk shape for the legacy single-file format. */ interface ParseCacheFile { version: string; /** key = chunk hash (hex) → cached chunk result list. */ entries: Record; } +/** On-disk shape for the sharded directory format. */ +interface ShardedParseCacheIndex { + version: string; + keys: string[]; +} + /** Runtime view: keyed Map for fast lookup; mutated in place during a run. */ export interface ParseCache { version: string; @@ -144,12 +157,19 @@ const mapReviver = (_key: string, value: unknown): unknown => { return value; }; -/** - * Load the parse cache. Returns an empty cache on any failure (missing - * file, corrupt JSON, version mismatch). Never throws on a normal load. - */ -export const loadParseCache = async (storagePath: string): Promise => { - const cachePath = path.join(storagePath, CACHE_FILENAME); +const getLegacyCachePath = (storagePath: string): string => + path.join(storagePath, LEGACY_CACHE_FILENAME); + +const getCacheDirPath = (storagePath: string): string => path.join(storagePath, CACHE_DIRNAME); + +const getCacheIndexPath = (storagePath: string): string => + path.join(getCacheDirPath(storagePath), CACHE_INDEX_FILENAME); + +const getCacheChunkPath = (storagePath: string, chunkHash: string): string => + path.join(getCacheDirPath(storagePath), `${chunkHash}.json`); + +const loadLegacyParseCache = async (storagePath: string): Promise => { + const cachePath = getLegacyCachePath(storagePath); try { const raw = await fs.readFile(cachePath, 'utf-8'); const data = JSON.parse(raw, mapReviver) as ParseCacheFile; @@ -172,22 +192,90 @@ export const loadParseCache = async (storagePath: string): Promise = } }; +const loadShardedParseCache = async (storagePath: string): Promise => { + const indexPath = getCacheIndexPath(storagePath); + try { + const raw = await fs.readFile(indexPath, 'utf-8'); + const data = JSON.parse(raw) as ShardedParseCacheIndex; + if ( + typeof data !== 'object' || + data === null || + data.version !== PARSE_CACHE_VERSION || + !Array.isArray(data.keys) + ) { + return emptyCache(); + } + + const entries = new Map(); + for (const chunkHash of data.keys) { + if (typeof chunkHash !== 'string' || !isValidChunkCacheKey(chunkHash)) continue; + try { + const chunkRaw = await fs.readFile(getCacheChunkPath(storagePath, chunkHash), 'utf-8'); + const chunkData = JSON.parse(chunkRaw, mapReviver) as ParseWorkerResult[]; + if (Array.isArray(chunkData)) entries.set(chunkHash, chunkData); + } catch { + /* skip corrupt or missing shard */ + } + } + + return { version: PARSE_CACHE_VERSION, entries, usedKeys: new Set() }; + } catch { + return null; + } +}; + /** - * Persist the cache to disk atomically (write-and-rename) so a crash - * mid-write doesn't leave a corrupt file. + * Load the parse cache. Returns an empty cache on any failure (missing + * file, corrupt JSON, version mismatch). Never throws on a normal load. + */ +export const loadParseCache = async (storagePath: string): Promise => { + const sharded = await loadShardedParseCache(storagePath); + if (sharded) return sharded; + return loadLegacyParseCache(storagePath); +}; + +/** + * Persist the cache to disk using a temp directory + rename. + * + * Writes shards under `${cacheDir}.tmp`, then removes the old `cacheDir` and + * renames the temp directory into place. There is a crash window after + * `rm(cacheDir)` and before `rename(tmpDir, cacheDir)` where no cache exists; + * that is acceptable — `loadParseCache` yields empty and the next run + * reparses. This is not a single atomic swap of the whole tree, but avoids + * leaving a half-written shard set visible to readers. */ export const saveParseCache = async (storagePath: string, cache: ParseCache): Promise => { await fs.mkdir(storagePath, { recursive: true }); - const cachePath = path.join(storagePath, CACHE_FILENAME); - const tmpPath = `${cachePath}.tmp`; - const out: ParseCacheFile = { + const cacheDir = getCacheDirPath(storagePath); + const tmpDir = `${cacheDir}.tmp`; + await fs.rm(tmpDir, { recursive: true, force: true }); + await fs.mkdir(tmpDir, { recursive: true }); + + const keys: string[] = []; + for (const [chunkHash, chunkResults] of cache.entries) { + if (!isValidChunkCacheKey(chunkHash)) continue; + let payload: string; + try { + payload = JSON.stringify(chunkResults, mapReplacer); + } catch { + // Extremely dense chunks could theoretically exceed string limits; skip + // rather than failing the entire save (orchestrator catches save errors). + continue; + } + keys.push(chunkHash); + const chunkPath = path.join(tmpDir, `${chunkHash}.json`); + await fs.writeFile(chunkPath, payload, 'utf-8'); + } + + const index: ShardedParseCacheIndex = { version: cache.version, - entries: Object.fromEntries(cache.entries), + keys, }; - // Compact JSON; this file can be tens of MB on a large repo and pretty- - // printing roughly doubles size for no value. - await fs.writeFile(tmpPath, JSON.stringify(out, mapReplacer), 'utf-8'); - await fs.rename(tmpPath, cachePath); + await fs.writeFile(path.join(tmpDir, CACHE_INDEX_FILENAME), JSON.stringify(index), 'utf-8'); + + await fs.rm(cacheDir, { recursive: true, force: true }); + await fs.rename(tmpDir, cacheDir); + await fs.rm(getLegacyCachePath(storagePath), { force: true }); }; /** diff --git a/gitnexus/test/unit/cross-file-impl.test.ts b/gitnexus/test/unit/cross-file-impl.test.ts index 55a47a5b5..bf5241d1d 100644 --- a/gitnexus/test/unit/cross-file-impl.test.ts +++ b/gitnexus/test/unit/cross-file-impl.test.ts @@ -49,17 +49,36 @@ vi.mock('../../src/core/tree-sitter/parser-loader.js', async (importOriginal) => }; }); +// Default to non-registry-primary so existing tests (which use .ts files) are +// not affected by the isRegistryPrimary guard added in cross-file-impl. Tests +// that verify the skip behavior can override this with mockReturnValue(true). +vi.mock('../../src/core/ingestion/registry-primary-flag.js', () => ({ + isRegistryPrimary: vi.fn(() => false), +})); + import { runCrossFileBindingPropagation } from '../../src/core/ingestion/pipeline-phases/cross-file-impl.js'; import { processCalls } from '../../src/core/ingestion/call-processor.js'; +import { isRegistryPrimary } from '../../src/core/ingestion/registry-primary-flag.js'; import { createResolutionContext } from '../../src/core/ingestion/model/resolution-context.js'; import { createKnowledgeGraph } from '../../src/core/graph/graph.js'; import type { ExportedTypeMap } from '../../src/core/ingestion/call-processor.js'; const processCallsMock = vi.mocked(processCalls); +const isRegistryPrimaryMock = vi.mocked(isRegistryPrimary); + +/** + * Index of the `compiledQueryCache` parameter in the `processCalls` signature. + * graph(0), files(1), astCache(2), ctx(3), onProgress?(4), exportedTypeMap?(5), + * importedBindingsMap?(6), importedReturnTypesMap?(7), + * importedRawReturnTypesMap?(8), heritageMap?(9), bindingAccumulator?(10), + * compiledQueryCache?(11). + */ +const COMPILED_QUERY_CACHE_ARG_INDEX = 11; describe('runCrossFileBindingPropagation', () => { beforeEach(() => { processCallsMock.mockClear(); + isRegistryPrimaryMock.mockReturnValue(false); // reset to non-primary before each test }); it('returns 0 immediately when namedImportMap is empty', async () => { @@ -162,6 +181,103 @@ describe('runCrossFileBindingPropagation', () => { } }); + it('passes the same compiledQueryCache Map instance to every processCalls call', async () => { + // Verifies that the O(N)→O(1) query-cache fix is correctly wired: the + // `compiledQueryCache` created in runCrossFileBindingPropagation is shared + // across all processCalls invocations so each language's Parser.Query is + // compiled exactly once, not once per file. + const graph = createKnowledgeGraph(); + const ctx = createResolutionContext(); + + const exportedTypeMap: ExportedTypeMap = new Map([ + ['upstream.ts', new Map([['User', 'User']])], + ]); + ctx.importMap.set('upstream.ts', new Set()); + + const allPaths = ['upstream.ts']; + for (let i = 0; i < 3; i++) { + const file = `downstream${i}.ts`; + allPaths.push(file); + const bindings = new Map(); + bindings.set('User', { sourcePath: 'upstream.ts', exportedName: 'User' }); + ctx.namedImportMap.set(file, bindings); + ctx.importMap.set(file, new Set(['upstream.ts'])); + } + + await runCrossFileBindingPropagation( + graph, + ctx, + exportedTypeMap, + new Set(allPaths), + allPaths.length, + '/repo', + Date.now(), + () => {}, + ); + + expect(processCallsMock).toHaveBeenCalledTimes(3); + + // Argument index 11 is compiledQueryCache — see COMPILED_QUERY_CACHE_ARG_INDEX. + const caches = processCallsMock.mock.calls.map((call) => call[COMPILED_QUERY_CACHE_ARG_INDEX]); + // Every call must receive a non-null Map (not undefined). + for (const cache of caches) { + expect(cache).toBeDefined(); + expect(cache).toBeInstanceOf(Map); + } + // All calls share the SAME instance — the whole point of the cache. + expect(caches[1]).toBe(caches[0]); + expect(caches[2]).toBe(caches[0]); + }); + + it('emits live onProgress events every 25 files with N/M format', async () => { + // Verifies that the frozen-progress-display fix is correctly wired: + // onProgress must be called multiple times from the processing loop, + // not just once at phase start, so large repos show real movement in + // the UI instead of a frozen percentage bar. + const graph = createKnowledgeGraph(); + const ctx = createResolutionContext(); + + const exportedTypeMap: ExportedTypeMap = new Map([ + ['upstream.ts', new Map([['User', 'User']])], + ]); + ctx.importMap.set('upstream.ts', new Set()); + + const allPaths = ['upstream.ts']; + for (let i = 0; i < 50; i++) { + const file = `downstream${i}.ts`; + allPaths.push(file); + const bindings = new Map(); + bindings.set('User', { sourcePath: 'upstream.ts', exportedName: 'User' }); + ctx.namedImportMap.set(file, bindings); + ctx.importMap.set(file, new Set(['upstream.ts'])); + } + + const progressMessages: string[] = []; + const onProgress = vi.fn((p: { phase: string; percent: number; message: string }) => { + progressMessages.push(p.message); + }); + + await runCrossFileBindingPropagation( + graph, + ctx, + exportedTypeMap, + new Set(allPaths), + allPaths.length, + '/repo', + Date.now(), + onProgress, + ); + + // 1 initial call at phase start + 2 loop calls (at 25 and 50 files). + expect(onProgress).toHaveBeenCalledTimes(3); + + // Loop messages must carry the "N/M files" format so the UI is informative. + const loopMessages = progressMessages.filter((m) => m.match(/\(\d+\/\d+ files\)/)); + expect(loopMessages).toHaveLength(2); + expect(loopMessages[0]).toContain('(25/50 files)'); + expect(loopMessages[1]).toContain('(50/50 files)'); + }); + it('caps processing at MAX_CROSS_FILE_REPROCESS (2000)', async () => { const graph = createKnowledgeGraph(); const ctx = createResolutionContext(); @@ -203,4 +319,47 @@ describe('runCrossFileBindingPropagation', () => { expect(result).toBe(2000); expect(processCallsMock).toHaveBeenCalledTimes(2000); }); + + it('skips registry-primary language files without calling processCalls', async () => { + // Finding 3: on large TypeScript/C++ repos (registry-primary since v1.6.4+) + // cross-file-impl was calling processCalls 595× per candidate only for + // processCalls to immediately return (isRegistryPrimary guard inside). + // Now cross-file-impl filters them out BEFORE readFileContents so we avoid + // the I/O cost and map-building overhead entirely. + const graph = createKnowledgeGraph(); + const ctx = createResolutionContext(); + + const exportedTypeMap: ExportedTypeMap = new Map([ + ['upstream.ts', new Map([['User', 'User']])], + ]); + ctx.importMap.set('upstream.ts', new Set()); + + const allPaths = ['upstream.ts']; + for (let i = 0; i < 5; i++) { + const file = `downstream${i}.ts`; + allPaths.push(file); + const bindings = new Map(); + bindings.set('User', { sourcePath: 'upstream.ts', exportedName: 'User' }); + ctx.namedImportMap.set(file, bindings); + ctx.importMap.set(file, new Set(['upstream.ts'])); + } + + // Simulate all files being registry-primary (e.g. TypeScript on main branch). + isRegistryPrimaryMock.mockReturnValue(true); + + const result = await runCrossFileBindingPropagation( + graph, + ctx, + exportedTypeMap, + new Set(allPaths), + allPaths.length, + '/repo', + Date.now(), + () => {}, + ); + + // No files are candidates; no processCalls invocations. + expect(result).toBe(0); + expect(processCallsMock).not.toHaveBeenCalled(); + }); }); diff --git a/gitnexus/test/unit/incremental-parse-cache.test.ts b/gitnexus/test/unit/incremental-parse-cache.test.ts index 757b9cf3a..17ec9c2e1 100644 --- a/gitnexus/test/unit/incremental-parse-cache.test.ts +++ b/gitnexus/test/unit/incremental-parse-cache.test.ts @@ -135,12 +135,15 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { it('round-trips an empty cache', async () => { const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { + const fs = await import('fs/promises'); const cache: ParseCache = { version: PARSE_CACHE_VERSION, entries: new Map(), usedKeys: new Set(), }; await saveParseCache(dir, cache); + await expect(fs.access(path.join(dir, 'parse-cache', 'index.json'))).resolves.toBeUndefined(); + await expect(fs.access(path.join(dir, 'parse-cache.json'))).rejects.toThrow(); const loaded = await loadParseCache(dir); expect(loaded.version).toBe(PARSE_CACHE_VERSION); expect(loaded.entries.size).toBe(0); @@ -189,6 +192,60 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { } }); + it('loads a legacy single-file cache for backwards compatibility', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { + legacyChunk: [minimalResult({ fileCount: 7 })], + }, + }), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get('legacyChunk')?.[0]?.fileCount).toBe(7); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('skips corrupt or missing shards while loading the sharded cache', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + const goodKey = 'a'.repeat(64); + const missingKey = 'b'.repeat(64); + const badKey = 'c'.repeat(64); + await fs.mkdir(cacheDir, { recursive: true }); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + keys: [goodKey, missingKey, badKey], + }), + 'utf-8', + ); + await fs.writeFile( + path.join(cacheDir, `${goodKey}.json`), + JSON.stringify([minimalResult({ fileCount: 3 })]), + 'utf-8', + ); + await fs.writeFile(path.join(cacheDir, `${badKey}.json`), '{not-json', 'utf-8'); + + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(goodKey)?.[0]?.fileCount).toBe(3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + it('round-trips Map and Set values through the JSON replacer/reviver', async () => { // ParsedFile.scopes[*].typeBindings is a ReadonlyMap. // Without the replacer/reviver pair, JSON.stringify collapses Maps to @@ -196,6 +253,7 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { // with "is not iterable". This test pins the round-trip behaviour. const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); try { + const fs = await import('fs/promises'); const innerMap = new Map([ ['k1', 'v1'], ['k2', 'v2'], @@ -218,14 +276,18 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { ], }); + const chunkKey = 'd'.repeat(64); const cache: ParseCache = { version: PARSE_CACHE_VERSION, - entries: new Map([['chunk-h', [fake]]]), - usedKeys: new Set(['chunk-h']), + entries: new Map([[chunkKey, [fake]]]), + usedKeys: new Set([chunkKey]), }; await saveParseCache(dir, cache); + const persisted = await fs.readdir(path.join(dir, 'parse-cache')); + expect(persisted).toContain('index.json'); + expect(persisted).toContain(`${chunkKey}.json`); const loaded = await loadParseCache(dir); - const reloaded = loaded.entries.get('chunk-h')?.[0]; + const reloaded = loaded.entries.get(chunkKey)?.[0]; expect(reloaded).toBeDefined(); const scope = (reloaded as ParseWorkerResult).parsedFiles[0]?.scopes[0] as unknown as { typeBindings?: unknown; @@ -240,4 +302,140 @@ describe('loadParseCache / saveParseCache (round-trip)', () => { await rm(dir, { recursive: true, force: true }); } }); + + it('ignores traversal-like and non-hex keys in sharded index.json', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + await fs.mkdir(cacheDir, { recursive: true }); + const safeKey = 'e'.repeat(64); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + keys: ['../evil', '/absolute', 'G'.repeat(64), safeKey], + }), + 'utf-8', + ); + await fs.writeFile( + path.join(cacheDir, `${safeKey}.json`), + JSON.stringify([minimalResult({ fileCount: 9 })]), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(safeKey)?.[0]?.fileCount).toBe(9); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('writes one shard file per cache entry (three distinct keys)', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const k1 = '1'.repeat(64); + const k2 = '2'.repeat(64); + const k3 = '3'.repeat(64); + const cache: ParseCache = { + version: PARSE_CACHE_VERSION, + entries: new Map([ + [k1, [minimalResult({ fileCount: 1 })]], + [k2, [minimalResult({ fileCount: 2 })]], + [k3, [minimalResult({ fileCount: 3 })]], + ]), + usedKeys: new Set([k1, k2, k3]), + }; + await saveParseCache(dir, cache); + const cacheDir = path.join(dir, 'parse-cache'); + const names = await fs.readdir(cacheDir); + expect(names).toContain('index.json'); + expect(names.filter((n) => n.endsWith('.json') && n !== 'index.json').length).toBe(3); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(3); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('returns empty when sharded index version mismatches even if legacy parse-cache.json is valid', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const cacheDir = path.join(dir, 'parse-cache'); + await fs.mkdir(cacheDir, { recursive: true }); + await fs.writeFile( + path.join(cacheDir, 'index.json'), + JSON.stringify({ version: 'foreign-sharded-1', keys: [] }), + 'utf-8', + ); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { legacyChunk: [minimalResult({ fileCount: 42 })] }, + }), + 'utf-8', + ); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(0); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('second saveParseCache replaces the first sharded cache', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + const k1 = '4'.repeat(64); + const k2 = '5'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k1, [minimalResult()]]]), + usedKeys: new Set([k1]), + }); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k2, [minimalResult({ fileCount: 99 })]]]), + usedKeys: new Set([k2]), + }); + const names = await fs.readdir(path.join(dir, 'parse-cache')); + expect(names).not.toContain(`${k1}.json`); + expect(names).toContain(`${k2}.json`); + const loaded = await loadParseCache(dir); + expect(loaded.entries.size).toBe(1); + expect(loaded.entries.get(k2)?.[0]?.fileCount).toBe(99); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); + + it('removes legacy parse-cache.json after a successful sharded save', async () => { + const dir = await mkdtemp(path.join(tmpdir(), 'gnx-pc-')); + try { + const fs = await import('fs/promises'); + await fs.writeFile( + path.join(dir, 'parse-cache.json'), + JSON.stringify({ + version: PARSE_CACHE_VERSION, + entries: { oldLegacy: [minimalResult({ fileCount: 5 })] }, + }), + 'utf-8', + ); + const k = '6'.repeat(64); + await saveParseCache(dir, { + version: PARSE_CACHE_VERSION, + entries: new Map([[k, [minimalResult({ fileCount: 6 })]]]), + usedKeys: new Set([k]), + }); + await expect(fs.access(path.join(dir, 'parse-cache.json'))).rejects.toThrow(); + const loaded = await loadParseCache(dir); + expect(loaded.entries.get(k)?.[0]?.fileCount).toBe(6); + expect(loaded.entries.has('oldLegacy')).toBe(false); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); });