feat(ci): vendored tree-sitter grammar update monitor

Adds a weekly (+ dispatchable) workflow that checks each vendored grammar against
its source-of-origin (npm for swift/kotlin, the GitHub default branch for
dart/proto; c is excluded — held at 0.21.4 for ABI safety) and opens a PR
re-vendoring any update that is ABI-COMPATIBLE with the pinned tree-sitter@0.21.1
(LANGUAGE_VERSION 13-14).

ABI awareness is the point: most upstreams have moved to ABI 15 (newer
tree-sitter), so a blind "bump to latest" would open PRs that can't build. The
monitor fetches the candidate source, reads its parser.c LANGUAGE_VERSION, and
only re-vendors 13/14 — incompatible updates are reported (notice + job summary),
never applied. (Confirmed live: dart/proto upstreams are ABI 15 today and are
correctly held; swift/kotlin are current.)

The re-vendor refreshes only the source-build inputs + runtime entrypoints,
preserving the GitNexus-hardened binding.gyp / README / prebuilds; the version
bump then triggers build-tree-sitter-prebuilds.yml, whose ABI-validation is the
final safety net so a subtly-wrong re-vendor can't silently ship. PR creation is
gated on the RELEASE_APP secret (skips with a notice if absent), mirroring the
build aggregate. Unit test locks the ABI gate; the script is import-safe.

.github/scripts/update-vendored-grammars.mjs

@@ -0,0 +1,240 @@
+#!/usr/bin/env node
+/**
+ * Vendored tree-sitter grammar update monitor.
+ *
+ * Checks each vendored grammar against its upstream source-of-origin and, for an
+ * available AND ABI-compatible update, re-vendors the grammar source in place so
+ * a PR can be opened. The version bump in vendor/<name>/package.json then triggers
+ * .github/workflows/build-tree-sitter-prebuilds.yml, which cross-builds + ABI-
+ * validates the prebuilds — so even an imperfect re-vendor can never silently
+ * ship: its PR's CI goes red.
+ *
+ * ABI awareness is load-bearing. Every grammar is pinned to tree-sitter@0.21.1
+ * (LANGUAGE_VERSION 13–14, the #1922 gate). Most upstream grammar releases target
+ * a newer tree-sitter, so a blind "bump to latest" would pull an ABI-incompatible
+ * parser and open doomed PRs. This monitor fetches the candidate source, reads its
+ * parser.c `#define LANGUAGE_VERSION`, and only re-vendors when it is 13 or 14;
+ * incompatible updates are reported (and surfaced as a workflow notice), not
+ * applied.
+ *
+ * Usage:
+ *   node update-vendored-grammars.mjs            # detect only → JSON report on stdout
+ *   node update-vendored-grammars.mjs --apply X  # re-vendor grammar X in place
+ *
+ * tree-sitter-c is intentionally absent: it is HELD at 0.21.4 for ABI safety
+ * (#1242) and must never be auto-bumped.
+ */
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(__dirname, '..', '..');
+const VENDOR = path.join(REPO_ROOT, 'gitnexus', 'vendor');
+
+const COMPATIBLE_ABI = new Set([13, 14]); // tree-sitter@0.21.1 LANGUAGE_VERSION range
+
+// Source-of-origin per grammar. npm grammars resolve `latest` via the registry;
+// github grammars (no usable npm release) track the default branch HEAD.
+const GRAMMARS = {
+  swift: { name: 'tree-sitter-swift', npm: 'tree-sitter-swift' },
+  kotlin: { name: 'tree-sitter-kotlin', npm: 'tree-sitter-kotlin' },
+  dart: { name: 'tree-sitter-dart', github: 'UserNobody14/tree-sitter-dart' },
+  proto: { name: 'tree-sitter-proto', github: 'coder3101/tree-sitter-proto' },
+};
+
+const sh = (cmd, args, opts = {}) =>
+  execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }).trim();
+
+const clean = (v) =>
+  String(v || '')
+    .replace(/^[v^~]/, '')
+    .trim();
+
+function vendoredVersion(g) {
+  const p = path.join(VENDOR, g.name, 'package.json');
+  return clean(JSON.parse(fs.readFileSync(p, 'utf8')).version);
+}
+
+/** Resolve the upstream candidate: { version, ref, kind }. */
+function resolveUpstream(g) {
+  if (g.npm) {
+    const version = clean(sh('npm', ['view', g.npm, 'version']));
+    return { version, ref: version, kind: 'npm' };
+  }
+  // github: no reliable release tags here, so track the default branch HEAD sha.
+  const meta = JSON.parse(sh('gh', ['api', `repos/${g.github}`]));
+  const branch = meta.default_branch;
+  const sha = JSON.parse(sh('gh', ['api', `repos/${g.github}/commits/${branch}`])).sha;
+  // Version key: "<upstreamPkgVersion>-g<sha7>" — safeRef-compatible (no `+`,
+  // which the build workflow's ref validator rejects) and changes on every commit.
+  let base = '0.0.0';
+  try {
+    const pkg = JSON.parse(
+      Buffer.from(
+        JSON.parse(sh('gh', ['api', `repos/${g.github}/contents/package.json?ref=${sha}`])).content,
+        'base64',
+      ).toString('utf8'),
+    );
+    if (pkg.version) base = clean(pkg.version);
+  } catch {
+    /* no upstream package.json — base stays 0.0.0 */
+  }
+  return { version: `${base}-g${sha.slice(0, 7)}`, ref: sha, kind: 'github' };
+}
+
+/** Fetch the candidate source into a temp dir; return the package root. */
+function fetchSource(g, ref) {
+  const work = fs.mkdtempSync(
+    path.join(os.tmpdir(), `revendor-${Object.keys(GRAMMARS).find((k) => GRAMMARS[k] === g)}-`),
+  );
+  if (g.npm) {
+    sh('npm', ['pack', `${g.npm}@${ref}`, '--silent'], { cwd: work });
+    const tgz = fs.readdirSync(work).find((f) => f.endsWith('.tgz'));
+    sh('tar', ['xzf', tgz], { cwd: work });
+    return path.join(work, 'package');
+  }
+  // github tarball at the resolved sha
+  sh('bash', ['-c', `gh api repos/${g.github}/tarball/${ref} > src.tgz && tar xzf src.tgz`], {
+    cwd: work,
+  });
+  const dir = fs.readdirSync(work).find((f) => fs.statSync(path.join(work, f)).isDirectory());
+  return path.join(work, dir);
+}
+
+/** Read parser.c's LANGUAGE_VERSION (ABI). Prefer the ABI-14 default parser.c. */
+function readAbi(srcRoot) {
+  const candidates = ['src/parser.c', 'parser.c'];
+  for (const rel of candidates) {
+    const p = path.join(srcRoot, rel);
+    if (!fs.existsSync(p)) continue;
+    // Read only the head — the #define is near the top.
+    const head = fs.readFileSync(p, 'utf8').slice(0, 4000);
+    const m = head.match(/#define\s+LANGUAGE_VERSION\s+(\d+)/);
+    if (m) return Number(m[1]);
+  }
+  return null; // unknown (e.g. parser.c only generated at build time)
+}
+
+function detect() {
+  const report = [];
+  for (const [key, g] of Object.entries(GRAMMARS)) {
+    const have = vendoredVersion(g);
+    let up;
+    try {
+      up = resolveUpstream(g);
+    } catch (err) {
+      report.push({ grammar: key, error: String(err.message || err) });
+      continue;
+    }
+    const newer = up.kind === 'npm' ? up.version !== have : !have || up.ref.slice(0, 7) !== have;
+    let abi = null;
+    if (newer) {
+      try {
+        abi = readAbi(fetchSource(g, up.ref));
+      } catch {
+        /* fetch/abi best-effort; null = unknown */
+      }
+    }
+    report.push({
+      grammar: key,
+      vendored: have,
+      upstream: up.version,
+      ref: up.ref,
+      kind: up.kind,
+      update: newer,
+      abi,
+      abiCompatible: abi == null ? null : COMPATIBLE_ABI.has(abi),
+      // Only auto-appliable when there's an update AND the ABI is known-compatible.
+      applicable: newer && abi != null && COMPATIBLE_ABI.has(abi),
+    });
+  }
+  return report;
+}
+
+const copyFile = (srcRoot, dest, rel) => {
+  const from = path.join(srcRoot, rel);
+  if (!fs.existsSync(from)) return false;
+  const to = path.join(dest, rel);
+  fs.mkdirSync(path.dirname(to), { recursive: true });
+  fs.copyFileSync(from, to);
+  return true;
+};
+
+/**
+ * Re-vendor one grammar in place from its ABI-compatible upstream candidate.
+ * Copies ONLY the generated source-build + runtime files; deliberately KEEPS the
+ * GitNexus-hardened binding.gyp (Windows cflags, target_name), README (vendor
+ * notice), LICENSE, and prebuilds/ (the build workflow refreshes those). Bumps the
+ * stripped vendor package.json version + provenance — never re-introduces
+ * scripts/dependencies (#836/#1728). Returns the new version.
+ */
+function apply(key) {
+  const g = GRAMMARS[key];
+  if (!g) {
+    console.error(`unknown grammar '${key}'`);
+    process.exit(2);
+  }
+  const have = vendoredVersion(g);
+  const up = resolveUpstream(g);
+  const newer = up.kind === 'npm' ? up.version !== have : !have || up.version !== have;
+  if (!newer) {
+    console.error(`${key}: already current (${have}); nothing to apply.`);
+    process.exit(0);
+  }
+  const srcRoot = fetchSource(g, up.ref);
+  const abi = readAbi(srcRoot);
+  if (abi == null || !COMPATIBLE_ABI.has(abi)) {
+    console.error(
+      `${key}: candidate ${up.version} is ABI ${abi ?? 'unknown'} — not tree-sitter@0.21.1 ` +
+        `compatible (need 13/14); refusing to re-vendor. Handle manually.`,
+    );
+    process.exit(3);
+  }
+
+  const dest = path.join(VENDOR, g.name);
+  // The source-build inputs + runtime entrypoints that change between versions.
+  // binding.gyp / README / LICENSE / prebuilds are intentionally NOT touched.
+  for (const rel of [
+    'src/parser.c',
+    'src/scanner.c',
+    'src/node-types.json',
+    'src/tree_sitter/alloc.h',
+    'src/tree_sitter/array.h',
+    'src/tree_sitter/parser.h',
+    'bindings/node/binding.cc',
+    'bindings/node/index.js',
+    'bindings/node/index.d.ts',
+  ]) {
+    copyFile(srcRoot, dest, rel);
+  }
+
+  const pkgPath = path.join(dest, 'package.json');
+  const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+  pkg.version = up.version;
+  pkg._vendoredBy =
+    `gitnexus - re-vendored from ${g.npm ? `npm ${g.npm}@${up.version}` : `${g.github}@${up.ref}`} ` +
+    `by grammar-update-monitor on ABI ${abi}. Source-build inputs (parser.c/scanner.c/src/) refreshed; ` +
+    `the GitNexus-hardened binding.gyp + vendor README + prebuilds are preserved (prebuilds are ` +
+    `rebuilt by build-tree-sitter-prebuilds.yml on this version change). No scripts/dependencies here ` +
+    `(#836/#1728).`;
+  fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n');
+
+  console.log(`${key}: re-vendored ${g.name} → ${up.version} (ABI ${abi}).`);
+  return up.version;
+}
+
+// Run the CLI only when invoked directly (not when imported by a test) — detect()
+// makes live network calls, so importing must be side-effect-free.
+const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isMain) {
+  if (process.argv[2] === '--apply') {
+    apply(process.argv[3]);
+  } else {
+    process.stdout.write(JSON.stringify(detect(), null, 2) + '\n');
+  }
+}
+
+export { detect, apply, resolveUpstream, readAbi, vendoredVersion, GRAMMARS, COMPATIBLE_ABI };

.github/workflows/grammar-update-monitor.yml

@@ -0,0 +1,144 @@
+name: Vendored grammar update monitor
+
+# Periodically checks each vendored tree-sitter grammar against its
+# source-of-origin and opens a PR re-vendoring any update that is ABI-COMPATIBLE
+# with the pinned tree-sitter@0.21.1 (LANGUAGE_VERSION 13–14, #1922). The version
+# bump then triggers build-tree-sitter-prebuilds.yml, which cross-builds + ABI-
+# validates the prebuilds — so a re-vendor that is subtly wrong can never silently
+# ship: its PR's CI goes red.
+#
+# ABI-INCOMPATIBLE updates (the common case — upstreams move to newer tree-sitter)
+# are reported as a notice + job summary, NOT applied, so the monitor never opens
+# doomed PRs. tree-sitter-c is excluded entirely: it is HELD at 0.21.4 for ABI
+# safety (#1242) and must never be auto-bumped.
+#
+# Concurrency convention: see CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention".
+
+on:
+  schedule:
+    - cron: '17 6 * * 1' # weekly, Monday 06:17 UTC
+  workflow_dispatch:
+
+# Least privilege; the actual writes use a short-lived App token minted below.
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: false
+
+jobs:
+  monitor:
+    name: Check upstreams + open update PRs
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 22
+
+      # secrets aren't usable in a job/step `if:`, so compute presence here.
+      - name: Check release App secret
+        id: relapp
+        env:
+          HAS_APP: ${{ secrets.RELEASE_APP_ID != '' && secrets.RELEASE_APP_PRIVATE_KEY != '' }}
+        run: echo "configured=$HAS_APP" >> "$GITHUB_OUTPUT"
+
+      - name: Mint GitHub App token
+        id: app-token
+        if: steps.relapp.outputs.configured == 'true'
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+
+      - name: Detect updates, re-vendor ABI-compatible ones, open PRs
+        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+        env:
+          HAS_APP: ${{ steps.relapp.outputs.configured }}
+          # App token writes; falls back to the read-only job token (PRs then skip).
+          GH_TOKEN: ${{ steps.app-token.outputs.token || github.token }}
+        with:
+          github-token: ${{ steps.app-token.outputs.token || github.token }}
+          script: |
+            const { execFileSync } = require('node:child_process');
+            const SCRIPT = '.github/scripts/update-vendored-grammars.mjs';
+            const run = (cmd, args, opts = {}) =>
+              execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts });
+
+            const report = JSON.parse(run('node', [SCRIPT]));
+            const { owner, repo } = context.repo;
+            const hasApp = process.env.HAS_APP === 'true';
+            const applied = [], held = [], errors = [], skipped = [];
+
+            run('git', ['config', 'user.name', 'gitnexus-release-bot[bot]']);
+            run('git', ['config', 'user.email', 'gitnexus-release-bot[bot]@users.noreply.github.com']);
+            const baseSha = run('git', ['rev-parse', 'HEAD']).trim();
+
+            for (const r of report) {
+              if (r.error) { errors.push(r); continue; }
+              if (!r.update) continue;
+              if (!r.applicable) { held.push(r); continue; } // ABI-incompatible / unknown
+
+              const name = `tree-sitter-${r.grammar}`;
+              const branch = `chore/update-${name}-${r.upstream}`.replace(/[^a-z0-9._/-]+/gi, '-');
+
+              // Idempotency: don't reopen an existing PR for this exact version.
+              const existing = await github.rest.pulls.list({ owner, repo, head: `${owner}:${branch}`, state: 'all' });
+              if (existing.data.length > 0) { skipped.push({ ...r, reason: 'PR exists' }); continue; }
+
+              // Re-vendor in place (refuses + exits non-zero if ABI turns out wrong).
+              try {
+                run('node', [SCRIPT, '--apply', r.grammar]);
+              } catch (e) {
+                errors.push({ ...r, error: `apply failed: ${String(e.message || e).slice(0, 200)}` });
+                run('git', ['checkout', '--', 'gitnexus/vendor']);
+                continue;
+              }
+
+              if (!hasApp) {
+                skipped.push({ ...r, reason: 'no RELEASE_APP secret — PR not opened' });
+                run('git', ['checkout', '--', 'gitnexus/vendor']);
+                continue;
+              }
+
+              const remote = `https://x-access-token:${process.env.GH_TOKEN}@github.com/${owner}/${repo}.git`;
+              run('git', ['checkout', '-B', branch, baseSha]);
+              run('git', ['add', `gitnexus/vendor/${name}`]);
+              run('git', ['commit', '-m', `chore(vendor): update ${name} to ${r.upstream}`]);
+              run('git', ['push', '--force-with-lease', remote, `HEAD:${branch}`]);
+              const body = [
+                `Automated re-vendor of **${name}** to \`${r.upstream}\` (from ${r.kind === 'npm' ? `npm \`${name}\`` : `\`${r.ref}\``}).`,
+                '',
+                `Verified ABI **${r.abi}** — compatible with the pinned \`tree-sitter@0.21.1\` (13–14).`,
+                'Source-build inputs refreshed; the GitNexus binding.gyp / README / prebuilds are preserved.',
+                'The version bump triggers `build-tree-sitter-prebuilds.yml` to rebuild + ABI-validate the',
+                'prebuilds — review its result before merging.',
+              ].join('\n');
+              const pr = await github.rest.pulls.create({
+                owner, repo, head: branch, base: 'main',
+                title: `chore(vendor): update ${name} to ${r.upstream}`, body,
+              });
+              applied.push({ ...r, pr: pr.data.number });
+              run('git', ['checkout', '--force', baseSha]);
+            }
+
+            // Summary
+            const s = core.summary.addHeading('Vendored grammar update monitor');
+            if (applied.length) s.addRaw(`\n**Opened PRs:** ${applied.map((a) => `${a.grammar}→${a.upstream} (#${a.pr})`).join(', ')}\n`);
+            if (held.length) s.addRaw(`\n**Held (ABI-incompatible / unknown — needs the tree-sitter runtime upgrade):** ${held.map((h) => `${h.grammar} ${h.upstream} (ABI ${h.abi ?? '?'})`).join(', ')}\n`);
+            if (skipped.length) s.addRaw(`\n**Skipped:** ${skipped.map((x) => `${x.grammar} (${x.reason})`).join(', ')}\n`);
+            if (errors.length) s.addRaw(`\n**Errors:** ${errors.map((e) => `${e.grammar}: ${e.error}`).join('; ')}\n`);
+            if (!applied.length && !held.length && !skipped.length && !errors.length) s.addRaw('\nAll vendored grammars are up to date. ✅\n');
+            await s.write();
+
+            for (const h of held) core.notice(`${h.grammar}: update to ${h.upstream} available but ABI ${h.abi ?? 'unknown'} (need 13/14) — held until the tree-sitter runtime upgrade.`);
+            if (!hasApp && (applied.length || skipped.some((x) => /secret/.test(x.reason)))) {
+              core.notice('RELEASE_APP_ID / RELEASE_APP_PRIVATE_KEY not configured — update PRs were not opened. Provision the App to enable auto-PRs.');
+            }

gitnexus/test/unit/grammar-update-monitor.test.ts

@@ -0,0 +1,71 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+/**
+ * Unit coverage for the ABI gate in the vendored-grammar update monitor
+ * (.github/scripts/update-vendored-grammars.mjs). The gate is load-bearing: every
+ * grammar is pinned to tree-sitter@0.21.1 (LANGUAGE_VERSION 13–14), so an update
+ * is only auto-applied when the candidate parser.c's ABI is 13 or 14 — otherwise
+ * the monitor would open PRs that can't build. We test the pure pieces (no
+ * network): reading the ABI from a parser.c and the compatibility set. The module
+ * is import-safe (its CLI is guarded behind an isMain check).
+ */
+const MOD = pathToFileURL(
+  path.resolve(
+    path.dirname(fileURLToPath(import.meta.url)),
+    '../../../.github/scripts/update-vendored-grammars.mjs',
+  ),
+).href;
+
+let mod: {
+  readAbi: (root: string) => number | null;
+  COMPATIBLE_ABI: Set<number>;
+  GRAMMARS: Record<string, { name: string; npm?: string; github?: string }>;
+};
+let tmp: string;
+
+beforeAll(async () => {
+  mod = await import(MOD);
+  tmp = mkdtempSync(path.join(tmpdir(), 'gum-'));
+});
+afterAll(() => rmSync(tmp, { recursive: true, force: true }));
+
+function fixture(abiLine: string): string {
+  const root = mkdtempSync(path.join(tmp, 'g-'));
+  mkdirSync(path.join(root, 'src'), { recursive: true });
+  writeFileSync(path.join(root, 'src', 'parser.c'), `${abiLine}\n#define STATE_COUNT 10\n`);
+  return root;
+}
+
+describe('readAbi', () => {
+  it('reads LANGUAGE_VERSION 14 from src/parser.c', () => {
+    expect(mod.readAbi(fixture('#define LANGUAGE_VERSION 14'))).toBe(14);
+  });
+  it('reads LANGUAGE_VERSION 15 (an incompatible upstream)', () => {
+    expect(mod.readAbi(fixture('#define LANGUAGE_VERSION 15'))).toBe(15);
+  });
+  it('returns null when parser.c is absent (generated-at-build-time grammars)', () => {
+    expect(mod.readAbi(mkdtempSync(path.join(tmp, 'empty-')))).toBeNull();
+  });
+});
+
+describe('COMPATIBLE_ABI gate', () => {
+  it('accepts ABI 13 and 14, rejects 12 and 15', () => {
+    expect(mod.COMPATIBLE_ABI.has(13)).toBe(true);
+    expect(mod.COMPATIBLE_ABI.has(14)).toBe(true);
+    expect(mod.COMPATIBLE_ABI.has(12)).toBe(false);
+    expect(mod.COMPATIBLE_ABI.has(15)).toBe(false);
+  });
+});
+
+describe('GRAMMARS registry', () => {
+  it('covers swift/kotlin (npm) + dart/proto (github) and EXCLUDES the ABI-pinned c', () => {
+    expect(Object.keys(mod.GRAMMARS).sort()).toEqual(['dart', 'kotlin', 'proto', 'swift']);
+    expect(mod.GRAMMARS.swift.npm).toBe('tree-sitter-swift');
+    expect(mod.GRAMMARS.dart.github).toContain('tree-sitter-dart');
+    expect(mod.GRAMMARS).not.toHaveProperty('c');
+  });
+});