From 7023b63605b475c4c53c644f7aafb3623577a0da Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 9 Jun 2026 15:57:41 +0000 Subject: [PATCH] feat(ci): vendored tree-sitter grammar update monitor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a weekly (+ dispatchable) workflow that checks each vendored grammar against its source-of-origin (npm for swift/kotlin, the GitHub default branch for dart/proto; c is excluded — held at 0.21.4 for ABI safety) and opens a PR re-vendoring any update that is ABI-COMPATIBLE with the pinned tree-sitter@0.21.1 (LANGUAGE_VERSION 13-14). ABI awareness is the point: most upstreams have moved to ABI 15 (newer tree-sitter), so a blind "bump to latest" would open PRs that can't build. The monitor fetches the candidate source, reads its parser.c LANGUAGE_VERSION, and only re-vendors 13/14 — incompatible updates are reported (notice + job summary), never applied. (Confirmed live: dart/proto upstreams are ABI 15 today and are correctly held; swift/kotlin are current.) The re-vendor refreshes only the source-build inputs + runtime entrypoints, preserving the GitNexus-hardened binding.gyp / README / prebuilds; the version bump then triggers build-tree-sitter-prebuilds.yml, whose ABI-validation is the final safety net so a subtly-wrong re-vendor can't silently ship. PR creation is gated on the RELEASE_APP secret (skips with a notice if absent), mirroring the build aggregate. Unit test locks the ABI gate; the script is import-safe. --- .github/scripts/update-vendored-grammars.mjs | 240 ++++++++++++++++++ .github/workflows/grammar-update-monitor.yml | 144 +++++++++++ .../test/unit/grammar-update-monitor.test.ts | 71 ++++++ 3 files changed, 455 insertions(+) create mode 100644 .github/scripts/update-vendored-grammars.mjs create mode 100644 .github/workflows/grammar-update-monitor.yml create mode 100644 gitnexus/test/unit/grammar-update-monitor.test.ts diff --git a/.github/scripts/update-vendored-grammars.mjs b/.github/scripts/update-vendored-grammars.mjs new file mode 100644 index 000000000..2b0151713 --- /dev/null +++ b/.github/scripts/update-vendored-grammars.mjs @@ -0,0 +1,240 @@ +#!/usr/bin/env node +/** + * Vendored tree-sitter grammar update monitor. + * + * Checks each vendored grammar against its upstream source-of-origin and, for an + * available AND ABI-compatible update, re-vendors the grammar source in place so + * a PR can be opened. The version bump in vendor//package.json then triggers + * .github/workflows/build-tree-sitter-prebuilds.yml, which cross-builds + ABI- + * validates the prebuilds — so even an imperfect re-vendor can never silently + * ship: its PR's CI goes red. + * + * ABI awareness is load-bearing. Every grammar is pinned to tree-sitter@0.21.1 + * (LANGUAGE_VERSION 13–14, the #1922 gate). Most upstream grammar releases target + * a newer tree-sitter, so a blind "bump to latest" would pull an ABI-incompatible + * parser and open doomed PRs. This monitor fetches the candidate source, reads its + * parser.c `#define LANGUAGE_VERSION`, and only re-vendors when it is 13 or 14; + * incompatible updates are reported (and surfaced as a workflow notice), not + * applied. + * + * Usage: + * node update-vendored-grammars.mjs # detect only → JSON report on stdout + * node update-vendored-grammars.mjs --apply X # re-vendor grammar X in place + * + * tree-sitter-c is intentionally absent: it is HELD at 0.21.4 for ABI safety + * (#1242) and must never be auto-bumped. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, '..', '..'); +const VENDOR = path.join(REPO_ROOT, 'gitnexus', 'vendor'); + +const COMPATIBLE_ABI = new Set([13, 14]); // tree-sitter@0.21.1 LANGUAGE_VERSION range + +// Source-of-origin per grammar. npm grammars resolve `latest` via the registry; +// github grammars (no usable npm release) track the default branch HEAD. +const GRAMMARS = { + swift: { name: 'tree-sitter-swift', npm: 'tree-sitter-swift' }, + kotlin: { name: 'tree-sitter-kotlin', npm: 'tree-sitter-kotlin' }, + dart: { name: 'tree-sitter-dart', github: 'UserNobody14/tree-sitter-dart' }, + proto: { name: 'tree-sitter-proto', github: 'coder3101/tree-sitter-proto' }, +}; + +const sh = (cmd, args, opts = {}) => + execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }).trim(); + +const clean = (v) => + String(v || '') + .replace(/^[v^~]/, '') + .trim(); + +function vendoredVersion(g) { + const p = path.join(VENDOR, g.name, 'package.json'); + return clean(JSON.parse(fs.readFileSync(p, 'utf8')).version); +} + +/** Resolve the upstream candidate: { version, ref, kind }. */ +function resolveUpstream(g) { + if (g.npm) { + const version = clean(sh('npm', ['view', g.npm, 'version'])); + return { version, ref: version, kind: 'npm' }; + } + // github: no reliable release tags here, so track the default branch HEAD sha. + const meta = JSON.parse(sh('gh', ['api', `repos/${g.github}`])); + const branch = meta.default_branch; + const sha = JSON.parse(sh('gh', ['api', `repos/${g.github}/commits/${branch}`])).sha; + // Version key: "-g" — safeRef-compatible (no `+`, + // which the build workflow's ref validator rejects) and changes on every commit. + let base = '0.0.0'; + try { + const pkg = JSON.parse( + Buffer.from( + JSON.parse(sh('gh', ['api', `repos/${g.github}/contents/package.json?ref=${sha}`])).content, + 'base64', + ).toString('utf8'), + ); + if (pkg.version) base = clean(pkg.version); + } catch { + /* no upstream package.json — base stays 0.0.0 */ + } + return { version: `${base}-g${sha.slice(0, 7)}`, ref: sha, kind: 'github' }; +} + +/** Fetch the candidate source into a temp dir; return the package root. */ +function fetchSource(g, ref) { + const work = fs.mkdtempSync( + path.join(os.tmpdir(), `revendor-${Object.keys(GRAMMARS).find((k) => GRAMMARS[k] === g)}-`), + ); + if (g.npm) { + sh('npm', ['pack', `${g.npm}@${ref}`, '--silent'], { cwd: work }); + const tgz = fs.readdirSync(work).find((f) => f.endsWith('.tgz')); + sh('tar', ['xzf', tgz], { cwd: work }); + return path.join(work, 'package'); + } + // github tarball at the resolved sha + sh('bash', ['-c', `gh api repos/${g.github}/tarball/${ref} > src.tgz && tar xzf src.tgz`], { + cwd: work, + }); + const dir = fs.readdirSync(work).find((f) => fs.statSync(path.join(work, f)).isDirectory()); + return path.join(work, dir); +} + +/** Read parser.c's LANGUAGE_VERSION (ABI). Prefer the ABI-14 default parser.c. */ +function readAbi(srcRoot) { + const candidates = ['src/parser.c', 'parser.c']; + for (const rel of candidates) { + const p = path.join(srcRoot, rel); + if (!fs.existsSync(p)) continue; + // Read only the head — the #define is near the top. + const head = fs.readFileSync(p, 'utf8').slice(0, 4000); + const m = head.match(/#define\s+LANGUAGE_VERSION\s+(\d+)/); + if (m) return Number(m[1]); + } + return null; // unknown (e.g. parser.c only generated at build time) +} + +function detect() { + const report = []; + for (const [key, g] of Object.entries(GRAMMARS)) { + const have = vendoredVersion(g); + let up; + try { + up = resolveUpstream(g); + } catch (err) { + report.push({ grammar: key, error: String(err.message || err) }); + continue; + } + const newer = up.kind === 'npm' ? up.version !== have : !have || up.ref.slice(0, 7) !== have; + let abi = null; + if (newer) { + try { + abi = readAbi(fetchSource(g, up.ref)); + } catch { + /* fetch/abi best-effort; null = unknown */ + } + } + report.push({ + grammar: key, + vendored: have, + upstream: up.version, + ref: up.ref, + kind: up.kind, + update: newer, + abi, + abiCompatible: abi == null ? null : COMPATIBLE_ABI.has(abi), + // Only auto-appliable when there's an update AND the ABI is known-compatible. + applicable: newer && abi != null && COMPATIBLE_ABI.has(abi), + }); + } + return report; +} + +const copyFile = (srcRoot, dest, rel) => { + const from = path.join(srcRoot, rel); + if (!fs.existsSync(from)) return false; + const to = path.join(dest, rel); + fs.mkdirSync(path.dirname(to), { recursive: true }); + fs.copyFileSync(from, to); + return true; +}; + +/** + * Re-vendor one grammar in place from its ABI-compatible upstream candidate. + * Copies ONLY the generated source-build + runtime files; deliberately KEEPS the + * GitNexus-hardened binding.gyp (Windows cflags, target_name), README (vendor + * notice), LICENSE, and prebuilds/ (the build workflow refreshes those). Bumps the + * stripped vendor package.json version + provenance — never re-introduces + * scripts/dependencies (#836/#1728). Returns the new version. + */ +function apply(key) { + const g = GRAMMARS[key]; + if (!g) { + console.error(`unknown grammar '${key}'`); + process.exit(2); + } + const have = vendoredVersion(g); + const up = resolveUpstream(g); + const newer = up.kind === 'npm' ? up.version !== have : !have || up.version !== have; + if (!newer) { + console.error(`${key}: already current (${have}); nothing to apply.`); + process.exit(0); + } + const srcRoot = fetchSource(g, up.ref); + const abi = readAbi(srcRoot); + if (abi == null || !COMPATIBLE_ABI.has(abi)) { + console.error( + `${key}: candidate ${up.version} is ABI ${abi ?? 'unknown'} — not tree-sitter@0.21.1 ` + + `compatible (need 13/14); refusing to re-vendor. Handle manually.`, + ); + process.exit(3); + } + + const dest = path.join(VENDOR, g.name); + // The source-build inputs + runtime entrypoints that change between versions. + // binding.gyp / README / LICENSE / prebuilds are intentionally NOT touched. + for (const rel of [ + 'src/parser.c', + 'src/scanner.c', + 'src/node-types.json', + 'src/tree_sitter/alloc.h', + 'src/tree_sitter/array.h', + 'src/tree_sitter/parser.h', + 'bindings/node/binding.cc', + 'bindings/node/index.js', + 'bindings/node/index.d.ts', + ]) { + copyFile(srcRoot, dest, rel); + } + + const pkgPath = path.join(dest, 'package.json'); + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')); + pkg.version = up.version; + pkg._vendoredBy = + `gitnexus - re-vendored from ${g.npm ? `npm ${g.npm}@${up.version}` : `${g.github}@${up.ref}`} ` + + `by grammar-update-monitor on ABI ${abi}. Source-build inputs (parser.c/scanner.c/src/) refreshed; ` + + `the GitNexus-hardened binding.gyp + vendor README + prebuilds are preserved (prebuilds are ` + + `rebuilt by build-tree-sitter-prebuilds.yml on this version change). No scripts/dependencies here ` + + `(#836/#1728).`; + fs.writeFileSync(pkgPath, JSON.stringify(pkg, null, 2) + '\n'); + + console.log(`${key}: re-vendored ${g.name} → ${up.version} (ABI ${abi}).`); + return up.version; +} + +// Run the CLI only when invoked directly (not when imported by a test) — detect() +// makes live network calls, so importing must be side-effect-free. +const isMain = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (isMain) { + if (process.argv[2] === '--apply') { + apply(process.argv[3]); + } else { + process.stdout.write(JSON.stringify(detect(), null, 2) + '\n'); + } +} + +export { detect, apply, resolveUpstream, readAbi, vendoredVersion, GRAMMARS, COMPATIBLE_ABI }; diff --git a/.github/workflows/grammar-update-monitor.yml b/.github/workflows/grammar-update-monitor.yml new file mode 100644 index 000000000..a0bb3307f --- /dev/null +++ b/.github/workflows/grammar-update-monitor.yml @@ -0,0 +1,144 @@ +name: Vendored grammar update monitor + +# Periodically checks each vendored tree-sitter grammar against its +# source-of-origin and opens a PR re-vendoring any update that is ABI-COMPATIBLE +# with the pinned tree-sitter@0.21.1 (LANGUAGE_VERSION 13–14, #1922). The version +# bump then triggers build-tree-sitter-prebuilds.yml, which cross-builds + ABI- +# validates the prebuilds — so a re-vendor that is subtly wrong can never silently +# ship: its PR's CI goes red. +# +# ABI-INCOMPATIBLE updates (the common case — upstreams move to newer tree-sitter) +# are reported as a notice + job summary, NOT applied, so the monitor never opens +# doomed PRs. tree-sitter-c is excluded entirely: it is HELD at 0.21.4 for ABI +# safety (#1242) and must never be auto-bumped. +# +# Concurrency convention: see CONTRIBUTING.md -> "GitHub Actions — Concurrency Convention". + +on: + schedule: + - cron: '17 6 * * 1' # weekly, Monday 06:17 UTC + workflow_dispatch: + +# Least privilege; the actual writes use a short-lived App token minted below. +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + monitor: + name: Check upstreams + open update PRs + runs-on: ubuntu-24.04 + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 22 + + # secrets aren't usable in a job/step `if:`, so compute presence here. + - name: Check release App secret + id: relapp + env: + HAS_APP: ${{ secrets.RELEASE_APP_ID != '' && secrets.RELEASE_APP_PRIVATE_KEY != '' }} + run: echo "configured=$HAS_APP" >> "$GITHUB_OUTPUT" + + - name: Mint GitHub App token + id: app-token + if: steps.relapp.outputs.configured == 'true' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ secrets.RELEASE_APP_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + + - name: Detect updates, re-vendor ABI-compatible ones, open PRs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + HAS_APP: ${{ steps.relapp.outputs.configured }} + # App token writes; falls back to the read-only job token (PRs then skip). + GH_TOKEN: ${{ steps.app-token.outputs.token || github.token }} + with: + github-token: ${{ steps.app-token.outputs.token || github.token }} + script: | + const { execFileSync } = require('node:child_process'); + const SCRIPT = '.github/scripts/update-vendored-grammars.mjs'; + const run = (cmd, args, opts = {}) => + execFileSync(cmd, args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], ...opts }); + + const report = JSON.parse(run('node', [SCRIPT])); + const { owner, repo } = context.repo; + const hasApp = process.env.HAS_APP === 'true'; + const applied = [], held = [], errors = [], skipped = []; + + run('git', ['config', 'user.name', 'gitnexus-release-bot[bot]']); + run('git', ['config', 'user.email', 'gitnexus-release-bot[bot]@users.noreply.github.com']); + const baseSha = run('git', ['rev-parse', 'HEAD']).trim(); + + for (const r of report) { + if (r.error) { errors.push(r); continue; } + if (!r.update) continue; + if (!r.applicable) { held.push(r); continue; } // ABI-incompatible / unknown + + const name = `tree-sitter-${r.grammar}`; + const branch = `chore/update-${name}-${r.upstream}`.replace(/[^a-z0-9._/-]+/gi, '-'); + + // Idempotency: don't reopen an existing PR for this exact version. + const existing = await github.rest.pulls.list({ owner, repo, head: `${owner}:${branch}`, state: 'all' }); + if (existing.data.length > 0) { skipped.push({ ...r, reason: 'PR exists' }); continue; } + + // Re-vendor in place (refuses + exits non-zero if ABI turns out wrong). + try { + run('node', [SCRIPT, '--apply', r.grammar]); + } catch (e) { + errors.push({ ...r, error: `apply failed: ${String(e.message || e).slice(0, 200)}` }); + run('git', ['checkout', '--', 'gitnexus/vendor']); + continue; + } + + if (!hasApp) { + skipped.push({ ...r, reason: 'no RELEASE_APP secret — PR not opened' }); + run('git', ['checkout', '--', 'gitnexus/vendor']); + continue; + } + + const remote = `https://x-access-token:${process.env.GH_TOKEN}@github.com/${owner}/${repo}.git`; + run('git', ['checkout', '-B', branch, baseSha]); + run('git', ['add', `gitnexus/vendor/${name}`]); + run('git', ['commit', '-m', `chore(vendor): update ${name} to ${r.upstream}`]); + run('git', ['push', '--force-with-lease', remote, `HEAD:${branch}`]); + const body = [ + `Automated re-vendor of **${name}** to \`${r.upstream}\` (from ${r.kind === 'npm' ? `npm \`${name}\`` : `\`${r.ref}\``}).`, + '', + `Verified ABI **${r.abi}** — compatible with the pinned \`tree-sitter@0.21.1\` (13–14).`, + 'Source-build inputs refreshed; the GitNexus binding.gyp / README / prebuilds are preserved.', + 'The version bump triggers `build-tree-sitter-prebuilds.yml` to rebuild + ABI-validate the', + 'prebuilds — review its result before merging.', + ].join('\n'); + const pr = await github.rest.pulls.create({ + owner, repo, head: branch, base: 'main', + title: `chore(vendor): update ${name} to ${r.upstream}`, body, + }); + applied.push({ ...r, pr: pr.data.number }); + run('git', ['checkout', '--force', baseSha]); + } + + // Summary + const s = core.summary.addHeading('Vendored grammar update monitor'); + if (applied.length) s.addRaw(`\n**Opened PRs:** ${applied.map((a) => `${a.grammar}→${a.upstream} (#${a.pr})`).join(', ')}\n`); + if (held.length) s.addRaw(`\n**Held (ABI-incompatible / unknown — needs the tree-sitter runtime upgrade):** ${held.map((h) => `${h.grammar} ${h.upstream} (ABI ${h.abi ?? '?'})`).join(', ')}\n`); + if (skipped.length) s.addRaw(`\n**Skipped:** ${skipped.map((x) => `${x.grammar} (${x.reason})`).join(', ')}\n`); + if (errors.length) s.addRaw(`\n**Errors:** ${errors.map((e) => `${e.grammar}: ${e.error}`).join('; ')}\n`); + if (!applied.length && !held.length && !skipped.length && !errors.length) s.addRaw('\nAll vendored grammars are up to date. ✅\n'); + await s.write(); + + for (const h of held) core.notice(`${h.grammar}: update to ${h.upstream} available but ABI ${h.abi ?? 'unknown'} (need 13/14) — held until the tree-sitter runtime upgrade.`); + if (!hasApp && (applied.length || skipped.some((x) => /secret/.test(x.reason)))) { + core.notice('RELEASE_APP_ID / RELEASE_APP_PRIVATE_KEY not configured — update PRs were not opened. Provision the App to enable auto-PRs.'); + } diff --git a/gitnexus/test/unit/grammar-update-monitor.test.ts b/gitnexus/test/unit/grammar-update-monitor.test.ts new file mode 100644 index 000000000..4e3944196 --- /dev/null +++ b/gitnexus/test/unit/grammar-update-monitor.test.ts @@ -0,0 +1,71 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +/** + * Unit coverage for the ABI gate in the vendored-grammar update monitor + * (.github/scripts/update-vendored-grammars.mjs). The gate is load-bearing: every + * grammar is pinned to tree-sitter@0.21.1 (LANGUAGE_VERSION 13–14), so an update + * is only auto-applied when the candidate parser.c's ABI is 13 or 14 — otherwise + * the monitor would open PRs that can't build. We test the pure pieces (no + * network): reading the ABI from a parser.c and the compatibility set. The module + * is import-safe (its CLI is guarded behind an isMain check). + */ +const MOD = pathToFileURL( + path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../../.github/scripts/update-vendored-grammars.mjs', + ), +).href; + +let mod: { + readAbi: (root: string) => number | null; + COMPATIBLE_ABI: Set; + GRAMMARS: Record; +}; +let tmp: string; + +beforeAll(async () => { + mod = await import(MOD); + tmp = mkdtempSync(path.join(tmpdir(), 'gum-')); +}); +afterAll(() => rmSync(tmp, { recursive: true, force: true })); + +function fixture(abiLine: string): string { + const root = mkdtempSync(path.join(tmp, 'g-')); + mkdirSync(path.join(root, 'src'), { recursive: true }); + writeFileSync(path.join(root, 'src', 'parser.c'), `${abiLine}\n#define STATE_COUNT 10\n`); + return root; +} + +describe('readAbi', () => { + it('reads LANGUAGE_VERSION 14 from src/parser.c', () => { + expect(mod.readAbi(fixture('#define LANGUAGE_VERSION 14'))).toBe(14); + }); + it('reads LANGUAGE_VERSION 15 (an incompatible upstream)', () => { + expect(mod.readAbi(fixture('#define LANGUAGE_VERSION 15'))).toBe(15); + }); + it('returns null when parser.c is absent (generated-at-build-time grammars)', () => { + expect(mod.readAbi(mkdtempSync(path.join(tmp, 'empty-')))).toBeNull(); + }); +}); + +describe('COMPATIBLE_ABI gate', () => { + it('accepts ABI 13 and 14, rejects 12 and 15', () => { + expect(mod.COMPATIBLE_ABI.has(13)).toBe(true); + expect(mod.COMPATIBLE_ABI.has(14)).toBe(true); + expect(mod.COMPATIBLE_ABI.has(12)).toBe(false); + expect(mod.COMPATIBLE_ABI.has(15)).toBe(false); + }); +}); + +describe('GRAMMARS registry', () => { + it('covers swift/kotlin (npm) + dart/proto (github) and EXCLUDES the ABI-pinned c', () => { + expect(Object.keys(mod.GRAMMARS).sort()).toEqual(['dart', 'kotlin', 'proto', 'swift']); + expect(mod.GRAMMARS.swift.npm).toBe('tree-sitter-swift'); + expect(mod.GRAMMARS.dart.github).toContain('tree-sitter-dart'); + expect(mod.GRAMMARS).not.toHaveProperty('c'); + }); +});