From e174fc7c3d2ed05f5bec55214463b3c7d9a58cd1 Mon Sep 17 00:00:00 2001 From: Gergo Magyar Date: Tue, 9 Jun 2026 11:51:16 +0000 Subject: [PATCH] feat(publish): gate a lean prebuilds-only npm tarball behind a coverage guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vendoring grammar source (parser.c) alongside the prebuilds means the npm tarball now carries ~50 MB of generated source it almost never compiles (every supported platform-arch has a prebuild). Prepare to drop it from the published package once all prebuilds exist — safely. - .npmignore: add a GATED, commented-out "lean publish" block that excludes the source-build inputs (parser.c/scanner.c/tree_sitter/binding.gyp/binding.cc) but keeps prebuilds/ + the runtime files. Uncommenting ships prebuilds-only. - scripts/assert-publish-grammar-coverage.cjs: a prepack guard that refuses to pack/publish if the source exclusion is active while any vendored grammar still lacks 6/6 prebuilds (which would ship a grammar with no loadable binding). Wired into `prepack` (runs on npm pack + publish, incl. the publish.yml dry-run) and exposed as `npm run assert-publish-coverage`. - test: pure-core decision cases + a real-repo publish-safety check that fails CI if .npmignore is activated prematurely. Net: the prebuilds already publish today (files: ["vendor"]); this makes the future switch to a prebuilds-only tarball a one-line uncomment that can't ship a dead grammar. The guard currently reports "source + prebuilds" (only swift has 6/6 prebuilds so far) and passes. Co-Authored-By: Claude Opus 4.8 (1M context) --- gitnexus/.npmignore | 21 +++ gitnexus/package.json | 3 +- .../assert-publish-grammar-coverage.cjs | 133 ++++++++++++++++++ .../assert-publish-grammar-coverage.test.ts | 67 +++++++++ 4 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 gitnexus/scripts/assert-publish-grammar-coverage.cjs create mode 100644 gitnexus/test/unit/assert-publish-grammar-coverage.test.ts diff --git a/gitnexus/.npmignore b/gitnexus/.npmignore index cf403314a..238ae3077 100644 --- a/gitnexus/.npmignore +++ b/gitnexus/.npmignore @@ -13,6 +13,27 @@ node_modules/ vendor/**/node_modules vendor/**/build +# ── Lean publish (GATED — do NOT uncomment until ALL prebuilds exist) ────────── +# Once the build-tree-sitter-prebuilds workflow has committed 6/6 prebuilds for +# EVERY vendored grammar (c, dart, proto, kotlin, swift), uncomment the lines +# below to ship prebuilds only and drop ~50 MB of generated parser.c source from +# the tarball. node-gyp-build never needs the source when a prebuild matches; the +# committed prebuilds still ship (this block does not touch vendor/**/prebuilds). +# +# WARNING: uncommenting while ANY grammar still lacks 6/6 prebuilds ships that +# grammar with neither a prebuild nor buildable source → its language is dead on +# the affected platforms. The prepack guard +# (scripts/assert-publish-grammar-coverage.cjs, also `npm run +# assert-publish-coverage`) ENFORCES this gate and fails the publish if violated, +# so do not bypass it. Keep the first toggle line below byte-identical to the +# guard's SOURCE_EXCLUSION_TOGGLE. +# +# vendor/**/src/parser.c +# vendor/**/src/scanner.c +# vendor/**/src/tree_sitter +# vendor/**/binding.gyp +# vendor/**/bindings/node/binding.cc + # Package lock (consumers use their own) package-lock.json diff --git a/gitnexus/package.json b/gitnexus/package.json index 8edbe1a3b..e68054a52 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -50,8 +50,9 @@ "test:coverage": "vitest run --coverage", "test:cross-platform": "tsx scripts/run-cross-platform.ts", "postinstall": "node scripts/materialize-vendor-grammars.cjs && node scripts/build-tree-sitter-c.cjs && node scripts/build-tree-sitter-dart.cjs && node scripts/build-tree-sitter-proto.cjs && node scripts/build-tree-sitter-swift.cjs && node scripts/build-tree-sitter-kotlin.cjs", + "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", - "prepack": "node scripts/build.js" + "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js" }, "dependencies": { "@huggingface/transformers": "^4.1.0", diff --git a/gitnexus/scripts/assert-publish-grammar-coverage.cjs b/gitnexus/scripts/assert-publish-grammar-coverage.cjs new file mode 100644 index 000000000..af44c4b73 --- /dev/null +++ b/gitnexus/scripts/assert-publish-grammar-coverage.cjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node +/** + * Publish guard: every vendored tree-sitter grammar must ship a loadable binding. + * + * The npm tarball includes gitnexus/vendor/ (package.json `files`). A grammar is + * "covered" on a platform-arch tuple if EITHER a prebuild ships for it OR the + * grammar source ships (so the install can source-build it, toolchain + * permitting). The lean-publish toggle in gitnexus/.npmignore + * (`vendor/**​/src/parser.c`, commented by default) drops the ~50 MB of generated + * source to ship prebuilds only — which is safe ONLY once every grammar has all + * six prebuilds. Excluding the source while any grammar is still missing a + * prebuild would ship a grammar with NO loadable binding (neither prebuild nor + * buildable source) → that language is silently dead for users. + * + * This guard fails `npm pack` / `npm publish` (wired via `prepack`) if that + * invariant is violated, so the gated .npmignore exclusions can never be + * activated early and silently ship a dead grammar. + */ +const fs = require('fs'); +const path = require('path'); + +const TUPLES = [ + 'linux-x64', + 'linux-arm64', + 'darwin-x64', + 'darwin-arm64', + 'win32-x64', + 'win32-arm64', +]; + +// The single .npmignore line that toggles source exclusion. Keep in exact sync +// with the lean-publish block in gitnexus/.npmignore. +const SOURCE_EXCLUSION_TOGGLE = 'vendor/**/src/parser.c'; + +function activeIgnorePatterns(npmignoreText) { + return npmignoreText + .split(/\r?\n/) + .map((l) => l.trim()) + .filter((l) => l && !l.startsWith('#')); +} + +function countPrebuiltTuples(grammarDir) { + const pdir = path.join(grammarDir, 'prebuilds'); + let n = 0; + for (const t of TUPLES) { + const td = path.join(pdir, t); + try { + if (fs.statSync(td).isDirectory() && fs.readdirSync(td).some((f) => f.endsWith('.node'))) { + n++; + } + } catch { + /* tuple dir absent — not covered */ + } + } + return n; +} + +/** + * Pure core (exported for tests). `grammars` is a list of + * `{ name, prebuilt: 0..6, hasSource: boolean }`. Returns human-readable problem + * strings; an empty array means the pack is publish-safe. + */ +function findCoverageProblems({ grammars, sourceExcluded }) { + const problems = []; + for (const g of grammars) { + const shipsSource = g.hasSource && !sourceExcluded; + if (g.prebuilt < 6 && !shipsSource) { + const missing = 6 - g.prebuilt; + problems.push( + `${g.name}: ${g.prebuilt}/6 prebuilds and source ${ + sourceExcluded ? 'EXCLUDED from the npm tarball' : 'absent' + } — would ship with no loadable binding on ${missing} platform-arch tuple(s).`, + ); + } + } + return problems; +} + +function collectGrammars(vendorDir) { + if (!fs.existsSync(vendorDir)) return []; + return fs + .readdirSync(vendorDir) + .filter((d) => /^tree-sitter-/.test(d)) + .map((name) => { + const dir = path.join(vendorDir, name); + return { + name, + prebuilt: countPrebuiltTuples(dir), + hasSource: fs.existsSync(path.join(dir, 'src', 'parser.c')), + }; + }); +} + +function main() { + const gitnexusRoot = path.join(__dirname, '..'); + const vendorDir = path.join(gitnexusRoot, 'vendor'); + const npmignorePath = path.join(gitnexusRoot, '.npmignore'); + const npmignoreText = fs.existsSync(npmignorePath) ? fs.readFileSync(npmignorePath, 'utf8') : ''; + const sourceExcluded = activeIgnorePatterns(npmignoreText).includes(SOURCE_EXCLUSION_TOGGLE); + const grammars = collectGrammars(vendorDir); + + if (grammars.length === 0) { + console.error(`[publish-guard] No vendored tree-sitter grammars found under ${vendorDir}.`); + process.exit(1); + } + + const problems = findCoverageProblems({ grammars, sourceExcluded }); + if (problems.length > 0) { + console.error('[publish-guard] Refusing to publish — a vendored grammar would ship unusable:'); + for (const p of problems) console.error(` - ${p}`); + console.error( + '\nFix: either commit the missing prebuilds (run the build-tree-sitter-prebuilds\n' + + 'workflow) or re-comment the lean-publish source exclusions in gitnexus/.npmignore.', + ); + process.exit(1); + } + + const mode = sourceExcluded ? 'prebuilds-only (source excluded)' : 'source + prebuilds'; + console.log( + `[publish-guard] OK — ${grammars.length} vendored grammar(s) covered; tarball mode: ${mode}.`, + ); +} + +if (require.main === module) main(); + +module.exports = { + findCoverageProblems, + activeIgnorePatterns, + countPrebuiltTuples, + collectGrammars, + TUPLES, + SOURCE_EXCLUSION_TOGGLE, +}; diff --git a/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts b/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts new file mode 100644 index 000000000..fb98c22c3 --- /dev/null +++ b/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts @@ -0,0 +1,67 @@ +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; + +/** + * Coverage for the publish guard `scripts/assert-publish-grammar-coverage.cjs`. + * + * The guard refuses to pack/publish if a vendored grammar would ship with no + * loadable binding — i.e. the lean-publish source exclusion was activated in + * .npmignore while a grammar still lacks 6/6 prebuilds. We test the pure decision + * core directly, and assert the real repo state is publish-safe (this catches a + * premature .npmignore activation in CI, not just at publish time). + */ +const requireCjs = createRequire(import.meta.url); +const SCRIPT = fileURLToPath( + new URL('../../scripts/assert-publish-grammar-coverage.cjs', import.meta.url), +); +const { findCoverageProblems, activeIgnorePatterns, SOURCE_EXCLUSION_TOGGLE } = requireCjs(SCRIPT); + +describe('findCoverageProblems (pure decision core)', () => { + it('passes when source ships, even with incomplete prebuilds (transitional state)', () => { + const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 0, hasSource: true }]; + expect(findCoverageProblems({ grammars, sourceExcluded: false })).toEqual([]); + }); + + it('fails when source is excluded but a grammar lacks 6/6 prebuilds', () => { + const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 4, hasSource: true }]; + const problems = findCoverageProblems({ grammars, sourceExcluded: true }); + expect(problems).toHaveLength(1); + expect(problems[0]).toContain('tree-sitter-kotlin'); + expect(problems[0]).toContain('EXCLUDED'); + expect(problems[0]).toContain('2 platform-arch tuple(s)'); + }); + + it('passes when source is excluded but every grammar has all 6 prebuilds', () => { + const grammars = [ + { name: 'tree-sitter-swift', prebuilt: 6, hasSource: true }, + { name: 'tree-sitter-c', prebuilt: 6, hasSource: false }, + ]; + expect(findCoverageProblems({ grammars, sourceExcluded: true })).toEqual([]); + }); + + it('fails when a grammar has neither prebuilds nor source', () => { + const grammars = [{ name: 'tree-sitter-x', prebuilt: 0, hasSource: false }]; + const problems = findCoverageProblems({ grammars, sourceExcluded: false }); + expect(problems).toHaveLength(1); + expect(problems[0]).toContain('absent'); + }); +}); + +describe('activeIgnorePatterns', () => { + it('ignores comments/blank lines and trims', () => { + const txt = ['# a comment', '', ' vendor/**/build ', '# vendor/**/src/parser.c'].join('\n'); + const active = activeIgnorePatterns(txt); + expect(active).toContain('vendor/**/build'); + expect(active).not.toContain(SOURCE_EXCLUSION_TOGGLE); // commented out → inert + }); +}); + +describe('real repo publish-safety (guards against premature .npmignore activation)', () => { + it('the script exits 0 against the committed repo state', () => { + const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 30_000 }); + expect(r.status, r.stderr).toBe(0); + expect(r.stdout).toContain('[publish-guard] OK'); + }); +});