diff --git a/gitnexus/.npmignore b/gitnexus/.npmignore index 238ae3077..bf2c8b7c9 100644 --- a/gitnexus/.npmignore +++ b/gitnexus/.npmignore @@ -13,26 +13,25 @@ node_modules/ vendor/**/node_modules vendor/**/build -# ── Lean publish (GATED — do NOT uncomment until ALL prebuilds exist) ────────── +# ── Lean publish (FUTURE optimization — NOT done here) ───────────────────────── # Once the build-tree-sitter-prebuilds workflow has committed 6/6 prebuilds for -# EVERY vendored grammar (c, dart, proto, kotlin, swift), uncomment the lines -# below to ship prebuilds only and drop ~50 MB of generated parser.c source from -# the tarball. node-gyp-build never needs the source when a prebuild matches; the -# committed prebuilds still ship (this block does not touch vendor/**/prebuilds). +# EVERY vendored grammar (c, dart, proto, kotlin, swift), the ~50 MB of generated +# source (parser.c etc.) can be dropped from the tarball — node-gyp-build never +# needs the source when a prebuild matches. # -# WARNING: uncommenting while ANY grammar still lacks 6/6 prebuilds ships that -# grammar with neither a prebuild nor buildable source → its language is dead on -# the affected platforms. The prepack guard +# IMPORTANT: this CANNOT be done from this file. package.json's `files: ["vendor"]` +# allow-list OVERRIDES .npmignore for the vendor/ subtree (verified: an active +# `vendor/**/src/parser.c` line here does NOT exclude it from `npm pack`). To slim +# the tarball, narrow the `files` field instead — replace the blanket "vendor" +# with the non-source subpaths only (vendor/**/prebuilds/**, +# vendor/**/bindings/node/index.*, vendor/**/src/node-types.json, +# vendor/**/package.json, vendor/**/LICENSE, vendor/**/README.md). +# +# Whatever the mechanism, the prepack guard # (scripts/assert-publish-grammar-coverage.cjs, also `npm run -# assert-publish-coverage`) ENFORCES this gate and fails the publish if violated, -# so do not bypass it. Keep the first toggle line below byte-identical to the -# guard's SOURCE_EXCLUSION_TOGGLE. -# -# vendor/**/src/parser.c -# vendor/**/src/scanner.c -# vendor/**/src/tree_sitter -# vendor/**/binding.gyp -# vendor/**/bindings/node/binding.cc +# assert-publish-coverage`) inspects the EFFECTIVE `npm pack` file list and FAILS +# the publish whenever a grammar with <6 prebuilds loses a source-build input — so +# the slim can never silently ship a dead grammar. Do not bypass it. # Package lock (consumers use their own) package-lock.json diff --git a/gitnexus/scripts/assert-publish-grammar-coverage.cjs b/gitnexus/scripts/assert-publish-grammar-coverage.cjs index af44c4b73..4ca9806e2 100644 --- a/gitnexus/scripts/assert-publish-grammar-coverage.cjs +++ b/gitnexus/scripts/assert-publish-grammar-coverage.cjs @@ -4,20 +4,31 @@ * * The npm tarball includes gitnexus/vendor/ (package.json `files`). A grammar is * "covered" on a platform-arch tuple if EITHER a prebuild ships for it OR the - * grammar source ships (so the install can source-build it, toolchain - * permitting). The lean-publish toggle in gitnexus/.npmignore - * (`vendor/**​/src/parser.c`, commented by default) drops the ~50 MB of generated - * source to ship prebuilds only — which is safe ONLY once every grammar has all - * six prebuilds. Excluding the source while any grammar is still missing a - * prebuild would ship a grammar with NO loadable binding (neither prebuild nor - * buildable source) → that language is silently dead for users. + * grammar's full source-build set ships (so the install can source-build it, + * toolchain permitting). A future lean publish — dropping the ~50 MB of generated + * source to ship prebuilds only — is safe ONLY once every grammar has all six + * prebuilds; doing it while any grammar is still missing a prebuild would ship a + * grammar with NO loadable binding (neither prebuild nor buildable source) → that + * language is silently dead for users. (Note: the slim must be done by narrowing + * package.json's `files` field, NOT via .npmignore — `files` overrides .npmignore + * for the vendored subtree; see gitnexus/.npmignore.) * - * This guard fails `npm pack` / `npm publish` (wired via `prepack`) if that - * invariant is violated, so the gated .npmignore exclusions can never be - * activated early and silently ship a dead grammar. + * Rather than infer "is source excluded?" from a single .npmignore toggle line + * (which a partial/out-of-order edit could defeat — exclude binding.gyp but leave + * parser.c, and the grammar is unbuildable yet still looks "source-shipping"), + * this guard inspects the EFFECTIVE tarball: `npm pack --dry-run --ignore-scripts + * --json` (the `--ignore-scripts` avoids re-entering this guard via prepack). A + * grammar "ships source" only when EVERY one of its on-disk source-build inputs + * (binding.gyp + binding.cc + parser.c + scanner.c when present + a tree_sitter + * header) is actually in the packed file list. + * + * Wired via `prepack`, so it fails `npm pack` / `npm publish` if the invariant is + * violated — the gated .npmignore exclusions can never be activated early and + * silently ship a dead grammar. */ const fs = require('fs'); const path = require('path'); +const { execSync } = require('child_process'); const TUPLES = [ 'linux-x64', @@ -28,65 +39,83 @@ const TUPLES = [ 'win32-arm64', ]; -// The single .npmignore line that toggles source exclusion. Keep in exact sync -// with the lean-publish block in gitnexus/.npmignore. -const SOURCE_EXCLUSION_TOGGLE = 'vendor/**/src/parser.c'; +// Source-build inputs (tarball-relative within vendor//) whose presence in +// the pack makes a grammar source-buildable. Per-grammar we only require the ones +// that actually exist on disk (e.g. tree-sitter-c has no external scanner.c). +const SOURCE_BUILD_REL = [ + 'binding.gyp', + 'bindings/node/binding.cc', + 'src/parser.c', + 'src/scanner.c', + 'src/tree_sitter/parser.h', +]; -function activeIgnorePatterns(npmignoreText) { - return npmignoreText - .split(/\r?\n/) - .map((l) => l.trim()) - .filter((l) => l && !l.startsWith('#')); +/** The on-disk source-build inputs for a grammar, as tarball-relative paths. */ +function sourceBuildSet(grammarDir, name) { + return SOURCE_BUILD_REL.filter((rel) => fs.existsSync(path.join(grammarDir, rel))).map( + (rel) => `vendor/${name}/${rel}`, + ); } -function countPrebuiltTuples(grammarDir) { - const pdir = path.join(grammarDir, 'prebuilds'); +/** Count platform-arch tuples whose prebuilt .node is present in the packed set. */ +function prebuiltTuplesInPack(name, packedFiles) { let n = 0; for (const t of TUPLES) { - const td = path.join(pdir, t); - try { - if (fs.statSync(td).isDirectory() && fs.readdirSync(td).some((f) => f.endsWith('.node'))) { - n++; - } - } catch { - /* tuple dir absent — not covered */ - } + const prefix = `vendor/${name}/prebuilds/${t}/`; + if ([...packedFiles].some((f) => f.startsWith(prefix) && f.endsWith('.node'))) n++; } return n; } /** * Pure core (exported for tests). `grammars` is a list of - * `{ name, prebuilt: 0..6, hasSource: boolean }`. Returns human-readable problem - * strings; an empty array means the pack is publish-safe. + * `{ name, prebuilt: 0..6, shipsSource: boolean }`. Returns human-readable + * problem strings; an empty array means the pack is publish-safe. */ -function findCoverageProblems({ grammars, sourceExcluded }) { +function findCoverageProblems({ grammars }) { const problems = []; for (const g of grammars) { - const shipsSource = g.hasSource && !sourceExcluded; - if (g.prebuilt < 6 && !shipsSource) { + if (g.prebuilt < 6 && !g.shipsSource) { const missing = 6 - g.prebuilt; problems.push( - `${g.name}: ${g.prebuilt}/6 prebuilds and source ${ - sourceExcluded ? 'EXCLUDED from the npm tarball' : 'absent' - } — would ship with no loadable binding on ${missing} platform-arch tuple(s).`, + `${g.name}: ${g.prebuilt}/6 prebuilds in the tarball and source NOT fully shipped ` + + `(a source-build input is missing/excluded) — would ship with no loadable binding on ` + + `${missing} platform-arch tuple(s).`, ); } } return problems; } -function collectGrammars(vendorDir) { +/** Build the set of tarball-relative paths `npm pack` would include. */ +function packFileSet(cwd) { + const out = execSync('npm pack --dry-run --ignore-scripts --json', { + cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], + }); + const parsed = JSON.parse(out); + const files = (parsed[0] && parsed[0].files) || []; + return new Set(files.map((f) => f.path.replace(/\\/g, '/'))); +} + +function collectGrammars(vendorDir, packedFiles) { if (!fs.existsSync(vendorDir)) return []; return fs .readdirSync(vendorDir) .filter((d) => /^tree-sitter-/.test(d)) .map((name) => { const dir = path.join(vendorDir, name); + const srcSet = sourceBuildSet(dir, name); + const buildable = + srcSet.includes(`vendor/${name}/src/parser.c`) && + srcSet.includes(`vendor/${name}/binding.gyp`); return { name, - prebuilt: countPrebuiltTuples(dir), - hasSource: fs.existsSync(path.join(dir, 'src', 'parser.c')), + prebuilt: prebuiltTuplesInPack(name, packedFiles), + // Source ships only when the grammar is buildable AND every one of its + // source-build inputs is actually in the packed file list. + shipsSource: buildable && srcSet.every((f) => packedFiles.has(f)), }; }); } @@ -94,17 +123,22 @@ function collectGrammars(vendorDir) { function main() { const gitnexusRoot = path.join(__dirname, '..'); const vendorDir = path.join(gitnexusRoot, 'vendor'); - const npmignorePath = path.join(gitnexusRoot, '.npmignore'); - const npmignoreText = fs.existsSync(npmignorePath) ? fs.readFileSync(npmignorePath, 'utf8') : ''; - const sourceExcluded = activeIgnorePatterns(npmignoreText).includes(SOURCE_EXCLUSION_TOGGLE); - const grammars = collectGrammars(vendorDir); + let packedFiles; + try { + packedFiles = packFileSet(gitnexusRoot); + } catch (err) { + console.error(`[publish-guard] Could not compute the npm pack file list: ${err.message}`); + process.exit(1); + } + + const grammars = collectGrammars(vendorDir, packedFiles); if (grammars.length === 0) { console.error(`[publish-guard] No vendored tree-sitter grammars found under ${vendorDir}.`); process.exit(1); } - const problems = findCoverageProblems({ grammars, sourceExcluded }); + const problems = findCoverageProblems({ grammars }); if (problems.length > 0) { console.error('[publish-guard] Refusing to publish — a vendored grammar would ship unusable:'); for (const p of problems) console.error(` - ${p}`); @@ -115,9 +149,10 @@ function main() { process.exit(1); } - const mode = sourceExcluded ? 'prebuilds-only (source excluded)' : 'source + prebuilds'; + const sourceShippers = grammars.filter((g) => g.shipsSource).length; console.log( - `[publish-guard] OK — ${grammars.length} vendored grammar(s) covered; tarball mode: ${mode}.`, + `[publish-guard] OK — ${grammars.length} vendored grammar(s) covered ` + + `(${sourceShippers} shipping source, ${grammars.length - sourceShippers} prebuilds-only).`, ); } @@ -125,9 +160,10 @@ if (require.main === module) main(); module.exports = { findCoverageProblems, - activeIgnorePatterns, - countPrebuiltTuples, + prebuiltTuplesInPack, + sourceBuildSet, collectGrammars, + packFileSet, TUPLES, - SOURCE_EXCLUSION_TOGGLE, + SOURCE_BUILD_REL, }; diff --git a/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts b/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts index fb98c22c3..d1a6e2782 100644 --- a/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts +++ b/gitnexus/test/unit/assert-publish-grammar-coverage.test.ts @@ -7,60 +7,68 @@ import { fileURLToPath } from 'node:url'; * Coverage for the publish guard `scripts/assert-publish-grammar-coverage.cjs`. * * The guard refuses to pack/publish if a vendored grammar would ship with no - * loadable binding — i.e. the lean-publish source exclusion was activated in - * .npmignore while a grammar still lacks 6/6 prebuilds. We test the pure decision - * core directly, and assert the real repo state is publish-safe (this catches a - * premature .npmignore activation in CI, not just at publish time). + * loadable binding — i.e. a lean-publish `.npmignore` edit dropped a source-build + * input while a grammar still lacks 6/6 prebuilds. It decides "ships source" by + * inspecting the EFFECTIVE `npm pack` file list, so a partial exclusion can't slip + * past. We test the pure decision core directly, and assert the real repo state is + * publish-safe (catching a premature .npmignore activation in CI, not just at + * publish time). */ const requireCjs = createRequire(import.meta.url); const SCRIPT = fileURLToPath( new URL('../../scripts/assert-publish-grammar-coverage.cjs', import.meta.url), ); -const { findCoverageProblems, activeIgnorePatterns, SOURCE_EXCLUSION_TOGGLE } = requireCjs(SCRIPT); +const { findCoverageProblems, prebuiltTuplesInPack } = requireCjs(SCRIPT); describe('findCoverageProblems (pure decision core)', () => { it('passes when source ships, even with incomplete prebuilds (transitional state)', () => { - const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 0, hasSource: true }]; - expect(findCoverageProblems({ grammars, sourceExcluded: false })).toEqual([]); + const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 0, shipsSource: true }]; + expect(findCoverageProblems({ grammars })).toEqual([]); }); - it('fails when source is excluded but a grammar lacks 6/6 prebuilds', () => { - const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 4, hasSource: true }]; - const problems = findCoverageProblems({ grammars, sourceExcluded: true }); + it('fails when source is NOT fully shipped and a grammar lacks 6/6 prebuilds', () => { + // e.g. a partial .npmignore edit excluded binding.gyp → shipsSource false. + const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 4, shipsSource: false }]; + const problems = findCoverageProblems({ grammars }); expect(problems).toHaveLength(1); expect(problems[0]).toContain('tree-sitter-kotlin'); - expect(problems[0]).toContain('EXCLUDED'); + expect(problems[0]).toContain('source NOT fully shipped'); expect(problems[0]).toContain('2 platform-arch tuple(s)'); }); - it('passes when source is excluded but every grammar has all 6 prebuilds', () => { + it('passes when source is not shipped but every grammar has all 6 prebuilds', () => { const grammars = [ - { name: 'tree-sitter-swift', prebuilt: 6, hasSource: true }, - { name: 'tree-sitter-c', prebuilt: 6, hasSource: false }, + { name: 'tree-sitter-swift', prebuilt: 6, shipsSource: false }, + { name: 'tree-sitter-c', prebuilt: 6, shipsSource: false }, ]; - expect(findCoverageProblems({ grammars, sourceExcluded: true })).toEqual([]); + expect(findCoverageProblems({ grammars })).toEqual([]); }); - it('fails when a grammar has neither prebuilds nor source', () => { - const grammars = [{ name: 'tree-sitter-x', prebuilt: 0, hasSource: false }]; - const problems = findCoverageProblems({ grammars, sourceExcluded: false }); + it('fails when a grammar has neither prebuilds nor shipped source', () => { + const grammars = [{ name: 'tree-sitter-x', prebuilt: 0, shipsSource: false }]; + const problems = findCoverageProblems({ grammars }); expect(problems).toHaveLength(1); - expect(problems[0]).toContain('absent'); + expect(problems[0]).toContain('no loadable binding'); }); }); -describe('activeIgnorePatterns', () => { - it('ignores comments/blank lines and trims', () => { - const txt = ['# a comment', '', ' vendor/**/build ', '# vendor/**/src/parser.c'].join('\n'); - const active = activeIgnorePatterns(txt); - expect(active).toContain('vendor/**/build'); - expect(active).not.toContain(SOURCE_EXCLUSION_TOGGLE); // commented out → inert +describe('prebuiltTuplesInPack', () => { + it('counts only tuples whose .node is in the packed set (ignores non-.node / other grammars)', () => { + const packed = new Set([ + 'vendor/tree-sitter-swift/prebuilds/linux-x64/tree-sitter-swift.node', + 'vendor/tree-sitter-swift/prebuilds/darwin-arm64/tree-sitter-swift.node', + 'vendor/tree-sitter-swift/prebuilds/win32-x64/README.md', // not a .node + 'vendor/tree-sitter-c/prebuilds/linux-x64/tree-sitter-c.node', // other grammar + ]); + expect(prebuiltTuplesInPack('tree-sitter-swift', packed)).toBe(2); + expect(prebuiltTuplesInPack('tree-sitter-c', packed)).toBe(1); }); }); describe('real repo publish-safety (guards against premature .npmignore activation)', () => { it('the script exits 0 against the committed repo state', () => { - const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 30_000 }); + // The guard shells out to `npm pack --dry-run` — allow time for it. + const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 120_000 }); expect(r.status, r.stderr).toBe(0); expect(r.stdout).toContain('[publish-guard] OK'); });