scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-10-03 02:21:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

fix(publish): make the coverage guard deterministic — read files, not npm pack

Browse source 
The npm-pack-based guard timed out in CI: `npm pack`'s prepare/prepack lifecycle
is not skipped by `--ignore-scripts` (flag or env config) on the CI npm, so the
inner pack ran the full build (~20s+) — fine for the slow smoke job, but it blew
past vitest's 30s test timeout in the coverage job (and risked re-entering this
prepack guard).

Replace it with a deterministic, fast (~0.1s) check that needs no subprocess:
since `files: ["vendor"]` OVERRIDES `.npmignore` for the vendored subtree (so
`.npmignore` can never drop vendored source — verified), the ONLY lever that can
exclude source is narrowing the package.json `files` field. The guard now reads
`files` directly: a grammar "ships source" iff `files` includes the vendor
subtree AND the grammar carries a buildable source set on disk. A lean publish
that narrows `files` while a grammar lacks 6/6 prebuilds still fails the gate.
This commit is contained in:
Gergo Magyar 2026-06-09 14:09:54 +00:00
parent 5cdd395f7e
commit e57097c903
2 changed files with 98 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
gitnexus/scripts/assert-publish-grammar-coverage.cjs
View file

@ -7,28 +7,25 @@
* grammar's full source-build set ships (so the install can source-build it,
* toolchain permitting). A future lean publish — dropping the ~50 MB of generated
* source to ship prebuilds only — is safe ONLY once every grammar has all six
* prebuilds; doing it while any grammar is still missing a prebuild would ship a
* prebuilds; doing it while any grammar still lacks a prebuild would ship a
* grammar with NO loadable binding (neither prebuild nor buildable source) → that
* language is silently dead for users. (Note: the slim must be done by narrowing
* package.json's `files` field, NOT via .npmignore — `files` overrides .npmignore
* for the vendored subtree; see gitnexus/.npmignore.)
* language is silently dead for users.
*
* Rather than infer "is source excluded?" from a single .npmignore toggle line
* (which a partial/out-of-order edit could defeat — exclude binding.gyp but leave
* parser.c, and the grammar is unbuildable yet still looks "source-shipping"),
* this guard inspects the EFFECTIVE tarball: `npm pack --dry-run --ignore-scripts
* --json` (the `--ignore-scripts` avoids re-entering this guard via prepack). A
* grammar "ships source" only when EVERY one of its on-disk source-build inputs
* (binding.gyp + binding.cc + parser.c + scanner.c when present + a tree_sitter
* header) is actually in the packed file list.
* HOW SOURCE INCLUSION IS DECIDED. The `files` allow-list OVERRIDES `.npmignore`
* for the vendored subtree (verified: an active "vendor/(star-star)/src/parser.c"
* in .npmignore does NOT drop it from `npm pack`). So `.npmignore` can never
* exclude vendored source — the ONLY lever is the `files` field. A broad `vendor`
* ships the whole subtree (source + prebuilds); a lean publish narrows `files` to
* non-source subpaths. This guard therefore reads `files` directly rather than
* shelling out to `npm pack` (which, in prepack, would re-enter this guard and,
* on npm versions that don't honor --ignore-scripts for prepare/prepack, run the
* full build — slow enough to time out and fragile).
*
* Wired via `prepack`, so it fails `npm pack` / `npm publish` if the invariant is
* violated — the gated .npmignore exclusions can never be activated early and
* silently ship a dead grammar.
* violated.
*/
const fs = require('fs');
const path = require('path');
const { execSync } = require('child_process');
const TUPLES = [
'linux-x64',
@ -39,9 +36,9 @@ const TUPLES = [
'win32-arm64',
];
// Source-build inputs (tarball-relative within vendor/<name>/) whose presence in
// the pack makes a grammar source-buildable. Per-grammar we only require the ones
// that actually exist on disk (e.g. tree-sitter-c has no external scanner.c).
// Source-build inputs (relative to vendor/<name>/) whose presence makes a grammar
// source-buildable. Per-grammar we only require the ones that exist on disk (e.g.
// tree-sitter-c has no external scanner.c).
const SOURCE_BUILD_REL = [
'binding.gyp',
'bindings/node/binding.cc',
@ -50,19 +47,47 @@ const SOURCE_BUILD_REL = [
'src/tree_sitter/parser.h',
];
/** The on-disk source-build inputs for a grammar, as tarball-relative paths. */
function sourceBuildSet(grammarDir, name) {
return SOURCE_BUILD_REL.filter((rel) => fs.existsSync(path.join(grammarDir, rel))).map(
(rel) => `vendor/${name}/${rel}`,
);
/**
* Does the package.json `files` allow-list ship the WHOLE vendor subtree (and
* therefore the vendored grammar source)? A bare `vendor` (optionally with a
* trailing slash or `/**`/`/*`) includes everything under vendor/. A lean publish
* replaces that with non-source subpaths, so this returns false and grammars must
* then rely on prebuilds.
*/
function filesShipsVendorSource(filesField) {
return (filesField || []).some((f) => {
const n = String(f)
.replace(/\\/g, '/')
.replace(/\/+$/, '')
.replace(/\/\*\*?$/, '');
return n === 'vendor';
});
}
/** Count platform-arch tuples whose prebuilt .node is present in the packed set. */
function prebuiltTuplesInPack(name, packedFiles) {
/** The on-disk source-build inputs for a grammar (relative paths). */
function sourceBuildSet(grammarDir) {
return SOURCE_BUILD_REL.filter((rel) => fs.existsSync(path.join(grammarDir, rel)));
}
/** True when a grammar can be source-built from its vendored files (has gyp + parser). */
function isBuildableFromSource(grammarDir) {
const set = sourceBuildSet(grammarDir);
return set.includes('binding.gyp') && set.includes('src/parser.c');
}
/** Count platform-arch tuples with a committed prebuilt .node on disk. */
function countPrebuiltTuples(grammarDir) {
const pdir = path.join(grammarDir, 'prebuilds');
let n = 0;
for (const t of TUPLES) {
const prefix = `vendor/${name}/prebuilds/${t}/`;
if ([...packedFiles].some((f) => f.startsWith(prefix) && f.endsWith('.node'))) n++;
const td = path.join(pdir, t);
try {
if (fs.statSync(td).isDirectory() && fs.readdirSync(td).some((f) => f.endsWith('.node'))) {
n++;
}
} catch {
/* tuple dir absent — not covered */
}
}
return n;
}
@ -78,54 +103,28 @@ function findCoverageProblems({ grammars }) {
if (g.prebuilt < 6 && !g.shipsSource) {
const missing = 6 - g.prebuilt;
problems.push(
`${g.name}: ${g.prebuilt}/6 prebuilds in the tarball and source NOT fully shipped ` +
`(a source-build input is missing/excluded) — would ship with no loadable binding on ` +
`${missing} platform-arch tuple(s).`,
`${g.name}: ${g.prebuilt}/6 prebuilds and its vendored source is not shipped ` +
`(the package.json \`files\` field excludes it, or it is not buildable) — would ship ` +
`with no loadable binding on ${missing} platform-arch tuple(s).`,
);
}
}
return problems;
}
/** Build the set of tarball-relative paths `npm pack` would include. */
function packFileSet(cwd) {
const out = execSync('npm pack --dry-run --ignore-scripts --json', {
cwd,
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'ignore'],
// The `--ignore-scripts` FLAG is not reliably honored by `npm pack`'s
// prepare/prepack lifecycle on every npm version — when it isn't, build.js
// runs (polluting this --json stdout with `[build] …`) AND, since this guard
// runs in prepack, the inner pack would re-enter the guard (recursion). The
// `npm_config_ignore_scripts` env config IS reliable, so set it too.
env: { ...process.env, npm_config_ignore_scripts: 'true' },
});
// Defensive: if any lifecycle/build output still precedes the JSON array on
// stdout (e.g. `[build] …`), parse from the array start (`[` then `{`) rather
// than the raw stream. `[build]` does NOT match (no `{` after the bracket).
const start = out.search(/\[\s*\{/);
const parsed = JSON.parse(start >= 0 ? out.slice(start) : out);
const files = (parsed[0] && parsed[0].files) || [];
return new Set(files.map((f) => f.path.replace(/\\/g, '/')));
}
function collectGrammars(vendorDir, packedFiles) {
function collectGrammars(vendorDir, shipsVendorSource) {
if (!fs.existsSync(vendorDir)) return [];
return fs
.readdirSync(vendorDir)
.filter((d) => /^tree-sitter-/.test(d))
.map((name) => {
const dir = path.join(vendorDir, name);
const srcSet = sourceBuildSet(dir, name);
const buildable =
srcSet.includes(`vendor/${name}/src/parser.c`) &&
srcSet.includes(`vendor/${name}/binding.gyp`);
return {
name,
prebuilt: prebuiltTuplesInPack(name, packedFiles),
// Source ships only when the grammar is buildable AND every one of its
// source-build inputs is actually in the packed file list.
shipsSource: buildable && srcSet.every((f) => packedFiles.has(f)),
prebuilt: countPrebuiltTuples(dir),
// Source ships when `files` includes the vendor subtree AND the grammar
// actually carries a buildable source set on disk.
shipsSource: shipsVendorSource && isBuildableFromSource(dir),
};
});
}
@ -133,16 +132,10 @@ function collectGrammars(vendorDir, packedFiles) {
function main() {
const gitnexusRoot = path.join(__dirname, '..');
const vendorDir = path.join(gitnexusRoot, 'vendor');
const pkg = JSON.parse(fs.readFileSync(path.join(gitnexusRoot, 'package.json'), 'utf8'));
const shipsVendorSource = filesShipsVendorSource(pkg.files);
let packedFiles;
try {
packedFiles = packFileSet(gitnexusRoot);
} catch (err) {
console.error(`[publish-guard] Could not compute the npm pack file list: ${err.message}`);
process.exit(1);
}
const grammars = collectGrammars(vendorDir, packedFiles);
const grammars = collectGrammars(vendorDir, shipsVendorSource);
if (grammars.length === 0) {
console.error(`[publish-guard] No vendored tree-sitter grammars found under ${vendorDir}.`);
process.exit(1);
@ -154,7 +147,7 @@ function main() {
for (const p of problems) console.error(` - ${p}`);
console.error(
'\nFix: either commit the missing prebuilds (run the build-tree-sitter-prebuilds\n' +
'workflow) or re-comment the lean-publish source exclusions in gitnexus/.npmignore.',
'workflow) or keep the vendored source in the package.json `files` field.',
);
process.exit(1);
}
@ -170,10 +163,11 @@ if (require.main === module) main();
module.exports = {
findCoverageProblems,
prebuiltTuplesInPack,
filesShipsVendorSource,
isBuildableFromSource,
sourceBuildSet,
countPrebuiltTuples,
collectGrammars,
packFileSet,
TUPLES,
SOURCE_BUILD_REL,
};

54
gitnexus/test/unit/assert-publish-grammar-coverage.test.ts
View file

@ -7,18 +7,18 @@ import { fileURLToPath } from 'node:url';
* Coverage for the publish guard `scripts/assert-publish-grammar-coverage.cjs`.
*
* The guard refuses to pack/publish if a vendored grammar would ship with no
* loadable binding — i.e. a lean-publish `.npmignore` edit dropped a source-build
* input while a grammar still lacks 6/6 prebuilds. It decides "ships source" by
* inspecting the EFFECTIVE `npm pack` file list, so a partial exclusion can't slip
* past. We test the pure decision core directly, and assert the real repo state is
* publish-safe (catching a premature .npmignore activation in CI, not just at
* publish time).
* loadable binding — i.e. the package.json `files` field was narrowed to drop the
* vendored source while a grammar still lacks 6/6 prebuilds. (`.npmignore` can't
* exclude the vendored subtree — `files` overrides it — so `files` is the only
* lever, and the guard reads it directly rather than shelling out to `npm pack`.)
* We test the pure decision core + the `files` check directly, and assert the real
* repo state is publish-safe (catching a premature narrowing in CI).
*/
const requireCjs = createRequire(import.meta.url);
const SCRIPT = fileURLToPath(
new URL('../../scripts/assert-publish-grammar-coverage.cjs', import.meta.url),
);
const { findCoverageProblems, prebuiltTuplesInPack } = requireCjs(SCRIPT);
const { findCoverageProblems, filesShipsVendorSource } = requireCjs(SCRIPT);
describe('findCoverageProblems (pure decision core)', () => {
it('passes when source ships, even with incomplete prebuilds (transitional state)', () => {
@ -26,13 +26,12 @@ describe('findCoverageProblems (pure decision core)', () => {
expect(findCoverageProblems({ grammars })).toEqual([]);
});
it('fails when source is NOT fully shipped and a grammar lacks 6/6 prebuilds', () => {
// e.g. a partial .npmignore edit excluded binding.gyp → shipsSource false.
it('fails when source is not shipped and a grammar lacks 6/6 prebuilds', () => {
const grammars = [{ name: 'tree-sitter-kotlin', prebuilt: 4, shipsSource: false }];
const problems = findCoverageProblems({ grammars });
expect(problems).toHaveLength(1);
expect(problems[0]).toContain('tree-sitter-kotlin');
expect(problems[0]).toContain('source NOT fully shipped');
expect(problems[0]).toContain('not shipped');
expect(problems[0]).toContain('2 platform-arch tuple(s)');
});
@ -52,23 +51,32 @@ describe('findCoverageProblems (pure decision core)', () => {
});
});
describe('prebuiltTuplesInPack', () => {
it('counts only tuples whose .node is in the packed set (ignores non-.node / other grammars)', () => {
const packed = new Set([
'vendor/tree-sitter-swift/prebuilds/linux-x64/tree-sitter-swift.node',
'vendor/tree-sitter-swift/prebuilds/darwin-arm64/tree-sitter-swift.node',
'vendor/tree-sitter-swift/prebuilds/win32-x64/README.md', // not a .node
'vendor/tree-sitter-c/prebuilds/linux-x64/tree-sitter-c.node', // other grammar
]);
expect(prebuiltTuplesInPack('tree-sitter-swift', packed)).toBe(2);
expect(prebuiltTuplesInPack('tree-sitter-c', packed)).toBe(1);
describe('filesShipsVendorSource', () => {
it('ships when a broad vendor entry is present', () => {
expect(filesShipsVendorSource(['dist', 'vendor', 'web'])).toBe(true);
expect(filesShipsVendorSource(['vendor/'])).toBe(true);
expect(filesShipsVendorSource(['vendor/**'])).toBe(true);
expect(filesShipsVendorSource(['vendor/*'])).toBe(true);
});
it('does NOT ship when files is narrowed to non-source subpaths (lean publish)', () => {
expect(
filesShipsVendorSource([
'dist',
'vendor/**/prebuilds/**',
'vendor/**/package.json',
'vendor/**/bindings/node/index.js',
]),
).toBe(false);
expect(filesShipsVendorSource([])).toBe(false);
expect(filesShipsVendorSource(undefined)).toBe(false);
});
});
describe('real repo publish-safety (guards against premature .npmignore activation)', () => {
describe('real repo publish-safety (guards against premature files narrowing)', () => {
it('the script exits 0 against the committed repo state', () => {
// The guard shells out to `npm pack --dry-run` — allow time for it.
const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 120_000 });
// Deterministic: reads package.json + walks vendor/ — no npm pack, fast.
const r = spawnSync(process.execPath, [SCRIPT], { encoding: 'utf8', timeout: 20_000 });
expect(r.status, r.stderr).toBe(0);
expect(r.stdout).toContain('[publish-guard] OK');
});