diff --git a/gitnexus/package.json b/gitnexus/package.json index e68054a52..3ab737638 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -49,7 +49,7 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:cross-platform": "tsx scripts/run-cross-platform.ts", - "postinstall": "node scripts/materialize-vendor-grammars.cjs && node scripts/build-tree-sitter-c.cjs && node scripts/build-tree-sitter-dart.cjs && node scripts/build-tree-sitter-proto.cjs && node scripts/build-tree-sitter-swift.cjs && node scripts/build-tree-sitter-kotlin.cjs", + "postinstall": "node scripts/materialize-vendor-grammars.cjs && node scripts/build-tree-sitter-grammars.cjs", "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", "prepack": "node scripts/assert-publish-grammar-coverage.cjs && node scripts/build.js" diff --git a/gitnexus/scripts/build-tree-sitter-c.cjs b/gitnexus/scripts/build-tree-sitter-c.cjs deleted file mode 100644 index 64d3df7bd..000000000 --- a/gitnexus/scripts/build-tree-sitter-c.cjs +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env node -/** - * Activate the tree-sitter-c native binding after materialize-vendor-grammars.cjs. - * - * tree-sitter-c is a REQUIRED grammar, vendored at 0.21.4 for ABI compatibility - * with the bundled tree-sitter@0.21.1 runtime (#1242). It is vendored — not left - * a plain npm dependency — because upstream ships prebuilds for only 4 of 6 - * platform-archs (#2116), and a required dep with no matching prebuild - * hard-fails `npm install` on toolchain-less ARM before any gitnexus script runs. - * - * Resolution order: prefer a committed prebuild (toolchain-free, the goal once - * the build-tree-sitter-prebuilds workflow has populated all six); otherwise - * build from the vendored source (binding.gyp + src/) so C parsing still works - * on any host with a toolchain — e.g. CI, where the prebuilds may not yet be - * vendored. No GITNEXUS_SKIP gate: C is required, not a user-opt-out grammar. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -const cDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-c'); -const bindingGyp = path.join(cDir, 'binding.gyp'); -const bindingNode = path.join(cDir, 'build', 'Release', 'tree_sitter_c_binding.node'); - -try { - if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Prefer a committed prebuild for this platform-arch (no toolchain needed). - try { - require('node-gyp-build').path(cDir); - process.exit(0); - } catch { - // No matching prebuild — fall through to the source build below. - } - - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-c] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-c] C parsing will be unavailable until a prebuild or toolchain is present.', - ); - process.exit(0); - } - - console.log( - '[tree-sitter-c] No prebuild for this platform — building native binding from source...', - ); - execSync('npx node-gyp rebuild', { cwd: cDir, stdio: 'pipe', timeout: 180000 }); - console.log('[tree-sitter-c] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-c] Could not build native binding:', err.message); - console.warn('[tree-sitter-c] C parsing will be unavailable. Other languages are unaffected.'); - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-dart.cjs b/gitnexus/scripts/build-tree-sitter-dart.cjs deleted file mode 100644 index a34ca8e92..000000000 --- a/gitnexus/scripts/build-tree-sitter-dart.cjs +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env node -/** - * Build tree-sitter-dart native binding in node_modules/ after materialize-vendor-grammars.cjs. - * Vendored source lives in vendor/ only; see #836 and #1728. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: skip the native rebuild entirely. Dart parsing becomes -// unavailable but `npm install gitnexus` finishes much faster on machines -// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0' -// (read as a string), and any other value all fall through to the rebuild. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-dart] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Dart parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const dartDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-dart'); -const bindingGyp = path.join(dartDir, 'binding.gyp'); -const bindingNode = path.join(dartDir, 'build', 'Release', 'tree_sitter_dart_binding.node'); - -try { - if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Prefer a committed prebuild (toolchain-free). If node-gyp-build resolves a - // binary for this platform-arch under prebuilds/, no source build is needed. - // Before any prebuilds are vendored this throws and we fall through to the - // source build below — no behavior change. - try { - require('node-gyp-build').path(dartDir); - process.exit(0); - } catch { - // No matching prebuild for this host — fall through to the source build. - } - - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-dart] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-dart] Dart parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', - ); - process.exit(0); - } - - console.log('[tree-sitter-dart] Building native binding...'); - execSync('npx node-gyp rebuild', { - cwd: dartDir, - stdio: 'pipe', - timeout: 180000, - }); - console.log('[tree-sitter-dart] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-dart] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-dart] Dart parsing will be unavailable. Non-Dart functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-grammars.cjs b/gitnexus/scripts/build-tree-sitter-grammars.cjs new file mode 100644 index 000000000..dd3d219e4 --- /dev/null +++ b/gitnexus/scripts/build-tree-sitter-grammars.cjs @@ -0,0 +1,120 @@ +#!/usr/bin/env node +/** + * Activate the vendored tree-sitter native bindings after + * materialize-vendor-grammars.cjs. One registry-driven script replaces the + * former per-grammar build-tree-sitter-.cjs files (they were ~95% + * identical). + * + * For each grammar the resolution order is identical: + * 1. If the package isn't materialized (no binding.gyp) or the binding is + * already built, do nothing. + * 2. Prefer a committed prebuild for this platform-arch (toolchain-free) via + * node-gyp-build — the goal once build-tree-sitter-prebuilds.yml has + * populated all six tuples. + * 3. Otherwise source-build from the vendored grammar source (binding.gyp + + * src/) so parsing still works on any toolchain host — e.g. CI, before the + * prebuilds land. + * + * HARD INVARIANT: this runs in `gitnexus`'s postinstall, so it MUST NEVER throw + * or exit non-zero — a failure for any single grammar must not break the install. + * + * Opt-out: GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (strict '1') skips the OPTIONAL + * grammars only. tree-sitter-c is REQUIRED (it backstops upstream's 4/6 ARM + * prebuild gap, #2116) and is always built. + * + * Usage: + * node build-tree-sitter-grammars.cjs # all grammars (postinstall) + * node build-tree-sitter-grammars.cjs swift c # only the named grammars + */ +const fs = require('fs'); +const path = require('path'); +const { execSync } = require('child_process'); + +// Registry. `display`/`ext` drive the human-readable warnings; `required` +// grammars ignore the opt-out gate. Insertion order == build order (c first). +const GRAMMARS = { + c: { required: true, display: 'C', ext: '.c' }, + dart: { required: false, display: 'Dart', ext: '.dart' }, + proto: { required: false, display: 'Proto', ext: '.proto' }, + swift: { required: false, display: 'Swift', ext: '.swift' }, + kotlin: { required: false, display: 'Kotlin', ext: '.kt/.kts' }, +}; + +const skipOptional = process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1'; + +function buildGrammar(short) { + const cfg = GRAMMARS[short]; + const tag = `[tree-sitter-${short}]`; + + if (!cfg.required && skipOptional) { + console.warn( + `${tag} Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). ${cfg.display} parsing will be unavailable until reinstalled without the env var.`, + ); + return; + } + + const dir = path.join(__dirname, '..', 'node_modules', `tree-sitter-${short}`); + const bindingGyp = path.join(dir, 'binding.gyp'); + const bindingNode = path.join(dir, 'build', 'Release', `tree_sitter_${short}_binding.node`); + + try { + // Not materialized (no source), or already built — nothing to do. + if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { + return; + } + + // Prefer a committed prebuild for this platform-arch (no toolchain needed). + try { + require('node-gyp-build').path(dir); + return; + } catch { + // No matching prebuild — fall through to the source build below. + } + + // The hoisted build deps must be resolvable to source-build. + try { + require.resolve('node-addon-api'); + require.resolve('node-gyp-build'); + } catch (resolveErr) { + console.warn( + `${tag} Skipping build: hoisted build deps not resolvable (${resolveErr.message}).`, + ); + console.warn( + `${tag} ${cfg.display} parsing will be unavailable until a prebuild or toolchain is present.`, + ); + return; + } + + console.log(`${tag} No prebuild for this platform — building native binding from source...`); + execSync('npx node-gyp rebuild', { cwd: dir, stdio: 'pipe', timeout: 180000 }); + console.log(`${tag} Native binding built successfully`); + } catch (err) { + console.warn(`${tag} Could not build native binding:`, err.message); + console.warn( + `${tag} ${cfg.display} (${cfg.ext}) parsing will be unavailable. Non-${cfg.display} functionality is unaffected.`, + ); + } +} + +function main() { + const args = process.argv.slice(2).filter(Boolean); + const targets = args.length > 0 ? args : Object.keys(GRAMMARS); + for (const short of targets) { + if (!GRAMMARS[short]) { + console.warn(`[tree-sitter] Unknown grammar '${short}' — skipping.`); + continue; + } + // Defensive: never let an unexpected throw escape and fail the install. + try { + buildGrammar(short); + } catch (err) { + console.warn(`[tree-sitter-${short}] Unexpected build error (ignored): ${err.message}`); + } + } + // Hard guarantee: postinstall must never exit non-zero. + process.exit(0); +} + +if (require.main === module) main(); + +module.exports = { GRAMMARS, buildGrammar }; diff --git a/gitnexus/scripts/build-tree-sitter-kotlin.cjs b/gitnexus/scripts/build-tree-sitter-kotlin.cjs deleted file mode 100644 index e2aeb1c82..000000000 --- a/gitnexus/scripts/build-tree-sitter-kotlin.cjs +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env node -/** - * Activate the tree-sitter-kotlin native binding after materialize-vendor-grammars.cjs. - * - * Kotlin is vendored (upstream ships source only; GitNexus cross-builds the - * prebuilds via .github/workflows/build-tree-sitter-prebuilds.yml). Resolution - * order, mirroring Dart/Proto/C: prefer a committed prebuild for this - * platform-arch (toolchain-free); otherwise build from the vendored source so - * Kotlin parsing still works on any host with a toolchain — e.g. CI, where the - * prebuilds may not yet be vendored. - * - * MUST NEVER throw or exit non-zero — it must never break `gitnexus` install. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: Kotlin is optional, so the env var skips its build entirely (also -// skipped at materialize). Strict `=== '1'` only. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-kotlin] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Kotlin parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const kotlinDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-kotlin'); -const bindingGyp = path.join(kotlinDir, 'binding.gyp'); -const bindingNode = path.join(kotlinDir, 'build', 'Release', 'tree_sitter_kotlin_binding.node'); - -try { - if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Prefer a committed prebuild for this platform-arch (no toolchain needed). - try { - require('node-gyp-build').path(kotlinDir); - process.exit(0); - } catch { - // No matching prebuild — fall through to the source build below. - } - - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-kotlin] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-kotlin] Kotlin parsing will be unavailable until a prebuild or toolchain is present.', - ); - process.exit(0); - } - - console.log( - '[tree-sitter-kotlin] No prebuild for this platform — building native binding from source...', - ); - execSync('npx node-gyp rebuild', { cwd: kotlinDir, stdio: 'pipe', timeout: 180000 }); - console.log('[tree-sitter-kotlin] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-kotlin] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-kotlin] Kotlin (.kt/.kts) parsing will be unavailable. Non-Kotlin functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-proto.cjs b/gitnexus/scripts/build-tree-sitter-proto.cjs deleted file mode 100644 index 3d425e217..000000000 --- a/gitnexus/scripts/build-tree-sitter-proto.cjs +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env node -/** - * Build tree-sitter-proto native binding. - * - * Why this script exists: - * tree-sitter-proto is vendored under gitnexus/vendor/tree-sitter-proto/ - * and copied into node_modules/ by materialize-vendor-grammars.cjs. Previously, the vendored - * package had its own `dependencies` and `install` script, which caused - * npm to create `vendor/tree-sitter-proto/node_modules/` and - * `vendor/tree-sitter-proto/build/` during install. Those directories - * blocked `rmdir` on global-install upgrade, producing: - * - * ENOTEMPTY: directory not empty, rmdir - * '.../gitnexus/vendor/tree-sitter-proto/node_modules/node-addon-api' - * - * (See https://github.com/abhigyanpatwari/GitNexus/issues/836.) - * - * We stripped `dependencies` and the `install` script from the vendored - * package.json, hoisted `node-addon-api` and `node-gyp-build` into - * gitnexus's own optionalDependencies, and moved native compilation here. - * - * What this does: - * Runs `npx node-gyp rebuild` inside `node_modules/tree-sitter-proto/`. - * Build output lands in - * `node_modules/tree-sitter-proto/build/Release/tree_sitter_proto_binding.node` - * — under npm-managed territory, safe on upgrade. - * - * Mirrors the tree-sitter-dart build helper. Best-effort: if any - * precondition fails (optional dep absent, no toolchain, --ignore-scripts), - * warn and exit 0 so gitnexus install still succeeds. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: skip the native rebuild entirely. Proto parsing becomes -// unavailable but `npm install gitnexus` finishes much faster on machines -// without a C++ toolchain. Strict `=== '1'` only — '=true', '=yes', '=0' -// (read as a string), and any other value all fall through to the rebuild. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-proto] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Proto parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const protoDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-proto'); -const bindingGyp = path.join(protoDir, 'binding.gyp'); -const bindingNode = path.join(protoDir, 'build', 'Release', 'tree_sitter_proto_binding.node'); - -try { - if (!fs.existsSync(bindingGyp)) { - // tree-sitter-proto is an optionalDependency; absent when install - // skipped optional deps or the file: dep was not resolved. - process.exit(0); - } - - // Skip if the native binding already exists (idempotent re-run). - if (fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Prefer a committed prebuild (toolchain-free). If node-gyp-build resolves a - // binary for this platform-arch under prebuilds/, no source build is needed. - // Before any prebuilds are vendored this throws and we fall through to the - // source build below — no behavior change. - try { - require('node-gyp-build').path(protoDir); - process.exit(0); - } catch { - // No matching prebuild for this host — fall through to the source build. - } - - // Pre-flight: the hoisted build deps must be resolvable. - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-proto] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-proto] Proto parsing will be unavailable. Install without --no-optional and with scripts enabled to build.', - ); - process.exit(0); - } - - console.log('[tree-sitter-proto] Building native binding...'); - execSync('npx node-gyp rebuild', { - cwd: protoDir, - stdio: 'pipe', - timeout: 180000, - }); - console.log('[tree-sitter-proto] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-proto] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-proto] Proto (.proto) parsing will be unavailable. Non-proto gitnexus functionality is unaffected.', - ); - // Exit 0: optionalDependency failures must not fail the gitnexus install. - process.exit(0); -} diff --git a/gitnexus/scripts/build-tree-sitter-swift.cjs b/gitnexus/scripts/build-tree-sitter-swift.cjs deleted file mode 100644 index 2228826fa..000000000 --- a/gitnexus/scripts/build-tree-sitter-swift.cjs +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env node -/** - * Activate the tree-sitter-swift native binding after materialize-vendor-grammars.cjs. - * - * Swift is vendored. Unlike its historical prebuild-only form, the grammar - * source (parser.c/scanner.c/binding.gyp + src/) is now ALSO vendored, so this - * script mirrors Dart/Proto/Kotlin/C exactly: prefer a committed prebuild for - * this platform-arch (toolchain-free); otherwise build from the vendored source - * so Swift parsing still works on any host with a toolchain — e.g. CI, where the - * GitNexus-cross-built prebuilds may not yet be vendored. The committed - * prebuilds for every platform-arch are produced by - * .github/workflows/build-tree-sitter-prebuilds.yml. - * - * MUST NEVER throw or exit non-zero — it must never break `gitnexus` install. - */ -const fs = require('fs'); -const path = require('path'); -const { execSync } = require('child_process'); - -// Opt-out: Swift is optional, so the env var skips its build entirely (also -// skipped at materialize). Strict `=== '1'` only. -if (process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1') { - console.warn( - '[tree-sitter-swift] Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1). Swift parsing will be unavailable until reinstalled without the env var.', - ); - process.exit(0); -} - -const swiftDir = path.join(__dirname, '..', 'node_modules', 'tree-sitter-swift'); -const bindingGyp = path.join(swiftDir, 'binding.gyp'); -const bindingNode = path.join(swiftDir, 'build', 'Release', 'tree_sitter_swift_binding.node'); - -try { - if (!fs.existsSync(bindingGyp) || fs.existsSync(bindingNode)) { - process.exit(0); - } - - // Prefer a committed prebuild for this platform-arch (no toolchain needed). - try { - require('node-gyp-build').path(swiftDir); - process.exit(0); - } catch { - // No matching prebuild — fall through to the source build below. - } - - try { - require.resolve('node-addon-api'); - require.resolve('node-gyp-build'); - } catch (resolveErr) { - console.warn( - '[tree-sitter-swift] Skipping build: hoisted build deps not resolvable (%s).', - resolveErr.message, - ); - console.warn( - '[tree-sitter-swift] Swift parsing will be unavailable until a prebuild or toolchain is present.', - ); - process.exit(0); - } - - console.log( - '[tree-sitter-swift] No prebuild for this platform — building native binding from source...', - ); - execSync('npx node-gyp rebuild', { cwd: swiftDir, stdio: 'pipe', timeout: 180000 }); - console.log('[tree-sitter-swift] Native binding built successfully'); -} catch (err) { - console.warn('[tree-sitter-swift] Could not build native binding:', err.message); - console.warn( - '[tree-sitter-swift] Swift (.swift) parsing will be unavailable. Non-Swift functionality is unaffected.', - ); - process.exit(0); -} diff --git a/gitnexus/test/unit/build-tree-sitter-grammars-probe.test.ts b/gitnexus/test/unit/build-tree-sitter-grammars-probe.test.ts new file mode 100644 index 000000000..6b05f53b6 --- /dev/null +++ b/gitnexus/test/unit/build-tree-sitter-grammars-probe.test.ts @@ -0,0 +1,125 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** + * Behavioral coverage for the consolidated activation script + * `scripts/build-tree-sitter-grammars.cjs` (replaces the per-grammar + * build-tree-sitter-.cjs files). + * + * For each grammar it prefers a committed prebuild (toolchain-free); if none + * matches it source-builds from the vendored source. Its hard invariant is that + * it MUST NEVER exit non-zero — it runs in `gitnexus`'s postinstall, so a + * non-zero exit would break `npm install gitnexus`. This suite runs the real + * script bytes (targeting one grammar via the CLI arg) across its branches and + * asserts exit code 0 every time, plus the required-vs-optional opt-out split. + * + * The script is copied into an isolated temp `scripts/` dir so its + * `__dirname`-relative `../node_modules/tree-sitter-` resolves under our + * control. The temp dir has no reachable `node-gyp-build` / `node-addon-api`, so + * the source-build path stops at the "hoisted build deps not resolvable" guard + * (still exit 0) instead of invoking a real compile. + */ + +const scriptSource = readFileSync( + fileURLToPath(new URL('../../scripts/build-tree-sitter-grammars.cjs', import.meta.url)), + 'utf8', +); + +let tmpRoot: string; +let scriptPath: string; + +beforeAll(() => { + tmpRoot = mkdtempSync(path.join(tmpdir(), 'gn-grammars-build-')); + mkdirSync(path.join(tmpRoot, 'scripts'), { recursive: true }); + scriptPath = path.join(tmpRoot, 'scripts', 'build-tree-sitter-grammars.cjs'); + writeFileSync(scriptPath, scriptSource); +}); + +afterAll(() => { + rmSync(tmpRoot, { recursive: true, force: true }); +}); + +function runBuild(grammar: string, overrides: Record) { + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (v !== undefined) env[k] = v; + } + delete env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS; + for (const [k, v] of Object.entries(overrides)) { + if (v === undefined) delete env[k]; + else env[k] = v; + } + return spawnSync(process.execPath, [scriptPath, grammar], { + env, + encoding: 'utf8', + timeout: 30_000, + }); +} + +function materializeShell(grammar: string) { + // A package shell with a binding.gyp present but no prebuild / built binary. + const pkg = path.join(tmpRoot, 'node_modules', `tree-sitter-${grammar}`); + mkdirSync(path.join(pkg, 'bindings', 'node'), { recursive: true }); + writeFileSync(path.join(pkg, 'binding.gyp'), '{ "targets": [] }'); + writeFileSync(path.join(pkg, 'bindings', 'node', 'index.js'), ''); +} + +describe('build-tree-sitter-grammars.cjs consolidated activation', () => { + it('optional grammar: exits 0 and reports skipping under GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1', () => { + const r = runBuild('swift', { GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }); + expect(r.status).toBe(0); + expect(r.signal).toBeNull(); + expect(r.stderr).toContain('[tree-sitter-swift] Skipping build'); + expect(r.stderr).not.toContain('Swift (.swift) parsing will be unavailable'); + }); + + it('REQUIRED grammar (c): ignores GITNEXUS_SKIP_OPTIONAL_GRAMMARS (no skip message)', () => { + // c is required — the opt-out must NOT short-circuit it. With nothing + // materialized it silently exits 0 at the binding.gyp-absent check. + const r = runBuild('c', { GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }); + expect(r.status).toBe(0); + expect(r.signal).toBeNull(); + expect(r.stderr).not.toContain('Skipping build (GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1)'); + }); + + it('exits 0 silently when the materialized package is absent (no binding.gyp)', () => { + const r = runBuild('kotlin', {}); + expect(r.status).toBe(0); + expect(r.signal).toBeNull(); + expect(r.stderr).not.toContain('Kotlin (.kt/.kts) parsing will be unavailable'); + }); + + it('exits 0 (warning) when a grammar has a binding.gyp but no prebuild/build deps', () => { + materializeShell('kotlin'); + try { + const r = runBuild('kotlin', {}); + expect(r.status).toBe(0); + expect(r.signal).toBeNull(); + expect(r.stderr).toMatch(/hoisted build deps not resolvable|Could not build native binding/); + expect(r.stderr).not.toContain('built successfully'); + } finally { + rmSync(path.join(tmpRoot, 'node_modules'), { recursive: true, force: true }); + } + }); + + it('unknown grammar arg: warns and exits 0', () => { + const r = runBuild('haskell', {}); + expect(r.status).toBe(0); + expect(r.signal).toBeNull(); + expect(r.stderr).toContain("Unknown grammar 'haskell'"); + }); + + it('never exits non-zero across grammars and env permutations (postinstall hard invariant)', () => { + for (const grammar of ['c', 'dart', 'proto', 'swift', 'kotlin']) { + for (const overrides of [{ GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }, {}]) { + const r = runBuild(grammar, overrides); + expect(r.status, `${grammar} ${JSON.stringify(overrides)}`).toBe(0); + expect(r.signal).toBeNull(); + } + } + }); +}); diff --git a/gitnexus/test/unit/build-tree-sitter-kotlin-probe.test.ts b/gitnexus/test/unit/build-tree-sitter-kotlin-probe.test.ts deleted file mode 100644 index f168a9b9d..000000000 --- a/gitnexus/test/unit/build-tree-sitter-kotlin-probe.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { spawnSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -/** - * Behavioral coverage for the postinstall activation script - * `scripts/build-tree-sitter-kotlin.cjs`. - * - * Kotlin is a vendored grammar (like Swift/Dart/Proto/C). The script prefers a - * committed prebuild for this platform-arch (toolchain-free); if none matches it - * source-builds from the vendored grammar source. Its hard invariant is that it - * MUST NEVER exit non-zero — it runs in `gitnexus`'s postinstall, so a non-zero - * exit would break `npm install gitnexus` for every user. This suite executes - * the real script bytes across its branches and asserts exit code 0 every time. - * - * The script is copied into an isolated temp `scripts/` dir so its - * `__dirname`-relative `../node_modules/tree-sitter-kotlin` resolves under our - * control (absent dir, or a present-but-unbuildable dir) without touching the - * repo's real node_modules. The temp dir has no reachable `node-gyp-build` / - * `node-addon-api`, so the source-build path stops at the "hoisted build deps - * not resolvable" guard (still exit 0) instead of invoking a real compile. - */ - -const probeSource = readFileSync( - fileURLToPath(new URL('../../scripts/build-tree-sitter-kotlin.cjs', import.meta.url)), - 'utf8', -); - -// Catch-branch sentinel (only printed when an actual node-gyp build is attempted -// and throws) — must NOT appear on the deps-unavailable guard path. -const CATCH_UNAVAILABLE = 'Kotlin (.kt/.kts) parsing will be unavailable'; - -let tmpRoot: string; -let scriptPath: string; - -beforeAll(() => { - tmpRoot = mkdtempSync(path.join(tmpdir(), 'gn-kotlin-build-')); - mkdirSync(path.join(tmpRoot, 'scripts'), { recursive: true }); - scriptPath = path.join(tmpRoot, 'scripts', 'build-tree-sitter-kotlin.cjs'); - writeFileSync(scriptPath, probeSource); -}); - -afterAll(() => { - rmSync(tmpRoot, { recursive: true, force: true }); -}); - -function runProbe(overrides: Record) { - const env: Record = {}; - for (const [k, v] of Object.entries(process.env)) { - if (v !== undefined) env[k] = v; - } - delete env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS; - for (const [k, v] of Object.entries(overrides)) { - if (v === undefined) delete env[k]; - else env[k] = v; - } - return spawnSync(process.execPath, [scriptPath], { env, encoding: 'utf8', timeout: 30_000 }); -} - -describe('build-tree-sitter-kotlin.cjs vendored grammar activation', () => { - it('exits 0 and reports skipping when GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1', () => { - const r = runProbe({ GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - expect(r.stderr).toContain('Skipping build'); - expect(r.stderr).not.toContain(CATCH_UNAVAILABLE); - }); - - it('exits 0 silently when the materialized package is absent (no binding.gyp)', () => { - // No node_modules/tree-sitter-kotlin next to the script — materialize was - // skipped/failed, so there is no binding.gyp to build. Silent exit 0. - const r = runProbe({}); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - expect(r.stderr).not.toContain(CATCH_UNAVAILABLE); - }); - - it('exits 0 (warning) when the package has a binding.gyp but no prebuild/build deps', () => { - // Materialize a package with binding.gyp present but no prebuild and no - // build/Release/*.node. The script falls through prefer-prebuild to the - // source-build path; in this temp env node-gyp-build/node-addon-api are not - // resolvable, so it stops at the deps guard (or, if they were resolvable, - // the node-gyp build would fail) — either way it warns and exits 0. - const pkg = path.join(tmpRoot, 'node_modules', 'tree-sitter-kotlin'); - mkdirSync(path.join(pkg, 'bindings', 'node'), { recursive: true }); - writeFileSync(path.join(pkg, 'binding.gyp'), '{ "targets": [] }'); - writeFileSync(path.join(pkg, 'bindings', 'node', 'index.js'), ''); - try { - const r = runProbe({}); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - expect(r.stderr).toMatch(/hoisted build deps not resolvable|Could not build native binding/); - expect(r.stderr).not.toContain('built successfully'); - } finally { - rmSync(path.join(tmpRoot, 'node_modules'), { recursive: true, force: true }); - } - }); - - it('never exits non-zero across env permutations (postinstall hard invariant)', () => { - for (const overrides of [{ GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }, {}]) { - const r = runProbe(overrides); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - } - }); -}); diff --git a/gitnexus/test/unit/build-tree-sitter-swift-probe.test.ts b/gitnexus/test/unit/build-tree-sitter-swift-probe.test.ts deleted file mode 100644 index 85a0c733a..000000000 --- a/gitnexus/test/unit/build-tree-sitter-swift-probe.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { spawnSync } from 'node:child_process'; -import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -/** - * Behavioral coverage for the postinstall activation script - * `scripts/build-tree-sitter-swift.cjs`. - * - * Swift is a vendored grammar, unified with Kotlin/Dart/Proto/C: the script - * prefers a committed prebuild for this platform-arch (toolchain-free); if none - * matches it source-builds from the vendored grammar source. Its hard invariant - * is that it MUST NEVER exit non-zero — it runs in `gitnexus`'s postinstall, so a - * non-zero exit would break `npm install gitnexus` for every user. This suite - * executes the real script bytes across its branches and asserts exit code 0 - * every time (mirrors build-tree-sitter-kotlin-probe.test.ts). - * - * The script is copied into an isolated temp `scripts/` dir so its - * `__dirname`-relative `../node_modules/tree-sitter-swift` resolves under our - * control. The temp dir has no reachable `node-gyp-build` / `node-addon-api`, so - * the source-build path stops at the "hoisted build deps not resolvable" guard - * (still exit 0) instead of invoking a real compile. - */ - -const probeSource = readFileSync( - fileURLToPath(new URL('../../scripts/build-tree-sitter-swift.cjs', import.meta.url)), - 'utf8', -); - -// Catch-branch sentinel (only printed when an actual node-gyp build is attempted -// and throws) — must NOT appear on the deps-unavailable guard path. -const CATCH_UNAVAILABLE = 'Swift (.swift) parsing will be unavailable'; - -let tmpRoot: string; -let scriptPath: string; - -beforeAll(() => { - tmpRoot = mkdtempSync(path.join(tmpdir(), 'gn-swift-build-')); - mkdirSync(path.join(tmpRoot, 'scripts'), { recursive: true }); - scriptPath = path.join(tmpRoot, 'scripts', 'build-tree-sitter-swift.cjs'); - writeFileSync(scriptPath, probeSource); -}); - -afterAll(() => { - rmSync(tmpRoot, { recursive: true, force: true }); -}); - -function runProbe(overrides: Record) { - const env: Record = {}; - for (const [k, v] of Object.entries(process.env)) { - if (v !== undefined) env[k] = v; - } - delete env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS; - for (const [k, v] of Object.entries(overrides)) { - if (v === undefined) delete env[k]; - else env[k] = v; - } - return spawnSync(process.execPath, [scriptPath], { env, encoding: 'utf8', timeout: 30_000 }); -} - -describe('build-tree-sitter-swift.cjs vendored grammar activation', () => { - it('exits 0 and reports skipping when GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1', () => { - const r = runProbe({ GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - expect(r.stderr).toContain('Skipping build'); - expect(r.stderr).not.toContain(CATCH_UNAVAILABLE); - }); - - it('exits 0 silently when the materialized package is absent (no binding.gyp)', () => { - // No node_modules/tree-sitter-swift next to the script — materialize was - // skipped/failed, so there is no binding.gyp to build. Silent exit 0. - const r = runProbe({}); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - expect(r.stderr).not.toContain(CATCH_UNAVAILABLE); - }); - - it('exits 0 (warning) when the package has a binding.gyp but no prebuild/build deps', () => { - // Materialize a package with binding.gyp present but no prebuild and no - // build/Release/*.node. The script falls through prefer-prebuild to the - // source-build path; in this temp env node-gyp-build/node-addon-api are not - // resolvable, so it stops at the deps guard (or, if they were resolvable, - // the node-gyp build would fail) — either way it warns and exits 0. - const pkg = path.join(tmpRoot, 'node_modules', 'tree-sitter-swift'); - mkdirSync(path.join(pkg, 'bindings', 'node'), { recursive: true }); - writeFileSync(path.join(pkg, 'binding.gyp'), '{ "targets": [] }'); - writeFileSync(path.join(pkg, 'bindings', 'node', 'index.js'), ''); - try { - const r = runProbe({}); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - expect(r.stderr).toMatch(/hoisted build deps not resolvable|Could not build native binding/); - expect(r.stderr).not.toContain('built successfully'); - } finally { - rmSync(path.join(tmpRoot, 'node_modules'), { recursive: true, force: true }); - } - }); - - it('never exits non-zero across env permutations (postinstall hard invariant)', () => { - for (const overrides of [{ GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }, {}]) { - const r = runProbe(overrides); - expect(r.status).toBe(0); - expect(r.signal).toBeNull(); - } - }); -}); diff --git a/gitnexus/test/unit/cli-commands.test.ts b/gitnexus/test/unit/cli-commands.test.ts index 127b7c5ef..c3b4ce9b9 100644 --- a/gitnexus/test/unit/cli-commands.test.ts +++ b/gitnexus/test/unit/cli-commands.test.ts @@ -93,14 +93,14 @@ describe('CLI commands', () => { // gate's assumptions (setTimeoutMicros semantics, ABI 13–14 grammar // range) can't drift under a minor bump. expect(pkg.default.dependencies['tree-sitter']).toBe('0.21.1'); - expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-swift.cjs'); + expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-grammars.cjs'); expect(swiftPkg.default.version).toBe('0.7.1'); // No scripts.install / dependencies inside vendor/ (#836 / #1728 hygiene). expect(swiftPkg.default.scripts?.install).toBeUndefined(); expect(swiftPkg.default.dependencies).toBeUndefined(); expect(swiftPkg.default.peerDependencies['tree-sitter']).toContain('^0.21.1'); // Swift is now unified with Dart/Proto/Kotlin/C: the grammar SOURCE is - // vendored so build-tree-sitter-swift.cjs can source-build the binding + // vendored so build-tree-sitter-grammars.cjs can source-build the binding // when no committed prebuild matches (e.g. CI before prebuilds land). const bindingGyp = await fs.readFile( path.join(REPO_ROOT, 'gitnexus/vendor/tree-sitter-swift/binding.gyp'), @@ -123,7 +123,7 @@ describe('CLI commands', () => { // optionalDependency. Its prebuilds are GitNexus-cross-built (upstream // ships source only) and materialized into node_modules/ at postinstall. expect(optional['tree-sitter-kotlin']).toBeUndefined(); - expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-kotlin.cjs'); + expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-grammars.cjs'); expect(kotlinPkg.default.version).toBe('0.3.8'); // No scripts.install / dependencies inside vendor/ (#836 / #1728 hygiene). expect(kotlinPkg.default.scripts?.install).toBeUndefined(); @@ -140,7 +140,7 @@ describe('CLI commands', () => { // (upstream ships 4/6). Vendored with GitNexus-built prebuilds for all 6, // held at 0.21.4 for ABI safety (#1242) — so it is NOT an npm dependency. expect(pkg.default.dependencies['tree-sitter-c']).toBeUndefined(); - expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-c.cjs'); + expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-grammars.cjs'); expect(cPkg.default.version).toBe('0.21.4'); expect(cPkg.default.scripts?.install).toBeUndefined(); expect(cPkg.default.dependencies).toBeUndefined(); diff --git a/gitnexus/vendor/tree-sitter-c/README.md b/gitnexus/vendor/tree-sitter-c/README.md index 52e7c86bc..03f7d0ec8 100644 --- a/gitnexus/vendor/tree-sitter-c/README.md +++ b/gitnexus/vendor/tree-sitter-c/README.md @@ -5,7 +5,7 @@ This directory is a GitNexus-managed **runtime** package derived from (`bindings/node/`, `src/node-types.json`, `LICENSE`), the native `prebuilds/`, **and** the grammar source (`binding.gyp`, `src/parser.c`, `src/tree_sitter/`). The prebuilds make C parsing toolchain-free; the source lets -`build-tree-sitter-c.cjs` compile the binding on a toolchain host when no +`build-tree-sitter-grammars.cjs` compile the binding on a toolchain host when no prebuild matches (e.g. CI before the prebuilds are vendored). ### Why this is vendored (unlike the other npm grammars) diff --git a/gitnexus/vendor/tree-sitter-kotlin/README.md b/gitnexus/vendor/tree-sitter-kotlin/README.md index 5a9af1970..216354156 100644 --- a/gitnexus/vendor/tree-sitter-kotlin/README.md +++ b/gitnexus/vendor/tree-sitter-kotlin/README.md @@ -15,8 +15,8 @@ Upstream `tree-sitter-kotlin` ships **source only** — its npm tarball has no and requires a C/C++ toolchain (`python3`/`make`/`g++`). To make Kotlin parsing toolchain-free on every host (Swift parity), GitNexus builds the platform prebuilds itself and vendors them here. `node-gyp-build` selects the correct -binary at require time; `build-tree-sitter-kotlin.cjs` probes availability at -install time. +binary at require time; `build-tree-sitter-grammars.cjs` activates the binding +(prefer prebuild, else source-build) at install time. `tree-sitter-swift` is handled the same way now: its prebuilds were originally **copied from upstream** (Swift ships them), but it is unified with this pipeline — diff --git a/gitnexus/vendor/tree-sitter-swift/README.md b/gitnexus/vendor/tree-sitter-swift/README.md index 53f7a26d7..cc2aaca9c 100644 --- a/gitnexus/vendor/tree-sitter-swift/README.md +++ b/gitnexus/vendor/tree-sitter-swift/README.md @@ -8,7 +8,7 @@ upgrade is handled separately. Unified with the Dart/Proto/Kotlin/C vendored grammars, this copy also vendors the grammar **source** — `binding.gyp`, `bindings/node/binding.cc`, `src/parser.c` (the ABI-14 default; ~18 MB, compresses heavily in git), -`src/scanner.c`, and `src/tree_sitter/` — so `gitnexus/scripts/build-tree-sitter-swift.cjs` +`src/scanner.c`, and `src/tree_sitter/` — so `gitnexus/scripts/build-tree-sitter-grammars.cjs` can source-build the native binding on any toolchain host when no committed prebuild matches (e.g. CI before the prebuilds land). Note: upstream deliberately omits the generated `parser.c` (see the FAQ below); GitNexus diff --git a/gitnexus/vendor/tree-sitter-swift/package.json b/gitnexus/vendor/tree-sitter-swift/package.json index 8a2ce7d61..3330c784c 100644 --- a/gitnexus/vendor/tree-sitter-swift/package.json +++ b/gitnexus/vendor/tree-sitter-swift/package.json @@ -9,7 +9,7 @@ "type": "git", "url": "git+https://github.com/alex-pinkus/tree-sitter-swift.git" }, - "_vendoredBy": "gitnexus - runtime package derived from official tree-sitter-swift@0.7.1 (gitHead 88bfd19a89be9d0481b14566fb6160cccea2fe0a). Unified with Dart/Proto/Kotlin/C: the grammar source (parser.c/scanner.c/binding.gyp + src/) is ALSO vendored so build-tree-sitter-swift.cjs can source-build the binding on a toolchain host when no prebuild matches (e.g. CI before prebuilds land); src/parser.c is the ABI-14 default (~18 MB on disk, compresses heavily in git — the upstream parser_abi13.c alternate is not vendored). The native prebuilds/ are GitNexus-cross-built by .github/workflows/build-tree-sitter-prebuilds.yml (originally upstream-shipped). Build activation runs via gitnexus/scripts/build-tree-sitter-swift.cjs after materialize-vendor-grammars.cjs (no scripts.install here — avoids #836 / #1728).", + "_vendoredBy": "gitnexus - runtime package derived from official tree-sitter-swift@0.7.1 (gitHead 88bfd19a89be9d0481b14566fb6160cccea2fe0a). Unified with Dart/Proto/Kotlin/C: the grammar source (parser.c/scanner.c/binding.gyp + src/) is ALSO vendored so build-tree-sitter-grammars.cjs can source-build the binding on a toolchain host when no prebuild matches (e.g. CI before prebuilds land); src/parser.c is the ABI-14 default (~18 MB on disk, compresses heavily in git — the upstream parser_abi13.c alternate is not vendored). The native prebuilds/ are GitNexus-cross-built by .github/workflows/build-tree-sitter-prebuilds.yml (originally upstream-shipped). Build activation runs via gitnexus/scripts/build-tree-sitter-grammars.cjs after materialize-vendor-grammars.cjs (no scripts.install here — avoids #836 / #1728).", "peerDependencies": { "tree-sitter": "^0.21.1 || ^0.22.1" },