From a532e08ef9950d360a122782135ce7df0d0b6365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Magyar?= Date: Sat, 10 Oct 2026 16:01:20 +0300 Subject: [PATCH] fix(deps): make tree-sitter peer dependencies install cleanly (#3555) * fix(deps): make tree-sitter peer installs clean Keep the eight audited lockfile peer ranges consistent with the bundled tree-sitter 0.25.1 runtime and apply the existing manifest preparation during postinstall. Ordinary and strict npm installs resolve without peer warnings. * docs: explain audited tree-sitter peer maintenance * fix(deps): pin audited tree-sitter peers to 0.25.1 * fix(deps): normalize audited peers to the exact runtime * docs: clarify exact tree-sitter runtime peer pins * test(deps): cover exact peer pins and existing-install migration Verify exact 0.25.1 peers, repeatable preparation, and migration from the previously widened peer metadata. * fix(deps): support pnpm symlinks in bundle preparation (#3555) Follow package symlinks and pnpm sibling dependency containers. Deduplicate real paths to avoid revisiting aliases or cycles while preserving the audited manifest and duplicate-instance checks. Validated with strict npm ci, both typechecks, npm pack, a real pnpm layout, and a pnpm-installed tarball parsing all 19 grammars on main and worker threads. * test(deps): cover pnpm layout and symlink cycles (#3555) Exercise real package symlinks, transitive sibling dependencies, a self-cycle, metadata drift before writes, and rejection of distinct duplicate grammar instances. The original script fails the pnpm regressions; the updated script passes all 10 bundle tests. * chore(autofix): apply prettier + eslint fixes via /autofix command --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- CONTRIBUTING.md | 19 ++-- gitnexus/package-lock.json | 16 +-- gitnexus/package.json | 2 +- .../scripts/prepare-tree-sitter-bundle.cjs | 36 +++++-- .../unit/prepare-tree-sitter-bundle.test.ts | 101 +++++++++++++++++- 5 files changed, 148 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f7b2209f..dc9cb884d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,14 +160,21 @@ Re-invoking `/autofix` after a successful apply is a safe no-op — the workflow The CLI pins and bundles the native `tree-sitter` runtime and its npm grammars. Several compatible grammars still declare older runtime peer ranges. Root npm -`overrides` work in this checkout but do not propagate to consumers, so -`gitnexus/scripts/prepare-tree-sitter-bundle.cjs` runs during `prepack` to add the -tested runtime version to eight audited dependency manifests. It leaves grammar -sources and native binaries unchanged and rejects unexpected versions, peer -ranges, or dependency layouts before writing any manifests. +`overrides` do not propagate to consumers, and older npm versions validate bundled +peers against their original ranges even when an override is configured. The +lockfile therefore pins the same eight audited peers to exactly `0.25.1`. +`gitnexus/scripts/prepare-tree-sitter-bundle.cjs` runs during `postinstall` +and `prepack` to keep installed dependency manifests consistent with that lockfile +and the published bundle. It leaves grammar sources and native binaries unchanged +and rejects unexpected versions, peer ranges, or dependency layouts before writing +any manifests. The upstream ranges remain in the script only as validation inputs; +installed and published audited manifests require exactly `0.25.1`. When upgrading these packages, update the exact pins, lockfile, and audited peer -list together. Run the parser, extraction, CFG, and worker tests, then use a real +list together. If regenerating the lockfile restores upstream's older peer ranges, +restore the same audited `0.25.1` peer pins before committing it. Verify ordinary +`npm ci` and a subsequent `npm install` complete without peer-resolution warnings. +Run the parser, extraction, CFG, and worker tests, then use a real `npm pack` (including `prepack`) to test a fresh consumer's `npm install`, `npm ci`, `npm ls`, and a subsequent dependency install. Check native parser loading on the supported Node.js versions and release platforms. A successful diff --git a/gitnexus/package-lock.json b/gitnexus/package-lock.json index a3e6b6615..2afc674e5 100644 --- a/gitnexus/package-lock.json +++ b/gitnexus/package-lock.json @@ -4738,7 +4738,7 @@ "node-gyp-build": "^4.8.4" }, "peerDependencies": { - "tree-sitter": "^0.22.1" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4779,7 +4779,7 @@ "tree-sitter-c": "^0.23.1" }, "peerDependencies": { - "tree-sitter": "^0.21.1" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4819,7 +4819,7 @@ "node-gyp-build": "^4.8.2" }, "peerDependencies": { - "tree-sitter": "^0.21.1" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4859,7 +4859,7 @@ "node-gyp-build": "^4.8.2" }, "peerDependencies": { - "tree-sitter": "^0.22.4" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4899,7 +4899,7 @@ "node-gyp-build": "^4.8.2" }, "peerDependencies": { - "tree-sitter": "^0.21.1" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4919,7 +4919,7 @@ "node-gyp-build": "^4.8.4" }, "peerDependencies": { - "tree-sitter": "^0.22.1" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4940,7 +4940,7 @@ "tree-sitter-javascript": "^0.23.1" }, "peerDependencies": { - "tree-sitter": "^0.21.0" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { @@ -4960,7 +4960,7 @@ "node-gyp-build": "^4.8.2" }, "peerDependencies": { - "tree-sitter": "^0.21.1" + "tree-sitter": "0.25.1" }, "peerDependenciesMeta": { "tree-sitter": { diff --git a/gitnexus/package.json b/gitnexus/package.json index 50c758ec8..8b9ccc5bc 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -60,7 +60,7 @@ "test:coverage": "vitest run --coverage", "test:cross-platform": "tsx scripts/run-cross-platform.ts", "test:benchmarks": "node --import tsx scripts/run-benchmarks.ts", - "postinstall": "node scripts/build-tree-sitter-grammars.cjs", + "postinstall": "node scripts/prepare-tree-sitter-bundle.cjs && node scripts/build-tree-sitter-grammars.cjs", "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", "prepack": "node scripts/prepare-tree-sitter-bundle.cjs && node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web", diff --git a/gitnexus/scripts/prepare-tree-sitter-bundle.cjs b/gitnexus/scripts/prepare-tree-sitter-bundle.cjs index 927843ce4..648cb895a 100644 --- a/gitnexus/scripts/prepare-tree-sitter-bundle.cjs +++ b/gitnexus/scripts/prepare-tree-sitter-bundle.cjs @@ -1,6 +1,6 @@ #!/usr/bin/env node /** - * Prepare the tested native parser bundle for npm pack/publish. + * Prepare the tested native parser dependencies for installation and publication. * * A dependency's npm overrides do not apply in a consumer project. Several * ABI-compatible grammars still advertise older runtime peers, so bundling @@ -11,6 +11,7 @@ const fs = require('node:fs'); const path = require('node:path'); const RUNTIME = '0.25.1'; +// Upstream metadata is an input to validate, never the peer range we ship. const AUDITED_PEERS = new Map([ ['tree-sitter-c@0.23.6', '^0.22.1'], ['tree-sitter-cpp@0.23.4', '^0.21.1'], @@ -46,29 +47,50 @@ function prepareTreeSitterBundle(packageRoot = path.resolve(__dirname, '..')) { } const patches = []; + const visited = new Set(); function visit(modules) { if (!fs.existsSync(modules)) return; + modules = fs.realpathSync(modules); + if (visited.has(modules)) return; + visited.add(modules); for (const entry of fs.readdirSync(modules, { withFileTypes: true })) { - if (!entry.isDirectory() || entry.name.startsWith('.')) continue; + if (entry.name.startsWith('.')) continue; const dir = path.join(modules, entry.name); + if ( + !entry.isDirectory() && + (!entry.isSymbolicLink() || !fs.statSync(dir, { throwIfNoEntry: false })?.isDirectory()) + ) { + continue; + } if (entry.name.startsWith('@')) { visit(dir); continue; } - const file = path.join(dir, 'package.json'); + const realDir = fs.realpathSync(dir); + if (visited.has(realDir)) continue; + visited.add(realDir); + const file = path.join(realDir, 'package.json'); if (fs.existsSync(file)) { const pkg = readJson(file); const original = AUDITED_PEERS.get(`${pkg.name}@${pkg.version}`); if (original) { - const peer = `${original} || ${RUNTIME}`; - if (![original, peer].includes(pkg.peerDependencies?.['tree-sitter'])) { + // Also accept bundles prepared before peers were pinned exactly, so + // an existing checkout can migrate through ordinary npm install. + const accepted = [original, `${original} || ${RUNTIME}`, RUNTIME]; + if (!accepted.includes(pkg.peerDependencies?.['tree-sitter'])) { throw new Error(`Unexpected tree-sitter peer metadata for ${pkg.name}@${pkg.version}`); } - pkg.peerDependencies['tree-sitter'] = peer; + pkg.peerDependencies['tree-sitter'] = RUNTIME; patches.push({ file, pkg }); } } - visit(path.join(dir, 'node_modules')); + visit(path.join(realDir, 'node_modules')); + // pnpm links packages into node_modules but places their dependencies + // beside the real package. Follow that container without traversing the + // entire virtual store; realpaths deduplicate aliases and break cycles. + let parent = path.dirname(realDir); + if (path.basename(parent).startsWith('@')) parent = path.dirname(parent); + if (path.basename(parent) === 'node_modules') visit(parent); } } visit(modulesRoot); diff --git a/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts b/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts index f4337bf06..620efea40 100644 --- a/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts +++ b/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts @@ -1,6 +1,14 @@ import { afterEach, describe, expect, it } from 'vitest'; import { createRequire } from 'node:module'; -import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { + mkdtempSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; @@ -68,12 +76,45 @@ function fixture() { return root; } +function pnpmFixture() { + const root = fixture(); + const modules = path.join(root, 'node_modules'); + const relocate = (source: string, name: string, version: string) => { + const target = path.join(modules, '.pnpm', `${name}@${version}`, 'node_modules', name); + mkdirSync(path.dirname(target), { recursive: true }); + renameSync(source, target); + return target; + }; + const c = relocate(path.join(modules, 'tree-sitter-c'), 'tree-sitter-c', '0.23.6'); + const javascript = relocate( + path.join(modules, 'tree-sitter-typescript/node_modules/tree-sitter-javascript'), + 'tree-sitter-javascript', + '0.23.1', + ); + rmSync(path.join(modules, 'tree-sitter-typescript/node_modules'), { recursive: true }); + for (const [name, version] of Object.entries(directVersions)) { + const target = relocate(path.join(modules, name), name, version); + symlinkSync(target, path.join(modules, name), 'junction'); + if (name === 'tree-sitter-cpp') { + symlinkSync(c, path.join(path.dirname(target), 'tree-sitter-c'), 'junction'); + } + if (name === 'tree-sitter-typescript') { + symlinkSync( + javascript, + path.join(path.dirname(target), 'tree-sitter-javascript'), + 'junction', + ); + } + } + return { root, c, javascript }; +} + afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); describe('native parser bundle preparation', () => { - it('admits the tested runtime for direct and transitive peers without changing binaries', () => { + it('pins direct and transitive peers to the tested runtime without changing binaries', () => { const root = fixture(); const native = path.join(root, 'node_modules/tree-sitter-cpp/parser.node'); writeFileSync(native, Buffer.from([0, 1, 128, 255])); @@ -83,7 +124,7 @@ describe('native parser bundle preparation', () => { read(path.join(root, 'node_modules/tree-sitter-cpp/package.json')).peerDependencies[ 'tree-sitter' ], - ).toBe('^0.21.1 || 0.25.1'); + ).toBe('0.25.1'); expect( read( path.join( @@ -91,7 +132,7 @@ describe('native parser bundle preparation', () => { 'node_modules/tree-sitter-typescript/node_modules/tree-sitter-javascript/package.json', ), ).peerDependencies['tree-sitter'], - ).toBe('^0.21.1 || 0.25.1'); + ).toBe('0.25.1'); expect( read(path.join(root, 'node_modules/tree-sitter-javascript/package.json')).peerDependencies[ 'tree-sitter' @@ -102,6 +143,58 @@ describe('native parser bundle preparation', () => { expect(read(path.join(root, 'package.json')).dependencies).toEqual(directVersions); }); + it('migrates previously widened peer ranges to the exact runtime', () => { + const root = fixture(); + const file = path.join(root, 'node_modules/tree-sitter-cpp/package.json'); + const pkg = read(file); + pkg.peerDependencies['tree-sitter'] = '^0.21.1 || 0.25.1'; + write(file, pkg); + expect(prepareTreeSitterBundle(root)).toHaveLength(8); + expect(read(file).peerDependencies['tree-sitter']).toBe('0.25.1'); + expect(prepareTreeSitterBundle(root)).toHaveLength(8); + }); + + it('prepares pnpm symlinks, transitive siblings, and aliased cycles exactly once', () => { + const { root, c, javascript } = pnpmFixture(); + const aliases = path.join(c, 'node_modules'); + mkdirSync(aliases); + symlinkSync(c, path.join(aliases, 'tree-sitter-c'), 'junction'); + expect(prepareTreeSitterBundle(root)).toHaveLength(8); + for (const dir of [c, javascript, path.join(root, 'node_modules/tree-sitter-cpp')]) { + expect(read(path.join(dir, 'package.json')).peerDependencies['tree-sitter']).toBe('0.25.1'); + } + expect( + read(path.join(root, 'node_modules/tree-sitter-javascript/package.json')).peerDependencies[ + 'tree-sitter' + ], + ).toBe('^0.25.0'); + expect(prepareTreeSitterBundle(root)).toHaveLength(8); + }); + + it('rejects drift behind pnpm symlinks before writing any package', () => { + const { root, c, javascript } = pnpmFixture(); + const first = path.join(c, 'package.json'); + const before = readFileSync(first, 'utf8'); + const last = path.join(javascript, 'package.json'); + const pkg = read(last); + pkg.peerDependencies['tree-sitter'] = '^0.26.0'; + write(last, pkg); + expect(() => prepareTreeSitterBundle(root)).toThrow('Unexpected tree-sitter peer metadata'); + expect(readFileSync(first, 'utf8')).toBe(before); + }); + + it('rejects distinct copies of an audited grammar before writing any package', () => { + const root = fixture(); + const first = path.join(root, 'node_modules/tree-sitter-c/package.json'); + const before = readFileSync(first, 'utf8'); + write( + path.join(root, 'node_modules/tree-sitter-cpp/node_modules/tree-sitter-c/package.json'), + read(first), + ); + expect(() => prepareTreeSitterBundle(root)).toThrow('dependency layout changed'); + expect(readFileSync(first, 'utf8')).toBe(before); + }); + it('rejects changed peer metadata before writing any package', () => { const root = fixture(); const first = path.join(root, 'node_modules/tree-sitter-c/package.json');