Merge branch 'main' into port/ingestion-precision

This commit is contained in:
Gergő Magyar 2026-09-05 10:35:53 +01:00 • committed by GitHub
commit b3b2281128
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
48 changed files with 181623 additions and 168 deletions

View file

@ -69,6 +69,7 @@ GRAMMARS: dict[str, tuple[str, str, str]] = {
# Vendored parsers — kept here so the upstream coords for drift
# detection are co-located with every other grammar's coords.
"tree-sitter-proto": ("coder3101/tree-sitter-proto", "main", "src/parser.c"),
"tree-sitter-zig": ("tree-sitter-grammars/tree-sitter-zig", "master", "src/parser.c"),
}
# npm-installed grammars deliberately held below npm latest (surfaced so reviewers

View file

@ -9,8 +9,8 @@ which is deliberately dependency-free so it runs on any vanilla runner. Run with
(pytest also discovers ``unittest.TestCase`` classes, so a future pytest CI job
picks these up unchanged.)
These tests lock in the #858 fix: the 5 vendored grammars
(c/swift/kotlin/dart/proto) are classified from the shared manifest
These tests lock in the #858 fix: the 6 vendored grammars
(c/swift/kotlin/dart/proto/zig) are classified from the shared manifest
(.github/vendored-grammars.json), their ABI is read from gitnexus/vendor/<name>,
and the report never renders a bare ``?`` placeholder. All network is mocked.
"""
@ -192,7 +192,7 @@ class AssertCurrent(TestCase):
def test_assert_current_is_network_free_and_passes(self):
report, code = self._run_assert_current() # raises if any urlopen fires
self.assertEqual(code, 0)
# All 5 vendored grammars are introspected from the repo (ABI 14), not skipped.
# All 6 vendored grammars are introspected from the repo (ABI 14), not skipped.
for name in readiness.VENDORED_NAMES:
self.assertIn(f"{name}: vendored ABI", report)
@ -324,6 +324,7 @@ class ReportRendering(TestCase):
# which is what removes the old "? (fetch failed)" for tree-sitter-proto.
self.assertNotIn("tree-sitter-proto", _render_report.last_npm_calls)
self.assertNotIn("tree-sitter-dart", _render_report.last_npm_calls)
self.assertNotIn("@tree-sitter-grammars/tree-sitter-zig", _render_report.last_npm_calls)
self.assertNotIn("Could not check", self.report)
self.assertNotIn("fetch failed", self.report)
@ -343,11 +344,11 @@ class ReportRendering(TestCase):
cells = [c.strip() for c in self._matrix_row("tree-sitter-swift").strip().strip("|").split("|")]
self.assertEqual(cells[6], "n/a") # Upstream ABI column
def test_row_diff_regex_captures_all_fifteen_grammar_statuses(self):
def test_row_diff_regex_captures_all_grammar_statuses(self):
# The change-detection bot keys on this regex: group 1 = grammar name,
# group 2 = the Status cell ONLY (not the whole tail). It must match every
# row after the format change so status transitions keep being detected.
self.assertEqual(len(self.rows), 15)
self.assertEqual(len(self.rows), len(readiness.GRAMMARS))
for name in readiness.VENDORED_NAMES:
self.assertIn(name, self.rows)
# group 2 is the Status cell — held c renders exactly "Vendored — held",

View file

@ -22,6 +22,10 @@
"proto": {
"name": "tree-sitter-proto",
"upstream": { "github": "coder3101/tree-sitter-proto" }
},
"zig": {
"name": "tree-sitter-zig",
"upstream": { "npm": "@tree-sitter-grammars/tree-sitter-zig" }
}
}
}

View file

@ -20,6 +20,11 @@ name: Build tree-sitter prebuilds
# - tree-sitter-swift (vendored source; built from gitnexus/vendor/ — its
# prebuilds were originally upstream-shipped, now
# GitNexus-cross-built like the rest for uniformity)
# - tree-sitter-zig (vendored source; built from gitnexus/vendor/ — moved
# off npm optionalDependency so `npm i -g gitnexus`
# no longer warns on peerOptional tree-sitter@^0.22.1.
# Upstream linux-arm64 prebuild is a mispackaged
# x86-64 binary; this workflow rebuilds all six.)
#
# Output: gitnexus/vendor/<grammar>/prebuilds/<platform-arch>/<grammar>.node for
# all 6 targets ({linux,darwin,win32}-{x64,arm64}). tree-sitter grammars are
@ -33,8 +38,9 @@ name: Build tree-sitter prebuilds
# OR an edit to the grammar's build-affecting source (parser.c / grammar.js /
# binding.gyp / scanner / bindings). The `guard` job is the real gate (it
# diffs BOTH the recorded version AND the source files vs the PR base); the
# `paths:` filter below keeps ordinary code PRs at ZERO matrix time and
# excludes the prebuilds the job commits back, so it never retriggers itself.
# `paths:` filter below keeps ordinary code PRs at ZERO matrix time. PR
# filters see the cumulative diff, so the guard separately skips updates
# containing only the prebuilds the job commits back.
# Net effect: an ordinary code PR triggers nothing; touching one grammar's source
# costs exactly one matrix run for that grammar. Delivery of the rebuilt binaries:
# - same-repo PR -> committed straight onto the PR's own branch (in the SAME PR);
@ -55,7 +61,7 @@ on:
workflow_dispatch:
inputs:
grammars:
description: 'Comma-separated grammar shortnames to build (c,dart,proto,kotlin,swift), or "all".'
description: 'Comma-separated grammar shortnames to build (c,dart,proto,kotlin,swift,zig), or "all".'
required: false
type: string
default: 'all'
@ -80,13 +86,14 @@ on:
# Any build-affecting change under a vendored grammar triggers a rebuild —
# not just a version bump — so editing the vendored source (parser.c,
# grammar.js, binding.gyp, scanner, bindings) re-cuts the prebuilds too.
# The prebuilds we commit back are EXCLUDED (negated last) so the bot's own
# in-PR commit can never retrigger this workflow (no build->commit->build loop).
# Excludes PRs containing only prebuilds. A source PR still matches after a
# bot commit because PR filters use the cumulative diff; `guard` stops the
# build->commit->build loop using the synchronize event's before/head diff.
- 'gitnexus/vendor/tree-sitter-*/**'
- '!gitnexus/vendor/tree-sitter-*/prebuilds/**'
# Self-test: re-run the guard if a future grammar pin is reintroduced in
# the main package.json (optionalDependencies fallback). No-op otherwise —
# all five grammars are now fully vendored (kotlin included).
# all six grammars are now fully vendored (kotlin and zig included).
- 'gitnexus/package.json'
# Self-test: re-run the guard (normally a no-op) when the recipe changes.
- '.github/workflows/build-tree-sitter-prebuilds.yml'
@ -123,6 +130,9 @@ jobs:
id: decide
env:
EVENT: ${{ github.event_name }}
ACTION: ${{ github.event.action }}
BEFORE_SHA: ${{ github.event.before }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
# Untrusted dispatch inputs — read via env only, validated in JS.
INPUT_GRAMMARS: ${{ inputs.grammars }}
INPUT_REF: ${{ inputs.ref }}
@ -131,7 +141,7 @@ jobs:
run: |
set -euo pipefail
node --input-type=module - <<'NODE'
import { execSync } from 'node:child_process';
import { execFileSync, execSync } from 'node:child_process';
import fs from 'node:fs';
import { appendFileSync } from 'node:fs';
@ -157,6 +167,11 @@ jobs:
// so it builds from gitnexus/vendor/ like dart/proto. Its prebuilds
// were originally upstream-shipped; rebuilding them here unifies it.
swift: { name: 'tree-sitter-swift', kind: 'vendored' },
// zig is vendored WITH its source (parser.c/binding.gyp). Moved off
// the npm optionalDependency so published installs no longer warn
// on peerOptional tree-sitter@^0.22.1. Upstream linux-arm64
// prebuild is a mispackaged x86-64 binary; rebuild here.
zig: { name: 'tree-sitter-zig', kind: 'vendored' },
};
const PLATFORMS = [
{ platform_arch: 'linux-x64', os: 'ubuntu-24.04' },
@ -187,6 +202,30 @@ jobs:
const event = process.env.EVENT;
const force = process.env.FORCE === 'true';
// PR path filters and the source/version checks below see the entire
// PR, so excluding prebuilds there does NOT prevent a rebuild loop.
// Check the whole push (not HEAD^ or the author's identity): a push
// containing source edits followed by a binary commit must still build.
if (event === 'pull_request' && process.env.ACTION === 'synchronize') {
const before = process.env.BEFORE_SHA;
const head = process.env.HEAD_SHA;
for (const sha of [before, head]) {
if (!sha || !/^[0-9a-fA-F]{40}$/.test(sha)) {
throw new Error('synchronize requires valid before/head SHAs; refusing an unbounded rebuild');
}
}
// Fail closed if either commit is unavailable. Never fall back to
// the cumulative PR diff, which would re-enable the loop.
const changed = execFileSync('git', [
'diff', '--name-only', '--no-renames', '-z', before, head, '--',
], { encoding: 'utf8' }).split('\0').filter(Boolean);
if (changed.every((p) => /^gitnexus\/vendor\/tree-sitter-[^/]+\/prebuilds\//.test(p))) {
appendFileSync(process.env.GITHUB_OUTPUT, 'any=false\nmatrix={"include":[]}\n');
console.log('::notice::Push changes only prebuild outputs (or no files) — skipping native matrix.');
process.exit(0);
}
}
// Select which grammar shortnames are in play.
let selected;
if (event === 'workflow_dispatch') {
@ -243,9 +282,9 @@ jobs:
} else {
// pull_request: build when the recorded version changed OR any
// build-affecting source file under the vendored grammar changed vs
// the PR base. The prebuilds/ subtree is excluded from the diff so
// the bot's own in-PR commit (which adds ONLY prebuilds) never reads
// as a source change — this is the other half of the no-loop guard.
// the PR base. Exclude generated outputs from build inputs; the
// synchronize check above prevents rebuilding the original source
// change after every generated-prebuild commit.
const base = recordedVersion(baseRoot, name);
const versionChanged = !!head && head !== base;
let sourceChanged = false;
@ -467,11 +506,14 @@ jobs:
proto: "syntax = \"proto3\";\nmessage M { int32 id = 1; }",
kotlin: "fun main() { println(\"hi\") }",
swift: "func greet() { print(\"hi\") }",
zig: "pub fn main() void {}",
};
const src = snippets[process.env.GRAMMAR];
if (!src) throw new Error("no validate snippet for grammar: " + process.env.GRAMMAR);
const lang = require("node-gyp-build")(process.cwd());
const Parser = require("tree-sitter");
const p = new Parser(); p.setLanguage(lang);
const tree = p.parse(snippets[process.env.GRAMMAR]);
const tree = p.parse(src);
if (!tree || !tree.rootNode || tree.rootNode.hasError) {
throw new Error("parse failed/error: " + (tree && tree.rootNode && tree.rootNode.type));
}

View file

@ -24,11 +24,11 @@ jobs:
shard: ${{ fromJSON(needs.shard-plan.outputs.cov_shards) }}
# Fail loudly (don't silently skip) if the FTS extension is unavailable, so
# FTS-dependent lbug integration suites are guaranteed to run in CI.
# Same contract for Zig's optionalDependency grammar: this runner is
# linux-x64, which @tree-sitter-grammars/tree-sitter-zig publishes a
# prebuild for, so an absent grammar here is a packaging regression and not
# an unsupported platform. Without it every Zig suite skips and the job is
# green having never executed the native Zig parser once.
# Same contract for Zig's vendored grammar: this runner is linux-x64,
# which vendor/tree-sitter-zig ships a prebuild for, so an absent grammar
# here is a packaging regression and not an unsupported platform. Without
# it every Zig suite skips and the job is green having never executed the
# native Zig parser once.
env:
GITNEXUS_REQUIRE_FTS: '1'
GITNEXUS_REQUIRE_ZIG: '1'
@ -284,10 +284,12 @@ jobs:
shell: bash
run: python3 .github/scripts/check-tree-sitter-upgrade-readiness.py --assert-current
# GITNEXUS_REQUIRE_ZIG=1: every OS in this matrix has a published
# tree-sitter-zig prebuild, so the smoke's "optional grammar may be
# absent" exemption is revoked here and an ABI-broken Zig binding fails
# the job instead of being accepted as a clean absence.
# GITNEXUS_REQUIRE_ZIG=1: every OS in this matrix has a committed
# vendored tree-sitter-zig prebuild (linux-arm64 is rebuilt by the
# prebuild workflow; ubuntu/windows/macos latest are x64/arm64 with
# shipped binaries), so the smoke's "optional grammar may be absent"
# exemption is revoked here and an ABI-broken Zig binding fails the
# job instead of being accepted as a clean absence.
- name: Run parser-loader ABI load-smoke (dynamic)
run: npx vitest run test/unit/parser-loader-abi.test.ts
env:

View file

@ -192,6 +192,6 @@ npx gitnexus serve # HTTP API on port 4747 (from any ind
### Gotchas
- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (materializes the vendored grammars into `node_modules/`, then prefers a committed prebuild per platform-arch and only source-builds when none matches). A C/C++ toolchain (`python3`, `make`, `g++`) is needed only for that source-build fallback.
- The vendored grammars `tree-sitter-{c,dart,proto,swift,kotlin}` are handled uniformly: c is required; dart/proto/swift/kotlin are optional and skippable via `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`. Install warnings appear only when no prebuild matches the platform-arch and no toolchain is present, and are non-fatal — only that language's parsing is unavailable.
- `npm install` in `gitnexus/` triggers `prepare` (builds via `tsc`) and `postinstall` (`build-tree-sitter-grammars.cjs` activates committed prebuilds in place under `vendor/`, and only source-builds when none matches). A C/C++ toolchain (`python3`, `make`, `g++`) is needed only for that source-build fallback.
- The vendored grammars `tree-sitter-{c,dart,proto,swift,kotlin,zig}` are handled uniformly: c is required; dart/proto/swift/kotlin/zig are optional and skippable via `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1`. Install warnings appear only when no prebuild matches the platform-arch and no toolchain is present, and are non-fatal — only that language's parsing is unavailable.
- ESLint configured via `eslint.config.mjs` (TS, React Hooks, unused-imports). No `npm run lint` script; use `npx eslint .`. Prettier runs via lint-staged. CI checks both in `ci-quality.yml`.

View file

@ -631,7 +631,7 @@ results until you re-run `gitnexus analyze --repair-fts` from a shell where the
### Installation fails with native module errors
Some optional language grammars (Dart, Proto, Swift, Kotlin) require native compilation. If they fail, GitNexus still works — those languages will be skipped. To skip them intentionally (no C++ toolchain needed), set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before installing. Zig (`@tree-sitter-grammars/tree-sitter-zig`, an npm `optionalDependency`) behaves the same way: if its native binding fails to install, `.zig` files are skipped.
Some optional language grammars (Dart, Proto, Swift, Kotlin, Zig) ship vendored native prebuilds. If a prebuild is missing and a source build is not possible, GitNexus still works — those languages will be skipped. To skip them intentionally (no C++ toolchain needed), set `GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1` before installing.
If `npm install -g gitnexus` fails on native modules:

View file

@ -75,7 +75,6 @@
},
"optionalDependencies": {
"@huggingface/transformers": "^4.1.0",
"@tree-sitter-grammars/tree-sitter-zig": "1.1.2",
"onnxruntime-node": "^1.24.0"
}
},
@ -1761,26 +1760,6 @@
"tslib": "^2.8.0"
}
},
"node_modules/@tree-sitter-grammars/tree-sitter-zig": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/@tree-sitter-grammars/tree-sitter-zig/-/tree-sitter-zig-1.1.2.tgz",
"integrity": "sha512-J0L31HZ2isy3F5zb2g5QWQOv2r/pbruQNL9ADhuQv2pn5BQOzxt80WcEJaYXBeuJ8GHxVT42slpCna8k1c8LOw==",
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"dependencies": {
"node-addon-api": "^8.3.0",
"node-gyp-build": "^4.8.4"
},
"peerDependencies": {
"tree-sitter": "^0.22.1"
},
"peerDependenciesMeta": {
"tree-sitter": {
"optional": true
}
}
},
"node_modules/@types/body-parser": {
"version": "1.19.6",
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",

View file

@ -99,7 +99,6 @@
},
"optionalDependencies": {
"@huggingface/transformers": "^4.1.0",
"@tree-sitter-grammars/tree-sitter-zig": "1.1.2",
"onnxruntime-node": "^1.24.0"
},
"trustedDependencies": [
@ -129,9 +128,6 @@
"sharp": ">=0.35.0",
"@huggingface/transformers": {
"onnxruntime-node": "$onnxruntime-node"
},
"@tree-sitter-grammars/tree-sitter-zig": {
"tree-sitter": "$tree-sitter"
}
},
"engines": {

View file

@ -4,7 +4,7 @@
* One registry-driven script replaces the former per-grammar
* build-tree-sitter-<name>.cjs files (they were ~95% identical).
*
* The grammars (tree-sitter-c/dart/proto/swift/kotlin) are loaded from
* The grammars (tree-sitter-c/dart/proto/swift/kotlin/zig) are loaded from
* `vendor/<name>/` by absolute path at runtime (see
* src/core/tree-sitter/vendored-grammars.ts) and are NEVER copied into
* node_modules — an undeclared package under node_modules is "extraneous" to
@ -44,6 +44,7 @@ const GRAMMARS = {
proto: { required: false, display: 'Proto', ext: '.proto' },
swift: { required: false, display: 'Swift', ext: '.swift' },
kotlin: { required: false, display: 'Kotlin', ext: '.kt/.kts' },
zig: { required: false, display: 'Zig', ext: '.zig' },
};
const skipOptional = process.env.GITNEXUS_SKIP_OPTIONAL_GRAMMARS === '1';

View file

@ -1,14 +1,13 @@
/**
* Optional grammar availability check.
*
* tree-sitter-dart, -proto, -swift, and -kotlin are vendored under vendor/ and
* loaded from there by absolute path (NEVER copied into node_modules — see
* core/tree-sitter/vendored-grammars.ts / #2111). Each ships committed platform
* prebuilds activated via node-gyp-build. All can be skipped via
* GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (postinstall scripts), or can silently
* soft-fail when no prebuild matches the host platform (and a source build was
* unavailable / not attempted). tree-sitter-zig is the one npm-installed
* optionalDependency in the list; its `probe` overrides the vendored load.
* tree-sitter-dart, -proto, -swift, -kotlin, and -zig are vendored under
* vendor/ and loaded from there by absolute path (NEVER copied into
* node_modules — see core/tree-sitter/vendored-grammars.ts / #2111). Each
* ships committed platform prebuilds activated via node-gyp-build. All can
* be skipped via GITNEXUS_SKIP_OPTIONAL_GRAMMARS=1 (postinstall scripts), or
* can silently soft-fail when no prebuild matches the host platform (and a
* source build was unavailable / not attempted).
*
* Either path produces the same observable: the .node binding is absent
* at runtime. This helper detects that condition and surfaces a single
@ -16,14 +15,11 @@
* support is unavailable instead of silently getting a degraded index.
*/
import { createRequire } from 'node:module';
import { SupportedLanguages } from 'gitnexus-shared';
import { isGrammarRuntimeSkipped } from '../core/tree-sitter/parser-loader.js';
import { requireVendoredGrammar } from '../core/tree-sitter/vendored-grammars.js';
import { cliWarn } from './cli-message.js';
const _require = createRequire(import.meta.url);
interface OptionalGrammar {
/** Display name in warnings */
name: string;
@ -38,12 +34,6 @@ interface OptionalGrammar {
* `.proto`, which is a gRPC-extractor concern, not a SupportedLanguages.
*/
language?: SupportedLanguages;
/**
* Availability probe. Defaults to `requireVendoredGrammar(pkg)`; grammars
* that install from npm as an optionalDependency (zig) override it with a
* plain `require` of the package.
*/
probe?: () => unknown;
}
const OPTIONAL_GRAMMARS: OptionalGrammar[] = [
@ -68,11 +58,9 @@ const OPTIONAL_GRAMMARS: OptionalGrammar[] = [
},
{
name: 'tree-sitter-zig',
pkg: '@tree-sitter-grammars/tree-sitter-zig',
pkg: 'tree-sitter-zig',
extensions: ['.zig'],
language: SupportedLanguages.Zig,
// npm optionalDependency, not vendored — probe via plain require.
probe: () => _require('@tree-sitter-grammars/tree-sitter-zig'),
},
];
@ -123,11 +111,7 @@ export function detectMissingOptionalGrammars(): MissingGrammar[] {
continue;
}
try {
if (g.probe !== undefined) {
g.probe();
} else {
requireVendoredGrammar(g.pkg);
}
requireVendoredGrammar(g.pkg);
} catch (err) {
const code = (err as NodeJS.ErrnoException | undefined)?.code;
const msg = err instanceof Error ? err.message : String(err);

View file

@ -1,16 +1,14 @@
import Parser from 'tree-sitter';
import { createRequire } from 'node:module';
const _require = createRequire(import.meta.url);
import { requireVendoredGrammar } from '../../../tree-sitter/vendored-grammars.js';
/**
* Zig scope-resolution query (RFC #909 Ring 3).
*
* The grammar is an optionalDependency (`@tree-sitter-grammars/tree-sitter-zig`),
* so the language module is required lazily and `getZigParser` /
* `getZigScopeQuery` throw only when actually invoked without the grammar
* installed. That is safe: the parse pipeline filters `.zig` files through
* `parser-loader.isLanguageAvailable` before any scope extraction runs.
* The grammar is vendored (`vendor/tree-sitter-zig`) and may be absent on a
* platform without a prebuild, so the language module is required lazily and
* `getZigParser` / `getZigScopeQuery` throw only when actually invoked without
* the grammar installed. That is safe: the parse pipeline filters `.zig` files
* through `parser-loader.isLanguageAvailable` before any scope extraction runs.
*
* Zig specifics encoded here:
* - Containers (struct/enum/union/opaque) are anonymous nodes bound by the
@ -413,7 +411,7 @@ let _parser: Parser | null = null;
let _query: Parser.Query | null = null;
function getZigLanguage(): Parameters<Parser['setLanguage']>[0] {
return _require('@tree-sitter-grammars/tree-sitter-zig');
return requireVendoredGrammar('tree-sitter-zig') as Parameters<Parser['setLanguage']>[0];
}
export function getZigParser(): Parser {

View file

@ -1,5 +1,4 @@
import { parentPort, threadId, workerData } from 'node:worker_threads';
import { createRequire } from 'node:module';
import {
boundCallableStartPosition,
localIdentity,
@ -59,7 +58,7 @@ type TreeSitterLanguage = Parameters<typeof Parser.prototype.setLanguage>[0];
// `isLanguageAvailable` must re-introduce the gate here. (The cleaner end-state
// — routing this table through `parser-loader.getLanguageGrammar` so there is
// one loader — is the deferred Tier-1 consolidation.)
// Swift/Dart/Kotlin/C are vendored grammars loaded from `vendor/` by absolute
// Swift/Dart/Kotlin/C/Zig are vendored grammars loaded from `vendor/` by absolute
// path (NEVER copied into node_modules — see vendored-grammars.ts / #2111). Each
// may be absent on a platform without a prebuild or a toolchain-less /
// `--ignore-scripts` install, so every load is guarded so a missing binding
@ -84,11 +83,9 @@ try {
C = requireVendoredGrammar('tree-sitter-c') as TreeSitterLanguage;
} catch {}
// @tree-sitter-grammars/tree-sitter-zig is an optionalDependency — may not be installed
const _require = createRequire(import.meta.url);
let Zig: TreeSitterLanguage | null = null;
try {
Zig = _require('@tree-sitter-grammars/tree-sitter-zig');
Zig = requireVendoredGrammar('tree-sitter-zig') as TreeSitterLanguage;
} catch {}
import { getLanguageFromFilename } from 'gitnexus-shared';
import {

View file

@ -43,7 +43,7 @@ interface GrammarSource {
/**
* When true, this grammar may be disabled at runtime via
* `GITNEXUS_SKIP_OPTIONAL_GRAMMARS`. Set ONLY on genuinely-optional grammars
* (optionalDependencies / vendored — swift/dart/kotlin). Required dependencies
* (optionalDependencies / vendored — swift/dart/kotlin/zig). Required dependencies
* routed through the optional machinery for ABI safety (e.g. C, which is
* `optional: true` + `severity: 'error'`) must NOT set this — opting out of a
* required parser is always an install/platform problem, never a user choice.
@ -177,22 +177,14 @@ const SOURCES: Record<string, GrammarSource> = {
'Likely cause: no prebuilt `.node` for this platform/architecture. ' +
`See ${ISSUES_URL}/2107.`,
},
// Zig grammar declares peerOptional `tree-sitter@^0.22.1` but its native
// binding is ABI-compatible with the bundled `tree-sitter@0.21.x` runtime
// (verified by load-time smoke test). The peer-dep mismatch is suppressed
// via the `overrides` block in package.json. Listed as `optional: true`
// because the package is in `optionalDependencies` — users on platforms
// without a prebuild may not have it — and `userSkippable` because it is
// a genuinely-optional grammar: `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` must be
// able to disable it at analyze time like swift/dart/kotlin.
[SupportedLanguages.Zig]: {
load: () => _require('@tree-sitter-grammars/tree-sitter-zig'),
load: () => requireVendoredGrammar('tree-sitter-zig'),
optional: true,
userSkippable: true,
unavailableNote:
'Zig parsing disabled: `@tree-sitter-grammars/tree-sitter-zig` is an ' +
'optionalDependency and is not installed (or its native binding failed ' +
'to load on this platform).',
'Zig parsing disabled: vendored `tree-sitter-zig` (under ' +
'`gitnexus/vendor/tree-sitter-zig`) failed to load. ' +
'Likely cause: no prebuilt `.node` for this platform/architecture.',
},
};

View file

@ -33,6 +33,7 @@ export const VENDORED_GRAMMAR_PACKAGES: ReadonlySet<string> = new Set([
'tree-sitter-proto',
'tree-sitter-swift',
'tree-sitter-kotlin',
'tree-sitter-zig',
]);
/** Absolute directory of a vendored grammar package under `vendor/`. */
@ -42,7 +43,7 @@ export const vendoredGrammarDir = (packageName: string): string =>
/**
* Load a vendored tree-sitter grammar by its absolute path under `vendor/`.
*
* GitNexus vendors five grammars (c/dart/proto/swift/kotlin) inside its own
* GitNexus vendors six grammars (c/dart/proto/swift/kotlin/zig) inside its own
* package under `vendor/`, each shipping committed per-platform prebuilds. They
* are deliberately NOT npm dependencies and must NEVER be copied into
* `node_modules`: an undeclared package under `node_modules` is "extraneous" to

View file

@ -70,10 +70,8 @@ const GRAMMAR_PACKAGES: Partial<Record<SupportedLanguages, { pkg: string; subpat
pkg: 'tree-sitter-typescript',
subpaths: ['typescript/src/node-types.json', 'tsx/src/node-types.json'],
},
// optionalDependency — loadGrammarModel returns null when absent, so the
// gate skips Zig literals ('unavailable', R9) instead of failing.
[SupportedLanguages.Zig]: {
pkg: '@tree-sitter-grammars/tree-sitter-zig',
pkg: 'tree-sitter-zig',
subpaths: ['src/node-types.json'],
},
};
@ -111,7 +109,7 @@ interface NodeTypeEntry {
/** Resolve the on-disk directory of an installed package, or null if absent. */
function resolvePackageDir(pkg: string): string | null {
// Vendored grammars (c/dart/proto/swift/kotlin) are NOT in node_modules — they
// Vendored grammars (c/dart/proto/swift/kotlin/zig) are NOT in node_modules — they
// load from vendor/ by absolute path (vendored-grammars.ts / #2111), so resolve
// their node-types.json from there rather than via _require.resolve.
if (VENDORED_GRAMMAR_PACKAGES.has(pkg)) {

View file

@ -20,8 +20,9 @@ import { isLanguageAvailable } from '../../src/core/tree-sitter/parser-loader.js
* a job it cannot satisfy.
*/
export const OPTIONAL_GRAMMAR_ENV: Readonly<Partial<Record<string, string>>> = {
// @tree-sitter-grammars/tree-sitter-zig@1.1.2 publishes prebuilds for
// {darwin,linux,win32}-{x64,arm64} — every OS in the CI matrix.
// Vendored tree-sitter-zig ships prebuilds for the CI matrix OS/arch
// tuples (linux-arm64 is rebuilt by the GitNexus prebuild workflow;
// ubuntu/windows/macos latest in required jobs have a committed binary).
[SupportedLanguages.Zig]: 'GITNEXUS_REQUIRE_ZIG',
} satisfies Partial<Record<SupportedLanguages, string>>;

View file

@ -15,9 +15,9 @@ import {
import { SupportedLanguages } from '../../../src/config/supported-languages.js';
import { describeGrammarPresence, optionalGrammarGate } from '../../helpers/optional-grammar.js';
// `@tree-sitter-grammars/tree-sitter-zig` is an optionalDependency: on a
// platform without a prebuild the grammar is absent and the pipeline skips
// `.zig` files by contract, so these suites skip too (Swift/Dart pattern).
// Vendored `tree-sitter-zig`: on a platform without a prebuild the grammar
// is absent and the pipeline skips `.zig` files by contract, so these
// suites skip too (Swift/Dart pattern).
// Under GITNEXUS_REQUIRE_ZIG=1 the skip is not acceptable — the presence
// assertion below fails the job instead of letting Zig vanish from a green run.
const zig = optionalGrammarGate(SupportedLanguages.Zig);

View file

@ -160,8 +160,8 @@ const NON_BRIDGE_CORPUS = [
] as const satisfies readonly CorpusEntry[];
/**
* Same contract, for fixtures whose grammar is an npm optionalDependency and
* may be absent on the runner. Gated per language rather than per case so a
* Same contract, for fixtures whose grammar is optional (vendored prebuild
* may be absent on the runner). Gated per language rather than per case so a
* missing grammar SKIPS (the pipeline drops the files by contract, and an
* empty emit would otherwise fail every sentinel for a reason that has
* nothing to do with the schema).

View file

@ -10,7 +10,7 @@ import {
import { SupportedLanguages, getLanguageFromFilename } from 'gitnexus-shared';
import { getProvider } from '../../src/core/ingestion/languages/index.js';
import Parser from 'tree-sitter';
import { createRequire } from 'module';
import { vendoredGrammarDir } from '../../src/core/tree-sitter/vendored-grammars.js';
const fixturesDir = path.resolve(__dirname, '..', 'fixtures', 'sample-code');
@ -690,15 +690,9 @@ describe('Tree-sitter multi-language parsing', () => {
// mismatch, bad export) must fail this test, not silently skip it. A
// deliberate `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` opt-out in the environment
// is the one non-failure reason an installed grammar reports unavailable.
const zigPackageInstalled = (() => {
if (isGrammarRuntimeSkipped(SupportedLanguages.Zig)) return false;
try {
createRequire(import.meta.url).resolve('@tree-sitter-grammars/tree-sitter-zig');
return true;
} catch {
return false;
}
})();
const zigPackageInstalled =
!isGrammarRuntimeSkipped(SupportedLanguages.Zig) &&
fs.existsSync(path.join(vendoredGrammarDir('tree-sitter-zig'), 'package.json'));
it.skipIf(!zigPackageInstalled)('parses functions, structs, enums, and imports', async () => {
expect(isLanguageAvailable(SupportedLanguages.Zig)).toBe(true);
await loadLanguage(SupportedLanguages.Zig);
@ -740,11 +734,10 @@ describe('Tree-sitter multi-language parsing', () => {
// Exercise the loader's real absent-binding branch (the `source.load()`
// catch in `loadGrammar`), not the `GITNEXUS_SKIP_OPTIONAL_GRAMMARS`
// opt-out, which is a separate code path with its own test
// (parser-loader-skip-optional.test.ts). The Zig row loads through the
// module-scoped `createRequire(import.meta.url)`, so stand in for
// `node:module` with a require that reports the package missing and
// delegates everything else. A fresh module copy is needed because the
// loader memoizes load results.
// (parser-loader-skip-optional.test.ts). Zig loads through
// `requireVendoredGrammar`, so stand in for that helper and report the
// grammar missing. A fresh module copy is needed because the loader
// memoizes load results.
//
// The fresh copy also re-reads `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` (parsed
// lazily once per module). Under `=zig` / `=all` — a supported way to
@ -752,25 +745,21 @@ describe('Tree-sitter multi-language parsing', () => {
// path below would never run, so the variable is cleared for this test
// and restored afterwards. Clearing (not skipping) keeps the branch
// exercised in every environment.
const ZIG_PKG = '@tree-sitter-grammars/tree-sitter-zig';
const SKIP_ENV = 'GITNEXUS_SKIP_OPTIONAL_GRAMMARS';
const savedSkip = process.env[SKIP_ENV];
delete process.env[SKIP_ENV];
vi.doMock('node:module', async (importOriginal) => {
const actual = await importOriginal<typeof import('node:module')>();
vi.doMock('../../src/core/tree-sitter/vendored-grammars.js', async (importOriginal) => {
const actual =
await importOriginal<typeof import('../../src/core/tree-sitter/vendored-grammars.js')>();
return {
...actual,
createRequire: (url: string | URL) => {
const real = actual.createRequire(url);
const stub = ((id: string) => {
if (id === ZIG_PKG) {
const err = new Error(`Cannot find module '${id}'`) as NodeJS.ErrnoException;
err.code = 'MODULE_NOT_FOUND';
throw err;
}
return real(id);
}) as unknown as NodeJS.Require;
return Object.assign(stub, real);
requireVendoredGrammar: (packageName: string) => {
if (packageName === 'tree-sitter-zig') {
const err = new Error(`Cannot find module '${packageName}'`) as NodeJS.ErrnoException;
err.code = 'MODULE_NOT_FOUND';
throw err;
}
return actual.requireVendoredGrammar(packageName);
},
};
});
@ -788,7 +777,7 @@ describe('Tree-sitter multi-language parsing', () => {
} finally {
if (savedSkip === undefined) delete process.env[SKIP_ENV];
else process.env[SKIP_ENV] = savedSkip;
vi.doUnmock('node:module');
vi.doUnmock('../../src/core/tree-sitter/vendored-grammars.js');
vi.resetModules();
}
});
@ -1005,8 +994,8 @@ describe('Tree-sitter multi-language parsing', () => {
[SupportedLanguages.CSharp, 'simple.cs'],
[SupportedLanguages.Rust, 'simple.rs'],
[SupportedLanguages.PHP, 'simple.php'],
// Dart and Swift (vendored optional grammars) and Zig (npm optionalDependency)
// are excluded — none of the three is guaranteed to be installed
// Dart, Swift, and Zig (vendored optional grammars) are excluded —
// none of the three is guaranteed to be installed on every platform.
];
for (const [lang, fixture, filePath] of langFixtures) {

View file

@ -115,7 +115,7 @@ describe('build-tree-sitter-grammars.cjs consolidated activation', () => {
});
it('never exits non-zero across grammars and env permutations (postinstall hard invariant)', () => {
for (const grammar of ['c', 'dart', 'proto', 'swift', 'kotlin']) {
for (const grammar of ['c', 'dart', 'proto', 'swift', 'kotlin', 'zig']) {
for (const overrides of [{ GITNEXUS_SKIP_OPTIONAL_GRAMMARS: '1' }, {}]) {
const r = runBuild(grammar, overrides);
expect(r.status, `${grammar} ${JSON.stringify(overrides)}`).toBe(0);

View file

@ -100,6 +100,8 @@ describe('CLI commands', () => {
expect(optional['tree-sitter-dart']).toBeUndefined();
expect(optional['tree-sitter-proto']).toBeUndefined();
expect(optional['tree-sitter-swift']).toBeUndefined();
expect(optional['tree-sitter-zig']).toBeUndefined();
expect(optional['@tree-sitter-grammars/tree-sitter-zig']).toBeUndefined();
// #2111: the grammars MUST NOT be copied into node_modules at install — an
// undeclared node_modules package is "extraneous" to every subsequent
// npm/npx reify, which prunes/relocates it (Windows EPERM symlink + silent
@ -174,6 +176,23 @@ describe('CLI commands', () => {
expect(kotlinPkg.default.peerDependencies['tree-sitter']).toContain('^0.21');
});
it('vendors tree-sitter-zig instead of an npm optionalDependency', async () => {
const pkg = await import('../../package.json', { with: { type: 'json' } });
const zigPkg = await import('../../vendor/tree-sitter-zig/package.json', {
with: { type: 'json' },
});
const optional = pkg.default.optionalDependencies ?? {};
expect(optional['@tree-sitter-grammars/tree-sitter-zig']).toBeUndefined();
expect(Object.keys(pkg.default.overrides ?? {})).not.toContain(
'@tree-sitter-grammars/tree-sitter-zig',
);
expect(pkg.default.scripts.postinstall).toContain('build-tree-sitter-grammars.cjs');
expect(zigPkg.default.version).toBe('1.1.2');
expect(zigPkg.default.scripts?.install).toBeUndefined();
expect(zigPkg.default.dependencies).toBeUndefined();
expect(zigPkg.default.peerDependencies['tree-sitter']).toContain('^0.21');
});
it('vendors tree-sitter-c prebuild-only at the 0.21.4 ABI pin instead of an npm dependency (#2116/#1242)', async () => {
const pkg = await import('../../package.json', { with: { type: 'json' } });
const cPkg = await import('../../vendor/tree-sitter-c/package.json', {

View file

@ -73,20 +73,28 @@ describe('COMPATIBLE_ABI gate', () => {
});
describe('GRAMMARS registry', () => {
it('covers all five grammars (swift/kotlin npm, dart/proto github, c npm)', () => {
expect(Object.keys(mod.GRAMMARS).sort()).toEqual(['c', 'dart', 'kotlin', 'proto', 'swift']);
it('covers all six grammars (swift/kotlin/zig npm, dart/proto github, c npm)', () => {
expect(Object.keys(mod.GRAMMARS).sort()).toEqual([
'c',
'dart',
'kotlin',
'proto',
'swift',
'zig',
]);
expect(mod.GRAMMARS.swift.npm).toBe('tree-sitter-swift');
expect(mod.GRAMMARS.dart.github).toContain('tree-sitter-dart');
expect(mod.GRAMMARS.zig.npm).toBe('@tree-sitter-grammars/tree-sitter-zig');
});
it('marks c and kotlin report-only (holds); swift/dart/proto are auto-updatable', () => {
it('marks c and kotlin report-only (holds); swift/dart/proto/zig are auto-updatable', () => {
expect(mod.GRAMMARS.c.npm).toBe('tree-sitter-c');
expect(mod.GRAMMARS.c.hold).toBeTruthy(); // ABI-pinned: detected/reported, never auto-applied
// kotlin is pinned to an unreleased fwcd main commit for `fun interface`
// support (#169); npm latest (0.3.8) lacks it, so the strict-inequality
// isNewer would auto-revert the pin without this hold.
expect(mod.GRAMMARS.kotlin.hold).toBeTruthy();
for (const k of ['swift', 'dart', 'proto']) {
for (const k of ['swift', 'dart', 'proto', 'zig']) {
expect(mod.GRAMMARS[k].hold).toBeUndefined();
}
});

View file

@ -6,7 +6,7 @@ import { SupportedLanguages } from '../../src/config/supported-languages.js';
*
* `GITNEXUS_SKIP_OPTIONAL_GRAMMARS` used to be an install-time-only env (the
* postinstall build scripts read it). `parser-loader` now also honors it at
* analyze time: when set, genuinely-optional grammars (swift/dart/kotlin)
* analyze time: when set, genuinely-optional grammars (swift/dart/kotlin/zig)
* report unavailable so the ingestion pipeline skips their files, mirroring a
* genuinely-absent binding. Grammars that are required `dependencies` routed
* through the optional machinery for ABI safety (C — `severity: 'error'`) are
@ -39,9 +39,8 @@ describe('parser-loader GITNEXUS_SKIP_OPTIONAL_GRAMMARS runtime gate', () => {
expect(isLanguageAvailable(SupportedLanguages.Swift)).toBe(false);
expect(isLanguageAvailable(SupportedLanguages.Dart)).toBe(false);
expect(isLanguageAvailable(SupportedLanguages.Kotlin)).toBe(false);
// Zig is an npm optionalDependency rather than a vendored grammar, but it
// is just as genuinely optional: the documented opt-out must cover it too
// (`optional-grammars.ts` lists it among the skippable grammars).
// Zig is a vendored optional grammar: the documented opt-out must cover
// it too (`optional-grammars.ts` lists it among the skippable grammars).
expect(isLanguageAvailable(SupportedLanguages.Zig)).toBe(false);
});
@ -58,6 +57,7 @@ describe('parser-loader GITNEXUS_SKIP_OPTIONAL_GRAMMARS runtime gate', () => {
expect(isLanguageAvailable(SupportedLanguages.Swift), `value=${v}`).toBe(false);
expect(isLanguageAvailable(SupportedLanguages.Dart), `value=${v}`).toBe(false);
expect(isLanguageAvailable(SupportedLanguages.Kotlin), `value=${v}`).toBe(false);
expect(isLanguageAvailable(SupportedLanguages.Zig), `value=${v}`).toBe(false);
}
});

View file

@ -1,7 +1,19 @@
import { describe, it, expect } from 'vitest';
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
import { afterEach, beforeEach, describe, it, expect } from 'vitest';
import {
existsSync,
mkdtempSync,
mkdirSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from 'node:fs';
import { execFileSync, spawnSync } from 'node:child_process';
import { tmpdir } from 'node:os';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { load } from 'js-yaml';
/**
* Regression guard: every tree-sitter grammar GitNexus ships must provide a
@ -175,3 +187,188 @@ describe('npm-dependency grammar prebuild coverage', () => {
});
}
});
describe('prebuild workflow validate snippets cover every REGISTRY grammar', () => {
// The "Validate the .node loads and parses" step looks up snippets[GRAMMAR].
// A missing key yields undefined, and tree-sitter's parse() throws
// "Input must be a function" — the six zig jobs on #3180.
const workflow = readFileSync(
path.join(GITNEXUS_ROOT, '..', '.github/workflows/build-tree-sitter-prebuilds.yml'),
'utf8',
);
const registry = [...workflow.matchAll(/^\s{12}(\w+):\s+\{\s+name:\s+'tree-sitter-/gm)].map(
(m) => m[1],
);
const snippets = [...workflow.matchAll(/^\s{14}(\w+):\s+"/gm)].map((m) => m[1]);
it('REGISTRY and snippets are both non-empty (the regex still matches the workflow)', () => {
expect(registry.length).toBeGreaterThan(0);
expect(snippets.length).toBeGreaterThan(0);
});
it('every REGISTRY grammar has a parse snippet', () => {
expect(snippets.sort(), 'add a snippets.<grammar> entry when extending REGISTRY').toEqual(
[...registry].sort(),
);
});
});
describe('prebuild workflow rebuild loop guard', () => {
const workflow = load(
readFileSync(
path.join(GITNEXUS_ROOT, '..', '.github/workflows/build-tree-sitter-prebuilds.yml'),
'utf8',
),
) as {
jobs: { guard: { steps: { id?: string; run?: string; env?: Record<string, string> }[] } };
};
const decide = workflow.jobs.guard.steps.find((step) => step.id === 'decide');
if (!decide?.run) throw new Error('Missing prebuild workflow Decide script');
// Execute the actual workflow script against real commits, including the
// cumulative PR diff that remains after generated binaries are committed.
const script = decide.run.match(/node --input-type=module - <<'NODE'\n([\s\S]*?)\nNODE/)?.[1];
if (!script) throw new Error('Missing prebuild workflow Decide Node heredoc');
let root: string;
let base: string;
let source: string;
const vendor = 'gitnexus/vendor/tree-sitter-zig';
function git(...args: string[]): string {
return execFileSync('git', args, { cwd: root, encoding: 'utf8' }).trim();
}
function write(rel: string, contents: string): void {
const dest = path.join(root, rel);
mkdirSync(path.dirname(dest), { recursive: true });
writeFileSync(dest, contents);
}
function commit(message: string): string {
git('add', 'gitnexus');
git('-c', 'commit.gpgsign=false', 'commit', '-qm', message);
return git('rev-parse', 'HEAD');
}
function runGuard(env: Record<string, string> = {}) {
const runnerTemp = mkdtempSync(path.join(root, 'runner-'));
const output = path.join(runnerTemp, 'output');
const result = spawnSync(process.execPath, ['--input-type=module', '-'], {
cwd: root,
input: script,
encoding: 'utf8',
env: {
...process.env,
EVENT: 'pull_request',
ACTION: 'synchronize',
BASE_SHA: base,
BEFORE_SHA: source,
HEAD_SHA: git('rev-parse', 'HEAD'),
INPUT_GRAMMARS: '',
INPUT_REF: '',
FORCE: 'false',
RUNNER_TEMP: runnerTemp,
GITHUB_OUTPUT: output,
...env,
},
});
return { ...result, output: existsSync(output) ? readFileSync(output, 'utf8') : '' };
}
function expectBuild(env: Record<string, string> = {}): void {
const result = runGuard(env);
expect(result.status, result.stderr).toBe(0);
expect(result.output).toContain('any=true\n');
const matrix = JSON.parse(result.output.split('matrix=')[1]);
expect(matrix.include).toHaveLength(6);
expect(matrix.include.every((entry: { grammar: string }) => entry.grammar === 'zig')).toBe(
true,
);
}
beforeEach(() => {
root = mkdtempSync(path.join(tmpdir(), 'gitnexus-prebuild-guard-'));
git('init', '-q');
git('config', 'user.name', 'Prebuild test');
git('config', 'user.email', 'prebuild-test@example.invalid');
mkdirSync(path.join(root, 'hooks'));
git('config', 'core.hooksPath', path.join(root, 'hooks'));
write('gitnexus/package.json', '{}');
base = commit('base without vendored zig');
write(`${vendor}/package.json`, '{"version":"1.1.2"}');
write(`${vendor}/src/parser.c`, 'original source');
source = commit('vendor zig source');
write(`${vendor}/prebuilds/win32-x64/tree-sitter-zig.node`, 'binary build 1');
write(`${vendor}/prebuilds/SHA256SUMS`, 'checksum 1');
commit('generated prebuilds');
});
afterEach(() => rmSync(root, { recursive: true, force: true }));
it('wires the event action and exact push endpoints into the guard', () => {
expect(decide.env).toMatchObject({
ACTION: '${{ github.event.action }}',
BEFORE_SHA: '${{ github.event.before }}',
HEAD_SHA: '${{ github.event.pull_request.head.sha }}',
});
});
it('stops repeated binary-only updates while the PR still contains new source', () => {
let before = source;
for (let build = 2; build <= 4; build++) {
const result = runGuard({ BEFORE_SHA: before });
expect(result.status, result.stderr).toBe(0);
expect(result.output).toBe('any=false\nmatrix={"include":[]}\n');
before = git('rev-parse', 'HEAD');
write(`${vendor}/prebuilds/win32-x64/tree-sitter-zig.node`, `binary build ${build}`);
write(`${vendor}/prebuilds/SHA256SUMS`, `checksum ${build}`);
commit('generated prebuilds');
}
});
it('builds a newly opened PR and supports manual recuts', () => {
expectBuild({ ACTION: 'opened', BEFORE_SHA: '' });
expectBuild({ EVENT: 'workflow_dispatch', ACTION: '', BEFORE_SHA: '', INPUT_GRAMMARS: 'zig' });
});
it('builds source changes even when the last commit in the push only updates binaries', () => {
expectBuild({ BEFORE_SHA: base });
const before = git('rev-parse', 'HEAD');
write(`${vendor}/src/parser.c`, 'updated source without a version bump');
commit('edit parser');
write(`${vendor}/prebuilds/SHA256SUMS`, 'checksum 2');
commit('generated prebuilds');
expectBuild({ BEFORE_SHA: before });
});
it('still builds after unrelated updates that may have cancelled an earlier build', () => {
const before = git('rev-parse', 'HEAD');
write('gitnexus/package.json', '{"description":"updated"}');
commit('update package');
expectBuild({ BEFORE_SHA: before });
});
it('uses event endpoints even when the checkout contains additional base-branch changes', () => {
const head = git('rev-parse', 'HEAD');
write(`${vendor}/src/parser.c`, 'source from an advanced PR merge ref');
commit('simulate merge ref changes');
const result = runGuard({ HEAD_SHA: head });
expect(result.status, result.stderr).toBe(0);
expect(result.output).toBe('any=false\nmatrix={"include":[]}\n');
});
it('does not hide source removal when a source file moves into prebuilds', () => {
const before = git('rev-parse', 'HEAD');
git('mv', `${vendor}/src/parser.c`, `${vendor}/prebuilds/parser.c`);
commit('move source');
expectBuild({ BEFORE_SHA: before });
});
it.each(['', 'not-a-sha', '0'.repeat(40)])(
'fails closed for an unavailable push endpoint: %s',
(before) => {
const result = runGuard({ BEFORE_SHA: before });
expect(result.status).not.toBe(0);
expect(result.output).not.toContain('any=true');
},
);
});

View file

@ -12,7 +12,7 @@ import {
/**
* Regression guard for #2111 / #1728.
*
* The five vendored tree-sitter grammars (c/dart/proto/swift/kotlin) MUST load
* The six vendored tree-sitter grammars (c/dart/proto/swift/kotlin/zig) MUST load
* from `vendor/` by absolute path and MUST NEVER be copied into / required from
* `node_modules`. An undeclared package under node_modules is "extraneous" to
* every subsequent npm/npx arborist reify, which prunes/relocates it — on

View file

@ -4,14 +4,14 @@
* from the PR review of the Zig provider — the assertion is the behavior that
* was wrong, not merely that extraction runs.
*
* `@tree-sitter-grammars/tree-sitter-zig` is an optionalDependency: the whole
* file skips cleanly when it is absent, mirroring the Dart/Kotlin suites.
* Vendored `tree-sitter-zig` may be absent on a platform without a prebuild:
* the whole file skips cleanly when it is, mirroring the Dart/Kotlin suites.
*/
import { describe, it, expect } from 'vitest';
import { createRequire } from 'node:module';
import Parser from 'tree-sitter';
import { SupportedLanguages, type BindingRef, type SymbolDefinition } from 'gitnexus-shared';
import { isOptionalGrammarRequired } from '../helpers/optional-grammar.js';
import { requireVendoredGrammar } from '../../src/core/tree-sitter/vendored-grammars.js';
import type { SyntaxNode } from '../../src/core/ingestion/utils/ast-helpers.js';
import { zigExportChecker } from '../../src/core/ingestion/export-detection.js';
import { createMethodExtractor } from '../../src/core/ingestion/method-extractors/generic.js';
@ -50,10 +50,9 @@ import { createSemanticModel } from '../../src/core/ingestion/model/semantic-mod
import { extract as extractScopes } from '../../src/core/ingestion/scope-extractor.js';
import { populateClassOwnedMembers } from '../../src/core/ingestion/scope-resolution/scope/walkers.js';
const _require = createRequire(import.meta.url);
let Zig: unknown = null;
try {
Zig = _require('@tree-sitter-grammars/tree-sitter-zig');
Zig = requireVendoredGrammar('tree-sitter-zig');
} catch {
// optional grammar absent on this platform — suite skips below
}
@ -71,7 +70,7 @@ describe.skipIf(!isOptionalGrammarRequired(SupportedLanguages.Zig))(
expect(
Zig,
'GITNEXUS_REQUIRE_ZIG=1 declares the Zig grammar mandatory on this runner, but ' +
"require('@tree-sitter-grammars/tree-sitter-zig') failed — every case below would " +
"requireVendoredGrammar('tree-sitter-zig') failed — every case below would " +
'have skipped and the job would still be green.',
).not.toBeNull();
});

21
gitnexus/vendor/tree-sitter-zig/LICENSE vendored Normal file
View file

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2024 Amaan Qureshi <amaanq12@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View file

@ -0,0 +1,25 @@
## GitNexus vendor notice
This directory is a GitNexus-managed vendored copy of
`@tree-sitter-grammars/tree-sitter-zig@1.1.2`. GitNexus keeps the top-level
`tree-sitter` dependency pinned to `0.21.1` until the broader parser runtime
upgrade is handled separately (#858). The upstream package declares
`peerOptional tree-sitter@^0.22.1`; that peer is ABI-compatible with 0.21.x
but npm still warns on `npm i -g gitnexus` because a published package's
`overrides` are ignored by dependents.
Unified with the Dart/Proto/Kotlin/Swift/C vendored grammars, this copy
vendors the grammar **source** — `binding.gyp`, `bindings/node/binding.cc`,
`src/parser.c` (ABI 14), and `src/tree_sitter/` — so
`gitnexus/scripts/build-tree-sitter-grammars.cjs` can source-build the native
binding on a toolchain host when no committed prebuild matches. The native
`prebuilds/` are loaded in place by `node-gyp-build`. Upstream's published
`linux-arm64` `.node` is a mispackaged x86-64 binary and is not shipped;
`.github/workflows/build-tree-sitter-prebuilds.yml` rebuilds all six tuples.
When updating this vendor package, replace it from the official
`@tree-sitter-grammars/tree-sitter-zig` npm release: refresh
`src/parser.c`/`src/tree_sitter/`/`binding.gyp`/`bindings/node/`, bump
`version` in `package.json` to retrigger the prebuild workflow, update the
`_vendoredBy` provenance, and verify the packed GitNexus tarball can both
load a committed prebuild and source-build `tree-sitter-zig`.

View file

@ -0,0 +1,29 @@
{
"targets": [
{
"target_name": "tree_sitter_zig_binding",
"dependencies": [
"<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
],
"include_dirs": [
"src",
],
"sources": [
"bindings/node/binding.cc",
"src/parser.c",
],
"conditions": [
["OS!='win'", {
"cflags_c": [
"-std=c11",
],
}, { # OS == "win"
"cflags_c": [
"/std:c11",
"/utf-8",
],
}],
],
}
]
}

View file

@ -0,0 +1,20 @@
#include <napi.h>
typedef struct TSLanguage TSLanguage;
extern "C" TSLanguage *tree_sitter_zig();
// "tree-sitter", "language" hashed with BLAKE2
const napi_type_tag LANGUAGE_TYPE_TAG = {
0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
};
Napi::Object Init(Napi::Env env, Napi::Object exports) {
exports["name"] = Napi::String::New(env, "zig");
auto language = Napi::External<TSLanguage>::New(env, tree_sitter_zig());
language.TypeTag(&LANGUAGE_TYPE_TAG);
exports["language"] = language;
return exports;
}
NODE_API_MODULE(tree_sitter_zig_binding, Init)

View file

@ -0,0 +1,28 @@
type BaseNode = {
type: string;
named: boolean;
};
type ChildNode = {
multiple: boolean;
required: boolean;
types: BaseNode[];
};
type NodeInfo =
| (BaseNode & {
subtypes: BaseNode[];
})
| (BaseNode & {
fields: { [name: string]: ChildNode };
children?: ChildNode;
});
type Language = {
name: string;
language: unknown;
nodeTypeInfo: NodeInfo[];
};
declare const language: Language;
export = language;

View file

@ -0,0 +1,11 @@
const root = require("path").join(__dirname, "..", "..");
module.exports =
typeof process.versions.bun === "string"
// Support `bun build --compile` by being statically analyzable enough to find the .node file at build-time
? require(`../../prebuilds/${process.platform}-${process.arch}/tree-sitter-zig.node`)
: require("node-gyp-build")(root);
try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");
} catch (_) {}

View file

@ -0,0 +1,900 @@
/**
* @file Zig grammar for tree-sitter
* @author Amaan Qureshi <amaanq12@gmail.com>
* @license MIT
*/
/// <reference types="tree-sitter-cli/dsl" />
// @ts-check
const PREC = {
PAREN_DECLARATOR: -10,
CONDITIONAL: -1,
DEFAULT: 0,
LOGICAL_OR: 1,
LOGICAL_AND: 2,
EQUAL: 3,
BITWISE: 4,
SHIFT: 5,
ADD: 6,
MULTIPLY: 7,
UNARY: 8,
STRUCT: 9,
MEMBER: 10,
};
const builtinTypes = [
'bool',
'f16',
'f32',
'f64',
'f128',
'void',
'type',
'anyerror',
'anyopaque',
'anytype',
'noreturn',
'isize',
'usize',
'comptime_int',
'comptime_float',
'c_short',
'c_ushort',
'c_int',
'c_uint',
'c_long',
'c_ulong',
'c_longlong',
'c_ulonglong',
'c_longdouble',
/(i|u)[1-9][0-9]*/,
];
module.exports = grammar({
name: 'zig',
conflicts: $ => [
[$.for_expression],
[$.while_expression],
[$.expression, $._function_prototype],
[$.expression, $.if_type_expression],
[$.comptime_type_expression, $.expression],
[$.comptime_type_expression, $.parameter],
],
extras: $ => [
$.comment,
/\s/,
],
inline: $ => [
$._reserved_identifier,
],
precedences: $ => [
[$.container_field, $.type_expression],
],
supertypes: $ => [
$.statement,
$.expression,
$.type_expression,
$.primary_type_expression,
],
word: $ => $._identifier,
rules: {
source_file: $ => optional($._container_members),
_container_members: $ => choice(
seq(
repeat1(choice(
$.test_declaration,
$.comptime_declaration,
$.variable_declaration,
$.function_declaration,
$.using_namespace_declaration,
seq($.container_field, ','),
)),
optional($.container_field),
),
$.container_field,
),
test_declaration: $ => seq(
optional('pub'),
'test',
optional(choice($.string, $.identifier)),
$.block,
),
comptime_declaration: $ => prec(1, seq(
optional('pub'),
'comptime',
$.block,
)),
container_field: $ => prec.right(prec.dynamic(1, seq(
optional('comptime'),
choice(
seq(
field('name', choice($.identifier, $._reserved_identifier, alias($.builtin_type, $.identifier))),
':',
field('type', choice($.primary_type_expression, $.if_type_expression, $.comptime_type_expression)),
),
field('name', choice($.primary_type_expression, $.if_type_expression, $.comptime_type_expression)),
),
optional($.byte_alignment),
optional(seq('=', $.expression)),
))),
variable_declaration: $ => seq(
optional('pub'),
optional(choice(
'export',
seq('extern', optional($.string)),
)),
optional('threadlocal'),
$._variable_declaration_header,
optional(seq('=', $.expression)),
';',
),
_variable_declaration_expression_statement: $ => choice(
seq(
$._variable_declaration_header,
repeat(prec(1, seq(',', choice($._variable_declaration_header, $.expression)))),
'=',
$.expression,
';',
),
seq(
$.expression,
choice(
seq(
choice(
'=', '*=', '*%=', '*|=', '/=', '%=',
'+=', '+%=', '+|=', '-=', '-%=', '-|=',
'<<=', '<<|=', '>>=', '&=', '^=', '|=',
),
$.expression,
),
seq(
repeat1(prec(1, seq(',', choice($._variable_declaration_header, $.expression)))),
'=',
$.expression,
),
),
';',
),
),
_variable_declaration_header: $ => prec(1, seq(
choice('const', 'var'),
$.identifier,
optional(seq(
':',
field('type', choice($.type_expression, $.if_type_expression, $.comptime_type_expression)),
)),
optional($.byte_alignment),
optional($.address_space),
optional($.link_section),
)),
function_declaration: $ => seq(
optional('pub'),
optional(choice(
'export',
seq('extern', optional($.string)),
'inline',
'noinline',
)),
$._function_prototype,
choice(
';',
field('body', $.block),
),
),
_function_prototype: $ => seq(
'fn',
optional(field('name', $.identifier)),
$.parameters,
optional($.byte_alignment),
optional($.address_space),
optional($.link_section),
optional($.calling_convention),
field('type', choice($.type_expression, $.if_type_expression, $.comptime_type_expression)),
),
parameters: $ => seq('(', optionalCommaSep($.parameter), ')'),
parameter: $ => choice(
seq(
optional(choice('noalias', 'comptime')),
optional(seq(
field('name', choice($.identifier, alias($.builtin_type, $.identifier))),
':',
)),
field('type', choice($.type_expression, $.if_type_expression, $.comptime_type_expression)),
),
'...',
),
using_namespace_declaration: $ => seq(
optional('pub'),
'usingnamespace',
$.expression,
';',
),
block: $ => seq(
'{',
repeat($.statement),
'}',
),
struct_declaration: $ => seq(
optional(choice('extern', 'packed')),
'struct',
optional(seq('(', $.expression, ')')),
'{',
$._container_members,
'}',
),
opaque_declaration: $ => seq(
optional(choice('extern', 'packed')),
'opaque',
'{',
$._container_members,
'}',
),
enum_declaration: $ => seq(
optional(choice('extern', 'packed')),
'enum',
optional(seq('(', $.expression, ')')),
'{',
$._container_members,
'}',
),
union_declaration: $ => seq(
optional(choice('extern', 'packed')),
'union',
optional(seq(
'(',
choice(
seq('enum', optional(seq('(', $.expression, ')'))),
$.expression,
),
')',
)),
'{',
$._container_members,
'}',
),
error_set_declaration: $ => seq(
'error',
'{',
optionalCommaSep($.identifier),
'}',
),
statement: $ => choice(
$.comptime_statement,
$.nosuspend_statement,
$.suspend_statement,
$.defer_statement,
$.errdefer_statement,
$.expression_statement,
alias($._variable_declaration_expression_statement, $.variable_declaration),
$.if_statement,
$.for_statement,
$.while_statement,
$.labeled_statement,
prec(1, $.switch_expression),
),
comptime_statement: $ => seq(
'comptime',
choice(
$._block_expr_statement,
alias($._variable_declaration_expression_statement, $.variable_declaration),
),
),
nosuspend_statement: $ => seq('nosuspend', $._block_expr_statement),
suspend_statement: $ => seq('suspend', $._block_expr_statement),
defer_statement: $ => seq('defer', $._block_expr_statement),
errdefer_statement: $ => seq('errdefer', optional($.payload), $._block_expr_statement),
_block_expr_statement: $ => prec(1, choice(
seq(optional($.block_label), $.block),
$.expression_statement,
)),
block_expression: $ => prec(1, seq(optional($.block_label), $.block)),
labeled_statement: $ => prec(1, seq(
optional($.block_label),
choice($.block, $.for_statement, $.while_statement),
)),
expression_statement: $ => seq($.expression, ';'),
if_statement: $ => seq(
$._if_prefix,
$._conditional_body,
),
_if_prefix: $ => seq(
'if',
'(',
field('condition', $.expression),
')',
optional($.payload),
),
else_clause: $ => seq(
'else',
optional($.payload),
field('alternative', $.statement),
),
for_statement: $ => seq(
optional('inline'),
$._for_prefix,
$._conditional_body,
),
_for_prefix: $ => seq(
'for',
'(',
optionalCommaSep(seq(
$.expression,
optional(seq('..', $.expression)),
)),
')',
$.payload,
),
while_statement: $ => seq(
optional('inline'),
$._while_prefix,
$._conditional_body,
),
_while_prefix: $ => seq(
'while',
'(',
field('condition', $.expression),
')',
optional($.payload),
optional(seq(':', '(', $.expression, ')')),
),
_conditional_body: $ => choice(
seq(
field('body', $.block_expression),
optional($.else_clause),
),
seq(
field('body', $.expression),
choice(';', $.else_clause),
),
),
payload: $ => seq('|', optionalCommaSep1(seq(optional('*'), $.identifier)), '|'),
byte_alignment: $ => seq('align', '(', $.expression, ')'),
address_space: $ => seq('addrspace', '(', $.expression, ')'),
link_section: $ => seq('linksection', '(', $.expression, ')'),
calling_convention: $ => seq('callconv', '(', $.expression, ')'),
expression: $ => prec.right(choice(
$.asm_expression,
$.if_expression,
$.for_expression,
$.while_expression,
$.assignment_expression,
$.unary_expression,
$.binary_expression,
$.comptime_expression,
$.async_expression,
$.await_expression,
$.nosuspend_expression,
$.continue_expression,
$.resume_expression,
$.return_expression,
$.break_expression,
$.try_expression,
$.catch_expression,
$.type_expression,
$.block,
)),
asm_expression: $ => seq(
'asm',
optional('volatile'),
'(',
$.expression,
optional($.asm_output),
')',
),
asm_output: $ => seq(':', optionalCommaSep($.asm_output_item), optional($.asm_input)),
asm_output_item: $ => seq(
'[',
$.identifier,
']',
choice($.string, $.multiline_string),
'(',
choice(seq('->', $.type_expression), $.identifier),
')',
),
asm_input: $ => seq(':', optionalCommaSep($.asm_input_item), optional($.asm_clobbers)),
asm_input_item: $ => seq(
'[',
$.identifier,
']',
choice($.string, $.multiline_string),
'(',
$.expression,
')',
),
asm_clobbers: $ => seq(':', optionalCommaSep(choice($.string, $.multiline_string))),
if_expression: $ => prec.right(seq(
$._if_prefix,
$.expression,
optional(seq('else', optional($.payload), $.expression)),
)),
for_expression: $ => prec.right(seq(
optional($.block_label),
optional('inline'),
$._for_prefix,
$.expression,
optional(seq('else', $.expression)),
)),
while_expression: $ => prec.right(seq(
optional($.block_label),
optional('inline'),
$._while_prefix,
$.expression,
optional(seq('else', optional($.payload), $.expression)),
)),
assignment_expression: $ => prec.right(seq(
field('left', $.expression),
field('operator', choice(
'=', '*=', '*%=', '*|=', '/=', '%=',
'+=', '+%=', '+|=', '-=', '-%=', '-|=',
'<<=', '<<|=', '>>=', '&=', '^=', '|=',
)),
field('right', $.expression),
)),
unary_expression: $ => prec.left(PREC.UNARY, seq(
field('operator', choice('!', '~', '-', '-%', '&')),
field('argument', $.expression),
)),
binary_expression: $ => {
const table = [
['or', PREC.LOGICAL_OR],
['and', PREC.LOGICAL_AND],
['==', PREC.EQUAL],
['!=', PREC.EQUAL],
['>', PREC.EQUAL],
['>=', PREC.EQUAL],
['<=', PREC.EQUAL],
['<', PREC.EQUAL],
['&', PREC.BITWISE],
['^', PREC.BITWISE],
['|', PREC.BITWISE],
['orelse', PREC.BITWISE],
['<<', PREC.SHIFT],
['>>', PREC.SHIFT],
['<<|', PREC.SHIFT],
['+', PREC.ADD],
['-', PREC.ADD],
['++', PREC.ADD],
['+%', PREC.ADD],
['-%', PREC.ADD],
['+|', PREC.ADD],
['-|', PREC.ADD],
['*', PREC.MULTIPLY],
['/', PREC.MULTIPLY],
['%', PREC.MULTIPLY],
['**', PREC.MULTIPLY],
['*%', PREC.MULTIPLY],
['*|', PREC.MULTIPLY],
['||', PREC.MULTIPLY],
];
return choice(...table.map(([operator, precedence]) => {
return prec.left(precedence, seq(
field('left', $.expression),
// @ts-ignore
field('operator', operator),
field('right', $.expression),
));
}));
},
comptime_expression: $ => prec.right(seq('comptime', $.expression)),
async_expression: $ => prec.right(seq('async', $.expression)),
await_expression: $ => prec.right(seq('await', $.expression)),
nosuspend_expression: $ => prec.right(seq('nosuspend', $.expression)),
continue_expression: $ => prec.right(seq(
'continue',
optional($.break_label),
optional($.expression),
)),
resume_expression: $ => prec.right(seq('resume', $.expression)),
return_expression: $ => prec.right(seq('return', optional($.expression))),
break_expression: $ => prec.right(seq(
'break',
optional($.break_label),
optional($.expression),
)),
try_expression: $ => prec.right(PREC.BITWISE, seq('try', $.expression)),
catch_expression: $ => prec.right(PREC.BITWISE, seq(
$.expression,
'catch',
optional($.payload),
$.expression,
)),
switch_expression: $ => seq(
optional($.block_label),
'switch',
'(', $.expression, ')',
'{',
optionalCommaSep($.switch_case),
'}',
),
switch_case: $ => seq(
$._switch_case_exp,
'=>',
optional($.payload),
choice($.expression),
),
_switch_case_exp: $ => seq(
optional('inline'),
choice(
optionalCommaSep1(seq($.expression, optional(seq('...', $.expression)))),
'else',
),
),
type_expression: $ => prec.right(choice(
$.anonymous_struct_initializer,
$.struct_initializer,
$.labeled_type_expression,
$.error_set_declaration,
$.parenthesized_expression,
$.primary_type_expression,
)),
primary_type_expression: $ => choice(
$.nullable_type,
$.anyframe_type,
$.slice_type,
$.pointer_type,
$.array_type,
$.error_union_type,
$.builtin_function,
$.character,
$.field_expression,
$.index_expression,
$.dereference_expression,
$.null_coercion_expression,
$.range_expression,
$.call_expression,
prec.right(alias($._function_prototype, $.function_signature)),
$.identifier,
$.float,
$.integer,
$.boolean,
$.error_type,
'anyframe',
'unreachable',
'undefined',
'null',
$.string,
$.multiline_string,
$.builtin_type,
$.struct_declaration,
$.opaque_declaration,
$.enum_declaration,
$.union_declaration,
$.switch_expression,
),
nullable_type: $ => prec(1, seq(
'?',
choice($.type_expression, $.if_type_expression, $.comptime_type_expression),
)),
anyframe_type: $ => prec(1, seq(
'anyframe',
'->',
$.type_expression,
)),
slice_type: $ => prec.right(1, seq(
'[',
optional(seq(
':',
field('sentinel', $.expression),
)),
']',
repeat(choice(
$.byte_alignment,
$.address_space,
'const',
'volatile',
'allowzero',
)),
$.type_expression,
)),
pointer_type: $ => prec.right(1, seq(
choice(
'*',
seq(
'[',
'*',
optional(choice('c', seq(':', $.expression))),
']',
),
),
repeat(choice(
$.address_space,
seq(
'align',
'(',
$.expression,
optional(seq(':', $.expression, ':', $.expression)),
')',
),
'const',
'volatile',
'allowzero',
)),
$.type_expression,
)),
array_type: $ => prec(1, seq(
'[',
$.expression,
optional(seq(':', $.expression)),
']',
$.type_expression,
)),
error_union_type: $ => prec.right(2, seq(
optional(field('error', $.type_expression)),
'!',
field('ok', $.type_expression),
)),
field_expression: $ => prec(PREC.MEMBER, seq(
optional(field('object', $.expression)),
'.',
field('member', $.identifier),
)),
index_expression: $ => prec(PREC.MEMBER, seq(
field('object', $.expression),
'[',
field('index', $.expression),
optional(seq(':', field('sentinel', $.expression))),
']',
)),
dereference_expression: $ => prec(PREC.MEMBER, seq($.expression, '.*')),
null_coercion_expression: $ => prec(PREC.MEMBER, seq($.expression, '.?')),
range_expression: $ => prec.right(PREC.MEMBER, seq(
field('left', $.expression),
'..',
optional(field('right', $.expression)),
)),
call_expression: $ => prec(PREC.MEMBER, seq(
field('function', $.expression),
'(',
optionalCommaSep($.expression),
')',
)),
anonymous_struct_initializer: $ => seq('.', $.initializer_list),
struct_initializer: $ => prec(-1, seq($.primary_type_expression, $.initializer_list)),
initializer_list: $ => seq(
'{',
choice(
optionalCommaSep($.field_initializer),
optionalCommaSep($.expression),
),
'}',
),
field_initializer: $ => seq(
'.',
$.identifier,
'=',
$.expression,
),
labeled_type_expression: $ => seq($.block_label, $.block),
comptime_type_expression: $ => seq('comptime', $.type_expression),
if_type_expression: $ => prec.right(seq(
$._if_prefix,
$.type_expression,
optional(seq('else', optional($.payload), $.type_expression)),
)),
parenthesized_expression: $ => seq('(', $.expression, ')'),
block_label: $ => prec(-1, seq(
choice($.identifier, alias($.builtin_type, $.identifier)),
':',
)),
break_label: $ => seq(':', $.identifier),
arguments: $ => seq('(', optionalCommaSep($.expression), ')'),
builtin_function: $ => seq(
$.builtin_identifier,
$.arguments,
),
string: $ => seq(
'"',
repeat(choice(
alias(token.immediate(prec(1, /[^\\"\n]+/)), $.string_content),
$.escape_sequence,
)),
'"',
),
multiline_string: _ => prec.right(repeat1(token(seq('\\\\', /[^\n]*/)))),
escape_sequence: _ => token(prec(1, seq(
'\\',
choice(
/[^xuU]/,
/\d{2,3}/,
/x[0-9a-fA-F]{2,}/,
/u\{[0-9a-fA-F]{1,6}\}/,
),
))),
character: $ => seq(
'\'',
choice(
alias(/[^'\n]/, $.character_content),
$.escape_sequence,
),
'\'',
),
integer: _ => {
const separator = '_';
const hex = /[0-9A-Fa-f]/;
const oct = /[0-7]/;
const bin = /[0-1]/;
const decimal = /[0-9]/;
const hexDigits = seq(repeat1(hex), repeat(seq(separator, repeat1(hex))));
const octDigits = seq(repeat1(oct), repeat(seq(separator, repeat1(oct))));
const binDigits = seq(repeat1(bin), repeat(seq(separator, repeat1(bin))));
const decimalDigits = seq(repeat1(decimal), repeat(seq(separator, repeat1(decimal))));
return token(choice(
seq('0x', hexDigits),
seq('0o', octDigits),
seq('0b', binDigits),
decimalDigits,
));
},
float: _ => {
const separator = '_';
const hex = /[0-9A-Fa-f]/;
const decimal = /[0-9]/;
const hexDigits = seq(repeat1(hex), repeat(seq(separator, repeat1(hex))));
const decimalDigits = seq(repeat1(decimal), repeat(seq(separator, repeat1(decimal))));
return token(choice(
seq('0x', hexDigits, '.', hexDigits, optional(seq(/[pP][+-]?/, decimalDigits))),
seq(decimalDigits, '.', decimalDigits, optional(seq(/[eE][+-]?/, decimalDigits))),
seq('0x', hexDigits, /[pP][+-]?/, decimalDigits),
seq(decimalDigits, /[eE][+-]?/, decimalDigits),
));
},
boolean: _ => choice('true', 'false'),
builtin_type: _ => choice(...builtinTypes),
error_type: $ => seq('error', '.', $.identifier),
builtin_identifier: _ => /@[A-Za-z_][A-Za-z0-9_]*/,
identifier: $ => choice($._identifier, seq('@', $.string)),
_identifier: _ => /[A-Za-z_][A-Za-z0-9_]*/,
_reserved_identifier: _ => choice(
'undefined',
'null',
'true',
'false',
),
comment: _ => token(seq('//', /.*/)),
},
});
/**
* Creates a rule to match optionally match one or more of the rules
* separated by a comma and optionally ending with a comma
*
* @param {RuleOrLiteral} rule
*
* @returns {ChoiceRule}
*/
function optionalCommaSep(rule) {
return optional(optionalCommaSep1(rule));
}
/**
* Creates a rule to match one or more of the rules separated by a comma
* and optionally ending with a comma
*
* @param {RuleOrLiteral} rule
*
* @returns {SeqRule}
*/
function optionalCommaSep1(rule) {
return seq(commaSep1(rule), optional(','));
}
/**
* Creates a rule to match one or more of the rules separated by a comma
*
* @param {RuleOrLiteral} rule
*
* @returns {SeqRule}
*/
function commaSep1(rule) {
return seq(rule, repeat(seq(',', rule)));
}

View file

@ -0,0 +1,18 @@
{
"name": "tree-sitter-zig",
"version": "1.1.2",
"description": "Zig grammar for tree-sitter",
"repository": "https://github.com/tree-sitter-grammars/tree-sitter-zig",
"license": "MIT",
"main": "bindings/node/index.js",
"types": "bindings/node/index.d.ts",
"_vendoredBy": "gitnexus - runtime package derived from @tree-sitter-grammars/tree-sitter-zig@1.1.2 (ABI 14). Vendored so `npm i -g gitnexus` no longer hits peerOptional tree-sitter@^0.22.1 against the bundled tree-sitter@0.21.1 (npm overrides on a published package are ignored by dependents). Loaded from vendor/ by absolute path at runtime (vendored-grammars.ts) — NEVER copied to node_modules (#2111). No scripts/dependencies here (#836/#1728). Upstream linux-arm64 prebuild is a mispackaged x86-64 binary and is omitted; GitNexus-cross-built prebuilds come from .github/workflows/build-tree-sitter-prebuilds.yml (regenerated whenever this `version` changes). To re-pin: bump `version`, refresh src/ + bindings/node/ from the new npm tarball, update this note, and let the prebuild workflow rebuild the binaries.",
"peerDependencies": {
"tree-sitter": "^0.21.0"
},
"peerDependenciesMeta": {
"tree-sitter": {
"optional": true
}
}
}

View file

@ -0,0 +1,6 @@
ad8b6ee26be562765ee377077a555c2225893a0c609143310d30dfbf49865fa2 ./darwin-arm64/tree-sitter-zig.node
67e1edd9147bbf213f8b2cceccba55590b505b6d696118e53b1d38f5ad0fc0e2 ./darwin-x64/tree-sitter-zig.node
9e77bd9cf21e71b741f71401081c958214f3b01fd73a6cfb5025ac1058ea0664 ./linux-arm64/tree-sitter-zig.node
1e12ccf46b093e5d878e18688b8284a29a4ceb742ef1cbf398ac17d15d5ea44f ./linux-x64/tree-sitter-zig.node
61c56d87d58a551fd48c890b1d03a2d26326da075ca8a4104554968456df82b7 ./win32-arm64/tree-sitter-zig.node
82dcf42dda4bc3faadaaf2bcedea0a07e78c55bfb7a315e9bbeb5427721d4406 ./win32-x64/tree-sitter-zig.node

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

169829
gitnexus/vendor/tree-sitter-zig/src/parser.c vendored Normal file

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,54 @@
#ifndef TREE_SITTER_ALLOC_H_
#define TREE_SITTER_ALLOC_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
// Allow clients to override allocation functions
#ifdef TREE_SITTER_REUSE_ALLOCATOR
extern void *(*ts_current_malloc)(size_t size);
extern void *(*ts_current_calloc)(size_t count, size_t size);
extern void *(*ts_current_realloc)(void *ptr, size_t size);
extern void (*ts_current_free)(void *ptr);
#ifndef ts_malloc
#define ts_malloc ts_current_malloc
#endif
#ifndef ts_calloc
#define ts_calloc ts_current_calloc
#endif
#ifndef ts_realloc
#define ts_realloc ts_current_realloc
#endif
#ifndef ts_free
#define ts_free ts_current_free
#endif
#else
#ifndef ts_malloc
#define ts_malloc malloc
#endif
#ifndef ts_calloc
#define ts_calloc calloc
#endif
#ifndef ts_realloc
#define ts_realloc realloc
#endif
#ifndef ts_free
#define ts_free free
#endif
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ALLOC_H_

View file

@ -0,0 +1,291 @@
#ifndef TREE_SITTER_ARRAY_H_
#define TREE_SITTER_ARRAY_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "./alloc.h"
#include <assert.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#ifdef _MSC_VER
#pragma warning(push)
#pragma warning(disable : 4101)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-variable"
#endif
#define Array(T) \
struct { \
T *contents; \
uint32_t size; \
uint32_t capacity; \
}
/// Initialize an array.
#define array_init(self) \
((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
/// Create an empty array.
#define array_new() \
{ NULL, 0, 0 }
/// Get a pointer to the element at a given `index` in the array.
#define array_get(self, _index) \
(assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
/// Get a pointer to the first element in the array.
#define array_front(self) array_get(self, 0)
/// Get a pointer to the last element in the array.
#define array_back(self) array_get(self, (self)->size - 1)
/// Clear the array, setting its size to zero. Note that this does not free any
/// memory allocated for the array's contents.
#define array_clear(self) ((self)->size = 0)
/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
/// less than the array's current capacity, this function has no effect.
#define array_reserve(self, new_capacity) \
_array__reserve((Array *)(self), array_elem_size(self), new_capacity)
/// Free any memory allocated for this array. Note that this does not free any
/// memory allocated for the array's contents.
#define array_delete(self) _array__delete((Array *)(self))
/// Push a new `element` onto the end of the array.
#define array_push(self, element) \
(_array__grow((Array *)(self), 1, array_elem_size(self)), \
(self)->contents[(self)->size++] = (element))
/// Increase the array's size by `count` elements.
/// New elements are zero-initialized.
#define array_grow_by(self, count) \
do { \
if ((count) == 0) break; \
_array__grow((Array *)(self), count, array_elem_size(self)); \
memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
(self)->size += (count); \
} while (0)
/// Append all elements from one array to the end of another.
#define array_push_all(self, other) \
array_extend((self), (other)->size, (other)->contents)
/// Append `count` elements to the end of the array, reading their values from the
/// `contents` pointer.
#define array_extend(self, count, contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), (self)->size, \
0, count, contents \
)
/// Remove `old_count` elements from the array starting at the given `index`. At
/// the same index, insert `new_count` new elements, reading their values from the
/// `new_contents` pointer.
#define array_splice(self, _index, old_count, new_count, new_contents) \
_array__splice( \
(Array *)(self), array_elem_size(self), _index, \
old_count, new_count, new_contents \
)
/// Insert one `element` into the array at the given `index`.
#define array_insert(self, _index, element) \
_array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
/// Remove one element from the array at the given `index`.
#define array_erase(self, _index) \
_array__erase((Array *)(self), array_elem_size(self), _index)
/// Pop the last element off the array, returning the element by value.
#define array_pop(self) ((self)->contents[--(self)->size])
/// Assign the contents of one array to another, reallocating if necessary.
#define array_assign(self, other) \
_array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
/// Swap one array with another
#define array_swap(self, other) \
_array__swap((Array *)(self), (Array *)(other))
/// Get the size of the array contents
#define array_elem_size(self) (sizeof *(self)->contents)
/// Search a sorted array for a given `needle` value, using the given `compare`
/// callback to determine the order.
///
/// If an existing element is found to be equal to `needle`, then the `index`
/// out-parameter is set to the existing value's index, and the `exists`
/// out-parameter is set to true. Otherwise, `index` is set to an index where
/// `needle` should be inserted in order to preserve the sorting, and `exists`
/// is set to false.
#define array_search_sorted_with(self, compare, needle, _index, _exists) \
_array__search_sorted(self, 0, compare, , needle, _index, _exists)
/// Search a sorted array for a given `needle` value, using integer comparisons
/// of a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_with`.
#define array_search_sorted_by(self, field, needle, _index, _exists) \
_array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
/// Insert a given `value` into a sorted array, using the given `compare`
/// callback to determine the order.
#define array_insert_sorted_with(self, compare, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
/// Insert a given `value` into a sorted array, using integer comparisons of
/// a given struct field (specified with a leading dot) to determine the order.
///
/// See also `array_search_sorted_by`.
#define array_insert_sorted_by(self, field, value) \
do { \
unsigned _index, _exists; \
array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
if (!_exists) array_insert(self, _index, value); \
} while (0)
// Private
typedef Array(void) Array;
/// This is not what you're looking for, see `array_delete`.
static inline void _array__delete(Array *self) {
if (self->contents) {
ts_free(self->contents);
self->contents = NULL;
self->size = 0;
self->capacity = 0;
}
}
/// This is not what you're looking for, see `array_erase`.
static inline void _array__erase(Array *self, size_t element_size,
uint32_t index) {
assert(index < self->size);
char *contents = (char *)self->contents;
memmove(contents + index * element_size, contents + (index + 1) * element_size,
(self->size - index - 1) * element_size);
self->size--;
}
/// This is not what you're looking for, see `array_reserve`.
static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
if (new_capacity > self->capacity) {
if (self->contents) {
self->contents = ts_realloc(self->contents, new_capacity * element_size);
} else {
self->contents = ts_malloc(new_capacity * element_size);
}
self->capacity = new_capacity;
}
}
/// This is not what you're looking for, see `array_assign`.
static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
_array__reserve(self, element_size, other->size);
self->size = other->size;
memcpy(self->contents, other->contents, self->size * element_size);
}
/// This is not what you're looking for, see `array_swap`.
static inline void _array__swap(Array *self, Array *other) {
Array swap = *other;
*other = *self;
*self = swap;
}
/// This is not what you're looking for, see `array_push` or `array_grow_by`.
static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
uint32_t new_size = self->size + count;
if (new_size > self->capacity) {
uint32_t new_capacity = self->capacity * 2;
if (new_capacity < 8) new_capacity = 8;
if (new_capacity < new_size) new_capacity = new_size;
_array__reserve(self, element_size, new_capacity);
}
}
/// This is not what you're looking for, see `array_splice`.
static inline void _array__splice(Array *self, size_t element_size,
uint32_t index, uint32_t old_count,
uint32_t new_count, const void *elements) {
uint32_t new_size = self->size + new_count - old_count;
uint32_t old_end = index + old_count;
uint32_t new_end = index + new_count;
assert(old_end <= self->size);
_array__reserve(self, element_size, new_size);
char *contents = (char *)self->contents;
if (self->size > old_end) {
memmove(
contents + new_end * element_size,
contents + old_end * element_size,
(self->size - old_end) * element_size
);
}
if (new_count > 0) {
if (elements) {
memcpy(
(contents + index * element_size),
elements,
new_count * element_size
);
} else {
memset(
(contents + index * element_size),
0,
new_count * element_size
);
}
}
self->size += new_count - old_count;
}
/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
do { \
*(_index) = start; \
*(_exists) = false; \
uint32_t size = (self)->size - *(_index); \
if (size == 0) break; \
int comparison; \
while (size > 1) { \
uint32_t half_size = size / 2; \
uint32_t mid_index = *(_index) + half_size; \
comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
if (comparison <= 0) *(_index) = mid_index; \
size -= half_size; \
} \
comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
if (comparison == 0) *(_exists) = true; \
else if (comparison < 0) *(_index) += 1; \
} while (0)
/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
/// parameter by reference in order to work with the generic sorting function above.
#define _compare_int(a, b) ((int)*(a) - (int)(b))
#ifdef _MSC_VER
#pragma warning(pop)
#elif defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic pop
#endif
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_ARRAY_H_

View file

@ -0,0 +1,266 @@
#ifndef TREE_SITTER_PARSER_H_
#define TREE_SITTER_PARSER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include <stdbool.h>
#include <stdint.h>
#include <stdlib.h>
#define ts_builtin_sym_error ((TSSymbol)-1)
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
#endif
typedef struct {
TSFieldId field_id;
uint8_t child_index;
bool inherited;
} TSFieldMapEntry;
typedef struct {
uint16_t index;
uint16_t length;
} TSFieldMapSlice;
typedef struct {
bool visible;
bool named;
bool supertype;
} TSSymbolMetadata;
typedef struct TSLexer TSLexer;
struct TSLexer {
int32_t lookahead;
TSSymbol result_symbol;
void (*advance)(TSLexer *, bool);
void (*mark_end)(TSLexer *);
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
TSParseActionTypeShift,
TSParseActionTypeReduce,
TSParseActionTypeAccept,
TSParseActionTypeRecover,
} TSParseActionType;
typedef union {
struct {
uint8_t type;
TSStateId state;
bool extra;
bool repetition;
} shift;
struct {
uint8_t type;
uint8_t child_count;
TSSymbol symbol;
int16_t dynamic_precedence;
uint16_t production_id;
} reduce;
uint8_t type;
} TSParseAction;
typedef struct {
uint16_t lex_state;
uint16_t external_lex_state;
} TSLexMode;
typedef union {
TSParseAction action;
struct {
uint8_t count;
bool reusable;
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
uint32_t alias_count;
uint32_t token_count;
uint32_t external_token_count;
uint32_t state_count;
uint32_t large_state_count;
uint32_t production_id_count;
uint32_t field_count;
uint16_t max_alias_sequence_length;
const uint16_t *parse_table;
const uint16_t *small_parse_table;
const uint32_t *small_parse_table_map;
const TSParseActionEntry *parse_actions;
const char * const *symbol_names;
const char * const *field_names;
const TSFieldMapSlice *field_map_slices;
const TSFieldMapEntry *field_map_entries;
const TSSymbolMetadata *symbol_metadata;
const TSSymbol *public_symbol_map;
const uint16_t *alias_map;
const TSSymbol *alias_sequences;
const TSLexMode *lex_modes;
bool (*lex_fn)(TSLexer *, TSStateId);
bool (*keyword_lex_fn)(TSLexer *, TSStateId);
TSSymbol keyword_capture_token;
struct {
const bool *states;
const TSSymbol *symbol_map;
void *(*create)(void);
void (*destroy)(void *);
bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
unsigned (*serialize)(void *, char *);
void (*deserialize)(void *, const char *, unsigned);
} external_scanner;
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
next_state: \
lexer->advance(lexer, skip); \
start: \
skip = false; \
lookahead = lexer->lookahead;
#define ADVANCE(state_value) \
{ \
state = state_value; \
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
state = state_value; \
goto next_state; \
}
#define ACCEPT_TOKEN(symbol_value) \
result = true; \
lexer->result_symbol = symbol_value; \
lexer->mark_end(lexer);
#define END_STATE() return result;
/*
* Parse Table Macros
*/
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
#define ACTIONS(id) id
#define SHIFT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value) \
} \
}}
#define SHIFT_REPEAT(state_value) \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = (state_value), \
.repetition = true \
} \
}}
#define SHIFT_EXTRA() \
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.extra = true \
} \
}}
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \
{{ \
.type = TSParseActionTypeRecover \
}}
#define ACCEPT_INPUT() \
{{ \
.type = TSParseActionTypeAccept \
}}
#ifdef __cplusplus
}
#endif
#endif // TREE_SITTER_PARSER_H_