diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9b6019aff..92c9887a0 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -380,12 +380,13 @@ jobs: # (#836) plus native grammar loading from the bundled runtime. Runs on windows-latest because that is the platform the fix # targets; the in-repo `npm ci` job above only exercises the dev-tree path # and skips the tarball reify step where the historical EPERM occurred. + # Native Linux ARM64 also verifies the corrected bundled prebuilds (#3599). packaged-install-smoke: name: packaged install smoke (${{ matrix.os }}) strategy: fail-fast: false matrix: - os: [windows-latest, ubuntu-latest] + os: [windows-latest, ubuntu-latest, ubuntu-24.04-arm] runs-on: ${{ matrix.os }} # Windows pack + web install regularly exceeds 15 minutes when setup also # runs prepare/postinstall/build before prepack compiles the same tree again. @@ -404,6 +405,18 @@ jobs: with: lifecycle-scripts: 'false' + # Reproduce #3599's runtime after the shared action selects Node 22.x. + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + if: matrix.os == 'ubuntu-24.04-arm' + with: + node-version: '22.23.1' + package-manager-cache: false + - name: Assert native ARM64 runtime + if: matrix.os == 'ubuntu-24.04-arm' + shell: bash + run: | + node -e "const assert = require('node:assert/strict'); assert.equal(process.platform, 'linux'); assert.equal(process.arch, 'arm64'); assert.equal(process.version, 'v22.23.1'); console.log(process.platform, process.arch, process.version)" + # `npm pack` runs prepack, which builds the web UI into gitnexus/web/ # so the tarball matches what `npm publish` ships. Install those deps # here, in their own visible step, rather than letting build.js do it @@ -418,6 +431,43 @@ jobs: run: npm pack working-directory: gitnexus + # Block native compilation only in the consumer phase. A correct bundle + # must install using prebuilds even when the runner has a toolchain. + - name: Reject ARM64 consumer compilation + if: matrix.os == 'ubuntu-24.04-arm' + shell: bash + run: | + set -euo pipefail + TOOLS="$RUNNER_TEMP/gitnexus-no-compile" + mkdir -p "$TOOLS" + export GITNEXUS_COMPILER_SENTINEL_LOG="$RUNNER_TEMP/gitnexus-compile-attempts.log" + cat > "$TOOLS/cc" <<'EOF' + #!/usr/bin/env bash + echo "Unexpected native compilation: $0 $*" >&2 + printf '%s\n' "$0 $*" >> "${GITNEXUS_COMPILER_SENTINEL_LOG:?}" + exit 97 + EOF + chmod +x "$TOOLS/cc" + for tool in c++ gcc g++ clang clang++ make gmake; do + cp "$TOOLS/cc" "$TOOLS/$tool" + done + # Prove the guard fails before relying on it, then clear its probe. + if "$TOOLS/cc" --version; then + echo 'Compiler sentinel unexpectedly succeeded' >&2 + exit 1 + else + test "$?" -eq 97 + fi + test -s "$GITNEXUS_COMPILER_SENTINEL_LOG" + : > "$GITNEXUS_COMPILER_SENTINEL_LOG" + echo "$TOOLS" >> "$GITHUB_PATH" + { + echo "GITNEXUS_COMPILER_SENTINEL_LOG=$GITNEXUS_COMPILER_SENTINEL_LOG" + echo "CC=$TOOLS/cc" + echo "CXX=$TOOLS/c++" + echo "MAKE=$TOOLS/make" + } >> "$GITHUB_ENV" + # Global installs do not exercise npm's bundled-dependency lockfile path # (#3563). Keep npm 10 here: newer npm can mask unresolved override refs. - name: Lock and install tarball in a fresh consumer project @@ -462,6 +512,23 @@ jobs: echo "PREFIX=$PREFIX" >> "$GITHUB_ENV" working-directory: gitnexus + # npm can ignore failed optional-dependency scripts, so also inspect the + # sentinel log after both installs rather than trusting exit status alone. + - name: Assert no consumer compilation attempts + if: always() && matrix.os == 'ubuntu-24.04-arm' + shell: bash + run: | + set -euo pipefail + if [ -z "${GITNEXUS_COMPILER_SENTINEL_LOG:-}" ] || [ ! -f "$GITNEXUS_COMPILER_SENTINEL_LOG" ]; then + echo 'Compiler sentinel was not initialized' >&2 + exit 1 + fi + if [ -s "$GITNEXUS_COMPILER_SENTINEL_LOG" ]; then + echo 'Unexpected native compilation during consumer installation:' >&2 + cat "$GITNEXUS_COMPILER_SENTINEL_LOG" >&2 + exit 1 + fi + - name: Assert no junctions or vendor build artifacts shell: bash run: | diff --git a/.github/workflows/release-evaluation.yml b/.github/workflows/release-evaluation.yml index fd7c086bc..57ce90627 100644 --- a/.github/workflows/release-evaluation.yml +++ b/.github/workflows/release-evaluation.yml @@ -294,7 +294,9 @@ jobs: GITNEXUS_REQUIRE_BWRAP_CANARY: '1' GITNEXUS_REQUIRE_CLAUDE_CANARY: '1' CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude - run: uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py -q + run: >- + uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py + tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q - name: Materialize both release graphs before paid sessions timeout-minutes: 120 working-directory: eval @@ -322,7 +324,7 @@ jobs: --tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \ --gitnexus-root "$GITHUB_WORKSPACE/$runtime" \ --arms baseline_nomcp baseline --runs 3 --include-expensive \ - --workers 3 --timeout 1800 --model "$MODEL" --effort "$EFFORT" \ + --workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \ --claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \ --out "$RUNNER_TEMP/release-evaluation/$runtime/raw" done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e8584e8c2..0d2f0c3a8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -165,10 +165,14 @@ peers against their original ranges even when an override is configured. The lockfile therefore pins the same nine audited peers to exactly `0.25.1`. `gitnexus/scripts/prepare-tree-sitter-bundle.cjs` runs during `postinstall` and `prepack` to keep installed dependency manifests consistent with that lockfile -and the published bundle. It leaves grammar sources and native binaries unchanged -and rejects unexpected versions, peer ranges, or dependency layouts before writing -any manifests. The upstream ranges remain in the script only as validation inputs; -installed and published audited manifests require exactly `0.25.1`. +and the published bundle. It leaves grammar sources unchanged and replaces the +incorrect `linux-arm64` prebuilds in five exact npm releases with audited AArch64 +binaries. It validates package versions, peer ranges, dependency layouts, and +replacement checksums and architecture before writing manifests or binaries. +The upstream ranges remain in the script only as validation inputs; installed and +published audited manifests require exactly `0.25.1`. See +[`gitnexus/prebuilds/bundled/README.md`](gitnexus/prebuilds/bundled/README.md) for +the affected releases, provenance, and regeneration instructions. When upgrading these packages, update the exact pins, lockfile, and audited peer list together. If regenerating the lockfile restores upstream's older peer ranges, diff --git a/eval/tests/test_ec2_workflows.py b/eval/tests/test_ec2_workflows.py index 3066f2c6d..a0f1b7d38 100644 --- a/eval/tests/test_ec2_workflows.py +++ b/eval/tests/test_ec2_workflows.py @@ -95,6 +95,22 @@ def test_release_comparison_is_scheduled_or_manual_and_gates_outcomes(): assert "--stable-sha" in scripts and "--candidate-sha" in scripts +def test_release_session_headroom_keeps_the_global_stop_reserve(): + steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"] + paid = next(step for step in steps if "workflow_bench.runner" in step.get("run", "")) + script = paid["run"] + assert "--timeout 3600" in script + assert "for runtime in stable candidate; do" in script + assert "stop_deadline_epoch }} - $(date +%s) - 5400" in script + assert 'timeout --signal=TERM --kill-after=60 "${remaining}s"' in script + assert paid["timeout-minutes"] == 1140 + canary = next(step for step in steps if "tests/test_proposer_sandbox.py" in step.get("run", "")) + assert "test_real_cli_background_completion_is_one_complete_measurement" in canary["run"] + assert canary["env"]["GITNEXUS_REQUIRE_CLAUDE_CANARY"] == "1" + assert steps.index(canary) < steps.index(paid) + assert not canary.get("continue-on-error", False) + + def test_both_runtimes_grade_tasks_against_one_pinned_task_dependency_checkout(): steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"] names = [step.get("name", step.get("uses", "")) for step in steps] diff --git a/eval/tests/test_workflow_bench_sessions.py b/eval/tests/test_workflow_bench_sessions.py index 0bd0a2e7f..f5a82beda 100644 --- a/eval/tests/test_workflow_bench_sessions.py +++ b/eval/tests/test_workflow_bench_sessions.py @@ -1295,6 +1295,136 @@ def test_final_result_event_must_be_last(monkeypatch, tmp_path): assert "not the last event" in rec["error_detail"]["event_stream_error"] +def background_completion_events(): + first = json.loads(report_variant(uuid="first", subtype="success", is_error=False)) + last = json.loads(report_variant(uuid="last", subtype="success", is_error=False, total_cost_usd=0.15)) + return [ + first, + {"type": "system", "subtype": "task_notification", "session_id": "s"}, + {"type": "system", "subtype": "init", "session_id": "s"}, + {"type": "assistant", "message": {"content": [{"type": "text", "text": "Task finished"}]}}, + last, + ] + + +def test_background_completion_sums_turn_usage_but_not_cumulative_cost(monkeypatch, tmp_path): + events = background_completion_events() + proc = replace(fake_cli_result("\n".join(map(json.dumps, events))), duration_s=12.5) + monkeypatch.setattr(runner_sessions, "run_managed", lambda *a, **k: proc) + rec = runner.run_claude("task", tmp_path, claude_bin="claude", timeout=30) + + assert rec["ok"] is True, rec["error_detail"] + assert rec["cost_usd"] == 0.15 # The second result includes the first turn's $0.10. + assert rec["num_turns"] == 6 + assert [rec[field] for field in runner_sessions.USAGE_FIELDS] == [2, 4, 6, 8] + assert rec["duration_s"] == 12.5 # Include background wait, not just the two 1s turns. + + +@pytest.mark.parametrize( + "mutate", + [ + lambda es: es[-1].update(session_id="different"), + lambda es: es[-1].update(uuid="first"), + lambda es: es[-1].update(total_cost_usd=0.05), + lambda es: es[0].update(is_error=True), + lambda es: es[0].update(subtype="error_max_turns"), + lambda es: es[0]["usage"].pop("input_tokens"), + lambda es: es[-1]["usage"].update(output_tokens=-1), + lambda es: es[-1]["usage"].update(output_tokens=True), + lambda es: es[-1].update(num_turns=-1), + lambda es: es[-1].update(total_cost_usd=None), + lambda es: es.pop(1), # No background completion to explain another turn. + lambda es: es.pop(3), # A second result without a new assistant turn is a replay. + lambda es: es[2].update(session_id="different"), + lambda es: es.append({"type": "assistant", "message": {"content": []}}), + ], +) +def test_unverifiable_background_continuation_stays_excluded(monkeypatch, tmp_path, mutate): + events = background_completion_events() + mutate(events) + monkeypatch.setattr( + runner_sessions, "run_managed", lambda *a, **k: fake_cli_result("\n".join(map(json.dumps, events))) + ) + rec = runner.run_claude("task", tmp_path, claude_bin="claude", timeout=30, transcript_output_dir=tmp_path) + assert rec["ok"] is False + assert rec["error_kind"] == "session-error" + assert rec["cost_usd"] is None + assert rec["transcript_missing"] is True + + +def test_real_cli_background_completion_is_one_complete_measurement(monkeypatch, tmp_path): + """Only the provider is scripted; the pinned CLI performs a real background Bash task.""" + from workflow_bench.mock_provider import MockProvider, Reply + + claude = os.environ.get("CLAUDE_CANARY_BIN") + if not claude or not Path(claude).is_file(): + if os.environ.get("GITNEXUS_REQUIRE_CLAUDE_CANARY") == "1": + pytest.fail("required pinned CLAUDE_CANARY_BIN is missing") + pytest.skip("the containment job supplies the pinned CLAUDE_CANARY_BIN") + home = tmp_path / "home" + home.mkdir() + # Release the real background command only after the first result. A fixed + # sleep could finish before that result on a loaded runner and miss the bug. + observe_line = runner_sessions.SessionProgress._observe_line + + def release_background_task(self, line): + observe_line(self, line) + if json.loads(line).get("type") == "result": + (home / "completion-ready").touch() + + monkeypatch.setattr(runner_sessions.SessionProgress, "_observe_line", release_background_task) + replies = [ + Reply( + tools=[ + { + "name": "Bash", + "input": { + "command": "while [ ! -f completion-ready ]; do sleep 0.01; done; printf done", + "run_in_background": True, + }, + } + ], + input_tokens=101, + output_tokens=11, + ), + Reply(text="First completion", input_tokens=102, output_tokens=12), + Reply(text="Background completion acknowledged", input_tokens=103, output_tokens=13), + ] + with MockProvider(replies) as provider: + rec = runner.run_claude( + "Run the command", + home, + claude_bin=claude, + timeout=40, + env={ + "PATH": os.environ["PATH"], + "HOME": str(home), + "ANTHROPIC_API_KEY": "offline-probe", + "ANTHROPIC_BASE_URL": provider.base_url, + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1", + "DISABLE_AUTOUPDATER": "1", + # Only loopback may be contacted; even CLI startup traffic + # cannot reach an external provider. No host credentials inherit. + "HTTPS_PROXY": "http://127.0.0.1:9", + "HTTP_PROXY": "http://127.0.0.1:9", + "NO_PROXY": "127.0.0.1,localhost", + }, + allowed_tools=["Bash"], + strict_mcp_config=True, + transcript_output_dir=tmp_path, + progress_label="offline-background-completion", + ) + assert rec["ok"] is True, rec["error_detail"] + assert len(provider.requests) == 3 + assert rec["input_tokens"] == 306 and rec["output_tokens"] == 36 + assert rec["num_turns"] == 3 + artifacts = list((tmp_path / "transcripts").glob("*.jsonl")) + assert len(artifacts) == 1 + results = [e for line in artifacts[0].read_text().splitlines() if (e := json.loads(line))["type"] == "result"] + assert len(results) == 2 + assert rec["cost_usd"] == results[-1]["total_cost_usd"] > results[0]["total_cost_usd"] + + def test_background_task_teardown_after_the_result_stays_valid_evidence(monkeypatch, tmp_path): # Claude Code drains background-task bookkeeping after the final result # event. Those `system` events carry no tool or usage payload, so they must diff --git a/eval/workflow_bench/README.md b/eval/workflow_bench/README.md index 2659f9f10..e8002e219 100644 --- a/eval/workflow_bench/README.md +++ b/eval/workflow_bench/README.md @@ -244,12 +244,26 @@ Every evaluated repo-local skill root is over-mounted read-only for the full model session, and an immutable empty user-level skills directory prevents a writable `$HOME` skill from shadowing it. Skill-use evidence comes only from the bounded stream captured directly from Claude stdout by the parent. The -runner parses every event through EOF, requires one final result, correlates +runner parses every event through EOF and requires a terminal result. A +background-task completion can start another foreground turn in the pinned +Claude CLI: those results must have distinct IDs in the same session, complete +per-turn usage, successful outcomes, and nondecreasing cumulative cost. The +runner sums their tokens and turn counts, takes the last cumulative cost, and +measures the whole process duration including the background wait. Missing, +replayed, mismatched, or unsuccessful continuations remain excluded. It correlates an exact Skill request ID with one later successful result, structurally redacts the event objects, and stores the canonical redacted JSONL with a digest. Files written beneath the agent's `$HOME` are never trusted as evidence. +Release evaluation allows 3600 seconds per session in both runtimes and arms. +Run `38873262106` hit the former 1800-second limit twice while still editing +and testing; the longer ceiling gives those sessions headroom, not a guarantee +of completion. The overall deadline and 5400-second evidence/cleanup reserve +remain enforced. Treat this ceiling and the result-accounting change as harness +differences when comparing against older runs; invalid historical measurements +are not repaired retroactively. + Sessions use a private home without stored Claude login/keychain state, and do not inherit `ANTHROPIC_AUTH_TOKEN`. Supply an Anthropic API key through `GITNEXUS_BENCH_ANTHROPIC_API_KEY` (preferred) or `--anthropic-api-key`; diff --git a/eval/workflow_bench/runner_sessions.py b/eval/workflow_bench/runner_sessions.py index e0fe991b4..a20e2ec87 100644 --- a/eval/workflow_bench/runner_sessions.py +++ b/eval/workflow_bench/runner_sessions.py @@ -619,6 +619,86 @@ def skill_was_invoked_events(events: Sequence[dict[str, Any]], skill_name: str) return successful +def _session_result(events: list[dict[str, Any]], duration_s: float) -> tuple[dict[str, Any], int]: + """Reduce proven background continuations, retaining one session's full accounting. + + Claude Code 2.1.214 emits a result per foreground turn, including a turn + started by a background task notification. Usage/turns are per-turn; cost + is cumulative. The real-CLI canary pins these different scopes. + """ + indexes = [index for index, event in enumerate(events) if event.get("type") == "result"] + if not indexes: + raise ValueError("expected a final result event, observed 0") + final_index = indexes[-1] + if any(event.get("type") != "system" for event in events[final_index + 1 :]): + raise ValueError("final result event is not the last event in the captured stream") + final = events[final_index] + if len(indexes) == 1: + return final, final_index + + session_id = final.get("session_id") + if not isinstance(session_id, str) or not session_id: + raise ValueError("background continuation has no session identity") + total_usage = dict.fromkeys(USAGE_FIELDS, 0) + turns = 0 + cost = 0.0 + result_ids: set[str] = set() + previous_index = -1 + for index in indexes: + result = events[index] + uuid = result.get("uuid") + if result.get("session_id") != session_id: + raise ValueError("background continuation changed session identity") + if not isinstance(uuid, str) or not uuid or uuid in result_ids: + raise ValueError("background continuation has a missing or repeated result id") + result_ids.add(uuid) + if result.get("subtype") != "success" or result.get("is_error") is not False: + raise ValueError("background continuation contains an unsuccessful result") + usage = result.get("usage") + if not isinstance(usage, dict) or any( + type(usage.get(field)) is not int or usage[field] < 0 for field in USAGE_FIELDS + ): + raise ValueError("background continuation has incomplete or invalid usage") + count = result.get("num_turns") + if type(count) is not int or count < 1: + raise ValueError("background continuation has invalid turn count") + next_cost = measured_cost(result.get("total_cost_usd")) + if next_cost is None or next_cost < cost: + raise ValueError("background continuation has missing or decreasing cumulative cost") + segment = events[previous_index + 1 : index] + if any(event.get("session_id", session_id) != session_id for event in segment): + raise ValueError("background continuation changed session identity") + if previous_index >= 0: + # Reject duplicate/replayed results and unrelated continuations. + # A task notification must precede a fresh foreground init and reply. + notification = next( + ( + i + for i, e in enumerate(segment) + if e.get("type") == "system" and e.get("subtype") == "task_notification" + ), + None, + ) + init = next( + (i for i, e in enumerate(segment) if e.get("type") == "system" and e.get("subtype") == "init"), None + ) + if ( + notification is None + or init is None + or notification >= init + or not any(e.get("type") == "assistant" for e in segment[init + 1 :]) + ): + raise ValueError("additional result lacks a background completion turn") + for field in USAGE_FIELDS: + total_usage[field] += usage[field] + turns += count + cost = next_cost + previous_index = index + # Per-turn duration_ms excludes time waiting for background work. Use the + # trusted parent's elapsed time for the whole process, including that wait. + return {**final, "usage": total_usage, "num_turns": turns, "duration_ms": duration_s * 1000}, final_index + + def run_claude( prompt: str, cwd: Path, @@ -718,19 +798,13 @@ def run_claude( if proc.stdout_capture_overflow: raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes") events = _parse_parent_event_stream(proc.stdout_capture) - result_indexes = [index for index, event in enumerate(events) if event.get("type") == "result"] - if len(result_indexes) != 1: - raise ValueError(f"expected exactly one final result event, observed {len(result_indexes)}") - result_index = result_indexes[0] - data = events[result_index] + data, result_index = _session_result(events, proc.duration_s) # A session that used a background task drains its bookkeeping after # the final result event (`background_tasks_changed`, `task_updated`, # `task_notification` — all `type: "system"`), so the result is last # only among the events that carry evidence. `system` events hold no # tool_use/tool_result/usage payload and so cannot forge skill or cost # evidence; any other event after the result still fails closed. - if any(event.get("type") != "system" for event in events[result_index + 1 :]): - raise ValueError("final result event is not the last event in the captured stream") # Nothing after the result is evidence. Cut the window here so that is # a property of what the evidence readers below can see, rather than an # assumption that a `system` event never carries a tool_use block. diff --git a/gitnexus/package.json b/gitnexus/package.json index 544adee41..de41608b7 100644 --- a/gitnexus/package.json +++ b/gitnexus/package.json @@ -32,6 +32,7 @@ }, "files": [ "dist", + "prebuilds/bundled", "hooks", "scripts", "skills", @@ -63,7 +64,7 @@ "postinstall": "node scripts/prepare-tree-sitter-bundle.cjs && node scripts/build-tree-sitter-grammars.cjs", "assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs", "prepare": "node scripts/build.js", - "prepack": "node scripts/prepare-tree-sitter-bundle.cjs && node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web", + "prepack": "node scripts/prepare-tree-sitter-bundle.cjs --for-publication && node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web", "version": "node scripts/sync-plugin-manifests.mjs" }, "dependencies": { diff --git a/gitnexus/prebuilds/bundled/README.md b/gitnexus/prebuilds/bundled/README.md new file mode 100644 index 000000000..9b78b4690 --- /dev/null +++ b/gitnexus/prebuilds/bundled/README.md @@ -0,0 +1,58 @@ +# Audited npm grammar prebuild repairs + +These five exact npm releases ship x86-64 binaries in their `linux-arm64` +prebuild directories: C++ 0.23.4, Java 0.23.5, Ruby 0.23.1, JavaScript 0.23.1 +(the TypeScript dependency), and TypeScript 0.23.2. The direct JavaScript 0.25.0 +package is unaffected. See GitNexus issue #3599. + +`manifest.json` records each immutable npm tarball URL, SHA-512 integrity, +SHA-256, upstream commit, unchanged source-file hashes, replacement SHA-256, +and build inputs. All generated parser C and scanner sources remain upstream +bytes. The TypeScript binary retains both TypeScript and TSX exports. + +The committed replacements were cross-built using GCC/G++ 13.3.0, binutils +2.42, node-gyp 11.5.0, node-addon-api 8.9.2 and Node 22.23.1 headers. Although +the build sysroot contains glibc 2.39, `readelf` confirms the outputs require +only GLIBC 2.17, GLIBCXX 3.4.21 and CXXABI 1.3.9, with a non-executable stack. +All six language exports loaded and parsed using tree-sitter 0.25.1 under ARM64 +Node 22.18.0, 22.23.1 and 24.19.0 via QEMU 8.2.2 user emulation. This is emulated +execution evidence; the packaged-install CI job supplies native ARM64 evidence. + +## Regeneration + +On Linux ARM64 with Node, npm, Python, make, GCC/G++, tar and readelf installed: + +```sh +cd gitnexus +node scripts/rebuild-bundled-grammar-prebuilds.cjs +node scripts/prepare-tree-sitter-bundle.cjs --for-publication +npx vitest run test/unit/prepare-tree-sitter-bundle.test.ts test/unit/prebuild-coverage.test.ts test/unit/parser-loader-abi.test.ts +``` + +The script downloads the recorded tarballs and headers, verifies their hashes, +installs the pinned build helpers with lifecycle scripts disabled, and builds +packages serially in a temporary tree. Builds use `gcc` and `g++` from `PATH`, +preserving only `PATH` and `HOME` from the caller's environment. Compiler, +make, and gyp overrides (including `CC`, `CXX`, and flags) are ignored so the +recorded toolchain matches the build. It checks unchanged source bytes, ELF +architecture, N-API, dynamic dependency baselines, and real native parsing +before replacing any committed output. Compiler changes can change the binary +hashes; review the updated binaries and provenance together. This is a bounded +npm-package overlay rebuild, separate from the source-vendored grammar matrix. + +## Packaging behavior + +Bundle preparation validates every repair before writing peer manifests or +binary files. `prepack` supplies `--for-publication` to remove local +`build/Release/*.node` and `build/Debug/*.node` outputs from repaired packages, +which would otherwise shadow their platform prebuilds. Normal consumer +postinstall keeps legitimate source-built fallbacks intact. Repair assets are +included in the npm package so postinstall remains self-contained. + +Corrected bytes must exist before consumer child install scripts run. The +upstream TypeScript and nested JavaScript builds share node-addon-api Makefile +outputs; configuring them at different dependency depths can overwrite those +outputs with conflicting relative stamp paths. Repairing this upstream build +behavior for explicit `--build-from-source` or raw source-checkout installation +is outside this fix. Ordinary supported Linux ARM64 glibc tarball installation +uses the corrected prebuilds and does not require compilation. diff --git a/gitnexus/prebuilds/bundled/manifest.json b/gitnexus/prebuilds/bundled/manifest.json new file mode 100644 index 000000000..7cef14acc --- /dev/null +++ b/gitnexus/prebuilds/bundled/manifest.json @@ -0,0 +1,255 @@ +{ + "schemaVersion": 1, + "tuple": "linux-arm64", + "build": { + "compiler": "aarch64-linux-gnu-gcc/g++ 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04.1)", + "linker": "GNU ld 2.42 (Ubuntu 2.42-4ubuntu2.10)", + "sysroot": "Ubuntu arm64 cross glibc 2.39-0ubuntu8cross1", + "hostArchitecture": "x64", + "hostNode": "24.19.0", + "nodeHeaders": { + "version": "22.23.1", + "url": "https://nodejs.org/dist/v22.23.1/node-v22.23.1-headers.tar.gz", + "sha256": "caf38d839815df92000b41b756056f575140f38f1d8c0cbef97448c33eb7d7b2" + }, + "dependencies": { + "node-addon-api": { + "version": "8.9.2", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==" + }, + "node-gyp": { + "version": "11.5.0", + "integrity": "sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ==" + }, + "node-gyp-build": { + "version": "4.8.4", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==" + } + }, + "runtime": "0.25.1", + "command": "node node_modules/node-gyp/bin/node-gyp.js rebuild --arch=arm64 --nodedir= --jobs=2", + "compilerFlags": ["--sysroot="], + "validation": { + "execution": "qemu-aarch64 8.2.2 user emulation", + "nodeVersions": ["22.18.0", "22.23.1", "24.19.0"], + "grammarExports": ["cpp", "java", "ruby", "javascript", "typescript", "tsx"], + "parseFailures": 0 + } + }, + "packages": [ + { + "name": "tree-sitter-cpp", + "version": "0.23.4", + "file": "tree-sitter-cpp@0.23.4/linux-arm64/tree-sitter-cpp.node", + "sha256": "57c248be4c90f8e9af63613bf49a2724a5de4482d4d3c30a99374ede9978d729", + "source": { + "url": "https://registry.npmjs.org/tree-sitter-cpp/-/tree-sitter-cpp-0.23.4.tgz", + "integrity": "sha512-qR5qUDyhZ5jJ6V8/umiBxokRbe89bCGmcq/dk94wI4kN86qfdV8k0GHIUEKaqWgcu42wKal5E97LKpLeVW8sKw==", + "gitHead": "f41e1a044c8a84ea9fa8577fdd2eab92ec96de02", + "sha256": "615d152f70e7a6bed8f15a93c3c0c264c40dc71c1b6a1445646fff9439e776bc", + "files": { + "binding.gyp": "e8c7ca4208ca882655c21a99b6ef0d153634eabb1cd901a45914a121b023555c", + "bindings/node/binding.cc": "9c50c8f087231e837648bd8660f84d2d9ea745ed6b886d05e94aa84cea584fd1", + "package.json": "bfde98d506b0847f2093336e5b4d7394dd311e78da69e22e5ecd73fba6db6066", + "src/grammar.json": "c0a93751d708440bb2c1929d738a5b03c4b835f61e5aeb580d0af269faf68f8d", + "src/node-types.json": "fdfd4b1f3dca1516616a1eb615bb6c1ad3082b8937ad21070d0def5dcfe7e535", + "src/parser.c": "2a35a43b4af6c9f7b69624ac00c2c50808912591450dc79c05dea03ac1bae814", + "src/scanner.c": "cf60387d290271f4d2fb558d0569b2b9ef879cb72206a9df5cc73b0b1a4b20ff", + "src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea", + "src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1", + "src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a" + }, + "generatedSourcesUnchanged": true + }, + "elf": { + "architecture": "AArch64", + "gnuStack": "RW", + "needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"], + "requiredSymbolVersions": [ + "CXXABI_1.3", + "CXXABI_1.3.9", + "GCC_3.0", + "GLIBCXX_3.4", + "GLIBCXX_3.4.21", + "GLIBC_2.17" + ], + "maxGLIBC": "2.17", + "maxGLIBCXX": "3.4.21", + "maxCXXABI": "1.3.9" + } + }, + { + "name": "tree-sitter-java", + "version": "0.23.5", + "file": "tree-sitter-java@0.23.5/linux-arm64/tree-sitter-java.node", + "sha256": "e91fa1c2dd4b7e13bebbd8c07c51d549d0b6da8b00d426a366ef1b515b159db8", + "source": { + "url": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.23.5.tgz", + "integrity": "sha512-Yju7oQ0Xx7GcUT01mUglPP+bYfvqjNCGdxqigTnew9nLGoII42PNVP3bHrYeMxswiCRM0yubWmN5qk+zsg0zMA==", + "gitHead": "94703d5a6bed02b98e438d7cad1136c01a60ba2c", + "sha256": "6b6c1b62605f9d8bb80efd2ebe5951a87b182faafa3eeb67c460a667818551b6", + "files": { + "binding.gyp": "42305f7d7305fc3a6d2cee4ee5e0e73e0db708e967f351bfad04872f46c8a3ac", + "bindings/node/binding.cc": "861ff9ef49c3e080dd00eebfd4d75b83f0ac64551a859caed23d4b416de88bf0", + "package.json": "c0c99e633aa28b2c1c58e61f91211391a9363d7d532aa5795bf4003f2dc14dba", + "src/grammar.json": "47ca521cd7a888d671d9f26301e8697fa686e04912032e959d242b5f25877eb6", + "src/node-types.json": "19c46facc653381c337ff6cad75dd8b052524179a366c80825d6d0010520eef2", + "src/parser.c": "4add5150cf4531eb5dd97f3343dcf65cd11704c84711348b328582b83424a0e4", + "src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea", + "src/tree_sitter/array.h": "5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94", + "src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a" + }, + "generatedSourcesUnchanged": true + }, + "elf": { + "architecture": "AArch64", + "gnuStack": "RW", + "needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"], + "requiredSymbolVersions": [ + "CXXABI_1.3", + "CXXABI_1.3.9", + "GCC_3.0", + "GLIBCXX_3.4", + "GLIBCXX_3.4.21", + "GLIBC_2.17" + ], + "maxGLIBC": "2.17", + "maxGLIBCXX": "3.4.21", + "maxCXXABI": "1.3.9" + } + }, + { + "name": "tree-sitter-ruby", + "version": "0.23.1", + "file": "tree-sitter-ruby@0.23.1/linux-arm64/tree-sitter-ruby.node", + "sha256": "e13ac3cbfcf40a5a2f350784d547d75b069174ca2955282d602f643a066a2c23", + "source": { + "url": "https://registry.npmjs.org/tree-sitter-ruby/-/tree-sitter-ruby-0.23.1.tgz", + "integrity": "sha512-d9/RXgWjR6HanN7wTYhS5bpBQLz1VkH048Vm3CodPGyJVnamXMGb8oEhDypVCBq4QnHui9sTXuJBBP3WtCw5RA==", + "gitHead": "71bd32fb7607035768799732addba884a37a6210", + "sha256": "1f0f0c100e31ddf650962ebb1f1fb8420bdbabccd49582f6e731ea7478db639d", + "files": { + "binding.gyp": "5890a49ccc6a81bf1da23c2b740725562acdbbb7c9813dfd87fc3fb228ea80c2", + "bindings/node/binding.cc": "b24ba437af1a31668a2765925f2f994a8516229fae81852451b21c35c32b67d2", + "package.json": "07fb6a02a020df5fcedd8cc968b266de69eb6e5a1f1e43bd7c5817306339019c", + "src/grammar.json": "ff0a0ab48788daea51cb418d9cfc14086dfcad50ac10e4981776be8b4d9296ab", + "src/node-types.json": "9683f80e7be3dd93f1d6f575800de3f3445cda8af1f313f1f80003229c328e8b", + "src/parser.c": "4ce468358b6f4e25a35c8cf6bc0eaf60665bc22d602f8c939323c2347255cd15", + "src/scanner.c": "e7a6196d6e78bf4c6728502e924c867dee5d851c6253e43fcdb8ba169009bc58", + "src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea", + "src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1", + "src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a" + }, + "generatedSourcesUnchanged": true + }, + "elf": { + "architecture": "AArch64", + "gnuStack": "RW", + "needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"], + "requiredSymbolVersions": [ + "CXXABI_1.3", + "CXXABI_1.3.9", + "GCC_3.0", + "GLIBCXX_3.4", + "GLIBCXX_3.4.21", + "GLIBC_2.17" + ], + "maxGLIBC": "2.17", + "maxGLIBCXX": "3.4.21", + "maxCXXABI": "1.3.9" + } + }, + { + "name": "tree-sitter-javascript", + "version": "0.23.1", + "file": "tree-sitter-javascript@0.23.1/linux-arm64/tree-sitter-javascript.node", + "sha256": "3e280c974f199cdd35a621b0bd556e238eb46ab0666e45c568d3bf477eee32b6", + "source": { + "url": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz", + "integrity": "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==", + "gitHead": "3a837b6f3658ca3618f2022f8707e29739c91364", + "sha256": "90e80b25a67517a4daf6ad751557bee21efbda7b7a5a554897933245d1734398", + "files": { + "binding.gyp": "e17df62e15b6b6a1590a77c61d0f614ba89c6fb77b807909050851d50008a2cb", + "bindings/node/binding.cc": "bd509e0a410b33707bf738d3b18ed9e250188155d10d3a13b778344c28b1b94a", + "package.json": "2d677a45985e03e329f78e2f8b0f7b401ae68698eec2743059fc34e56f1f6644", + "src/grammar.json": "f952064b8f29a084b6d879e8018a5c86b06efddc0e9f7c8951ae752832d41b05", + "src/node-types.json": "0d80ab597fcf1310efb9694d4276c407655d060b8bbab8f4ffda0223c43e94bf", + "src/parser.c": "1150721590eca3c7b6623c7dc3498184f3e22c5d895a08b4806ebf5804f817c3", + "src/scanner.c": "b3d3f64284d97bf80749c026862427782cf7ecc0b7dc094e6698ab311c9a42c7", + "src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea", + "src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1", + "src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a" + }, + "generatedSourcesUnchanged": true + }, + "elf": { + "architecture": "AArch64", + "gnuStack": "RW", + "needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"], + "requiredSymbolVersions": [ + "CXXABI_1.3", + "CXXABI_1.3.9", + "GCC_3.0", + "GLIBCXX_3.4", + "GLIBCXX_3.4.21", + "GLIBC_2.17" + ], + "maxGLIBC": "2.17", + "maxGLIBCXX": "3.4.21", + "maxCXXABI": "1.3.9" + } + }, + { + "name": "tree-sitter-typescript", + "version": "0.23.2", + "file": "tree-sitter-typescript@0.23.2/linux-arm64/tree-sitter-typescript.node", + "sha256": "d1a7c91ce8346de4aa4828649762625a0e8d51625472b33949932483d70e872e", + "source": { + "url": "https://registry.npmjs.org/tree-sitter-typescript/-/tree-sitter-typescript-0.23.2.tgz", + "integrity": "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==", + "gitHead": "f975a621f4e7f532fe322e13c4f79495e0a7b2e7", + "sha256": "0fdf63c35930a75885145d75ee63cb879da2d49db9bfe454bd8e60b08ba778a1", + "files": { + "binding.gyp": "5451214724030d5150581c5af614d2d37c10379d9801c1170035f9777058b9e1", + "bindings/node/binding.cc": "4cd4aa17fc240701572efe79b016aae0f2fa067f28a551de3cdba0e98537bc00", + "common/scanner.h": "da66ef2bd14a3f7ea743e25ba068c6c9aae2c3509db200ff80c4a0e6116e564c", + "package.json": "26892ef49837678a747f81e344ed26938ec5804e29e003ad34fd5a3c87c68baf", + "tsx/package.json": "d91e8e983679ba8bb4736e0680dc895513f2f10a7e7ff833bcc5f9e92ef0dfeb", + "tsx/src/grammar.json": "f98a7830b67be266c9e9effb673a5fc154ad6aa26f87f7c6d4a9191e09d8b357", + "tsx/src/node-types.json": "78b5789145286799a27a0a7ecc36cc1bcb151f94ec7fa631b248459867010c8c", + "tsx/src/parser.c": "1902cb53fa7ff5179df89b2eea863165e84c8cc866226419dc26921d8c055885", + "tsx/src/scanner.c": "d563cd30b2f39718c9ae4292795c5ce03a2ad01954ba3a86ef84c2781a736673", + "tsx/src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea", + "tsx/src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1", + "tsx/src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a", + "typescript/package.json": "fcbd2e0fb7e7eed1af20cfd562cd3c3b22c839cb95b9d18df5793b850bd601dd", + "typescript/src/grammar.json": "99dc4b95424dff47c89acb50d5d3f25b081b07eaff43994b7881dccd4186b214", + "typescript/src/node-types.json": "c790a733fc756b54d4e54dceeb7d2d51e40d8b57136e70277753a75804cce3e3", + "typescript/src/parser.c": "74fe453edd70f4eae9af0a1050cbd7943d8971d59165b6aaebbaa0a0b716d1aa", + "typescript/src/scanner.c": "9125013b42cb888379d9be909f1d73dfb75a37626c2cdbf4122718a2b431a6d3", + "typescript/src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea", + "typescript/src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1", + "typescript/src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a" + }, + "generatedSourcesUnchanged": true + }, + "elf": { + "architecture": "AArch64", + "gnuStack": "RW", + "needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"], + "requiredSymbolVersions": [ + "CXXABI_1.3", + "CXXABI_1.3.9", + "GCC_3.0", + "GLIBCXX_3.4", + "GLIBCXX_3.4.21", + "GLIBC_2.17" + ], + "maxGLIBC": "2.17", + "maxGLIBCXX": "3.4.21", + "maxCXXABI": "1.3.9" + } + } + ] +} diff --git a/gitnexus/prebuilds/bundled/tree-sitter-cpp@0.23.4/linux-arm64/tree-sitter-cpp.node b/gitnexus/prebuilds/bundled/tree-sitter-cpp@0.23.4/linux-arm64/tree-sitter-cpp.node new file mode 100644 index 000000000..98a0e10a0 Binary files /dev/null and b/gitnexus/prebuilds/bundled/tree-sitter-cpp@0.23.4/linux-arm64/tree-sitter-cpp.node differ diff --git a/gitnexus/prebuilds/bundled/tree-sitter-java@0.23.5/linux-arm64/tree-sitter-java.node b/gitnexus/prebuilds/bundled/tree-sitter-java@0.23.5/linux-arm64/tree-sitter-java.node new file mode 100644 index 000000000..975ac427c Binary files /dev/null and b/gitnexus/prebuilds/bundled/tree-sitter-java@0.23.5/linux-arm64/tree-sitter-java.node differ diff --git a/gitnexus/prebuilds/bundled/tree-sitter-javascript@0.23.1/linux-arm64/tree-sitter-javascript.node b/gitnexus/prebuilds/bundled/tree-sitter-javascript@0.23.1/linux-arm64/tree-sitter-javascript.node new file mode 100644 index 000000000..18d3a3e68 Binary files /dev/null and b/gitnexus/prebuilds/bundled/tree-sitter-javascript@0.23.1/linux-arm64/tree-sitter-javascript.node differ diff --git a/gitnexus/prebuilds/bundled/tree-sitter-ruby@0.23.1/linux-arm64/tree-sitter-ruby.node b/gitnexus/prebuilds/bundled/tree-sitter-ruby@0.23.1/linux-arm64/tree-sitter-ruby.node new file mode 100644 index 000000000..6f27cf7cf Binary files /dev/null and b/gitnexus/prebuilds/bundled/tree-sitter-ruby@0.23.1/linux-arm64/tree-sitter-ruby.node differ diff --git a/gitnexus/prebuilds/bundled/tree-sitter-typescript@0.23.2/linux-arm64/tree-sitter-typescript.node b/gitnexus/prebuilds/bundled/tree-sitter-typescript@0.23.2/linux-arm64/tree-sitter-typescript.node new file mode 100644 index 000000000..57ab3f871 Binary files /dev/null and b/gitnexus/prebuilds/bundled/tree-sitter-typescript@0.23.2/linux-arm64/tree-sitter-typescript.node differ diff --git a/gitnexus/scripts/arm64-prebuild.cjs b/gitnexus/scripts/arm64-prebuild.cjs new file mode 100644 index 000000000..0db84ce1c --- /dev/null +++ b/gitnexus/scripts/arm64-prebuild.cjs @@ -0,0 +1,15 @@ +// A tuple directory and an N-API symbol alone also accept the broken upstream +// x86-64 binaries. Require an ELF64 little-endian AArch64 shared object. +function isArm64NapiSharedObject(bytes) { + return ( + bytes.length >= 64 && + bytes.subarray(0, 4).equals(Buffer.from([127, 69, 76, 70])) && + bytes[4] === 2 && + bytes[5] === 1 && + bytes.readUInt16LE(16) === 3 && + bytes.readUInt16LE(18) === 183 && + bytes.includes('napi_register_module_v1') + ); +} + +module.exports = { isArm64NapiSharedObject }; diff --git a/gitnexus/scripts/assert-native-parsers.mjs b/gitnexus/scripts/assert-native-parsers.mjs index af3d68081..765ab9e6b 100644 --- a/gitnexus/scripts/assert-native-parsers.mjs +++ b/gitnexus/scripts/assert-native-parsers.mjs @@ -36,7 +36,8 @@ const SAMPLES = { async function check(installedDir) { const load = (rel) => import(pathToFileURL(path.join(installedDir, rel)).href); - const Parser = createRequire(path.join(installedDir, 'package.json'))('tree-sitter'); + const require = createRequire(path.join(installedDir, 'package.json')); + const Parser = require('tree-sitter'); const { listGrammarSources, getLanguageGrammar } = await load( 'dist/core/tree-sitter/parser-loader.js', ); @@ -60,6 +61,26 @@ async function check(installedDir) { failures.push(`${key}: ${err instanceof Error ? err.message : String(err)}`); } } + // TypeScript's nested JavaScript 0.23.1 has its own install lifecycle and + // prebuild (#3599). The registry's direct JavaScript grammar is 0.25.0, so + // loading it above does not exercise this bundled dependency. + try { + const typescriptRequire = createRequire(require.resolve('tree-sitter-typescript')); + const { version } = typescriptRequire('tree-sitter-javascript/package.json'); + if (version !== '0.23.1') throw new Error(`expected nested JavaScript 0.23.1, got ${version}`); + const parser = new Parser(); + parser.setLanguage(typescriptRequire('tree-sitter-javascript')); + const root = parser.parse(SAMPLES.javascript).rootNode; + if (root.hasError || root.namedChildCount === 0) { + failures.push( + `typescript:nested-javascript: parsed with errors: ${root.toString().slice(0, 200)}`, + ); + } + } catch (err) { + failures.push( + `typescript:nested-javascript: ${err instanceof Error ? err.message : String(err)}`, + ); + } const { PROTO_GRPC_PLUGIN } = await load('dist/core/group/extractors/grpc-patterns/proto.js'); if (PROTO_GRPC_PLUGIN === null) { failures.push('proto: native Protobuf grammar failed to initialize'); @@ -71,7 +92,7 @@ async function check(installedDir) { ).map((d) => d.symbolName); if (found.join() !== 'S.M') failures.push(`proto: expected [S.M], got [${found}]`); } - return { checked: keys.length + 1, failures }; + return { checked: keys.length + 2, failures }; } if (!isMainThread) { diff --git a/gitnexus/scripts/prepare-tree-sitter-bundle.cjs b/gitnexus/scripts/prepare-tree-sitter-bundle.cjs index e5ba9354c..24919ab95 100644 --- a/gitnexus/scripts/prepare-tree-sitter-bundle.cjs +++ b/gitnexus/scripts/prepare-tree-sitter-bundle.cjs @@ -5,10 +5,13 @@ * A dependency's npm overrides do not apply in a consumer project. Several * ABI-compatible grammars still advertise older runtime peers, so bundling * alone leaves an invalid dependency tree. Admit only the runtime version we - * tested, in only the audited manifests below. Sources and binaries are intact. + * tested, in only the audited manifests below. Five audited upstream Linux + * ARM64 binaries are repaired before packing. */ const fs = require('node:fs'); const path = require('node:path'); +const { createHash } = require('node:crypto'); +const { isArm64NapiSharedObject } = require('./arm64-prebuild.cjs'); const RUNTIME = '0.25.1'; // Upstream metadata is an input to validate, never the peer range we ship. @@ -23,9 +26,51 @@ const AUDITED_PEERS = new Map([ ['tree-sitter-javascript@0.23.1', '^0.21.1'], ['tree-sitter-typescript@0.23.2', '^0.21.0'], ]); +// These npm releases put x86-64 bytes in their linux-arm64 prebuild directory. +// Keep the replacement set explicit: upstream upgrades require a new audit. +const ARM64_REPAIRS = new Set([ + 'tree-sitter-cpp@0.23.4', + 'tree-sitter-java@0.23.5', + 'tree-sitter-ruby@0.23.1', + 'tree-sitter-javascript@0.23.1', + 'tree-sitter-typescript@0.23.2', +]); const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8')); -function prepareTreeSitterBundle(packageRoot = path.resolve(__dirname, '..')) { +function readArm64Repairs(packageRoot) { + const root = path.join(packageRoot, 'prebuilds', 'bundled'); + const manifest = readJson(path.join(root, 'manifest.json')); + if ( + manifest.schemaVersion !== 1 || + manifest.tuple !== 'linux-arm64' || + !Array.isArray(manifest.packages) || + manifest.packages.length !== ARM64_REPAIRS.size + ) { + throw new Error('Invalid bundled ARM64 prebuild manifest'); + } + const repairs = new Map(); + for (const entry of manifest.packages) { + const identity = `${entry.name}@${entry.version}`; + const expectedFile = `${identity}/linux-arm64/${entry.name}.node`; + if (!ARM64_REPAIRS.has(identity) || repairs.has(identity) || entry.file !== expectedFile) { + throw new Error(`Unexpected bundled ARM64 prebuild identity: ${identity}`); + } + const bytes = fs.readFileSync(path.join(root, entry.file)); + if (createHash('sha256').update(bytes).digest('hex') !== entry.sha256) { + throw new Error(`Bundled ARM64 prebuild checksum mismatch: ${identity}`); + } + if (!isArm64NapiSharedObject(bytes)) { + throw new Error(`Bundled prebuild is not an AArch64 N-API shared object: ${identity}`); + } + repairs.set(identity, bytes); + } + return repairs; +} + +function prepareTreeSitterBundle( + packageRoot = path.resolve(__dirname, '..'), + { forPublication = false } = {}, +) { const modulesRoot = path.join(packageRoot, 'node_modules'); const manifest = readJson(path.join(packageRoot, 'package.json')); if (manifest.dependencies['tree-sitter'] !== RUNTIME) { @@ -102,16 +147,54 @@ function prepareTreeSitterBundle(packageRoot = path.resolve(__dirname, '..')) { throw new Error('Tree-sitter dependency layout changed; revalidate the audited peer manifests'); } + const repairs = readArm64Repairs(packageRoot); + const binaries = []; + const shadows = []; + for (const { pkg, file } of patches) { + const dir = path.dirname(file); + const bytes = repairs.get(`${pkg.name}@${pkg.version}`); + if (!bytes) continue; + const target = path.join(dir, 'prebuilds', 'linux-arm64', `${pkg.name}.node`); + // A changed upstream layout needs review rather than creating a second + // candidate that node-gyp-build may not select. + const candidates = fs + .readdirSync(path.dirname(target)) + .filter((name) => name.endsWith('.node')); + if (candidates.length !== 1 || candidates[0] !== path.basename(target)) { + throw new Error(`Unexpected ARM64 prebuild layout for ${pkg.name}@${pkg.version}`); + } + binaries.push({ target, bytes }); + if (forPublication) { + // node-gyp-build prefers local build outputs over prebuilds, even when + // those outputs were compiled on the publisher's other architecture. + // Preserve legitimate consumer source builds during ordinary postinstall. + for (const mode of ['Release', 'Debug']) { + const build = path.join(dir, 'build', mode); + if (fs.existsSync(build)) { + for (const name of fs.readdirSync(build)) { + if (name.endsWith('.node')) shadows.push(path.join(build, name)); + } + } + } + } + } + // Validate the complete set before writing, so dependency drift cannot leave // a partially prepared bundle behind. Repeated pack operations are idempotent. for (const { file, pkg } of patches) { fs.writeFileSync(file, JSON.stringify(pkg, null, 2) + '\n'); } + for (const { target, bytes } of binaries) { + if (!fs.readFileSync(target).equals(bytes)) fs.writeFileSync(target, bytes); + } + for (const file of shadows) fs.unlinkSync(file); return patches.map(({ pkg }) => `${pkg.name}@${pkg.version}`); } if (require.main === module) { - const prepared = prepareTreeSitterBundle(); + const prepared = prepareTreeSitterBundle(undefined, { + forPublication: process.argv.includes('--for-publication'), + }); console.log( `[tree-sitter-bundle] Validated runtime ${RUNTIME}; prepared ${prepared.length} peer manifests.`, ); diff --git a/gitnexus/scripts/rebuild-bundled-grammar-prebuilds.cjs b/gitnexus/scripts/rebuild-bundled-grammar-prebuilds.cjs new file mode 100644 index 000000000..e79da3c5f --- /dev/null +++ b/gitnexus/scripts/rebuild-bundled-grammar-prebuilds.cjs @@ -0,0 +1,212 @@ +#!/usr/bin/env node +/** Rebuild only the five audited npm-package Linux ARM64 overlays. */ +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { createHash } = require('node:crypto'); +const { createRequire } = require('node:module'); +const { execFileSync } = require('node:child_process'); +const { isArm64NapiSharedObject } = require('./arm64-prebuild.cjs'); + +const output = path.resolve(__dirname, '../prebuilds/bundled'); +const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); +const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8')); +const samples = { + 'tree-sitter-cpp': [[null, 'class Box { public: int value; };']], + 'tree-sitter-java': [[null, 'class Box { int value; }']], + 'tree-sitter-ruby': [[null, 'class Box\n def value\n 1\n end\nend']], + 'tree-sitter-javascript': [[null, 'export function value() { return 1; }']], + 'tree-sitter-typescript': [ + ['typescript', 'const value: number = 1;'], + ['tsx', 'const value =
hello
;'], + ], +}; + +function buildEnvironment(env = process.env) { + // Keep tool discovery and the user directory, but exclude compiler, make and + // gyp overrides so the recorded GCC toolchain and flags match the build. + return { PATH: env.PATH, HOME: env.HOME, CC: 'gcc', CXX: 'g++' }; +} + +async function download(url, file, expectedSha256, integrity) { + const response = await fetch(url); + if (!response.ok) throw new Error(`Download failed: ${url} (${response.status})`); + const bytes = Buffer.from(await response.arrayBuffer()); + if ( + sha256(bytes) !== expectedSha256 || + (integrity && `sha512-${createHash('sha512').update(bytes).digest('base64')}` !== integrity) + ) { + throw new Error(`Source checksum mismatch: ${url}`); + } + fs.writeFileSync(file, bytes); +} + +function verifySources(dir, files) { + for (const [file, expected] of Object.entries(files)) { + if (sha256(fs.readFileSync(path.join(dir, file))) !== expected) { + throw new Error(`Upstream source changed: ${dir}/${file}`); + } + } +} + +function inspectBinary(file) { + const bytes = fs.readFileSync(file); + if (!isArm64NapiSharedObject(bytes)) { + throw new Error(`Not an AArch64 N-API shared object: ${file}`); + } + const readelf = (...args) => execFileSync('readelf', args.concat(file), { encoding: 'utf8' }); + const versions = [ + ...new Set(readelf('--version-info').match(/(?:GLIBCXX|GLIBC|CXXABI|GCC)_[\d.]+/g)), + ].sort(); + const limits = { GLIBC: '2.17', GLIBCXX: '3.4.21', CXXABI: '1.3.9' }; + for (const [family, max] of Object.entries(limits)) { + for (const version of versions.filter((value) => value.startsWith(`${family}_`))) { + const parts = version + .slice(family.length + 1) + .split('.') + .map(Number); + const ceiling = max.split('.').map(Number); + const difference = parts + .map((part, i) => part - (ceiling[i] ?? 0)) + .find((part) => part !== 0); + if (difference > 0) throw new Error(`${file} requires ${version}, above ${family}_${max}`); + } + } + const stack = readelf('--program-headers', '--wide') + .split('\n') + .find((line) => line.includes('GNU_STACK')); + if (!stack || !/\sRW\s/.test(stack)) throw new Error(`Missing non-executable GNU_STACK: ${file}`); + return { + architecture: 'AArch64', + gnuStack: 'RW', + needed: [...readelf('--dynamic').matchAll(/\(NEEDED\).*\[([^\]]+)\]/g)].map( + (match) => match[1], + ), + requiredSymbolVersions: versions, + maxGLIBC: limits.GLIBC, + maxGLIBCXX: limits.GLIBCXX, + maxCXXABI: limits.CXXABI, + }; +} + +async function main() { + if (process.argv.includes('--help')) { + console.log( + 'Run on Linux ARM64 with Node, npm, Python, make, GCC/G++, tar and readelf.\nRebuilds the five pinned upstream packages into prebuilds/bundled; see its README.md.', + ); + return; + } + if (process.platform !== 'linux' || process.arch !== 'arm64') { + throw new Error( + 'Regeneration requires an actual Linux ARM64 Node process; npm_config_arch is insufficient.', + ); + } + const manifest = readJson(path.join(output, 'manifest.json')); + const work = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-bundled-prebuilds-')); + const buildEnv = buildEnvironment(); + const run = (command, args, cwd = work, env = process.env) => + execFileSync(command, args, { cwd, env, stdio: 'inherit' }); + try { + const dependencies = Object.fromEntries( + Object.entries(manifest.build.dependencies).map(([name, entry]) => [name, entry.version]), + ); + dependencies['tree-sitter'] = manifest.build.runtime; + fs.writeFileSync( + path.join(work, 'package.json'), + JSON.stringify({ private: true, dependencies }), + ); + run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--legacy-peer-deps']); + const lock = readJson(path.join(work, 'package-lock.json')); + for (const [name, entry] of Object.entries(manifest.build.dependencies)) { + const installed = lock.packages[`node_modules/${name}`]; + if (installed.version !== entry.version || installed.integrity !== entry.integrity) { + throw new Error(`Build dependency differs from recorded input: ${name}`); + } + } + const header = manifest.build.nodeHeaders; + const headerArchive = path.join(work, 'node-headers.tar.gz'); + await download(header.url, headerArchive, header.sha256); + run('tar', ['-xzf', headerArchive, '-C', work]); + const headers = path.join(work, `node-v${header.version}`); + const Parser = createRequire(path.join(work, 'package.json'))('tree-sitter'); + const built = []; + // Build serially: node-gyp writes a shared node-addon-api Makefile whose + // relative target paths depend on the grammar's position in node_modules. + for (const entry of manifest.packages) { + const archive = path.join(work, `${entry.name}.tgz`); + await download(entry.source.url, archive, entry.source.sha256, entry.source.integrity); + const dir = path.join(work, 'node_modules', entry.name); + fs.mkdirSync(dir, { recursive: true }); + run('tar', ['-xzf', archive, '-C', dir, '--strip-components=1']); + const pkg = readJson(path.join(dir, 'package.json')); + if (pkg.name !== entry.name || pkg.version !== entry.version) { + throw new Error(`Unexpected upstream package identity: ${entry.name}`); + } + verifySources(dir, entry.source.files); + run( + process.execPath, + [ + path.join(work, 'node_modules/node-gyp/bin/node-gyp.js'), + 'rebuild', + '--arch=arm64', + `--nodedir=${headers}`, + '--jobs=2', + ], + dir, + buildEnv, + ); + verifySources(dir, entry.source.files); + const release = path.join(dir, 'build/Release'); + const files = fs.readdirSync(release).filter((name) => name.endsWith('.node')); + if (files.length !== 1) throw new Error(`Expected one native binding: ${entry.name}`); + const binary = path.join(release, files[0]); + const elf = inspectBinary(binary); + const language = require(binary); + for (const [variant, source] of samples[entry.name]) { + const parser = new Parser(); + parser.setLanguage(variant ? language[variant] : language); + const root = parser.parse(source).rootNode; + if (root.hasError || root.namedChildCount === 0) + throw new Error(`Parse failed: ${entry.name} ${variant ?? ''}`); + } + const bytes = fs.readFileSync(binary); + built.push({ entry: { ...entry, sha256: sha256(bytes), elf }, bytes }); + } + const version = (command) => + execFileSync(command, ['--version'], { env: buildEnv, encoding: 'utf8' }).split('\n')[0]; + manifest.build = { + ...manifest.build, + compiler: `${version(buildEnv.CC)}; ${version(buildEnv.CXX)}`, + linker: version('ld'), + sysroot: version('ldd'), + hostArchitecture: process.arch, + hostNode: process.versions.node, + compilerFlags: [], + validation: { + execution: 'Linux ARM64 Node process', + nodeVersions: [process.versions.node], + grammarExports: ['cpp', 'java', 'ruby', 'javascript', 'typescript', 'tsx'], + parseFailures: 0, + }, + }; + manifest.packages = built.map(({ entry }) => entry); + // No committed output changes until every package passes provenance, + // ABI-baseline, native loading and parsing checks. + for (const { entry, bytes } of built) fs.writeFileSync(path.join(output, entry.file), bytes); + fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n'); + console.log( + `Rebuilt and parsed ${built.length} audited ARM64 bindings; review binary and manifest changes.`, + ); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +} + +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} + +module.exports = { buildEnvironment }; diff --git a/gitnexus/test/unit/build-web-optin.test.ts b/gitnexus/test/unit/build-web-optin.test.ts index efb07190e..4d205311e 100644 --- a/gitnexus/test/unit/build-web-optin.test.ts +++ b/gitnexus/test/unit/build-web-optin.test.ts @@ -27,6 +27,7 @@ interface WorkflowStep { interface WorkflowJob { 'timeout-minutes'?: number; + strategy?: { matrix?: { os?: string[] } }; steps?: WorkflowStep[]; } @@ -280,6 +281,101 @@ describe('workflows that need the web UI install it themselves', () => { expect(setup?.with?.['lifecycle-scripts']).toBe('false'); expect(setup?.with?.build).toBeUndefined(); }); + + it('exercises the reported native ARM64 runtime alongside Windows and Linux x64', () => { + const job = ciJobs['packaged-install-smoke']; + expect(job?.strategy?.matrix?.os).toEqual([ + 'windows-latest', + 'ubuntu-latest', + 'ubuntu-24.04-arm', + ]); + const steps = job?.steps ?? []; + const setupIdx = stepIndex(steps, (step) => step.uses === './.github/actions/setup-gitnexus'); + const armNodeIdx = stepIndex(steps, (step) => step.with?.['node-version'] === '22.23.1'); + expect(armNodeIdx).toBeGreaterThan(setupIdx); + expect(steps[armNodeIdx]?.if).toContain("matrix.os == 'ubuntu-24.04-arm'"); + const assertArch = steps.find((step) => step.name === 'Assert native ARM64 runtime'); + expect(assertArch?.if).toContain("matrix.os == 'ubuntu-24.04-arm'"); + expect(String(assertArch?.run)).toContain("process.arch, 'arm64'"); + expect(String(assertArch?.run)).toContain("process.version, 'v22.23.1'"); + }); + + it('keeps consumer compilation guards after packing and checks both npm lockfile paths', () => { + const steps = ciJobs['packaged-install-smoke']?.steps ?? []; + const packIdx = stepIndex(steps, (step) => step.name === 'Pack gitnexus tarball'); + const guardIdx = stepIndex(steps, (step) => step.name === 'Reject ARM64 consumer compilation'); + const localIdx = stepIndex( + steps, + (step) => step.name === 'Lock and install tarball in a fresh consumer project', + ); + const globalIdx = stepIndex( + steps, + (step) => step.name === 'Install gitnexus tarball into isolated prefix', + ); + const checkIdx = stepIndex( + steps, + (step) => step.name === 'Assert no consumer compilation attempts', + ); + expect(guardIdx).toBeGreaterThan(packIdx); + expect(localIdx).toBeGreaterThan(guardIdx); + expect(globalIdx).toBeGreaterThan(guardIdx); + expect(checkIdx).toBeGreaterThan(globalIdx); + expect(steps[guardIdx]?.if).toContain("matrix.os == 'ubuntu-24.04-arm'"); + expect(steps[checkIdx]?.if).toContain('always()'); + expect(String(steps[localIdx]?.run)).toContain('for NPM_VERSION in 10.9.4 11.9.0'); + expect(String(steps[localIdx]?.run)).toContain('cmp package-lock.json expected-lock.json'); + }); + + it.skipIf(process.platform === 'win32')('records a rejected compiler invocation', () => { + const steps = ciJobs['packaged-install-smoke']?.steps ?? []; + const guard = steps.find((step) => step.name === 'Reject ARM64 consumer compilation'); + const check = steps.find((step) => step.name === 'Assert no consumer compilation attempts'); + expect(guard?.run).toBeTypeOf('string'); + expect(check?.run).toBeTypeOf('string'); + const dir = mkdtempSync(path.join(os.tmpdir(), 'gitnexus-compile-guard-')); + tempDirs.push(dir); + const githubPath = path.join(dir, 'github-path'); + const githubEnv = path.join(dir, 'github-env'); + const setup = spawnSync('bash', ['-eu', '-o', 'pipefail', '-c', String(guard?.run)], { + env: { ...process.env, RUNNER_TEMP: dir, GITHUB_PATH: githubPath, GITHUB_ENV: githubEnv }, + encoding: 'utf8', + }); + expect(setup.status, setup.stderr).toBe(0); + const guardEnv = Object.fromEntries( + readFileSync(githubEnv, 'utf8') + .trim() + .split('\n') + .map((line) => { + const separator = line.indexOf('='); + return [line.slice(0, separator), line.slice(separator + 1)]; + }), + ); + const env = { + ...process.env, + ...guardEnv, + PATH: `${readFileSync(githubPath, 'utf8').trim()}${path.delimiter}${process.env.PATH}`, + }; + const clean = spawnSync('bash', ['-eu', '-o', 'pipefail', '-c', String(check?.run)], { + env, + encoding: 'utf8', + }); + expect(clean.status, clean.stderr).toBe(0); + writeFileSync(path.join(dir, 'probe.c'), 'int main(void) { return 0; }\n'); + const compile = spawnSync('cc', ['-c', 'probe.c', '-o', 'probe.o'], { + cwd: dir, + env, + encoding: 'utf8', + }); + expect(compile.status).toBe(97); + expect(compile.stderr).toContain('Unexpected native compilation'); + expect(existsSync(path.join(dir, 'probe.o'))).toBe(false); + const attempted = spawnSync('bash', ['-eu', '-o', 'pipefail', '-c', String(check?.run)], { + env, + encoding: 'utf8', + }); + expect(attempted.status).toBe(1); + expect(attempted.stderr).toContain('Unexpected native compilation'); + }); }); describe('setup-gitnexus job budget', () => { diff --git a/gitnexus/test/unit/prebuild-coverage.test.ts b/gitnexus/test/unit/prebuild-coverage.test.ts index 2c9442f5d..94d46e43d 100644 --- a/gitnexus/test/unit/prebuild-coverage.test.ts +++ b/gitnexus/test/unit/prebuild-coverage.test.ts @@ -14,6 +14,8 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { load } from 'js-yaml'; +import { createHash } from 'node:crypto'; +import { classifyBinaryHeader } from '../../src/core/lbug/extension-load-error.js'; /** * Regression guard: every tree-sitter grammar GitNexus ships must provide a @@ -381,3 +383,51 @@ describe('prebuild workflow rebuild loop guard', () => { }, ); }); + +describe('bundled Linux ARM64 repair architecture', () => { + const repairs = [ + ['tree-sitter-cpp', '0.23.4'], + ['tree-sitter-java', '0.23.5'], + ['tree-sitter-ruby', '0.23.1'], + ['tree-sitter-javascript', '0.23.1'], + ['tree-sitter-typescript', '0.23.2'], + ]; + for (const [name, version] of repairs) { + it(`${name}@${version}: installed ARM64 prebuild is an AArch64 N-API binary`, () => { + const dir = + name === 'tree-sitter-javascript' + ? path.join(NODE_MODULES, 'tree-sitter-typescript/node_modules', name) + : path.join(NODE_MODULES, name); + const bytes = readFileSync(path.join(dir, 'prebuilds/linux-arm64', `${name}.node`)); + expect(classifyBinaryHeader(bytes, bytes.length, 'linux', 'arm64')).toBe('valid'); + expect(classifyBinaryHeader(bytes, bytes.length, 'linux', 'x64')).toBe('corrupt'); + expect(bytes.includes(NAPI_SYMBOL)).toBe(true); + }); + } + + it('pins the five repair identities, pristine source provenance, and replacement checksums', () => { + const dir = path.join(GITNEXUS_ROOT, 'prebuilds/bundled'); + const manifest = JSON.parse(readFileSync(path.join(dir, 'manifest.json'), 'utf8')); + // Independent audit pin: review regenerated artifacts and their provenance + // before updating this hash. Parse/stringify ignores formatting-only changes. + expect(createHash('sha256').update(JSON.stringify(manifest)).digest('hex')).toBe( + '2aa9eda6086233ce620564a63f9ed422c424b0d89b10e58b0e906e091db2aa3f', + ); + expect(manifest.schemaVersion).toBe(1); + expect(manifest.tuple).toBe('linux-arm64'); + expect( + manifest.packages.map((entry: { name: string; version: string }) => [ + entry.name, + entry.version, + ]), + ).toEqual(repairs); + for (const entry of manifest.packages) { + const bytes = readFileSync(path.join(dir, entry.file)); + expect(createHash('sha256').update(bytes).digest('hex')).toBe(entry.sha256); + expect(classifyBinaryHeader(bytes, bytes.length, 'linux', 'arm64')).toBe('valid'); + expect(entry.source.integrity).toMatch(/^sha512-/); + expect(entry.source.sha256).toMatch(/^[0-9a-f]{64}$/); + expect(entry.source.generatedSourcesUnchanged).toBe(true); + } + }); +}); diff --git a/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts b/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts index d2e9c8298..89c487a5e 100644 --- a/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts +++ b/gitnexus/test/unit/prepare-tree-sitter-bundle.test.ts @@ -1,6 +1,8 @@ import { afterEach, describe, expect, it } from 'vitest'; import { createRequire } from 'node:module'; +import { createHash } from 'node:crypto'; import { + existsSync, mkdtempSync, mkdirSync, readFileSync, @@ -15,7 +17,35 @@ import { cleanupTempDirSync } from '../helpers/test-db.js'; const require = createRequire(import.meta.url); const { prepareTreeSitterBundle } = require('../../scripts/prepare-tree-sitter-bundle.cjs'); +const { TUPLES: tuples } = require('../../scripts/assert-publish-grammar-coverage.cjs'); const roots: string[] = []; +const repairs = [ + ['tree-sitter-cpp', '0.23.4'], + ['tree-sitter-java', '0.23.5'], + ['tree-sitter-ruby', '0.23.1'], + ['tree-sitter-javascript', '0.23.1'], + ['tree-sitter-typescript', '0.23.2'], +]; +const sha256 = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex'); +const repairRoot = (root: string) => path.join(root, 'prebuilds/bundled'); +const grammarRoot = (root: string, name: string) => + path.join( + root, + 'node_modules', + name === 'tree-sitter-javascript' + ? 'tree-sitter-typescript/node_modules/tree-sitter-javascript' + : name, + ); +const replacement = () => { + const bytes = Buffer.alloc(96); + bytes.write('\x7fELF'); + bytes[4] = 2; + bytes[5] = 1; + bytes.writeUInt16LE(3, 16); + bytes.writeUInt16LE(183, 18); + bytes.write('napi_register_module_v1', 64); + return bytes; +}; const directVersions: Record = { '@eagleoutice/tree-sitter-r': '1.1.2', @@ -76,6 +106,23 @@ function fixture() { peerDependencies: { 'tree-sitter': '^0.21.1' }, }, ); + const overlays = repairs.map(([name, version]) => { + const file = `${name}@${version}/linux-arm64/${name}.node`; + const bytes = replacement(); + mkdirSync(path.dirname(path.join(repairRoot(root), file)), { recursive: true }); + writeFileSync(path.join(repairRoot(root), file), bytes); + for (const tuple of tuples) { + const dir = path.join(grammarRoot(root, name), 'prebuilds', tuple); + mkdirSync(dir, { recursive: true }); + writeFileSync(path.join(dir, `${name}.node`), `original ${name} ${tuple}`); + } + return { name, version, file, sha256: sha256(bytes) }; + }); + write(path.join(repairRoot(root), 'manifest.json'), { + schemaVersion: 1, + tuple: 'linux-arm64', + packages: overlays, + }); return root; } @@ -117,7 +164,7 @@ afterEach(() => { }); describe('native parser bundle preparation', () => { - it('pins direct and transitive peers to the tested runtime without changing binaries', () => { + it('pins audited peers without changing unrelated binaries', () => { const root = fixture(); const native = path.join(root, 'node_modules/tree-sitter-cpp/parser.node'); writeFileSync(native, Buffer.from([0, 1, 128, 255])); @@ -271,3 +318,85 @@ describe('native parser bundle preparation', () => { expect(() => prepareTreeSitterBundle(root)).toThrow('installed version differs'); }); }); + +describe('audited Linux ARM64 bundle repairs', () => { + it('repairs all five identities and preserves every other tuple and the direct JavaScript dependency', () => { + const root = fixture(); + const direct = path.join( + root, + 'node_modules/tree-sitter-javascript/prebuilds/linux-arm64/tree-sitter-javascript.node', + ); + mkdirSync(path.dirname(direct), { recursive: true }); + writeFileSync(direct, 'direct JavaScript 0.25.0'); + prepareTreeSitterBundle(root); + for (const [name] of repairs) { + for (const tuple of tuples) { + const bytes = readFileSync( + path.join(grammarRoot(root, name), 'prebuilds', tuple, `${name}.node`), + ); + expect(bytes).toEqual( + tuple === 'linux-arm64' ? replacement() : Buffer.from(`original ${name} ${tuple}`), + ); + } + } + expect(readFileSync(direct, 'utf8')).toBe('direct JavaScript 0.25.0'); + expect(prepareTreeSitterBundle(root)).toHaveLength(8); + }); + + it.each(['missing', 'corrupt', 'architecture', 'version'])( + 'rejects %s repair input before any manifest or binary writes', + (failure) => { + const root = fixture(); + const manifestFile = path.join(repairRoot(root), 'manifest.json'); + const manifest = read(manifestFile); + const last = manifest.packages.at(-1); + const file = path.join(repairRoot(root), last.file); + if (failure === 'missing') rmSync(file); + if (failure === 'corrupt') writeFileSync(file, 'corrupt binary'); + if (failure === 'architecture') { + const bytes = replacement(); + bytes.writeUInt16LE(62, 18); + writeFileSync(file, bytes); + last.sha256 = sha256(bytes); + } + if (failure === 'version') last.version = '99.0.0'; + write(manifestFile, manifest); + const peer = path.join(root, 'node_modules/tree-sitter-c/package.json'); + const before = readFileSync(peer); + expect(() => prepareTreeSitterBundle(root)).toThrow(); + expect(readFileSync(peer)).toEqual(before); + for (const [name] of repairs) { + expect( + readFileSync( + path.join(grammarRoot(root, name), 'prebuilds/linux-arm64', `${name}.node`), + 'utf8', + ), + ).toBe(`original ${name} linux-arm64`); + } + }, + ); + + it('repairs nested JavaScript through linked dependency containers', () => { + const { root, javascript } = pnpmFixture(); + prepareTreeSitterBundle(root); + expect( + readFileSync(path.join(javascript, 'prebuilds/linux-arm64/tree-sitter-javascript.node')), + ).toEqual(replacement()); + }); + + it('removes shadowing local builds only for publication, preserving consumer source-build fallbacks', () => { + const root = fixture(); + const shadows = repairs.flatMap(([name]) => + ['Release', 'Debug'].map((mode) => { + const file = path.join(grammarRoot(root, name), 'build', mode, 'binding.node'); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, 'source-built fallback'); + return file; + }), + ); + prepareTreeSitterBundle(root); + expect(shadows.every(existsSync)).toBe(true); + prepareTreeSitterBundle(root, { forPublication: true }); + expect(shadows.some(existsSync)).toBe(false); + }); +}); diff --git a/gitnexus/test/unit/rebuild-bundled-grammar-prebuilds.test.ts b/gitnexus/test/unit/rebuild-bundled-grammar-prebuilds.test.ts new file mode 100644 index 000000000..bbda307b9 --- /dev/null +++ b/gitnexus/test/unit/rebuild-bundled-grammar-prebuilds.test.ts @@ -0,0 +1,35 @@ +import { createRequire } from 'node:module'; +import { describe, expect, it } from 'vitest'; + +const require = createRequire(import.meta.url); +const { buildEnvironment } = require('../../scripts/rebuild-bundled-grammar-prebuilds.cjs') as { + buildEnvironment: (env: NodeJS.ProcessEnv) => NodeJS.ProcessEnv; +}; + +describe('bundled grammar rebuild environment', () => { + it('keeps compiler and flag overrides out of the recorded GCC build', () => { + const inherited = { + PATH: '/usr/bin:/bin', + HOME: '/home/builder', + CC: 'clang', + CXX: 'clang++', + CC_target: 'clang', + CXX_target: 'clang++', + LINK_target: 'clang++', + CFLAGS: '-O1', + CXXFLAGS: '-O1', + CPPFLAGS: '-DOVERRIDE', + LDFLAGS: '-fuse-ld=lld', + MAKEFLAGS: 'CC=clang', + GYP_DEFINES: 'clang=1', + }; + + expect(buildEnvironment(inherited)).toEqual({ + PATH: inherited.PATH, + HOME: inherited.HOME, + CC: 'gcc', + CXX: 'g++', + }); + expect(inherited.CC).toBe('clang'); + }); +});