mirror of
https://github.com/abhigyanpatwari/GitNexus.git
synced 2026-10-29 06:32:28 +00:00
Merge upstream/main into feat/r-language-support
This commit is contained in:
commit
3d207378ec
23 changed files with 1284 additions and 22 deletions
69
.github/workflows/ci-tests.yml
vendored
69
.github/workflows/ci-tests.yml
vendored
|
|
@ -380,12 +380,13 @@ jobs:
|
|||
# (#836) plus native grammar loading from the bundled runtime. Runs on windows-latest because that is the platform the fix
|
||||
# targets; the in-repo `npm ci` job above only exercises the dev-tree path
|
||||
# and skips the tarball reify step where the historical EPERM occurred.
|
||||
# Native Linux ARM64 also verifies the corrected bundled prebuilds (#3599).
|
||||
packaged-install-smoke:
|
||||
name: packaged install smoke (${{ matrix.os }})
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [windows-latest, ubuntu-latest]
|
||||
os: [windows-latest, ubuntu-latest, ubuntu-24.04-arm]
|
||||
runs-on: ${{ matrix.os }}
|
||||
# Windows pack + web install regularly exceeds 15 minutes when setup also
|
||||
# runs prepare/postinstall/build before prepack compiles the same tree again.
|
||||
|
|
@ -404,6 +405,18 @@ jobs:
|
|||
with:
|
||||
lifecycle-scripts: 'false'
|
||||
|
||||
# Reproduce #3599's runtime after the shared action selects Node 22.x.
|
||||
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
|
||||
if: matrix.os == 'ubuntu-24.04-arm'
|
||||
with:
|
||||
node-version: '22.23.1'
|
||||
package-manager-cache: false
|
||||
- name: Assert native ARM64 runtime
|
||||
if: matrix.os == 'ubuntu-24.04-arm'
|
||||
shell: bash
|
||||
run: |
|
||||
node -e "const assert = require('node:assert/strict'); assert.equal(process.platform, 'linux'); assert.equal(process.arch, 'arm64'); assert.equal(process.version, 'v22.23.1'); console.log(process.platform, process.arch, process.version)"
|
||||
|
||||
# `npm pack` runs prepack, which builds the web UI into gitnexus/web/
|
||||
# so the tarball matches what `npm publish` ships. Install those deps
|
||||
# here, in their own visible step, rather than letting build.js do it
|
||||
|
|
@ -418,6 +431,43 @@ jobs:
|
|||
run: npm pack
|
||||
working-directory: gitnexus
|
||||
|
||||
# Block native compilation only in the consumer phase. A correct bundle
|
||||
# must install using prebuilds even when the runner has a toolchain.
|
||||
- name: Reject ARM64 consumer compilation
|
||||
if: matrix.os == 'ubuntu-24.04-arm'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TOOLS="$RUNNER_TEMP/gitnexus-no-compile"
|
||||
mkdir -p "$TOOLS"
|
||||
export GITNEXUS_COMPILER_SENTINEL_LOG="$RUNNER_TEMP/gitnexus-compile-attempts.log"
|
||||
cat > "$TOOLS/cc" <<'EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "Unexpected native compilation: $0 $*" >&2
|
||||
printf '%s\n' "$0 $*" >> "${GITNEXUS_COMPILER_SENTINEL_LOG:?}"
|
||||
exit 97
|
||||
EOF
|
||||
chmod +x "$TOOLS/cc"
|
||||
for tool in c++ gcc g++ clang clang++ make gmake; do
|
||||
cp "$TOOLS/cc" "$TOOLS/$tool"
|
||||
done
|
||||
# Prove the guard fails before relying on it, then clear its probe.
|
||||
if "$TOOLS/cc" --version; then
|
||||
echo 'Compiler sentinel unexpectedly succeeded' >&2
|
||||
exit 1
|
||||
else
|
||||
test "$?" -eq 97
|
||||
fi
|
||||
test -s "$GITNEXUS_COMPILER_SENTINEL_LOG"
|
||||
: > "$GITNEXUS_COMPILER_SENTINEL_LOG"
|
||||
echo "$TOOLS" >> "$GITHUB_PATH"
|
||||
{
|
||||
echo "GITNEXUS_COMPILER_SENTINEL_LOG=$GITNEXUS_COMPILER_SENTINEL_LOG"
|
||||
echo "CC=$TOOLS/cc"
|
||||
echo "CXX=$TOOLS/c++"
|
||||
echo "MAKE=$TOOLS/make"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
# Global installs do not exercise npm's bundled-dependency lockfile path
|
||||
# (#3563). Keep npm 10 here: newer npm can mask unresolved override refs.
|
||||
- name: Lock and install tarball in a fresh consumer project
|
||||
|
|
@ -462,6 +512,23 @@ jobs:
|
|||
echo "PREFIX=$PREFIX" >> "$GITHUB_ENV"
|
||||
working-directory: gitnexus
|
||||
|
||||
# npm can ignore failed optional-dependency scripts, so also inspect the
|
||||
# sentinel log after both installs rather than trusting exit status alone.
|
||||
- name: Assert no consumer compilation attempts
|
||||
if: always() && matrix.os == 'ubuntu-24.04-arm'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GITNEXUS_COMPILER_SENTINEL_LOG:-}" ] || [ ! -f "$GITNEXUS_COMPILER_SENTINEL_LOG" ]; then
|
||||
echo 'Compiler sentinel was not initialized' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -s "$GITNEXUS_COMPILER_SENTINEL_LOG" ]; then
|
||||
echo 'Unexpected native compilation during consumer installation:' >&2
|
||||
cat "$GITNEXUS_COMPILER_SENTINEL_LOG" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Assert no junctions or vendor build artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
|
|||
6
.github/workflows/release-evaluation.yml
vendored
6
.github/workflows/release-evaluation.yml
vendored
|
|
@ -294,7 +294,9 @@ jobs:
|
|||
GITNEXUS_REQUIRE_BWRAP_CANARY: '1'
|
||||
GITNEXUS_REQUIRE_CLAUDE_CANARY: '1'
|
||||
CLAUDE_CANARY_BIN: ${{ runner.temp }}/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude
|
||||
run: uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py -q
|
||||
run: >-
|
||||
uv run --locked --extra dev python -m pytest tests/test_proposer_sandbox.py
|
||||
tests/test_workflow_bench_sessions.py::test_real_cli_background_completion_is_one_complete_measurement -q
|
||||
- name: Materialize both release graphs before paid sessions
|
||||
timeout-minutes: 120
|
||||
working-directory: eval
|
||||
|
|
@ -322,7 +324,7 @@ jobs:
|
|||
--tasks "$RUNNER_TEMP/release-evaluation/$runtime/tasks.yaml" \
|
||||
--gitnexus-root "$GITHUB_WORKSPACE/$runtime" \
|
||||
--arms baseline_nomcp baseline --runs 3 --include-expensive \
|
||||
--workers 3 --timeout 1800 --model "$MODEL" --effort "$EFFORT" \
|
||||
--workers 3 --timeout 3600 --model "$MODEL" --effort "$EFFORT" \
|
||||
--claude-bin "$RUNNER_TEMP/claude-canary/node_modules/@anthropic-ai/claude-code-linux-x64/claude" \
|
||||
--out "$RUNNER_TEMP/release-evaluation/$runtime/raw"
|
||||
done
|
||||
|
|
|
|||
|
|
@ -165,10 +165,14 @@ peers against their original ranges even when an override is configured. The
|
|||
lockfile therefore pins the same nine audited peers to exactly `0.25.1`.
|
||||
`gitnexus/scripts/prepare-tree-sitter-bundle.cjs` runs during `postinstall`
|
||||
and `prepack` to keep installed dependency manifests consistent with that lockfile
|
||||
and the published bundle. It leaves grammar sources and native binaries unchanged
|
||||
and rejects unexpected versions, peer ranges, or dependency layouts before writing
|
||||
any manifests. The upstream ranges remain in the script only as validation inputs;
|
||||
installed and published audited manifests require exactly `0.25.1`.
|
||||
and the published bundle. It leaves grammar sources unchanged and replaces the
|
||||
incorrect `linux-arm64` prebuilds in five exact npm releases with audited AArch64
|
||||
binaries. It validates package versions, peer ranges, dependency layouts, and
|
||||
replacement checksums and architecture before writing manifests or binaries.
|
||||
The upstream ranges remain in the script only as validation inputs; installed and
|
||||
published audited manifests require exactly `0.25.1`. See
|
||||
[`gitnexus/prebuilds/bundled/README.md`](gitnexus/prebuilds/bundled/README.md) for
|
||||
the affected releases, provenance, and regeneration instructions.
|
||||
|
||||
When upgrading these packages, update the exact pins, lockfile, and audited peer
|
||||
list together. If regenerating the lockfile restores upstream's older peer ranges,
|
||||
|
|
|
|||
|
|
@ -95,6 +95,22 @@ def test_release_comparison_is_scheduled_or_manual_and_gates_outcomes():
|
|||
assert "--stable-sha" in scripts and "--candidate-sha" in scripts
|
||||
|
||||
|
||||
def test_release_session_headroom_keeps_the_global_stop_reserve():
|
||||
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
||||
paid = next(step for step in steps if "workflow_bench.runner" in step.get("run", ""))
|
||||
script = paid["run"]
|
||||
assert "--timeout 3600" in script
|
||||
assert "for runtime in stable candidate; do" in script
|
||||
assert "stop_deadline_epoch }} - $(date +%s) - 5400" in script
|
||||
assert 'timeout --signal=TERM --kill-after=60 "${remaining}s"' in script
|
||||
assert paid["timeout-minutes"] == 1140
|
||||
canary = next(step for step in steps if "tests/test_proposer_sandbox.py" in step.get("run", ""))
|
||||
assert "test_real_cli_background_completion_is_one_complete_measurement" in canary["run"]
|
||||
assert canary["env"]["GITNEXUS_REQUIRE_CLAUDE_CANARY"] == "1"
|
||||
assert steps.index(canary) < steps.index(paid)
|
||||
assert not canary.get("continue-on-error", False)
|
||||
|
||||
|
||||
def test_both_runtimes_grade_tasks_against_one_pinned_task_dependency_checkout():
|
||||
steps = workflow("release-evaluation.yml")["jobs"]["evaluate"]["steps"]
|
||||
names = [step.get("name", step.get("uses", "")) for step in steps]
|
||||
|
|
|
|||
|
|
@ -1295,6 +1295,136 @@ def test_final_result_event_must_be_last(monkeypatch, tmp_path):
|
|||
assert "not the last event" in rec["error_detail"]["event_stream_error"]
|
||||
|
||||
|
||||
def background_completion_events():
|
||||
first = json.loads(report_variant(uuid="first", subtype="success", is_error=False))
|
||||
last = json.loads(report_variant(uuid="last", subtype="success", is_error=False, total_cost_usd=0.15))
|
||||
return [
|
||||
first,
|
||||
{"type": "system", "subtype": "task_notification", "session_id": "s"},
|
||||
{"type": "system", "subtype": "init", "session_id": "s"},
|
||||
{"type": "assistant", "message": {"content": [{"type": "text", "text": "Task finished"}]}},
|
||||
last,
|
||||
]
|
||||
|
||||
|
||||
def test_background_completion_sums_turn_usage_but_not_cumulative_cost(monkeypatch, tmp_path):
|
||||
events = background_completion_events()
|
||||
proc = replace(fake_cli_result("\n".join(map(json.dumps, events))), duration_s=12.5)
|
||||
monkeypatch.setattr(runner_sessions, "run_managed", lambda *a, **k: proc)
|
||||
rec = runner.run_claude("task", tmp_path, claude_bin="claude", timeout=30)
|
||||
|
||||
assert rec["ok"] is True, rec["error_detail"]
|
||||
assert rec["cost_usd"] == 0.15 # The second result includes the first turn's $0.10.
|
||||
assert rec["num_turns"] == 6
|
||||
assert [rec[field] for field in runner_sessions.USAGE_FIELDS] == [2, 4, 6, 8]
|
||||
assert rec["duration_s"] == 12.5 # Include background wait, not just the two 1s turns.
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda es: es[-1].update(session_id="different"),
|
||||
lambda es: es[-1].update(uuid="first"),
|
||||
lambda es: es[-1].update(total_cost_usd=0.05),
|
||||
lambda es: es[0].update(is_error=True),
|
||||
lambda es: es[0].update(subtype="error_max_turns"),
|
||||
lambda es: es[0]["usage"].pop("input_tokens"),
|
||||
lambda es: es[-1]["usage"].update(output_tokens=-1),
|
||||
lambda es: es[-1]["usage"].update(output_tokens=True),
|
||||
lambda es: es[-1].update(num_turns=-1),
|
||||
lambda es: es[-1].update(total_cost_usd=None),
|
||||
lambda es: es.pop(1), # No background completion to explain another turn.
|
||||
lambda es: es.pop(3), # A second result without a new assistant turn is a replay.
|
||||
lambda es: es[2].update(session_id="different"),
|
||||
lambda es: es.append({"type": "assistant", "message": {"content": []}}),
|
||||
],
|
||||
)
|
||||
def test_unverifiable_background_continuation_stays_excluded(monkeypatch, tmp_path, mutate):
|
||||
events = background_completion_events()
|
||||
mutate(events)
|
||||
monkeypatch.setattr(
|
||||
runner_sessions, "run_managed", lambda *a, **k: fake_cli_result("\n".join(map(json.dumps, events)))
|
||||
)
|
||||
rec = runner.run_claude("task", tmp_path, claude_bin="claude", timeout=30, transcript_output_dir=tmp_path)
|
||||
assert rec["ok"] is False
|
||||
assert rec["error_kind"] == "session-error"
|
||||
assert rec["cost_usd"] is None
|
||||
assert rec["transcript_missing"] is True
|
||||
|
||||
|
||||
def test_real_cli_background_completion_is_one_complete_measurement(monkeypatch, tmp_path):
|
||||
"""Only the provider is scripted; the pinned CLI performs a real background Bash task."""
|
||||
from workflow_bench.mock_provider import MockProvider, Reply
|
||||
|
||||
claude = os.environ.get("CLAUDE_CANARY_BIN")
|
||||
if not claude or not Path(claude).is_file():
|
||||
if os.environ.get("GITNEXUS_REQUIRE_CLAUDE_CANARY") == "1":
|
||||
pytest.fail("required pinned CLAUDE_CANARY_BIN is missing")
|
||||
pytest.skip("the containment job supplies the pinned CLAUDE_CANARY_BIN")
|
||||
home = tmp_path / "home"
|
||||
home.mkdir()
|
||||
# Release the real background command only after the first result. A fixed
|
||||
# sleep could finish before that result on a loaded runner and miss the bug.
|
||||
observe_line = runner_sessions.SessionProgress._observe_line
|
||||
|
||||
def release_background_task(self, line):
|
||||
observe_line(self, line)
|
||||
if json.loads(line).get("type") == "result":
|
||||
(home / "completion-ready").touch()
|
||||
|
||||
monkeypatch.setattr(runner_sessions.SessionProgress, "_observe_line", release_background_task)
|
||||
replies = [
|
||||
Reply(
|
||||
tools=[
|
||||
{
|
||||
"name": "Bash",
|
||||
"input": {
|
||||
"command": "while [ ! -f completion-ready ]; do sleep 0.01; done; printf done",
|
||||
"run_in_background": True,
|
||||
},
|
||||
}
|
||||
],
|
||||
input_tokens=101,
|
||||
output_tokens=11,
|
||||
),
|
||||
Reply(text="First completion", input_tokens=102, output_tokens=12),
|
||||
Reply(text="Background completion acknowledged", input_tokens=103, output_tokens=13),
|
||||
]
|
||||
with MockProvider(replies) as provider:
|
||||
rec = runner.run_claude(
|
||||
"Run the command",
|
||||
home,
|
||||
claude_bin=claude,
|
||||
timeout=40,
|
||||
env={
|
||||
"PATH": os.environ["PATH"],
|
||||
"HOME": str(home),
|
||||
"ANTHROPIC_API_KEY": "offline-probe",
|
||||
"ANTHROPIC_BASE_URL": provider.base_url,
|
||||
"CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC": "1",
|
||||
"DISABLE_AUTOUPDATER": "1",
|
||||
# Only loopback may be contacted; even CLI startup traffic
|
||||
# cannot reach an external provider. No host credentials inherit.
|
||||
"HTTPS_PROXY": "http://127.0.0.1:9",
|
||||
"HTTP_PROXY": "http://127.0.0.1:9",
|
||||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
},
|
||||
allowed_tools=["Bash"],
|
||||
strict_mcp_config=True,
|
||||
transcript_output_dir=tmp_path,
|
||||
progress_label="offline-background-completion",
|
||||
)
|
||||
assert rec["ok"] is True, rec["error_detail"]
|
||||
assert len(provider.requests) == 3
|
||||
assert rec["input_tokens"] == 306 and rec["output_tokens"] == 36
|
||||
assert rec["num_turns"] == 3
|
||||
artifacts = list((tmp_path / "transcripts").glob("*.jsonl"))
|
||||
assert len(artifacts) == 1
|
||||
results = [e for line in artifacts[0].read_text().splitlines() if (e := json.loads(line))["type"] == "result"]
|
||||
assert len(results) == 2
|
||||
assert rec["cost_usd"] == results[-1]["total_cost_usd"] > results[0]["total_cost_usd"]
|
||||
|
||||
|
||||
def test_background_task_teardown_after_the_result_stays_valid_evidence(monkeypatch, tmp_path):
|
||||
# Claude Code drains background-task bookkeeping after the final result
|
||||
# event. Those `system` events carry no tool or usage payload, so they must
|
||||
|
|
|
|||
|
|
@ -244,12 +244,26 @@ Every evaluated repo-local skill root is over-mounted read-only for the full
|
|||
model session, and an immutable empty user-level skills directory prevents a
|
||||
writable `$HOME` skill from shadowing it. Skill-use evidence comes only from
|
||||
the bounded stream captured directly from Claude stdout by the parent. The
|
||||
runner parses every event through EOF, requires one final result, correlates
|
||||
runner parses every event through EOF and requires a terminal result. A
|
||||
background-task completion can start another foreground turn in the pinned
|
||||
Claude CLI: those results must have distinct IDs in the same session, complete
|
||||
per-turn usage, successful outcomes, and nondecreasing cumulative cost. The
|
||||
runner sums their tokens and turn counts, takes the last cumulative cost, and
|
||||
measures the whole process duration including the background wait. Missing,
|
||||
replayed, mismatched, or unsuccessful continuations remain excluded. It correlates
|
||||
an exact Skill request ID with one later successful result, structurally
|
||||
redacts the event objects, and stores the canonical redacted JSONL with a
|
||||
digest. Files written beneath the agent's `$HOME` are never trusted as
|
||||
evidence.
|
||||
|
||||
Release evaluation allows 3600 seconds per session in both runtimes and arms.
|
||||
Run `38873262106` hit the former 1800-second limit twice while still editing
|
||||
and testing; the longer ceiling gives those sessions headroom, not a guarantee
|
||||
of completion. The overall deadline and 5400-second evidence/cleanup reserve
|
||||
remain enforced. Treat this ceiling and the result-accounting change as harness
|
||||
differences when comparing against older runs; invalid historical measurements
|
||||
are not repaired retroactively.
|
||||
|
||||
Sessions use a private home without stored Claude login/keychain state, and
|
||||
do not inherit `ANTHROPIC_AUTH_TOKEN`. Supply an Anthropic API key
|
||||
through `GITNEXUS_BENCH_ANTHROPIC_API_KEY` (preferred) or `--anthropic-api-key`;
|
||||
|
|
|
|||
|
|
@ -619,6 +619,86 @@ def skill_was_invoked_events(events: Sequence[dict[str, Any]], skill_name: str)
|
|||
return successful
|
||||
|
||||
|
||||
def _session_result(events: list[dict[str, Any]], duration_s: float) -> tuple[dict[str, Any], int]:
|
||||
"""Reduce proven background continuations, retaining one session's full accounting.
|
||||
|
||||
Claude Code 2.1.214 emits a result per foreground turn, including a turn
|
||||
started by a background task notification. Usage/turns are per-turn; cost
|
||||
is cumulative. The real-CLI canary pins these different scopes.
|
||||
"""
|
||||
indexes = [index for index, event in enumerate(events) if event.get("type") == "result"]
|
||||
if not indexes:
|
||||
raise ValueError("expected a final result event, observed 0")
|
||||
final_index = indexes[-1]
|
||||
if any(event.get("type") != "system" for event in events[final_index + 1 :]):
|
||||
raise ValueError("final result event is not the last event in the captured stream")
|
||||
final = events[final_index]
|
||||
if len(indexes) == 1:
|
||||
return final, final_index
|
||||
|
||||
session_id = final.get("session_id")
|
||||
if not isinstance(session_id, str) or not session_id:
|
||||
raise ValueError("background continuation has no session identity")
|
||||
total_usage = dict.fromkeys(USAGE_FIELDS, 0)
|
||||
turns = 0
|
||||
cost = 0.0
|
||||
result_ids: set[str] = set()
|
||||
previous_index = -1
|
||||
for index in indexes:
|
||||
result = events[index]
|
||||
uuid = result.get("uuid")
|
||||
if result.get("session_id") != session_id:
|
||||
raise ValueError("background continuation changed session identity")
|
||||
if not isinstance(uuid, str) or not uuid or uuid in result_ids:
|
||||
raise ValueError("background continuation has a missing or repeated result id")
|
||||
result_ids.add(uuid)
|
||||
if result.get("subtype") != "success" or result.get("is_error") is not False:
|
||||
raise ValueError("background continuation contains an unsuccessful result")
|
||||
usage = result.get("usage")
|
||||
if not isinstance(usage, dict) or any(
|
||||
type(usage.get(field)) is not int or usage[field] < 0 for field in USAGE_FIELDS
|
||||
):
|
||||
raise ValueError("background continuation has incomplete or invalid usage")
|
||||
count = result.get("num_turns")
|
||||
if type(count) is not int or count < 1:
|
||||
raise ValueError("background continuation has invalid turn count")
|
||||
next_cost = measured_cost(result.get("total_cost_usd"))
|
||||
if next_cost is None or next_cost < cost:
|
||||
raise ValueError("background continuation has missing or decreasing cumulative cost")
|
||||
segment = events[previous_index + 1 : index]
|
||||
if any(event.get("session_id", session_id) != session_id for event in segment):
|
||||
raise ValueError("background continuation changed session identity")
|
||||
if previous_index >= 0:
|
||||
# Reject duplicate/replayed results and unrelated continuations.
|
||||
# A task notification must precede a fresh foreground init and reply.
|
||||
notification = next(
|
||||
(
|
||||
i
|
||||
for i, e in enumerate(segment)
|
||||
if e.get("type") == "system" and e.get("subtype") == "task_notification"
|
||||
),
|
||||
None,
|
||||
)
|
||||
init = next(
|
||||
(i for i, e in enumerate(segment) if e.get("type") == "system" and e.get("subtype") == "init"), None
|
||||
)
|
||||
if (
|
||||
notification is None
|
||||
or init is None
|
||||
or notification >= init
|
||||
or not any(e.get("type") == "assistant" for e in segment[init + 1 :])
|
||||
):
|
||||
raise ValueError("additional result lacks a background completion turn")
|
||||
for field in USAGE_FIELDS:
|
||||
total_usage[field] += usage[field]
|
||||
turns += count
|
||||
cost = next_cost
|
||||
previous_index = index
|
||||
# Per-turn duration_ms excludes time waiting for background work. Use the
|
||||
# trusted parent's elapsed time for the whole process, including that wait.
|
||||
return {**final, "usage": total_usage, "num_turns": turns, "duration_ms": duration_s * 1000}, final_index
|
||||
|
||||
|
||||
def run_claude(
|
||||
prompt: str,
|
||||
cwd: Path,
|
||||
|
|
@ -718,19 +798,13 @@ def run_claude(
|
|||
if proc.stdout_capture_overflow:
|
||||
raise ValueError(f"parent-captured event stream exceeds {MAX_TRANSCRIPT_BYTES} bytes")
|
||||
events = _parse_parent_event_stream(proc.stdout_capture)
|
||||
result_indexes = [index for index, event in enumerate(events) if event.get("type") == "result"]
|
||||
if len(result_indexes) != 1:
|
||||
raise ValueError(f"expected exactly one final result event, observed {len(result_indexes)}")
|
||||
result_index = result_indexes[0]
|
||||
data = events[result_index]
|
||||
data, result_index = _session_result(events, proc.duration_s)
|
||||
# A session that used a background task drains its bookkeeping after
|
||||
# the final result event (`background_tasks_changed`, `task_updated`,
|
||||
# `task_notification` — all `type: "system"`), so the result is last
|
||||
# only among the events that carry evidence. `system` events hold no
|
||||
# tool_use/tool_result/usage payload and so cannot forge skill or cost
|
||||
# evidence; any other event after the result still fails closed.
|
||||
if any(event.get("type") != "system" for event in events[result_index + 1 :]):
|
||||
raise ValueError("final result event is not the last event in the captured stream")
|
||||
# Nothing after the result is evidence. Cut the window here so that is
|
||||
# a property of what the evidence readers below can see, rather than an
|
||||
# assumption that a `system` event never carries a tool_use block.
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@
|
|||
},
|
||||
"files": [
|
||||
"dist",
|
||||
"prebuilds/bundled",
|
||||
"hooks",
|
||||
"scripts",
|
||||
"skills",
|
||||
|
|
@ -63,7 +64,7 @@
|
|||
"postinstall": "node scripts/prepare-tree-sitter-bundle.cjs && node scripts/build-tree-sitter-grammars.cjs",
|
||||
"assert-publish-coverage": "node scripts/assert-publish-grammar-coverage.cjs",
|
||||
"prepare": "node scripts/build.js",
|
||||
"prepack": "node scripts/prepare-tree-sitter-bundle.cjs && node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web",
|
||||
"prepack": "node scripts/prepare-tree-sitter-bundle.cjs --for-publication && node scripts/assert-publish-grammar-coverage.cjs && node scripts/assert-publish-fts-coverage.cjs && node scripts/build.js --web && node scripts/assert-web-assets.mjs web",
|
||||
"version": "node scripts/sync-plugin-manifests.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
|
|
|
|||
58
gitnexus/prebuilds/bundled/README.md
Normal file
58
gitnexus/prebuilds/bundled/README.md
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
# Audited npm grammar prebuild repairs
|
||||
|
||||
These five exact npm releases ship x86-64 binaries in their `linux-arm64`
|
||||
prebuild directories: C++ 0.23.4, Java 0.23.5, Ruby 0.23.1, JavaScript 0.23.1
|
||||
(the TypeScript dependency), and TypeScript 0.23.2. The direct JavaScript 0.25.0
|
||||
package is unaffected. See GitNexus issue #3599.
|
||||
|
||||
`manifest.json` records each immutable npm tarball URL, SHA-512 integrity,
|
||||
SHA-256, upstream commit, unchanged source-file hashes, replacement SHA-256,
|
||||
and build inputs. All generated parser C and scanner sources remain upstream
|
||||
bytes. The TypeScript binary retains both TypeScript and TSX exports.
|
||||
|
||||
The committed replacements were cross-built using GCC/G++ 13.3.0, binutils
|
||||
2.42, node-gyp 11.5.0, node-addon-api 8.9.2 and Node 22.23.1 headers. Although
|
||||
the build sysroot contains glibc 2.39, `readelf` confirms the outputs require
|
||||
only GLIBC 2.17, GLIBCXX 3.4.21 and CXXABI 1.3.9, with a non-executable stack.
|
||||
All six language exports loaded and parsed using tree-sitter 0.25.1 under ARM64
|
||||
Node 22.18.0, 22.23.1 and 24.19.0 via QEMU 8.2.2 user emulation. This is emulated
|
||||
execution evidence; the packaged-install CI job supplies native ARM64 evidence.
|
||||
|
||||
## Regeneration
|
||||
|
||||
On Linux ARM64 with Node, npm, Python, make, GCC/G++, tar and readelf installed:
|
||||
|
||||
```sh
|
||||
cd gitnexus
|
||||
node scripts/rebuild-bundled-grammar-prebuilds.cjs
|
||||
node scripts/prepare-tree-sitter-bundle.cjs --for-publication
|
||||
npx vitest run test/unit/prepare-tree-sitter-bundle.test.ts test/unit/prebuild-coverage.test.ts test/unit/parser-loader-abi.test.ts
|
||||
```
|
||||
|
||||
The script downloads the recorded tarballs and headers, verifies their hashes,
|
||||
installs the pinned build helpers with lifecycle scripts disabled, and builds
|
||||
packages serially in a temporary tree. Builds use `gcc` and `g++` from `PATH`,
|
||||
preserving only `PATH` and `HOME` from the caller's environment. Compiler,
|
||||
make, and gyp overrides (including `CC`, `CXX`, and flags) are ignored so the
|
||||
recorded toolchain matches the build. It checks unchanged source bytes, ELF
|
||||
architecture, N-API, dynamic dependency baselines, and real native parsing
|
||||
before replacing any committed output. Compiler changes can change the binary
|
||||
hashes; review the updated binaries and provenance together. This is a bounded
|
||||
npm-package overlay rebuild, separate from the source-vendored grammar matrix.
|
||||
|
||||
## Packaging behavior
|
||||
|
||||
Bundle preparation validates every repair before writing peer manifests or
|
||||
binary files. `prepack` supplies `--for-publication` to remove local
|
||||
`build/Release/*.node` and `build/Debug/*.node` outputs from repaired packages,
|
||||
which would otherwise shadow their platform prebuilds. Normal consumer
|
||||
postinstall keeps legitimate source-built fallbacks intact. Repair assets are
|
||||
included in the npm package so postinstall remains self-contained.
|
||||
|
||||
Corrected bytes must exist before consumer child install scripts run. The
|
||||
upstream TypeScript and nested JavaScript builds share node-addon-api Makefile
|
||||
outputs; configuring them at different dependency depths can overwrite those
|
||||
outputs with conflicting relative stamp paths. Repairing this upstream build
|
||||
behavior for explicit `--build-from-source` or raw source-checkout installation
|
||||
is outside this fix. Ordinary supported Linux ARM64 glibc tarball installation
|
||||
uses the corrected prebuilds and does not require compilation.
|
||||
255
gitnexus/prebuilds/bundled/manifest.json
Normal file
255
gitnexus/prebuilds/bundled/manifest.json
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
{
|
||||
"schemaVersion": 1,
|
||||
"tuple": "linux-arm64",
|
||||
"build": {
|
||||
"compiler": "aarch64-linux-gnu-gcc/g++ 13.3.0 (Ubuntu 13.3.0-6ubuntu2~24.04.1)",
|
||||
"linker": "GNU ld 2.42 (Ubuntu 2.42-4ubuntu2.10)",
|
||||
"sysroot": "Ubuntu arm64 cross glibc 2.39-0ubuntu8cross1",
|
||||
"hostArchitecture": "x64",
|
||||
"hostNode": "24.19.0",
|
||||
"nodeHeaders": {
|
||||
"version": "22.23.1",
|
||||
"url": "https://nodejs.org/dist/v22.23.1/node-v22.23.1-headers.tar.gz",
|
||||
"sha256": "caf38d839815df92000b41b756056f575140f38f1d8c0cbef97448c33eb7d7b2"
|
||||
},
|
||||
"dependencies": {
|
||||
"node-addon-api": {
|
||||
"version": "8.9.2",
|
||||
"integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg=="
|
||||
},
|
||||
"node-gyp": {
|
||||
"version": "11.5.0",
|
||||
"integrity": "sha512-ra7Kvlhxn5V9Slyus0ygMa2h+UqExPqUIkfk7Pc8QTLT956JLSy51uWFwHtIYy0vI8cB4BDhc/S03+880My/LQ=="
|
||||
},
|
||||
"node-gyp-build": {
|
||||
"version": "4.8.4",
|
||||
"integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="
|
||||
}
|
||||
},
|
||||
"runtime": "0.25.1",
|
||||
"command": "node node_modules/node-gyp/bin/node-gyp.js rebuild --arch=arm64 --nodedir=<Node 22.23.1 headers> --jobs=2",
|
||||
"compilerFlags": ["--sysroot=<ARM64 glibc sysroot>"],
|
||||
"validation": {
|
||||
"execution": "qemu-aarch64 8.2.2 user emulation",
|
||||
"nodeVersions": ["22.18.0", "22.23.1", "24.19.0"],
|
||||
"grammarExports": ["cpp", "java", "ruby", "javascript", "typescript", "tsx"],
|
||||
"parseFailures": 0
|
||||
}
|
||||
},
|
||||
"packages": [
|
||||
{
|
||||
"name": "tree-sitter-cpp",
|
||||
"version": "0.23.4",
|
||||
"file": "tree-sitter-cpp@0.23.4/linux-arm64/tree-sitter-cpp.node",
|
||||
"sha256": "57c248be4c90f8e9af63613bf49a2724a5de4482d4d3c30a99374ede9978d729",
|
||||
"source": {
|
||||
"url": "https://registry.npmjs.org/tree-sitter-cpp/-/tree-sitter-cpp-0.23.4.tgz",
|
||||
"integrity": "sha512-qR5qUDyhZ5jJ6V8/umiBxokRbe89bCGmcq/dk94wI4kN86qfdV8k0GHIUEKaqWgcu42wKal5E97LKpLeVW8sKw==",
|
||||
"gitHead": "f41e1a044c8a84ea9fa8577fdd2eab92ec96de02",
|
||||
"sha256": "615d152f70e7a6bed8f15a93c3c0c264c40dc71c1b6a1445646fff9439e776bc",
|
||||
"files": {
|
||||
"binding.gyp": "e8c7ca4208ca882655c21a99b6ef0d153634eabb1cd901a45914a121b023555c",
|
||||
"bindings/node/binding.cc": "9c50c8f087231e837648bd8660f84d2d9ea745ed6b886d05e94aa84cea584fd1",
|
||||
"package.json": "bfde98d506b0847f2093336e5b4d7394dd311e78da69e22e5ecd73fba6db6066",
|
||||
"src/grammar.json": "c0a93751d708440bb2c1929d738a5b03c4b835f61e5aeb580d0af269faf68f8d",
|
||||
"src/node-types.json": "fdfd4b1f3dca1516616a1eb615bb6c1ad3082b8937ad21070d0def5dcfe7e535",
|
||||
"src/parser.c": "2a35a43b4af6c9f7b69624ac00c2c50808912591450dc79c05dea03ac1bae814",
|
||||
"src/scanner.c": "cf60387d290271f4d2fb558d0569b2b9ef879cb72206a9df5cc73b0b1a4b20ff",
|
||||
"src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea",
|
||||
"src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1",
|
||||
"src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a"
|
||||
},
|
||||
"generatedSourcesUnchanged": true
|
||||
},
|
||||
"elf": {
|
||||
"architecture": "AArch64",
|
||||
"gnuStack": "RW",
|
||||
"needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"],
|
||||
"requiredSymbolVersions": [
|
||||
"CXXABI_1.3",
|
||||
"CXXABI_1.3.9",
|
||||
"GCC_3.0",
|
||||
"GLIBCXX_3.4",
|
||||
"GLIBCXX_3.4.21",
|
||||
"GLIBC_2.17"
|
||||
],
|
||||
"maxGLIBC": "2.17",
|
||||
"maxGLIBCXX": "3.4.21",
|
||||
"maxCXXABI": "1.3.9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tree-sitter-java",
|
||||
"version": "0.23.5",
|
||||
"file": "tree-sitter-java@0.23.5/linux-arm64/tree-sitter-java.node",
|
||||
"sha256": "e91fa1c2dd4b7e13bebbd8c07c51d549d0b6da8b00d426a366ef1b515b159db8",
|
||||
"source": {
|
||||
"url": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.23.5.tgz",
|
||||
"integrity": "sha512-Yju7oQ0Xx7GcUT01mUglPP+bYfvqjNCGdxqigTnew9nLGoII42PNVP3bHrYeMxswiCRM0yubWmN5qk+zsg0zMA==",
|
||||
"gitHead": "94703d5a6bed02b98e438d7cad1136c01a60ba2c",
|
||||
"sha256": "6b6c1b62605f9d8bb80efd2ebe5951a87b182faafa3eeb67c460a667818551b6",
|
||||
"files": {
|
||||
"binding.gyp": "42305f7d7305fc3a6d2cee4ee5e0e73e0db708e967f351bfad04872f46c8a3ac",
|
||||
"bindings/node/binding.cc": "861ff9ef49c3e080dd00eebfd4d75b83f0ac64551a859caed23d4b416de88bf0",
|
||||
"package.json": "c0c99e633aa28b2c1c58e61f91211391a9363d7d532aa5795bf4003f2dc14dba",
|
||||
"src/grammar.json": "47ca521cd7a888d671d9f26301e8697fa686e04912032e959d242b5f25877eb6",
|
||||
"src/node-types.json": "19c46facc653381c337ff6cad75dd8b052524179a366c80825d6d0010520eef2",
|
||||
"src/parser.c": "4add5150cf4531eb5dd97f3343dcf65cd11704c84711348b328582b83424a0e4",
|
||||
"src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea",
|
||||
"src/tree_sitter/array.h": "5bdf6ed1a78e3409fd443e085ca967a64c188a5d082aaf7f819bccd53a471c94",
|
||||
"src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a"
|
||||
},
|
||||
"generatedSourcesUnchanged": true
|
||||
},
|
||||
"elf": {
|
||||
"architecture": "AArch64",
|
||||
"gnuStack": "RW",
|
||||
"needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"],
|
||||
"requiredSymbolVersions": [
|
||||
"CXXABI_1.3",
|
||||
"CXXABI_1.3.9",
|
||||
"GCC_3.0",
|
||||
"GLIBCXX_3.4",
|
||||
"GLIBCXX_3.4.21",
|
||||
"GLIBC_2.17"
|
||||
],
|
||||
"maxGLIBC": "2.17",
|
||||
"maxGLIBCXX": "3.4.21",
|
||||
"maxCXXABI": "1.3.9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tree-sitter-ruby",
|
||||
"version": "0.23.1",
|
||||
"file": "tree-sitter-ruby@0.23.1/linux-arm64/tree-sitter-ruby.node",
|
||||
"sha256": "e13ac3cbfcf40a5a2f350784d547d75b069174ca2955282d602f643a066a2c23",
|
||||
"source": {
|
||||
"url": "https://registry.npmjs.org/tree-sitter-ruby/-/tree-sitter-ruby-0.23.1.tgz",
|
||||
"integrity": "sha512-d9/RXgWjR6HanN7wTYhS5bpBQLz1VkH048Vm3CodPGyJVnamXMGb8oEhDypVCBq4QnHui9sTXuJBBP3WtCw5RA==",
|
||||
"gitHead": "71bd32fb7607035768799732addba884a37a6210",
|
||||
"sha256": "1f0f0c100e31ddf650962ebb1f1fb8420bdbabccd49582f6e731ea7478db639d",
|
||||
"files": {
|
||||
"binding.gyp": "5890a49ccc6a81bf1da23c2b740725562acdbbb7c9813dfd87fc3fb228ea80c2",
|
||||
"bindings/node/binding.cc": "b24ba437af1a31668a2765925f2f994a8516229fae81852451b21c35c32b67d2",
|
||||
"package.json": "07fb6a02a020df5fcedd8cc968b266de69eb6e5a1f1e43bd7c5817306339019c",
|
||||
"src/grammar.json": "ff0a0ab48788daea51cb418d9cfc14086dfcad50ac10e4981776be8b4d9296ab",
|
||||
"src/node-types.json": "9683f80e7be3dd93f1d6f575800de3f3445cda8af1f313f1f80003229c328e8b",
|
||||
"src/parser.c": "4ce468358b6f4e25a35c8cf6bc0eaf60665bc22d602f8c939323c2347255cd15",
|
||||
"src/scanner.c": "e7a6196d6e78bf4c6728502e924c867dee5d851c6253e43fcdb8ba169009bc58",
|
||||
"src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea",
|
||||
"src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1",
|
||||
"src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a"
|
||||
},
|
||||
"generatedSourcesUnchanged": true
|
||||
},
|
||||
"elf": {
|
||||
"architecture": "AArch64",
|
||||
"gnuStack": "RW",
|
||||
"needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"],
|
||||
"requiredSymbolVersions": [
|
||||
"CXXABI_1.3",
|
||||
"CXXABI_1.3.9",
|
||||
"GCC_3.0",
|
||||
"GLIBCXX_3.4",
|
||||
"GLIBCXX_3.4.21",
|
||||
"GLIBC_2.17"
|
||||
],
|
||||
"maxGLIBC": "2.17",
|
||||
"maxGLIBCXX": "3.4.21",
|
||||
"maxCXXABI": "1.3.9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tree-sitter-javascript",
|
||||
"version": "0.23.1",
|
||||
"file": "tree-sitter-javascript@0.23.1/linux-arm64/tree-sitter-javascript.node",
|
||||
"sha256": "3e280c974f199cdd35a621b0bd556e238eb46ab0666e45c568d3bf477eee32b6",
|
||||
"source": {
|
||||
"url": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz",
|
||||
"integrity": "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==",
|
||||
"gitHead": "3a837b6f3658ca3618f2022f8707e29739c91364",
|
||||
"sha256": "90e80b25a67517a4daf6ad751557bee21efbda7b7a5a554897933245d1734398",
|
||||
"files": {
|
||||
"binding.gyp": "e17df62e15b6b6a1590a77c61d0f614ba89c6fb77b807909050851d50008a2cb",
|
||||
"bindings/node/binding.cc": "bd509e0a410b33707bf738d3b18ed9e250188155d10d3a13b778344c28b1b94a",
|
||||
"package.json": "2d677a45985e03e329f78e2f8b0f7b401ae68698eec2743059fc34e56f1f6644",
|
||||
"src/grammar.json": "f952064b8f29a084b6d879e8018a5c86b06efddc0e9f7c8951ae752832d41b05",
|
||||
"src/node-types.json": "0d80ab597fcf1310efb9694d4276c407655d060b8bbab8f4ffda0223c43e94bf",
|
||||
"src/parser.c": "1150721590eca3c7b6623c7dc3498184f3e22c5d895a08b4806ebf5804f817c3",
|
||||
"src/scanner.c": "b3d3f64284d97bf80749c026862427782cf7ecc0b7dc094e6698ab311c9a42c7",
|
||||
"src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea",
|
||||
"src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1",
|
||||
"src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a"
|
||||
},
|
||||
"generatedSourcesUnchanged": true
|
||||
},
|
||||
"elf": {
|
||||
"architecture": "AArch64",
|
||||
"gnuStack": "RW",
|
||||
"needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"],
|
||||
"requiredSymbolVersions": [
|
||||
"CXXABI_1.3",
|
||||
"CXXABI_1.3.9",
|
||||
"GCC_3.0",
|
||||
"GLIBCXX_3.4",
|
||||
"GLIBCXX_3.4.21",
|
||||
"GLIBC_2.17"
|
||||
],
|
||||
"maxGLIBC": "2.17",
|
||||
"maxGLIBCXX": "3.4.21",
|
||||
"maxCXXABI": "1.3.9"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "tree-sitter-typescript",
|
||||
"version": "0.23.2",
|
||||
"file": "tree-sitter-typescript@0.23.2/linux-arm64/tree-sitter-typescript.node",
|
||||
"sha256": "d1a7c91ce8346de4aa4828649762625a0e8d51625472b33949932483d70e872e",
|
||||
"source": {
|
||||
"url": "https://registry.npmjs.org/tree-sitter-typescript/-/tree-sitter-typescript-0.23.2.tgz",
|
||||
"integrity": "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==",
|
||||
"gitHead": "f975a621f4e7f532fe322e13c4f79495e0a7b2e7",
|
||||
"sha256": "0fdf63c35930a75885145d75ee63cb879da2d49db9bfe454bd8e60b08ba778a1",
|
||||
"files": {
|
||||
"binding.gyp": "5451214724030d5150581c5af614d2d37c10379d9801c1170035f9777058b9e1",
|
||||
"bindings/node/binding.cc": "4cd4aa17fc240701572efe79b016aae0f2fa067f28a551de3cdba0e98537bc00",
|
||||
"common/scanner.h": "da66ef2bd14a3f7ea743e25ba068c6c9aae2c3509db200ff80c4a0e6116e564c",
|
||||
"package.json": "26892ef49837678a747f81e344ed26938ec5804e29e003ad34fd5a3c87c68baf",
|
||||
"tsx/package.json": "d91e8e983679ba8bb4736e0680dc895513f2f10a7e7ff833bcc5f9e92ef0dfeb",
|
||||
"tsx/src/grammar.json": "f98a7830b67be266c9e9effb673a5fc154ad6aa26f87f7c6d4a9191e09d8b357",
|
||||
"tsx/src/node-types.json": "78b5789145286799a27a0a7ecc36cc1bcb151f94ec7fa631b248459867010c8c",
|
||||
"tsx/src/parser.c": "1902cb53fa7ff5179df89b2eea863165e84c8cc866226419dc26921d8c055885",
|
||||
"tsx/src/scanner.c": "d563cd30b2f39718c9ae4292795c5ce03a2ad01954ba3a86ef84c2781a736673",
|
||||
"tsx/src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea",
|
||||
"tsx/src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1",
|
||||
"tsx/src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a",
|
||||
"typescript/package.json": "fcbd2e0fb7e7eed1af20cfd562cd3c3b22c839cb95b9d18df5793b850bd601dd",
|
||||
"typescript/src/grammar.json": "99dc4b95424dff47c89acb50d5d3f25b081b07eaff43994b7881dccd4186b214",
|
||||
"typescript/src/node-types.json": "c790a733fc756b54d4e54dceeb7d2d51e40d8b57136e70277753a75804cce3e3",
|
||||
"typescript/src/parser.c": "74fe453edd70f4eae9af0a1050cbd7943d8971d59165b6aaebbaa0a0b716d1aa",
|
||||
"typescript/src/scanner.c": "9125013b42cb888379d9be909f1d73dfb75a37626c2cdbf4122718a2b431a6d3",
|
||||
"typescript/src/tree_sitter/alloc.h": "b29c1c9fb7cc82f58c84b376df1297d6e2737a1d655fd356db0859e3c29c2fea",
|
||||
"typescript/src/tree_sitter/array.h": "4ff743903dc46f5db6aa54f31c6b4d160a8a9779e5b2ab1ee59ae7ebcd850ea1",
|
||||
"typescript/src/tree_sitter/parser.h": "a1f6ef161fbaf48a0e10fca90ef5290a062462b307b3898aa562993853b9f80a"
|
||||
},
|
||||
"generatedSourcesUnchanged": true
|
||||
},
|
||||
"elf": {
|
||||
"architecture": "AArch64",
|
||||
"gnuStack": "RW",
|
||||
"needed": ["libstdc++.so.6", "libgcc_s.so.1", "libc.so.6", "ld-linux-aarch64.so.1"],
|
||||
"requiredSymbolVersions": [
|
||||
"CXXABI_1.3",
|
||||
"CXXABI_1.3.9",
|
||||
"GCC_3.0",
|
||||
"GLIBCXX_3.4",
|
||||
"GLIBCXX_3.4.21",
|
||||
"GLIBC_2.17"
|
||||
],
|
||||
"maxGLIBC": "2.17",
|
||||
"maxGLIBCXX": "3.4.21",
|
||||
"maxCXXABI": "1.3.9"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
15
gitnexus/scripts/arm64-prebuild.cjs
Normal file
15
gitnexus/scripts/arm64-prebuild.cjs
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
// A tuple directory and an N-API symbol alone also accept the broken upstream
|
||||
// x86-64 binaries. Require an ELF64 little-endian AArch64 shared object.
|
||||
function isArm64NapiSharedObject(bytes) {
|
||||
return (
|
||||
bytes.length >= 64 &&
|
||||
bytes.subarray(0, 4).equals(Buffer.from([127, 69, 76, 70])) &&
|
||||
bytes[4] === 2 &&
|
||||
bytes[5] === 1 &&
|
||||
bytes.readUInt16LE(16) === 3 &&
|
||||
bytes.readUInt16LE(18) === 183 &&
|
||||
bytes.includes('napi_register_module_v1')
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = { isArm64NapiSharedObject };
|
||||
|
|
@ -36,7 +36,8 @@ const SAMPLES = {
|
|||
|
||||
async function check(installedDir) {
|
||||
const load = (rel) => import(pathToFileURL(path.join(installedDir, rel)).href);
|
||||
const Parser = createRequire(path.join(installedDir, 'package.json'))('tree-sitter');
|
||||
const require = createRequire(path.join(installedDir, 'package.json'));
|
||||
const Parser = require('tree-sitter');
|
||||
const { listGrammarSources, getLanguageGrammar } = await load(
|
||||
'dist/core/tree-sitter/parser-loader.js',
|
||||
);
|
||||
|
|
@ -60,6 +61,26 @@ async function check(installedDir) {
|
|||
failures.push(`${key}: ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
// TypeScript's nested JavaScript 0.23.1 has its own install lifecycle and
|
||||
// prebuild (#3599). The registry's direct JavaScript grammar is 0.25.0, so
|
||||
// loading it above does not exercise this bundled dependency.
|
||||
try {
|
||||
const typescriptRequire = createRequire(require.resolve('tree-sitter-typescript'));
|
||||
const { version } = typescriptRequire('tree-sitter-javascript/package.json');
|
||||
if (version !== '0.23.1') throw new Error(`expected nested JavaScript 0.23.1, got ${version}`);
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(typescriptRequire('tree-sitter-javascript'));
|
||||
const root = parser.parse(SAMPLES.javascript).rootNode;
|
||||
if (root.hasError || root.namedChildCount === 0) {
|
||||
failures.push(
|
||||
`typescript:nested-javascript: parsed with errors: ${root.toString().slice(0, 200)}`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
failures.push(
|
||||
`typescript:nested-javascript: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
const { PROTO_GRPC_PLUGIN } = await load('dist/core/group/extractors/grpc-patterns/proto.js');
|
||||
if (PROTO_GRPC_PLUGIN === null) {
|
||||
failures.push('proto: native Protobuf grammar failed to initialize');
|
||||
|
|
@ -71,7 +92,7 @@ async function check(installedDir) {
|
|||
).map((d) => d.symbolName);
|
||||
if (found.join() !== 'S.M') failures.push(`proto: expected [S.M], got [${found}]`);
|
||||
}
|
||||
return { checked: keys.length + 1, failures };
|
||||
return { checked: keys.length + 2, failures };
|
||||
}
|
||||
|
||||
if (!isMainThread) {
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@
|
|||
* A dependency's npm overrides do not apply in a consumer project. Several
|
||||
* ABI-compatible grammars still advertise older runtime peers, so bundling
|
||||
* alone leaves an invalid dependency tree. Admit only the runtime version we
|
||||
* tested, in only the audited manifests below. Sources and binaries are intact.
|
||||
* tested, in only the audited manifests below. Five audited upstream Linux
|
||||
* ARM64 binaries are repaired before packing.
|
||||
*/
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { isArm64NapiSharedObject } = require('./arm64-prebuild.cjs');
|
||||
|
||||
const RUNTIME = '0.25.1';
|
||||
// Upstream metadata is an input to validate, never the peer range we ship.
|
||||
|
|
@ -23,9 +26,51 @@ const AUDITED_PEERS = new Map([
|
|||
['tree-sitter-javascript@0.23.1', '^0.21.1'],
|
||||
['tree-sitter-typescript@0.23.2', '^0.21.0'],
|
||||
]);
|
||||
// These npm releases put x86-64 bytes in their linux-arm64 prebuild directory.
|
||||
// Keep the replacement set explicit: upstream upgrades require a new audit.
|
||||
const ARM64_REPAIRS = new Set([
|
||||
'tree-sitter-cpp@0.23.4',
|
||||
'tree-sitter-java@0.23.5',
|
||||
'tree-sitter-ruby@0.23.1',
|
||||
'tree-sitter-javascript@0.23.1',
|
||||
'tree-sitter-typescript@0.23.2',
|
||||
]);
|
||||
const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
|
||||
function prepareTreeSitterBundle(packageRoot = path.resolve(__dirname, '..')) {
|
||||
function readArm64Repairs(packageRoot) {
|
||||
const root = path.join(packageRoot, 'prebuilds', 'bundled');
|
||||
const manifest = readJson(path.join(root, 'manifest.json'));
|
||||
if (
|
||||
manifest.schemaVersion !== 1 ||
|
||||
manifest.tuple !== 'linux-arm64' ||
|
||||
!Array.isArray(manifest.packages) ||
|
||||
manifest.packages.length !== ARM64_REPAIRS.size
|
||||
) {
|
||||
throw new Error('Invalid bundled ARM64 prebuild manifest');
|
||||
}
|
||||
const repairs = new Map();
|
||||
for (const entry of manifest.packages) {
|
||||
const identity = `${entry.name}@${entry.version}`;
|
||||
const expectedFile = `${identity}/linux-arm64/${entry.name}.node`;
|
||||
if (!ARM64_REPAIRS.has(identity) || repairs.has(identity) || entry.file !== expectedFile) {
|
||||
throw new Error(`Unexpected bundled ARM64 prebuild identity: ${identity}`);
|
||||
}
|
||||
const bytes = fs.readFileSync(path.join(root, entry.file));
|
||||
if (createHash('sha256').update(bytes).digest('hex') !== entry.sha256) {
|
||||
throw new Error(`Bundled ARM64 prebuild checksum mismatch: ${identity}`);
|
||||
}
|
||||
if (!isArm64NapiSharedObject(bytes)) {
|
||||
throw new Error(`Bundled prebuild is not an AArch64 N-API shared object: ${identity}`);
|
||||
}
|
||||
repairs.set(identity, bytes);
|
||||
}
|
||||
return repairs;
|
||||
}
|
||||
|
||||
function prepareTreeSitterBundle(
|
||||
packageRoot = path.resolve(__dirname, '..'),
|
||||
{ forPublication = false } = {},
|
||||
) {
|
||||
const modulesRoot = path.join(packageRoot, 'node_modules');
|
||||
const manifest = readJson(path.join(packageRoot, 'package.json'));
|
||||
if (manifest.dependencies['tree-sitter'] !== RUNTIME) {
|
||||
|
|
@ -102,16 +147,54 @@ function prepareTreeSitterBundle(packageRoot = path.resolve(__dirname, '..')) {
|
|||
throw new Error('Tree-sitter dependency layout changed; revalidate the audited peer manifests');
|
||||
}
|
||||
|
||||
const repairs = readArm64Repairs(packageRoot);
|
||||
const binaries = [];
|
||||
const shadows = [];
|
||||
for (const { pkg, file } of patches) {
|
||||
const dir = path.dirname(file);
|
||||
const bytes = repairs.get(`${pkg.name}@${pkg.version}`);
|
||||
if (!bytes) continue;
|
||||
const target = path.join(dir, 'prebuilds', 'linux-arm64', `${pkg.name}.node`);
|
||||
// A changed upstream layout needs review rather than creating a second
|
||||
// candidate that node-gyp-build may not select.
|
||||
const candidates = fs
|
||||
.readdirSync(path.dirname(target))
|
||||
.filter((name) => name.endsWith('.node'));
|
||||
if (candidates.length !== 1 || candidates[0] !== path.basename(target)) {
|
||||
throw new Error(`Unexpected ARM64 prebuild layout for ${pkg.name}@${pkg.version}`);
|
||||
}
|
||||
binaries.push({ target, bytes });
|
||||
if (forPublication) {
|
||||
// node-gyp-build prefers local build outputs over prebuilds, even when
|
||||
// those outputs were compiled on the publisher's other architecture.
|
||||
// Preserve legitimate consumer source builds during ordinary postinstall.
|
||||
for (const mode of ['Release', 'Debug']) {
|
||||
const build = path.join(dir, 'build', mode);
|
||||
if (fs.existsSync(build)) {
|
||||
for (const name of fs.readdirSync(build)) {
|
||||
if (name.endsWith('.node')) shadows.push(path.join(build, name));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the complete set before writing, so dependency drift cannot leave
|
||||
// a partially prepared bundle behind. Repeated pack operations are idempotent.
|
||||
for (const { file, pkg } of patches) {
|
||||
fs.writeFileSync(file, JSON.stringify(pkg, null, 2) + '\n');
|
||||
}
|
||||
for (const { target, bytes } of binaries) {
|
||||
if (!fs.readFileSync(target).equals(bytes)) fs.writeFileSync(target, bytes);
|
||||
}
|
||||
for (const file of shadows) fs.unlinkSync(file);
|
||||
return patches.map(({ pkg }) => `${pkg.name}@${pkg.version}`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
const prepared = prepareTreeSitterBundle();
|
||||
const prepared = prepareTreeSitterBundle(undefined, {
|
||||
forPublication: process.argv.includes('--for-publication'),
|
||||
});
|
||||
console.log(
|
||||
`[tree-sitter-bundle] Validated runtime ${RUNTIME}; prepared ${prepared.length} peer manifests.`,
|
||||
);
|
||||
|
|
|
|||
212
gitnexus/scripts/rebuild-bundled-grammar-prebuilds.cjs
Normal file
212
gitnexus/scripts/rebuild-bundled-grammar-prebuilds.cjs
Normal file
|
|
@ -0,0 +1,212 @@
|
|||
#!/usr/bin/env node
|
||||
/** Rebuild only the five audited npm-package Linux ARM64 overlays. */
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const os = require('node:os');
|
||||
const { createHash } = require('node:crypto');
|
||||
const { createRequire } = require('node:module');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
const { isArm64NapiSharedObject } = require('./arm64-prebuild.cjs');
|
||||
|
||||
const output = path.resolve(__dirname, '../prebuilds/bundled');
|
||||
const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex');
|
||||
const readJson = (file) => JSON.parse(fs.readFileSync(file, 'utf8'));
|
||||
const samples = {
|
||||
'tree-sitter-cpp': [[null, 'class Box { public: int value; };']],
|
||||
'tree-sitter-java': [[null, 'class Box { int value; }']],
|
||||
'tree-sitter-ruby': [[null, 'class Box\n def value\n 1\n end\nend']],
|
||||
'tree-sitter-javascript': [[null, 'export function value() { return 1; }']],
|
||||
'tree-sitter-typescript': [
|
||||
['typescript', 'const value: number = 1;'],
|
||||
['tsx', 'const value = <div>hello</div>;'],
|
||||
],
|
||||
};
|
||||
|
||||
function buildEnvironment(env = process.env) {
|
||||
// Keep tool discovery and the user directory, but exclude compiler, make and
|
||||
// gyp overrides so the recorded GCC toolchain and flags match the build.
|
||||
return { PATH: env.PATH, HOME: env.HOME, CC: 'gcc', CXX: 'g++' };
|
||||
}
|
||||
|
||||
async function download(url, file, expectedSha256, integrity) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) throw new Error(`Download failed: ${url} (${response.status})`);
|
||||
const bytes = Buffer.from(await response.arrayBuffer());
|
||||
if (
|
||||
sha256(bytes) !== expectedSha256 ||
|
||||
(integrity && `sha512-${createHash('sha512').update(bytes).digest('base64')}` !== integrity)
|
||||
) {
|
||||
throw new Error(`Source checksum mismatch: ${url}`);
|
||||
}
|
||||
fs.writeFileSync(file, bytes);
|
||||
}
|
||||
|
||||
function verifySources(dir, files) {
|
||||
for (const [file, expected] of Object.entries(files)) {
|
||||
if (sha256(fs.readFileSync(path.join(dir, file))) !== expected) {
|
||||
throw new Error(`Upstream source changed: ${dir}/${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function inspectBinary(file) {
|
||||
const bytes = fs.readFileSync(file);
|
||||
if (!isArm64NapiSharedObject(bytes)) {
|
||||
throw new Error(`Not an AArch64 N-API shared object: ${file}`);
|
||||
}
|
||||
const readelf = (...args) => execFileSync('readelf', args.concat(file), { encoding: 'utf8' });
|
||||
const versions = [
|
||||
...new Set(readelf('--version-info').match(/(?:GLIBCXX|GLIBC|CXXABI|GCC)_[\d.]+/g)),
|
||||
].sort();
|
||||
const limits = { GLIBC: '2.17', GLIBCXX: '3.4.21', CXXABI: '1.3.9' };
|
||||
for (const [family, max] of Object.entries(limits)) {
|
||||
for (const version of versions.filter((value) => value.startsWith(`${family}_`))) {
|
||||
const parts = version
|
||||
.slice(family.length + 1)
|
||||
.split('.')
|
||||
.map(Number);
|
||||
const ceiling = max.split('.').map(Number);
|
||||
const difference = parts
|
||||
.map((part, i) => part - (ceiling[i] ?? 0))
|
||||
.find((part) => part !== 0);
|
||||
if (difference > 0) throw new Error(`${file} requires ${version}, above ${family}_${max}`);
|
||||
}
|
||||
}
|
||||
const stack = readelf('--program-headers', '--wide')
|
||||
.split('\n')
|
||||
.find((line) => line.includes('GNU_STACK'));
|
||||
if (!stack || !/\sRW\s/.test(stack)) throw new Error(`Missing non-executable GNU_STACK: ${file}`);
|
||||
return {
|
||||
architecture: 'AArch64',
|
||||
gnuStack: 'RW',
|
||||
needed: [...readelf('--dynamic').matchAll(/\(NEEDED\).*\[([^\]]+)\]/g)].map(
|
||||
(match) => match[1],
|
||||
),
|
||||
requiredSymbolVersions: versions,
|
||||
maxGLIBC: limits.GLIBC,
|
||||
maxGLIBCXX: limits.GLIBCXX,
|
||||
maxCXXABI: limits.CXXABI,
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.argv.includes('--help')) {
|
||||
console.log(
|
||||
'Run on Linux ARM64 with Node, npm, Python, make, GCC/G++, tar and readelf.\nRebuilds the five pinned upstream packages into prebuilds/bundled; see its README.md.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (process.platform !== 'linux' || process.arch !== 'arm64') {
|
||||
throw new Error(
|
||||
'Regeneration requires an actual Linux ARM64 Node process; npm_config_arch is insufficient.',
|
||||
);
|
||||
}
|
||||
const manifest = readJson(path.join(output, 'manifest.json'));
|
||||
const work = fs.mkdtempSync(path.join(os.tmpdir(), 'gitnexus-bundled-prebuilds-'));
|
||||
const buildEnv = buildEnvironment();
|
||||
const run = (command, args, cwd = work, env = process.env) =>
|
||||
execFileSync(command, args, { cwd, env, stdio: 'inherit' });
|
||||
try {
|
||||
const dependencies = Object.fromEntries(
|
||||
Object.entries(manifest.build.dependencies).map(([name, entry]) => [name, entry.version]),
|
||||
);
|
||||
dependencies['tree-sitter'] = manifest.build.runtime;
|
||||
fs.writeFileSync(
|
||||
path.join(work, 'package.json'),
|
||||
JSON.stringify({ private: true, dependencies }),
|
||||
);
|
||||
run('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--legacy-peer-deps']);
|
||||
const lock = readJson(path.join(work, 'package-lock.json'));
|
||||
for (const [name, entry] of Object.entries(manifest.build.dependencies)) {
|
||||
const installed = lock.packages[`node_modules/${name}`];
|
||||
if (installed.version !== entry.version || installed.integrity !== entry.integrity) {
|
||||
throw new Error(`Build dependency differs from recorded input: ${name}`);
|
||||
}
|
||||
}
|
||||
const header = manifest.build.nodeHeaders;
|
||||
const headerArchive = path.join(work, 'node-headers.tar.gz');
|
||||
await download(header.url, headerArchive, header.sha256);
|
||||
run('tar', ['-xzf', headerArchive, '-C', work]);
|
||||
const headers = path.join(work, `node-v${header.version}`);
|
||||
const Parser = createRequire(path.join(work, 'package.json'))('tree-sitter');
|
||||
const built = [];
|
||||
// Build serially: node-gyp writes a shared node-addon-api Makefile whose
|
||||
// relative target paths depend on the grammar's position in node_modules.
|
||||
for (const entry of manifest.packages) {
|
||||
const archive = path.join(work, `${entry.name}.tgz`);
|
||||
await download(entry.source.url, archive, entry.source.sha256, entry.source.integrity);
|
||||
const dir = path.join(work, 'node_modules', entry.name);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
run('tar', ['-xzf', archive, '-C', dir, '--strip-components=1']);
|
||||
const pkg = readJson(path.join(dir, 'package.json'));
|
||||
if (pkg.name !== entry.name || pkg.version !== entry.version) {
|
||||
throw new Error(`Unexpected upstream package identity: ${entry.name}`);
|
||||
}
|
||||
verifySources(dir, entry.source.files);
|
||||
run(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(work, 'node_modules/node-gyp/bin/node-gyp.js'),
|
||||
'rebuild',
|
||||
'--arch=arm64',
|
||||
`--nodedir=${headers}`,
|
||||
'--jobs=2',
|
||||
],
|
||||
dir,
|
||||
buildEnv,
|
||||
);
|
||||
verifySources(dir, entry.source.files);
|
||||
const release = path.join(dir, 'build/Release');
|
||||
const files = fs.readdirSync(release).filter((name) => name.endsWith('.node'));
|
||||
if (files.length !== 1) throw new Error(`Expected one native binding: ${entry.name}`);
|
||||
const binary = path.join(release, files[0]);
|
||||
const elf = inspectBinary(binary);
|
||||
const language = require(binary);
|
||||
for (const [variant, source] of samples[entry.name]) {
|
||||
const parser = new Parser();
|
||||
parser.setLanguage(variant ? language[variant] : language);
|
||||
const root = parser.parse(source).rootNode;
|
||||
if (root.hasError || root.namedChildCount === 0)
|
||||
throw new Error(`Parse failed: ${entry.name} ${variant ?? ''}`);
|
||||
}
|
||||
const bytes = fs.readFileSync(binary);
|
||||
built.push({ entry: { ...entry, sha256: sha256(bytes), elf }, bytes });
|
||||
}
|
||||
const version = (command) =>
|
||||
execFileSync(command, ['--version'], { env: buildEnv, encoding: 'utf8' }).split('\n')[0];
|
||||
manifest.build = {
|
||||
...manifest.build,
|
||||
compiler: `${version(buildEnv.CC)}; ${version(buildEnv.CXX)}`,
|
||||
linker: version('ld'),
|
||||
sysroot: version('ldd'),
|
||||
hostArchitecture: process.arch,
|
||||
hostNode: process.versions.node,
|
||||
compilerFlags: [],
|
||||
validation: {
|
||||
execution: 'Linux ARM64 Node process',
|
||||
nodeVersions: [process.versions.node],
|
||||
grammarExports: ['cpp', 'java', 'ruby', 'javascript', 'typescript', 'tsx'],
|
||||
parseFailures: 0,
|
||||
},
|
||||
};
|
||||
manifest.packages = built.map(({ entry }) => entry);
|
||||
// No committed output changes until every package passes provenance,
|
||||
// ABI-baseline, native loading and parsing checks.
|
||||
for (const { entry, bytes } of built) fs.writeFileSync(path.join(output, entry.file), bytes);
|
||||
fs.writeFileSync(path.join(output, 'manifest.json'), JSON.stringify(manifest, null, 2) + '\n');
|
||||
console.log(
|
||||
`Rebuilt and parsed ${built.length} audited ARM64 bindings; review binary and manifest changes.`,
|
||||
);
|
||||
} finally {
|
||||
fs.rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { buildEnvironment };
|
||||
|
|
@ -27,6 +27,7 @@ interface WorkflowStep {
|
|||
|
||||
interface WorkflowJob {
|
||||
'timeout-minutes'?: number;
|
||||
strategy?: { matrix?: { os?: string[] } };
|
||||
steps?: WorkflowStep[];
|
||||
}
|
||||
|
||||
|
|
@ -280,6 +281,101 @@ describe('workflows that need the web UI install it themselves', () => {
|
|||
expect(setup?.with?.['lifecycle-scripts']).toBe('false');
|
||||
expect(setup?.with?.build).toBeUndefined();
|
||||
});
|
||||
|
||||
it('exercises the reported native ARM64 runtime alongside Windows and Linux x64', () => {
|
||||
const job = ciJobs['packaged-install-smoke'];
|
||||
expect(job?.strategy?.matrix?.os).toEqual([
|
||||
'windows-latest',
|
||||
'ubuntu-latest',
|
||||
'ubuntu-24.04-arm',
|
||||
]);
|
||||
const steps = job?.steps ?? [];
|
||||
const setupIdx = stepIndex(steps, (step) => step.uses === './.github/actions/setup-gitnexus');
|
||||
const armNodeIdx = stepIndex(steps, (step) => step.with?.['node-version'] === '22.23.1');
|
||||
expect(armNodeIdx).toBeGreaterThan(setupIdx);
|
||||
expect(steps[armNodeIdx]?.if).toContain("matrix.os == 'ubuntu-24.04-arm'");
|
||||
const assertArch = steps.find((step) => step.name === 'Assert native ARM64 runtime');
|
||||
expect(assertArch?.if).toContain("matrix.os == 'ubuntu-24.04-arm'");
|
||||
expect(String(assertArch?.run)).toContain("process.arch, 'arm64'");
|
||||
expect(String(assertArch?.run)).toContain("process.version, 'v22.23.1'");
|
||||
});
|
||||
|
||||
it('keeps consumer compilation guards after packing and checks both npm lockfile paths', () => {
|
||||
const steps = ciJobs['packaged-install-smoke']?.steps ?? [];
|
||||
const packIdx = stepIndex(steps, (step) => step.name === 'Pack gitnexus tarball');
|
||||
const guardIdx = stepIndex(steps, (step) => step.name === 'Reject ARM64 consumer compilation');
|
||||
const localIdx = stepIndex(
|
||||
steps,
|
||||
(step) => step.name === 'Lock and install tarball in a fresh consumer project',
|
||||
);
|
||||
const globalIdx = stepIndex(
|
||||
steps,
|
||||
(step) => step.name === 'Install gitnexus tarball into isolated prefix',
|
||||
);
|
||||
const checkIdx = stepIndex(
|
||||
steps,
|
||||
(step) => step.name === 'Assert no consumer compilation attempts',
|
||||
);
|
||||
expect(guardIdx).toBeGreaterThan(packIdx);
|
||||
expect(localIdx).toBeGreaterThan(guardIdx);
|
||||
expect(globalIdx).toBeGreaterThan(guardIdx);
|
||||
expect(checkIdx).toBeGreaterThan(globalIdx);
|
||||
expect(steps[guardIdx]?.if).toContain("matrix.os == 'ubuntu-24.04-arm'");
|
||||
expect(steps[checkIdx]?.if).toContain('always()');
|
||||
expect(String(steps[localIdx]?.run)).toContain('for NPM_VERSION in 10.9.4 11.9.0');
|
||||
expect(String(steps[localIdx]?.run)).toContain('cmp package-lock.json expected-lock.json');
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === 'win32')('records a rejected compiler invocation', () => {
|
||||
const steps = ciJobs['packaged-install-smoke']?.steps ?? [];
|
||||
const guard = steps.find((step) => step.name === 'Reject ARM64 consumer compilation');
|
||||
const check = steps.find((step) => step.name === 'Assert no consumer compilation attempts');
|
||||
expect(guard?.run).toBeTypeOf('string');
|
||||
expect(check?.run).toBeTypeOf('string');
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'gitnexus-compile-guard-'));
|
||||
tempDirs.push(dir);
|
||||
const githubPath = path.join(dir, 'github-path');
|
||||
const githubEnv = path.join(dir, 'github-env');
|
||||
const setup = spawnSync('bash', ['-eu', '-o', 'pipefail', '-c', String(guard?.run)], {
|
||||
env: { ...process.env, RUNNER_TEMP: dir, GITHUB_PATH: githubPath, GITHUB_ENV: githubEnv },
|
||||
encoding: 'utf8',
|
||||
});
|
||||
expect(setup.status, setup.stderr).toBe(0);
|
||||
const guardEnv = Object.fromEntries(
|
||||
readFileSync(githubEnv, 'utf8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
const separator = line.indexOf('=');
|
||||
return [line.slice(0, separator), line.slice(separator + 1)];
|
||||
}),
|
||||
);
|
||||
const env = {
|
||||
...process.env,
|
||||
...guardEnv,
|
||||
PATH: `${readFileSync(githubPath, 'utf8').trim()}${path.delimiter}${process.env.PATH}`,
|
||||
};
|
||||
const clean = spawnSync('bash', ['-eu', '-o', 'pipefail', '-c', String(check?.run)], {
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
expect(clean.status, clean.stderr).toBe(0);
|
||||
writeFileSync(path.join(dir, 'probe.c'), 'int main(void) { return 0; }\n');
|
||||
const compile = spawnSync('cc', ['-c', 'probe.c', '-o', 'probe.o'], {
|
||||
cwd: dir,
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
expect(compile.status).toBe(97);
|
||||
expect(compile.stderr).toContain('Unexpected native compilation');
|
||||
expect(existsSync(path.join(dir, 'probe.o'))).toBe(false);
|
||||
const attempted = spawnSync('bash', ['-eu', '-o', 'pipefail', '-c', String(check?.run)], {
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
});
|
||||
expect(attempted.status).toBe(1);
|
||||
expect(attempted.stderr).toContain('Unexpected native compilation');
|
||||
});
|
||||
});
|
||||
|
||||
describe('setup-gitnexus job budget', () => {
|
||||
|
|
|
|||
|
|
@ -14,6 +14,8 @@ import { tmpdir } from 'node:os';
|
|||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { load } from 'js-yaml';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { classifyBinaryHeader } from '../../src/core/lbug/extension-load-error.js';
|
||||
|
||||
/**
|
||||
* Regression guard: every tree-sitter grammar GitNexus ships must provide a
|
||||
|
|
@ -381,3 +383,51 @@ describe('prebuild workflow rebuild loop guard', () => {
|
|||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe('bundled Linux ARM64 repair architecture', () => {
|
||||
const repairs = [
|
||||
['tree-sitter-cpp', '0.23.4'],
|
||||
['tree-sitter-java', '0.23.5'],
|
||||
['tree-sitter-ruby', '0.23.1'],
|
||||
['tree-sitter-javascript', '0.23.1'],
|
||||
['tree-sitter-typescript', '0.23.2'],
|
||||
];
|
||||
for (const [name, version] of repairs) {
|
||||
it(`${name}@${version}: installed ARM64 prebuild is an AArch64 N-API binary`, () => {
|
||||
const dir =
|
||||
name === 'tree-sitter-javascript'
|
||||
? path.join(NODE_MODULES, 'tree-sitter-typescript/node_modules', name)
|
||||
: path.join(NODE_MODULES, name);
|
||||
const bytes = readFileSync(path.join(dir, 'prebuilds/linux-arm64', `${name}.node`));
|
||||
expect(classifyBinaryHeader(bytes, bytes.length, 'linux', 'arm64')).toBe('valid');
|
||||
expect(classifyBinaryHeader(bytes, bytes.length, 'linux', 'x64')).toBe('corrupt');
|
||||
expect(bytes.includes(NAPI_SYMBOL)).toBe(true);
|
||||
});
|
||||
}
|
||||
|
||||
it('pins the five repair identities, pristine source provenance, and replacement checksums', () => {
|
||||
const dir = path.join(GITNEXUS_ROOT, 'prebuilds/bundled');
|
||||
const manifest = JSON.parse(readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
|
||||
// Independent audit pin: review regenerated artifacts and their provenance
|
||||
// before updating this hash. Parse/stringify ignores formatting-only changes.
|
||||
expect(createHash('sha256').update(JSON.stringify(manifest)).digest('hex')).toBe(
|
||||
'2aa9eda6086233ce620564a63f9ed422c424b0d89b10e58b0e906e091db2aa3f',
|
||||
);
|
||||
expect(manifest.schemaVersion).toBe(1);
|
||||
expect(manifest.tuple).toBe('linux-arm64');
|
||||
expect(
|
||||
manifest.packages.map((entry: { name: string; version: string }) => [
|
||||
entry.name,
|
||||
entry.version,
|
||||
]),
|
||||
).toEqual(repairs);
|
||||
for (const entry of manifest.packages) {
|
||||
const bytes = readFileSync(path.join(dir, entry.file));
|
||||
expect(createHash('sha256').update(bytes).digest('hex')).toBe(entry.sha256);
|
||||
expect(classifyBinaryHeader(bytes, bytes.length, 'linux', 'arm64')).toBe('valid');
|
||||
expect(entry.source.integrity).toMatch(/^sha512-/);
|
||||
expect(entry.source.sha256).toMatch(/^[0-9a-f]{64}$/);
|
||||
expect(entry.source.generatedSourcesUnchanged).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { createRequire } from 'node:module';
|
||||
import { createHash } from 'node:crypto';
|
||||
import {
|
||||
existsSync,
|
||||
mkdtempSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
|
|
@ -15,7 +17,35 @@ import { cleanupTempDirSync } from '../helpers/test-db.js';
|
|||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { prepareTreeSitterBundle } = require('../../scripts/prepare-tree-sitter-bundle.cjs');
|
||||
const { TUPLES: tuples } = require('../../scripts/assert-publish-grammar-coverage.cjs');
|
||||
const roots: string[] = [];
|
||||
const repairs = [
|
||||
['tree-sitter-cpp', '0.23.4'],
|
||||
['tree-sitter-java', '0.23.5'],
|
||||
['tree-sitter-ruby', '0.23.1'],
|
||||
['tree-sitter-javascript', '0.23.1'],
|
||||
['tree-sitter-typescript', '0.23.2'],
|
||||
];
|
||||
const sha256 = (bytes: Buffer) => createHash('sha256').update(bytes).digest('hex');
|
||||
const repairRoot = (root: string) => path.join(root, 'prebuilds/bundled');
|
||||
const grammarRoot = (root: string, name: string) =>
|
||||
path.join(
|
||||
root,
|
||||
'node_modules',
|
||||
name === 'tree-sitter-javascript'
|
||||
? 'tree-sitter-typescript/node_modules/tree-sitter-javascript'
|
||||
: name,
|
||||
);
|
||||
const replacement = () => {
|
||||
const bytes = Buffer.alloc(96);
|
||||
bytes.write('\x7fELF');
|
||||
bytes[4] = 2;
|
||||
bytes[5] = 1;
|
||||
bytes.writeUInt16LE(3, 16);
|
||||
bytes.writeUInt16LE(183, 18);
|
||||
bytes.write('napi_register_module_v1', 64);
|
||||
return bytes;
|
||||
};
|
||||
|
||||
const directVersions: Record<string, string> = {
|
||||
'@eagleoutice/tree-sitter-r': '1.1.2',
|
||||
|
|
@ -76,6 +106,23 @@ function fixture() {
|
|||
peerDependencies: { 'tree-sitter': '^0.21.1' },
|
||||
},
|
||||
);
|
||||
const overlays = repairs.map(([name, version]) => {
|
||||
const file = `${name}@${version}/linux-arm64/${name}.node`;
|
||||
const bytes = replacement();
|
||||
mkdirSync(path.dirname(path.join(repairRoot(root), file)), { recursive: true });
|
||||
writeFileSync(path.join(repairRoot(root), file), bytes);
|
||||
for (const tuple of tuples) {
|
||||
const dir = path.join(grammarRoot(root, name), 'prebuilds', tuple);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
writeFileSync(path.join(dir, `${name}.node`), `original ${name} ${tuple}`);
|
||||
}
|
||||
return { name, version, file, sha256: sha256(bytes) };
|
||||
});
|
||||
write(path.join(repairRoot(root), 'manifest.json'), {
|
||||
schemaVersion: 1,
|
||||
tuple: 'linux-arm64',
|
||||
packages: overlays,
|
||||
});
|
||||
return root;
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +164,7 @@ afterEach(() => {
|
|||
});
|
||||
|
||||
describe('native parser bundle preparation', () => {
|
||||
it('pins direct and transitive peers to the tested runtime without changing binaries', () => {
|
||||
it('pins audited peers without changing unrelated binaries', () => {
|
||||
const root = fixture();
|
||||
const native = path.join(root, 'node_modules/tree-sitter-cpp/parser.node');
|
||||
writeFileSync(native, Buffer.from([0, 1, 128, 255]));
|
||||
|
|
@ -271,3 +318,85 @@ describe('native parser bundle preparation', () => {
|
|||
expect(() => prepareTreeSitterBundle(root)).toThrow('installed version differs');
|
||||
});
|
||||
});
|
||||
|
||||
describe('audited Linux ARM64 bundle repairs', () => {
|
||||
it('repairs all five identities and preserves every other tuple and the direct JavaScript dependency', () => {
|
||||
const root = fixture();
|
||||
const direct = path.join(
|
||||
root,
|
||||
'node_modules/tree-sitter-javascript/prebuilds/linux-arm64/tree-sitter-javascript.node',
|
||||
);
|
||||
mkdirSync(path.dirname(direct), { recursive: true });
|
||||
writeFileSync(direct, 'direct JavaScript 0.25.0');
|
||||
prepareTreeSitterBundle(root);
|
||||
for (const [name] of repairs) {
|
||||
for (const tuple of tuples) {
|
||||
const bytes = readFileSync(
|
||||
path.join(grammarRoot(root, name), 'prebuilds', tuple, `${name}.node`),
|
||||
);
|
||||
expect(bytes).toEqual(
|
||||
tuple === 'linux-arm64' ? replacement() : Buffer.from(`original ${name} ${tuple}`),
|
||||
);
|
||||
}
|
||||
}
|
||||
expect(readFileSync(direct, 'utf8')).toBe('direct JavaScript 0.25.0');
|
||||
expect(prepareTreeSitterBundle(root)).toHaveLength(8);
|
||||
});
|
||||
|
||||
it.each(['missing', 'corrupt', 'architecture', 'version'])(
|
||||
'rejects %s repair input before any manifest or binary writes',
|
||||
(failure) => {
|
||||
const root = fixture();
|
||||
const manifestFile = path.join(repairRoot(root), 'manifest.json');
|
||||
const manifest = read(manifestFile);
|
||||
const last = manifest.packages.at(-1);
|
||||
const file = path.join(repairRoot(root), last.file);
|
||||
if (failure === 'missing') rmSync(file);
|
||||
if (failure === 'corrupt') writeFileSync(file, 'corrupt binary');
|
||||
if (failure === 'architecture') {
|
||||
const bytes = replacement();
|
||||
bytes.writeUInt16LE(62, 18);
|
||||
writeFileSync(file, bytes);
|
||||
last.sha256 = sha256(bytes);
|
||||
}
|
||||
if (failure === 'version') last.version = '99.0.0';
|
||||
write(manifestFile, manifest);
|
||||
const peer = path.join(root, 'node_modules/tree-sitter-c/package.json');
|
||||
const before = readFileSync(peer);
|
||||
expect(() => prepareTreeSitterBundle(root)).toThrow();
|
||||
expect(readFileSync(peer)).toEqual(before);
|
||||
for (const [name] of repairs) {
|
||||
expect(
|
||||
readFileSync(
|
||||
path.join(grammarRoot(root, name), 'prebuilds/linux-arm64', `${name}.node`),
|
||||
'utf8',
|
||||
),
|
||||
).toBe(`original ${name} linux-arm64`);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it('repairs nested JavaScript through linked dependency containers', () => {
|
||||
const { root, javascript } = pnpmFixture();
|
||||
prepareTreeSitterBundle(root);
|
||||
expect(
|
||||
readFileSync(path.join(javascript, 'prebuilds/linux-arm64/tree-sitter-javascript.node')),
|
||||
).toEqual(replacement());
|
||||
});
|
||||
|
||||
it('removes shadowing local builds only for publication, preserving consumer source-build fallbacks', () => {
|
||||
const root = fixture();
|
||||
const shadows = repairs.flatMap(([name]) =>
|
||||
['Release', 'Debug'].map((mode) => {
|
||||
const file = path.join(grammarRoot(root, name), 'build', mode, 'binding.node');
|
||||
mkdirSync(path.dirname(file), { recursive: true });
|
||||
writeFileSync(file, 'source-built fallback');
|
||||
return file;
|
||||
}),
|
||||
);
|
||||
prepareTreeSitterBundle(root);
|
||||
expect(shadows.every(existsSync)).toBe(true);
|
||||
prepareTreeSitterBundle(root, { forPublication: true });
|
||||
expect(shadows.some(existsSync)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
35
gitnexus/test/unit/rebuild-bundled-grammar-prebuilds.test.ts
Normal file
35
gitnexus/test/unit/rebuild-bundled-grammar-prebuilds.test.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { createRequire } from 'node:module';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const { buildEnvironment } = require('../../scripts/rebuild-bundled-grammar-prebuilds.cjs') as {
|
||||
buildEnvironment: (env: NodeJS.ProcessEnv) => NodeJS.ProcessEnv;
|
||||
};
|
||||
|
||||
describe('bundled grammar rebuild environment', () => {
|
||||
it('keeps compiler and flag overrides out of the recorded GCC build', () => {
|
||||
const inherited = {
|
||||
PATH: '/usr/bin:/bin',
|
||||
HOME: '/home/builder',
|
||||
CC: 'clang',
|
||||
CXX: 'clang++',
|
||||
CC_target: 'clang',
|
||||
CXX_target: 'clang++',
|
||||
LINK_target: 'clang++',
|
||||
CFLAGS: '-O1',
|
||||
CXXFLAGS: '-O1',
|
||||
CPPFLAGS: '-DOVERRIDE',
|
||||
LDFLAGS: '-fuse-ld=lld',
|
||||
MAKEFLAGS: 'CC=clang',
|
||||
GYP_DEFINES: 'clang=1',
|
||||
};
|
||||
|
||||
expect(buildEnvironment(inherited)).toEqual({
|
||||
PATH: inherited.PATH,
|
||||
HOME: inherited.HOME,
|
||||
CC: 'gcc',
|
||||
CXX: 'g++',
|
||||
});
|
||||
expect(inherited.CC).toBe('clang');
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Reference in a new issue