scriptoriumadmin/GitNexus
1
0
Fork 0
mirror of https://github.com/abhigyanpatwari/GitNexus.git synced 2026-09-08 22:22:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

fix(eval): hide the hidden harness from the proposer; drop inert hooks

Browse source 
The proposer authors the overlay the benchmark arms are scored with, but its
clone was never sanitized: it could read eval/workflow_bench, i.e. the task
prompts and the hidden oracles it was about to be graded against. The last
diagnostic run did exactly that, reading
inv-feature-list-repos-filter.oracle.test.ts directly, so a proposal could
win the gate by encoding expected behavior into a skill instead of being a
better skill. Sanitize the proposer clone exactly as run_cell already does.

Also remove the PreToolUse tool-input normalizer. It never ran: headless
`claude -p` (2.1.247) dispatches no hooks from inline --settings, a settings
file, project/user/local --setting-sources, or a trusted ~/.claude.json
project entry. Keeping it would read as a control in review while enforcing
nothing, and it was the sole reason the proposer stopped using --bare —
which stays off on its own merits, since bare ignores --tools and would cost
the proposer Grep and Glob.

Blank optional arguments from the OpenAI adapter remain handled where the
code is ours: MCP aliases in local-backend normalizeToolParams. Built-in
Read still rejects pages:"" and the model self-corrects on the next turn.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Gergo Magyar 2026-09-03 20:03:09 +00:00
parent aea20ccf72
commit a2fdf9e93c
5 changed files with 52 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
eval/tests/test_evolve.py
View file

@ -560,22 +560,34 @@ def test_proposer_refuses_a_prior_proposal_that_lost_its_trust_boundary(tmp_path
)
def test_run_proposer_keeps_trusted_input_hook_enabled(monkeypatch, tmp_path):
def test_run_proposer_hides_the_hidden_harness_and_keeps_the_full_tool_surface(monkeypatch, tmp_path):
"""The proposer writes the artifact the arms are scored with.
So its clone must be sanitized before the session starts — a proposer that
can read eval/workflow_bench reads the task prompts and hidden oracles it
is about to be graded against, and can encode the answers into the skill.
"""
evidence = tmp_path / "evidence"
evidence.mkdir()
transcript_projects = tmp_path / "transcript-projects"
transcript_projects.mkdir()
captured: dict[str, object] = {}
sanitized: list[Path] = []
def fake_make_worktree(_repo, _ref, destination):
clone = destination / "clone"
clone.mkdir()
return clone
def fake_sanitize(clone):
sanitized.append(clone)
return "0" * 40
class FakeSandbox:
claude_bin = "claude"
command_prefix: list[str] = []
settings_json = '{"hooks":{"PreToolUse":[]}}'
settings_json = '{"permissions":{"allow":["Read"]}}'
@property
def transcript_projects(self):
@ -591,6 +603,7 @@ def test_run_proposer_keeps_trusted_input_hook_enabled(monkeypatch, tmp_path):
monkeypatch.setattr(evolve.runner, "make_worktree", fake_make_worktree)
monkeypatch.setattr(evolve.runner, "remove_clone", lambda _clone: None)
monkeypatch.setattr(evolve, "sanitize_clone_for_hidden_oracles", fake_sanitize)
monkeypatch.setattr(evolve, "prepare_sandbox", fake_prepare_sandbox)
monkeypatch.setattr(evolve.runner, "run_claude", fake_run_claude)
args = build_parser().parse_args(["--tasks", "tasks.yaml", "--model", "model"])
@ -605,6 +618,10 @@ def test_run_proposer_keeps_trusted_input_hook_enabled(monkeypatch, tmp_path):
)
assert record["ok"] is False
# Sanitization has to happen on the clone the session actually runs in,
# and before the sandbox is prepared around it.
assert [clone.name for clone in sanitized] == ["clone"]
# Not --bare: bare ignores --tools and would cost the proposer Grep/Glob.
assert captured.get("bare", False) is False
assert captured["allowed_tools"] == evolve.PROPOSER_ALLOWED_TOOLS
assert captured["settings_json"] == FakeSandbox.settings_json

36
eval/tests/test_proposer_sandbox.py
View file

@ -29,7 +29,6 @@ from workflow_bench.proposer_sandbox import (
SANDBOX_PATH,
SANDBOX_PYTHON3,
SANDBOX_SHELL_PREFIX,
SANDBOX_TOOL_INPUT_NORMALIZER,
SANDBOX_USER_SKILLS,
ReadOnlyMount,
SandboxError,
@ -70,9 +69,10 @@ def test_environment_is_allowlisted_and_shell_children_are_credential_free(monke
assert settings["sandbox"]["allowUnsandboxedCommands"] is False
assert settings["sandbox"]["network"]["deniedDomains"] == ["*"]
assert SANDBOX_EVIDENCE in settings["sandbox"]["filesystem"]["allowRead"]
normalizer_hook = settings["hooks"]["PreToolUse"][0]
assert normalizer_hook["matcher"] == r"Read|mcp__gitnexus__.*"
assert normalizer_hook["hooks"][0]["command"] == SANDBOX_TOOL_INPUT_NORMALIZER
# Headless `claude -p` (2.1.247) never dispatches PreToolUse from any
# settings source, so a hook here would be confinement theater: it would
# read as a control in review while enforcing nothing at runtime.
assert "hooks" not in settings
# ENV_SCRUB forces "default" mode; the proposer's tools (Bash writes the
# overlay) run headless only because they are explicitly pre-approved.
# Requesting a non-default defaultMode would merely warn, so it must be gone.
@ -178,32 +178,6 @@ def test_sandbox_command_has_minimal_mounts_and_no_host_root_bind(tmp_path: Path
assert probe.returncode == 0, probe.stderr
assert probe.stdout == f"/home/agent|{SANDBOX_PATH}"
normalizer_index = argv.index(SANDBOX_TOOL_INPUT_NORMALIZER)
normalizer = Path(argv[normalizer_index - 1])
event = json.dumps(
{
"hook_event_name": "PreToolUse",
"tool_name": "Read",
"tool_input": {
"file_path": "/evidence/rows.json",
"pages": "",
"limit": 100,
},
}
)
normalized = subprocess.run(
[sys.executable, normalizer],
input=event,
text=True,
capture_output=True,
check=True,
)
output = json.loads(normalized.stdout)
assert output["hookSpecificOutput"]["updatedInput"] == {
"file_path": "/evidence/rows.json",
"limit": 100,
}
gitnexus_index = argv.index(SANDBOX_GITNEXUS_CLI)
gitnexus_wrapper = Path(argv[gitnexus_index - 1])
assert stat.S_IMODE(gitnexus_wrapper.stat().st_mode) == 0o500
@ -883,7 +857,7 @@ def test_clone_controlled_mcp_replacement_is_never_executed_or_credentialed(tmp_
os.environ.get("GITNEXUS_REQUIRE_CLAUDE_CANARY") != "1",
reason="real Claude/Bash/MCP canary is mandatory in the named Ubuntu CI job",
)
def test_real_claude_hooks_auth_inner_sandbox_and_mcp_permissions(tmp_path: Path) -> None:
def test_real_claude_auth_inner_sandbox_and_mcp_permissions(tmp_path: Path) -> None:
"""Exercise the exact CLI boundary without contacting a paid model."""
claude = Path(os.environ["CLAUDE_CANARY_BIN"]).resolve()

7
eval/workflow_bench/README.md
View file

@ -258,7 +258,12 @@ loser rows, their session transcripts and patches, the rejected
`proposal.md` when available (including a workflow seed from a prior run), and
the learning queue,
then writes ONE bounded candidate overlay plus a reviewer-facing
`proposal.md`. The overlay is re-validated by `candidate_overlay_files`
`proposal.md`. The proposer's clone is sanitized exactly like an arm's before
its session starts: it authors the artifact the arms are scored with, so
letting it read `eval/workflow_bench` would hand it the task prompts and the
hidden oracles it is about to be graded against, and a proposal could win the
gate by encoding the expected behavior into a skill rather than by being a
better skill. The overlay is re-validated by `candidate_overlay_files`
(same boundary: Markdown under the plan/work trees, nothing else), frozen,
and exercised only by its exact required pairs. Task refs are resolved once
before generation zero and the immutable task bindings are forwarded to every

18
eval/workflow_bench/evolve.py
View file

@ -66,7 +66,7 @@ from .evolution import (
candidate_overlay_files,
required_candidate_arms,
)
from .oracle_assets import MAX_CLONE_REFS
from .oracle_assets import MAX_CLONE_REFS, sanitize_clone_for_hidden_oracles
from .promotion_apply import (
apply_promoted_overlay as apply_promoted_overlay,
committed_destination_base_digests as committed_destination_base_digests,
@ -639,7 +639,7 @@ def stage_proposer_evidence_bundle(
# The proposer's exact tool surface. Read/Grep/Glob observe the read-only
# evidence bundle and the incumbent skills; Bash writes the candidate overlay.
# `--tools` restricts non-bare Claude to this list, so Write/Edit/Skill/Web are
# unavailable while the trusted PreToolUse normalizer remains enabled. Settings
# unavailable, and Grep/Glob stay available (--bare would drop them). Settings
# pre-authorize Bash via autoAllowBashIfSandboxed, and the sandbox filesystem
# policy confines writes to workspace/tmp/home. Exported so containment tests
# exercise the production allowlist without drift.
@ -662,6 +662,14 @@ def run_proposer(
clone = runner.make_worktree(REPO_ROOT, "HEAD", Path(tmp))
primary: BaseException | None = None
try:
# The proposer authors the skill overlay that the arms are then
# scored with, so it must not see what it is scored against. Its
# clone carries eval/workflow_bench — the task prompts and the
# hidden oracles — which would let a proposal encode the expected
# behavior directly into a skill and win the gate without the
# skill being any better. Strip it from the working tree and from
# recoverable history exactly as the benchmark arms do.
sanitize_clone_for_hidden_oracles(clone)
output_root = clone / ".wfbench-output"
output_root.mkdir(mode=0o700)
internal_overlay = output_root / "overlay"
@ -693,9 +701,9 @@ def run_proposer(
# No permission_mode: CLAUDE_CODE_SUBPROCESS_ENV_SCRUB
# forces "default", so requesting dontAsk only warns. Tools
# are pre-approved via settings permissions.allow
# (proposer_sandbox.build_claude_settings). Do not use
# Claude's --bare flag here: it disables that trusted
# PreToolUse hook along with untrusted hooks/plugins.
# (proposer_sandbox.build_claude_settings). Not --bare:
# bare ignores --tools and imposes its own Bash/Edit/Read
# ceiling, which would cost the proposer Grep and Glob.
command_prefix=sandbox.command_prefix,
require_pid_namespace=True,
settings_json=sandbox.settings_json,

57
eval/workflow_bench/proposer_sandbox.py
View file

@ -28,7 +28,6 @@ SANDBOX_CLAUDE = "/opt/claude/claude"
SANDBOX_SHELL_PREFIX = "/opt/claude/shell-prefix"
SANDBOX_PYTHON3 = "/opt/claude/python3"
SANDBOX_GITNEXUS_CLI = "/opt/claude/gitnexus"
SANDBOX_TOOL_INPUT_NORMALIZER = "/opt/claude/normalize-tool-input"
SANDBOX_GIT_EXCLUDES = "/opt/claude/git-excludes"
SANDBOX_NODE = "/opt/claude/node"
SANDBOX_NODE_PREFIX = "/opt/claude/nodejs"
@ -328,7 +327,15 @@ def build_sandbox_environment(
def build_claude_settings() -> str:
"""Inline settings: one trusted normalizer hook; every Bash stays sandboxed."""
"""Inline settings that keep every Bash sandboxed and pre-approve the tools.
Deliberately hook-free: headless ``claude -p`` (2.1.247) never dispatches
``PreToolUse``, whatever source the hook is declared in — inline
``--settings``, a settings file, project/user/local ``--setting-sources``,
or a trusted project entry in ``~/.claude.json``. Confinement therefore
rests only on mechanisms the CLI honors in this mode: the sandbox policy
below, ``--tools``/``--allowedTools``, and the bwrap mounts.
"""
settings = {
"sandbox": {
@ -362,20 +369,6 @@ def build_claude_settings() -> str:
],
},
},
"hooks": {
"PreToolUse": [
{
"matcher": r"Read|mcp__gitnexus__.*",
"hooks": [
{
"type": "command",
"command": SANDBOX_TOOL_INPUT_NORMALIZER,
"timeout": 5,
}
],
}
]
},
"permissions": {
# CLAUDE_CODE_SUBPROCESS_ENV_SCRUB forces permission mode to
# "default" (allowed_non_write_users hardening), so requesting a
@ -509,36 +502,6 @@ def _create_gitnexus_wrapper(private_root: Path) -> Path:
return wrapper
def _create_tool_input_normalizer(private_root: Path) -> Path:
"""Create the trusted PreToolUse adapter for strict provider defaults."""
wrapper = private_root / "normalize-tool-input"
wrapper.write_text(
f"#!{SANDBOX_PYTHON3}\n"
"import json, sys\n"
"try:\n"
" event = json.load(sys.stdin)\n"
" tool_input = event.get('tool_input')\n"
" if not isinstance(tool_input, dict):\n"
" raise ValueError('tool_input is not an object')\n"
" normalized = {\n"
" key: value for key, value in tool_input.items()\n"
" if not (isinstance(value, str) and not value.strip())\n"
" }\n"
" if normalized != tool_input:\n"
" print(json.dumps({'hookSpecificOutput': {\n"
" 'hookEventName': 'PreToolUse',\n"
" 'permissionDecision': 'allow',\n"
" 'updatedInput': normalized,\n"
" }}, separators=(',', ':')))\n"
"except Exception as exc:\n"
" print(f'tool-input normalizer failed closed: {exc}', file=sys.stderr)\n"
" sys.exit(2)\n"
)
wrapper.chmod(0o500)
return wrapper
def _create_git_excludes(private_root: Path) -> Path:
"""Create the immutable excludes for nested-sandbox mount artifacts."""
@ -821,7 +784,6 @@ def prepare_sandbox(
shell_prefix = _create_shell_prefix_wrapper(private_root)
python3_wrapper = _create_python3_wrapper(private_root)
gitnexus_wrapper = _create_gitnexus_wrapper(private_root)
tool_input_normalizer = _create_tool_input_normalizer(private_root)
git_excludes = _create_git_excludes(private_root)
# Claude may discover user-level skills below HOME. Keep the rest of HOME
# writable for normal CLI state, but overlay an immutable empty skills root
@ -835,7 +797,6 @@ def prepare_sandbox(
ReadOnlyMount(source=shell_prefix, target=SANDBOX_SHELL_PREFIX),
ReadOnlyMount(source=python3_wrapper, target=SANDBOX_PYTHON3),
ReadOnlyMount(source=gitnexus_wrapper, target=SANDBOX_GITNEXUS_CLI),
ReadOnlyMount(source=tool_input_normalizer, target=SANDBOX_TOOL_INPUT_NORMALIZER),
ReadOnlyMount(source=git_excludes, target=SANDBOX_GIT_EXCLUDES),
)
primary: BaseException | None = None