diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pins_resolver_version.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pins_resolver_version.py new file mode 100644 index 00000000000..15147bff121 --- /dev/null +++ b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_pins_resolver_version.py @@ -0,0 +1,256 @@ +"""Pin: run_daily.sh must install the resolver-selected claude-code version. + +PR #32548 deleted the CircleCI job `claude_code_compat_pr_gate`, which +was the only automated consumer of `pr_gate_version_resolver.py` (newest +`@anthropic-ai/claude-code` npm version published at least 3 days ago; a +security-review buffer, PRD #26476). The daily runner used to probe +whatever `claude` happened to be on the cron VM's PATH, so nothing +automated enforced the buffer anymore. These tests pin the replacement +flow in run_daily.sh: + + 1. The target version comes from running `pr_gate_version_resolver.py` + and exactly `@anthropic-ai/claude-code@` is npm-installed + into a per-run prefix under `${WORKDIR}`. + 2. Both the resolver and the npm install run under the same `env -i` + credential scrub (with the isolated per-run HOME) as the probe and + pytest steps: npm postinstall runs package code, which is exactly + the supply-chain vector the 3-day buffer exists for. + 3. The `claude --version` probe verifies the binary now on PATH + reports exactly the resolved version and dies on a mismatch. + +The resolve/install/probe block is extracted out of run_daily.sh and +executed with a stub resolver and a fake `npm`, mirroring the fake-curl +technique in `test_run_daily_release_pagination.py`. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import textwrap +from dataclasses import dataclass +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[4] +RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh" + +RESOLVED_VERSION = "0.0.99" + +_PREAMBLE = ( + "set -Eeuo pipefail\n" + "log() { printf '==> %s\\n' \"$*\" >&2; }\n" + "die() { printf 'ERROR: %s\\n' \"$*\" >&2; exit 1; }\n" +) + +_SECRET_ENV = { + "ANTHROPIC_API_KEY": "test-secret-anthropic", + "AWS_BEARER_TOKEN_BEDROCK": "test-secret-bedrock", + "AZURE_FOUNDRY_API_KEY": "test-secret-foundry", + "AGENT_SHIN_GITHUB_TOKEN": "test-secret-agent-shin", + "GITHUB_TOKEN": "test-secret-github", +} + + +def _extract_pin_snippet() -> str: + body = RUN_DAILY.read_text() + start = body.index('CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home"') + end = body.index('log "pinned claude code:') + return body[start:end] + + +def _executable_lines(block: str) -> str: + return "\n".join( + line for line in block.splitlines() if line.lstrip()[:1] != "#" + ) + + +@dataclass(frozen=True, slots=True) +class PinHarness: + workdir: Path + resolver_env_json: Path + npm_env_txt: Path + npm_argv_txt: Path + script: str + env: dict[str, str] + + def run(self) -> "subprocess.CompletedProcess[str]": + return subprocess.run( + ["bash", "-c", self.script], + capture_output=True, + text=True, + env=self.env, + ) + + +def _build_harness(tmp_path: Path, installed_version: str) -> PinHarness: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + dumps = tmp_path / "dumps" + dumps.mkdir() + workdir = tmp_path / "work" + workdir.mkdir() + populator_dir = tmp_path / "claude_code" / "cron_vm" + populator_dir.mkdir(parents=True) + + resolver_env_json = dumps / "resolver_env.json" + resolver = tmp_path / "claude_code" / "pr_gate_version_resolver.py" + resolver.write_text( + textwrap.dedent( + f"""\ + import json + import os + + with open({str(resolver_env_json)!r}, "w") as fh: + json.dump(dict(os.environ), fh) + print({RESOLVED_VERSION!r}) + """ + ) + ) + + npm_env_txt = dumps / "npm_env.txt" + npm_argv_txt = dumps / "npm_argv.txt" + fake_npm = fake_bin / "npm" + fake_npm.write_text( + textwrap.dedent( + f"""\ + #!/usr/bin/env bash + printf '%s\\n' "$@" > "{npm_argv_txt}" + env > "{npm_env_txt}" + prefix="" + pkg="" + while [[ $# -gt 0 ]]; do + case "$1" in + --prefix) prefix="$2"; shift 2 ;; + install) shift ;; + *) pkg="$1"; shift ;; + esac + done + mkdir -p "${{prefix}}/node_modules/.bin" + cat > "${{prefix}}/node_modules/.bin/claude" <<'CLAUDE_EOF' + #!/usr/bin/env bash + echo "{installed_version} (Claude Code)" + CLAUDE_EOF + chmod +x "${{prefix}}/node_modules/.bin/claude" + """ + ) + ) + fake_npm.chmod(0o755) + + script = ( + _PREAMBLE + + f'WORKDIR="{workdir}"\n' + + f'POPULATOR_DIR="{populator_dir}"\n' + + _extract_pin_snippet() + + 'printf "%s" "${CLAUDE_CODE_VERSION}"\n' + ) + env = { + **os.environ, + **_SECRET_ENV, + "PATH": f"{fake_bin}:{os.environ.get('PATH', '')}", + } + return PinHarness( + workdir=workdir, + resolver_env_json=resolver_env_json, + npm_env_txt=npm_env_txt, + npm_argv_txt=npm_argv_txt, + script=script, + env=env, + ) + + +def test_run_daily_installs_exactly_the_resolver_selected_version( + tmp_path: Path, +) -> None: + harness = _build_harness(tmp_path, installed_version=RESOLVED_VERSION) + result = harness.run() + assert result.returncode == 0, ( + f"pin snippet failed: stderr={result.stderr!r}" + ) + assert result.stdout == RESOLVED_VERSION, ( + f"CLAUDE_CODE_VERSION must be the resolver's output, got " + f"{result.stdout!r}" + ) + argv = harness.npm_argv_txt.read_text().splitlines() + assert f"@anthropic-ai/claude-code@{RESOLVED_VERSION}" in argv, ( + f"run_daily.sh must `npm install` the exact resolver-selected " + f"version; npm argv was {argv!r}" + ) + assert "install" in argv + prefix = argv[argv.index("--prefix") + 1] + assert Path(prefix) == harness.workdir / "claude-cli", ( + "the per-run install prefix must live inside ${WORKDIR} so the " + "cleanup trap removes it" + ) + assert ( + harness.workdir / "claude-cli" / "node_modules" / ".bin" / "claude" + ).exists() + + +def test_resolver_and_npm_install_run_under_env_scrub(tmp_path: Path) -> None: + harness = _build_harness(tmp_path, installed_version=RESOLVED_VERSION) + result = harness.run() + assert result.returncode == 0, ( + f"pin snippet failed: stderr={result.stderr!r}" + ) + + resolver_env = json.loads(harness.resolver_env_json.read_text()) + npm_env = dict( + line.split("=", 1) + for line in harness.npm_env_txt.read_text().splitlines() + if "=" in line + ) + probe_home = str(harness.workdir / "claude-probe-home") + for name, seen_env in (("resolver", resolver_env), ("npm", npm_env)): + for secret in _SECRET_ENV: + assert secret not in seen_env, ( + f"run_daily.sh: the {name} step leaked {secret} through " + f"its `env -i` scrub. npm postinstall runs package code, " + f"which is exactly the supply-chain vector the 3-day " + f"buffer exists for." + ) + assert seen_env.get("HOME") == probe_home, ( + f"run_daily.sh: the {name} step must run under the isolated " + f"per-run HOME, not the runtime user's; got " + f"{seen_env.get('HOME')!r}" + ) + + +def test_probe_dies_when_installed_version_mismatches_resolved( + tmp_path: Path, +) -> None: + harness = _build_harness(tmp_path, installed_version="9.9.9") + result = harness.run() + assert result.returncode != 0, ( + "run_daily.sh must die when the claude binary on PATH does not " + "report the resolver-selected version; a silent pass here means " + "the 3-day security buffer is not actually enforced." + ) + assert "9.9.9" in result.stderr + assert RESOLVED_VERSION in result.stderr + + +def test_static_resolver_and_install_are_env_i_wrapped() -> None: + body = RUN_DAILY.read_text() + lines = _executable_lines(_extract_pin_snippet()) + resolver_call = lines.index("pr_gate_version_resolver.py") + assert "env -i" in lines[:resolver_call], ( + "run_daily.sh must run pr_gate_version_resolver.py under `env -i`" + ) + install_call = lines.index('@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}') + assert "env -i" in lines[resolver_call:install_call], ( + "run_daily.sh must run the npm install under its own `env -i`" + ) + assert "claude" not in _required_commands(body), ( + "run_daily.sh must not require a pre-provisioned `claude` on " + "PATH anymore; the pinned install provides it" + ) + for required in ("npm", "python3"): + assert required in _required_commands(body) + + +def _required_commands(body: str) -> tuple[str, ...]: + marker = "for cmd in " + start = body.index(marker) + len(marker) + end = body.index(";", start) + return tuple(body[start:end].split()) diff --git a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py index 1c3959764f3..203972b9e94 100644 --- a/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py +++ b/tests/e2e/claude_code/_publisher_unit_tests/test_run_daily_version_probe_scrubs_env.py @@ -10,7 +10,7 @@ compromised `@anthropic-ai/claude-code` release could read those secrets out of `os.environ` before the proxy or test harness ever starts. The version probe must be wrapped in `env -i` with a minimal PATH/HOME/USER/TERM/LANG/LC_ALL/TMPDIR allowlist — matching the -PR-gate's resolver/npm-install/pytest scrubs. +resolver/npm-install/pytest scrubs in the same script. """ from __future__ import annotations @@ -23,8 +23,8 @@ RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily def _version_probe_block() -> str: body = RUN_DAILY.read_text() - start = body.index("CLAUDE_CODE_VERSION=") - end = body.index('[[ -n "${CLAUDE_CODE_VERSION}" ]]', start) + start = body.index("PROBED_CLAUDE_VERSION=") + end = body.index('[[ -n "${PROBED_CLAUDE_VERSION}" ]]', start) return body[start:end] diff --git a/tests/e2e/claude_code/cron_vm/run_daily.sh b/tests/e2e/claude_code/cron_vm/run_daily.sh index ae2d67c070c..25545b8b797 100755 --- a/tests/e2e/claude_code/cron_vm/run_daily.sh +++ b/tests/e2e/claude_code/cron_vm/run_daily.sh @@ -22,7 +22,7 @@ # rather than spawning a new one. If the JSON is byte-identical to the # docs branch, we skip the push entirely. # -# Required commands on $PATH: git, uv, gh, jq, curl, claude. +# Required commands on $PATH: git, uv, gh, jq, curl, npm, python3. # Required state: ~/litellm/litellm checked out (this file lives in it), # $WORKTREE is created on first run, gh is already authenticated. # @@ -90,7 +90,7 @@ trap cleanup EXIT INT TERM log() { printf '==> %s\n' "$*" >&2; } die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } -for cmd in git uv gh jq curl claude; do +for cmd in git uv gh jq curl npm python3; do command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}" done @@ -165,26 +165,52 @@ log "resolved litellm: ${LITELLM_VERSION}" # The systemd unit loads provider credentials and the agent-shin GitHub # token from /etc/litellm-compat-matrix.env into this script's -# environment. Running the npm-installed `claude` binary directly here -# would hand that full env to package code -- a compromised -# @anthropic-ai/claude-code release could read ANTHROPIC_API_KEY / -# AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY / +# environment. Running npm install or the npm-installed `claude` binary +# directly here would hand that full env to package code -- a +# compromised @anthropic-ai/claude-code release could read +# ANTHROPIC_API_KEY / AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY / # AGENT_SHIN_GITHUB_TOKEN from os.environ and exfiltrate them before -# the proxy or test harness ever starts. Probe under `env -i` with the -# same minimal allowlist the PR-gate uses (the matrix run itself goes -# through cli_driver.py, which already scrubs the CLI env). +# the proxy or test harness ever starts. Resolve, install, and probe +# under `env -i` with the same minimal allowlist the pytest step below +# uses (the matrix run itself goes through cli_driver.py, which already +# scrubs the CLI env). # -# The probe also runs under a fresh empty HOME instead of the runtime +# These steps also run under a fresh empty HOME instead of the runtime # user's real $HOME. `ProtectHome=read-only` in the systemd unit # blocks *writes* to /home/mateo but still allows reads, so a # compromised claude package invoked here with HOME=/home/mateo could # read ~/.config/gh/hosts.yml (the gh-host token), ~/.bash_history, # or ~/.ssh/. Pointing HOME at a per-run dir under ${WORKDIR} hides # those entirely from the subprocess; ${WORKDIR} is rm -rf'd by the -# script-wide cleanup() trap regardless of probe outcome. +# script-wide cleanup() trap regardless of outcome. CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home" mkdir -p "${CLAUDE_PROBE_HOME}" CLAUDE_CODE_VERSION="$(env -i \ + PATH="${PATH}" \ + HOME="${CLAUDE_PROBE_HOME}" \ + USER="${USER:-mateo}" \ + TERM="${TERM:-dumb}" \ + LANG="${LANG:-C.UTF-8}" \ + LC_ALL="${LC_ALL:-}" \ + TMPDIR="${TMPDIR:-/tmp}" \ + python3 "${POPULATOR_DIR}/../pr_gate_version_resolver.py")" +[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "pr_gate_version_resolver.py printed an empty version" +log "resolved claude code: ${CLAUDE_CODE_VERSION}" + +CLAUDE_CLI_PREFIX="${WORKDIR}/claude-cli" +env -i \ + PATH="${PATH}" \ + HOME="${CLAUDE_PROBE_HOME}" \ + USER="${USER:-mateo}" \ + TERM="${TERM:-dumb}" \ + LANG="${LANG:-C.UTF-8}" \ + LC_ALL="${LC_ALL:-}" \ + TMPDIR="${TMPDIR:-/tmp}" \ + npm install --prefix "${CLAUDE_CLI_PREFIX}" "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \ + || die "npm install of @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} failed" +PATH="${CLAUDE_CLI_PREFIX}/node_modules/.bin:${PATH}" + +PROBED_CLAUDE_VERSION="$(env -i \ PATH="${PATH}" \ HOME="${CLAUDE_PROBE_HOME}" \ USER="${USER:-mateo}" \ @@ -199,8 +225,10 @@ CLAUDE_CODE_VERSION="$(env -i \ # `grep` finds no match (exit 1) — without it the assignment inherits the # pipeline's non-zero exit, `set -e` kills the script, and the operator # never sees the helpful diagnostic below. -[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not parse semver from 'claude --version'" -log "local claude code: ${CLAUDE_CODE_VERSION}" +[[ -n "${PROBED_CLAUDE_VERSION}" ]] || die "could not parse semver from 'claude --version'" +[[ "${PROBED_CLAUDE_VERSION}" == "${CLAUDE_CODE_VERSION}" ]] \ + || die "claude on PATH reports ${PROBED_CLAUDE_VERSION}, expected pinned ${CLAUDE_CODE_VERSION}" +log "pinned claude code: ${CLAUDE_CODE_VERSION}" # --------------------------------------------------------------------------- # 2. Update the worktree to that tag @@ -385,9 +413,8 @@ set +e # 2. a model-directed `Read` tool call during a PDF/vision cell # cannot reach /proc//environ and pull the creds out # of the parent process the way it can today; -# 3. this matches the PR-gate pytest step in `.circleci/config.yml`, -# which already runs under `env -i` with the same minimal -# allowlist. +# 3. this matches the resolver/npm-install/probe steps above, which +# already run under `env -i` with the same minimal allowlist. # # `cli_driver.py` re-allowlists its own subset (PATH/USER/LOGNAME/etc.) # when spawning the `claude` binary, so the CLI still finds Node + the diff --git a/tests/e2e/claude_code/pr_gate_version_resolver.py b/tests/e2e/claude_code/pr_gate_version_resolver.py index 82e12a2bf15..64eb5427d96 100644 --- a/tests/e2e/claude_code/pr_gate_version_resolver.py +++ b/tests/e2e/claude_code/pr_gate_version_resolver.py @@ -1,23 +1,24 @@ """Claude Code PR-Gate Version Resolver. -Resolves the `@anthropic-ai/claude-code` npm version that the PR-gate CI -job installs. Selects the newest version (by publish timestamp) whose -publish timestamp is at least 3 days old. The 3-day window is a security -review buffer — see PRD #26476, "Version resolvers". +Resolves the `@anthropic-ai/claude-code` npm version that the daily +compatibility-matrix runner installs. Selects the newest version (by +publish timestamp) whose publish timestamp is at least 3 days old. The +3-day window is a security review buffer — see PRD #26476, "Version +resolvers". Two surfaces: - ``resolve_pr_gate_version(...)`` — the importable function. Accepts pre-fetched npm metadata (for unit tests) or a custom ``fetcher`` callable. The default fetcher hits the public npm registry. -- ``python -m claude_code.pr_gate_version_resolver`` — prints the - resolved version string to stdout, suitable for piping into a shell - ``$(...)`` substitution inside the CircleCI job. +- ``python3 pr_gate_version_resolver.py`` — prints the resolved version + string to stdout, suitable for piping into a shell ``$(...)`` + substitution. -The CLI form is what CircleCI runs at job start; engineers reading the -job log can see the selected version on a single line above the -``npm install -g`` step (acceptance criterion: "the selected Claude -Code version is logged in the CI output"). +The CLI form is what ``cron_vm/run_daily.sh`` runs at the start of each +daily run; engineers reading the run log can see the selected version +on a single line above the ``npm install`` step (acceptance criterion: +"the selected Claude Code version is logged in the CI output"). """ from __future__ import annotations