fix(claude_code): pin daily compat run to the 3-day-buffered claude-code version

PR #32548 deleted the CircleCI claude_code_compat_pr_gate job, the only
automated consumer of pr_gate_version_resolver.py, so the 3-day npm
publish-age security buffer (PRD #26476) stopped gating any automated
path; run_daily.sh just probed whatever claude was on the cron VM's
PATH. The daily runner now resolves the buffered version with the
resolver, npm-installs exactly that version into a per-run prefix under
WORKDIR (both steps under the script's env -i credential scrub with the
isolated per-run HOME), prepends the install's bin dir to PATH, and
turns the existing version probe into a verification that dies on a
mismatch with the resolved version.
This commit is contained in:
mateo-berri 2026-07-15 16:30:08 -07:00
parent c6d49a85b2
commit 8220c57468
4 changed files with 314 additions and 30 deletions

View file

@ -0,0 +1,256 @@
"""Pin: run_daily.sh must install the resolver-selected claude-code version.
PR #32548 deleted the CircleCI job `claude_code_compat_pr_gate`, which
was the only automated consumer of `pr_gate_version_resolver.py` (newest
`@anthropic-ai/claude-code` npm version published at least 3 days ago; a
security-review buffer, PRD #26476). The daily runner used to probe
whatever `claude` happened to be on the cron VM's PATH, so nothing
automated enforced the buffer anymore. These tests pin the replacement
flow in run_daily.sh:
1. The target version comes from running `pr_gate_version_resolver.py`
and exactly `@anthropic-ai/claude-code@<resolved>` is npm-installed
into a per-run prefix under `${WORKDIR}`.
2. Both the resolver and the npm install run under the same `env -i`
credential scrub (with the isolated per-run HOME) as the probe and
pytest steps: npm postinstall runs package code, which is exactly
the supply-chain vector the 3-day buffer exists for.
3. The `claude --version` probe verifies the binary now on PATH
reports exactly the resolved version and dies on a mismatch.
The resolve/install/probe block is extracted out of run_daily.sh and
executed with a stub resolver and a fake `npm`, mirroring the fake-curl
technique in `test_run_daily_release_pagination.py`.
"""
from __future__ import annotations
import json
import os
import subprocess
import textwrap
from dataclasses import dataclass
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[4]
RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh"
RESOLVED_VERSION = "0.0.99"
_PREAMBLE = (
"set -Eeuo pipefail\n"
"log() { printf '==> %s\\n' \"$*\" >&2; }\n"
"die() { printf 'ERROR: %s\\n' \"$*\" >&2; exit 1; }\n"
)
_SECRET_ENV = {
"ANTHROPIC_API_KEY": "test-secret-anthropic",
"AWS_BEARER_TOKEN_BEDROCK": "test-secret-bedrock",
"AZURE_FOUNDRY_API_KEY": "test-secret-foundry",
"AGENT_SHIN_GITHUB_TOKEN": "test-secret-agent-shin",
"GITHUB_TOKEN": "test-secret-github",
}
def _extract_pin_snippet() -> str:
body = RUN_DAILY.read_text()
start = body.index('CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home"')
end = body.index('log "pinned claude code:')
return body[start:end]
def _executable_lines(block: str) -> str:
return "\n".join(
line for line in block.splitlines() if line.lstrip()[:1] != "#"
)
@dataclass(frozen=True, slots=True)
class PinHarness:
workdir: Path
resolver_env_json: Path
npm_env_txt: Path
npm_argv_txt: Path
script: str
env: dict[str, str]
def run(self) -> "subprocess.CompletedProcess[str]":
return subprocess.run(
["bash", "-c", self.script],
capture_output=True,
text=True,
env=self.env,
)
def _build_harness(tmp_path: Path, installed_version: str) -> PinHarness:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
dumps = tmp_path / "dumps"
dumps.mkdir()
workdir = tmp_path / "work"
workdir.mkdir()
populator_dir = tmp_path / "claude_code" / "cron_vm"
populator_dir.mkdir(parents=True)
resolver_env_json = dumps / "resolver_env.json"
resolver = tmp_path / "claude_code" / "pr_gate_version_resolver.py"
resolver.write_text(
textwrap.dedent(
f"""\
import json
import os
with open({str(resolver_env_json)!r}, "w") as fh:
json.dump(dict(os.environ), fh)
print({RESOLVED_VERSION!r})
"""
)
)
npm_env_txt = dumps / "npm_env.txt"
npm_argv_txt = dumps / "npm_argv.txt"
fake_npm = fake_bin / "npm"
fake_npm.write_text(
textwrap.dedent(
f"""\
#!/usr/bin/env bash
printf '%s\\n' "$@" > "{npm_argv_txt}"
env > "{npm_env_txt}"
prefix=""
pkg=""
while [[ $# -gt 0 ]]; do
case "$1" in
--prefix) prefix="$2"; shift 2 ;;
install) shift ;;
*) pkg="$1"; shift ;;
esac
done
mkdir -p "${{prefix}}/node_modules/.bin"
cat > "${{prefix}}/node_modules/.bin/claude" <<'CLAUDE_EOF'
#!/usr/bin/env bash
echo "{installed_version} (Claude Code)"
CLAUDE_EOF
chmod +x "${{prefix}}/node_modules/.bin/claude"
"""
)
)
fake_npm.chmod(0o755)
script = (
_PREAMBLE
+ f'WORKDIR="{workdir}"\n'
+ f'POPULATOR_DIR="{populator_dir}"\n'
+ _extract_pin_snippet()
+ 'printf "%s" "${CLAUDE_CODE_VERSION}"\n'
)
env = {
**os.environ,
**_SECRET_ENV,
"PATH": f"{fake_bin}:{os.environ.get('PATH', '')}",
}
return PinHarness(
workdir=workdir,
resolver_env_json=resolver_env_json,
npm_env_txt=npm_env_txt,
npm_argv_txt=npm_argv_txt,
script=script,
env=env,
)
def test_run_daily_installs_exactly_the_resolver_selected_version(
tmp_path: Path,
) -> None:
harness = _build_harness(tmp_path, installed_version=RESOLVED_VERSION)
result = harness.run()
assert result.returncode == 0, (
f"pin snippet failed: stderr={result.stderr!r}"
)
assert result.stdout == RESOLVED_VERSION, (
f"CLAUDE_CODE_VERSION must be the resolver's output, got "
f"{result.stdout!r}"
)
argv = harness.npm_argv_txt.read_text().splitlines()
assert f"@anthropic-ai/claude-code@{RESOLVED_VERSION}" in argv, (
f"run_daily.sh must `npm install` the exact resolver-selected "
f"version; npm argv was {argv!r}"
)
assert "install" in argv
prefix = argv[argv.index("--prefix") + 1]
assert Path(prefix) == harness.workdir / "claude-cli", (
"the per-run install prefix must live inside ${WORKDIR} so the "
"cleanup trap removes it"
)
assert (
harness.workdir / "claude-cli" / "node_modules" / ".bin" / "claude"
).exists()
def test_resolver_and_npm_install_run_under_env_scrub(tmp_path: Path) -> None:
harness = _build_harness(tmp_path, installed_version=RESOLVED_VERSION)
result = harness.run()
assert result.returncode == 0, (
f"pin snippet failed: stderr={result.stderr!r}"
)
resolver_env = json.loads(harness.resolver_env_json.read_text())
npm_env = dict(
line.split("=", 1)
for line in harness.npm_env_txt.read_text().splitlines()
if "=" in line
)
probe_home = str(harness.workdir / "claude-probe-home")
for name, seen_env in (("resolver", resolver_env), ("npm", npm_env)):
for secret in _SECRET_ENV:
assert secret not in seen_env, (
f"run_daily.sh: the {name} step leaked {secret} through "
f"its `env -i` scrub. npm postinstall runs package code, "
f"which is exactly the supply-chain vector the 3-day "
f"buffer exists for."
)
assert seen_env.get("HOME") == probe_home, (
f"run_daily.sh: the {name} step must run under the isolated "
f"per-run HOME, not the runtime user's; got "
f"{seen_env.get('HOME')!r}"
)
def test_probe_dies_when_installed_version_mismatches_resolved(
tmp_path: Path,
) -> None:
harness = _build_harness(tmp_path, installed_version="9.9.9")
result = harness.run()
assert result.returncode != 0, (
"run_daily.sh must die when the claude binary on PATH does not "
"report the resolver-selected version; a silent pass here means "
"the 3-day security buffer is not actually enforced."
)
assert "9.9.9" in result.stderr
assert RESOLVED_VERSION in result.stderr
def test_static_resolver_and_install_are_env_i_wrapped() -> None:
body = RUN_DAILY.read_text()
lines = _executable_lines(_extract_pin_snippet())
resolver_call = lines.index("pr_gate_version_resolver.py")
assert "env -i" in lines[:resolver_call], (
"run_daily.sh must run pr_gate_version_resolver.py under `env -i`"
)
install_call = lines.index('@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}')
assert "env -i" in lines[resolver_call:install_call], (
"run_daily.sh must run the npm install under its own `env -i`"
)
assert "claude" not in _required_commands(body), (
"run_daily.sh must not require a pre-provisioned `claude` on "
"PATH anymore; the pinned install provides it"
)
for required in ("npm", "python3"):
assert required in _required_commands(body)
def _required_commands(body: str) -> tuple[str, ...]:
marker = "for cmd in "
start = body.index(marker) + len(marker)
end = body.index(";", start)
return tuple(body[start:end].split())

View file

@ -10,7 +10,7 @@ compromised `@anthropic-ai/claude-code` release could read those
secrets out of `os.environ` before the proxy or test harness ever
starts. The version probe must be wrapped in `env -i` with a minimal
PATH/HOME/USER/TERM/LANG/LC_ALL/TMPDIR allowlist matching the
PR-gate's resolver/npm-install/pytest scrubs.
resolver/npm-install/pytest scrubs in the same script.
"""
from __future__ import annotations
@ -23,8 +23,8 @@ RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily
def _version_probe_block() -> str:
body = RUN_DAILY.read_text()
start = body.index("CLAUDE_CODE_VERSION=")
end = body.index('[[ -n "${CLAUDE_CODE_VERSION}" ]]', start)
start = body.index("PROBED_CLAUDE_VERSION=")
end = body.index('[[ -n "${PROBED_CLAUDE_VERSION}" ]]', start)
return body[start:end]

View file

@ -22,7 +22,7 @@
# rather than spawning a new one. If the JSON is byte-identical to the
# docs branch, we skip the push entirely.
#
# Required commands on $PATH: git, uv, gh, jq, curl, claude.
# Required commands on $PATH: git, uv, gh, jq, curl, npm, python3.
# Required state: ~/litellm/litellm checked out (this file lives in it),
# $WORKTREE is created on first run, gh is already authenticated.
#
@ -90,7 +90,7 @@ trap cleanup EXIT INT TERM
log() { printf '==> %s\n' "$*" >&2; }
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
for cmd in git uv gh jq curl claude; do
for cmd in git uv gh jq curl npm python3; do
command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}"
done
@ -165,26 +165,52 @@ log "resolved litellm: ${LITELLM_VERSION}"
# The systemd unit loads provider credentials and the agent-shin GitHub
# token from /etc/litellm-compat-matrix.env into this script's
# environment. Running the npm-installed `claude` binary directly here
# would hand that full env to package code -- a compromised
# @anthropic-ai/claude-code release could read ANTHROPIC_API_KEY /
# AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY /
# environment. Running npm install or the npm-installed `claude` binary
# directly here would hand that full env to package code -- a
# compromised @anthropic-ai/claude-code release could read
# ANTHROPIC_API_KEY / AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY /
# AGENT_SHIN_GITHUB_TOKEN from os.environ and exfiltrate them before
# the proxy or test harness ever starts. Probe under `env -i` with the
# same minimal allowlist the PR-gate uses (the matrix run itself goes
# through cli_driver.py, which already scrubs the CLI env).
# the proxy or test harness ever starts. Resolve, install, and probe
# under `env -i` with the same minimal allowlist the pytest step below
# uses (the matrix run itself goes through cli_driver.py, which already
# scrubs the CLI env).
#
# The probe also runs under a fresh empty HOME instead of the runtime
# These steps also run under a fresh empty HOME instead of the runtime
# user's real $HOME. `ProtectHome=read-only` in the systemd unit
# blocks *writes* to /home/mateo but still allows reads, so a
# compromised claude package invoked here with HOME=/home/mateo could
# read ~/.config/gh/hosts.yml (the gh-host token), ~/.bash_history,
# or ~/.ssh/. Pointing HOME at a per-run dir under ${WORKDIR} hides
# those entirely from the subprocess; ${WORKDIR} is rm -rf'd by the
# script-wide cleanup() trap regardless of probe outcome.
# script-wide cleanup() trap regardless of outcome.
CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home"
mkdir -p "${CLAUDE_PROBE_HOME}"
CLAUDE_CODE_VERSION="$(env -i \
PATH="${PATH}" \
HOME="${CLAUDE_PROBE_HOME}" \
USER="${USER:-mateo}" \
TERM="${TERM:-dumb}" \
LANG="${LANG:-C.UTF-8}" \
LC_ALL="${LC_ALL:-}" \
TMPDIR="${TMPDIR:-/tmp}" \
python3 "${POPULATOR_DIR}/../pr_gate_version_resolver.py")"
[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "pr_gate_version_resolver.py printed an empty version"
log "resolved claude code: ${CLAUDE_CODE_VERSION}"
CLAUDE_CLI_PREFIX="${WORKDIR}/claude-cli"
env -i \
PATH="${PATH}" \
HOME="${CLAUDE_PROBE_HOME}" \
USER="${USER:-mateo}" \
TERM="${TERM:-dumb}" \
LANG="${LANG:-C.UTF-8}" \
LC_ALL="${LC_ALL:-}" \
TMPDIR="${TMPDIR:-/tmp}" \
npm install --prefix "${CLAUDE_CLI_PREFIX}" "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \
|| die "npm install of @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} failed"
PATH="${CLAUDE_CLI_PREFIX}/node_modules/.bin:${PATH}"
PROBED_CLAUDE_VERSION="$(env -i \
PATH="${PATH}" \
HOME="${CLAUDE_PROBE_HOME}" \
USER="${USER:-mateo}" \
@ -199,8 +225,10 @@ CLAUDE_CODE_VERSION="$(env -i \
# `grep` finds no match (exit 1) — without it the assignment inherits the
# pipeline's non-zero exit, `set -e` kills the script, and the operator
# never sees the helpful diagnostic below.
[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not parse semver from 'claude --version'"
log "local claude code: ${CLAUDE_CODE_VERSION}"
[[ -n "${PROBED_CLAUDE_VERSION}" ]] || die "could not parse semver from 'claude --version'"
[[ "${PROBED_CLAUDE_VERSION}" == "${CLAUDE_CODE_VERSION}" ]] \
|| die "claude on PATH reports ${PROBED_CLAUDE_VERSION}, expected pinned ${CLAUDE_CODE_VERSION}"
log "pinned claude code: ${CLAUDE_CODE_VERSION}"
# ---------------------------------------------------------------------------
# 2. Update the worktree to that tag
@ -385,9 +413,8 @@ set +e
# 2. a model-directed `Read` tool call during a PDF/vision cell
# cannot reach /proc/<pytest-pid>/environ and pull the creds out
# of the parent process the way it can today;
# 3. this matches the PR-gate pytest step in `.circleci/config.yml`,
# which already runs under `env -i` with the same minimal
# allowlist.
# 3. this matches the resolver/npm-install/probe steps above, which
# already run under `env -i` with the same minimal allowlist.
#
# `cli_driver.py` re-allowlists its own subset (PATH/USER/LOGNAME/etc.)
# when spawning the `claude` binary, so the CLI still finds Node + the

View file

@ -1,23 +1,24 @@
"""Claude Code PR-Gate Version Resolver.
Resolves the `@anthropic-ai/claude-code` npm version that the PR-gate CI
job installs. Selects the newest version (by publish timestamp) whose
publish timestamp is at least 3 days old. The 3-day window is a security
review buffer see PRD #26476, "Version resolvers".
Resolves the `@anthropic-ai/claude-code` npm version that the daily
compatibility-matrix runner installs. Selects the newest version (by
publish timestamp) whose publish timestamp is at least 3 days old. The
3-day window is a security review buffer see PRD #26476, "Version
resolvers".
Two surfaces:
- ``resolve_pr_gate_version(...)`` the importable function. Accepts
pre-fetched npm metadata (for unit tests) or a custom ``fetcher``
callable. The default fetcher hits the public npm registry.
- ``python -m claude_code.pr_gate_version_resolver`` prints the
resolved version string to stdout, suitable for piping into a shell
``$(...)`` substitution inside the CircleCI job.
- ``python3 pr_gate_version_resolver.py`` prints the resolved version
string to stdout, suitable for piping into a shell ``$(...)``
substitution.
The CLI form is what CircleCI runs at job start; engineers reading the
job log can see the selected version on a single line above the
``npm install -g`` step (acceptance criterion: "the selected Claude
Code version is logged in the CI output").
The CLI form is what ``cron_vm/run_daily.sh`` runs at the start of each
daily run; engineers reading the run log can see the selected version
on a single line above the ``npm install`` step (acceptance criterion:
"the selected Claude Code version is logged in the CI output").
"""
from __future__ import annotations