mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(claude_code): pin daily compat run to the 3-day-buffered claude-code version
PR #32548 deleted the CircleCI claude_code_compat_pr_gate job, the only automated consumer of pr_gate_version_resolver.py, so the 3-day npm publish-age security buffer (PRD #26476) stopped gating any automated path; run_daily.sh just probed whatever claude was on the cron VM's PATH. The daily runner now resolves the buffered version with the resolver, npm-installs exactly that version into a per-run prefix under WORKDIR (both steps under the script's env -i credential scrub with the isolated per-run HOME), prepends the install's bin dir to PATH, and turns the existing version probe into a verification that dies on a mismatch with the resolved version.
This commit is contained in:
parent
c6d49a85b2
commit
8220c57468
4 changed files with 314 additions and 30 deletions
|
|
@ -0,0 +1,256 @@
|
|||
"""Pin: run_daily.sh must install the resolver-selected claude-code version.
|
||||
|
||||
PR #32548 deleted the CircleCI job `claude_code_compat_pr_gate`, which
|
||||
was the only automated consumer of `pr_gate_version_resolver.py` (newest
|
||||
`@anthropic-ai/claude-code` npm version published at least 3 days ago; a
|
||||
security-review buffer, PRD #26476). The daily runner used to probe
|
||||
whatever `claude` happened to be on the cron VM's PATH, so nothing
|
||||
automated enforced the buffer anymore. These tests pin the replacement
|
||||
flow in run_daily.sh:
|
||||
|
||||
1. The target version comes from running `pr_gate_version_resolver.py`
|
||||
and exactly `@anthropic-ai/claude-code@<resolved>` is npm-installed
|
||||
into a per-run prefix under `${WORKDIR}`.
|
||||
2. Both the resolver and the npm install run under the same `env -i`
|
||||
credential scrub (with the isolated per-run HOME) as the probe and
|
||||
pytest steps: npm postinstall runs package code, which is exactly
|
||||
the supply-chain vector the 3-day buffer exists for.
|
||||
3. The `claude --version` probe verifies the binary now on PATH
|
||||
reports exactly the resolved version and dies on a mismatch.
|
||||
|
||||
The resolve/install/probe block is extracted out of run_daily.sh and
|
||||
executed with a stub resolver and a fake `npm`, mirroring the fake-curl
|
||||
technique in `test_run_daily_release_pagination.py`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import textwrap
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily.sh"
|
||||
|
||||
RESOLVED_VERSION = "0.0.99"
|
||||
|
||||
_PREAMBLE = (
|
||||
"set -Eeuo pipefail\n"
|
||||
"log() { printf '==> %s\\n' \"$*\" >&2; }\n"
|
||||
"die() { printf 'ERROR: %s\\n' \"$*\" >&2; exit 1; }\n"
|
||||
)
|
||||
|
||||
_SECRET_ENV = {
|
||||
"ANTHROPIC_API_KEY": "test-secret-anthropic",
|
||||
"AWS_BEARER_TOKEN_BEDROCK": "test-secret-bedrock",
|
||||
"AZURE_FOUNDRY_API_KEY": "test-secret-foundry",
|
||||
"AGENT_SHIN_GITHUB_TOKEN": "test-secret-agent-shin",
|
||||
"GITHUB_TOKEN": "test-secret-github",
|
||||
}
|
||||
|
||||
|
||||
def _extract_pin_snippet() -> str:
|
||||
body = RUN_DAILY.read_text()
|
||||
start = body.index('CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home"')
|
||||
end = body.index('log "pinned claude code:')
|
||||
return body[start:end]
|
||||
|
||||
|
||||
def _executable_lines(block: str) -> str:
|
||||
return "\n".join(
|
||||
line for line in block.splitlines() if line.lstrip()[:1] != "#"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PinHarness:
|
||||
workdir: Path
|
||||
resolver_env_json: Path
|
||||
npm_env_txt: Path
|
||||
npm_argv_txt: Path
|
||||
script: str
|
||||
env: dict[str, str]
|
||||
|
||||
def run(self) -> "subprocess.CompletedProcess[str]":
|
||||
return subprocess.run(
|
||||
["bash", "-c", self.script],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=self.env,
|
||||
)
|
||||
|
||||
|
||||
def _build_harness(tmp_path: Path, installed_version: str) -> PinHarness:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
dumps = tmp_path / "dumps"
|
||||
dumps.mkdir()
|
||||
workdir = tmp_path / "work"
|
||||
workdir.mkdir()
|
||||
populator_dir = tmp_path / "claude_code" / "cron_vm"
|
||||
populator_dir.mkdir(parents=True)
|
||||
|
||||
resolver_env_json = dumps / "resolver_env.json"
|
||||
resolver = tmp_path / "claude_code" / "pr_gate_version_resolver.py"
|
||||
resolver.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
import json
|
||||
import os
|
||||
|
||||
with open({str(resolver_env_json)!r}, "w") as fh:
|
||||
json.dump(dict(os.environ), fh)
|
||||
print({RESOLVED_VERSION!r})
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
npm_env_txt = dumps / "npm_env.txt"
|
||||
npm_argv_txt = dumps / "npm_argv.txt"
|
||||
fake_npm = fake_bin / "npm"
|
||||
fake_npm.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\\n' "$@" > "{npm_argv_txt}"
|
||||
env > "{npm_env_txt}"
|
||||
prefix=""
|
||||
pkg=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--prefix) prefix="$2"; shift 2 ;;
|
||||
install) shift ;;
|
||||
*) pkg="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
mkdir -p "${{prefix}}/node_modules/.bin"
|
||||
cat > "${{prefix}}/node_modules/.bin/claude" <<'CLAUDE_EOF'
|
||||
#!/usr/bin/env bash
|
||||
echo "{installed_version} (Claude Code)"
|
||||
CLAUDE_EOF
|
||||
chmod +x "${{prefix}}/node_modules/.bin/claude"
|
||||
"""
|
||||
)
|
||||
)
|
||||
fake_npm.chmod(0o755)
|
||||
|
||||
script = (
|
||||
_PREAMBLE
|
||||
+ f'WORKDIR="{workdir}"\n'
|
||||
+ f'POPULATOR_DIR="{populator_dir}"\n'
|
||||
+ _extract_pin_snippet()
|
||||
+ 'printf "%s" "${CLAUDE_CODE_VERSION}"\n'
|
||||
)
|
||||
env = {
|
||||
**os.environ,
|
||||
**_SECRET_ENV,
|
||||
"PATH": f"{fake_bin}:{os.environ.get('PATH', '')}",
|
||||
}
|
||||
return PinHarness(
|
||||
workdir=workdir,
|
||||
resolver_env_json=resolver_env_json,
|
||||
npm_env_txt=npm_env_txt,
|
||||
npm_argv_txt=npm_argv_txt,
|
||||
script=script,
|
||||
env=env,
|
||||
)
|
||||
|
||||
|
||||
def test_run_daily_installs_exactly_the_resolver_selected_version(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
harness = _build_harness(tmp_path, installed_version=RESOLVED_VERSION)
|
||||
result = harness.run()
|
||||
assert result.returncode == 0, (
|
||||
f"pin snippet failed: stderr={result.stderr!r}"
|
||||
)
|
||||
assert result.stdout == RESOLVED_VERSION, (
|
||||
f"CLAUDE_CODE_VERSION must be the resolver's output, got "
|
||||
f"{result.stdout!r}"
|
||||
)
|
||||
argv = harness.npm_argv_txt.read_text().splitlines()
|
||||
assert f"@anthropic-ai/claude-code@{RESOLVED_VERSION}" in argv, (
|
||||
f"run_daily.sh must `npm install` the exact resolver-selected "
|
||||
f"version; npm argv was {argv!r}"
|
||||
)
|
||||
assert "install" in argv
|
||||
prefix = argv[argv.index("--prefix") + 1]
|
||||
assert Path(prefix) == harness.workdir / "claude-cli", (
|
||||
"the per-run install prefix must live inside ${WORKDIR} so the "
|
||||
"cleanup trap removes it"
|
||||
)
|
||||
assert (
|
||||
harness.workdir / "claude-cli" / "node_modules" / ".bin" / "claude"
|
||||
).exists()
|
||||
|
||||
|
||||
def test_resolver_and_npm_install_run_under_env_scrub(tmp_path: Path) -> None:
|
||||
harness = _build_harness(tmp_path, installed_version=RESOLVED_VERSION)
|
||||
result = harness.run()
|
||||
assert result.returncode == 0, (
|
||||
f"pin snippet failed: stderr={result.stderr!r}"
|
||||
)
|
||||
|
||||
resolver_env = json.loads(harness.resolver_env_json.read_text())
|
||||
npm_env = dict(
|
||||
line.split("=", 1)
|
||||
for line in harness.npm_env_txt.read_text().splitlines()
|
||||
if "=" in line
|
||||
)
|
||||
probe_home = str(harness.workdir / "claude-probe-home")
|
||||
for name, seen_env in (("resolver", resolver_env), ("npm", npm_env)):
|
||||
for secret in _SECRET_ENV:
|
||||
assert secret not in seen_env, (
|
||||
f"run_daily.sh: the {name} step leaked {secret} through "
|
||||
f"its `env -i` scrub. npm postinstall runs package code, "
|
||||
f"which is exactly the supply-chain vector the 3-day "
|
||||
f"buffer exists for."
|
||||
)
|
||||
assert seen_env.get("HOME") == probe_home, (
|
||||
f"run_daily.sh: the {name} step must run under the isolated "
|
||||
f"per-run HOME, not the runtime user's; got "
|
||||
f"{seen_env.get('HOME')!r}"
|
||||
)
|
||||
|
||||
|
||||
def test_probe_dies_when_installed_version_mismatches_resolved(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
harness = _build_harness(tmp_path, installed_version="9.9.9")
|
||||
result = harness.run()
|
||||
assert result.returncode != 0, (
|
||||
"run_daily.sh must die when the claude binary on PATH does not "
|
||||
"report the resolver-selected version; a silent pass here means "
|
||||
"the 3-day security buffer is not actually enforced."
|
||||
)
|
||||
assert "9.9.9" in result.stderr
|
||||
assert RESOLVED_VERSION in result.stderr
|
||||
|
||||
|
||||
def test_static_resolver_and_install_are_env_i_wrapped() -> None:
|
||||
body = RUN_DAILY.read_text()
|
||||
lines = _executable_lines(_extract_pin_snippet())
|
||||
resolver_call = lines.index("pr_gate_version_resolver.py")
|
||||
assert "env -i" in lines[:resolver_call], (
|
||||
"run_daily.sh must run pr_gate_version_resolver.py under `env -i`"
|
||||
)
|
||||
install_call = lines.index('@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}')
|
||||
assert "env -i" in lines[resolver_call:install_call], (
|
||||
"run_daily.sh must run the npm install under its own `env -i`"
|
||||
)
|
||||
assert "claude" not in _required_commands(body), (
|
||||
"run_daily.sh must not require a pre-provisioned `claude` on "
|
||||
"PATH anymore; the pinned install provides it"
|
||||
)
|
||||
for required in ("npm", "python3"):
|
||||
assert required in _required_commands(body)
|
||||
|
||||
|
||||
def _required_commands(body: str) -> tuple[str, ...]:
|
||||
marker = "for cmd in "
|
||||
start = body.index(marker) + len(marker)
|
||||
end = body.index(";", start)
|
||||
return tuple(body[start:end].split())
|
||||
|
|
@ -10,7 +10,7 @@ compromised `@anthropic-ai/claude-code` release could read those
|
|||
secrets out of `os.environ` before the proxy or test harness ever
|
||||
starts. The version probe must be wrapped in `env -i` with a minimal
|
||||
PATH/HOME/USER/TERM/LANG/LC_ALL/TMPDIR allowlist — matching the
|
||||
PR-gate's resolver/npm-install/pytest scrubs.
|
||||
resolver/npm-install/pytest scrubs in the same script.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -23,8 +23,8 @@ RUN_DAILY = REPO_ROOT / "tests" / "e2e" / "claude_code" / "cron_vm" / "run_daily
|
|||
|
||||
def _version_probe_block() -> str:
|
||||
body = RUN_DAILY.read_text()
|
||||
start = body.index("CLAUDE_CODE_VERSION=")
|
||||
end = body.index('[[ -n "${CLAUDE_CODE_VERSION}" ]]', start)
|
||||
start = body.index("PROBED_CLAUDE_VERSION=")
|
||||
end = body.index('[[ -n "${PROBED_CLAUDE_VERSION}" ]]', start)
|
||||
return body[start:end]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@
|
|||
# rather than spawning a new one. If the JSON is byte-identical to the
|
||||
# docs branch, we skip the push entirely.
|
||||
#
|
||||
# Required commands on $PATH: git, uv, gh, jq, curl, claude.
|
||||
# Required commands on $PATH: git, uv, gh, jq, curl, npm, python3.
|
||||
# Required state: ~/litellm/litellm checked out (this file lives in it),
|
||||
# $WORKTREE is created on first run, gh is already authenticated.
|
||||
#
|
||||
|
|
@ -90,7 +90,7 @@ trap cleanup EXIT INT TERM
|
|||
log() { printf '==> %s\n' "$*" >&2; }
|
||||
die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
||||
|
||||
for cmd in git uv gh jq curl claude; do
|
||||
for cmd in git uv gh jq curl npm python3; do
|
||||
command -v "${cmd}" >/dev/null 2>&1 || die "missing required command: ${cmd}"
|
||||
done
|
||||
|
||||
|
|
@ -165,26 +165,52 @@ log "resolved litellm: ${LITELLM_VERSION}"
|
|||
|
||||
# The systemd unit loads provider credentials and the agent-shin GitHub
|
||||
# token from /etc/litellm-compat-matrix.env into this script's
|
||||
# environment. Running the npm-installed `claude` binary directly here
|
||||
# would hand that full env to package code -- a compromised
|
||||
# @anthropic-ai/claude-code release could read ANTHROPIC_API_KEY /
|
||||
# AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY /
|
||||
# environment. Running npm install or the npm-installed `claude` binary
|
||||
# directly here would hand that full env to package code -- a
|
||||
# compromised @anthropic-ai/claude-code release could read
|
||||
# ANTHROPIC_API_KEY / AWS_BEARER_TOKEN_BEDROCK / AZURE_FOUNDRY_API_KEY /
|
||||
# AGENT_SHIN_GITHUB_TOKEN from os.environ and exfiltrate them before
|
||||
# the proxy or test harness ever starts. Probe under `env -i` with the
|
||||
# same minimal allowlist the PR-gate uses (the matrix run itself goes
|
||||
# through cli_driver.py, which already scrubs the CLI env).
|
||||
# the proxy or test harness ever starts. Resolve, install, and probe
|
||||
# under `env -i` with the same minimal allowlist the pytest step below
|
||||
# uses (the matrix run itself goes through cli_driver.py, which already
|
||||
# scrubs the CLI env).
|
||||
#
|
||||
# The probe also runs under a fresh empty HOME instead of the runtime
|
||||
# These steps also run under a fresh empty HOME instead of the runtime
|
||||
# user's real $HOME. `ProtectHome=read-only` in the systemd unit
|
||||
# blocks *writes* to /home/mateo but still allows reads, so a
|
||||
# compromised claude package invoked here with HOME=/home/mateo could
|
||||
# read ~/.config/gh/hosts.yml (the gh-host token), ~/.bash_history,
|
||||
# or ~/.ssh/. Pointing HOME at a per-run dir under ${WORKDIR} hides
|
||||
# those entirely from the subprocess; ${WORKDIR} is rm -rf'd by the
|
||||
# script-wide cleanup() trap regardless of probe outcome.
|
||||
# script-wide cleanup() trap regardless of outcome.
|
||||
CLAUDE_PROBE_HOME="${WORKDIR}/claude-probe-home"
|
||||
mkdir -p "${CLAUDE_PROBE_HOME}"
|
||||
CLAUDE_CODE_VERSION="$(env -i \
|
||||
PATH="${PATH}" \
|
||||
HOME="${CLAUDE_PROBE_HOME}" \
|
||||
USER="${USER:-mateo}" \
|
||||
TERM="${TERM:-dumb}" \
|
||||
LANG="${LANG:-C.UTF-8}" \
|
||||
LC_ALL="${LC_ALL:-}" \
|
||||
TMPDIR="${TMPDIR:-/tmp}" \
|
||||
python3 "${POPULATOR_DIR}/../pr_gate_version_resolver.py")"
|
||||
[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "pr_gate_version_resolver.py printed an empty version"
|
||||
log "resolved claude code: ${CLAUDE_CODE_VERSION}"
|
||||
|
||||
CLAUDE_CLI_PREFIX="${WORKDIR}/claude-cli"
|
||||
env -i \
|
||||
PATH="${PATH}" \
|
||||
HOME="${CLAUDE_PROBE_HOME}" \
|
||||
USER="${USER:-mateo}" \
|
||||
TERM="${TERM:-dumb}" \
|
||||
LANG="${LANG:-C.UTF-8}" \
|
||||
LC_ALL="${LC_ALL:-}" \
|
||||
TMPDIR="${TMPDIR:-/tmp}" \
|
||||
npm install --prefix "${CLAUDE_CLI_PREFIX}" "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" \
|
||||
|| die "npm install of @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} failed"
|
||||
PATH="${CLAUDE_CLI_PREFIX}/node_modules/.bin:${PATH}"
|
||||
|
||||
PROBED_CLAUDE_VERSION="$(env -i \
|
||||
PATH="${PATH}" \
|
||||
HOME="${CLAUDE_PROBE_HOME}" \
|
||||
USER="${USER:-mateo}" \
|
||||
|
|
@ -199,8 +225,10 @@ CLAUDE_CODE_VERSION="$(env -i \
|
|||
# `grep` finds no match (exit 1) — without it the assignment inherits the
|
||||
# pipeline's non-zero exit, `set -e` kills the script, and the operator
|
||||
# never sees the helpful diagnostic below.
|
||||
[[ -n "${CLAUDE_CODE_VERSION}" ]] || die "could not parse semver from 'claude --version'"
|
||||
log "local claude code: ${CLAUDE_CODE_VERSION}"
|
||||
[[ -n "${PROBED_CLAUDE_VERSION}" ]] || die "could not parse semver from 'claude --version'"
|
||||
[[ "${PROBED_CLAUDE_VERSION}" == "${CLAUDE_CODE_VERSION}" ]] \
|
||||
|| die "claude on PATH reports ${PROBED_CLAUDE_VERSION}, expected pinned ${CLAUDE_CODE_VERSION}"
|
||||
log "pinned claude code: ${CLAUDE_CODE_VERSION}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Update the worktree to that tag
|
||||
|
|
@ -385,9 +413,8 @@ set +e
|
|||
# 2. a model-directed `Read` tool call during a PDF/vision cell
|
||||
# cannot reach /proc/<pytest-pid>/environ and pull the creds out
|
||||
# of the parent process the way it can today;
|
||||
# 3. this matches the PR-gate pytest step in `.circleci/config.yml`,
|
||||
# which already runs under `env -i` with the same minimal
|
||||
# allowlist.
|
||||
# 3. this matches the resolver/npm-install/probe steps above, which
|
||||
# already run under `env -i` with the same minimal allowlist.
|
||||
#
|
||||
# `cli_driver.py` re-allowlists its own subset (PATH/USER/LOGNAME/etc.)
|
||||
# when spawning the `claude` binary, so the CLI still finds Node + the
|
||||
|
|
|
|||
|
|
@ -1,23 +1,24 @@
|
|||
"""Claude Code PR-Gate Version Resolver.
|
||||
|
||||
Resolves the `@anthropic-ai/claude-code` npm version that the PR-gate CI
|
||||
job installs. Selects the newest version (by publish timestamp) whose
|
||||
publish timestamp is at least 3 days old. The 3-day window is a security
|
||||
review buffer — see PRD #26476, "Version resolvers".
|
||||
Resolves the `@anthropic-ai/claude-code` npm version that the daily
|
||||
compatibility-matrix runner installs. Selects the newest version (by
|
||||
publish timestamp) whose publish timestamp is at least 3 days old. The
|
||||
3-day window is a security review buffer — see PRD #26476, "Version
|
||||
resolvers".
|
||||
|
||||
Two surfaces:
|
||||
|
||||
- ``resolve_pr_gate_version(...)`` — the importable function. Accepts
|
||||
pre-fetched npm metadata (for unit tests) or a custom ``fetcher``
|
||||
callable. The default fetcher hits the public npm registry.
|
||||
- ``python -m claude_code.pr_gate_version_resolver`` — prints the
|
||||
resolved version string to stdout, suitable for piping into a shell
|
||||
``$(...)`` substitution inside the CircleCI job.
|
||||
- ``python3 pr_gate_version_resolver.py`` — prints the resolved version
|
||||
string to stdout, suitable for piping into a shell ``$(...)``
|
||||
substitution.
|
||||
|
||||
The CLI form is what CircleCI runs at job start; engineers reading the
|
||||
job log can see the selected version on a single line above the
|
||||
``npm install -g`` step (acceptance criterion: "the selected Claude
|
||||
Code version is logged in the CI output").
|
||||
The CLI form is what ``cron_vm/run_daily.sh`` runs at the start of each
|
||||
daily run; engineers reading the run log can see the selected version
|
||||
on a single line above the ``npm install`` step (acceptance criterion:
|
||||
"the selected Claude Code version is logged in the CI output").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue