compat-matrix: parallel-fanout refactor + 5 new feature dirs + rate limiter

- Switch all per-cell tests from @pytest.mark.parametrize("model", ...)
  (3 sequential invocations) to a single test that fans out to all 3
  Claude tiers via run_claude_models_parallel. Per-cell wall time is now
  bounded by the slowest model rather than the sum.

- Add 5 new v0 feature dirs (5 providers each, 25 new test files):
    web_search, pdf_input, prompt_caching_1h,
    tool_use_streaming, thinking_with_tool_use
  Manifest expanded to match.

- Add cross-process token-bucket rate limiter (rate_limiter.py + tests)
  so xdist workers stay under per-provider req/s limits during full-grid
  runs. New env knobs: LITELLM_COMPAT_RATE_{ANTHROPIC,AZURE,VERTEX_AI,
  BEDROCK_CONVERSE,BEDROCK_INVOKE}.

- conftest.py: write per-worker shards under <artifact>.shards/, merge
  in the controller; preserve the "don't write empty artifact" guard so
  unit-test runs don't clobber a real compat-results.json.

- Vertex test_config.yaml: route project/location through env so the
  cron VM can target a different GCP project than the upstream default.

- Add run_compat.sh wrapper for binary-searching ideal req/s per
  provider against compat-rate-limit-summary.json output.

.gitignore

@@ -104,5 +104,7 @@ test-config
 
 # Claude Code compatibility-matrix pytest artifact (CI-only output).
 compat-results.json
+compat-results.json.shards/
+compat-rate-limit-summary.json
 # Matrix JSON produced by the daily-cron publisher (pushed to litellm-docs).
 compatibility-matrix.json

tests/claude_code/_builder_unit_tests/test_matrix_builder.py

@@ -192,7 +192,28 @@ def test_build_matrix_6x5_grid_matches_published_sample():
     so any future schema drift surfaces here in review.
     """
     repo_root = Path(__file__).resolve().parents[1]
-    manifest = load_manifest(repo_root / "manifest.yaml")
+    full_manifest = load_manifest(repo_root / "manifest.yaml")
+
+    # The v0 sample matrix is a frozen baseline: it covers exactly the
+    # six features the PRD shipped with, in their canonical order. The
+    # live manifest may carry additional rows (extensions added after
+    # v0 shipped), but the sample is derived only from the v0 slice so
+    # this test stays a meaningful regression gate for the v0 cell
+    # shape rather than chasing every new row added downstream.
+    v0_feature_ids = [
+        "basic_messaging_non_streaming",
+        "basic_messaging_streaming",
+        "tool_use",
+        "prompt_caching_5m",
+        "vision",
+        "extended_thinking",
+    ]
+    v0_features = [
+        feature
+        for feature in full_manifest["features"]
+        if feature["id"] in v0_feature_ids
+    ]
+    manifest = {**full_manifest, "features": v0_features}
 
     feature_ids = [feature["id"] for feature in manifest["features"]]
     providers = manifest["providers"]

tests/claude_code/_builder_unit_tests/test_v0_layout.py

@@ -46,8 +46,14 @@ def manifest() -> dict:
 
 
 def test_manifest_lists_all_six_v0_features_in_order(manifest):
+    """The PRD's v0 row set must appear at the top of the manifest in
+    order. Features beyond v0 (extensions added after the matrix
+    shipped) are allowed but must not reorder or displace the v0
+    rows — the docs page anchors row links by index, so v0 stays
+    pinned at positions [0:6] for the lifetime of the schema.
+    """
     ids = [feature["id"] for feature in manifest["features"]]
-    assert ids == EXPECTED_FEATURE_IDS
+    assert ids[: len(EXPECTED_FEATURE_IDS)] == EXPECTED_FEATURE_IDS
 
 
 def test_manifest_lists_all_five_v0_providers_in_order(manifest):

tests/claude_code/_driver_unit_tests/conftest.py

@@ -0,0 +1,32 @@
+"""Local conftest for the driver unit tests.
+
+Installs a hermetic, no-op rate limiter for every test in this
+subdirectory. Without this, importing `cli_driver` and calling
+`run_claude(..., runner=fake)` would silently consume tokens from the
+shared default limiter (which writes to `$TMPDIR/...`), polluting the
+on-disk state another test run might rely on and adding flakiness if
+the env vars say "rate=0.1/s".
+
+A no-op limiter (rate=0 for every provider) returns immediately from
+`acquire(...)`, so unit tests behave exactly as they did before the
+limiter was added.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.claude_code.rate_limiter import (
+    ALL_PROVIDERS,
+    ProviderConfig,
+    RateLimiter,
+    use_limiter,
+)
+
+
+@pytest.fixture(autouse=True)
+def _hermetic_rate_limiter(tmp_path):
+    config = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS}
+    limiter = RateLimiter(config=config, state_dir=tmp_path)
+    with use_limiter(limiter):
+        yield

tests/claude_code/_driver_unit_tests/test_cli_driver.py

@@ -20,6 +20,7 @@ from tests.claude_code.cli_driver import (
     DriverResult,
     failure_diagnostic,
     run_claude,
+    run_claude_models_parallel,
 )
 
 
@@ -345,6 +346,241 @@ def test_failure_diagnostic_ignores_non_int_api_error_status():
     assert "text=oops" in diag
 
 
+# ---------------------------------------------------------------------------
+# run_claude_models_parallel
+#
+# The matrix runs three Claude tiers per cell, so the parallel helper has to
+# (a) invoke `run_claude` once per model, (b) preserve each model's outcome
+# separately, and (c) return errors as values rather than raising — callers
+# need both the failed and the succeeded model results to report per-cell
+# rows accurately.
+# ---------------------------------------------------------------------------
+
+
+def test_run_claude_models_parallel_returns_one_result_per_model():
+    """Each model gets its own DriverResult keyed under the helper's dict."""
+    seen_models: List[str] = []
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        # The model id is two slots after `--model` in the assembled command.
+        idx = cmd.index("--model")
+        model = cmd[idx + 1]
+        seen_models.append(model)
+        return _Completed(
+            returncode=0,
+            stdout=json.dumps(
+                {
+                    "type": "assistant",
+                    "message": {
+                        "content": [{"type": "text", "text": f"reply-{model}"}]
+                    },
+                }
+            )
+            + "\n",
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=["a", "b", "c"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+    )
+
+    assert set(outcomes.keys()) == {"a", "b", "c"}
+    for model in ("a", "b", "c"):
+        result = outcomes[model]
+        assert isinstance(result, DriverResult)
+        assert result.text == f"reply-{model}"
+        assert result.exit_code == 0
+    assert sorted(seen_models) == ["a", "b", "c"]
+
+
+def test_run_claude_models_parallel_returns_errors_as_values():
+    """A model whose CLI is missing surfaces as a ClaudeCLIError, not a raise."""
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        idx = cmd.index("--model")
+        model = cmd[idx + 1]
+        if model == "boom":
+            raise FileNotFoundError(2, "no such file", "claude")
+        return _Completed(
+            returncode=0,
+            stdout=json.dumps(
+                {
+                    "type": "assistant",
+                    "message": {"content": [{"type": "text", "text": "ok"}]},
+                }
+            )
+            + "\n",
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=["ok-model", "boom"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+    )
+
+    assert isinstance(outcomes["ok-model"], DriverResult)
+    assert outcomes["ok-model"].text == "ok"
+    assert isinstance(outcomes["boom"], ClaudeCLIError)
+    assert "claude CLI not found" in str(outcomes["boom"])
+
+
+def test_run_claude_models_parallel_preserves_nonzero_exit_codes():
+    """Mixed success/failure on exit code should not collapse into one verdict."""
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        idx = cmd.index("--model")
+        model = cmd[idx + 1]
+        if model == "fail":
+            return _Completed(returncode=2, stdout="", stderr="auth failed")
+        return _Completed(
+            returncode=0,
+            stdout=json.dumps(
+                {
+                    "type": "assistant",
+                    "message": {"content": [{"type": "text", "text": "ok"}]},
+                }
+            )
+            + "\n",
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=["ok-model", "fail"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+    )
+
+    assert outcomes["ok-model"].exit_code == 0
+    assert outcomes["fail"].exit_code == 2
+    assert outcomes["fail"].stderr == "auth failed"
+
+
+def test_run_claude_models_parallel_rejects_empty_models():
+    with pytest.raises(ValueError, match="non-empty"):
+        run_claude_models_parallel(
+            models=[],
+            prompt="hi",
+            base_url="http://x",
+            api_key="k",
+        )
+
+
+def test_run_claude_models_parallel_stamps_duration_on_each_result():
+    """Each DriverResult carries the per-model wall time so callers can
+    attribute slow cells without re-timing the work themselves.
+
+    The fake runner sleeps for very different durations per model so
+    we can prove each result is timing its own work (not the batch
+    wall time). We use generous absolute bounds because thread-pool
+    scheduling on a loaded CI box adds noise on the order of tens of
+    milliseconds.
+    """
+    import time
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        idx = cmd.index("--model")
+        model = cmd[idx + 1]
+        time.sleep(0.05 if model == "fast" else 0.40)
+        return _Completed(returncode=0, stdout="")
+
+    outcomes = run_claude_models_parallel(
+        models=["fast", "slow"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+    )
+
+    fast_ms = outcomes["fast"].duration_ms
+    slow_ms = outcomes["slow"].duration_ms
+    assert fast_ms is not None and slow_ms is not None
+    # 50ms sleep ⇒ ~50–250ms after scheduling overhead; 400ms sleep ⇒
+    # 400–700ms. We just need the two distributions to be non-overlapping
+    # so we know each row's duration is its own work, not the batch's.
+    assert fast_ms < 300, fast_ms
+    assert slow_ms >= 350, slow_ms
+    assert slow_ms > fast_ms
+
+
+def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys):
+    """The breakdown helper must emit a per-model timing block so users
+    can answer "why didn't parallel help?" without re-instrumenting."""
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        return _Completed(returncode=0, stdout="")
+
+    run_claude_models_parallel(
+        models=["model-x", "model-y"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+    )
+
+    captured = capsys.readouterr()
+    assert "[parallel] per-model wall time:" in captured.err
+    assert "model-x" in captured.err
+    assert "model-y" in captured.err
+    assert "speedup=" in captured.err
+    assert "slowest=" in captured.err
+
+
+def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys):
+    """When a model raises ClaudeCLIError, the breakdown should still
+    show its row tagged as `cli-error` rather than crashing or omitting it."""
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        idx = cmd.index("--model")
+        if cmd[idx + 1] == "boom":
+            raise FileNotFoundError(2, "no such file", "claude")
+        return _Completed(returncode=0, stdout="")
+
+    run_claude_models_parallel(
+        models=["ok-model", "boom"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+    )
+
+    captured = capsys.readouterr()
+    assert "ok-model" in captured.err
+    assert "boom" in captured.err
+    assert "cli-error" in captured.err
+
+
+def test_run_claude_models_parallel_forwards_extra_args_and_env():
+    """Shared kwargs must reach every per-model invocation unchanged."""
+    captured_envs: List[dict] = []
+    captured_cmds: List[List[str]] = []
+
+    def runner(cmd, env, capture_output, text, timeout, check):
+        captured_envs.append(env)
+        captured_cmds.append(cmd)
+        return _Completed(returncode=0, stdout="")
+
+    run_claude_models_parallel(
+        models=["a", "b"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        extra_env={"MAX_THINKING_TOKENS": "4096"},
+        extra_args=["--allowed-tools", "Bash"],
+        runner=runner,
+    )
+
+    assert all(env["MAX_THINKING_TOKENS"] == "4096" for env in captured_envs)
+    for cmd in captured_cmds:
+        assert "--allowed-tools" in cmd
+        assert "Bash" in cmd
+
+
 def test_failure_diagnostic_uses_last_result_event_status():
     """If multiple `result` events appear, the most recent status wins."""
     result = DriverResult(

tests/claude_code/_driver_unit_tests/test_compat_result.py

@@ -68,3 +68,71 @@ def test_set_copies_input():
     r.set(payload)
     payload["error"] = "mutated"
     assert r.value["error"] == "x"
+
+
+# ---------------------------------------------------------------------------
+# add() / collected()
+#
+# When a single test exercises three Claude tiers in parallel, each tier
+# needs its own row in the results artifact so the matrix builder can
+# apply its "all three must pass" aggregation. `add()` is the per-tier
+# recorder; `collected()` is what the conftest hook reads.
+# ---------------------------------------------------------------------------
+
+
+def test_add_appends_each_call_to_values():
+    r = CompatResult()
+    r.add({"status": "pass"})
+    r.add({"status": "fail", "error": "bad"})
+    assert r.values == [
+        {"status": "pass"},
+        {"status": "fail", "error": "bad"},
+    ]
+
+
+def test_add_validates_like_set():
+    """The add() and set() validators are the same; both must reject bad payloads."""
+    r = CompatResult()
+    with pytest.raises(ValueError, match="requires 'error'"):
+        r.add({"status": "fail"})
+    with pytest.raises(ValueError, match="requires 'reason'"):
+        r.add({"status": "not_applicable"})
+    with pytest.raises(ValueError, match="status must be one of"):
+        r.add({"status": "maybe"})
+    with pytest.raises(TypeError):
+        r.add("pass")  # type: ignore[arg-type]
+
+
+def test_add_copies_input():
+    """Same defensive copy contract as set()."""
+    r = CompatResult()
+    payload = {"status": "fail", "error": "x"}
+    r.add(payload)
+    payload["error"] = "mutated"
+    assert r.values[0]["error"] == "x"
+
+
+def test_collected_returns_values_when_added():
+    r = CompatResult()
+    r.add({"status": "pass"})
+    r.add({"status": "pass"})
+    assert r.collected() == [{"status": "pass"}, {"status": "pass"}]
+
+
+def test_collected_returns_single_value_when_only_set_called():
+    """Legacy single-result tests should still surface their one outcome."""
+    r = CompatResult()
+    r.set({"status": "pass"})
+    assert r.collected() == [{"status": "pass"}]
+
+
+def test_collected_prefers_added_values_over_set_value():
+    """If both are populated, the per-tier list wins — that's the multi-model shape."""
+    r = CompatResult()
+    r.set({"status": "pass"})
+    r.add({"status": "fail", "error": "tier-2 broke"})
+    assert r.collected() == [{"status": "fail", "error": "tier-2 broke"}]
+
+
+def test_collected_returns_empty_when_nothing_reported():
+    assert CompatResult().collected() == []

tests/claude_code/_driver_unit_tests/test_rate_limiter.py

@@ -0,0 +1,329 @@
+"""Unit tests for the cross-process token-bucket rate limiter.
+
+The tests cover three layers:
+
+1. Provider inference from model alias — the matrix-column mapping the
+   live tests rely on (`-bedrock-converse` vs `-bedrock-invoke` vs
+   `-azure` vs `-vertex` vs bare = anthropic).
+
+2. Config parsing — env-var precedence, fallback to default, malformed
+   input handling, burst override semantics. These run against
+   `os.environ`-shaped dicts so we don't have to monkeypatch globals.
+
+3. Token-bucket behavior — enforcing rate, accumulating burst, never
+   over-spending across a fake clock. Filesystem state is exercised
+   with a real `tmp_path` because the persistence is the whole point;
+   the only injected seam is `clock` (and `sleep`, so tests don't
+   actually wait on wall time).
+
+The cross-process flock semantics are exercised indirectly: every
+test creates a fresh `RateLimiter` rooted at `tmp_path`, so the same
+file lock that protects production is exercised here too. We don't
+fork to test multi-process behavior in this file because pytest
+fixtures + xdist already do that for the integration suite.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import List
+
+import pytest
+
+from tests.claude_code.rate_limiter import (
+    ALL_PROVIDERS,
+    BURST_ENV,
+    DEFAULT_RATE,
+    PROVIDER_ANTHROPIC,
+    PROVIDER_AZURE,
+    PROVIDER_BEDROCK_CONVERSE,
+    PROVIDER_BEDROCK_INVOKE,
+    PROVIDER_VERTEX_AI,
+    ProviderConfig,
+    RateLimiter,
+    get_default_limiter,
+    infer_provider,
+    load_config,
+    reset_default_limiter,
+    use_limiter,
+)
+
+
+# ---------------------------------------------------------------------------
+# Provider inference
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    "model, expected",
+    [
+        ("claude-haiku-4-5", PROVIDER_ANTHROPIC),
+        ("claude-sonnet-4-6", PROVIDER_ANTHROPIC),
+        ("claude-opus-4-7", PROVIDER_ANTHROPIC),
+        ("claude-haiku-4-5-azure", PROVIDER_AZURE),
+        ("claude-sonnet-4-6-azure", PROVIDER_AZURE),
+        ("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI),
+        ("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE),
+        ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE),
+    ],
+)
+def test_infer_provider_maps_alias_suffix_to_column(model, expected):
+    assert infer_provider(model) == expected
+
+
+def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order():
+    """Both bedrock suffixes contain `bedrock`; the more-specific suffix wins."""
+    assert infer_provider("claude-foo-bedrock-converse") == PROVIDER_BEDROCK_CONVERSE
+    assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE
+
+
+def test_infer_provider_rejects_empty_string():
+    with pytest.raises(ValueError, match="non-empty"):
+        infer_provider("")
+
+
+def test_infer_provider_is_case_insensitive():
+    """Aliases in the proxy config sometimes drift between cases; we
+    should still route them to the right column."""
+    assert infer_provider("CLAUDE-OPUS-4-7-AZURE") == PROVIDER_AZURE
+
+
+# ---------------------------------------------------------------------------
+# Config parsing
+# ---------------------------------------------------------------------------
+
+
+def test_load_config_uses_default_rate_when_env_absent():
+    cfg = load_config(env={})
+    for provider in ALL_PROVIDERS:
+        assert cfg[provider].rate_per_sec == DEFAULT_RATE
+        assert cfg[provider].burst == DEFAULT_RATE
+
+
+def test_load_config_reads_per_provider_rate():
+    cfg = load_config(
+        env={
+            "LITELLM_COMPAT_RATE_ANTHROPIC": "10",
+            "LITELLM_COMPAT_RATE_AZURE": "0.5",
+        }
+    )
+    assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == 10.0
+    assert cfg[PROVIDER_AZURE].rate_per_sec == 0.5
+    assert cfg[PROVIDER_VERTEX_AI].rate_per_sec == DEFAULT_RATE
+
+
+def test_load_config_zero_rate_disables_provider():
+    cfg = load_config(env={"LITELLM_COMPAT_RATE_BEDROCK_INVOKE": "0"})
+    assert cfg[PROVIDER_BEDROCK_INVOKE].enabled is False
+
+
+def test_load_config_burst_override_applies_to_every_provider():
+    cfg = load_config(
+        env={
+            "LITELLM_COMPAT_RATE_ANTHROPIC": "5",
+            BURST_ENV: "20",
+        }
+    )
+    for provider in ALL_PROVIDERS:
+        assert cfg[provider].burst == 20.0
+
+
+def test_load_config_falls_back_on_malformed_value():
+    cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "not-a-number"})
+    assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == DEFAULT_RATE
+
+
+def test_load_config_burst_floors_at_one_when_rate_is_low():
+    """A 0.5/s rate with no burst override must still allow at least
+    one immediate request — otherwise the very first call would block."""
+    cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "0.5"})
+    assert cfg[PROVIDER_ANTHROPIC].burst == 1.0
+
+
+# ---------------------------------------------------------------------------
+# Token bucket
+# ---------------------------------------------------------------------------
+
+
+@pytest.fixture
+def fake_clock():
+    """A controllable monotonic clock + sleep for the limiter under test.
+
+    Tests advance `clock.now` to simulate elapsed wall time. `sleep`
+    adds the requested duration to `clock.now` instead of actually
+    sleeping, so a "wait 200ms" code path runs in microseconds and
+    is deterministic.
+    """
+
+    class Clock:
+        def __init__(self):
+            self.now = 1_000.0
+            self.sleeps: List[float] = []
+
+        def __call__(self):
+            return self.now
+
+        def sleep(self, seconds: float) -> None:
+            self.sleeps.append(seconds)
+            self.now += seconds
+
+    return Clock()
+
+
+def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None):
+    cfg = {
+        p: ProviderConfig(rate_per_sec=rate, burst=burst if burst is not None else rate)
+        for p in ALL_PROVIDERS
+    }
+    return RateLimiter(
+        config=cfg,
+        state_dir=tmp_path,
+        clock=fake_clock,
+        sleep=fake_clock.sleep,
+    )
+
+
+def test_acquire_first_call_does_not_wait(tmp_path, fake_clock):
+    """A freshly-initialized bucket starts full; the first acquire is free."""
+    limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=10.0)
+    waited = limiter.acquire(PROVIDER_ANTHROPIC)
+    assert waited == 0.0
+    assert fake_clock.sleeps == []
+
+
+def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock):
+    """rate=0 ⇒ no throttling, even if every other provider is throttled."""
+    cfg = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS}
+    limiter = RateLimiter(
+        config=cfg, state_dir=tmp_path, clock=fake_clock, sleep=fake_clock.sleep
+    )
+    for _ in range(100):
+        assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+    assert fake_clock.sleeps == []
+
+
+def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock):
+    """`burst` immediate requests succeed; the next one waits 1/rate seconds."""
+    limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=3.0)
+
+    for _ in range(3):
+        assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+
+    # Bucket is empty; next call must sleep ~0.5s to earn one token at 2/s.
+    waited = limiter.acquire(PROVIDER_ANTHROPIC)
+    assert waited == pytest.approx(0.5, abs=0.01)
+
+
+def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock):
+    """Advancing the clock between calls credits tokens at the configured rate."""
+    limiter = _make_limiter(tmp_path, fake_clock, rate=4.0, burst=1.0)
+
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0  # consumes the 1-token burst
+    fake_clock.now += 0.25  # 0.25s × 4/s = 1 token earned
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+
+
+def test_acquire_caps_refill_at_burst(tmp_path, fake_clock):
+    """A long quiet period must not let the bucket grow past `burst`."""
+    limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=2.0)
+
+    fake_clock.now += 1_000  # would earn 10_000 tokens uncapped
+    # Only `burst` (=2) immediate calls should succeed before throttling.
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+    waited = limiter.acquire(PROVIDER_ANTHROPIC)
+    assert waited > 0
+
+
+def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock):
+    """Anthropic exhaustion must not throttle Azure (each column has its own bucket)."""
+    limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=1.0)
+
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+    # Anthropic bucket is now empty; Azure is untouched.
+    assert limiter.acquire(PROVIDER_AZURE) == 0.0
+
+
+def test_acquire_persists_state_across_limiter_instances(tmp_path):
+    """A fresh RateLimiter must read the on-disk state, not start fresh.
+
+    This is the property that makes the limiter cross-process: an
+    xdist worker created mid-run sees the credit consumed by other
+    workers, instead of getting its own private bucket.
+    """
+    cfg = {p: ProviderConfig(rate_per_sec=10.0, burst=2.0) for p in ALL_PROVIDERS}
+    state = {"now": 1_000.0, "sleeps": []}
+
+    def clock():
+        return state["now"]
+
+    def sleep(seconds):
+        state["sleeps"].append(seconds)
+        state["now"] += seconds
+
+    first = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep)
+    first.acquire(PROVIDER_ANTHROPIC)
+    first.acquire(PROVIDER_ANTHROPIC)
+    # bucket is now empty
+
+    second = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep)
+    waited = second.acquire(PROVIDER_ANTHROPIC)
+    assert waited > 0  # had to wait, didn't see a fresh full bucket
+
+
+def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock):
+    """A truncated/garbage state file must not crash the test session."""
+    state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json"
+    state_file.write_text("not-json {{")
+
+    limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0)
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+
+
+def test_acquire_handles_clock_going_backward(tmp_path, fake_clock):
+    """Across a host suspend/resume the monotonic clock can briefly
+    go backward; we must not interpret that as removing tokens."""
+    limiter = _make_limiter(tmp_path, fake_clock, rate=1.0, burst=2.0)
+    limiter.acquire(PROVIDER_ANTHROPIC)
+    fake_clock.now -= 10  # clock moved backward
+    # Bucket should still have ~1 token left from the burst, not -9.
+    assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
+
+
+# ---------------------------------------------------------------------------
+# Process-default singleton
+# ---------------------------------------------------------------------------
+
+
+def test_use_limiter_swaps_default_for_block(tmp_path):
+    sentinel_cfg = {
+        p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS
+    }
+    sentinel = RateLimiter(config=sentinel_cfg, state_dir=tmp_path)
+    reset_default_limiter()
+    try:
+        with use_limiter(sentinel):
+            assert get_default_limiter() is sentinel
+        # After the context exits, the default goes back to whatever it
+        # was — in this test that's "rebuilt on next access" because we
+        # called reset_default_limiter() above.
+        assert get_default_limiter() is not sentinel
+    finally:
+        reset_default_limiter()
+
+
+# ---------------------------------------------------------------------------
+# Persistence shape
+# ---------------------------------------------------------------------------
+
+
+def test_state_file_is_json_after_acquire(tmp_path, fake_clock):
+    limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0)
+    limiter.acquire(PROVIDER_ANTHROPIC)
+    state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json"
+    payload = json.loads(state_file.read_text())
+    assert "tokens" in payload
+    assert "last_refill" in payload
+    assert payload["tokens"] == pytest.approx(4.0)

tests/claude_code/basic_messaging_non_streaming/test_anthropic.py

@@ -12,9 +12,11 @@ The (feature, provider) for this cell is inferred from the file path by
                        feature_id                         provider
 
 Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus
-4.7; the cell only goes green if all three pass. We parametrize over the
-three models and the conftest aggregator produces one cell from the three
-results.
+4.7; the cell only goes green if all three pass. We fan the three model
+runs out in parallel inside this single test so the per-cell wall time is
+bounded by the slowest model rather than the sum, and report one
+`compat_result.add(...)` entry per model so the matrix builder still sees
+three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -23,7 +25,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -38,8 +44,7 @@ ANTHROPIC_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", ANTHROPIC_MODELS)
-def test_basic_messaging_non_streaming_anthropic(compat_result, model):
+def test_basic_messaging_non_streaming_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
 
     "Basic messaging" means: send a single user prompt, receive any
@@ -63,38 +68,35 @@ def test_basic_messaging_non_streaming_anthropic(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_non_streaming/test_azure.py

@@ -17,10 +17,10 @@ The (feature, provider) for this cell is inferred from the file path by
                        feature_id                          provider
 
 Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus
-4.7; the cell only goes green if all three pass. We parametrize over the
-three models and the conftest aggregator produces one cell from the
-three results, naming the failing model in the error string when any
-model fails.
+4.7; the cell only goes green if all three pass. We fan the three model
+runs out in parallel inside this single test and report one
+`compat_result.add(...)` entry per model so the matrix builder still sees
+three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -29,7 +29,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -45,8 +49,7 @@ AZURE_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", AZURE_MODELS)
-def test_basic_messaging_non_streaming_azure(compat_result, model):
+def test_basic_messaging_non_streaming_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
 
     "Basic messaging" means: send a single user prompt, receive any
@@ -70,38 +73,35 @@ def test_basic_messaging_non_streaming_azure(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py

@@ -12,10 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by
                        feature_id                          provider
 
 Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus
-4.7; the cell only goes green if all three pass. We parametrize over the
-three models and the conftest aggregator produces one cell from the
-three results, naming the failing model in the error string when any
-model fails.
+4.7; the cell only goes green if all three pass. We fan the three model
+runs out in parallel inside this single test and report one
+`compat_result.add(...)` entry per model so the matrix builder still sees
+three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -24,7 +24,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -40,8 +44,7 @@ BEDROCK_CONVERSE_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS)
-def test_basic_messaging_non_streaming_bedrock_converse(compat_result, model):
+def test_basic_messaging_non_streaming_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
     api_key = os.environ.get(PROXY_API_KEY_ENV)
@@ -59,38 +62,35 @@ def test_basic_messaging_non_streaming_bedrock_converse(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py

@@ -12,10 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by
                        feature_id                          provider
 
 Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus
-4.7; the cell only goes green if all three pass. We parametrize over the
-three models and the conftest aggregator produces one cell from the
-three results, naming the failing model in the error string when any
-model fails.
+4.7; the cell only goes green if all three pass. We fan the three model
+runs out in parallel inside this single test and report one
+`compat_result.add(...)` entry per model so the matrix builder still sees
+three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -24,7 +24,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -40,8 +44,7 @@ BEDROCK_INVOKE_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS)
-def test_basic_messaging_non_streaming_bedrock_invoke(compat_result, model):
+def test_basic_messaging_non_streaming_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
     api_key = os.environ.get(PROXY_API_KEY_ENV)
@@ -59,38 +62,35 @@ def test_basic_messaging_non_streaming_bedrock_invoke(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py

@@ -12,10 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by
                        feature_id                          provider
 
 Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus
-4.7; the cell only goes green if all three pass. We parametrize over the
-three models and the conftest aggregator produces one cell from the
-three results, naming the failing model in the error string when any
-model fails.
+4.7; the cell only goes green if all three pass. We fan the three model
+runs out in parallel inside this single test and report one
+`compat_result.add(...)` entry per model so the matrix builder still sees
+three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -24,7 +24,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -40,8 +44,7 @@ VERTEX_AI_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", VERTEX_AI_MODELS)
-def test_basic_messaging_non_streaming_vertex_ai(compat_result, model):
+def test_basic_messaging_non_streaming_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
     api_key = os.environ.get(PROXY_API_KEY_ENV)
@@ -59,38 +62,35 @@ def test_basic_messaging_non_streaming_vertex_ai(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_streaming/test_anthropic.py

@@ -17,6 +17,10 @@ The (feature, provider) for this cell is inferred from the file path by
     tests/claude_code/basic_messaging_streaming/test_anthropic.py
                        ^^^^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^
                        feature_id                     provider
+
+The three Claude tiers run in parallel inside this single test, with
+one `compat_result.add(...)` entry per model so the matrix builder
+still sees three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -25,7 +29,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -37,8 +45,7 @@ ANTHROPIC_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", ANTHROPIC_MODELS)
-def test_basic_messaging_streaming_anthropic(compat_result, model):
+def test_basic_messaging_streaming_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (at least one stream-json event observed).
     """
@@ -58,49 +65,41 @@ def test_basic_messaging_streaming_anthropic(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Count from 1 to 5, one number per line.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.events:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no stream-json events emitted; streaming wire silent",
-            }
-        )
-        pytest.fail(f"no stream events for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if not outcome.events:
+            error = f"[{model}] no stream-json events emitted; streaming wire silent"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_streaming/test_azure.py

@@ -23,7 +23,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -35,8 +39,7 @@ AZURE_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", AZURE_MODELS)
-def test_basic_messaging_streaming_azure(compat_result, model):
+def test_basic_messaging_streaming_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (at least one stream-json event observed).
     """
@@ -56,48 +59,41 @@ def test_basic_messaging_streaming_azure(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Count from 1 to 5, one number per line.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.events:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no stream-json events emitted; streaming wire silent",
-            }
-        )
-        pytest.fail(f"no stream events for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if not outcome.events:
+            error = f"[{model}] no stream-json events emitted; streaming wire silent"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py

@@ -19,7 +19,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -31,8 +35,7 @@ BEDROCK_CONVERSE_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS)
-def test_basic_messaging_streaming_bedrock_converse(compat_result, model):
+def test_basic_messaging_streaming_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (at least one stream-json event observed).
     """
@@ -52,48 +55,41 @@ def test_basic_messaging_streaming_bedrock_converse(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Count from 1 to 5, one number per line.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.events:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no stream-json events emitted; streaming wire silent",
-            }
-        )
-        pytest.fail(f"no stream events for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if not outcome.events:
+            error = f"[{model}] no stream-json events emitted; streaming wire silent"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py

@@ -19,7 +19,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -31,8 +35,7 @@ BEDROCK_INVOKE_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS)
-def test_basic_messaging_streaming_bedrock_invoke(compat_result, model):
+def test_basic_messaging_streaming_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (at least one stream-json event observed).
     """
@@ -52,48 +55,41 @@ def test_basic_messaging_streaming_bedrock_invoke(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Count from 1 to 5, one number per line.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.events:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no stream-json events emitted; streaming wire silent",
-            }
-        )
-        pytest.fail(f"no stream events for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if not outcome.events:
+            error = f"[{model}] no stream-json events emitted; streaming wire silent"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/basic_messaging_streaming/test_vertex_ai.py

@@ -19,7 +19,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -31,8 +35,7 @@ VERTEX_AI_MODELS = [
 ]
 
 
-@pytest.mark.parametrize("model", VERTEX_AI_MODELS)
-def test_basic_messaging_streaming_vertex_ai(compat_result, model):
+def test_basic_messaging_streaming_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     non-empty streamed reply (at least one stream-json event observed).
     """
@@ -52,48 +55,41 @@ def test_basic_messaging_streaming_vertex_ai(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Count from 1 to 5, one number per line.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt="Count from 1 to 5, one number per line.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.events:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no stream-json events emitted; streaming wire silent",
-            }
-        )
-        pytest.fail(f"no stream events for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if not outcome.events:
+            error = f"[{model}] no stream-json events emitted; streaming wire silent"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/cli_driver.py

@@ -16,11 +16,27 @@ from __future__ import annotations
 import json
 import os
 import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Mapping, Optional, Sequence
+from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union
+
+from tests.claude_code.rate_limiter import (
+    RateLimiter,
+    get_default_limiter,
+    infer_provider,
+)
 
 CLAUDE_CLI_DEFAULT = "claude"
-DEFAULT_TIMEOUT_SECONDS = 120
+# 120s is fine for a single isolated CLI call against an unloaded
+# upstream, but the matrix run launches up to 75 concurrent calls and
+# upstreams can take several minutes to respond under that contention.
+# We expose the timeout as an env var so binary-search runs can grow
+# it without touching the test code.
+DEFAULT_TIMEOUT_SECONDS = float(
+    os.environ.get("LITELLM_COMPAT_CLI_TIMEOUT_SECONDS") or 120
+)
 
 
 class ClaudeCLIError(RuntimeError):
@@ -57,6 +73,7 @@ def run_claude(
     cli_path: str = CLAUDE_CLI_DEFAULT,
     timeout: float = DEFAULT_TIMEOUT_SECONDS,
     runner: Optional[Any] = None,
+    rate_limiter: Optional[RateLimiter] = None,
 ) -> DriverResult:
     """Invoke `claude` once in headless stream-JSON mode and return the result.
 
@@ -68,6 +85,12 @@ def run_claude(
     `runner` is an injection seam used by the unit tests: by default we call
     `subprocess.run`, but the test suite swaps in a fake that yields canned
     stream-JSON. Production callers should never set it.
+
+    `rate_limiter` is the second injection seam: the cross-process
+    token-bucket limiter throttles outbound calls per provider so a
+    fully-parallel matrix run doesn't trip 429s. Defaults to the
+    process-wide singleton; unit tests pass a no-op limiter or one
+    backed by a tmp dir to keep tests hermetic.
     """
     if not prompt:
         raise ValueError("prompt must be a non-empty string")
@@ -100,6 +123,15 @@ def run_claude(
     env["ANTHROPIC_BASE_URL"] = base_url
     env["ANTHROPIC_AUTH_TOKEN"] = api_key
 
+    # Throttle by provider *before* launching the CLI. Doing this here
+    # (rather than per-test) means every code path that lands on
+    # `run_claude` is rate-limited automatically — including
+    # `run_claude_models_parallel`, which is the hot path during the
+    # full matrix run.
+    limiter = rate_limiter if rate_limiter is not None else get_default_limiter()
+    provider = infer_provider(model)
+    limiter.acquire(provider)
+
     run_fn = runner or subprocess.run
     try:
         completed = run_fn(
@@ -130,6 +162,134 @@ def run_claude(
     )
 
 
+ModelResult = Union[DriverResult, ClaudeCLIError]
+
+
+def run_claude_models_parallel(
+    *,
+    models: Sequence[str],
+    prompt: str,
+    base_url: str,
+    api_key: str,
+    extra_env: Optional[Mapping[str, str]] = None,
+    extra_args: Optional[Sequence[str]] = None,
+    cli_path: str = CLAUDE_CLI_DEFAULT,
+    timeout: float = DEFAULT_TIMEOUT_SECONDS,
+    runner: Optional[Callable[..., Any]] = None,
+) -> Dict[str, ModelResult]:
+    """Invoke `run_claude` for every `models[i]` concurrently and collect outcomes.
+
+    Each `claude` CLI invocation is a long-lived subprocess that spends
+    almost all of its time waiting on the upstream API; running the
+    three Claude tiers in parallel cuts the per-cell wall time roughly
+    threefold without changing what each invocation does.
+
+    Threads (rather than asyncio) are the right primitive here because
+    `subprocess.run` releases the GIL while it waits, and we want to
+    keep the synchronous CLI driver unchanged so unit tests can keep
+    injecting a fake `runner`.
+
+    Returns a dict keyed by model id. Each value is either the
+    `DriverResult` produced by `run_claude` or the `ClaudeCLIError`
+    that aborted that model's run — callers decide how to map either
+    into a `compat_result` entry. The shared kwargs (prompt, env, args,
+    timeout, runner) are forwarded verbatim so the per-model wire is
+    identical to what the sequential path produces.
+    """
+    if not models:
+        raise ValueError("models must be a non-empty sequence")
+
+    def _one(model: str) -> Tuple[str, ModelResult, float]:
+        # Per-model wall clock: this is what the matrix run actually pays for.
+        # We record it whether the run succeeded or raised so the breakdown
+        # log below covers both code paths and surfaces "which model is the
+        # long pole?" without requiring per-test instrumentation.
+        started = time.monotonic()
+        try:
+            result = run_claude(
+                prompt=prompt,
+                model=model,
+                base_url=base_url,
+                api_key=api_key,
+                extra_env=extra_env,
+                extra_args=extra_args,
+                cli_path=cli_path,
+                timeout=timeout,
+                runner=runner,
+            )
+            elapsed = time.monotonic() - started
+            # Stamp the duration onto the DriverResult so callers (tests,
+            # diagnostics) can attribute slow cells without re-timing.
+            result.duration_ms = int(elapsed * 1000)
+            return model, result, elapsed
+        except ClaudeCLIError as exc:
+            elapsed = time.monotonic() - started
+            return model, exc, elapsed
+
+    outcomes: Dict[str, ModelResult] = {}
+    durations: Dict[str, float] = {}
+    overall_started = time.monotonic()
+    with ThreadPoolExecutor(max_workers=len(models)) as pool:
+        futures = [pool.submit(_one, model) for model in models]
+        for future in as_completed(futures):
+            model, outcome, elapsed = future.result()
+            outcomes[model] = outcome
+            durations[model] = elapsed
+    overall_elapsed = time.monotonic() - overall_started
+
+    _log_parallel_breakdown(models, durations, outcomes, overall_elapsed)
+    return outcomes
+
+
+def _log_parallel_breakdown(
+    models: Sequence[str],
+    durations: Mapping[str, float],
+    outcomes: Mapping[str, ModelResult],
+    overall_elapsed: float,
+) -> None:
+    """Emit a one-block timing breakdown to stderr.
+
+    Pytest only shows captured output for failing tests by default, but
+    `-s` surfaces it for passing tests too — which is exactly when you
+    care about "did parallelization actually help?". The block reports:
+
+      - per-model wall time and outcome (ok / cli-error / non-zero exit)
+      - the slowest model (the parallel run's wall-time floor)
+      - the sum of sequential model times (what the old serial path
+        would have paid)
+      - the overall parallel wall time and the speedup ratio
+
+    If one model dominates, `slowest ≈ overall ≈ sequential / 1`, and
+    the speedup will be near 1× — exactly the diagnostic that explains
+    "why didn't this get faster?".
+    """
+    sequential_total = sum(durations.values())
+    slowest_model = max(durations, key=durations.get) if durations else None
+    slowest = durations[slowest_model] if slowest_model else 0.0
+    speedup = sequential_total / overall_elapsed if overall_elapsed > 0 else 0.0
+
+    lines: List[str] = []
+    lines.append("[parallel] per-model wall time:")
+    for model in models:
+        elapsed = durations.get(model, 0.0)
+        outcome = outcomes.get(model)
+        if isinstance(outcome, ClaudeCLIError):
+            status = "cli-error"
+        elif isinstance(outcome, DriverResult):
+            status = f"exit={outcome.exit_code}"
+        else:
+            status = "missing"
+        lines.append(f"  {model:<40s} {elapsed:6.2f}s  ({status})")
+    if slowest_model is not None:
+        lines.append(
+            f"[parallel] slowest={slowest_model} ({slowest:.2f}s); "
+            f"sequential_sum={sequential_total:.2f}s; "
+            f"parallel_wall={overall_elapsed:.2f}s; "
+            f"speedup={speedup:.2f}x"
+        )
+    print("\n".join(lines), file=sys.stderr, flush=True)
+
+
 def _parse_stream_json(stdout: str) -> List[Dict[str, Any]]:
     """Parse newline-delimited JSON emitted by `claude --output-format stream-json`.
 

tests/claude_code/conftest.py

@@ -1,16 +1,26 @@
 """Pytest plumbing for the Claude Code compatibility matrix.
 
-Two responsibilities live here:
+Three responsibilities live here:
 
 1. The `compat_result` fixture — the only API a test author needs to learn.
    Tests call `compat_result.set({"status": "pass"})` (or fail / not_applicable)
-   to report their outcome as a tagged union. The fixture is per-test and
-   stores the last value reported.
+   to report their outcome as a tagged union. Multi-model tests call
+   `.add(...)` once per Claude tier so each tier lands as its own row in
+   the results artifact.
 
 2. The `pytest_runtest_makereport` hook — captures each test's reported result,
-   infers (feature, provider) from the file path, and writes a single
-   `compat-results.json` artifact next to JUnit XML. The Matrix JSON Builder
-   consumes this artifact to produce the published `compatibility-matrix.json`.
+   infers (feature, provider) from the file path, and accumulates rows into
+   a per-process collector. At session end we serialize them to
+   `compat-results.json` (or a per-worker file under xdist) so the Matrix
+   JSON Builder can consume them.
+
+3. xdist coordination — when `pytest -n auto` is used, every worker writes
+   its own results shard and the controller merges them into the canonical
+   `compat-results.json` in `pytest_sessionfinish`. Without this, the
+   workers race on the same path and the artifact only reflects whichever
+   worker finished last. The same merge step also emits a rate-limit
+   summary that the binary-search helper consumes to decide whether the
+   current X/Y/Z values were too aggressive.
 
 The (feature, provider) inference comes from the test file path: the parent
 directory name is the feature_id (matching `manifest.yaml`), and the file
@@ -22,43 +32,105 @@ from __future__ import annotations
 
 import json
 import os
+import re
+import sys
+from collections import Counter, defaultdict
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import pytest
 
 VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"}
 RESULTS_ARTIFACT_ENV = "COMPAT_RESULTS_PATH"
 DEFAULT_ARTIFACT_PATH = "compat-results.json"
+RATE_LIMIT_SUMMARY_ENV = "COMPAT_RATE_LIMIT_SUMMARY_PATH"
+DEFAULT_RATE_LIMIT_SUMMARY_PATH = "compat-rate-limit-summary.json"
+
+# Heuristic: detect 429s and rate-limit-shaped errors anywhere in the
+# error string. The CLI buries upstream errors in `assistant.message.content`
+# text on stdout (see `failure_diagnostic`), so we don't get a structured
+# status code in every code path — a regex over the joined error text is
+# the most reliable signal we have.
+#
+# We also treat a CLI timeout (`claude CLI timed out after Ns`) as a
+# rate-limit-shaped failure for binary-search purposes: in practice the
+# only reason every model in a cell stalls past the timeout is the
+# upstream collapsing under concurrency, which is exactly the situation
+# the rate limiter is supposed to back off from. False positives on a
+# genuinely slow upstream are tolerable here because the worst case is
+# the binary search runs at a slightly lower rate than necessary.
+_RATE_LIMIT_RE = re.compile(
+    r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|"
+    r"claude\s+CLI\s+timed\s+out)",
+    re.IGNORECASE,
+)
 
 
 @dataclass
 class CompatResult:
     """Per-test recorder for compatibility outcomes.
 
-    Tests interact only via `.set(...)`. `.value` is read by the
+    Tests interact via `.set(...)` (single result) or `.add(...)` (one
+    result per Claude tier when the test fans the three models out in
+    parallel). `.value` and `.values` are read by the
     `pytest_runtest_makereport` hook after the test body finishes.
+
+    Multi-result usage exists because every cell in the compat matrix is
+    backed by three model invocations (Haiku/Sonnet/Opus) per (feature,
+    provider). When a test runs them concurrently in a single pytest
+    node, each model needs its own entry in the results artifact so the
+    matrix builder's per-cell aggregator can apply its "all three must
+    pass" rule.
     """
 
     value: Optional[Dict[str, Any]] = None
+    values: List[Dict[str, Any]] = field(default_factory=list)
 
     def set(self, result: Dict[str, Any]) -> None:
+        validated = self._validate(result)
+        self.value = validated
+
+    def add(self, result: Dict[str, Any]) -> None:
+        """Append one model's outcome to the per-test results list.
+
+        Use this when a single test exercises multiple Claude tiers
+        concurrently and needs to report one outcome per tier. The
+        conftest hook will emit one entry per appended result.
+        """
+        validated = self._validate(result)
+        self.values.append(validated)
+
+    @staticmethod
+    def _validate(result: Dict[str, Any]) -> Dict[str, Any]:
         if not isinstance(result, dict):
-            raise TypeError("compat_result.set() requires a dict")
+            raise TypeError("compat_result requires a dict")
         status = result.get("status")
         if status not in VALID_STATUSES:
             raise ValueError(
-                f"compat_result.set() status must be one of {sorted(VALID_STATUSES)}, "
+                f"compat_result status must be one of {sorted(VALID_STATUSES)}, "
                 f"got {status!r}"
             )
         if status == "fail" and not result.get("error"):
-            raise ValueError("compat_result.set({'status': 'fail'}) requires 'error'")
+            raise ValueError("compat_result {'status': 'fail'} requires 'error'")
         if status == "not_applicable" and not result.get("reason"):
             raise ValueError(
-                "compat_result.set({'status': 'not_applicable'}) requires 'reason'"
+                "compat_result {'status': 'not_applicable'} requires 'reason'"
             )
-        self.value = dict(result)
+        return dict(result)
+
+    def collected(self) -> List[Dict[str, Any]]:
+        """Return every result reported during the test, preserving order.
+
+        Multi-model tests use `.add(...)` per model; legacy tests use
+        `.set(...)` once. We surface both shapes in a single list so
+        the makereport hook only has to think about a list of results.
+        """
+        if self.values:
+            return list(self.values)
+        if self.value is not None:
+            return [dict(self.value)]
+        return []
 
 
 @dataclass
@@ -109,7 +181,14 @@ def _infer_feature_and_provider(node_path: Path) -> Optional[tuple]:
 
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
-    """Capture compat_result.value at end-of-test and remember it for the artifact."""
+    """Capture compat_result reports at end-of-test and remember them for the artifact.
+
+    A single test may report multiple results (one per Claude tier when
+    the three are run in parallel inside one node). We emit one
+    `_CollectedResult` per reported entry so the matrix builder's
+    per-cell aggregator sees the same shape it would have seen if the
+    test were parametrized — every model lands in the artifact.
+    """
     outcome = yield
     report = outcome.get_result()
     if report.when != "call":
@@ -121,30 +200,200 @@ def pytest_runtest_makereport(item, call):
     feature_id, provider = inferred
 
     fixture = item.funcargs.get("compat_result") if hasattr(item, "funcargs") else None
-    reported: Optional[Dict[str, Any]] = getattr(fixture, "value", None)
-
-    if reported is None:
-        if report.passed:
-            reported = {
-                "status": "fail",
-                "error": "test passed without calling compat_result.set(); "
-                "every compat test must report a status.",
-            }
-        else:
-            reported = {
-                "status": "fail",
-                "error": (str(report.longrepr) if report.longrepr else "test failed"),
-            }
-
-    _COLLECTOR.items.append(
-        _CollectedResult(
-            feature_id=feature_id,
-            provider=provider,
-            nodeid=report.nodeid,
-            result=reported,
-        )
+    collected: List[Dict[str, Any]] = (
+        fixture.collected() if isinstance(fixture, CompatResult) else []
     )
 
+    if not collected:
+        if report.passed:
+            collected = [
+                {
+                    "status": "fail",
+                    "error": "test passed without reporting via compat_result; "
+                    "every compat test must report a status.",
+                }
+            ]
+        else:
+            collected = [
+                {
+                    "status": "fail",
+                    "error": (
+                        str(report.longrepr) if report.longrepr else "test failed"
+                    ),
+                }
+            ]
+
+    for reported in collected:
+        _COLLECTOR.items.append(
+            _CollectedResult(
+                feature_id=feature_id,
+                provider=provider,
+                nodeid=report.nodeid,
+                result=reported,
+            )
+        )
+
+
+def _is_xdist_worker(session) -> bool:
+    """Return True iff the current pytest session is an xdist worker.
+
+    The standard idiom is to look up `workerinput` on the config; the
+    controller process doesn't have it, the workers do. We deliberately
+    don't `import xdist` because the suite must keep running when xdist
+    isn't installed at all.
+    """
+    return hasattr(session.config, "workerinput")
+
+
+def _xdist_worker_id(session) -> Optional[str]:
+    info = getattr(session.config, "workerinput", None)
+    if not info:
+        return None
+    return info.get("workerid")
+
+
+def _shard_dir(artifact_path: Path) -> Path:
+    """Workers write their shards next to the canonical results path.
+
+    Putting shards in a sibling directory (rather than inline JSON
+    files in the same dir) keeps the controller's merge step simple
+    — it just lists `*.json` in `<artifact>.shards/` — and avoids
+    accidental shard/canonical filename collisions.
+    """
+    return artifact_path.with_name(artifact_path.name + ".shards")
+
+
+def _serialize_items(items: List["_CollectedResult"]) -> List[Dict[str, Any]]:
+    return [
+        {
+            "feature_id": item.feature_id,
+            "provider": item.provider,
+            "nodeid": item.nodeid,
+            "result": item.result,
+        }
+        for item in items
+    ]
+
+
+def _build_rate_limit_summary(
+    rows: List[Dict[str, Any]],
+) -> Dict[str, Any]:
+    """Aggregate per-provider rate-limit signals from the result rows.
+
+    We classify any failure whose error string matches `_RATE_LIMIT_RE`
+    as a "rate-limited" failure. The binary-search helper reads this
+    summary to decide whether the current X/Y/Z values were too
+    aggressive: if any provider has `rate_limited > 0`, the harness
+    should back off that provider's rate and retry.
+
+    Returns a dict shaped:
+
+        {
+          "totals": {"pass": N, "fail": N, "rate_limited": N, ...},
+          "per_provider": {
+            "anthropic": {"pass": ..., "fail": ..., "rate_limited": ...},
+            ...
+          },
+          "rate_limited_examples": [
+            {"feature_id": ..., "provider": ..., "error": "..."}, ...
+          ],
+        }
+    """
+    totals: Counter = Counter()
+    per_provider: Dict[str, Counter] = defaultdict(Counter)
+    rate_limited_examples: List[Dict[str, Any]] = []
+
+    for row in rows:
+        result = row.get("result") or {}
+        status = result.get("status") or "unknown"
+        provider = row.get("provider") or "unknown"
+        totals[status] += 1
+        per_provider[provider][status] += 1
+
+        if status == "fail":
+            error = str(result.get("error") or "")
+            if _RATE_LIMIT_RE.search(error):
+                totals["rate_limited"] += 1
+                per_provider[provider]["rate_limited"] += 1
+                # Cap examples so a stuck-throttled run doesn't write
+                # a multi-MB summary file the helper has to slurp.
+                if len(rate_limited_examples) < 25:
+                    rate_limited_examples.append(
+                        {
+                            "feature_id": row.get("feature_id"),
+                            "provider": provider,
+                            "error": error[:500],
+                        }
+                    )
+
+    return {
+        "totals": dict(totals),
+        "per_provider": {p: dict(c) for p, c in per_provider.items()},
+        "rate_limited_examples": rate_limited_examples,
+    }
+
+
+def _print_rate_limit_summary(summary: Dict[str, Any]) -> None:
+    """Emit a human-readable per-provider table to stderr.
+
+    Pytest only captures stderr when `-s` isn't set; we deliberately
+    write here anyway because the binary-search workflow runs pytest
+    with `-q` and grep-checks the structured JSON artifact, while a
+    human running locally with `-s` sees the same numbers inline.
+    """
+    totals = summary.get("totals", {})
+    per_provider = summary.get("per_provider", {})
+    lines: List[str] = []
+    lines.append("[compat] session totals:")
+    for status in ("pass", "fail", "rate_limited", "not_applicable", "not_tested"):
+        if status in totals:
+            lines.append(f"  {status:<16s} {totals[status]}")
+    if per_provider:
+        lines.append("[compat] per-provider breakdown:")
+        for provider in sorted(per_provider):
+            counts = per_provider[provider]
+            parts = " ".join(
+                f"{k}={v}"
+                for k, v in sorted(counts.items())
+                if k != "not_tested" or v > 0
+            )
+            lines.append(f"  {provider:<20s} {parts}")
+    if totals.get("rate_limited", 0):
+        lines.append(
+            "[compat] WARNING: at least one cell hit a rate-limit-shaped error; "
+            "lower the corresponding LITELLM_COMPAT_RATE_<PROVIDER> and retry"
+        )
+    print("\n".join(lines), file=sys.stderr, flush=True)
+
+
+def pytest_sessionstart(session):
+    """Clear stale per-worker shards from any prior session.
+
+    Without this, a previous run's shard directory leaks into the next
+    `pytest_sessionfinish` merge — yielding a `compat-results.json`
+    that includes results from runs that aren't part of the current
+    session, and a misleading rate-limit summary that re-flags
+    failures the user already saw and addressed.
+
+    Only the controller (non-xdist-worker) clears; workers must not
+    race the controller while it's wiping the directory.
+    """
+    if _is_xdist_worker(session):
+        return
+    artifact_path = Path(os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH)
+    shard_dir = _shard_dir(artifact_path)
+    if not shard_dir.exists():
+        return
+    for stale in shard_dir.glob("*.json"):
+        try:
+            stale.unlink()
+        except OSError:
+            # If we can't remove a stale shard (permissions, race with
+            # an unrelated process), keep going — the merge step is
+            # robust to malformed shards, and a stale row landing in
+            # the artifact is recoverable; aborting the session isn't.
+            continue
+
 
 def pytest_sessionstart(session):
     """Clear collected results at the start of each session.
@@ -158,27 +407,74 @@ def pytest_sessionstart(session):
 
 
 def pytest_sessionfinish(session, exitstatus):
-    """Write the structured results artifact at end of session.
+    """Write the per-process results shard, then merge if we're the controller.
 
-    Skip writing when no compat results were collected — this conftest is
-    loaded by pytest for every test under `tests/claude_code/`, including
-    sibling unit-test trees (e.g. `_driver_unit_tests/`). Writing an empty
-    artifact in those runs would silently overwrite a real artifact from a
-    prior compat-test run on the same checkout.
+    Worker processes (xdist `gw0`, `gw1`, ...) only write their shard
+    under `<artifact>.shards/<workerid>.json`. The controller writes
+    its own shard if it ran any tests itself, then walks the shards
+    directory and produces the canonical `compat-results.json` plus
+    the rate-limit summary. Single-process runs (no xdist) take the
+    same code path with a single shard, so behavior is consistent.
+
+    Skip when no compat results were collected — this conftest is
+    loaded for every test under `tests/claude_code/`, including sibling
+    unit-test trees (e.g. `_driver_unit_tests/`). Writing an empty
+    artifact would silently overwrite a real artifact from a prior
+    compat-test run on the same checkout.
     """
-    if not _COLLECTOR.items:
+    if not _COLLECTOR.items and not _is_xdist_worker(session):
         return
-    artifact_path = os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH
-    payload = {
-        "schema_version": "1",
-        "results": [
+    artifact_path = Path(os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH)
+    shard_dir = _shard_dir(artifact_path)
+    shard_dir.mkdir(parents=True, exist_ok=True)
+
+    worker_id = _xdist_worker_id(session) or "main"
+    shard_path = shard_dir / f"{worker_id}.json"
+    shard_path.write_text(
+        json.dumps(
             {
-                "feature_id": item.feature_id,
-                "provider": item.provider,
-                "nodeid": item.nodeid,
-                "result": item.result,
-            }
-            for item in _COLLECTOR.items
-        ],
-    }
-    Path(artifact_path).write_text(json.dumps(payload, indent=2, sort_keys=True))
+                "schema_version": "1",
+                "worker_id": worker_id,
+                "results": _serialize_items(_COLLECTOR.items),
+            },
+            indent=2,
+            sort_keys=True,
+        )
+    )
+
+    # Workers stop here. The controller merges; if we're not running
+    # under xdist, we are effectively the controller.
+    if _is_xdist_worker(session):
+        return
+
+    merged_rows: List[Dict[str, Any]] = []
+    for shard_file in sorted(shard_dir.glob("*.json")):
+        try:
+            shard = json.loads(shard_file.read_text())
+        except (OSError, ValueError):
+            continue
+        rows = shard.get("results")
+        if isinstance(rows, list):
+            merged_rows.extend(rows)
+
+    # Skip writing artifact + summary entirely for unit-test-only runs
+    # (no per-feature compat rows). Otherwise every `pytest tests/...`
+    # run — including local unit-test invocations — would silently
+    # overwrite a real artifact from a prior compat-test run.
+    if not merged_rows:
+        return
+
+    artifact_path.write_text(
+        json.dumps(
+            {"schema_version": "1", "results": merged_rows},
+            indent=2,
+            sort_keys=True,
+        )
+    )
+
+    summary = _build_rate_limit_summary(merged_rows)
+    summary_path = Path(
+        os.environ.get(RATE_LIMIT_SUMMARY_ENV) or DEFAULT_RATE_LIMIT_SUMMARY_PATH
+    )
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True))
+    _print_rate_limit_summary(summary)

tests/claude_code/extended_thinking/test_anthropic.py

@@ -12,6 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by
     tests/claude_code/extended_thinking/test_anthropic.py
                        ^^^^^^^^^^^^^^^^^      ^^^^^^^^^
                        feature_id             provider
+
+The three Claude tiers run in parallel inside this single test, with
+one `compat_result.add(...)` entry per model so the matrix builder
+still sees three rows for this (feature, provider).
 """
 
 from __future__ import annotations
@@ -21,7 +25,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -58,8 +66,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", ANTHROPIC_MODELS)
-def test_extended_thinking_anthropic(compat_result, model):
+def test_extended_thinking_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy with thinking
     enabled and assert a `thinking` content block was emitted."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -78,39 +85,38 @@ def test_extended_thinking_anthropic(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=THINKING_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_env=THINKING_ENV,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt=THINKING_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_thinking_block(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no `thinking` content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no thinking block for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_thinking_block(outcome.events):
+            error = (
+                f"[{model}] no `thinking` content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/extended_thinking/test_azure.py

@@ -28,7 +28,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -60,8 +64,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", AZURE_MODELS)
-def test_extended_thinking_azure(compat_result, model):
+def test_extended_thinking_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy with thinking
     enabled and assert a `thinking` content block was emitted."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -80,39 +83,38 @@ def test_extended_thinking_azure(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=THINKING_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_env=THINKING_ENV,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt=THINKING_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_thinking_block(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no `thinking` content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no thinking block for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_thinking_block(outcome.events):
+            error = (
+                f"[{model}] no `thinking` content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/extended_thinking/test_bedrock_converse.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -52,8 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS)
-def test_extended_thinking_bedrock_converse(compat_result, model):
+def test_extended_thinking_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy with thinking
     enabled and assert a `thinking` content block was emitted."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -72,39 +75,38 @@ def test_extended_thinking_bedrock_converse(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=THINKING_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_env=THINKING_ENV,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt=THINKING_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_thinking_block(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no `thinking` content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no thinking block for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_thinking_block(outcome.events):
+            error = (
+                f"[{model}] no `thinking` content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/extended_thinking/test_bedrock_invoke.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -52,8 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS)
-def test_extended_thinking_bedrock_invoke(compat_result, model):
+def test_extended_thinking_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy with thinking
     enabled and assert a `thinking` content block was emitted."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -72,39 +75,38 @@ def test_extended_thinking_bedrock_invoke(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=THINKING_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_env=THINKING_ENV,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt=THINKING_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_thinking_block(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no `thinking` content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no thinking block for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_thinking_block(outcome.events):
+            error = (
+                f"[{model}] no `thinking` content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/extended_thinking/test_vertex_ai.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -52,8 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", VERTEX_AI_MODELS)
-def test_extended_thinking_vertex_ai(compat_result, model):
+def test_extended_thinking_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy with thinking
     enabled and assert a `thinking` content block was emitted."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -72,39 +75,38 @@ def test_extended_thinking_vertex_ai(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=THINKING_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_env=THINKING_ENV,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt=THINKING_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_thinking_block(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no `thinking` content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no thinking block for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_thinking_block(outcome.events):
+            error = (
+                f"[{model}] no `thinking` content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/manifest.yaml

@@ -34,3 +34,13 @@ features:
     name: Vision
   - id: extended_thinking
     name: Extended thinking
+  - id: tool_use_streaming
+    name: Tool use (streaming / fine-grained)
+  - id: thinking_with_tool_use
+    name: Extended thinking + tool use
+  - id: pdf_input
+    name: PDF document input
+  - id: prompt_caching_1h
+    name: Prompt caching (1h TTL)
+  - id: web_search
+    name: Web search (server tool)

tests/claude_code/pdf_input/test_anthropic.py

@@ -0,0 +1,176 @@
+"""pdf_input x Anthropic.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+to Anthropic, write a tiny valid PDF to disk, allow the built-in `Read`
+tool, and ask Claude to read the PDF and report what it contains.
+
+The Read tool inlines the PDF bytes as `document` content blocks on the
+next assistant turn, which is exactly the gateway path we want to
+exercise: it's distinct from image content blocks (which are tested in
+`vision/`) and uses a different transformation in LiteLLM's Anthropic
+provider. We assert the upstream produces a non-empty reply that
+references the contents of the PDF — proving the proxy preserved the
+document content block end-to-end.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/pdf_input/test_anthropic.py
+                       ^^^^^^^^^      ^^^^^^^^^
+                       feature_id     provider
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+ANTHROPIC_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+# Smallest valid PDF that renders a single visible word ("PONG"). Built
+# inline rather than checked in as a binary fixture so the test stays
+# self-contained and the marker word is easy to grep for in CI logs.
+# The structure is a hand-crafted single-page PDF with one Helvetica
+# text show; offsets are computed at write time so the xref table
+# stays consistent regardless of platform line endings.
+PDF_MARKER = "PONG"
+
+
+def _build_minimal_pdf(marker: str) -> bytes:
+    """Return a single-page PDF whose only visible text is `marker`.
+
+    We construct the PDF imperatively because `pypdf`/`reportlab` are
+    not in the test deps and we want the cell to work in a clean
+    environment. The xref offsets are recomputed for each `marker`
+    length so the file stays well-formed.
+    """
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        (
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
+            b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>"
+        ),
+        # Page content stream: position the text and show the marker.
+        # `BT ... ET` is a text object; `Tf` selects font, `Td` moves
+        # the cursor, `Tj` paints a string.
+        (
+            b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td ("
+            + marker.encode("ascii")
+            + b") Tj ET\nendstream"
+        ),
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ]
+    # Fix up the /Length on the content stream to match its body.
+    body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n")
+    body_close = objects[3].index(b"\nendstream")
+    body_len = body_close - body_open
+    objects[3] = (
+        b"<< /Length "
+        + str(body_len).encode("ascii")
+        + b" >>\nstream\n"
+        + objects[3][body_open:body_close]
+        + b"\nendstream"
+    )
+
+    out = bytearray(b"%PDF-1.4\n")
+    offsets = []
+    for i, obj in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
+
+    xref_offset = len(out)
+    out += b"xref\n0 %d\n" % (len(objects) + 1)
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += f"{off:010d} 00000 n \n".encode("ascii")
+    out += (
+        b"trailer\n<< /Size "
+        + str(len(objects) + 1).encode("ascii")
+        + b" /Root 1 0 R >>\nstartxref\n"
+        + str(xref_offset).encode("ascii")
+        + b"\n%%EOF\n"
+    )
+    return bytes(out)
+
+
+def test_pdf_input_anthropic(compat_result, tmp_path):
+    """Drive the `claude` CLI against the LiteLLM proxy with a PDF
+    attached via the Read tool and assert the reply references it."""
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    pdf_path = tmp_path / "marker.pdf"
+    pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER))
+
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt=(
+            f"Use the Read tool to read the file at {pdf_path}. "
+            "Report the single word that appears in the document."
+        ),
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--allowed-tools", "Read"],
+    )
+
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        # The strongest gateway-level signal we can assert without
+        # parsing every event type: the model's final user-visible
+        # reply names the marker word that only the PDF carries.
+        # If the proxy dropped the `document` content block, the
+        # model has no way to produce this token.
+        if PDF_MARKER not in outcome.text.upper():
+            error = (
+                f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; "
+                f"got: {outcome.text.strip()!r}"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/pdf_input/test_azure.py

@@ -0,0 +1,146 @@
+"""pdf_input x Microsoft Foundry (Azure).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to Microsoft Foundry's Anthropic deployments on Azure,
+write a tiny valid PDF to disk, allow the built-in `Read` tool, and
+ask Claude to read the PDF and report what it contains.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/pdf_input/test_azure.py
+                       ^^^^^^^^^      ^^^^^
+                       feature_id     provider
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+AZURE_MODELS = [
+    "claude-haiku-4-5-azure",
+    "claude-sonnet-4-6-azure",
+    "claude-opus-4-7-azure",
+]
+
+PDF_MARKER = "PONG"
+
+
+def _build_minimal_pdf(marker: str) -> bytes:
+    """Return a single-page PDF whose only visible text is `marker`."""
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        (
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
+            b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>"
+        ),
+        (
+            b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td ("
+            + marker.encode("ascii")
+            + b") Tj ET\nendstream"
+        ),
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ]
+    body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n")
+    body_close = objects[3].index(b"\nendstream")
+    body_len = body_close - body_open
+    objects[3] = (
+        b"<< /Length "
+        + str(body_len).encode("ascii")
+        + b" >>\nstream\n"
+        + objects[3][body_open:body_close]
+        + b"\nendstream"
+    )
+
+    out = bytearray(b"%PDF-1.4\n")
+    offsets = []
+    for i, obj in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
+
+    xref_offset = len(out)
+    out += b"xref\n0 %d\n" % (len(objects) + 1)
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += f"{off:010d} 00000 n \n".encode("ascii")
+    out += (
+        b"trailer\n<< /Size "
+        + str(len(objects) + 1).encode("ascii")
+        + b" /Root 1 0 R >>\nstartxref\n"
+        + str(xref_offset).encode("ascii")
+        + b"\n%%EOF\n"
+    )
+    return bytes(out)
+
+
+def test_pdf_input_azure(compat_result, tmp_path):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    pdf_path = tmp_path / "marker.pdf"
+    pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER))
+
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt=(
+            f"Use the Read tool to read the file at {pdf_path}. "
+            "Report the single word that appears in the document."
+        ),
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--allowed-tools", "Read"],
+    )
+
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if PDF_MARKER not in outcome.text.upper():
+            error = (
+                f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; "
+                f"got: {outcome.text.strip()!r}"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/pdf_input/test_bedrock_converse.py

@@ -0,0 +1,152 @@
+"""pdf_input x Bedrock (Converse).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the `Converse` API, write a tiny
+valid PDF to disk, allow the built-in `Read` tool, and ask Claude to
+read the PDF and report what it contains.
+
+Bedrock Converse expresses documents via its own
+`document = { format, name, source: { bytes } }` shape; this cell
+catches gateway regressions where the proxy fails to translate
+Anthropic's `document` content block to Converse's document format
+(or vice versa on the response).
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/pdf_input/test_bedrock_converse.py
+                       ^^^^^^^^^      ^^^^^^^^^^^^^^^^
+                       feature_id     provider
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_CONVERSE_MODELS = [
+    "claude-haiku-4-5-bedrock-converse",
+    "claude-sonnet-4-6-bedrock-converse",
+    "claude-opus-4-7-bedrock-converse",
+]
+
+PDF_MARKER = "PONG"
+
+
+def _build_minimal_pdf(marker: str) -> bytes:
+    """Return a single-page PDF whose only visible text is `marker`."""
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        (
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
+            b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>"
+        ),
+        (
+            b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td ("
+            + marker.encode("ascii")
+            + b") Tj ET\nendstream"
+        ),
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ]
+    body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n")
+    body_close = objects[3].index(b"\nendstream")
+    body_len = body_close - body_open
+    objects[3] = (
+        b"<< /Length "
+        + str(body_len).encode("ascii")
+        + b" >>\nstream\n"
+        + objects[3][body_open:body_close]
+        + b"\nendstream"
+    )
+
+    out = bytearray(b"%PDF-1.4\n")
+    offsets = []
+    for i, obj in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
+
+    xref_offset = len(out)
+    out += b"xref\n0 %d\n" % (len(objects) + 1)
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += f"{off:010d} 00000 n \n".encode("ascii")
+    out += (
+        b"trailer\n<< /Size "
+        + str(len(objects) + 1).encode("ascii")
+        + b" /Root 1 0 R >>\nstartxref\n"
+        + str(xref_offset).encode("ascii")
+        + b"\n%%EOF\n"
+    )
+    return bytes(out)
+
+
+def test_pdf_input_bedrock_converse(compat_result, tmp_path):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    pdf_path = tmp_path / "marker.pdf"
+    pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER))
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt=(
+            f"Use the Read tool to read the file at {pdf_path}. "
+            "Report the single word that appears in the document."
+        ),
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--allowed-tools", "Read"],
+    )
+
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if PDF_MARKER not in outcome.text.upper():
+            error = (
+                f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; "
+                f"got: {outcome.text.strip()!r}"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/pdf_input/test_bedrock_invoke.py

@@ -0,0 +1,151 @@
+"""pdf_input x Bedrock (Invoke).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the legacy `InvokeModel` API path,
+write a tiny valid PDF to disk, allow the built-in `Read` tool, and
+ask Claude to read the PDF and report what it contains.
+
+Bedrock InvokeModel for Anthropic models accepts the native
+`document` content block shape; this cell catches gateway regressions
+where the proxy drops or mis-encodes the document content block on
+the way through (e.g. base64-only encoding, missing media_type, etc.).
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/pdf_input/test_bedrock_invoke.py
+                       ^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id     provider
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_INVOKE_MODELS = [
+    "claude-haiku-4-5-bedrock-invoke",
+    "claude-sonnet-4-6-bedrock-invoke",
+    "claude-opus-4-7-bedrock-invoke",
+]
+
+PDF_MARKER = "PONG"
+
+
+def _build_minimal_pdf(marker: str) -> bytes:
+    """Return a single-page PDF whose only visible text is `marker`."""
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        (
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
+            b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>"
+        ),
+        (
+            b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td ("
+            + marker.encode("ascii")
+            + b") Tj ET\nendstream"
+        ),
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ]
+    body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n")
+    body_close = objects[3].index(b"\nendstream")
+    body_len = body_close - body_open
+    objects[3] = (
+        b"<< /Length "
+        + str(body_len).encode("ascii")
+        + b" >>\nstream\n"
+        + objects[3][body_open:body_close]
+        + b"\nendstream"
+    )
+
+    out = bytearray(b"%PDF-1.4\n")
+    offsets = []
+    for i, obj in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
+
+    xref_offset = len(out)
+    out += b"xref\n0 %d\n" % (len(objects) + 1)
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += f"{off:010d} 00000 n \n".encode("ascii")
+    out += (
+        b"trailer\n<< /Size "
+        + str(len(objects) + 1).encode("ascii")
+        + b" /Root 1 0 R >>\nstartxref\n"
+        + str(xref_offset).encode("ascii")
+        + b"\n%%EOF\n"
+    )
+    return bytes(out)
+
+
+def test_pdf_input_bedrock_invoke(compat_result, tmp_path):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    pdf_path = tmp_path / "marker.pdf"
+    pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER))
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt=(
+            f"Use the Read tool to read the file at {pdf_path}. "
+            "Report the single word that appears in the document."
+        ),
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--allowed-tools", "Read"],
+    )
+
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if PDF_MARKER not in outcome.text.upper():
+            error = (
+                f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; "
+                f"got: {outcome.text.strip()!r}"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/pdf_input/test_vertex_ai.py

@@ -0,0 +1,146 @@
+"""pdf_input x Vertex AI.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to GCP Vertex AI, write a tiny valid PDF to disk, allow
+the built-in `Read` tool, and ask Claude to read the PDF and report
+what it contains.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/pdf_input/test_vertex_ai.py
+                       ^^^^^^^^^      ^^^^^^^^^
+                       feature_id     provider
+"""
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+VERTEX_AI_MODELS = [
+    "claude-haiku-4-5-vertex",
+    "claude-sonnet-4-6-vertex",
+    "claude-opus-4-7-vertex",
+]
+
+PDF_MARKER = "PONG"
+
+
+def _build_minimal_pdf(marker: str) -> bytes:
+    """Return a single-page PDF whose only visible text is `marker`."""
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        (
+            b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] "
+            b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>"
+        ),
+        (
+            b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td ("
+            + marker.encode("ascii")
+            + b") Tj ET\nendstream"
+        ),
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+    ]
+    body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n")
+    body_close = objects[3].index(b"\nendstream")
+    body_len = body_close - body_open
+    objects[3] = (
+        b"<< /Length "
+        + str(body_len).encode("ascii")
+        + b" >>\nstream\n"
+        + objects[3][body_open:body_close]
+        + b"\nendstream"
+    )
+
+    out = bytearray(b"%PDF-1.4\n")
+    offsets = []
+    for i, obj in enumerate(objects, start=1):
+        offsets.append(len(out))
+        out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
+
+    xref_offset = len(out)
+    out += b"xref\n0 %d\n" % (len(objects) + 1)
+    out += b"0000000000 65535 f \n"
+    for off in offsets:
+        out += f"{off:010d} 00000 n \n".encode("ascii")
+    out += (
+        b"trailer\n<< /Size "
+        + str(len(objects) + 1).encode("ascii")
+        + b" /Root 1 0 R >>\nstartxref\n"
+        + str(xref_offset).encode("ascii")
+        + b"\n%%EOF\n"
+    )
+    return bytes(out)
+
+
+def test_pdf_input_vertex_ai(compat_result, tmp_path):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    pdf_path = tmp_path / "marker.pdf"
+    pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER))
+
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt=(
+            f"Use the Read tool to read the file at {pdf_path}. "
+            "Report the single word that appears in the document."
+        ),
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--allowed-tools", "Read"],
+    )
+
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if PDF_MARKER not in outcome.text.upper():
+            error = (
+                f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; "
+                f"got: {outcome.text.strip()!r}"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_1h/test_anthropic.py

@@ -0,0 +1,124 @@
+"""prompt_caching_1h x Anthropic.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+to Anthropic, opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`,
+and assert that the upstream's usage block reports either
+`cache_creation_input_tokens` or `cache_read_input_tokens` > 0 — i.e.
+the proxy preserved Claude Code's `cache_control: { ttl: "1h" }`
+annotations end-to-end and the upstream actually honored them.
+
+This is the 1-hour-TTL companion to `prompt_caching_5m/`. It exists as
+its own cell because the 1h TTL travels through the proxy with a
+distinct `cache_control` shape (and a distinct beta-header gate on
+some providers); a regression that strips or downgrades the TTL on the
+way through is invisible to the 5m cell, which would still see cache
+hits with a default-TTL annotation.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/prompt_caching_1h/test_anthropic.py
+                       ^^^^^^^^^^^^^^^^^      ^^^^^^^^^
+                       feature_id             provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Optional
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+ANTHROPIC_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+# Per the changelog (2.1.108): `ENABLE_PROMPT_CACHING_1H` flips Claude
+# Code from the default 5-minute cache TTL to a 1-hour TTL on the
+# `cache_control` annotations it adds to the system prompt and the
+# most recent user turn. Setting it here is what we are validating
+# the proxy faithfully forwards to the upstream.
+CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"}
+
+
+def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
+    """Return cache_creation_input_tokens + cache_read_input_tokens from
+    the upstream usage block, or 0 if the keys are missing."""
+    if not isinstance(usage, Mapping):
+        return 0
+    creation = usage.get("cache_creation_input_tokens") or 0
+    read = usage.get("cache_read_input_tokens") or 0
+    try:
+        return int(creation) + int(read)
+    except (TypeError, ValueError):
+        return 0
+
+
+def test_prompt_caching_1h_anthropic(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy with the 1h
+    TTL opt-in env var set, and assert the upstream usage block
+    surfaces a non-zero cache token count."""
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=CACHE_1H_ENV,
+    )
+
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens with "
+                "ENABLE_PROMPT_CACHING_1H=1; the proxy likely stripped the "
+                "1h TTL beta header or rejected the cache_control shape"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_1h/test_azure.py

@@ -0,0 +1,104 @@
+"""prompt_caching_1h x Microsoft Foundry (Azure).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to Microsoft Foundry's Anthropic deployments on Azure,
+opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and
+assert the upstream's usage block reports a non-zero cache token count.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/prompt_caching_1h/test_azure.py
+                       ^^^^^^^^^^^^^^^^^      ^^^^^
+                       feature_id             provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Optional
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+AZURE_MODELS = [
+    "claude-haiku-4-5-azure",
+    "claude-sonnet-4-6-azure",
+    "claude-opus-4-7-azure",
+]
+
+CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"}
+
+
+def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
+    if not isinstance(usage, Mapping):
+        return 0
+    creation = usage.get("cache_creation_input_tokens") or 0
+    read = usage.get("cache_read_input_tokens") or 0
+    try:
+        return int(creation) + int(read)
+    except (TypeError, ValueError):
+        return 0
+
+
+def test_prompt_caching_1h_azure(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=CACHE_1H_ENV,
+    )
+
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens with "
+                "ENABLE_PROMPT_CACHING_1H=1"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_1h/test_bedrock_converse.py

@@ -0,0 +1,112 @@
+"""prompt_caching_1h x Bedrock (Converse).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the `Converse` API, opt into the
+1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and assert the
+upstream's usage block reports a non-zero cache token count.
+
+Bedrock Converse expresses prompt caching via `cachePoint` markers in
+the message list, with TTL controlled out-of-band; this cell catches
+proxy regressions where the 1h opt-in fails to translate into the
+correct Converse cache configuration.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/prompt_caching_1h/test_bedrock_converse.py
+                       ^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^^^
+                       feature_id             provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Optional
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_CONVERSE_MODELS = [
+    "claude-haiku-4-5-bedrock-converse",
+    "claude-sonnet-4-6-bedrock-converse",
+    "claude-opus-4-7-bedrock-converse",
+]
+
+CACHE_1H_ENV = {
+    "ENABLE_PROMPT_CACHING_1H": "1",
+    "ENABLE_PROMPT_CACHING_1H_BEDROCK": "1",
+}
+
+
+def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
+    if not isinstance(usage, Mapping):
+        return 0
+    creation = usage.get("cache_creation_input_tokens") or 0
+    read = usage.get("cache_read_input_tokens") or 0
+    try:
+        return int(creation) + int(read)
+    except (TypeError, ValueError):
+        return 0
+
+
+def test_prompt_caching_1h_bedrock_converse(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=CACHE_1H_ENV,
+    )
+
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens with "
+                "1h-TTL opt-in env vars set"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py

@@ -0,0 +1,117 @@
+"""prompt_caching_1h x Bedrock (Invoke).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the legacy `InvokeModel` API path,
+opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and
+assert the upstream's usage block reports a non-zero cache token count.
+
+Bedrock historically gated 1h prompt caching behind a separate
+`ENABLE_PROMPT_CACHING_1H_BEDROCK` env var (see 2.1.108: deprecated but
+still honored). The proxy must accept either env var and forward an
+appropriate `cache_control` shape to the Bedrock InvokeModel endpoint;
+this cell catches regressions where the TTL is silently downgraded to
+5 minutes on the way through.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py
+                       ^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id             provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Optional
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_INVOKE_MODELS = [
+    "claude-haiku-4-5-bedrock-invoke",
+    "claude-sonnet-4-6-bedrock-invoke",
+    "claude-opus-4-7-bedrock-invoke",
+]
+
+# Set both the modern and the deprecated-but-honored Bedrock var so we
+# match whichever code path the proxy is following.
+CACHE_1H_ENV = {
+    "ENABLE_PROMPT_CACHING_1H": "1",
+    "ENABLE_PROMPT_CACHING_1H_BEDROCK": "1",
+}
+
+
+def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
+    if not isinstance(usage, Mapping):
+        return 0
+    creation = usage.get("cache_creation_input_tokens") or 0
+    read = usage.get("cache_read_input_tokens") or 0
+    try:
+        return int(creation) + int(read)
+    except (TypeError, ValueError):
+        return 0
+
+
+def test_prompt_caching_1h_bedrock_invoke(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=CACHE_1H_ENV,
+    )
+
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens with "
+                "1h-TTL opt-in env vars set; the proxy likely stripped or "
+                "downgraded the cache_control TTL"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_1h/test_vertex_ai.py

@@ -0,0 +1,104 @@
+"""prompt_caching_1h x Vertex AI.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to GCP Vertex AI, opt into the 1-hour cache TTL via
+`ENABLE_PROMPT_CACHING_1H`, and assert the upstream's usage block
+reports a non-zero cache token count.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/prompt_caching_1h/test_vertex_ai.py
+                       ^^^^^^^^^^^^^^^^^      ^^^^^^^^^
+                       feature_id             provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Optional
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+VERTEX_AI_MODELS = [
+    "claude-haiku-4-5-vertex",
+    "claude-sonnet-4-6-vertex",
+    "claude-opus-4-7-vertex",
+]
+
+CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"}
+
+
+def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
+    if not isinstance(usage, Mapping):
+        return 0
+    creation = usage.get("cache_creation_input_tokens") or 0
+    read = usage.get("cache_read_input_tokens") or 0
+    try:
+        return int(creation) + int(read)
+    except (TypeError, ValueError):
+        return 0
+
+
+def test_prompt_caching_1h_vertex_ai(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=CACHE_1H_ENV,
+    )
+
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens with "
+                "ENABLE_PROMPT_CACHING_1H=1"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_5m/test_anthropic.py

@@ -27,7 +27,11 @@ from typing import Any, Mapping, Optional
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -52,8 +56,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
         return 0
 
 
-@pytest.mark.parametrize("model", ANTHROPIC_MODELS)
-def test_prompt_caching_5m_anthropic(compat_result, model):
+def test_prompt_caching_5m_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert the
     upstream usage block surfaces a non-zero cache token count."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -72,42 +75,39 @@ def test_prompt_caching_5m_anthropic(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if _cache_tokens(result.usage) <= 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": (
-                    f"[{model}] usage block reported zero cache tokens; "
-                    "expected cache_control on the system prompt to produce a non-zero "
-                    "cache_creation_input_tokens or cache_read_input_tokens"
-                ),
-            }
-        )
-        pytest.fail(f"no cache tokens for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens; "
+                "expected cache_control on the system prompt to produce a non-zero "
+                "cache_creation_input_tokens or cache_read_input_tokens"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_5m/test_azure.py

@@ -27,7 +27,11 @@ from typing import Any, Mapping, Optional
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -50,8 +54,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
         return 0
 
 
-@pytest.mark.parametrize("model", AZURE_MODELS)
-def test_prompt_caching_5m_azure(compat_result, model):
+def test_prompt_caching_5m_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert the
     upstream usage block surfaces a non-zero cache token count."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -70,42 +73,39 @@ def test_prompt_caching_5m_azure(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if _cache_tokens(result.usage) <= 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": (
-                    f"[{model}] usage block reported zero cache tokens; "
-                    "expected cache_control on the system prompt to produce a non-zero "
-                    "cache_creation_input_tokens or cache_read_input_tokens"
-                ),
-            }
-        )
-        pytest.fail(f"no cache tokens for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens; "
+                "expected cache_control on the system prompt to produce a non-zero "
+                "cache_creation_input_tokens or cache_read_input_tokens"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_5m/test_bedrock_converse.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Optional
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -43,8 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
         return 0
 
 
-@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS)
-def test_prompt_caching_5m_bedrock_converse(compat_result, model):
+def test_prompt_caching_5m_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert the
     upstream usage block surfaces a non-zero cache token count."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -63,42 +66,39 @@ def test_prompt_caching_5m_bedrock_converse(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if _cache_tokens(result.usage) <= 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": (
-                    f"[{model}] usage block reported zero cache tokens; "
-                    "expected cache_control on the system prompt to produce a non-zero "
-                    "cache_creation_input_tokens or cache_read_input_tokens"
-                ),
-            }
-        )
-        pytest.fail(f"no cache tokens for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens; "
+                "expected cache_control on the system prompt to produce a non-zero "
+                "cache_creation_input_tokens or cache_read_input_tokens"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Optional
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -43,8 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
         return 0
 
 
-@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS)
-def test_prompt_caching_5m_bedrock_invoke(compat_result, model):
+def test_prompt_caching_5m_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert the
     upstream usage block surfaces a non-zero cache token count."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -63,42 +66,39 @@ def test_prompt_caching_5m_bedrock_invoke(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if _cache_tokens(result.usage) <= 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": (
-                    f"[{model}] usage block reported zero cache tokens; "
-                    "expected cache_control on the system prompt to produce a non-zero "
-                    "cache_creation_input_tokens or cache_read_input_tokens"
-                ),
-            }
-        )
-        pytest.fail(f"no cache tokens for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens; "
+                "expected cache_control on the system prompt to produce a non-zero "
+                "cache_creation_input_tokens or cache_read_input_tokens"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/prompt_caching_5m/test_vertex_ai.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Optional
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -43,8 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
         return 0
 
 
-@pytest.mark.parametrize("model", VERTEX_AI_MODELS)
-def test_prompt_caching_5m_vertex_ai(compat_result, model):
+def test_prompt_caching_5m_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert the
     upstream usage block surfaces a non-zero cache token count."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -63,42 +66,39 @@ def test_prompt_caching_5m_vertex_ai(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt="Reply with the single word 'pong' and nothing else.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        base_url=base_url,
+        api_key=api_key,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if _cache_tokens(result.usage) <= 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": (
-                    f"[{model}] usage block reported zero cache tokens; "
-                    "expected cache_control on the system prompt to produce a non-zero "
-                    "cache_creation_input_tokens or cache_read_input_tokens"
-                ),
-            }
-        )
-        pytest.fail(f"no cache tokens for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if _cache_tokens(outcome.usage) <= 0:
+            error = (
+                f"[{model}] usage block reported zero cache tokens; "
+                "expected cache_control on the system prompt to produce a non-zero "
+                "cache_creation_input_tokens or cache_read_input_tokens"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/rate_limiter.py

@@ -0,0 +1,358 @@
+"""Cross-process token-bucket rate limiter for the Claude Code compat suite.
+
+The compat matrix runs 75 live `claude` CLI invocations (25 cells × 3
+Claude tiers per cell). When pytest-xdist fans these out across worker
+processes, each worker would maintain its own in-memory rate limiter
+and the *aggregate* request rate hitting any one upstream provider
+would be `workers × per-worker rate` — exactly the situation that
+trips Anthropic / Azure / Bedrock / Vertex 429s in the middle of a
+matrix run and silently flips green cells red.
+
+The fix is a **shared, cross-process** token bucket per provider,
+backed by a small JSON state file and an OS-level `flock`. Each
+`run_claude` invocation acquires one token (sleeping if the bucket is
+empty) before launching the CLI; refills happen lazily based on wall
+time, so workers can be killed and restarted without losing or
+double-spending budget.
+
+Configuration is driven entirely by environment variables so a
+binary-search workflow can shift per-provider rates without code
+edits:
+
+    LITELLM_COMPAT_RATE_ANTHROPIC          (req/s, default 5.0)
+    LITELLM_COMPAT_RATE_AZURE              (req/s, default 5.0)
+    LITELLM_COMPAT_RATE_VERTEX_AI          (req/s, default 5.0)
+    LITELLM_COMPAT_RATE_BEDROCK_CONVERSE   (req/s, default 5.0)
+    LITELLM_COMPAT_RATE_BEDROCK_INVOKE     (req/s, default 5.0)
+    LITELLM_COMPAT_RATE_BURST              (per-bucket burst override;
+                                            default = rate)
+    LITELLM_COMPAT_RATE_STATE_DIR          (state file directory;
+                                            default = $TMPDIR/litellm-claude-compat-ratelimit)
+
+A rate of 0 (or any non-positive value) disables throttling for that
+provider — useful when you trust the upstream to handle the burst or
+when running unit-test-shaped workloads that never actually hit the
+network.
+
+The provider id is inferred from the model id by `infer_provider`,
+mirroring the matrix's column layout (`anthropic`, `azure`,
+`vertex_ai`, `bedrock_converse`, `bedrock_invoke`).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import math
+import os
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, Iterator, Mapping, Optional
+
+# `fcntl` is POSIX-only; the suite is Linux/macOS only, so we don't
+# attempt a Windows fallback. Importing at module load fails fast on
+# the (currently non-existent) Windows runner so we don't silently
+# degrade to no-locking behavior.
+import fcntl
+
+PROVIDER_ANTHROPIC = "anthropic"
+PROVIDER_AZURE = "azure"
+PROVIDER_VERTEX_AI = "vertex_ai"
+PROVIDER_BEDROCK_CONVERSE = "bedrock_converse"
+PROVIDER_BEDROCK_INVOKE = "bedrock_invoke"
+
+ALL_PROVIDERS = (
+    PROVIDER_ANTHROPIC,
+    PROVIDER_AZURE,
+    PROVIDER_VERTEX_AI,
+    PROVIDER_BEDROCK_CONVERSE,
+    PROVIDER_BEDROCK_INVOKE,
+)
+
+DEFAULT_RATE = 5.0  # req/s per provider, conservative starting point
+RATE_ENV_PREFIX = "LITELLM_COMPAT_RATE_"
+BURST_ENV = "LITELLM_COMPAT_RATE_BURST"
+STATE_DIR_ENV = "LITELLM_COMPAT_RATE_STATE_DIR"
+DEFAULT_STATE_DIR_NAME = "litellm-claude-compat-ratelimit"
+
+
+def infer_provider(model: str) -> str:
+    """Map a model alias to its compat-matrix provider id.
+
+    The matrix column layout is fixed; aliases registered in the proxy
+    encode the provider via a suffix (`-bedrock-converse`,
+    `-bedrock-invoke`, `-azure`, `-vertex`) or its absence (Anthropic).
+    Order matters: the bedrock suffixes both contain `bedrock`, so we
+    test the more-specific ones first.
+    """
+    if not model:
+        raise ValueError("model must be a non-empty string")
+    lower = model.lower()
+    if lower.endswith("-bedrock-converse"):
+        return PROVIDER_BEDROCK_CONVERSE
+    if lower.endswith("-bedrock-invoke"):
+        return PROVIDER_BEDROCK_INVOKE
+    if lower.endswith("-azure"):
+        return PROVIDER_AZURE
+    if lower.endswith("-vertex"):
+        return PROVIDER_VERTEX_AI
+    return PROVIDER_ANTHROPIC
+
+
+@dataclass(frozen=True)
+class ProviderConfig:
+    """Static config snapshot for one provider's bucket.
+
+    Captured up front (rather than re-read per acquire) so the limiter's
+    behavior in a single process is stable even if env vars are mutated
+    mid-run. A fresh `RateLimiter` picks up env changes on construction.
+    """
+
+    rate_per_sec: float
+    burst: float
+
+    @property
+    def enabled(self) -> bool:
+        return self.rate_per_sec > 0 and self.burst > 0
+
+
+def load_config(
+    env: Optional[Mapping[str, str]] = None,
+) -> Dict[str, ProviderConfig]:
+    """Build a {provider: ProviderConfig} from env, applying defaults.
+
+    Parsing failures fall back to the default rate rather than
+    crashing the test session — a typo in `LITELLM_COMPAT_RATE_AZURE`
+    should not silently disable throttling, but it also shouldn't
+    abort 75 live tests with a `ValueError` ten minutes in.
+    """
+    src = env if env is not None else os.environ
+
+    def _as_float(value: Optional[str], default: float) -> float:
+        if value is None or value == "":
+            return default
+        try:
+            return float(value)
+        except (TypeError, ValueError):
+            return default
+
+    burst_override = _as_float(src.get(BURST_ENV), -1.0)
+
+    out: Dict[str, ProviderConfig] = {}
+    for provider in ALL_PROVIDERS:
+        env_key = RATE_ENV_PREFIX + provider.upper()
+        rate = _as_float(src.get(env_key), DEFAULT_RATE)
+        burst = burst_override if burst_override > 0 else max(rate, 1.0)
+        out[provider] = ProviderConfig(rate_per_sec=rate, burst=burst)
+    return out
+
+
+def _state_dir(env: Optional[Mapping[str, str]] = None) -> Path:
+    """Resolve the directory holding per-provider state files.
+
+    A user-supplied `LITELLM_COMPAT_RATE_STATE_DIR` wins for tests and
+    container environments where `$TMPDIR` may be ephemeral or shared
+    in surprising ways. The directory is created lazily with
+    `parents=True, exist_ok=True` so first-run setup needs no
+    fixture wiring.
+    """
+    src = env if env is not None else os.environ
+    explicit = src.get(STATE_DIR_ENV)
+    if explicit:
+        return Path(explicit)
+    return Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME
+
+
+class RateLimiter:
+    """Cross-process token bucket per provider.
+
+    Each `acquire(provider)` call:
+      1. Opens (creating if needed) `<state_dir>/<provider>.json`.
+      2. Holds an exclusive `flock` while reading + updating the
+         {tokens, last_refill} state.
+      3. Refills tokens based on `now - last_refill`, capped at burst.
+      4. If tokens >= 1, subtracts one and returns immediately.
+      5. Otherwise, computes the wall-time delay needed to earn a
+         single token at the configured rate, releases the lock, and
+         sleeps. After sleeping it retries — staying under the lock
+         while sleeping would serialize all workers behind whichever
+         one held it longest.
+
+    This bounds the *aggregate* req/s seen by the upstream, regardless
+    of how many xdist workers, threads, or processes are concurrently
+    running tests against the same provider.
+
+    A `_clock` / `_sleep` injection seam keeps the unit tests fast and
+    deterministic; production callers should never override either.
+    """
+
+    def __init__(
+        self,
+        config: Optional[Mapping[str, ProviderConfig]] = None,
+        state_dir: Optional[Path] = None,
+        clock: Optional[callable] = None,
+        sleep: Optional[callable] = None,
+    ) -> None:
+        self._config = dict(config) if config is not None else load_config()
+        self._state_dir = Path(state_dir) if state_dir is not None else _state_dir()
+        self._clock = clock or time.monotonic
+        self._sleep = sleep or time.sleep
+        # `_state_dir.mkdir` once on construction is fine; concurrent
+        # workers all racing to create the same directory is benign.
+        self._state_dir.mkdir(parents=True, exist_ok=True)
+
+    def acquire(self, provider: str) -> float:
+        """Block until one token is available for `provider`.
+
+        Returns the cumulative wall-time spent waiting (0.0 when the
+        bucket had budget and we returned immediately). Callers can
+        log this to attribute slow cells to throttling vs. upstream
+        latency — same role `DriverResult.duration_ms` plays for the
+        actual CLI invocation.
+        """
+        cfg = self._config.get(provider)
+        if cfg is None or not cfg.enabled:
+            return 0.0
+
+        path = self._state_path(provider)
+        total_waited = 0.0
+        while True:
+            now = self._clock()
+            sleep_for = self._try_consume(path, cfg, now)
+            if sleep_for <= 0:
+                return total_waited
+            self._sleep(sleep_for)
+            total_waited += sleep_for
+
+    def _state_path(self, provider: str) -> Path:
+        return self._state_dir / f"{provider}.json"
+
+    def _try_consume(self, path: Path, cfg: ProviderConfig, now: float) -> float:
+        """Atomically refill and try to take one token.
+
+        Returns 0.0 if a token was consumed, or a positive sleep
+        duration (seconds) if the caller must wait before retrying.
+
+        We hold an exclusive `flock` only across the read-modify-write
+        of the JSON state — never across a `sleep` — so workers don't
+        serialize while one of them is parked.
+        """
+        # `os.open` + `os.O_CREAT | os.O_RDWR` gives us a fd we can
+        # both lock and read/write through. Opening with `"a+"` then
+        # seeking is equivalent but uglier; this version is closer to
+        # the canonical flock recipe.
+        fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            try:
+                tokens, last_refill = self._read_state(fd, cfg, now)
+                tokens, last_refill = self._refill(tokens, last_refill, now, cfg)
+                if tokens >= 1.0:
+                    tokens -= 1.0
+                    self._write_state(fd, tokens, last_refill)
+                    return 0.0
+                # Not enough budget. Persist the refilled state so a
+                # subsequent caller doesn't have to redo the math, then
+                # release the lock and tell the caller how long to
+                # sleep before retrying.
+                self._write_state(fd, tokens, last_refill)
+                deficit = 1.0 - tokens
+                # `deficit / rate` seconds will earn exactly enough
+                # for one token. Add a tiny safety margin so we don't
+                # wake up nanoseconds early and spin.
+                return (deficit / cfg.rate_per_sec) + 1e-3
+            finally:
+                fcntl.flock(fd, fcntl.LOCK_UN)
+        finally:
+            os.close(fd)
+
+    @staticmethod
+    def _read_state(fd: int, cfg: ProviderConfig, now: float) -> tuple:
+        """Read {tokens, last_refill} from `fd`, defaulting to a full
+        bucket on a missing/empty/corrupt file.
+
+        New files start full so the very first request never waits;
+        corrupt files are treated like new files because the
+        alternative — refusing to run — is worse than briefly
+        over-spending one bucket's worth of budget.
+        """
+        os.lseek(fd, 0, os.SEEK_SET)
+        raw = os.read(fd, 4096).decode("utf-8")
+        if not raw.strip():
+            return cfg.burst, now
+        try:
+            obj = json.loads(raw)
+            tokens = float(obj.get("tokens", cfg.burst))
+            last_refill = float(obj.get("last_refill", now))
+            return tokens, last_refill
+        except (ValueError, TypeError):
+            return cfg.burst, now
+
+    @staticmethod
+    def _refill(
+        tokens: float, last_refill: float, now: float, cfg: ProviderConfig
+    ) -> tuple:
+        """Apply elapsed time to the bucket, capped at burst.
+
+        Negative elapsed (clock went backward, e.g. across host
+        sleep/resume or a manually-tweaked monotonic mock) is clamped
+        to zero so we never *remove* tokens.
+        """
+        elapsed = max(0.0, now - last_refill)
+        tokens = min(cfg.burst, tokens + elapsed * cfg.rate_per_sec)
+        return tokens, now
+
+    @staticmethod
+    def _write_state(fd: int, tokens: float, last_refill: float) -> None:
+        payload = json.dumps({"tokens": tokens, "last_refill": last_refill}).encode(
+            "utf-8"
+        )
+        os.lseek(fd, 0, os.SEEK_SET)
+        os.ftruncate(fd, 0)
+        os.write(fd, payload)
+
+
+# A single process-wide instance is all we need: provider-keyed state
+# is stored in files, so multiple `RateLimiter` instances would just
+# duplicate the in-process bookkeeping. We expose a getter rather than
+# the instance directly so unit tests can install a custom limiter
+# scoped to a tmp directory without monkeypatching globals.
+_default: Optional[RateLimiter] = None
+
+
+def get_default_limiter() -> RateLimiter:
+    global _default
+    if _default is None:
+        _default = RateLimiter()
+    return _default
+
+
+def reset_default_limiter() -> None:
+    """Drop the cached default limiter; the next `get_default_limiter`
+    call rebuilds it from the current environment.
+
+    Useful between unit tests that patch env vars: without this they'd
+    keep reading the stale config snapshot from the first call.
+    """
+    global _default
+    _default = None
+
+
+@contextlib.contextmanager
+def use_limiter(limiter: RateLimiter) -> Iterator[RateLimiter]:
+    """Temporarily install `limiter` as the process default.
+
+    The driver's `run_claude` calls `get_default_limiter()`; tests that
+    want a controlled tmp-dir-backed limiter use this contextmanager
+    to swap one in without touching env vars or the on-disk default
+    state.
+    """
+    global _default
+    previous = _default
+    _default = limiter
+    try:
+        yield limiter
+    finally:
+        _default = previous

tests/claude_code/run_compat.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Run the Claude Code compat matrix end-to-end against a live LiteLLM
+# proxy, with per-provider rate limits applied via the cross-process
+# token bucket in `tests/claude_code/rate_limiter.py`.
+#
+# Designed for binary-searching the ideal X / Y / Z req/s per provider:
+#   1. Pick an initial rate (e.g. 5/s for everyone).
+#   2. Run this script.
+#   3. Read `compat-rate-limit-summary.json` to see whether any provider
+#      hit a 429-shaped error during the run.
+#   4. If a provider has `rate_limited > 0`, halve its rate; else, double it.
+#   5. Repeat until the highest no-429 rate is found.
+#
+# Required env (proxy connection):
+#   LITELLM_PROXY_BASE_URL   e.g. http://localhost:4000
+#   LITELLM_PROXY_API_KEY    e.g. sk-1234
+#
+# Optional env (rate limits, all default to 5 req/s; 0 disables a column):
+#   LITELLM_COMPAT_RATE_ANTHROPIC
+#   LITELLM_COMPAT_RATE_AZURE
+#   LITELLM_COMPAT_RATE_VERTEX_AI
+#   LITELLM_COMPAT_RATE_BEDROCK_CONVERSE
+#   LITELLM_COMPAT_RATE_BEDROCK_INVOKE
+#   LITELLM_COMPAT_RATE_BURST            override per-bucket burst
+#
+# Optional env (parallelism):
+#   COMPAT_XDIST_WORKERS                  passed to `pytest -n` (default: auto)
+#
+# Optional env (artifacts):
+#   COMPAT_RESULTS_PATH                   default: compat-results.json
+#   COMPAT_RATE_LIMIT_SUMMARY_PATH        default: compat-rate-limit-summary.json
+
+set -euo pipefail
+
+if [[ -z "${LITELLM_PROXY_BASE_URL:-}" || -z "${LITELLM_PROXY_API_KEY:-}" ]]; then
+    echo "error: LITELLM_PROXY_BASE_URL and LITELLM_PROXY_API_KEY must be set" >&2
+    exit 64
+fi
+
+# Reset the cross-process rate-limiter state from any prior run. Stale
+# token-bucket files would let a previous run's accumulated budget bleed
+# into the new one, which subtly biases the binary search.
+state_dir="${LITELLM_COMPAT_RATE_STATE_DIR:-${TMPDIR:-/tmp}/litellm-claude-compat-ratelimit}"
+if [[ -d "$state_dir" ]]; then
+    rm -rf "$state_dir"
+fi
+
+# Worker count. `auto` picks one worker per CPU; the rate limiter
+# enforces aggregate provider rates regardless of worker count, so
+# this is a "go as fast as the limiter allows" knob, not a tuning knob.
+workers="${COMPAT_XDIST_WORKERS:-auto}"
+
+# Where the artifacts land. We resolve them now so the summary file is
+# always at a known path the caller can grep, even if they didn't set
+# the env explicitly.
+results_path="${COMPAT_RESULTS_PATH:-compat-results.json}"
+summary_path="${COMPAT_RATE_LIMIT_SUMMARY_PATH:-compat-rate-limit-summary.json}"
+
+echo "[run_compat] rates:"
+for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE; do
+    var="LITELLM_COMPAT_RATE_${provider}"
+    echo "  ${provider}=${!var:-default(5/s)}"
+done
+echo "  BURST=${LITELLM_COMPAT_RATE_BURST:-default(=rate)}"
+echo "[run_compat] xdist workers: ${workers}"
+echo "[run_compat] results:  ${results_path}"
+echo "[run_compat] summary:  ${summary_path}"
+
+# Run only the per-feature live tests; skip the unit-test directories
+# (they're under directories starting with `_`). The dist=loadfile
+# scheduler keeps each test file pinned to a single worker, which is
+# what we want — every test in a file shares a single ThreadPoolExecutor
+# fanout, and we don't gain anything by splitting it across workers.
+start=$(date +%s)
+COMPAT_RESULTS_PATH="${results_path}" \
+COMPAT_RATE_LIMIT_SUMMARY_PATH="${summary_path}" \
+PATH="$HOME/.local/bin:$PATH" \
+uv run pytest \
+    tests/claude_code/basic_messaging_non_streaming \
+    tests/claude_code/basic_messaging_streaming \
+    tests/claude_code/extended_thinking \
+    tests/claude_code/tool_use \
+    tests/claude_code/vision \
+    tests/claude_code/prompt_caching_5m \
+    -n "${workers}" \
+    --dist=loadfile \
+    -q \
+    "$@"
+
+exit_code=$?
+end=$(date +%s)
+echo "[run_compat] wall time: $((end - start))s"
+
+# Surface the rate-limit summary inline so a human reader doesn't have
+# to `cat` the JSON file. The full file is still on disk for the binary
+# search loop.
+if [[ -f "${summary_path}" ]]; then
+    echo "[run_compat] summary: ${summary_path}"
+    if command -v jq >/dev/null 2>&1; then
+        jq '.totals, .per_provider' "${summary_path}"
+    else
+        cat "${summary_path}"
+    fi
+fi
+
+exit "${exit_code}"

tests/claude_code/test_config.yaml

@@ -62,18 +62,18 @@ model_list:
   - model_name: claude-haiku-4-5-vertex
     litellm_params:
       model: vertex_ai/claude-haiku-4-5
-      vertex_ai_project: pathrise-convert-1606954137718
-      vertex_ai_location: us-east5
+      vertex_ai_project: os.environ/VERTEXAI_PROJECT
+      vertex_ai_location: os.environ/VERTEXAI_LOCATION
   - model_name: claude-sonnet-4-6-vertex
     litellm_params:
       model: vertex_ai/claude-sonnet-4-6
-      vertex_ai_project: pathrise-convert-1606954137718
-      vertex_ai_location: us-east5
+      vertex_ai_project: os.environ/VERTEXAI_PROJECT
+      vertex_ai_location: os.environ/VERTEXAI_LOCATION
   - model_name: claude-opus-4-7-vertex
     litellm_params:
       model: vertex_ai/claude-opus-4-7
-      vertex_ai_project: pathrise-convert-1606954137718
-      vertex_ai_location: us-east5
+      vertex_ai_project: os.environ/VERTEXAI_PROJECT
+      vertex_ai_location: os.environ/VERTEXAI_LOCATION
 
   # ---- Microsoft Foundry (Anthropic deployments on Azure) ----
   - model_name: claude-haiku-4-5-azure

tests/claude_code/thinking_with_tool_use/test_anthropic.py

@@ -0,0 +1,148 @@
+"""thinking_with_tool_use x Anthropic.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+to Anthropic, enable extended thinking via `MAX_THINKING_TOKENS`, allow
+the built-in `Bash` tool, and ask Claude to plan-and-execute a task
+that requires both reasoning and a tool call. Assert the upstream
+returned both a `thinking` content block and a `tool_use` content
+block in the same turn — proving the proxy preserves the wire shape
+where extended thinking and tool use coexist.
+
+This is the cell that historically catches the most provider bugs:
+"thinking blocks cannot be modified" 400s, the recurring Bedrock
+"thinking.type.enabled is not supported" error, and the
+`fine-grained-tool-streaming` + `interleaved-thinking` beta-header
+interactions. A regression in any of those collapses this cell.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/thinking_with_tool_use/test_anthropic.py
+                       ^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^
+                       feature_id                  provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+ANTHROPIC_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+# Extended thinking on, with a small budget — enough to surface a
+# non-empty thinking block on a trivial reasoning prompt without
+# blowing up wall time.
+THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"}
+
+# Prompt designed to force both blocks: the model has to *reason* about
+# what command to run before *invoking* the Bash tool. Using a fixed
+# expected output keeps the assertion focused on the wire shape rather
+# than on answer quality.
+THINKING_TOOL_PROMPT = (
+    "Think step by step about which shell command would print just the word "
+    "'pong'. Then use the Bash tool to run that exact command and report what "
+    "it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+
+def _has_block_type(
+    events: Sequence[Mapping[str, Any]],
+    block_type: str,
+) -> bool:
+    """Walk the stream-json events and return True if any assistant
+    message included a content block of the given type."""
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == block_type:
+                return True
+    return False
+
+
+def test_thinking_with_tool_use_anthropic(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy with thinking
+    enabled and tool use, and assert both `thinking` and `tool_use`
+    content blocks landed in the same turn."""
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt=THINKING_TOOL_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "thinking"):
+            error = (
+                f"[{model}] no `thinking` content block observed; thinking "
+                f"either disabled by the proxy or stripped by the upstream"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "tool_use"):
+            error = (
+                f"[{model}] no `tool_use` content block observed alongside "
+                f"thinking; the proxy may have dropped tools when thinking is on"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/thinking_with_tool_use/test_azure.py

@@ -0,0 +1,123 @@
+"""thinking_with_tool_use x Microsoft Foundry (Azure).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to Microsoft Foundry's Anthropic deployments on Azure,
+enable extended thinking via `MAX_THINKING_TOKENS`, allow the built-in
+`Bash` tool, and ask Claude to plan-and-execute a task that requires
+both reasoning and a tool call. Assert the upstream returned both a
+`thinking` content block and a `tool_use` content block in the same
+turn.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/thinking_with_tool_use/test_azure.py
+                       ^^^^^^^^^^^^^^^^^^^^^^      ^^^^^
+                       feature_id                  provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+AZURE_MODELS = [
+    "claude-haiku-4-5-azure",
+    "claude-sonnet-4-6-azure",
+    "claude-opus-4-7-azure",
+]
+
+THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"}
+THINKING_TOOL_PROMPT = (
+    "Think step by step about which shell command would print just the word "
+    "'pong'. Then use the Bash tool to run that exact command and report what "
+    "it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+
+def _has_block_type(
+    events: Sequence[Mapping[str, Any]],
+    block_type: str,
+) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == block_type:
+                return True
+    return False
+
+
+def test_thinking_with_tool_use_azure(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt=THINKING_TOOL_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "thinking"):
+            error = f"[{model}] no `thinking` content block observed"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "tool_use"):
+            error = f"[{model}] no `tool_use` content block observed alongside thinking"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py

@@ -0,0 +1,128 @@
+"""thinking_with_tool_use x Bedrock (Converse).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the `Converse` API, enable extended
+thinking via `MAX_THINKING_TOKENS`, allow the built-in `Bash` tool, and
+ask Claude to plan-and-execute a task that requires both reasoning and
+a tool call. Assert the upstream returned both a `thinking` content
+block and a `tool_use` content block in the same turn.
+
+The Converse API has its own `additionalModelRequestFields.thinking`
+shape and its own tool-use envelope; this cell catches gateway
+regressions where the proxy fails to translate between Anthropic's
+`thinking` parameter and Converse's reasoning configuration when tools
+are also present.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py
+                       ^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^^^
+                       feature_id                  provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_CONVERSE_MODELS = [
+    "claude-haiku-4-5-bedrock-converse",
+    "claude-sonnet-4-6-bedrock-converse",
+    "claude-opus-4-7-bedrock-converse",
+]
+
+THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"}
+THINKING_TOOL_PROMPT = (
+    "Think step by step about which shell command would print just the word "
+    "'pong'. Then use the Bash tool to run that exact command and report what "
+    "it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+
+def _has_block_type(
+    events: Sequence[Mapping[str, Any]],
+    block_type: str,
+) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == block_type:
+                return True
+    return False
+
+
+def test_thinking_with_tool_use_bedrock_converse(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt=THINKING_TOOL_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "thinking"):
+            error = f"[{model}] no `thinking` content block observed"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "tool_use"):
+            error = f"[{model}] no `tool_use` content block observed alongside thinking"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py

@@ -0,0 +1,130 @@
+"""thinking_with_tool_use x Bedrock (Invoke).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the legacy `InvokeModel` API path,
+enable extended thinking via `MAX_THINKING_TOKENS`, allow the built-in
+`Bash` tool, and ask Claude to plan-and-execute a task that requires
+both reasoning and a tool call. Assert the upstream returned both a
+`thinking` content block and a `tool_use` content block in the same
+turn.
+
+This is the cell most likely to flush out Bedrock-specific bugs in
+LiteLLM's Anthropic <-> Bedrock translation: the recurring
+"thinking.type.enabled is not supported" 400 error has reappeared on
+several Bedrock model routes (notably application inference profile
+ARNs), and the only reliable signal that the fix is wired through the
+proxy is a successful round-trip on this cell.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py
+                       ^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id                  provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_INVOKE_MODELS = [
+    "claude-haiku-4-5-bedrock-invoke",
+    "claude-sonnet-4-6-bedrock-invoke",
+    "claude-opus-4-7-bedrock-invoke",
+]
+
+THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"}
+THINKING_TOOL_PROMPT = (
+    "Think step by step about which shell command would print just the word "
+    "'pong'. Then use the Bash tool to run that exact command and report what "
+    "it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+
+def _has_block_type(
+    events: Sequence[Mapping[str, Any]],
+    block_type: str,
+) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == block_type:
+                return True
+    return False
+
+
+def test_thinking_with_tool_use_bedrock_invoke(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt=THINKING_TOOL_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "thinking"):
+            error = f"[{model}] no `thinking` content block observed"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "tool_use"):
+            error = f"[{model}] no `tool_use` content block observed alongside thinking"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/thinking_with_tool_use/test_vertex_ai.py

@@ -0,0 +1,128 @@
+"""thinking_with_tool_use x Vertex AI.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to GCP Vertex AI, enable extended thinking via
+`MAX_THINKING_TOKENS`, allow the built-in `Bash` tool, and ask Claude
+to plan-and-execute a task that requires both reasoning and a tool
+call. Assert the upstream returned both a `thinking` content block and
+a `tool_use` content block in the same turn.
+
+Vertex AI exposes Anthropic models via `:rawPredict` /
+`:streamRawPredict` and has its own beta-header allowlist. This cell
+catches gateway regressions where the proxy strips
+`anthropic-beta: interleaved-thinking-2025-05-14` (or the equivalent
+header set the upstream needs) on the way to Vertex.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/thinking_with_tool_use/test_vertex_ai.py
+                       ^^^^^^^^^^^^^^^^^^^^^^      ^^^^^^^^^
+                       feature_id                  provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+VERTEX_AI_MODELS = [
+    "claude-haiku-4-5-vertex",
+    "claude-sonnet-4-6-vertex",
+    "claude-opus-4-7-vertex",
+]
+
+THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"}
+THINKING_TOOL_PROMPT = (
+    "Think step by step about which shell command would print just the word "
+    "'pong'. Then use the Bash tool to run that exact command and report what "
+    "it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+
+def _has_block_type(
+    events: Sequence[Mapping[str, Any]],
+    block_type: str,
+) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == block_type:
+                return True
+    return False
+
+
+def test_thinking_with_tool_use_vertex_ai(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt=THINKING_TOOL_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_env=THINKING_ENV,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "thinking"):
+            error = f"[{model}] no `thinking` content block observed"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_block_type(outcome.events, "tool_use"):
+            error = f"[{model}] no `tool_use` content block observed alongside thinking"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use/test_anthropic.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -56,8 +60,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", ANTHROPIC_MODELS)
-def test_tool_use_anthropic(compat_result, model):
+def test_tool_use_anthropic(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     tool call was emitted on the wire."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -76,39 +79,38 @@ def test_tool_use_anthropic(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=TOOL_USE_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=TOOL_USE_ARGS,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_tool_use_event(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no tool_use content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no tool_use for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use/test_azure.py

@@ -24,7 +24,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -55,8 +59,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", AZURE_MODELS)
-def test_tool_use_azure(compat_result, model):
+def test_tool_use_azure(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     tool call was emitted on the wire."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -75,39 +78,38 @@ def test_tool_use_azure(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=TOOL_USE_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=TOOL_USE_ARGS,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_tool_use_event(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no tool_use content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no tool_use for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use/test_bedrock_converse.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -51,8 +55,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS)
-def test_tool_use_bedrock_converse(compat_result, model):
+def test_tool_use_bedrock_converse(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     tool call was emitted on the wire."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -71,39 +74,38 @@ def test_tool_use_bedrock_converse(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=TOOL_USE_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=TOOL_USE_ARGS,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_tool_use_event(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no tool_use content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no tool_use for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use/test_bedrock_invoke.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -51,8 +55,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS)
-def test_tool_use_bedrock_invoke(compat_result, model):
+def test_tool_use_bedrock_invoke(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     tool call was emitted on the wire."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -71,39 +74,38 @@ def test_tool_use_bedrock_invoke(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=TOOL_USE_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=TOOL_USE_ARGS,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_tool_use_event(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no tool_use content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no tool_use for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use/test_vertex_ai.py

@@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -51,8 +55,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
     return False
 
 
-@pytest.mark.parametrize("model", VERTEX_AI_MODELS)
-def test_tool_use_vertex_ai(compat_result, model):
+def test_tool_use_vertex_ai(compat_result):
     """Drive the `claude` CLI against the LiteLLM proxy and assert a
     tool call was emitted on the wire."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -71,39 +74,38 @@ def test_tool_use_vertex_ai(compat_result, model):
             f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
         )
 
-    try:
-        result = run_claude(
-            prompt=TOOL_USE_PROMPT,
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=TOOL_USE_ARGS,
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not _has_tool_use_event(result.events):
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] no tool_use content block observed in stream-json events",
-            }
-        )
-        pytest.fail(f"no tool_use for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use_streaming/test_anthropic.py

@@ -0,0 +1,147 @@
+"""tool_use_streaming x Anthropic.
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes to Anthropic, ask
+Claude to invoke a built-in tool (`Bash`), and assert that the upstream
+(a) emitted a `tool_use` content block and (b) actually streamed the
+events incrementally — i.e. more than one stream-json record was
+observed before the final `result`.
+
+This is the "fine-grained tool streaming" path. Historically gateways
+break it in two ways: they either buffer the entire response before
+flushing (in which case `len(events)` collapses to ~1 final record) or
+they strip the `fine-grained-tool-streaming-2025-05-14` beta header and
+the upstream falls back to non-streaming tool_use. Both regressions are
+caught by the assertions below.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/tool_use_streaming/test_anthropic.py
+                       ^^^^^^^^^^^^^^^^^^      ^^^^^^^^^
+                       feature_id              provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+ANTHROPIC_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+# Same shape as the non-streaming `tool_use` cell: ask Claude to call
+# the built-in `Bash` tool. The CLI is already in stream-json mode by
+# default in `run_claude`, so we don't need to toggle anything to
+# exercise the streaming wire — what we want to assert is that the
+# stream-json transport actually carried more than one record, which
+# is the wire-level signal that the proxy didn't buffer the upstream.
+TOOL_USE_PROMPT = (
+    "Use the Bash tool to run the command `echo pong` and report what it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+# Floor on the number of stream-json records we expect to see for a
+# tool-use turn. A buffered (non-streamed) wire collapses to one
+# `system` init record + one `assistant` final + one `result`, total 3.
+# Real fine-grained streaming produces many more (incremental
+# input_json_delta events, intermediate assistant deltas, etc.). We
+# pick a floor above the buffered case so the assertion catches the
+# regression without being flaky on short responses.
+MIN_STREAM_EVENTS = 4
+
+
+def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
+    """Walk the stream-json events and return True if any assistant
+    message included a `tool_use` content block."""
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                return True
+    return False
+
+
+def test_tool_use_streaming_anthropic(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert the
+    proxy preserves fine-grained tool streaming end-to-end."""
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if len(outcome.events) < MIN_STREAM_EVENTS:
+            error = (
+                f"[{model}] only {len(outcome.events)} stream-json events observed "
+                f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response or "
+                f"stripped fine-grained tool streaming"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use_streaming/test_azure.py

@@ -0,0 +1,122 @@
+"""tool_use_streaming x Microsoft Foundry (Azure).
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Claude requests to
+Microsoft Foundry's Anthropic deployments on Azure, ask Claude to
+invoke a built-in tool (`Bash`), and assert that the upstream (a)
+emitted a `tool_use` content block and (b) actually streamed events
+incrementally.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/tool_use_streaming/test_azure.py
+                       ^^^^^^^^^^^^^^^^^^      ^^^^^
+                       feature_id              provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+AZURE_MODELS = [
+    "claude-haiku-4-5-azure",
+    "claude-sonnet-4-6-azure",
+    "claude-opus-4-7-azure",
+]
+
+TOOL_USE_PROMPT = (
+    "Use the Bash tool to run the command `echo pong` and report what it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+MIN_STREAM_EVENTS = 4
+
+
+def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                return True
+    return False
+
+
+def test_tool_use_streaming_azure(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if len(outcome.events) < MIN_STREAM_EVENTS:
+            error = (
+                f"[{model}] only {len(outcome.events)} stream-json events observed "
+                f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use_streaming/test_bedrock_converse.py

@@ -0,0 +1,128 @@
+"""tool_use_streaming x Bedrock (Converse).
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Claude requests to
+AWS Bedrock via the `Converse` API (`ConverseStream`), ask Claude to
+invoke a built-in tool (`Bash`), and assert that the upstream (a)
+emitted a `tool_use` content block and (b) actually streamed events
+incrementally.
+
+Bedrock Converse has its own tool-streaming envelope (`toolUse` blocks
+with `delta` chunks); this cell catches gateway regressions where the
+proxy buffers the response or fails to translate Converse's streaming
+envelope back to the Anthropic `message_*` event shape Claude Code
+expects.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/tool_use_streaming/test_bedrock_converse.py
+                       ^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^^^
+                       feature_id              provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_CONVERSE_MODELS = [
+    "claude-haiku-4-5-bedrock-converse",
+    "claude-sonnet-4-6-bedrock-converse",
+    "claude-opus-4-7-bedrock-converse",
+]
+
+TOOL_USE_PROMPT = (
+    "Use the Bash tool to run the command `echo pong` and report what it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+MIN_STREAM_EVENTS = 4
+
+
+def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                return True
+    return False
+
+
+def test_tool_use_streaming_bedrock_converse(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if len(outcome.events) < MIN_STREAM_EVENTS:
+            error = (
+                f"[{model}] only {len(outcome.events)} stream-json events observed "
+                f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use_streaming/test_bedrock_invoke.py

@@ -0,0 +1,126 @@
+"""tool_use_streaming x Bedrock (Invoke).
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Claude requests to
+AWS Bedrock via the legacy `InvokeModel` API path, ask Claude to invoke
+a built-in tool (`Bash`), and assert that the upstream (a) emitted a
+`tool_use` content block and (b) actually streamed events incrementally.
+
+Bedrock InvokeModel surfaces tool-streaming via `InvokeModelWithResponseStream`;
+this cell catches gateway regressions where the proxy buffers the
+response or fails to translate the streaming envelope to Anthropic
+`message_*` event shape.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/tool_use_streaming/test_bedrock_invoke.py
+                       ^^^^^^^^^^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id              provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_INVOKE_MODELS = [
+    "claude-haiku-4-5-bedrock-invoke",
+    "claude-sonnet-4-6-bedrock-invoke",
+    "claude-opus-4-7-bedrock-invoke",
+]
+
+TOOL_USE_PROMPT = (
+    "Use the Bash tool to run the command `echo pong` and report what it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+MIN_STREAM_EVENTS = 4
+
+
+def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                return True
+    return False
+
+
+def test_tool_use_streaming_bedrock_invoke(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if len(outcome.events) < MIN_STREAM_EVENTS:
+            error = (
+                f"[{model}] only {len(outcome.events)} stream-json events observed "
+                f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/tool_use_streaming/test_vertex_ai.py

@@ -0,0 +1,125 @@
+"""tool_use_streaming x Vertex AI.
+
+Drive the real `claude` CLI in headless `--output-format stream-json`
+mode against a running LiteLLM proxy that routes Claude requests to
+GCP Vertex AI, ask Claude to invoke a built-in tool (`Bash`), and
+assert that the upstream (a) emitted a `tool_use` content block and
+(b) actually streamed events incrementally.
+
+Vertex AI exposes Anthropic models via `:streamRawPredict`; this cell
+catches gateway regressions where the proxy buffers the response or
+strips the streaming beta header on the way to Vertex.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/tool_use_streaming/test_vertex_ai.py
+                       ^^^^^^^^^^^^^^^^^^      ^^^^^^^^^
+                       feature_id              provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+VERTEX_AI_MODELS = [
+    "claude-haiku-4-5-vertex",
+    "claude-sonnet-4-6-vertex",
+    "claude-opus-4-7-vertex",
+]
+
+TOOL_USE_PROMPT = (
+    "Use the Bash tool to run the command `echo pong` and report what it printed."
+)
+TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
+
+MIN_STREAM_EVENTS = 4
+
+
+def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "tool_use":
+                return True
+    return False
+
+
+def test_tool_use_streaming_vertex_ai(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt=TOOL_USE_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=TOOL_USE_ARGS,
+    )
+
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_tool_use_event(outcome.events):
+            error = (
+                f"[{model}] no tool_use content block observed in stream-json events"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if len(outcome.events) < MIN_STREAM_EVENTS:
+            error = (
+                f"[{model}] only {len(outcome.events)} stream-json events observed "
+                f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/vision/test_anthropic.py

@@ -21,7 +21,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -38,8 +42,7 @@ ANTHROPIC_MODELS = [
 RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
 
 
-@pytest.mark.parametrize("model", ANTHROPIC_MODELS)
-def test_vision_anthropic(compat_result, model, tmp_path):
+def test_vision_anthropic(compat_result, tmp_path):
     """Drive the `claude` CLI against the LiteLLM proxy with an image
     attached and assert a non-empty reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -61,39 +64,36 @@ def test_vision_anthropic(compat_result, model, tmp_path):
     image_path = tmp_path / "red_pixel.png"
     image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64))
 
-    try:
-        result = run_claude(
-            prompt="What single color do you see in the attached image? Answer in one word.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=["--image", str(image_path)],
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt="What single color do you see in the attached image? Answer in one word.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--image", str(image_path)],
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text on a vision prompt",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text on a vision prompt"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/vision/test_azure.py

@@ -25,7 +25,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -39,8 +43,7 @@ AZURE_MODELS = [
 RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
 
 
-@pytest.mark.parametrize("model", AZURE_MODELS)
-def test_vision_azure(compat_result, model, tmp_path):
+def test_vision_azure(compat_result, tmp_path):
     """Drive the `claude` CLI against the LiteLLM proxy with an image
     attached and assert a non-empty reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -62,39 +65,36 @@ def test_vision_azure(compat_result, model, tmp_path):
     image_path = tmp_path / "red_pixel.png"
     image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64))
 
-    try:
-        result = run_claude(
-            prompt="What single color do you see in the attached image? Answer in one word.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=["--image", str(image_path)],
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt="What single color do you see in the attached image? Answer in one word.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--image", str(image_path)],
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text on a vision prompt",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text on a vision prompt"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/vision/test_bedrock_converse.py

@@ -20,7 +20,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -34,8 +38,7 @@ BEDROCK_CONVERSE_MODELS = [
 RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
 
 
-@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS)
-def test_vision_bedrock_converse(compat_result, model, tmp_path):
+def test_vision_bedrock_converse(compat_result, tmp_path):
     """Drive the `claude` CLI against the LiteLLM proxy with an image
     attached and assert a non-empty reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -57,39 +60,36 @@ def test_vision_bedrock_converse(compat_result, model, tmp_path):
     image_path = tmp_path / "red_pixel.png"
     image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64))
 
-    try:
-        result = run_claude(
-            prompt="What single color do you see in the attached image? Answer in one word.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=["--image", str(image_path)],
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt="What single color do you see in the attached image? Answer in one word.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--image", str(image_path)],
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text on a vision prompt",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text on a vision prompt"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/vision/test_bedrock_invoke.py

@@ -20,7 +20,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -34,8 +38,7 @@ BEDROCK_INVOKE_MODELS = [
 RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
 
 
-@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS)
-def test_vision_bedrock_invoke(compat_result, model, tmp_path):
+def test_vision_bedrock_invoke(compat_result, tmp_path):
     """Drive the `claude` CLI against the LiteLLM proxy with an image
     attached and assert a non-empty reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -57,39 +60,36 @@ def test_vision_bedrock_invoke(compat_result, model, tmp_path):
     image_path = tmp_path / "red_pixel.png"
     image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64))
 
-    try:
-        result = run_claude(
-            prompt="What single color do you see in the attached image? Answer in one word.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=["--image", str(image_path)],
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt="What single color do you see in the attached image? Answer in one word.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--image", str(image_path)],
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text on a vision prompt",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text on a vision prompt"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/vision/test_vertex_ai.py

@@ -20,7 +20,11 @@ import os
 
 import pytest
 
-from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
 
 PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
 PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@@ -34,8 +38,7 @@ VERTEX_AI_MODELS = [
 RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
 
 
-@pytest.mark.parametrize("model", VERTEX_AI_MODELS)
-def test_vision_vertex_ai(compat_result, model, tmp_path):
+def test_vision_vertex_ai(compat_result, tmp_path):
     """Drive the `claude` CLI against the LiteLLM proxy with an image
     attached and assert a non-empty reply."""
     base_url = os.environ.get(PROXY_BASE_URL_ENV)
@@ -57,39 +60,36 @@ def test_vision_vertex_ai(compat_result, model, tmp_path):
     image_path = tmp_path / "red_pixel.png"
     image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64))
 
-    try:
-        result = run_claude(
-            prompt="What single color do you see in the attached image? Answer in one word.",
-            model=model,
-            base_url=base_url,
-            api_key=api_key,
-            extra_args=["--image", str(image_path)],
-        )
-    except ClaudeCLIError as exc:
-        compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
-        pytest.fail(str(exc), pytrace=False)
-        return
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt="What single color do you see in the attached image? Answer in one word.",
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=["--image", str(image_path)],
+    )
 
-    if result.exit_code != 0:
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}",
-            }
-        )
-        pytest.fail(
-            f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False
-        )
-        return
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    if not result.text.strip():
-        compat_result.set(
-            {
-                "status": "fail",
-                "error": f"[{model}] claude returned empty assistant text on a vision prompt",
-            }
-        )
-        pytest.fail(f"empty reply for {model}", pytrace=False)
-        return
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
 
-    compat_result.set({"status": "pass"})
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text on a vision prompt"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/web_search/test_anthropic.py

@@ -0,0 +1,142 @@
+"""web_search x Anthropic.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+to Anthropic, allow the built-in `WebSearch` tool, ask a question that
+requires fresh web data, and assert that the upstream emitted a
+`server_tool_use` or `web_search_tool_result` content block — proving
+the proxy preserves Anthropic's server-side web search end-to-end.
+
+Web search is a *server tool*: unlike `Bash`/`Read`/etc., the upstream
+executes the search itself and embeds the results inline in the
+response. The wire shape is distinctive:
+
+  - `server_tool_use` block with `name: "web_search"`
+  - `web_search_tool_result` block carrying the encrypted result content
+
+A regression where the proxy strips the `web_search_20250305` tool
+from the request, drops the result block from the response, or fails
+to forward the required beta header collapses both signals.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/web_search/test_anthropic.py
+                       ^^^^^^^^^^      ^^^^^^^^^
+                       feature_id      provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+ANTHROPIC_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+# A prompt the model cannot answer from training data alone — it forces
+# the model to actually hit the web_search server tool rather than
+# replying from memory. We pick "this week" as the freshness anchor
+# because it's stable across long-running test schedules without
+# pinning to a specific date that would go stale.
+WEB_SEARCH_PROMPT = (
+    "Use web search to find a news headline published this week about "
+    "Anthropic. Reply with one sentence summarizing what you found."
+)
+# Allow only WebSearch so the model has no fallback path: if the proxy
+# strips the server tool, the run will fail loudly rather than silently
+# answering from training data via a different tool.
+WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
+
+# Block types that prove the server tool actually executed end-to-end.
+SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"}
+
+
+def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool:
+    """Walk the stream-json events and return True if any assistant
+    message included a server-tool block (server_tool_use or
+    web_search_tool_result)."""
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") in SERVER_TOOL_BLOCK_TYPES:
+                return True
+    return False
+
+
+def test_web_search_anthropic(compat_result):
+    """Drive the `claude` CLI against the LiteLLM proxy and assert the
+    upstream emitted a server-tool block proving web search ran."""
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=ANTHROPIC_MODELS,
+        prompt=WEB_SEARCH_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=WEB_SEARCH_ARGS,
+    )
+
+    failures = []
+    for model in ANTHROPIC_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_server_tool_block(outcome.events):
+            error = (
+                f"[{model}] no server_tool_use / web_search_tool_result block "
+                "observed; the proxy may have stripped the WebSearch server tool "
+                "or its result block"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/web_search/test_azure.py

@@ -0,0 +1,115 @@
+"""web_search x Microsoft Foundry (Azure).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to Microsoft Foundry's Anthropic deployments on Azure,
+allow the built-in `WebSearch` tool, ask a question that requires
+fresh web data, and assert that the upstream emitted a
+`server_tool_use` or `web_search_tool_result` content block.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/web_search/test_azure.py
+                       ^^^^^^^^^^      ^^^^^
+                       feature_id      provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+AZURE_MODELS = [
+    "claude-haiku-4-5-azure",
+    "claude-sonnet-4-6-azure",
+    "claude-opus-4-7-azure",
+]
+
+WEB_SEARCH_PROMPT = (
+    "Use web search to find a news headline published this week about "
+    "Anthropic. Reply with one sentence summarizing what you found."
+)
+WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
+SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"}
+
+
+def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") in SERVER_TOOL_BLOCK_TYPES:
+                return True
+    return False
+
+
+def test_web_search_azure(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=AZURE_MODELS,
+        prompt=WEB_SEARCH_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=WEB_SEARCH_ARGS,
+    )
+
+    failures = []
+    for model in AZURE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_server_tool_block(outcome.events):
+            error = (
+                f"[{model}] no server_tool_use / web_search_tool_result block "
+                "observed"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/web_search/test_bedrock_converse.py

@@ -0,0 +1,115 @@
+"""web_search x Bedrock (Converse).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the `Converse` API, allow the
+built-in `WebSearch` tool, ask a question that requires fresh web
+data, and assert that the upstream emitted a `server_tool_use` or
+`web_search_tool_result` content block.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/web_search/test_bedrock_converse.py
+                       ^^^^^^^^^^      ^^^^^^^^^^^^^^^^
+                       feature_id      provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_CONVERSE_MODELS = [
+    "claude-haiku-4-5-bedrock-converse",
+    "claude-sonnet-4-6-bedrock-converse",
+    "claude-opus-4-7-bedrock-converse",
+]
+
+WEB_SEARCH_PROMPT = (
+    "Use web search to find a news headline published this week about "
+    "Anthropic. Reply with one sentence summarizing what you found."
+)
+WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
+SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"}
+
+
+def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") in SERVER_TOOL_BLOCK_TYPES:
+                return True
+    return False
+
+
+def test_web_search_bedrock_converse(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_CONVERSE_MODELS,
+        prompt=WEB_SEARCH_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=WEB_SEARCH_ARGS,
+    )
+
+    failures = []
+    for model in BEDROCK_CONVERSE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_server_tool_block(outcome.events):
+            error = (
+                f"[{model}] no server_tool_use / web_search_tool_result block "
+                "observed"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/web_search/test_bedrock_invoke.py

@@ -0,0 +1,122 @@
+"""web_search x Bedrock (Invoke).
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to AWS Bedrock via the legacy `InvokeModel` API path,
+allow the built-in `WebSearch` tool, ask a question that requires
+fresh web data, and assert that the upstream emitted a
+`server_tool_use` or `web_search_tool_result` content block.
+
+Bedrock support for the Anthropic-hosted web_search server tool has
+been historically uneven — when it works, the wire shape is identical
+to Anthropic's native API; when it doesn't, the upstream returns a
+400 ("server tools not supported") that the proxy must surface
+faithfully rather than silently dropping the tool from the request.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/web_search/test_bedrock_invoke.py
+                       ^^^^^^^^^^      ^^^^^^^^^^^^^^
+                       feature_id      provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+BEDROCK_INVOKE_MODELS = [
+    "claude-haiku-4-5-bedrock-invoke",
+    "claude-sonnet-4-6-bedrock-invoke",
+    "claude-opus-4-7-bedrock-invoke",
+]
+
+WEB_SEARCH_PROMPT = (
+    "Use web search to find a news headline published this week about "
+    "Anthropic. Reply with one sentence summarizing what you found."
+)
+WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
+SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"}
+
+
+def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") in SERVER_TOOL_BLOCK_TYPES:
+                return True
+    return False
+
+
+def test_web_search_bedrock_invoke(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=BEDROCK_INVOKE_MODELS,
+        prompt=WEB_SEARCH_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=WEB_SEARCH_ARGS,
+    )
+
+    failures = []
+    for model in BEDROCK_INVOKE_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_server_tool_block(outcome.events):
+            error = (
+                f"[{model}] no server_tool_use / web_search_tool_result block "
+                "observed; Bedrock may not support the web_search server tool "
+                "for this model, or the proxy stripped it"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/claude_code/web_search/test_vertex_ai.py

@@ -0,0 +1,120 @@
+"""web_search x Vertex AI.
+
+Drive the real `claude` CLI against a running LiteLLM proxy that routes
+Claude requests to GCP Vertex AI, allow the built-in `WebSearch` tool,
+ask a question that requires fresh web data, and assert that the
+upstream emitted a `server_tool_use` or `web_search_tool_result`
+content block.
+
+Per the changelog (1.0.110, 2.1.79+), Vertex has had inconsistent
+support for the Anthropic web_search server tool — the cell will fail
+loudly when the upstream rejects the tool, which is the diagnostic we
+want for a compat matrix.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/claude_code/conftest.py`:
+
+    tests/claude_code/web_search/test_vertex_ai.py
+                       ^^^^^^^^^^      ^^^^^^^^^
+                       feature_id      provider
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Mapping, Sequence
+
+import pytest
+
+from tests.claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+VERTEX_AI_MODELS = [
+    "claude-haiku-4-5-vertex",
+    "claude-sonnet-4-6-vertex",
+    "claude-opus-4-7-vertex",
+]
+
+WEB_SEARCH_PROMPT = (
+    "Use web search to find a news headline published this week about "
+    "Anthropic. Reply with one sentence summarizing what you found."
+)
+WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
+SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"}
+
+
+def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool:
+    for event in events:
+        if event.get("type") != "assistant":
+            continue
+        message = event.get("message") or {}
+        content = message.get("content")
+        if not isinstance(content, list):
+            continue
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") in SERVER_TOOL_BLOCK_TYPES:
+                return True
+    return False
+
+
+def test_web_search_vertex_ai(compat_result):
+    base_url = os.environ.get(PROXY_BASE_URL_ENV)
+    api_key = os.environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
+        )
+
+    outcomes = run_claude_models_parallel(
+        models=VERTEX_AI_MODELS,
+        prompt=WEB_SEARCH_PROMPT,
+        base_url=base_url,
+        api_key=api_key,
+        extra_args=WEB_SEARCH_ARGS,
+    )
+
+    failures = []
+    for model in VERTEX_AI_MODELS:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not _has_server_tool_block(outcome.events):
+            error = (
+                f"[{model}] no server_tool_use / web_search_tool_result block "
+                "observed"
+            )
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)