Merge pull request #33473 from BerriAI/litellm_claude_code_passthrough

tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py

@@ -19,6 +19,7 @@ from claude_code.cli_driver import (
     ClaudeCLIError,
     DriverResult,
     failure_diagnostic,
+    is_rate_limit_shaped,
     run_claude,
     run_claude_models_parallel,
 )
@@ -793,3 +794,199 @@ def test_failure_diagnostic_uses_last_result_event_status():
     diag = failure_diagnostic(result)
     assert "api_status=429" in diag
     assert "500" not in diag
+
+
+_RATE_LIMITED_STDOUT = (
+    json.dumps(
+        {
+            "type": "assistant",
+            "message": {
+                "content": [
+                    {"type": "text", "text": "API Error: 429 Too Many Requests"}
+                ]
+            },
+        }
+    )
+    + "\n"
+    + json.dumps({"type": "result", "api_error_status": 429})
+    + "\n"
+)
+
+_OK_STDOUT = (
+    json.dumps(
+        {
+            "type": "assistant",
+            "message": {"content": [{"type": "text", "text": "pong"}]},
+        }
+    )
+    + "\n"
+)
+
+
+class _FlakyRunner:
+    """Fake runner that rate-limits each model N times before succeeding.
+
+    Keeps a per-model call count so tests can assert exactly how many
+    attempts the retry loop made — the load-bearing detail a canned
+    single-response runner can't express.
+    """
+
+    def __init__(self, failures_before_success: dict):
+        self.failures_before_success = dict(failures_before_success)
+        self.calls: dict = {}
+
+    def __call__(self, cmd, env, capture_output, text, timeout, check, input=None):
+        model = cmd[cmd.index("--model") + 1]
+        self.calls[model] = self.calls.get(model, 0) + 1
+        if self.calls[model] <= self.failures_before_success.get(model, 0):
+            return _Completed(returncode=1, stdout=_RATE_LIMITED_STDOUT)
+        return _Completed(returncode=0, stdout=_OK_STDOUT)
+
+
+@pytest.mark.parametrize(
+    "outcome,expected",
+    [
+        (ClaudeCLIError("claude CLI timed out after 120.0s"), True),
+        (ClaudeCLIError("claude CLI not found at 'claude'"), False),
+        (
+            DriverResult(
+                text="",
+                events=[{"type": "result", "api_error_status": 429}],
+                exit_code=1,
+            ),
+            True,
+        ),
+        (DriverResult(text="Too Many Requests", exit_code=1), True),
+        (DriverResult(text="", stderr="throttled by upstream", exit_code=1), True),
+        (DriverResult(text="rate limit exceeded", exit_code=0), False),
+        (DriverResult(text="", stderr="auth failed", exit_code=2), False),
+    ],
+)
+def test_is_rate_limit_shaped_classification(outcome, expected):
+    """The retry trigger must match 429/throttle/timeout markers on
+    failures only — a passing result mentioning '429' in its reply text
+    must never be classified as retryable."""
+    assert is_rate_limit_shaped(outcome) is expected
+
+
+def test_run_claude_models_parallel_retries_rate_limited_model_until_success():
+    """A model that 429s once must be retried after the backoff sleep and
+    end up green, while an untroubled sibling model runs exactly once."""
+    runner = _FlakyRunner({"flaky": 1})
+    sleeps: List[float] = []
+
+    outcomes = run_claude_models_parallel(
+        models=["flaky", "steady"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+        rate_limit_retries=2,
+        rate_limit_backoff_seconds=0.5,
+        sleep=sleeps.append,
+    )
+
+    assert isinstance(outcomes["flaky"], DriverResult)
+    assert outcomes["flaky"].exit_code == 0
+    assert outcomes["flaky"].text == "pong"
+    assert runner.calls == {"flaky": 2, "steady": 1}
+    assert sleeps == [0.5]
+
+
+def test_run_claude_models_parallel_does_not_retry_non_rate_limit_failures():
+    """A deterministic failure (bad auth) must fail fast: no sleeps, one
+    attempt — retrying it would just triple the matrix wall time."""
+
+    def runner(cmd, env, capture_output, text, timeout, check, input=None):
+        return _Completed(returncode=2, stdout="", stderr="auth failed")
+
+    sleeps: List[float] = []
+    outcomes = run_claude_models_parallel(
+        models=["a"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+        rate_limit_retries=2,
+        rate_limit_backoff_seconds=0.5,
+        sleep=sleeps.append,
+    )
+
+    assert outcomes["a"].exit_code == 2
+    assert sleeps == []
+
+
+def test_run_claude_models_parallel_returns_last_failure_when_retries_exhausted():
+    """A persistently rate-limited model exhausts its budget (initial
+    attempt + N retries, each preceded by one backoff sleep) and still
+    surfaces the 429 diagnostic instead of masking it."""
+    runner = _FlakyRunner({"stuck": 99})
+    sleeps: List[float] = []
+
+    outcomes = run_claude_models_parallel(
+        models=["stuck"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+        rate_limit_retries=2,
+        rate_limit_backoff_seconds=0.25,
+        sleep=sleeps.append,
+    )
+
+    assert runner.calls == {"stuck": 3}
+    assert sleeps == [0.25, 0.25]
+    assert outcomes["stuck"].exit_code == 1
+    assert "429" in failure_diagnostic(outcomes["stuck"])
+
+
+def test_run_claude_models_parallel_retries_timeout_shaped_cli_errors():
+    """CLI timeouts are how saturated upstreams usually present (the CLI
+    retries 429s internally until the harness kills it), so a timeout
+    must be retried like an explicit 429."""
+    calls: List[int] = []
+
+    def runner(cmd, env, capture_output, text, timeout, check, input=None):
+        calls.append(1)
+        if len(calls) == 1:
+            raise subprocess.TimeoutExpired(cmd="claude", timeout=1)
+        return _Completed(returncode=0, stdout=_OK_STDOUT)
+
+    sleeps: List[float] = []
+    outcomes = run_claude_models_parallel(
+        models=["a"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+        rate_limit_retries=1,
+        rate_limit_backoff_seconds=0.5,
+        sleep=sleeps.append,
+    )
+
+    assert isinstance(outcomes["a"], DriverResult)
+    assert outcomes["a"].text == "pong"
+    assert len(calls) == 2
+    assert sleeps == [0.5]
+
+
+def test_run_claude_models_parallel_zero_retries_disables_backoff():
+    """`rate_limit_retries=0` must restore the old single-attempt
+    behavior exactly: one call, no sleeps, failure returned as-is."""
+    runner = _FlakyRunner({"stuck": 99})
+    sleeps: List[float] = []
+
+    outcomes = run_claude_models_parallel(
+        models=["stuck"],
+        prompt="hi",
+        base_url="http://x",
+        api_key="k",
+        runner=runner,
+        rate_limit_retries=0,
+        rate_limit_backoff_seconds=0.5,
+        sleep=sleeps.append,
+    )
+
+    assert runner.calls == {"stuck": 1}
+    assert sleeps == []
+    assert outcomes["stuck"].exit_code == 1

tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py

@@ -0,0 +1,195 @@
+"""Unit tests for the shared `run_passthrough_cell` helper.
+
+These tests inject a fake `run_models` callable and an explicit `env`
+mapping (both are first-class parameters, no monkeypatching), so they
+exercise the helper's branching -- env-missing guard, base-URL
+assembly, extra-env forwarding, per-model pass/fail -- without
+spawning the real CLI.
+
+The env-builder tests pin the provider-mode contract itself: the
+CLAUDE_CODE_USE_* / CLAUDE_CODE_SKIP_*_AUTH flags and the passthrough
+route each mode must target. Those values are the feature -- e.g.
+dropping the `/v1` from the vertex base URL produces a request Google
+404s on -- so a mutation to any of them must fail here before it burns
+a live matrix run.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Mapping, Optional
+
+import pytest
+
+from claude_code._passthrough import (
+    ANTHROPIC_PASSTHROUGH_BASE_PATH,
+    CLIENT_SIDE_AWS_REGION,
+    VERTEX_PLACEHOLDER_PROJECT,
+    VERTEX_PLACEHOLDER_REGION,
+    bedrock_extra_env,
+    foundry_extra_env,
+    run_passthrough_cell,
+    vertex_extra_env,
+)
+from claude_code.cli_driver import ClaudeCLIError, DriverResult
+
+PROXY_ENV = {
+    "LITELLM_PROXY_BASE_URL": "http://localhost:4000",
+    "LITELLM_PROXY_API_KEY": "sk-test",
+}
+
+
+class _FakeResult:
+    def __init__(self) -> None:
+        self.rows: List[Dict[str, Any]] = []
+        self.single: Optional[Dict[str, Any]] = None
+
+    def set(self, payload: Mapping[str, Any]) -> None:
+        self.single = dict(payload)
+
+    def add(self, payload: Mapping[str, Any]) -> None:
+        self.rows.append(dict(payload))
+
+
+def _fake_run_models(outcomes_by_model, captured: Dict[str, Any]):
+    def fake(*, models, prompt, base_url, api_key, extra_env=None, **_kwargs):
+        captured["models"] = list(models)
+        captured["prompt"] = prompt
+        captured["base_url"] = base_url
+        captured["api_key"] = api_key
+        captured["extra_env"] = dict(extra_env) if extra_env is not None else None
+        return {model: outcomes_by_model[model] for model in models}
+
+    return fake
+
+
+def test_env_missing_guard_reports_fail_and_aborts():
+    fake_result = _FakeResult()
+    with pytest.raises(pytest.fail.Exception):
+        run_passthrough_cell(
+            compat_result=fake_result,
+            models=["claude-haiku-4-5"],
+            prompt="ping",
+            env={},
+        )
+    assert fake_result.single is not None
+    assert fake_result.single["status"] == "fail"
+    assert "LITELLM_PROXY_BASE_URL" in fake_result.single["error"]
+
+
+def test_anthropic_base_path_appended_to_normalized_proxy_url():
+    fake_result = _FakeResult()
+    captured: Dict[str, Any] = {}
+    outcome = DriverResult(text="pong")
+
+    run_passthrough_cell(
+        compat_result=fake_result,
+        models=["claude-haiku-4-5"],
+        prompt="ping",
+        passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH,
+        run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
+        env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"},
+    )
+
+    assert captured["base_url"] == "http://localhost:4000/anthropic"
+    assert captured["extra_env"] is None
+    assert fake_result.rows == [{"status": "pass"}]
+
+
+def test_extra_env_builder_receives_normalized_base_and_is_forwarded():
+    fake_result = _FakeResult()
+    captured: Dict[str, Any] = {}
+    outcome = DriverResult(text="pong")
+    seen_bases: List[str] = []
+
+    def build(proxy_base: str) -> Dict[str, str]:
+        seen_bases.append(proxy_base)
+        return {"SOME_FLAG": "1"}
+
+    run_passthrough_cell(
+        compat_result=fake_result,
+        models=["claude-haiku-4-5"],
+        prompt="ping",
+        build_extra_env=build,
+        run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
+        env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"},
+    )
+
+    assert seen_bases == ["http://localhost:4000"]
+    assert captured["extra_env"] == {"SOME_FLAG": "1"}
+    assert captured["base_url"] == "http://localhost:4000"
+
+
+def test_per_model_failures_reported_individually():
+    fake_result = _FakeResult()
+    captured: Dict[str, Any] = {}
+    outcomes = {
+        "claude-haiku-4-5": DriverResult(text="pong"),
+        "claude-sonnet-4-6": ClaudeCLIError("claude CLI timed out after 120s"),
+        "claude-opus-4-7": DriverResult(text="", exit_code=1),
+    }
+
+    with pytest.raises(pytest.fail.Exception):
+        run_passthrough_cell(
+            compat_result=fake_result,
+            models=list(outcomes.keys()),
+            prompt="ping",
+            run_models=_fake_run_models(outcomes, captured),
+            env=PROXY_ENV,
+        )
+
+    statuses = [row["status"] for row in fake_result.rows]
+    assert statuses == ["pass", "fail", "fail"]
+    assert "timed out" in fake_result.rows[1]["error"]
+    assert "claude CLI failed" in fake_result.rows[2]["error"]
+
+
+def test_empty_assistant_text_is_a_fail():
+    fake_result = _FakeResult()
+    captured: Dict[str, Any] = {}
+    outcomes = {"claude-haiku-4-5": DriverResult(text="   ")}
+
+    with pytest.raises(pytest.fail.Exception):
+        run_passthrough_cell(
+            compat_result=fake_result,
+            models=["claude-haiku-4-5"],
+            prompt="ping",
+            run_models=_fake_run_models(outcomes, captured),
+            env=PROXY_ENV,
+        )
+
+    assert fake_result.rows == [
+        {
+            "status": "fail",
+            "error": "[claude-haiku-4-5] claude returned empty assistant text",
+        }
+    ]
+
+
+def test_bedrock_extra_env_targets_proxy_bedrock_route():
+    env = bedrock_extra_env("http://localhost:4000")
+    assert env == {
+        "CLAUDE_CODE_USE_BEDROCK": "1",
+        "CLAUDE_CODE_SKIP_BEDROCK_AUTH": "1",
+        "ANTHROPIC_BEDROCK_BASE_URL": "http://localhost:4000/bedrock",
+        "AWS_REGION": CLIENT_SIDE_AWS_REGION,
+    }
+
+
+def test_vertex_extra_env_keeps_the_api_version_in_the_base_url():
+    env = vertex_extra_env("http://localhost:4000")
+    assert env == {
+        "CLAUDE_CODE_USE_VERTEX": "1",
+        "CLAUDE_CODE_SKIP_VERTEX_AUTH": "1",
+        "ANTHROPIC_VERTEX_BASE_URL": "http://localhost:4000/vertex_ai/v1",
+        "ANTHROPIC_VERTEX_PROJECT_ID": VERTEX_PLACEHOLDER_PROJECT,
+        "CLOUD_ML_REGION": VERTEX_PLACEHOLDER_REGION,
+    }
+
+
+def test_foundry_extra_env_targets_proxy_azure_route():
+    env = foundry_extra_env("http://localhost:4000")
+    assert env == {
+        "CLAUDE_CODE_USE_FOUNDRY": "1",
+        "CLAUDE_CODE_SKIP_FOUNDRY_AUTH": "1",
+        "ANTHROPIC_FOUNDRY_BASE_URL": "http://localhost:4000/azure",
+    }

tests/e2e/claude_code/_passthrough.py

@@ -0,0 +1,196 @@
+"""Shared body for the `passthrough` × <provider> compat cells.
+
+Every other matrix row drives the proxy's `/v1/messages` translation
+layer: Claude Code speaks the first-party Anthropic wire and LiteLLM
+transforms the request per provider. This row instead exercises
+LiteLLM's *native passthrough* routes -- the "LLM gateway"
+configuration documented at https://code.claude.com/docs/en/gateway --
+where Claude Code speaks each cloud's own wire format and the proxy
+forwards it, attaching provider credentials on the way out:
+
+    anthropic        ANTHROPIC_BASE_URL={proxy}/anthropic. The CLI's
+                     first-party wire, forwarded verbatim to
+                     api.anthropic.com, so the model ids are real
+                     Anthropic ids rather than proxy aliases.
+    bedrock_invoke   CLAUDE_CODE_USE_BEDROCK=1 +
+                     ANTHROPIC_BEDROCK_BASE_URL={proxy}/bedrock. The
+                     CLI POSTs /model/{model}/invoke-with-response-stream;
+                     the proxy recognizes a router alias in the model
+                     segment, rewrites it to the deployment's upstream
+                     model id, and SigV4-signs with its own AWS creds.
+    vertex_ai        CLAUDE_CODE_USE_VERTEX=1 +
+                     ANTHROPIC_VERTEX_BASE_URL={proxy}/vertex_ai/v1.
+                     The CLI POSTs
+                     .../models/{model}:streamRawPredict; the proxy
+                     resolves a router alias in the model segment and
+                     takes project, location, and credentials from the
+                     deployment (which is why the deployment must set
+                     `use_in_pass_through: true` -- see
+                     test_config.yaml).
+    azure            CLAUDE_CODE_USE_FOUNDRY=1 +
+                     ANTHROPIC_FOUNDRY_BASE_URL={proxy}/azure. Foundry
+                     mode sends the model in the JSON body, not the
+                     URL, so the proxy's /azure route cannot resolve a
+                     router alias and falls back to the env-configured
+                     AZURE_API_BASE / AZURE_API_KEY target.
+    bedrock_converse not applicable -- Claude Code's bedrock mode is
+                     InvokeModel-only; no Converse-wire client exists.
+
+Auth is the same in every mode: the CLI's provider-native signing is
+disabled via CLAUDE_CODE_SKIP_<PROVIDER>_AUTH, and the LiteLLM virtual
+key travels as `Authorization: Bearer` (ANTHROPIC_AUTH_TOKEN), exactly
+like the translation rows. The proxy holds the real provider
+credentials.
+
+The per-mode env vars and URL shapes above were captured from a real
+`claude` CLI (2.1.210) run against a request-logging sink, not from
+docs; if a CLI release changes them, the cells fail with the CLI's own
+diagnostic rather than silently testing the wrong wire.
+
+`run_models` and `env` are injection seams for
+`_driver_unit_tests/test_passthrough.py`; production callers leave
+them unset.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any, Callable, Dict, Mapping, Optional, Sequence
+
+import pytest
+
+from claude_code.cli_driver import (
+    ClaudeCLIError,
+    failure_diagnostic,
+    run_claude_models_parallel,
+)
+
+PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
+PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
+
+ANTHROPIC_PASSTHROUGH_BASE_PATH = "/anthropic"
+
+CLIENT_SIDE_AWS_REGION = "us-east-1"
+"""Satisfies the CLI's embedded AWS SDK, which refuses to construct a
+client without a region. The value never influences routing: the proxy
+signs the upstream request with its own credentials and region."""
+
+VERTEX_PLACEHOLDER_PROJECT = "proxy-resolved-project"
+VERTEX_PLACEHOLDER_REGION = "us-east5"
+"""The CLI refuses to build a Vertex URL without a project id and
+region, but the proxy replaces both path segments with the resolved
+deployment's `vertex_project` / `vertex_location` before forwarding,
+so deliberately-fake values prove the resolution actually happened."""
+
+
+def bedrock_extra_env(proxy_base_url: str) -> Dict[str, str]:
+    return {
+        "CLAUDE_CODE_USE_BEDROCK": "1",
+        "CLAUDE_CODE_SKIP_BEDROCK_AUTH": "1",
+        "ANTHROPIC_BEDROCK_BASE_URL": f"{proxy_base_url}/bedrock",
+        "AWS_REGION": CLIENT_SIDE_AWS_REGION,
+    }
+
+
+def vertex_extra_env(proxy_base_url: str) -> Dict[str, str]:
+    """Vertex-mode CLI env pointed at the proxy's /vertex_ai route.
+
+    The `/v1` suffix on ANTHROPIC_VERTEX_BASE_URL is load-bearing: the
+    CLI's Vertex SDK ships its API version inside its *default* base
+    URL (`https://{region}-aiplatform.googleapis.com/v1`), so
+    overriding the base drops the version from the request path unless
+    the override carries it. LiteLLM's /vertex_ai route reuses the
+    incoming path verbatim when it contains `/projects/.../locations/...`,
+    so a version-less path would reach Google as
+    `aiplatform.googleapis.com/projects/...` and 404.
+    """
+    return {
+        "CLAUDE_CODE_USE_VERTEX": "1",
+        "CLAUDE_CODE_SKIP_VERTEX_AUTH": "1",
+        "ANTHROPIC_VERTEX_BASE_URL": f"{proxy_base_url}/vertex_ai/v1",
+        "ANTHROPIC_VERTEX_PROJECT_ID": VERTEX_PLACEHOLDER_PROJECT,
+        "CLOUD_ML_REGION": VERTEX_PLACEHOLDER_REGION,
+    }
+
+
+def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]:
+    return {
+        "CLAUDE_CODE_USE_FOUNDRY": "1",
+        "CLAUDE_CODE_SKIP_FOUNDRY_AUTH": "1",
+        "ANTHROPIC_FOUNDRY_BASE_URL": f"{proxy_base_url}/azure",
+    }
+
+
+def run_passthrough_cell(
+    *,
+    compat_result,
+    models: Sequence[str],
+    prompt: str,
+    passthrough_base_path: str = "",
+    build_extra_env: Optional[Callable[[str], Mapping[str, str]]] = None,
+    run_models: Callable[..., Mapping[str, Any]] = run_claude_models_parallel,
+    env: Optional[Mapping[str, str]] = None,
+) -> None:
+    """Run the shared `passthrough` × <provider> cell body.
+
+    `passthrough_base_path` is appended to the proxy base URL and
+    becomes the CLI's ANTHROPIC_BASE_URL (only the anthropic column
+    uses it; the cloud columns ignore ANTHROPIC_BASE_URL entirely once
+    their CLAUDE_CODE_USE_* flag is set). `build_extra_env` receives
+    the trailing-slash-normalized proxy base URL and returns the
+    provider-mode env for the CLI subprocess.
+    """
+    environ = env if env is not None else os.environ
+    base_url = environ.get(PROXY_BASE_URL_ENV)
+    api_key = environ.get(PROXY_API_KEY_ENV)
+    if not base_url or not api_key:
+        compat_result.set(
+            {
+                "status": "fail",
+                "error": (
+                    f"missing required env: set {PROXY_BASE_URL_ENV} and "
+                    f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
+                ),
+            }
+        )
+        pytest.fail(
+            f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured",
+            pytrace=False,
+        )
+
+    proxy_base = base_url.rstrip("/")
+    extra_env = dict(build_extra_env(proxy_base)) if build_extra_env else None
+
+    outcomes = run_models(
+        models=models,
+        prompt=prompt,
+        base_url=proxy_base + passthrough_base_path,
+        api_key=api_key,
+        extra_env=extra_env,
+    )
+
+    failures = []
+    for model in models:
+        outcome = outcomes[model]
+        if isinstance(outcome, ClaudeCLIError):
+            error = f"[{model}] {outcome}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if outcome.exit_code != 0:
+            error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        if not outcome.text.strip():
+            error = f"[{model}] claude returned empty assistant text"
+            compat_result.add({"status": "fail", "error": error})
+            failures.append(error)
+            continue
+
+        compat_result.add({"status": "pass"})
+
+    if failures:
+        pytest.fail("; ".join(failures), pytrace=False)

tests/e2e/claude_code/cli_driver.py

@@ -15,6 +15,7 @@ from __future__ import annotations
 
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -40,6 +41,29 @@ DEFAULT_TIMEOUT_SECONDS = float(
     os.environ.get("LITELLM_COMPAT_CLI_TIMEOUT_SECONDS") or 120
 )
 
+RATE_LIMIT_SHAPED_RE = re.compile(
+    r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|"
+    r"claude\s+CLI\s+timed\s+out)",
+    re.IGNORECASE,
+)
+"""Heuristic shared with the conftest rate-limit summary: 429s and
+throttle markers anywhere in the failure text, plus CLI timeouts --
+the CLI retries 429s internally until the harness timeout kills it,
+so a saturated upstream usually surfaces as a timeout rather than a
+clean 429."""
+
+DEFAULT_RATE_LIMIT_RETRIES = int(
+    os.environ.get("LITELLM_COMPAT_RATE_LIMIT_RETRIES") or 2
+)
+DEFAULT_RATE_LIMIT_BACKOFF_SECONDS = float(
+    os.environ.get("LITELLM_COMPAT_RATE_LIMIT_BACKOFF_SECONDS") or 65
+)
+"""Rate-limit-shaped failures are retried after a backoff long enough
+for a per-minute quota window (the dominant 429 source across
+Anthropic / Bedrock / Vertex) to reset. Both knobs are env-tunable so
+a matrix run can trade wall time for resilience without code edits;
+retries=0 disables the behavior entirely."""
+
 # Env vars the `claude` Node CLI legitimately needs to function:
 # locating its own binary + node, basic locale/terminal plumbing.
 # Deliberately excludes every credential-bearing var that the
@@ -265,6 +289,22 @@ def run_claude(
 ModelResult = Union[DriverResult, ClaudeCLIError]
 
 
+def is_rate_limit_shaped(outcome: ModelResult) -> bool:
+    """Classify an outcome as a retryable rate-limit-shaped failure.
+
+    A `ClaudeCLIError` matches on its message (which is where the
+    driver's own timeout diagnostic lands); a failing `DriverResult`
+    matches on its full `failure_diagnostic` so 429s buried in the
+    CLI's stdout text or `api_error_status` are both caught. Passing
+    results are never rate-limit-shaped.
+    """
+    if isinstance(outcome, ClaudeCLIError):
+        return bool(RATE_LIMIT_SHAPED_RE.search(str(outcome)))
+    if outcome.exit_code == 0:
+        return False
+    return bool(RATE_LIMIT_SHAPED_RE.search(failure_diagnostic(outcome)))
+
+
 def run_claude_models_parallel(
     *,
     models: Sequence[str],
@@ -277,6 +317,9 @@ def run_claude_models_parallel(
     cli_path: str = CLAUDE_CLI_DEFAULT,
     timeout: float = DEFAULT_TIMEOUT_SECONDS,
     runner: Optional[Callable[..., Any]] = None,
+    rate_limit_retries: Optional[int] = None,
+    rate_limit_backoff_seconds: Optional[float] = None,
+    sleep: Callable[[float], None] = time.sleep,
 ) -> Dict[str, ModelResult]:
     """Invoke `run_claude` for every `models[i]` concurrently and collect outcomes.
 
@@ -290,6 +333,14 @@ def run_claude_models_parallel(
     keep the synchronous CLI driver unchanged so unit tests can keep
     injecting a fake `runner`.
 
+    Rate-limit-shaped failures (see `is_rate_limit_shaped`) are retried
+    per model up to `rate_limit_retries` times, sleeping
+    `rate_limit_backoff_seconds` before each retry so per-minute quota
+    windows can reset; both default to the `LITELLM_COMPAT_RATE_LIMIT_*`
+    env knobs. Each retry goes back through `run_claude`, so it
+    re-acquires a token from the provider rate limiter like any other
+    invocation. `sleep` is an injection seam for unit tests.
+
     Returns a dict keyed by model id. Each value is either the
     `DriverResult` produced by `run_claude` or the `ClaudeCLIError`
     that aborted that model's run — callers decide how to map either
@@ -300,14 +351,20 @@ def run_claude_models_parallel(
     if not models:
         raise ValueError("models must be a non-empty sequence")
 
-    def _one(model: str) -> Tuple[str, ModelResult, float]:
-        # Per-model wall clock: this is what the matrix run actually pays for.
-        # We record it whether the run succeeded or raised so the breakdown
-        # log below covers both code paths and surfaces "which model is the
-        # long pole?" without requiring per-test instrumentation.
-        started = time.monotonic()
+    retries = (
+        DEFAULT_RATE_LIMIT_RETRIES
+        if rate_limit_retries is None
+        else max(0, rate_limit_retries)
+    )
+    backoff = (
+        DEFAULT_RATE_LIMIT_BACKOFF_SECONDS
+        if rate_limit_backoff_seconds is None
+        else max(0.0, rate_limit_backoff_seconds)
+    )
+
+    def _run_once(model: str) -> ModelResult:
         try:
-            result = run_claude(
+            return run_claude(
                 prompt=prompt,
                 model=model,
                 base_url=base_url,
@@ -319,14 +376,8 @@ def run_claude_models_parallel(
                 timeout=timeout,
                 runner=runner,
             )
-            elapsed = time.monotonic() - started
-            # Stamp the duration onto the DriverResult so callers (tests,
-            # diagnostics) can attribute slow cells without re-timing.
-            result.duration_ms = int(elapsed * 1000)
-            return model, result, elapsed
         except ClaudeCLIError as exc:
-            elapsed = time.monotonic() - started
-            return model, exc, elapsed
+            return exc
         except Exception as exc:
             # Honor the documented "errors as values" contract for any
             # exception type — not just ClaudeCLIError. The rate
@@ -334,13 +385,38 @@ def run_claude_models_parallel(
             # raise ValueError on edge-case model strings, and a future
             # bug elsewhere in the call stack must not abort the entire
             # parallel batch and lose the other models' outcomes.
-            elapsed = time.monotonic() - started
             wrapped = ClaudeCLIError(
                 f"unexpected error running model {model!r}: "
                 f"{type(exc).__name__}: {exc}"
             )
             wrapped.__cause__ = exc
-            return model, wrapped, elapsed
+            return wrapped
+
+    def _one(model: str) -> Tuple[str, ModelResult, float]:
+        # Per-model wall clock: this is what the matrix run actually pays
+        # for, retries and backoff sleeps included. We record it whether
+        # the run succeeded or raised so the breakdown log below covers
+        # both code paths and surfaces "which model is the long pole?"
+        # without requiring per-test instrumentation.
+        started = time.monotonic()
+        outcome = _run_once(model)
+        for attempt in range(retries):
+            if not is_rate_limit_shaped(outcome):
+                break
+            print(
+                f"[retry] {model}: rate-limit-shaped failure; sleeping "
+                f"{backoff:.0f}s before attempt {attempt + 2}/{retries + 1}",
+                file=sys.stderr,
+                flush=True,
+            )
+            sleep(backoff)
+            outcome = _run_once(model)
+        elapsed = time.monotonic() - started
+        if isinstance(outcome, DriverResult):
+            # Stamp the duration onto the DriverResult so callers (tests,
+            # diagnostics) can attribute slow cells without re-timing.
+            outcome.duration_ms = int(elapsed * 1000)
+        return model, outcome, elapsed
 
     outcomes: Dict[str, ModelResult] = {}
     durations: Dict[str, float] = {}

tests/e2e/claude_code/conftest.py

@@ -33,7 +33,6 @@ from __future__ import annotations
 import functools
 import json
 import os
-import re
 import sys
 from collections import Counter, defaultdict
 from dataclasses import dataclass, field
@@ -43,6 +42,8 @@ from typing import Any, Dict, FrozenSet, List, Optional, Tuple
 import pytest
 import yaml
 
+from claude_code.cli_driver import RATE_LIMIT_SHAPED_RE
+
 VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"}
 RESULTS_ARTIFACT_ENV = "COMPAT_RESULTS_PATH"
 DEFAULT_ARTIFACT_PATH = "compat-results.json"
@@ -62,11 +63,10 @@ DEFAULT_RATE_LIMIT_SUMMARY_PATH = "compat-rate-limit-summary.json"
 # the rate limiter is supposed to back off from. False positives on a
 # genuinely slow upstream are tolerable here because the worst case is
 # the binary search runs at a slightly lower rate than necessary.
-_RATE_LIMIT_RE = re.compile(
-    r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|"
-    r"claude\s+CLI\s+timed\s+out)",
-    re.IGNORECASE,
-)
+#
+# The pattern lives in `cli_driver` so the driver's retry-on-rate-limit
+# logic and this summary classify failures identically.
+_RATE_LIMIT_RE = RATE_LIMIT_SHAPED_RE
 
 
 @dataclass

tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example

@@ -27,6 +27,15 @@ VERTEXAI_LOCATION=global
 AZURE_FOUNDRY_API_KEY=
 AZURE_FOUNDRY_API_BASE=
 
+# Azure cell of the `passthrough` row. Foundry-mode Claude Code sends
+# the model in the request body, so the proxy's /azure passthrough
+# cannot resolve a router alias and falls back to these env vars.
+# AZURE_API_BASE is the Foundry resource's Anthropic surface, i.e.
+# https://<resource>.services.ai.azure.com/anthropic ; AZURE_API_KEY
+# is the same key as AZURE_FOUNDRY_API_KEY.
+AZURE_API_BASE=
+AZURE_API_KEY=
+
 # REQUIRED for publishing: PAT for the `agent-shin` user, used to push
 # the daily compat-matrix branch to its fork (agent-shin/litellm-docs)
 # and open the cross-repo PR against BerriAI/litellm-docs. Scopes:

tests/e2e/claude_code/manifest.yaml

@@ -91,6 +91,23 @@ features:
     # Code releases. The HTTP probe hits the bug surface LiteLLM
     # has actually shipped fixes for (2.1.117, 2.1.72, 2.1.70 per
     # the Claude Code release notes).
+  - id: passthrough
+    name: Native API passthrough
+    # Drives the CLI in each cloud's native mode against LiteLLM's
+    # passthrough routes instead of the /v1/messages translation
+    # layer -- the "LLM gateway" setup from
+    # https://code.claude.com/docs/en/gateway. anthropic uses
+    # ANTHROPIC_BASE_URL={proxy}/anthropic; bedrock_invoke uses
+    # CLAUDE_CODE_USE_BEDROCK=1 against {proxy}/bedrock (InvokeModel
+    # wire, alias resolved from the URL by the router); vertex_ai
+    # uses CLAUDE_CODE_USE_VERTEX=1 against {proxy}/vertex_ai/v1
+    # (rawPredict wire, alias + project + location resolved from the
+    # deployment, which therefore needs `use_in_pass_through: true`);
+    # azure uses CLAUDE_CODE_USE_FOUNDRY=1 against {proxy}/azure and
+    # needs AZURE_API_BASE/AZURE_API_KEY on the proxy (see
+    # passthrough/test_azure.py and the cron env example).
+    # bedrock_converse is structurally not_applicable: Claude Code
+    # has no Converse-wire client.
   - id: long_context_1m
     name: Long context (1M)
     # Sends a ~210k-token padded prompt with the

tests/e2e/claude_code/passthrough/test_anthropic.py

@@ -0,0 +1,44 @@
+"""passthrough x Anthropic.
+
+Drive the real `claude` CLI in its default first-party mode, but with
+ANTHROPIC_BASE_URL aimed at the proxy's `/anthropic` passthrough route
+instead of the `/v1/messages` translation endpoint. The proxy forwards
+the request verbatim to api.anthropic.com, swapping the virtual-key
+bearer for its own ANTHROPIC_API_KEY.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/passthrough/test_anthropic.py
+                       ^^^^^^^^^^^     ^^^^^^^^^
+                       feature_id      provider
+
+Because nothing is translated, the model ids are the real Anthropic API
+ids (which happen to equal the proxy aliases for this column). A red
+cell here means the passthrough route broke forwarding itself -- auth
+header swap, streaming SSE relay, or beta-header propagation -- since
+no per-provider transformation is involved.
+"""
+
+from __future__ import annotations
+
+from claude_code._passthrough import (
+    ANTHROPIC_PASSTHROUGH_BASE_PATH,
+    run_passthrough_cell,
+)
+
+ANTHROPIC_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+
+def test_passthrough_anthropic(compat_result):
+    """Drive the `claude` CLI through `{proxy}/anthropic` and assert a reply."""
+    run_passthrough_cell(
+        compat_result=compat_result,
+        models=ANTHROPIC_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH,
+    )

tests/e2e/claude_code/passthrough/test_azure.py

@@ -0,0 +1,59 @@
+"""passthrough x Azure (Microsoft Foundry).
+
+Drive the real `claude` CLI in foundry mode (CLAUDE_CODE_USE_FOUNDRY=1)
+with ANTHROPIC_FOUNDRY_BASE_URL aimed at the proxy's `/azure`
+passthrough route. The CLI POSTs `/v1/messages` with the model in the
+JSON body -- unlike the bedrock/vertex modes there is no model segment
+in the URL, so the proxy's router-alias resolution cannot engage and
+the `/azure` route falls back to its env-configured target: the proxy
+must set AZURE_API_BASE to the Foundry resource's Anthropic surface
+(`https://<resource>.services.ai.azure.com/anthropic`) and
+AZURE_API_KEY to the Foundry key (see
+cron_vm/litellm-compat-matrix.env.example). The model ids are the
+Foundry deployment names, which this matrix provisions to match the
+Anthropic ids.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/passthrough/test_azure.py
+                       ^^^^^^^^^^^     ^^^^^
+                       feature_id      provider
+
+Wiring verified live at authoring time: through `{proxy}/azure` the
+Foundry Anthropic surface accepted the `api-key` / `Authorization:
+Bearer` headers the fallback sends (a bogus key 401s, the real key
+proceeds to deployment lookup), so a red cell here means missing
+AZURE_API_BASE/AZURE_API_KEY on the proxy, missing Foundry deployments
+for the three tiers, or a genuine forwarding gap -- not an auth-scheme
+mismatch.
+
+Known-red at authoring time against a healthy Foundry resource: the
+`/azure` fallback assembles only its own auth headers and drops the
+rest of the client's headers, including the `anthropic-version` header
+the CLI sends, and Foundry's Anthropic surface rejects the request
+with 400 "anthropic-version: header is required" (the same request
+sent directly to Foundry with that header succeeds). This cell stays
+red until that forwarding gap is fixed, which is precisely the class
+of bug the row exists to surface.
+"""
+
+from __future__ import annotations
+
+from claude_code._passthrough import foundry_extra_env, run_passthrough_cell
+
+AZURE_MODELS = [
+    "claude-haiku-4-5",
+    "claude-sonnet-4-6",
+    "claude-opus-4-7",
+]
+
+
+def test_passthrough_azure(compat_result):
+    """Drive the `claude` CLI through `{proxy}/azure` and assert a reply."""
+    run_passthrough_cell(
+        compat_result=compat_result,
+        models=AZURE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        build_extra_env=foundry_extra_env,
+    )

tests/e2e/claude_code/passthrough/test_bedrock_converse.py

@@ -0,0 +1,34 @@
+"""passthrough x Bedrock (Converse).
+
+Structurally not applicable. In bedrock mode the `claude` CLI speaks
+only the InvokeModel wire (`/model/{id}/invoke-with-response-stream`);
+it has no Converse-wire client, so there is no Claude Code traffic a
+Converse passthrough could serve. LiteLLM's `/bedrock` route does
+accept `/model/{id}/converse-stream`, but exercising it would test a
+wire no Claude Code user can produce, which is out of scope for this
+matrix.
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/passthrough/test_bedrock_converse.py
+                       ^^^^^^^^^^^     ^^^^^^^^^^^^^^^^
+                       feature_id      provider
+"""
+
+from __future__ import annotations
+
+
+def test_passthrough_bedrock_converse(compat_result):
+    """Report not_applicable: Claude Code has no Converse-wire mode."""
+    compat_result.set(
+        {
+            "status": "not_applicable",
+            "reason": (
+                "Claude Code's bedrock mode speaks only the InvokeModel wire "
+                "(/model/{id}/invoke-with-response-stream); it has no "
+                "Converse-wire client, so there is no Claude Code surface "
+                "for Converse passthrough."
+            ),
+        }
+    )

tests/e2e/claude_code/passthrough/test_bedrock_invoke.py

@@ -0,0 +1,42 @@
+"""passthrough x Bedrock (Invoke).
+
+Drive the real `claude` CLI in bedrock mode (CLAUDE_CODE_USE_BEDROCK=1)
+with ANTHROPIC_BEDROCK_BASE_URL aimed at the proxy's `/bedrock`
+passthrough route. The CLI speaks the native InvokeModel wire --
+`POST /model/{model}/invoke-with-response-stream` -- with the proxy
+alias in the model segment; the proxy resolves the alias through its
+router, rewrites the path to the deployment's upstream model id, and
+SigV4-signs the forwarded request with its own AWS credentials
+(CLAUDE_CODE_SKIP_BEDROCK_AUTH=1 keeps the CLI from signing).
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/passthrough/test_bedrock_invoke.py
+                       ^^^^^^^^^^^     ^^^^^^^^^^^^^^
+                       feature_id      provider
+
+The CLI also fires a best-effort `GET /bedrock/inference-profiles`
+listing at startup; its failure is non-fatal and does not gate this
+cell.
+"""
+
+from __future__ import annotations
+
+from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell
+
+BEDROCK_INVOKE_MODELS = [
+    "claude-haiku-4-5-bedrock-invoke",
+    "claude-sonnet-4-6-bedrock-invoke",
+    "claude-opus-4-7-bedrock-invoke",
+]
+
+
+def test_passthrough_bedrock_invoke(compat_result):
+    """Drive the `claude` CLI through `{proxy}/bedrock` and assert a reply."""
+    run_passthrough_cell(
+        compat_result=compat_result,
+        models=BEDROCK_INVOKE_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        build_extra_env=bedrock_extra_env,
+    )

tests/e2e/claude_code/passthrough/test_vertex_ai.py

@@ -0,0 +1,45 @@
+"""passthrough x Vertex AI.
+
+Drive the real `claude` CLI in vertex mode (CLAUDE_CODE_USE_VERTEX=1)
+with ANTHROPIC_VERTEX_BASE_URL aimed at the proxy's `/vertex_ai`
+passthrough route. The CLI speaks the native rawPredict wire --
+`POST .../projects/{p}/locations/{l}/publishers/anthropic/models/{model}:streamRawPredict`
+-- with the proxy alias in the model segment; the proxy resolves the
+alias through its router, replaces the placeholder project/location
+path segments with the deployment's `vertex_project` /
+`vertex_location`, and attaches its own Google credentials
+(CLAUDE_CODE_SKIP_VERTEX_AUTH=1 keeps the CLI from minting a token).
+
+The (feature, provider) for this cell is inferred from the file path by
+`tests/e2e/claude_code/conftest.py`:
+
+    tests/e2e/claude_code/passthrough/test_vertex_ai.py
+                       ^^^^^^^^^^^     ^^^^^^^^^
+                       feature_id      provider
+
+This cell requires the vertex deployments in the proxy config to carry
+`use_in_pass_through: true` (see test_config.yaml) -- that is what
+registers their credentials with the passthrough router. Without it
+the proxy forwards the CLI's own headers (the virtual-key bearer) to
+Google and every tier fails with a 401.
+"""
+
+from __future__ import annotations
+
+from claude_code._passthrough import run_passthrough_cell, vertex_extra_env
+
+VERTEX_MODELS = [
+    "claude-haiku-4-5-vertex",
+    "claude-sonnet-4-6-vertex",
+    "claude-opus-4-7-vertex",
+]
+
+
+def test_passthrough_vertex_ai(compat_result):
+    """Drive the `claude` CLI through `{proxy}/vertex_ai` and assert a reply."""
+    run_passthrough_cell(
+        compat_result=compat_result,
+        models=VERTEX_MODELS,
+        prompt="Reply with the single word 'pong' and nothing else.",
+        build_extra_env=vertex_extra_env,
+    )

tests/e2e/claude_code/test_config.yaml

@@ -59,21 +59,31 @@ model_list:
       aws_region_name: us-east-1
 
   # ---- Vertex AI ----
+  # `use_in_pass_through: true` registers each deployment's
+  # project/location/credentials with the /vertex_ai passthrough
+  # router, which the `passthrough` row needs to resolve
+  # .../models/{alias}:streamRawPredict URLs. That registration only
+  # reads the canonical `vertex_project`/`vertex_location` param names
+  # (not the `vertex_ai_*` aliases); the chat translation path accepts
+  # both.
   - model_name: claude-haiku-4-5-vertex
     litellm_params:
       model: vertex_ai/claude-haiku-4-5
-      vertex_ai_project: os.environ/VERTEXAI_PROJECT
-      vertex_ai_location: os.environ/VERTEXAI_LOCATION
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: os.environ/VERTEXAI_LOCATION
+      use_in_pass_through: true
   - model_name: claude-sonnet-4-6-vertex
     litellm_params:
       model: vertex_ai/claude-sonnet-4-6
-      vertex_ai_project: os.environ/VERTEXAI_PROJECT
-      vertex_ai_location: os.environ/VERTEXAI_LOCATION
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: os.environ/VERTEXAI_LOCATION
+      use_in_pass_through: true
   - model_name: claude-opus-4-7-vertex
     litellm_params:
       model: vertex_ai/claude-opus-4-7
-      vertex_ai_project: os.environ/VERTEXAI_PROJECT
-      vertex_ai_location: os.environ/VERTEXAI_LOCATION
+      vertex_project: os.environ/VERTEXAI_PROJECT
+      vertex_location: os.environ/VERTEXAI_LOCATION
+      use_in_pass_through: true
 
   # ---- Microsoft Foundry (Anthropic deployments on Azure) ----
   - model_name: claude-haiku-4-5-azure