mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
test(e2e/claude_code): add passthrough matrix row for the big-3 clouds and Anthropic API
Adds a 'passthrough' feature row to the Claude Code compat matrix that
drives the real claude CLI in each cloud's native mode against
LiteLLM's passthrough routes (the LLM-gateway setup from
code.claude.com/docs/en/gateway) instead of the /v1/messages
translation layer:
- anthropic: ANTHROPIC_BASE_URL={proxy}/anthropic, forwarded verbatim
to api.anthropic.com
- bedrock_invoke: CLAUDE_CODE_USE_BEDROCK=1 against {proxy}/bedrock;
the router resolves the alias in /model/{alias}/invoke-with-response-stream
- vertex_ai: CLAUDE_CODE_USE_VERTEX=1 against {proxy}/vertex_ai/v1;
alias, project, location and credentials resolve from the deployment,
which now sets use_in_pass_through: true (and the canonical
vertex_project/vertex_location param names) in test_config.yaml
- azure: CLAUDE_CODE_USE_FOUNDRY=1 against {proxy}/azure via the
AZURE_API_BASE/AZURE_API_KEY fallback (documented in the cron env
example)
- bedrock_converse: not_applicable; Claude Code has no Converse-wire
client
The shared cell body lives in _passthrough.py with injectable runner
and env (no monkeypatching), unit-covered in
_driver_unit_tests/test_passthrough.py including pins on the per-mode
CLI env contracts captured from a real claude CLI (2.1.210) run
against a request-logging sink.
This commit is contained in:
parent
b907378f02
commit
95fe8687c8
11 changed files with 648 additions and 6 deletions
195
tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py
Normal file
195
tests/e2e/claude_code/_driver_unit_tests/test_passthrough.py
Normal file
|
|
@ -0,0 +1,195 @@
|
|||
"""Unit tests for the shared `run_passthrough_cell` helper.
|
||||
|
||||
These tests inject a fake `run_models` callable and an explicit `env`
|
||||
mapping (both are first-class parameters, no monkeypatching), so they
|
||||
exercise the helper's branching -- env-missing guard, base-URL
|
||||
assembly, extra-env forwarding, per-model pass/fail -- without
|
||||
spawning the real CLI.
|
||||
|
||||
The env-builder tests pin the provider-mode contract itself: the
|
||||
CLAUDE_CODE_USE_* / CLAUDE_CODE_SKIP_*_AUTH flags and the passthrough
|
||||
route each mode must target. Those values are the feature -- e.g.
|
||||
dropping the `/v1` from the vertex base URL produces a request Google
|
||||
404s on -- so a mutation to any of them must fail here before it burns
|
||||
a live matrix run.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Mapping, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code._passthrough import (
|
||||
ANTHROPIC_PASSTHROUGH_BASE_PATH,
|
||||
CLIENT_SIDE_AWS_REGION,
|
||||
VERTEX_PLACEHOLDER_PROJECT,
|
||||
VERTEX_PLACEHOLDER_REGION,
|
||||
bedrock_extra_env,
|
||||
foundry_extra_env,
|
||||
run_passthrough_cell,
|
||||
vertex_extra_env,
|
||||
)
|
||||
from claude_code.cli_driver import ClaudeCLIError, DriverResult
|
||||
|
||||
PROXY_ENV = {
|
||||
"LITELLM_PROXY_BASE_URL": "http://localhost:4000",
|
||||
"LITELLM_PROXY_API_KEY": "sk-test",
|
||||
}
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def __init__(self) -> None:
|
||||
self.rows: List[Dict[str, Any]] = []
|
||||
self.single: Optional[Dict[str, Any]] = None
|
||||
|
||||
def set(self, payload: Mapping[str, Any]) -> None:
|
||||
self.single = dict(payload)
|
||||
|
||||
def add(self, payload: Mapping[str, Any]) -> None:
|
||||
self.rows.append(dict(payload))
|
||||
|
||||
|
||||
def _fake_run_models(outcomes_by_model, captured: Dict[str, Any]):
|
||||
def fake(*, models, prompt, base_url, api_key, extra_env=None, **_kwargs):
|
||||
captured["models"] = list(models)
|
||||
captured["prompt"] = prompt
|
||||
captured["base_url"] = base_url
|
||||
captured["api_key"] = api_key
|
||||
captured["extra_env"] = dict(extra_env) if extra_env is not None else None
|
||||
return {model: outcomes_by_model[model] for model in models}
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
def test_env_missing_guard_reports_fail_and_aborts():
|
||||
fake_result = _FakeResult()
|
||||
with pytest.raises(pytest.fail.Exception):
|
||||
run_passthrough_cell(
|
||||
compat_result=fake_result,
|
||||
models=["claude-haiku-4-5"],
|
||||
prompt="ping",
|
||||
env={},
|
||||
)
|
||||
assert fake_result.single is not None
|
||||
assert fake_result.single["status"] == "fail"
|
||||
assert "LITELLM_PROXY_BASE_URL" in fake_result.single["error"]
|
||||
|
||||
|
||||
def test_anthropic_base_path_appended_to_normalized_proxy_url():
|
||||
fake_result = _FakeResult()
|
||||
captured: Dict[str, Any] = {}
|
||||
outcome = DriverResult(text="pong")
|
||||
|
||||
run_passthrough_cell(
|
||||
compat_result=fake_result,
|
||||
models=["claude-haiku-4-5"],
|
||||
prompt="ping",
|
||||
passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH,
|
||||
run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
|
||||
env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"},
|
||||
)
|
||||
|
||||
assert captured["base_url"] == "http://localhost:4000/anthropic"
|
||||
assert captured["extra_env"] is None
|
||||
assert fake_result.rows == [{"status": "pass"}]
|
||||
|
||||
|
||||
def test_extra_env_builder_receives_normalized_base_and_is_forwarded():
|
||||
fake_result = _FakeResult()
|
||||
captured: Dict[str, Any] = {}
|
||||
outcome = DriverResult(text="pong")
|
||||
seen_bases: List[str] = []
|
||||
|
||||
def build(proxy_base: str) -> Dict[str, str]:
|
||||
seen_bases.append(proxy_base)
|
||||
return {"SOME_FLAG": "1"}
|
||||
|
||||
run_passthrough_cell(
|
||||
compat_result=fake_result,
|
||||
models=["claude-haiku-4-5"],
|
||||
prompt="ping",
|
||||
build_extra_env=build,
|
||||
run_models=_fake_run_models({"claude-haiku-4-5": outcome}, captured),
|
||||
env={**PROXY_ENV, "LITELLM_PROXY_BASE_URL": "http://localhost:4000/"},
|
||||
)
|
||||
|
||||
assert seen_bases == ["http://localhost:4000"]
|
||||
assert captured["extra_env"] == {"SOME_FLAG": "1"}
|
||||
assert captured["base_url"] == "http://localhost:4000"
|
||||
|
||||
|
||||
def test_per_model_failures_reported_individually():
|
||||
fake_result = _FakeResult()
|
||||
captured: Dict[str, Any] = {}
|
||||
outcomes = {
|
||||
"claude-haiku-4-5": DriverResult(text="pong"),
|
||||
"claude-sonnet-4-6": ClaudeCLIError("claude CLI timed out after 120s"),
|
||||
"claude-opus-4-7": DriverResult(text="", exit_code=1),
|
||||
}
|
||||
|
||||
with pytest.raises(pytest.fail.Exception):
|
||||
run_passthrough_cell(
|
||||
compat_result=fake_result,
|
||||
models=list(outcomes.keys()),
|
||||
prompt="ping",
|
||||
run_models=_fake_run_models(outcomes, captured),
|
||||
env=PROXY_ENV,
|
||||
)
|
||||
|
||||
statuses = [row["status"] for row in fake_result.rows]
|
||||
assert statuses == ["pass", "fail", "fail"]
|
||||
assert "timed out" in fake_result.rows[1]["error"]
|
||||
assert "claude CLI failed" in fake_result.rows[2]["error"]
|
||||
|
||||
|
||||
def test_empty_assistant_text_is_a_fail():
|
||||
fake_result = _FakeResult()
|
||||
captured: Dict[str, Any] = {}
|
||||
outcomes = {"claude-haiku-4-5": DriverResult(text=" ")}
|
||||
|
||||
with pytest.raises(pytest.fail.Exception):
|
||||
run_passthrough_cell(
|
||||
compat_result=fake_result,
|
||||
models=["claude-haiku-4-5"],
|
||||
prompt="ping",
|
||||
run_models=_fake_run_models(outcomes, captured),
|
||||
env=PROXY_ENV,
|
||||
)
|
||||
|
||||
assert fake_result.rows == [
|
||||
{
|
||||
"status": "fail",
|
||||
"error": "[claude-haiku-4-5] claude returned empty assistant text",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_bedrock_extra_env_targets_proxy_bedrock_route():
|
||||
env = bedrock_extra_env("http://localhost:4000")
|
||||
assert env == {
|
||||
"CLAUDE_CODE_USE_BEDROCK": "1",
|
||||
"CLAUDE_CODE_SKIP_BEDROCK_AUTH": "1",
|
||||
"ANTHROPIC_BEDROCK_BASE_URL": "http://localhost:4000/bedrock",
|
||||
"AWS_REGION": CLIENT_SIDE_AWS_REGION,
|
||||
}
|
||||
|
||||
|
||||
def test_vertex_extra_env_keeps_the_api_version_in_the_base_url():
|
||||
env = vertex_extra_env("http://localhost:4000")
|
||||
assert env == {
|
||||
"CLAUDE_CODE_USE_VERTEX": "1",
|
||||
"CLAUDE_CODE_SKIP_VERTEX_AUTH": "1",
|
||||
"ANTHROPIC_VERTEX_BASE_URL": "http://localhost:4000/vertex_ai/v1",
|
||||
"ANTHROPIC_VERTEX_PROJECT_ID": VERTEX_PLACEHOLDER_PROJECT,
|
||||
"CLOUD_ML_REGION": VERTEX_PLACEHOLDER_REGION,
|
||||
}
|
||||
|
||||
|
||||
def test_foundry_extra_env_targets_proxy_azure_route():
|
||||
env = foundry_extra_env("http://localhost:4000")
|
||||
assert env == {
|
||||
"CLAUDE_CODE_USE_FOUNDRY": "1",
|
||||
"CLAUDE_CODE_SKIP_FOUNDRY_AUTH": "1",
|
||||
"ANTHROPIC_FOUNDRY_BASE_URL": "http://localhost:4000/azure",
|
||||
}
|
||||
196
tests/e2e/claude_code/_passthrough.py
Normal file
196
tests/e2e/claude_code/_passthrough.py
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
"""Shared body for the `passthrough` × <provider> compat cells.
|
||||
|
||||
Every other matrix row drives the proxy's `/v1/messages` translation
|
||||
layer: Claude Code speaks the first-party Anthropic wire and LiteLLM
|
||||
transforms the request per provider. This row instead exercises
|
||||
LiteLLM's *native passthrough* routes -- the "LLM gateway"
|
||||
configuration documented at https://code.claude.com/docs/en/gateway --
|
||||
where Claude Code speaks each cloud's own wire format and the proxy
|
||||
forwards it, attaching provider credentials on the way out:
|
||||
|
||||
anthropic ANTHROPIC_BASE_URL={proxy}/anthropic. The CLI's
|
||||
first-party wire, forwarded verbatim to
|
||||
api.anthropic.com, so the model ids are real
|
||||
Anthropic ids rather than proxy aliases.
|
||||
bedrock_invoke CLAUDE_CODE_USE_BEDROCK=1 +
|
||||
ANTHROPIC_BEDROCK_BASE_URL={proxy}/bedrock. The
|
||||
CLI POSTs /model/{model}/invoke-with-response-stream;
|
||||
the proxy recognizes a router alias in the model
|
||||
segment, rewrites it to the deployment's upstream
|
||||
model id, and SigV4-signs with its own AWS creds.
|
||||
vertex_ai CLAUDE_CODE_USE_VERTEX=1 +
|
||||
ANTHROPIC_VERTEX_BASE_URL={proxy}/vertex_ai/v1.
|
||||
The CLI POSTs
|
||||
.../models/{model}:streamRawPredict; the proxy
|
||||
resolves a router alias in the model segment and
|
||||
takes project, location, and credentials from the
|
||||
deployment (which is why the deployment must set
|
||||
`use_in_pass_through: true` -- see
|
||||
test_config.yaml).
|
||||
azure CLAUDE_CODE_USE_FOUNDRY=1 +
|
||||
ANTHROPIC_FOUNDRY_BASE_URL={proxy}/azure. Foundry
|
||||
mode sends the model in the JSON body, not the
|
||||
URL, so the proxy's /azure route cannot resolve a
|
||||
router alias and falls back to the env-configured
|
||||
AZURE_API_BASE / AZURE_API_KEY target.
|
||||
bedrock_converse not applicable -- Claude Code's bedrock mode is
|
||||
InvokeModel-only; no Converse-wire client exists.
|
||||
|
||||
Auth is the same in every mode: the CLI's provider-native signing is
|
||||
disabled via CLAUDE_CODE_SKIP_<PROVIDER>_AUTH, and the LiteLLM virtual
|
||||
key travels as `Authorization: Bearer` (ANTHROPIC_AUTH_TOKEN), exactly
|
||||
like the translation rows. The proxy holds the real provider
|
||||
credentials.
|
||||
|
||||
The per-mode env vars and URL shapes above were captured from a real
|
||||
`claude` CLI (2.1.210) run against a request-logging sink, not from
|
||||
docs; if a CLI release changes them, the cells fail with the CLI's own
|
||||
diagnostic rather than silently testing the wrong wire.
|
||||
|
||||
`run_models` and `env` are injection seams for
|
||||
`_driver_unit_tests/test_passthrough.py`; production callers leave
|
||||
them unset.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Callable, Dict, Mapping, Optional, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from claude_code.cli_driver import (
|
||||
ClaudeCLIError,
|
||||
failure_diagnostic,
|
||||
run_claude_models_parallel,
|
||||
)
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
ANTHROPIC_PASSTHROUGH_BASE_PATH = "/anthropic"
|
||||
|
||||
CLIENT_SIDE_AWS_REGION = "us-east-1"
|
||||
"""Satisfies the CLI's embedded AWS SDK, which refuses to construct a
|
||||
client without a region. The value never influences routing: the proxy
|
||||
signs the upstream request with its own credentials and region."""
|
||||
|
||||
VERTEX_PLACEHOLDER_PROJECT = "proxy-resolved-project"
|
||||
VERTEX_PLACEHOLDER_REGION = "us-east5"
|
||||
"""The CLI refuses to build a Vertex URL without a project id and
|
||||
region, but the proxy replaces both path segments with the resolved
|
||||
deployment's `vertex_project` / `vertex_location` before forwarding,
|
||||
so deliberately-fake values prove the resolution actually happened."""
|
||||
|
||||
|
||||
def bedrock_extra_env(proxy_base_url: str) -> Dict[str, str]:
|
||||
return {
|
||||
"CLAUDE_CODE_USE_BEDROCK": "1",
|
||||
"CLAUDE_CODE_SKIP_BEDROCK_AUTH": "1",
|
||||
"ANTHROPIC_BEDROCK_BASE_URL": f"{proxy_base_url}/bedrock",
|
||||
"AWS_REGION": CLIENT_SIDE_AWS_REGION,
|
||||
}
|
||||
|
||||
|
||||
def vertex_extra_env(proxy_base_url: str) -> Dict[str, str]:
|
||||
"""Vertex-mode CLI env pointed at the proxy's /vertex_ai route.
|
||||
|
||||
The `/v1` suffix on ANTHROPIC_VERTEX_BASE_URL is load-bearing: the
|
||||
CLI's Vertex SDK ships its API version inside its *default* base
|
||||
URL (`https://{region}-aiplatform.googleapis.com/v1`), so
|
||||
overriding the base drops the version from the request path unless
|
||||
the override carries it. LiteLLM's /vertex_ai route reuses the
|
||||
incoming path verbatim when it contains `/projects/.../locations/...`,
|
||||
so a version-less path would reach Google as
|
||||
`aiplatform.googleapis.com/projects/...` and 404.
|
||||
"""
|
||||
return {
|
||||
"CLAUDE_CODE_USE_VERTEX": "1",
|
||||
"CLAUDE_CODE_SKIP_VERTEX_AUTH": "1",
|
||||
"ANTHROPIC_VERTEX_BASE_URL": f"{proxy_base_url}/vertex_ai/v1",
|
||||
"ANTHROPIC_VERTEX_PROJECT_ID": VERTEX_PLACEHOLDER_PROJECT,
|
||||
"CLOUD_ML_REGION": VERTEX_PLACEHOLDER_REGION,
|
||||
}
|
||||
|
||||
|
||||
def foundry_extra_env(proxy_base_url: str) -> Dict[str, str]:
|
||||
return {
|
||||
"CLAUDE_CODE_USE_FOUNDRY": "1",
|
||||
"CLAUDE_CODE_SKIP_FOUNDRY_AUTH": "1",
|
||||
"ANTHROPIC_FOUNDRY_BASE_URL": f"{proxy_base_url}/azure",
|
||||
}
|
||||
|
||||
|
||||
def run_passthrough_cell(
|
||||
*,
|
||||
compat_result,
|
||||
models: Sequence[str],
|
||||
prompt: str,
|
||||
passthrough_base_path: str = "",
|
||||
build_extra_env: Optional[Callable[[str], Mapping[str, str]]] = None,
|
||||
run_models: Callable[..., Mapping[str, Any]] = run_claude_models_parallel,
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
) -> None:
|
||||
"""Run the shared `passthrough` × <provider> cell body.
|
||||
|
||||
`passthrough_base_path` is appended to the proxy base URL and
|
||||
becomes the CLI's ANTHROPIC_BASE_URL (only the anthropic column
|
||||
uses it; the cloud columns ignore ANTHROPIC_BASE_URL entirely once
|
||||
their CLAUDE_CODE_USE_* flag is set). `build_extra_env` receives
|
||||
the trailing-slash-normalized proxy base URL and returns the
|
||||
provider-mode env for the CLI subprocess.
|
||||
"""
|
||||
environ = env if env is not None else os.environ
|
||||
base_url = environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured",
|
||||
pytrace=False,
|
||||
)
|
||||
|
||||
proxy_base = base_url.rstrip("/")
|
||||
extra_env = dict(build_extra_env(proxy_base)) if build_extra_env else None
|
||||
|
||||
outcomes = run_models(
|
||||
models=models,
|
||||
prompt=prompt,
|
||||
base_url=proxy_base + passthrough_base_path,
|
||||
api_key=api_key,
|
||||
extra_env=extra_env,
|
||||
)
|
||||
|
||||
failures = []
|
||||
for model in models:
|
||||
outcome = outcomes[model]
|
||||
if isinstance(outcome, ClaudeCLIError):
|
||||
error = f"[{model}] {outcome}"
|
||||
compat_result.add({"status": "fail", "error": error})
|
||||
failures.append(error)
|
||||
continue
|
||||
|
||||
if outcome.exit_code != 0:
|
||||
error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
|
||||
compat_result.add({"status": "fail", "error": error})
|
||||
failures.append(error)
|
||||
continue
|
||||
|
||||
if not outcome.text.strip():
|
||||
error = f"[{model}] claude returned empty assistant text"
|
||||
compat_result.add({"status": "fail", "error": error})
|
||||
failures.append(error)
|
||||
continue
|
||||
|
||||
compat_result.add({"status": "pass"})
|
||||
|
||||
if failures:
|
||||
pytest.fail("; ".join(failures), pytrace=False)
|
||||
|
|
@ -27,6 +27,15 @@ VERTEXAI_LOCATION=global
|
|||
AZURE_FOUNDRY_API_KEY=
|
||||
AZURE_FOUNDRY_API_BASE=
|
||||
|
||||
# Azure cell of the `passthrough` row. Foundry-mode Claude Code sends
|
||||
# the model in the request body, so the proxy's /azure passthrough
|
||||
# cannot resolve a router alias and falls back to these env vars.
|
||||
# AZURE_API_BASE is the Foundry resource's Anthropic surface, i.e.
|
||||
# https://<resource>.services.ai.azure.com/anthropic ; AZURE_API_KEY
|
||||
# is the same key as AZURE_FOUNDRY_API_KEY.
|
||||
AZURE_API_BASE=
|
||||
AZURE_API_KEY=
|
||||
|
||||
# REQUIRED for publishing: PAT for the `agent-shin` user, used to push
|
||||
# the daily compat-matrix branch to its fork (agent-shin/litellm-docs)
|
||||
# and open the cross-repo PR against BerriAI/litellm-docs. Scopes:
|
||||
|
|
|
|||
|
|
@ -91,6 +91,23 @@ features:
|
|||
# Code releases. The HTTP probe hits the bug surface LiteLLM
|
||||
# has actually shipped fixes for (2.1.117, 2.1.72, 2.1.70 per
|
||||
# the Claude Code release notes).
|
||||
- id: passthrough
|
||||
name: Native API passthrough
|
||||
# Drives the CLI in each cloud's native mode against LiteLLM's
|
||||
# passthrough routes instead of the /v1/messages translation
|
||||
# layer -- the "LLM gateway" setup from
|
||||
# https://code.claude.com/docs/en/gateway. anthropic uses
|
||||
# ANTHROPIC_BASE_URL={proxy}/anthropic; bedrock_invoke uses
|
||||
# CLAUDE_CODE_USE_BEDROCK=1 against {proxy}/bedrock (InvokeModel
|
||||
# wire, alias resolved from the URL by the router); vertex_ai
|
||||
# uses CLAUDE_CODE_USE_VERTEX=1 against {proxy}/vertex_ai/v1
|
||||
# (rawPredict wire, alias + project + location resolved from the
|
||||
# deployment, which therefore needs `use_in_pass_through: true`);
|
||||
# azure uses CLAUDE_CODE_USE_FOUNDRY=1 against {proxy}/azure and
|
||||
# needs AZURE_API_BASE/AZURE_API_KEY on the proxy (see
|
||||
# passthrough/test_azure.py and the cron env example).
|
||||
# bedrock_converse is structurally not_applicable: Claude Code
|
||||
# has no Converse-wire client.
|
||||
- id: long_context_1m
|
||||
name: Long context (1M)
|
||||
# Sends a ~210k-token padded prompt with the
|
||||
|
|
|
|||
0
tests/e2e/claude_code/passthrough/__init__.py
Normal file
0
tests/e2e/claude_code/passthrough/__init__.py
Normal file
44
tests/e2e/claude_code/passthrough/test_anthropic.py
Normal file
44
tests/e2e/claude_code/passthrough/test_anthropic.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
"""passthrough x Anthropic.
|
||||
|
||||
Drive the real `claude` CLI in its default first-party mode, but with
|
||||
ANTHROPIC_BASE_URL aimed at the proxy's `/anthropic` passthrough route
|
||||
instead of the `/v1/messages` translation endpoint. The proxy forwards
|
||||
the request verbatim to api.anthropic.com, swapping the virtual-key
|
||||
bearer for its own ANTHROPIC_API_KEY.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
tests/e2e/claude_code/passthrough/test_anthropic.py
|
||||
^^^^^^^^^^^ ^^^^^^^^^
|
||||
feature_id provider
|
||||
|
||||
Because nothing is translated, the model ids are the real Anthropic API
|
||||
ids (which happen to equal the proxy aliases for this column). A red
|
||||
cell here means the passthrough route broke forwarding itself -- auth
|
||||
header swap, streaming SSE relay, or beta-header propagation -- since
|
||||
no per-provider transformation is involved.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._passthrough import (
|
||||
ANTHROPIC_PASSTHROUGH_BASE_PATH,
|
||||
run_passthrough_cell,
|
||||
)
|
||||
|
||||
ANTHROPIC_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
]
|
||||
|
||||
|
||||
def test_passthrough_anthropic(compat_result):
|
||||
"""Drive the `claude` CLI through `{proxy}/anthropic` and assert a reply."""
|
||||
run_passthrough_cell(
|
||||
compat_result=compat_result,
|
||||
models=ANTHROPIC_MODELS,
|
||||
prompt="Reply with the single word 'pong' and nothing else.",
|
||||
passthrough_base_path=ANTHROPIC_PASSTHROUGH_BASE_PATH,
|
||||
)
|
||||
50
tests/e2e/claude_code/passthrough/test_azure.py
Normal file
50
tests/e2e/claude_code/passthrough/test_azure.py
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
"""passthrough x Azure (Microsoft Foundry).
|
||||
|
||||
Drive the real `claude` CLI in foundry mode (CLAUDE_CODE_USE_FOUNDRY=1)
|
||||
with ANTHROPIC_FOUNDRY_BASE_URL aimed at the proxy's `/azure`
|
||||
passthrough route. The CLI POSTs `/v1/messages` with the model in the
|
||||
JSON body -- unlike the bedrock/vertex modes there is no model segment
|
||||
in the URL, so the proxy's router-alias resolution cannot engage and
|
||||
the `/azure` route falls back to its env-configured target: the proxy
|
||||
must set AZURE_API_BASE to the Foundry resource's Anthropic surface
|
||||
(`https://<resource>.services.ai.azure.com/anthropic`) and
|
||||
AZURE_API_KEY to the Foundry key (see
|
||||
cron_vm/litellm-compat-matrix.env.example). The model ids are the
|
||||
Foundry deployment names, which this matrix provisions to match the
|
||||
Anthropic ids.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
tests/e2e/claude_code/passthrough/test_azure.py
|
||||
^^^^^^^^^^^ ^^^^^
|
||||
feature_id provider
|
||||
|
||||
Wiring verified live at authoring time: through `{proxy}/azure` the
|
||||
Foundry Anthropic surface accepted the `api-key` / `Authorization:
|
||||
Bearer` headers the fallback sends (a bogus key 401s, the real key
|
||||
proceeds to deployment lookup), so a red cell here means missing
|
||||
AZURE_API_BASE/AZURE_API_KEY on the proxy, missing Foundry deployments
|
||||
for the three tiers, or a genuine forwarding regression -- not an
|
||||
auth-scheme mismatch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._passthrough import foundry_extra_env, run_passthrough_cell
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
]
|
||||
|
||||
|
||||
def test_passthrough_azure(compat_result):
|
||||
"""Drive the `claude` CLI through `{proxy}/azure` and assert a reply."""
|
||||
run_passthrough_cell(
|
||||
compat_result=compat_result,
|
||||
models=AZURE_MODELS,
|
||||
prompt="Reply with the single word 'pong' and nothing else.",
|
||||
build_extra_env=foundry_extra_env,
|
||||
)
|
||||
34
tests/e2e/claude_code/passthrough/test_bedrock_converse.py
Normal file
34
tests/e2e/claude_code/passthrough/test_bedrock_converse.py
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
"""passthrough x Bedrock (Converse).
|
||||
|
||||
Structurally not applicable. In bedrock mode the `claude` CLI speaks
|
||||
only the InvokeModel wire (`/model/{id}/invoke-with-response-stream`);
|
||||
it has no Converse-wire client, so there is no Claude Code traffic a
|
||||
Converse passthrough could serve. LiteLLM's `/bedrock` route does
|
||||
accept `/model/{id}/converse-stream`, but exercising it would test a
|
||||
wire no Claude Code user can produce, which is out of scope for this
|
||||
matrix.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
tests/e2e/claude_code/passthrough/test_bedrock_converse.py
|
||||
^^^^^^^^^^^ ^^^^^^^^^^^^^^^^
|
||||
feature_id provider
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def test_passthrough_bedrock_converse(compat_result):
|
||||
"""Report not_applicable: Claude Code has no Converse-wire mode."""
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "not_applicable",
|
||||
"reason": (
|
||||
"Claude Code's bedrock mode speaks only the InvokeModel wire "
|
||||
"(/model/{id}/invoke-with-response-stream); it has no "
|
||||
"Converse-wire client, so there is no Claude Code surface "
|
||||
"for Converse passthrough."
|
||||
),
|
||||
}
|
||||
)
|
||||
42
tests/e2e/claude_code/passthrough/test_bedrock_invoke.py
Normal file
42
tests/e2e/claude_code/passthrough/test_bedrock_invoke.py
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
"""passthrough x Bedrock (Invoke).
|
||||
|
||||
Drive the real `claude` CLI in bedrock mode (CLAUDE_CODE_USE_BEDROCK=1)
|
||||
with ANTHROPIC_BEDROCK_BASE_URL aimed at the proxy's `/bedrock`
|
||||
passthrough route. The CLI speaks the native InvokeModel wire --
|
||||
`POST /model/{model}/invoke-with-response-stream` -- with the proxy
|
||||
alias in the model segment; the proxy resolves the alias through its
|
||||
router, rewrites the path to the deployment's upstream model id, and
|
||||
SigV4-signs the forwarded request with its own AWS credentials
|
||||
(CLAUDE_CODE_SKIP_BEDROCK_AUTH=1 keeps the CLI from signing).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
tests/e2e/claude_code/passthrough/test_bedrock_invoke.py
|
||||
^^^^^^^^^^^ ^^^^^^^^^^^^^^
|
||||
feature_id provider
|
||||
|
||||
The CLI also fires a best-effort `GET /bedrock/inference-profiles`
|
||||
listing at startup; its failure is non-fatal and does not gate this
|
||||
cell.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._passthrough import bedrock_extra_env, run_passthrough_cell
|
||||
|
||||
BEDROCK_INVOKE_MODELS = [
|
||||
"claude-haiku-4-5-bedrock-invoke",
|
||||
"claude-sonnet-4-6-bedrock-invoke",
|
||||
"claude-opus-4-7-bedrock-invoke",
|
||||
]
|
||||
|
||||
|
||||
def test_passthrough_bedrock_invoke(compat_result):
|
||||
"""Drive the `claude` CLI through `{proxy}/bedrock` and assert a reply."""
|
||||
run_passthrough_cell(
|
||||
compat_result=compat_result,
|
||||
models=BEDROCK_INVOKE_MODELS,
|
||||
prompt="Reply with the single word 'pong' and nothing else.",
|
||||
build_extra_env=bedrock_extra_env,
|
||||
)
|
||||
45
tests/e2e/claude_code/passthrough/test_vertex_ai.py
Normal file
45
tests/e2e/claude_code/passthrough/test_vertex_ai.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
"""passthrough x Vertex AI.
|
||||
|
||||
Drive the real `claude` CLI in vertex mode (CLAUDE_CODE_USE_VERTEX=1)
|
||||
with ANTHROPIC_VERTEX_BASE_URL aimed at the proxy's `/vertex_ai`
|
||||
passthrough route. The CLI speaks the native rawPredict wire --
|
||||
`POST .../projects/{p}/locations/{l}/publishers/anthropic/models/{model}:streamRawPredict`
|
||||
-- with the proxy alias in the model segment; the proxy resolves the
|
||||
alias through its router, replaces the placeholder project/location
|
||||
path segments with the deployment's `vertex_project` /
|
||||
`vertex_location`, and attaches its own Google credentials
|
||||
(CLAUDE_CODE_SKIP_VERTEX_AUTH=1 keeps the CLI from minting a token).
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/e2e/claude_code/conftest.py`:
|
||||
|
||||
tests/e2e/claude_code/passthrough/test_vertex_ai.py
|
||||
^^^^^^^^^^^ ^^^^^^^^^
|
||||
feature_id provider
|
||||
|
||||
This cell requires the vertex deployments in the proxy config to carry
|
||||
`use_in_pass_through: true` (see test_config.yaml) -- that is what
|
||||
registers their credentials with the passthrough router. Without it
|
||||
the proxy forwards the CLI's own headers (the virtual-key bearer) to
|
||||
Google and every tier fails with a 401.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from claude_code._passthrough import run_passthrough_cell, vertex_extra_env
|
||||
|
||||
VERTEX_MODELS = [
|
||||
"claude-haiku-4-5-vertex",
|
||||
"claude-sonnet-4-6-vertex",
|
||||
"claude-opus-4-7-vertex",
|
||||
]
|
||||
|
||||
|
||||
def test_passthrough_vertex_ai(compat_result):
|
||||
"""Drive the `claude` CLI through `{proxy}/vertex_ai` and assert a reply."""
|
||||
run_passthrough_cell(
|
||||
compat_result=compat_result,
|
||||
models=VERTEX_MODELS,
|
||||
prompt="Reply with the single word 'pong' and nothing else.",
|
||||
build_extra_env=vertex_extra_env,
|
||||
)
|
||||
|
|
@ -59,21 +59,31 @@ model_list:
|
|||
aws_region_name: us-east-1
|
||||
|
||||
# ---- Vertex AI ----
|
||||
# `use_in_pass_through: true` registers each deployment's
|
||||
# project/location/credentials with the /vertex_ai passthrough
|
||||
# router, which the `passthrough` row needs to resolve
|
||||
# .../models/{alias}:streamRawPredict URLs. That registration only
|
||||
# reads the canonical `vertex_project`/`vertex_location` param names
|
||||
# (not the `vertex_ai_*` aliases); the chat translation path accepts
|
||||
# both.
|
||||
- model_name: claude-haiku-4-5-vertex
|
||||
litellm_params:
|
||||
model: vertex_ai/claude-haiku-4-5
|
||||
vertex_ai_project: os.environ/VERTEXAI_PROJECT
|
||||
vertex_ai_location: os.environ/VERTEXAI_LOCATION
|
||||
vertex_project: os.environ/VERTEXAI_PROJECT
|
||||
vertex_location: os.environ/VERTEXAI_LOCATION
|
||||
use_in_pass_through: true
|
||||
- model_name: claude-sonnet-4-6-vertex
|
||||
litellm_params:
|
||||
model: vertex_ai/claude-sonnet-4-6
|
||||
vertex_ai_project: os.environ/VERTEXAI_PROJECT
|
||||
vertex_ai_location: os.environ/VERTEXAI_LOCATION
|
||||
vertex_project: os.environ/VERTEXAI_PROJECT
|
||||
vertex_location: os.environ/VERTEXAI_LOCATION
|
||||
use_in_pass_through: true
|
||||
- model_name: claude-opus-4-7-vertex
|
||||
litellm_params:
|
||||
model: vertex_ai/claude-opus-4-7
|
||||
vertex_ai_project: os.environ/VERTEXAI_PROJECT
|
||||
vertex_ai_location: os.environ/VERTEXAI_LOCATION
|
||||
vertex_project: os.environ/VERTEXAI_PROJECT
|
||||
vertex_location: os.environ/VERTEXAI_LOCATION
|
||||
use_in_pass_through: true
|
||||
|
||||
# ---- Microsoft Foundry (Anthropic deployments on Azure) ----
|
||||
- model_name: claude-haiku-4-5-azure
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue