scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-04 02:31:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 70

refactor(e2e): type remaining claude_code suite files and drop pyright exclusion

Browse source
This commit is contained in:
mateo-berri 2026-07-15 17:06:11 -07:00
parent 949aa53096
commit cfa7f54ec2
84 changed files with 919 additions and 601 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

138
tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py
View file

@ -15,6 +15,7 @@ from pathlib import Path
import pytest
from claude_code.json_types import JSON_OBJECT_ADAPTER, JSONValue
from claude_code.matrix_builder import (
ManifestError,
ResultsError,
@ -27,7 +28,22 @@ from claude_code.matrix_builder import (
FIXTURES = Path(__file__).parent / "fixtures"
def test_build_matrix_matches_golden_file(tmp_path):
def _as_dict(value: JSONValue) -> dict[str, JSONValue]:
assert isinstance(value, dict)
return value
def _as_list(value: JSONValue) -> list[JSONValue]:
assert isinstance(value, list)
return value
def _as_str(value: JSONValue) -> str:
assert isinstance(value, str)
return value
def test_build_matrix_matches_golden_file(tmp_path: Path) -> None:
manifest = load_manifest(FIXTURES / "manifest.yaml")
results = load_results(FIXTURES / "results.json")
matrix = build_matrix(
@ -37,18 +53,18 @@ def test_build_matrix_matches_golden_file(tmp_path):
claude_code_version="2.1.120",
generated_at="2026-04-25T00:00:00Z",
)
expected = json.loads((FIXTURES / "expected_matrix.json").read_text())
expected = JSON_OBJECT_ADAPTER.validate_python(json.loads((FIXTURES / "expected_matrix.json").read_text()))
assert matrix == expected
def test_build_matrix_pass_requires_all_models_pass():
def test_build_matrix_pass_requires_all_models_pass() -> None:
"""Multiple results in one cell must all be pass for the cell to be pass."""
manifest = {
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
@ -60,16 +76,16 @@ def test_build_matrix_pass_requires_all_models_pass():
claude_code_version="c",
generated_at="t",
)
assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"}
assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "pass"}
def test_build_matrix_any_fail_makes_cell_fail():
manifest = {
def test_build_matrix_any_fail_makes_cell_fail() -> None:
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
{
"feature_id": "f",
@ -85,22 +101,22 @@ def test_build_matrix_any_fail_makes_cell_fail():
claude_code_version="c",
generated_at="t",
)
cell = matrix["features"][0]["providers"]["anthropic"]
cell = _as_dict(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"])
assert cell["status"] == "fail"
assert cell["error"] == "[claude-opus-4-7] timeout"
def test_build_matrix_joins_all_failure_errors_in_one_cell():
def test_build_matrix_joins_all_failure_errors_in_one_cell() -> None:
"""When multiple tiers fail for different reasons within the same cell,
every failure's error must appear in the published cell so triage
isn't reduced to a single tier's diagnostic.
"""
manifest = {
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{
"feature_id": "f",
"provider": "anthropic",
@ -120,25 +136,25 @@ def test_build_matrix_joins_all_failure_errors_in_one_cell():
claude_code_version="c",
generated_at="t",
)
cell = matrix["features"][0]["providers"]["anthropic"]
cell = _as_dict(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"])
assert cell["status"] == "fail"
assert "[claude-haiku-4-5] 429" in cell["error"]
assert "[claude-opus-4-7] timeout" in cell["error"]
assert "[claude-haiku-4-5] 429" in _as_str(cell["error"])
assert "[claude-opus-4-7] timeout" in _as_str(cell["error"])
def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass():
def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass() -> None:
"""A `not_tested` row mixed with `pass` rows must not silently demote
the cell to `not_tested` — `not_tested` is "absent data", not a
negative signal. Otherwise a partial crash mid-test, or a test that
explicitly recorded "tier didn't run", would discard real passing
results from the published cell.
"""
manifest = {
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
{
"feature_id": "f",
@ -154,20 +170,20 @@ def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass():
claude_code_version="c",
generated_at="t",
)
assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"}
assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "pass"}
def test_build_matrix_all_not_tested_stays_not_tested():
def test_build_matrix_all_not_tested_stays_not_tested() -> None:
"""A cell whose every row is `not_tested` (or empty) must remain
`not_tested` — the absent-data rule only drops `not_tested` rows
when there's other signal to surface.
"""
manifest = {
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{
"feature_id": "f",
"provider": "anthropic",
@ -186,22 +202,22 @@ def test_build_matrix_all_not_tested_stays_not_tested():
claude_code_version="c",
generated_at="t",
)
assert matrix["features"][0]["providers"]["anthropic"] == {"status": "not_tested"}
assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "not_tested"}
def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass():
def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass() -> None:
"""A `not_applicable` row mixed with `pass` rows must surface as
`pass`, not `not_applicable`. The published cell answers "does this
feature work on this provider?"; if any tier passes, the feature
works there. Discarding passing tiers because one tier is NA would
misrepresent the cell as unsupported.
"""
manifest = {
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
{
"feature_id": "f",
@ -220,20 +236,20 @@ def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass():
claude_code_version="c",
generated_at="t",
)
assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"}
assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "pass"}
def test_build_matrix_all_not_applicable_stays_not_applicable():
def test_build_matrix_all_not_applicable_stays_not_applicable() -> None:
"""When every observed row is `not_applicable`, the cell remains
`not_applicable` and the first row's reason carries through to the
published matrix.
"""
manifest = {
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{
"feature_id": "f",
"provider": "anthropic",
@ -255,19 +271,19 @@ def test_build_matrix_all_not_applicable_stays_not_applicable():
claude_code_version="c",
generated_at="t",
)
assert matrix["features"][0]["providers"]["anthropic"] == {
assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {
"status": "not_applicable",
"reason": "feature unsupported on this provider",
}
def test_build_matrix_fills_not_tested_for_missing_cells():
manifest = {
def test_build_matrix_fills_not_tested_for_missing_cells() -> None:
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic", "azure"],
"features": [{"id": "f", "name": "F"}],
}
results = [
results: list[JSONValue] = [
{"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}},
]
matrix = build_matrix(
@ -277,13 +293,13 @@ def test_build_matrix_fills_not_tested_for_missing_cells():
claude_code_version="c",
generated_at="t",
)
cells = matrix["features"][0]["providers"]
cells = _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])
assert cells["anthropic"] == {"status": "pass"}
assert cells["azure"] == {"status": "not_tested"}
def test_build_matrix_preserves_provider_and_feature_order():
manifest = {
def test_build_matrix_preserves_provider_and_feature_order() -> None:
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["azure", "anthropic", "vertex_ai"],
"features": [
@ -299,16 +315,16 @@ def test_build_matrix_preserves_provider_and_feature_order():
generated_at="t",
)
assert matrix["providers"] == ["azure", "anthropic", "vertex_ai"]
assert [f["id"] for f in matrix["features"]] == ["z", "a"]
assert list(matrix["features"][0]["providers"].keys()) == [
assert [_as_dict(f)["id"] for f in _as_list(matrix["features"])] == ["z", "a"]
assert list(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"]).keys()) == [
"azure",
"anthropic",
"vertex_ai",
]
def test_build_matrix_emits_schema_version_one():
manifest = {
def test_build_matrix_emits_schema_version_one() -> None:
manifest: dict[str, JSONValue] = {
"schema_version": "1",
"providers": ["anthropic"],
"features": [{"id": "f", "name": "F"}],
@ -323,7 +339,7 @@ def test_build_matrix_emits_schema_version_one():
assert matrix["schema_version"] == "1"
def test_load_manifest_rejects_wrong_schema_version(tmp_path):
def test_load_manifest_rejects_wrong_schema_version(tmp_path: Path) -> None:
bad = tmp_path / "manifest.yaml"
bad.write_text(
'schema_version: "2"\nproviders: [anthropic]\nfeatures:\n - id: f\n name: F\n'
@ -332,21 +348,21 @@ def test_load_manifest_rejects_wrong_schema_version(tmp_path):
load_manifest(bad)
def test_load_manifest_rejects_empty_features(tmp_path):
def test_load_manifest_rejects_empty_features(tmp_path: Path) -> None:
bad = tmp_path / "manifest.yaml"
bad.write_text('schema_version: "1"\nproviders: [anthropic]\nfeatures: []\n')
with pytest.raises(ManifestError):
load_manifest(bad)
def test_load_results_rejects_missing_results_key(tmp_path):
def test_load_results_rejects_missing_results_key(tmp_path: Path) -> None:
bad = tmp_path / "results.json"
bad.write_text(json.dumps({"schema_version": "1"}))
with pytest.raises(ResultsError):
load_results(bad)
def test_build_matrix_6x5_grid_matches_published_sample():
def test_build_matrix_6x5_grid_matches_published_sample() -> None:
"""Slice 5 acceptance: feeding the per-model results the full v0
row set produces reproduces the hand-authored 6x5 sample that the
docs page renders.
@ -386,16 +402,16 @@ def test_build_matrix_6x5_grid_matches_published_sample():
]
v0_features = [
feature
for feature in full_manifest["features"]
if feature["id"] in v0_feature_ids
for feature in _as_list(full_manifest["features"])
if _as_dict(feature)["id"] in v0_feature_ids
]
manifest = {**full_manifest, "features": v0_features}
manifest: dict[str, JSONValue] = {**full_manifest, "features": v0_features}
feature_ids = [feature["id"] for feature in manifest["features"]]
providers = manifest["providers"]
feature_ids = [_as_str(_as_dict(feature)["id"]) for feature in _as_list(manifest["features"])]
providers = [_as_str(provider) for provider in _as_list(manifest["providers"])]
models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"]
results = []
results: list[JSONValue] = []
for feature_id in feature_ids:
for provider in providers:
for model in models:
@ -418,18 +434,20 @@ def test_build_matrix_6x5_grid_matches_published_sample():
claude_code_version="2.1.120",
generated_at="2026-04-25T00:00:00Z",
)
expected = json.loads((repo_root / "sample_compatibility-matrix.json").read_text())
expected = JSON_OBJECT_ADAPTER.validate_python(
json.loads((repo_root / "sample_compatibility-matrix.json").read_text())
)
assert matrix == expected
def test_build_matrix_1x5_grid_one_failing_model_breaks_cell():
def test_build_matrix_1x5_grid_one_failing_model_breaks_cell() -> None:
"""If even one of three models fails on a provider, that cell is fail
and the error string carries the failing model id so the docs
tooltip can name the outlier."""
repo_root = Path(__file__).resolve().parents[1]
manifest = load_manifest(repo_root / "manifest.yaml")
results = [
results: list[JSONValue] = [
{
"feature_id": "basic_messaging_non_streaming",
"provider": "bedrock_invoke",
@ -457,12 +475,12 @@ def test_build_matrix_1x5_grid_one_failing_model_breaks_cell():
claude_code_version="c",
generated_at="t",
)
cell = matrix["features"][0]["providers"]["bedrock_invoke"]
cell = _as_dict(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["bedrock_invoke"])
assert cell["status"] == "fail"
assert "claude-opus-4-7-bedrock-invoke" in cell["error"]
assert "claude-opus-4-7-bedrock-invoke" in _as_str(cell["error"])
def test_build_from_paths_writes_output(tmp_path):
def test_build_from_paths_writes_output(tmp_path: Path) -> None:
out = tmp_path / "compatibility-matrix.json"
matrix = build_from_paths(
manifest_path=FIXTURES / "manifest.yaml",
@ -473,7 +491,7 @@ def test_build_from_paths_writes_output(tmp_path):
output_path=out,
)
assert out.exists()
on_disk = json.loads(out.read_text())
on_disk = JSON_OBJECT_ADAPTER.validate_python(json.loads(out.read_text()))
assert on_disk == matrix
expected = json.loads((FIXTURES / "expected_matrix.json").read_text())
expected = JSON_OBJECT_ADAPTER.validate_python(json.loads((FIXTURES / "expected_matrix.json").read_text()))
assert on_disk == expected

58
tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py
View file

@ -16,9 +16,26 @@ from pathlib import Path
import pytest
import yaml
from claude_code.json_types import JSON_OBJECT_ADAPTER, JSONValue
REPO_ROOT = Path(__file__).resolve().parents[1]
MANIFEST_PATH = REPO_ROOT / "manifest.yaml"
def _as_dict(value: JSONValue) -> dict[str, JSONValue]:
assert isinstance(value, dict)
return value
def _as_list(value: JSONValue) -> list[JSONValue]:
assert isinstance(value, list)
return value
def _as_str(value: JSONValue) -> str:
assert isinstance(value, str)
return value
# The PRD's "Features in v0" section, in row order.
EXPECTED_FEATURE_IDS = [
"basic_messaging_non_streaming",
@ -55,8 +72,10 @@ def _all_manifest_feature_ids() -> list[str]:
constants above only validate the original six rows by design.
"""
return [
feature["id"]
for feature in yaml.safe_load(MANIFEST_PATH.read_text())["features"]
_as_str(_as_dict(feature)["id"])
for feature in _as_list(
JSON_OBJECT_ADAPTER.validate_python(yaml.safe_load(MANIFEST_PATH.read_text()))["features"]
)
]
@ -64,45 +83,46 @@ ALL_FEATURE_IDS = _all_manifest_feature_ids()
@pytest.fixture(scope="module")
def manifest() -> dict:
return yaml.safe_load(MANIFEST_PATH.read_text())
def manifest() -> dict[str, JSONValue]:
return JSON_OBJECT_ADAPTER.validate_python(yaml.safe_load(MANIFEST_PATH.read_text()))
def test_manifest_lists_all_six_v0_features_in_order(manifest):
def test_manifest_lists_all_six_v0_features_in_order(manifest: dict[str, JSONValue]) -> None:
"""The PRD's v0 row set must appear at the top of the manifest in
order. Features beyond v0 (extensions added after the matrix
shipped) are allowed but must not reorder or displace the v0
rows — the docs page anchors row links by index, so v0 stays
pinned at positions [0:6] for the lifetime of the schema.
"""
ids = [feature["id"] for feature in manifest["features"]]
ids = [_as_dict(feature)["id"] for feature in _as_list(manifest["features"])]
assert ids[: len(EXPECTED_FEATURE_IDS)] == EXPECTED_FEATURE_IDS
def test_manifest_lists_all_five_v0_providers_in_order(manifest):
def test_manifest_lists_all_five_v0_providers_in_order(manifest: dict[str, JSONValue]) -> None:
assert manifest["providers"] == EXPECTED_PROVIDERS
def test_manifest_every_feature_has_human_readable_name(manifest):
for feature in manifest["features"]:
assert isinstance(feature["name"], str) and feature["name"].strip()
def test_manifest_every_feature_has_human_readable_name(manifest: dict[str, JSONValue]) -> None:
for feature in _as_list(manifest["features"]):
name = _as_dict(feature)["name"]
assert isinstance(name, str) and name.strip()
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
def test_feature_directory_exists(feature_id):
def test_feature_directory_exists(feature_id: str) -> None:
feature_dir = REPO_ROOT / feature_id
assert feature_dir.is_dir(), f"missing feature directory: {feature_dir}"
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS)
def test_per_provider_test_file_exists(feature_id, provider):
def test_per_provider_test_file_exists(feature_id: str, provider: str) -> None:
test_file = REPO_ROOT / feature_id / f"test_{provider}.py"
assert test_file.is_file(), f"missing per-provider test file: {test_file}"
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
def test_feature_directory_has_init_file(feature_id):
def test_feature_directory_has_init_file(feature_id: str) -> None:
"""Each feature directory needs an __init__.py so pytest collects
the per-provider test files as a package — matches the layout
established by `basic_messaging_non_streaming/`."""
@ -116,7 +136,7 @@ def test_feature_directory_has_init_file(feature_id):
# extend the same structural guarantees to any row added afterward so
# a broken post-v0 directory still fails CI.
@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS)
def test_every_manifest_feature_has_directory(feature_id):
def test_every_manifest_feature_has_directory(feature_id: str) -> None:
feature_dir = REPO_ROOT / feature_id
assert feature_dir.is_dir(), (
f"manifest declares {feature_id!r} but {feature_dir} is missing — "
@ -125,14 +145,14 @@ def test_every_manifest_feature_has_directory(feature_id):
@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS)
def test_every_manifest_feature_has_init_file(feature_id):
def test_every_manifest_feature_has_init_file(feature_id: str) -> None:
init_file = REPO_ROOT / feature_id / "__init__.py"
assert init_file.is_file(), f"missing __init__.py: {init_file}"
@pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS)
@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS)
def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider):
def test_every_manifest_feature_has_per_provider_test_file(feature_id: str, provider: str) -> None:
"""Every (feature, provider) cell in the rendered matrix must be
backed by a per-provider test file. Without this check, a missing
file silently becomes a `not_tested` cell in the published matrix
@ -144,8 +164,8 @@ def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider)
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS)
def test_per_provider_test_file_imports_and_parametrizes_three_models(
feature_id, provider
):
feature_id: str, provider: str
) -> None:
"""Every test file must reference the three Claude tiers required
by the PRD: Haiku 4.5, Sonnet 4.6, Opus 4.7. Implementations may
use plain aliases or per-provider-suffixed aliases (e.g.
@ -159,7 +179,7 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models(
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
def test_azure_test_file_drives_the_proxy(feature_id):
def test_azure_test_file_drives_the_proxy(feature_id: str) -> None:
"""Azure (Microsoft Foundry) hosts Anthropic Claude as of 2025-11-18,
so every Azure cell in the v0 matrix exercises a real route through
the LiteLLM proxy — same shape as the other provider columns. Pin

7
tests/e2e/claude_code/_driver_unit_tests/conftest.py
View file

@ -14,6 +14,9 @@ limiter was added.
from __future__ import annotations
from pathlib import Path
from typing import Generator
import pytest
from claude_code.rate_limiter import (
@ -25,7 +28,9 @@ from claude_code.rate_limiter import (
@pytest.fixture(autouse=True)
def _hermetic_rate_limiter(tmp_path):
def _hermetic_rate_limiter( # pyright: ignore[reportUnusedFunction] # autouse fixture, invoked by pytest
tmp_path: Path,
) -> Generator[None]:
config = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS}
limiter = RateLimiter(config=config, state_dir=tmp_path)
with use_limiter(limiter):

79
tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py
View file

@ -9,21 +9,22 @@ upstream stream must turn the cell red, not green.
from __future__ import annotations
import os
from typing import Any, Dict, List, Mapping, Optional, Sequence
from typing import Mapping, Sequence, TypedDict
import pytest
from claude_code import _basic_messaging
from claude_code._basic_messaging import (
MIN_STREAM_DELTA_EVENTS,
_count_stream_event_deltas,
_count_stream_event_deltas, # pyright: ignore[reportPrivateUsage] # unit tests exercise the private helper
run_basic_messaging_cell,
)
from claude_code.cli_driver import DriverResult
from claude_code.cli_driver import DriverResult, ModelResult
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONObject, JSONValue
class _FakeResult:
class _FakeResult(CompatResult):
"""Stand-in for the test's `compat_result` fixture.
Records every `set` / `add` payload so assertions can inspect what
@ -32,24 +33,25 @@ class _FakeResult:
"""
def __init__(self) -> None:
self.rows: List[Dict[str, Any]] = []
self.single: Optional[Dict[str, Any]] = None
super().__init__()
self.rows: list[dict[str, JSONValue]] = []
self.single: dict[str, JSONValue] | None = None
def set(self, payload: Mapping[str, Any]) -> None:
self.single = dict(payload)
def set(self, result: Mapping[str, JSONValue]) -> None:
self.single = dict(result)
def add(self, payload: Mapping[str, Any]) -> None:
self.rows.append(dict(payload))
def add(self, result: Mapping[str, JSONValue]) -> None:
self.rows.append(dict(result))
def _streamed_events(n_deltas: int = 5) -> List[Dict[str, Any]]:
def _streamed_events(n_deltas: int = 5) -> list[JSONObject]:
"""Build a stream-json event list that *looks* streamed.
Includes `n_deltas` `stream_event` records (matching what
`--include-partial-messages` produces) plus the usual
`system`/`assistant`/`result` boilerplate the CLI always emits.
"""
events: List[Dict[str, Any]] = [{"type": "system", "subtype": "init"}]
events: list[JSONObject] = [{"type": "system", "subtype": "init"}]
for i in range(n_deltas):
events.append(
{
@ -70,7 +72,7 @@ def _streamed_events(n_deltas: int = 5) -> List[Dict[str, Any]]:
return events
def _buffered_events() -> List[Dict[str, Any]]:
def _buffered_events() -> list[JSONObject]:
"""Event list a buffering proxy would produce: zero `stream_event`s."""
return [
{"type": "system", "subtype": "init"},
@ -82,16 +84,40 @@ def _buffered_events() -> List[Dict[str, Any]]:
]
def _install_fake_runner(monkeypatch, *, outcomes_by_model):
class _CapturedCall(TypedDict):
models: list[str]
prompt: str | None
base_url: str
api_key: str
extra_args: list[str]
def _install_fake_runner(
monkeypatch: pytest.MonkeyPatch, *, outcomes_by_model: Mapping[str, ModelResult]
) -> _CapturedCall:
"""Patch `run_claude_models_parallel` to return canned outcomes.
Captures the kwargs the cell passed in so tests can assert on
`extra_args` (which is how the streaming variant opts into
`--include-partial-messages`).
"""
captured: Dict[str, Any] = {}
captured: _CapturedCall = {
"models": [],
"prompt": None,
"base_url": "",
"api_key": "",
"extra_args": [],
}
def fake(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs):
def fake(
*,
models: Sequence[str],
prompt: str | None,
base_url: str,
api_key: str,
extra_args: Sequence[str] | None = None,
**_kwargs: object,
) -> dict[str, ModelResult]:
captured["models"] = list(models)
captured["prompt"] = prompt
captured["base_url"] = base_url
@ -104,13 +130,15 @@ def _install_fake_runner(monkeypatch, *, outcomes_by_model):
@pytest.fixture(autouse=True)
def _proxy_env(monkeypatch):
def _proxy_env( # pyright: ignore[reportUnusedFunction] # autouse fixture, invoked by pytest
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("LITELLM_PROXY_BASE_URL", "http://localhost:4000")
monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test")
def test_count_stream_event_deltas_only_counts_records_with_event_payload():
events = [
def test_count_stream_event_deltas_only_counts_records_with_event_payload() -> None:
events: list[JSONObject] = [
{"type": "system"},
{"type": "stream_event", "event": {"type": "message_start"}},
{"type": "stream_event", "event": {"type": "content_block_delta"}},
@ -123,7 +151,7 @@ def test_count_stream_event_deltas_only_counts_records_with_event_payload():
assert _count_stream_event_deltas(events) == 2
def test_verify_streaming_passes_when_proxy_streams(monkeypatch):
def test_verify_streaming_passes_when_proxy_streams(monkeypatch: pytest.MonkeyPatch) -> None:
fake_result = _FakeResult()
model = "claude-haiku-4-5"
outcome = DriverResult(text="1\n2\n3", events=_streamed_events(n_deltas=5))
@ -140,7 +168,7 @@ def test_verify_streaming_passes_when_proxy_streams(monkeypatch):
assert fake_result.rows == [{"status": "pass"}]
def test_verify_streaming_fails_when_proxy_buffers(monkeypatch):
def test_verify_streaming_fails_when_proxy_buffers(monkeypatch: pytest.MonkeyPatch) -> None:
fake_result = _FakeResult()
model = "claude-haiku-4-5"
outcome = DriverResult(text="1\n2\n3", events=_buffered_events())
@ -157,11 +185,12 @@ def test_verify_streaming_fails_when_proxy_buffers(monkeypatch):
assert len(fake_result.rows) == 1
row = fake_result.rows[0]
assert row["status"] == "fail"
assert isinstance(row["error"], str)
assert "stream_event" in row["error"]
assert f"< {MIN_STREAM_DELTA_EVENTS}" in row["error"]
def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch):
def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch: pytest.MonkeyPatch) -> None:
"""Default `verify_streaming=False` keeps the non-streaming wire identical."""
fake_result = _FakeResult()
model = "claude-haiku-4-5"
@ -178,11 +207,11 @@ def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch):
assert fake_result.rows == [{"status": "pass"}]
def test_verify_streaming_requires_all_models_to_stream(monkeypatch):
def test_verify_streaming_requires_all_models_to_stream(monkeypatch: pytest.MonkeyPatch) -> None:
"""If any one tier buffers, the cell fails — same all-must-pass shape as
the non-streaming check."""
fake_result = _FakeResult()
outcomes = {
outcomes: dict[str, ModelResult] = {
"claude-haiku-4-5": DriverResult(text="ok", events=_streamed_events(5)),
"claude-sonnet-4-6": DriverResult(text="ok", events=_buffered_events()),
"claude-opus-4-7": DriverResult(text="ok", events=_streamed_events(5)),

204
tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py
View file

@ -10,31 +10,47 @@ from __future__ import annotations
import json
import subprocess
from dataclasses import dataclass
from typing import List, Optional
from typing import Mapping, Sequence, TypedDict
import pytest
from claude_code.cli_driver import (
ClaudeCLIError,
CommandRunner,
DriverResult,
failure_diagnostic,
run_claude,
run_claude_models_parallel,
)
from claude_code.json_types import JSONObject
@dataclass
class _Completed:
returncode: int = 0
stdout: str = ""
stderr: str = ""
class _Completed(subprocess.CompletedProcess[str]):
def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None:
super().__init__(args=(), returncode=returncode, stdout=stdout, stderr=stderr)
def _make_runner(*, stdout: str = "", returncode: int = 0, stderr: str = ""):
captured = {}
class _Captured(TypedDict):
cmd: Sequence[str]
env: Mapping[str, str]
timeout: float
input: str | None
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def _make_runner(
*, stdout: str = "", returncode: int = 0, stderr: str = ""
) -> tuple[CommandRunner, _Captured]:
captured: _Captured = {"cmd": (), "env": {}, "timeout": 0.0, "input": None}
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
captured["cmd"] = cmd
captured["env"] = env
captured["timeout"] = timeout
@ -44,7 +60,7 @@ def _make_runner(*, stdout: str = "", returncode: int = 0, stderr: str = ""):
return runner, captured
def test_run_claude_assembles_command_correctly():
def test_run_claude_assembles_command_correctly() -> None:
runner, captured = _make_runner(
stdout='{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}}\n'
)
@ -66,7 +82,7 @@ def test_run_claude_assembles_command_correctly():
assert cmd[-2:] == ["--", "hello"]
def test_run_claude_places_extra_args_before_prompt():
def test_run_claude_places_extra_args_before_prompt() -> None:
"""`claude --print` expects the prompt as the final positional arg.
Flags appearing after the prompt are ignored or eaten by the prompt
@ -93,7 +109,7 @@ def test_run_claude_places_extra_args_before_prompt():
assert cmd.index("--allowed-tools") < cmd.index("--")
def test_run_claude_overlays_proxy_env():
def test_run_claude_overlays_proxy_env() -> None:
runner, captured = _make_runner(stdout="")
run_claude(
prompt="hi",
@ -107,7 +123,7 @@ def test_run_claude_overlays_proxy_env():
assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-abc"
def test_run_claude_extra_env_is_added_to_subprocess_env():
def test_run_claude_extra_env_is_added_to_subprocess_env() -> None:
"""Caller-supplied extra_env entries land on the subprocess env."""
runner, captured = _make_runner(stdout="")
run_claude(
@ -121,7 +137,7 @@ def test_run_claude_extra_env_is_added_to_subprocess_env():
assert captured["env"]["MAX_THINKING_TOKENS"] == "4096"
def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch):
def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch: pytest.MonkeyPatch) -> None:
"""Process-runtime vars (PATH) flow through; credentials don't.
The `claude` CLI is a Node binary installed dynamically from npm in
@ -160,7 +176,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch):
assert "GITHUB_TOKEN" not in env
def test_run_claude_uses_isolated_per_invocation_home(monkeypatch, tmp_path):
def test_run_claude_uses_isolated_per_invocation_home(monkeypatch: pytest.MonkeyPatch) -> None:
"""`claude` subprocess never sees the runtime user's real $HOME.
The CLI needs *a* HOME (it caches per-session state under
@ -196,7 +212,7 @@ def test_run_claude_uses_isolated_per_invocation_home(monkeypatch, tmp_path):
assert "claude-cli-home-" in env["HOME"]
def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch):
def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch: pytest.MonkeyPatch) -> None:
"""Two consecutive calls get two different isolated HOMEs.
Reusing a single tmpdir across calls would defeat the isolation
@ -229,7 +245,7 @@ def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch):
assert home_a != home_b
def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch):
def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch: pytest.MonkeyPatch) -> None:
"""The per-invocation HOME tmpdir is rm-rf'd when run_claude returns.
Without cleanup, a long matrix run would accumulate one tmpdir
@ -254,7 +270,9 @@ def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch):
), f"isolated HOME {isolated_home!r} should be removed after run_claude returns"
def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch):
def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Cleanup runs even when the CLI subprocess raises.
If the CLI is missing or times out, `run_claude` raises
@ -266,9 +284,17 @@ def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch):
monkeypatch.setenv("HOME", "/home/runner")
captured: dict = {}
captured: dict[str, Mapping[str, str]] = {}
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
captured["env"] = env
raise subprocess.TimeoutExpired(cmd=cmd, timeout=timeout)
@ -287,7 +313,9 @@ def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch):
), f"isolated HOME {isolated_home!r} should be removed even on timeout"
def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(monkeypatch):
def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The allowlist applies to inherited os.environ; extra_env is the
sanctioned way for a test to opt-in to passing something extra."""
monkeypatch.setenv("ANTHROPIC_API_KEY", "from-os")
@ -303,8 +331,8 @@ def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(monkeypatch
assert captured["env"]["ANTHROPIC_API_KEY"] == "from-arg"
def test_run_claude_parses_stream_json_assistant_text():
events = [
def test_run_claude_parses_stream_json_assistant_text() -> None:
events: list[JSONObject] = [
{"type": "system", "session_id": "abc"},
{
"type": "assistant",
@ -333,7 +361,7 @@ def test_run_claude_parses_stream_json_assistant_text():
assert result.exit_code == 0
def test_run_claude_handles_string_message_content():
def test_run_claude_handles_string_message_content() -> None:
"""Some CLI versions emit `message.content` as a plain string."""
stdout = (
json.dumps({"type": "assistant", "message": {"content": "bare text"}}) + "\n"
@ -349,7 +377,7 @@ def test_run_claude_handles_string_message_content():
assert result.text == "bare text"
def test_run_claude_skips_malformed_lines():
def test_run_claude_skips_malformed_lines() -> None:
stdout = (
"not-json\n"
+ json.dumps(
@ -373,7 +401,7 @@ def test_run_claude_skips_malformed_lines():
assert len(result.events) == 1
def test_run_claude_propagates_nonzero_exit_code():
def test_run_claude_propagates_nonzero_exit_code() -> None:
runner, _ = _make_runner(stdout="", returncode=2, stderr="auth failed")
result = run_claude(
prompt="hi",
@ -387,8 +415,8 @@ def test_run_claude_propagates_nonzero_exit_code():
assert result.text == ""
def test_run_claude_raises_on_missing_cli():
def runner(*args, **kwargs):
def test_run_claude_raises_on_missing_cli() -> None:
def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]:
raise FileNotFoundError(2, "no such file", "claude")
with pytest.raises(ClaudeCLIError, match="claude CLI not found"):
@ -401,8 +429,8 @@ def test_run_claude_raises_on_missing_cli():
)
def test_run_claude_raises_on_timeout():
def runner(*args, **kwargs):
def test_run_claude_raises_on_timeout() -> None:
def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]:
raise subprocess.TimeoutExpired(cmd="claude", timeout=1)
with pytest.raises(ClaudeCLIError, match="timed out"):
@ -416,7 +444,7 @@ def test_run_claude_raises_on_timeout():
)
def test_run_claude_validates_required_params():
def test_run_claude_validates_required_params() -> None:
runner, _ = _make_runner()
with pytest.raises(ValueError, match="prompt"):
run_claude(
@ -473,7 +501,7 @@ def test_run_claude_validates_required_params():
# ---------------------------------------------------------------------------
def test_failure_diagnostic_surfaces_api_error_text_from_stdout():
def test_failure_diagnostic_surfaces_api_error_text_from_stdout() -> None:
"""The CLI hides 4xx/5xx from the proxy in `assistant.message.content` text."""
api_error_text = (
'API Error: 400 {"error":{"message":"litellm.BadRequestError: '
@ -505,14 +533,14 @@ def test_failure_diagnostic_surfaces_api_error_text_from_stdout():
assert "There are no healthy deployments" in diag
def test_failure_diagnostic_falls_back_to_stderr_when_no_text():
def test_failure_diagnostic_falls_back_to_stderr_when_no_text() -> None:
result = DriverResult(text="", events=[], exit_code=2, stderr="boom\n")
diag = failure_diagnostic(result)
assert "exit=2" in diag
assert "stderr=boom" in diag
def test_failure_diagnostic_handles_completely_empty_result():
def test_failure_diagnostic_handles_completely_empty_result() -> None:
"""A run that produced literally nothing should still yield a useful string."""
result = DriverResult(text="", events=[], exit_code=137, stderr="")
diag = failure_diagnostic(result)
@ -520,7 +548,7 @@ def test_failure_diagnostic_handles_completely_empty_result():
assert "no diagnostic output" in diag
def test_failure_diagnostic_truncates_long_text():
def test_failure_diagnostic_truncates_long_text() -> None:
"""Don't let a 5MB HTML 502 page from a load balancer wreck the matrix JSON."""
huge = "x" * 5000
result = DriverResult(text=huge, events=[], exit_code=1, stderr="")
@ -530,7 +558,7 @@ def test_failure_diagnostic_truncates_long_text():
assert len(diag) < 300
def test_failure_diagnostic_ignores_non_int_api_error_status():
def test_failure_diagnostic_ignores_non_int_api_error_status() -> None:
"""The CLI sometimes emits api_error_status as a string; don't crash."""
result = DriverResult(
text="oops",
@ -554,11 +582,19 @@ def test_failure_diagnostic_ignores_non_int_api_error_status():
# ---------------------------------------------------------------------------
def test_run_claude_models_parallel_returns_one_result_per_model():
def test_run_claude_models_parallel_returns_one_result_per_model() -> None:
"""Each model gets its own DriverResult keyed under the helper's dict."""
seen_models: List[str] = []
seen_models: list[str] = []
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
# The model id is two slots after `--model` in the assembled command.
idx = cmd.index("--model")
model = cmd[idx + 1]
@ -593,10 +629,18 @@ def test_run_claude_models_parallel_returns_one_result_per_model():
assert sorted(seen_models) == ["a", "b", "c"]
def test_run_claude_models_parallel_returns_errors_as_values():
def test_run_claude_models_parallel_returns_errors_as_values() -> None:
"""A model whose CLI is missing surfaces as a ClaudeCLIError, not a raise."""
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
idx = cmd.index("--model")
model = cmd[idx + 1]
if model == "boom":
@ -626,10 +670,18 @@ def test_run_claude_models_parallel_returns_errors_as_values():
assert "claude CLI not found" in str(outcomes["boom"])
def test_run_claude_models_parallel_preserves_nonzero_exit_codes():
def test_run_claude_models_parallel_preserves_nonzero_exit_codes() -> None:
"""Mixed success/failure on exit code should not collapse into one verdict."""
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
idx = cmd.index("--model")
model = cmd[idx + 1]
if model == "fail":
@ -653,12 +705,14 @@ def test_run_claude_models_parallel_preserves_nonzero_exit_codes():
runner=runner,
)
assert isinstance(outcomes["ok-model"], DriverResult)
assert isinstance(outcomes["fail"], DriverResult)
assert outcomes["ok-model"].exit_code == 0
assert outcomes["fail"].exit_code == 2
assert outcomes["fail"].stderr == "auth failed"
def test_run_claude_models_parallel_rejects_empty_models():
def test_run_claude_models_parallel_rejects_empty_models() -> None:
with pytest.raises(ValueError, match="non-empty"):
run_claude_models_parallel(
models=[],
@ -668,7 +722,7 @@ def test_run_claude_models_parallel_rejects_empty_models():
)
def test_run_claude_models_parallel_stamps_duration_on_each_result():
def test_run_claude_models_parallel_stamps_duration_on_each_result() -> None:
"""Each DriverResult carries the per-model wall time so callers can
attribute slow cells without re-timing the work themselves.
@ -680,7 +734,15 @@ def test_run_claude_models_parallel_stamps_duration_on_each_result():
"""
import time
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
idx = cmd.index("--model")
model = cmd[idx + 1]
time.sleep(0.05 if model == "fast" else 0.40)
@ -694,6 +756,8 @@ def test_run_claude_models_parallel_stamps_duration_on_each_result():
runner=runner,
)
assert isinstance(outcomes["fast"], DriverResult)
assert isinstance(outcomes["slow"], DriverResult)
fast_ms = outcomes["fast"].duration_ms
slow_ms = outcomes["slow"].duration_ms
assert fast_ms is not None and slow_ms is not None
@ -705,11 +769,21 @@ def test_run_claude_models_parallel_stamps_duration_on_each_result():
assert slow_ms > fast_ms
def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys):
def test_run_claude_models_parallel_breakdown_logs_to_stderr(
capsys: pytest.CaptureFixture[str],
) -> None:
"""The breakdown helper must emit a per-model timing block so users
can answer "why didn't parallel help?" without re-instrumenting."""
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
return _Completed(returncode=0, stdout="")
run_claude_models_parallel(
@ -728,11 +802,21 @@ def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys):
assert "slowest=" in captured.err
def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys):
def test_run_claude_models_parallel_breakdown_marks_cli_errors(
capsys: pytest.CaptureFixture[str],
) -> None:
"""When a model raises ClaudeCLIError, the breakdown should still
show its row tagged as `cli-error` rather than crashing or omitting it."""
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
idx = cmd.index("--model")
if cmd[idx + 1] == "boom":
raise FileNotFoundError(2, "no such file", "claude")
@ -752,12 +836,20 @@ def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys):
assert "cli-error" in captured.err
def test_run_claude_models_parallel_forwards_extra_args_and_env():
def test_run_claude_models_parallel_forwards_extra_args_and_env() -> None:
"""Shared kwargs must reach every per-model invocation unchanged."""
captured_envs: List[dict] = []
captured_cmds: List[List[str]] = []
captured_envs: list[Mapping[str, str]] = []
captured_cmds: list[Sequence[str]] = []
def runner(cmd, env, capture_output, text, timeout, check, input=None):
def runner(
cmd: Sequence[str],
env: Mapping[str, str],
capture_output: bool,
text: bool,
timeout: float,
check: bool,
input: str | None = None,
) -> subprocess.CompletedProcess[str]:
captured_envs.append(env)
captured_cmds.append(cmd)
return _Completed(returncode=0, stdout="")
@ -778,7 +870,7 @@ def test_run_claude_models_parallel_forwards_extra_args_and_env():
assert "Bash" in cmd
def test_failure_diagnostic_uses_last_result_event_status():
def test_failure_diagnostic_uses_last_result_event_status() -> None:
"""If multiple `result` events appear, the most recent status wins."""
result = DriverResult(
text="",

37
tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py
View file

@ -13,60 +13,61 @@ import pytest
from claude_code.conftest import CompatResult
def test_set_pass_is_accepted():
def test_set_pass_is_accepted() -> None:
r = CompatResult()
r.set({"status": "pass"})
assert r.value == {"status": "pass"}
def test_set_fail_requires_error():
def test_set_fail_requires_error() -> None:
r = CompatResult()
with pytest.raises(ValueError, match="requires 'error'"):
r.set({"status": "fail"})
def test_set_fail_with_error_is_accepted():
def test_set_fail_with_error_is_accepted() -> None:
r = CompatResult()
r.set({"status": "fail", "error": "boom"})
assert r.value == {"status": "fail", "error": "boom"}
def test_set_not_applicable_requires_reason():
def test_set_not_applicable_requires_reason() -> None:
r = CompatResult()
with pytest.raises(ValueError, match="requires 'reason'"):
r.set({"status": "not_applicable"})
def test_set_not_applicable_with_reason_is_accepted():
def test_set_not_applicable_with_reason_is_accepted() -> None:
r = CompatResult()
r.set({"status": "not_applicable", "reason": "Bedrock has no /thinking"})
assert r.value == {"status": "not_applicable", "reason": "Bedrock has no /thinking"}
def test_set_not_tested_is_accepted():
def test_set_not_tested_is_accepted() -> None:
r = CompatResult()
r.set({"status": "not_tested"})
assert r.value == {"status": "not_tested"}
def test_set_rejects_unknown_status():
def test_set_rejects_unknown_status() -> None:
r = CompatResult()
with pytest.raises(ValueError, match="status must be one of"):
r.set({"status": "maybe"})
def test_set_rejects_non_dict():
def test_set_rejects_non_dict() -> None:
r = CompatResult()
with pytest.raises(TypeError):
r.set("pass") # type: ignore[arg-type]
r.set("pass") # pyright: ignore[reportArgumentType] # deliberately wrong type to assert the TypeError
def test_set_copies_input():
def test_set_copies_input() -> None:
"""Mutating the dict after set() must not change the stored value."""
r = CompatResult()
payload = {"status": "fail", "error": "x"}
r.set(payload)
payload["error"] = "mutated"
assert r.value is not None
assert r.value["error"] == "x"
@ -80,7 +81,7 @@ def test_set_copies_input():
# ---------------------------------------------------------------------------
def test_add_appends_each_call_to_values():
def test_add_appends_each_call_to_values() -> None:
r = CompatResult()
r.add({"status": "pass"})
r.add({"status": "fail", "error": "bad"})
@ -90,7 +91,7 @@ def test_add_appends_each_call_to_values():
]
def test_add_validates_like_set():
def test_add_validates_like_set() -> None:
"""The add() and set() validators are the same; both must reject bad payloads."""
r = CompatResult()
with pytest.raises(ValueError, match="requires 'error'"):
@ -100,10 +101,10 @@ def test_add_validates_like_set():
with pytest.raises(ValueError, match="status must be one of"):
r.add({"status": "maybe"})
with pytest.raises(TypeError):
r.add("pass") # type: ignore[arg-type]
r.add("pass") # pyright: ignore[reportArgumentType] # deliberately wrong type to assert the TypeError
def test_add_copies_input():
def test_add_copies_input() -> None:
"""Same defensive copy contract as set()."""
r = CompatResult()
payload = {"status": "fail", "error": "x"}
@ -112,21 +113,21 @@ def test_add_copies_input():
assert r.values[0]["error"] == "x"
def test_collected_returns_values_when_added():
def test_collected_returns_values_when_added() -> None:
r = CompatResult()
r.add({"status": "pass"})
r.add({"status": "pass"})
assert r.collected() == [{"status": "pass"}, {"status": "pass"}]
def test_collected_returns_single_value_when_only_set_called():
def test_collected_returns_single_value_when_only_set_called() -> None:
"""Legacy single-result tests should still surface their one outcome."""
r = CompatResult()
r.set({"status": "pass"})
assert r.collected() == [{"status": "pass"}]
def test_collected_prefers_added_values_over_set_value():
def test_collected_prefers_added_values_over_set_value() -> None:
"""If both are populated, the per-tier list wins — that's the multi-model shape."""
r = CompatResult()
r.set({"status": "pass"})
@ -134,5 +135,5 @@ def test_collected_prefers_added_values_over_set_value():
assert r.collected() == [{"status": "fail", "error": "tier-2 broke"}]
def test_collected_returns_empty_when_nothing_reported():
def test_collected_returns_empty_when_nothing_reported() -> None:
assert CompatResult().collected() == []

103
tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py
View file

@ -25,13 +25,11 @@ fixtures + xdist already do that for the integration suite.
from __future__ import annotations
import json
import time
from pathlib import Path
from typing import List
import pytest
from claude_code.json_types import JSON_OBJECT_ADAPTER
from claude_code.rate_limiter import (
ALL_PROVIDERS,
BURST_ENV,
@ -69,22 +67,22 @@ from claude_code.rate_limiter import (
("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE),
],
)
def test_infer_provider_maps_alias_suffix_to_column(model, expected):
def test_infer_provider_maps_alias_suffix_to_column(model: str, expected: str) -> None:
assert infer_provider(model) == expected
def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order():
def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order() -> None:
"""Both bedrock suffixes contain `bedrock`; the more-specific suffix wins."""
assert infer_provider("claude-foo-bedrock-converse") == PROVIDER_BEDROCK_CONVERSE
assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE
def test_infer_provider_rejects_empty_string():
def test_infer_provider_rejects_empty_string() -> None:
with pytest.raises(ValueError, match="non-empty"):
infer_provider("")
def test_infer_provider_is_case_insensitive():
def test_infer_provider_is_case_insensitive() -> None:
"""Aliases in the proxy config sometimes drift between cases; we
should still route them to the right column."""
assert infer_provider("CLAUDE-OPUS-4-7-AZURE") == PROVIDER_AZURE
@ -95,14 +93,14 @@ def test_infer_provider_is_case_insensitive():
# ---------------------------------------------------------------------------
def test_load_config_uses_default_rate_when_env_absent():
def test_load_config_uses_default_rate_when_env_absent() -> None:
cfg = load_config(env={})
for provider in ALL_PROVIDERS:
assert cfg[provider].rate_per_sec == DEFAULT_RATE
assert cfg[provider].burst == DEFAULT_RATE
def test_load_config_reads_per_provider_rate():
def test_load_config_reads_per_provider_rate() -> None:
cfg = load_config(
env={
"LITELLM_COMPAT_RATE_ANTHROPIC": "10",
@ -114,12 +112,12 @@ def test_load_config_reads_per_provider_rate():
assert cfg[PROVIDER_VERTEX_AI].rate_per_sec == DEFAULT_RATE
def test_load_config_zero_rate_disables_provider():
def test_load_config_zero_rate_disables_provider() -> None:
cfg = load_config(env={"LITELLM_COMPAT_RATE_BEDROCK_INVOKE": "0"})
assert cfg[PROVIDER_BEDROCK_INVOKE].enabled is False
def test_load_config_burst_override_applies_to_every_provider():
def test_load_config_burst_override_applies_to_every_provider() -> None:
cfg = load_config(
env={
"LITELLM_COMPAT_RATE_ANTHROPIC": "5",
@ -130,12 +128,12 @@ def test_load_config_burst_override_applies_to_every_provider():
assert cfg[provider].burst == 20.0
def test_load_config_falls_back_on_malformed_value():
def test_load_config_falls_back_on_malformed_value() -> None:
cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "not-a-number"})
assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == DEFAULT_RATE
def test_load_config_burst_floors_at_one_when_rate_is_low():
def test_load_config_burst_floors_at_one_when_rate_is_low() -> None:
"""A 0.5/s rate with no burst override must still allow at least
one immediate request — otherwise the very first call would block."""
cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "0.5"})
@ -147,8 +145,21 @@ def test_load_config_burst_floors_at_one_when_rate_is_low():
# ---------------------------------------------------------------------------
class _Clock:
def __init__(self) -> None:
self.now: float = 1_000.0
self.sleeps: list[float] = []
def __call__(self) -> float:
return self.now
def sleep(self, seconds: float) -> None:
self.sleeps.append(seconds)
self.now += seconds
@pytest.fixture
def fake_clock():
def fake_clock() -> _Clock:
"""A controllable monotonic clock + sleep for the limiter under test.
Tests advance `clock.now` to simulate elapsed wall time. `sleep`
@ -156,23 +167,12 @@ def fake_clock():
sleeping, so a "wait 200ms" code path runs in microseconds and
is deterministic.
"""
class Clock:
def __init__(self):
self.now = 1_000.0
self.sleeps: List[float] = []
def __call__(self):
return self.now
def sleep(self, seconds: float) -> None:
self.sleeps.append(seconds)
self.now += seconds
return Clock()
return _Clock()
def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None):
def _make_limiter(
tmp_path: Path, fake_clock: _Clock, *, rate: float = 10.0, burst: float | None = None
) -> RateLimiter:
cfg = {
p: ProviderConfig(rate_per_sec=rate, burst=burst if burst is not None else rate)
for p in ALL_PROVIDERS
@ -185,7 +185,7 @@ def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None):
)
def test_acquire_first_call_does_not_wait(tmp_path, fake_clock):
def test_acquire_first_call_does_not_wait(tmp_path: Path, fake_clock: _Clock) -> None:
"""A freshly-initialized bucket starts full; the first acquire is free."""
limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=10.0)
waited = limiter.acquire(PROVIDER_ANTHROPIC)
@ -193,7 +193,7 @@ def test_acquire_first_call_does_not_wait(tmp_path, fake_clock):
assert fake_clock.sleeps == []
def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock):
def test_acquire_disabled_provider_returns_immediately(tmp_path: Path, fake_clock: _Clock) -> None:
"""rate=0 ⇒ no throttling, even if every other provider is throttled."""
cfg = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS}
limiter = RateLimiter(
@ -204,7 +204,7 @@ def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock):
assert fake_clock.sleeps == []
def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock):
def test_acquire_burns_through_burst_then_throttles(tmp_path: Path, fake_clock: _Clock) -> None:
"""`burst` immediate requests succeed; the next one waits 1/rate seconds."""
limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=3.0)
@ -213,10 +213,10 @@ def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock):
# Bucket is empty; next call must sleep ~0.5s to earn one token at 2/s.
waited = limiter.acquire(PROVIDER_ANTHROPIC)
assert waited == pytest.approx(0.5, abs=0.01)
assert waited == pytest.approx(0.5, abs=0.01) # pyright: ignore[reportUnknownMemberType] # untyped upstream
def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock):
def test_acquire_refills_with_elapsed_time(tmp_path: Path, fake_clock: _Clock) -> None:
"""Advancing the clock between calls credits tokens at the configured rate."""
limiter = _make_limiter(tmp_path, fake_clock, rate=4.0, burst=1.0)
@ -225,7 +225,7 @@ def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock):
assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
def test_acquire_caps_refill_at_burst(tmp_path, fake_clock):
def test_acquire_caps_refill_at_burst(tmp_path: Path, fake_clock: _Clock) -> None:
"""A long quiet period must not let the bucket grow past `burst`."""
limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=2.0)
@ -237,7 +237,7 @@ def test_acquire_caps_refill_at_burst(tmp_path, fake_clock):
assert waited > 0
def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock):
def test_acquire_independent_buckets_per_provider(tmp_path: Path, fake_clock: _Clock) -> None:
"""Anthropic exhaustion must not throttle Azure (each column has its own bucket)."""
limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=1.0)
@ -246,7 +246,7 @@ def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock):
assert limiter.acquire(PROVIDER_AZURE) == 0.0
def test_acquire_persists_state_across_limiter_instances(tmp_path):
def test_acquire_persists_state_across_limiter_instances(tmp_path: Path) -> None:
"""A fresh RateLimiter must read the on-disk state, not start fresh.
This is the property that makes the limiter cross-process: an
@ -254,26 +254,23 @@ def test_acquire_persists_state_across_limiter_instances(tmp_path):
workers, instead of getting its own private bucket.
"""
cfg = {p: ProviderConfig(rate_per_sec=10.0, burst=2.0) for p in ALL_PROVIDERS}
state = {"now": 1_000.0, "sleeps": []}
shared_clock = _Clock()
def clock():
return state["now"]
def sleep(seconds):
state["sleeps"].append(seconds)
state["now"] += seconds
first = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep)
first = RateLimiter(
config=cfg, state_dir=tmp_path, clock=shared_clock, sleep=shared_clock.sleep
)
first.acquire(PROVIDER_ANTHROPIC)
first.acquire(PROVIDER_ANTHROPIC)
# bucket is now empty
second = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep)
second = RateLimiter(
config=cfg, state_dir=tmp_path, clock=shared_clock, sleep=shared_clock.sleep
)
waited = second.acquire(PROVIDER_ANTHROPIC)
assert waited > 0 # had to wait, didn't see a fresh full bucket
def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock):
def test_acquire_recovers_from_corrupt_state_file(tmp_path: Path, fake_clock: _Clock) -> None:
"""A truncated/garbage state file must not crash the test session."""
state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json"
state_file.write_text("not-json {{")
@ -282,7 +279,7 @@ def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock):
assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0
def test_acquire_handles_clock_going_backward(tmp_path, fake_clock):
def test_acquire_handles_clock_going_backward(tmp_path: Path, fake_clock: _Clock) -> None:
"""Across a host suspend/resume the monotonic clock can briefly
go backward; we must not interpret that as removing tokens."""
limiter = _make_limiter(tmp_path, fake_clock, rate=1.0, burst=2.0)
@ -297,7 +294,7 @@ def test_acquire_handles_clock_going_backward(tmp_path, fake_clock):
# ---------------------------------------------------------------------------
def test_use_limiter_swaps_default_for_block(tmp_path):
def test_use_limiter_swaps_default_for_block(tmp_path: Path) -> None:
sentinel_cfg = {
p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS
}
@ -319,11 +316,11 @@ def test_use_limiter_swaps_default_for_block(tmp_path):
# ---------------------------------------------------------------------------
def test_state_file_is_json_after_acquire(tmp_path, fake_clock):
def test_state_file_is_json_after_acquire(tmp_path: Path, fake_clock: _Clock) -> None:
limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0)
limiter.acquire(PROVIDER_ANTHROPIC)
state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json"
payload = json.loads(state_file.read_text())
payload = JSON_OBJECT_ADAPTER.validate_json(state_file.read_text())
assert "tokens" in payload
assert "last_refill" in payload
assert payload["tokens"] == pytest.approx(4.0)
assert payload["tokens"] == pytest.approx(4.0) # pyright: ignore[reportUnknownMemberType] # untyped upstream

18
tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py
View file

@ -49,6 +49,10 @@ BASH_FEATURE_DIRS = (
)
def _cell_id(path: Path) -> str:
return str(path.relative_to(REPO_ROOT))
def _bash_cells() -> Iterable[Path]:
for feature in BASH_FEATURE_DIRS:
feature_dir = CLAUDE_CODE_DIR / feature
@ -73,9 +77,7 @@ def _has_bare_bash_token(text: str) -> bool:
return '"Bash"' in text.replace('"Bash(echo pong)"', "")
@pytest.mark.parametrize(
"cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT))
)
@pytest.mark.parametrize("cell", list(_bash_cells()), ids=_cell_id)
def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None:
"""The cell must pass `Bash(echo pong)` as the allow rule, not the
unrestricted `Bash` value that was originally flagged."""
@ -101,7 +103,7 @@ def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None:
)
def test_has_bare_bash_token_flags_unrestricted_value():
def test_has_bare_bash_token_flags_unrestricted_value() -> None:
"""A file that allows the bare `"Bash"` token alongside the
exact-match rule must be flagged. Without this guard the security
pin reverts to the dead-code `or` it had originally, which let
@ -111,7 +113,7 @@ def test_has_bare_bash_token_flags_unrestricted_value():
assert _has_bare_bash_token(text)
def test_has_bare_bash_token_accepts_only_exact_match():
def test_has_bare_bash_token_accepts_only_exact_match() -> None:
"""The standard pattern — only the exact-match allow rule, no bare
`"Bash"` — must be accepted. This is the shape every Bash-using
cell in the suite is required to take.
@ -120,7 +122,7 @@ def test_has_bare_bash_token_accepts_only_exact_match():
assert not _has_bare_bash_token(text)
def test_has_bare_bash_token_ignores_unrelated_substrings():
def test_has_bare_bash_token_ignores_unrelated_substrings() -> None:
"""`Bash(echo pong)` is the only allowed shape; substrings like
`BashTool` or `Bashing` are unrelated identifiers and must not be
confused with the bare `"Bash"` token (i.e. the exact quoted
@ -129,9 +131,7 @@ def test_has_bare_bash_token_ignores_unrelated_substrings():
assert not _has_bare_bash_token(text)
@pytest.mark.parametrize(
"cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT))
)
@pytest.mark.parametrize("cell", list(_bash_cells()), ids=_cell_id)
def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None:
"""The cell must pair the allow rule with `--permission-mode dontAsk`
so tool calls that don't match the allow rule are auto-denied (as

31
tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py
View file

@ -17,6 +17,7 @@ from datetime import datetime, timedelta, timezone
import pytest
from claude_code.json_types import JSONValue
from claude_code.pr_gate_version_resolver import (
NoEligibleVersionError,
resolve_pr_gate_version,
@ -32,7 +33,7 @@ def _t(iso: str) -> str:
NOW = datetime(2026, 4, 25, 12, 0, 0, tzinfo=timezone.utc)
def _metadata_with_times(times: dict) -> dict:
def _metadata_with_times(times: dict[str, JSONValue]) -> dict[str, JSONValue]:
"""Shape an npm `packument`-like dict with the `time` field populated.
The npm registry response includes `time.created` / `time.modified`
@ -48,7 +49,7 @@ def _metadata_with_times(times: dict) -> dict:
}
def test_picks_newest_version_at_least_three_days_old():
def test_picks_newest_version_at_least_three_days_old() -> None:
metadata = _metadata_with_times(
{
"2.1.118": _t("2026-04-15T10:00:00.000Z"),
@ -60,9 +61,9 @@ def test_picks_newest_version_at_least_three_days_old():
assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.119"
def test_skips_created_and_modified_meta_keys():
def test_skips_created_and_modified_meta_keys() -> None:
"""`time` contains `created` / `modified` non-version entries — must be ignored."""
metadata = {
metadata: dict[str, JSONValue] = {
"name": "@anthropic-ai/claude-code",
"time": {
"created": _t("2024-01-01T00:00:00.000Z"),
@ -73,7 +74,7 @@ def test_skips_created_and_modified_meta_keys():
assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.0.0"
def test_min_age_boundary_is_inclusive():
def test_min_age_boundary_is_inclusive() -> None:
"""A version published exactly 3 days ago is eligible (>= cutoff)."""
three_days_ago = NOW - timedelta(days=3)
metadata = _metadata_with_times(
@ -84,7 +85,7 @@ def test_min_age_boundary_is_inclusive():
assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.0"
def test_raises_when_every_version_is_too_new():
def test_raises_when_every_version_is_too_new() -> None:
metadata = _metadata_with_times(
{
"2.1.121": _t("2026-04-25T08:00:00.000Z"), # 4h old
@ -95,13 +96,13 @@ def test_raises_when_every_version_is_too_new():
resolve_pr_gate_version(metadata=metadata, as_of=NOW)
def test_raises_when_metadata_has_no_versions():
metadata = {"name": "@anthropic-ai/claude-code", "time": {}}
def test_raises_when_metadata_has_no_versions() -> None:
metadata: dict[str, JSONValue] = {"name": "@anthropic-ai/claude-code", "time": {}}
with pytest.raises(NoEligibleVersionError):
resolve_pr_gate_version(metadata=metadata, as_of=NOW)
def test_picks_latest_publish_time_not_largest_semver():
def test_picks_latest_publish_time_not_largest_semver() -> None:
"""If a patch is published to an old major after a newer release,
"newest" is by publish time, not semver string ordering."""
metadata = _metadata_with_times(
@ -113,7 +114,7 @@ def test_picks_latest_publish_time_not_largest_semver():
assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "1.9.99"
def test_uses_custom_min_age():
def test_uses_custom_min_age() -> None:
metadata = _metadata_with_times(
{
"1.0.0": _t("2026-04-23T10:00:00.000Z"), # 2d 2h old
@ -127,7 +128,7 @@ def test_uses_custom_min_age():
assert out == "0.9.0"
def test_excludes_prerelease_versions():
def test_excludes_prerelease_versions() -> None:
"""Pre-release tags (1.0.0-alpha.1, 2.0.0-rc.1, etc.) must never win,
even if their publish timestamp is the newest eligible one."""
metadata = _metadata_with_times(
@ -141,7 +142,7 @@ def test_excludes_prerelease_versions():
assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.119"
def test_raises_when_only_prereleases_are_eligible():
def test_raises_when_only_prereleases_are_eligible() -> None:
metadata = _metadata_with_times(
{
"2.2.0-alpha.1": _t("2026-04-22T10:00:00.000Z"),
@ -152,10 +153,10 @@ def test_raises_when_only_prereleases_are_eligible():
resolve_pr_gate_version(metadata=metadata, as_of=NOW)
def test_resolver_uses_fetcher_when_metadata_not_provided():
captured = {}
def test_resolver_uses_fetcher_when_metadata_not_provided() -> None:
captured: dict[str, str] = {}
def fake_fetch(package_name: str) -> dict:
def fake_fetch(package_name: str) -> dict[str, JSONValue]:
captured["package"] = package_name
return _metadata_with_times({"3.0.0": _t("2026-04-10T10:00:00.000Z")})

3
tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
# Per the PRD: each cell is exercised against three Claude tiers via the
# Anthropic provider. Aliases are configured in the LiteLLM proxy's
@ -32,7 +33,7 @@ ANTHROPIC_MODELS = [
]
def test_basic_messaging_non_streaming_anthropic(compat_result):
def test_basic_messaging_non_streaming_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
"Basic messaging" means: send a single user prompt, receive any

3
tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py
View file

@ -26,6 +26,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
# Per-model aliases registered in the LiteLLM proxy's routing config to
# point at Microsoft Foundry's Anthropic deployments. The driver only
@ -38,7 +39,7 @@ AZURE_MODELS = [
]
def test_basic_messaging_non_streaming_azure(compat_result):
def test_basic_messaging_non_streaming_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
"Basic messaging" means: send a single user prompt, receive any

3
tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
# Per-model aliases registered in the LiteLLM proxy's routing config to
# point at Bedrock's Converse endpoint. The driver only sends the alias;
@ -33,7 +34,7 @@ BEDROCK_CONVERSE_MODELS = [
]
def test_basic_messaging_non_streaming_bedrock_converse(compat_result):
def test_basic_messaging_non_streaming_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
run_basic_messaging_cell(
compat_result=compat_result,

3
tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
# Per-model aliases registered in the LiteLLM proxy's routing config to
# point at Bedrock's legacy InvokeModel endpoint. The driver only sends
@ -33,7 +34,7 @@ BEDROCK_INVOKE_MODELS = [
]
def test_basic_messaging_non_streaming_bedrock_invoke(compat_result):
def test_basic_messaging_non_streaming_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
run_basic_messaging_cell(
compat_result=compat_result,

3
tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py
View file

@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider).
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
# Per-model aliases registered in the LiteLLM proxy's routing config to
# point at Vertex AI's Anthropic model endpoints. The driver only sends
@ -33,7 +34,7 @@ VERTEX_AI_MODELS = [
]
def test_basic_messaging_non_streaming_vertex_ai(compat_result):
def test_basic_messaging_non_streaming_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply."""
run_basic_messaging_cell(
compat_result=compat_result,

3
tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py
View file

@ -26,6 +26,7 @@ sees three rows for this (feature, provider).
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
ANTHROPIC_MODELS = [
"claude-haiku-4-5",
@ -34,7 +35,7 @@ ANTHROPIC_MODELS = [
]
def test_basic_messaging_streaming_anthropic(compat_result):
def test_basic_messaging_streaming_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).
"""

3
tests/e2e/claude_code/basic_messaging_streaming/test_azure.py
View file

@ -20,6 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
AZURE_MODELS = [
"claude-haiku-4-5-azure",
@ -28,7 +29,7 @@ AZURE_MODELS = [
]
def test_basic_messaging_streaming_azure(compat_result):
def test_basic_messaging_streaming_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).
"""

3
tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
BEDROCK_CONVERSE_MODELS = [
"claude-haiku-4-5-bedrock-converse",
@ -24,7 +25,7 @@ BEDROCK_CONVERSE_MODELS = [
]
def test_basic_messaging_streaming_bedrock_converse(compat_result):
def test_basic_messaging_streaming_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).
"""

3
tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
BEDROCK_INVOKE_MODELS = [
"claude-haiku-4-5-bedrock-invoke",
@ -24,7 +25,7 @@ BEDROCK_INVOKE_MODELS = [
]
def test_basic_messaging_streaming_bedrock_invoke(compat_result):
def test_basic_messaging_streaming_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).
"""

3
tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
from claude_code._basic_messaging import run_basic_messaging_cell
from claude_code.conftest import CompatResult
VERTEX_AI_MODELS = [
"claude-haiku-4-5-vertex",
@ -24,7 +25,7 @@ VERTEX_AI_MODELS = [
]
def test_basic_messaging_streaming_vertex_ai(compat_result):
def test_basic_messaging_streaming_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
non-empty streamed reply (one row per Claude tier).
"""

5
tests/e2e/claude_code/count_tokens/test_anthropic.py
View file

@ -41,6 +41,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_count_tokens_shape,
probe_count_tokens,
@ -56,7 +57,7 @@ ANTHROPIC_MODELS = [
]
def test_count_tokens_anthropic(compat_result):
def test_count_tokens_anthropic(compat_result: CompatResult) -> None:
"""Probe `/v1/messages/count_tokens` for each Anthropic tier and
assert the response shape."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -76,7 +77,7 @@ def test_count_tokens_anthropic(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
result = probe_count_tokens(
base_url=base_url, api_key=api_key, model=model

5
tests/e2e/claude_code/count_tokens/test_azure.py
View file

@ -41,6 +41,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_count_tokens_shape,
probe_count_tokens,
@ -56,7 +57,7 @@ AZURE_MODELS = [
]
def test_count_tokens_azure(compat_result):
def test_count_tokens_azure(compat_result: CompatResult) -> None:
"""Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and
assert the response shape."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -76,7 +77,7 @@ def test_count_tokens_azure(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
result = probe_count_tokens(
base_url=base_url, api_key=api_key, model=model

5
tests/e2e/claude_code/count_tokens/test_bedrock_converse.py
View file

@ -41,6 +41,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_count_tokens_shape,
probe_count_tokens,
@ -56,7 +57,7 @@ BEDROCK_CONVERSE_MODELS = [
]
def test_count_tokens_bedrock_converse(compat_result):
def test_count_tokens_bedrock_converse(compat_result: CompatResult) -> None:
"""Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and
assert the response shape."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -76,7 +77,7 @@ def test_count_tokens_bedrock_converse(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
result = probe_count_tokens(
base_url=base_url, api_key=api_key, model=model

5
tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py
View file

@ -41,6 +41,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_count_tokens_shape,
probe_count_tokens,
@ -56,7 +57,7 @@ BEDROCK_INVOKE_MODELS = [
]
def test_count_tokens_bedrock_invoke(compat_result):
def test_count_tokens_bedrock_invoke(compat_result: CompatResult) -> None:
"""Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and
assert the response shape."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -76,7 +77,7 @@ def test_count_tokens_bedrock_invoke(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
result = probe_count_tokens(
base_url=base_url, api_key=api_key, model=model

5
tests/e2e/claude_code/count_tokens/test_vertex_ai.py
View file

@ -41,6 +41,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_count_tokens_shape,
probe_count_tokens,
@ -56,7 +57,7 @@ VERTEX_AI_MODELS = [
]
def test_count_tokens_vertex_ai(compat_result):
def test_count_tokens_vertex_ai(compat_result: CompatResult) -> None:
"""Probe `/v1/messages/count_tokens` for each Vertex AI tier and
assert the response shape."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -76,7 +77,7 @@ def test_count_tokens_vertex_ai(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
result = probe_count_tokens(
base_url=base_url, api_key=api_key, model=model

7
tests/e2e/claude_code/long_context_1m/test_anthropic.py
View file

@ -64,6 +64,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'."
pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing)
pad_lines = []
pad_lines: list[str] = []
pad_len = 0
idx = 0
while pad_len < pad_target_chars:
@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
return preamble + "".join(pad_lines) + closing
def test_long_context_1m_anthropic(compat_result):
def test_long_context_1m_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
non-empty reply for Sonnet + Opus."""
@ -197,7 +198,7 @@ def test_long_context_1m_anthropic(compat_result):
timeout=300.0,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

7
tests/e2e/claude_code/long_context_1m/test_azure.py
View file

@ -64,6 +64,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'."
pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing)
pad_lines = []
pad_lines: list[str] = []
pad_len = 0
idx = 0
while pad_len < pad_target_chars:
@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
return preamble + "".join(pad_lines) + closing
def test_long_context_1m_azure(compat_result):
def test_long_context_1m_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
non-empty reply for Sonnet + Opus."""
@ -197,7 +198,7 @@ def test_long_context_1m_azure(compat_result):
timeout=300.0,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

7
tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py
View file

@ -64,6 +64,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'."
pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing)
pad_lines = []
pad_lines: list[str] = []
pad_len = 0
idx = 0
while pad_len < pad_target_chars:
@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
return preamble + "".join(pad_lines) + closing
def test_long_context_1m_bedrock_converse(compat_result):
def test_long_context_1m_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
non-empty reply for Sonnet + Opus."""
@ -197,7 +198,7 @@ def test_long_context_1m_bedrock_converse(compat_result):
timeout=300.0,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

7
tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py
View file

@ -64,6 +64,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'."
pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing)
pad_lines = []
pad_lines: list[str] = []
pad_len = 0
idx = 0
while pad_len < pad_target_chars:
@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
return preamble + "".join(pad_lines) + closing
def test_long_context_1m_bedrock_invoke(compat_result):
def test_long_context_1m_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
non-empty reply for Sonnet + Opus."""
@ -197,7 +198,7 @@ def test_long_context_1m_bedrock_invoke(compat_result):
timeout=300.0,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

7
tests/e2e/claude_code/long_context_1m/test_vertex_ai.py
View file

@ -64,6 +64,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'."
pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing)
pad_lines = []
pad_lines: list[str] = []
pad_len = 0
idx = 0
while pad_len < pad_target_chars:
@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str:
return preamble + "".join(pad_lines) + closing
def test_long_context_1m_vertex_ai(compat_result):
def test_long_context_1m_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the
`context-1m-2025-08-07` beta header; assert no 400 / 413 and a
non-empty reply for Sonnet + Opus."""
@ -197,7 +198,7 @@ def test_long_context_1m_vertex_ai(compat_result):
timeout=300.0,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

8
tests/e2e/claude_code/pdf_input/test_anthropic.py
View file

@ -23,6 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from pathlib import Path
import pytest
@ -31,6 +32,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -88,7 +90,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
)
out = bytearray(b"%PDF-1.4\n")
offsets = []
offsets: list[int] = []
for i, obj in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
@ -108,7 +110,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
return bytes(out)
def test_pdf_input_anthropic(compat_result, tmp_path):
def test_pdf_input_anthropic(compat_result: CompatResult, tmp_path: Path) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with a PDF
attached via the Read tool and assert the reply references it."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -141,7 +143,7 @@ def test_pdf_input_anthropic(compat_result, tmp_path):
extra_args=["--allowed-tools", "Read"],
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

8
tests/e2e/claude_code/pdf_input/test_azure.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from pathlib import Path
import pytest
@ -24,6 +25,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,7 +67,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
)
out = bytearray(b"%PDF-1.4\n")
offsets = []
offsets: list[int] = []
for i, obj in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
@ -85,7 +87,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
return bytes(out)
def test_pdf_input_azure(compat_result, tmp_path):
def test_pdf_input_azure(compat_result: CompatResult, tmp_path: Path) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -116,7 +118,7 @@ def test_pdf_input_azure(compat_result, tmp_path):
extra_args=["--allowed-tools", "Read"],
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

8
tests/e2e/claude_code/pdf_input/test_bedrock_converse.py
View file

@ -22,6 +22,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from pathlib import Path
import pytest
@ -30,6 +31,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -71,7 +73,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
)
out = bytearray(b"%PDF-1.4\n")
offsets = []
offsets: list[int] = []
for i, obj in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
@ -91,7 +93,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
return bytes(out)
def test_pdf_input_bedrock_converse(compat_result, tmp_path):
def test_pdf_input_bedrock_converse(compat_result: CompatResult, tmp_path: Path) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -122,7 +124,7 @@ def test_pdf_input_bedrock_converse(compat_result, tmp_path):
extra_args=["--allowed-tools", "Read"],
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

8
tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py
View file

@ -21,6 +21,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from pathlib import Path
import pytest
@ -29,6 +30,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -70,7 +72,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
)
out = bytearray(b"%PDF-1.4\n")
offsets = []
offsets: list[int] = []
for i, obj in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
@ -90,7 +92,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
return bytes(out)
def test_pdf_input_bedrock_invoke(compat_result, tmp_path):
def test_pdf_input_bedrock_invoke(compat_result: CompatResult, tmp_path: Path) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -121,7 +123,7 @@ def test_pdf_input_bedrock_invoke(compat_result, tmp_path):
extra_args=["--allowed-tools", "Read"],
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

8
tests/e2e/claude_code/pdf_input/test_vertex_ai.py
View file

@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from pathlib import Path
import pytest
@ -24,6 +25,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,7 +67,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
)
out = bytearray(b"%PDF-1.4\n")
offsets = []
offsets: list[int] = []
for i, obj in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n"
@ -85,7 +87,7 @@ def _build_minimal_pdf(marker: str) -> bytes:
return bytes(out)
def test_pdf_input_vertex_ai(compat_result, tmp_path):
def test_pdf_input_vertex_ai(compat_result: CompatResult, tmp_path: Path) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -116,7 +118,7 @@ def test_pdf_input_vertex_ai(compat_result, tmp_path):
extra_args=["--allowed-tools", "Read"],
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py
View file

@ -25,7 +25,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -34,6 +34,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -52,20 +54,22 @@ ANTHROPIC_MODELS = [
CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"}
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
"""Return cache_creation_input_tokens + cache_read_input_tokens from
the upstream usage block, or 0 if the keys are missing."""
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_1h_anthropic(compat_result):
def test_prompt_caching_1h_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with the 1h
TTL opt-in env var set, and assert the upstream usage block
surfaces a non-zero cache token count."""
@ -93,7 +97,7 @@ def test_prompt_caching_1h_anthropic(compat_result):
extra_env=CACHE_1H_ENV,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_1h/test_azure.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -38,18 +40,20 @@ AZURE_MODELS = [
CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"}
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_1h_azure(compat_result):
def test_prompt_caching_1h_azure(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -74,7 +78,7 @@ def test_prompt_caching_1h_azure(compat_result):
extra_env=CACHE_1H_ENV,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py
View file

@ -21,7 +21,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -30,6 +30,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -46,18 +48,20 @@ CACHE_1H_ENV = {
}
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_1h_bedrock_converse(compat_result):
def test_prompt_caching_1h_bedrock_converse(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -82,7 +86,7 @@ def test_prompt_caching_1h_bedrock_converse(compat_result):
extra_env=CACHE_1H_ENV,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py
View file

@ -23,7 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -32,6 +32,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -50,18 +52,20 @@ CACHE_1H_ENV = {
}
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_1h_bedrock_invoke(compat_result):
def test_prompt_caching_1h_bedrock_invoke(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -86,7 +90,7 @@ def test_prompt_caching_1h_bedrock_invoke(compat_result):
extra_env=CACHE_1H_ENV,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -38,18 +40,20 @@ VERTEX_AI_MODELS = [
CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"}
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_1h_vertex_ai(compat_result):
def test_prompt_caching_1h_vertex_ai(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -74,7 +78,7 @@ def test_prompt_caching_1h_vertex_ai(compat_result):
extra_env=CACHE_1H_ENV,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py
View file

@ -23,7 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -32,6 +32,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -43,20 +45,22 @@ ANTHROPIC_MODELS = [
]
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
"""Return cache_creation_input_tokens + cache_read_input_tokens from
the upstream usage block, or 0 if the keys are missing."""
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_5m_anthropic(compat_result):
def test_prompt_caching_5m_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -82,7 +86,7 @@ def test_prompt_caching_5m_anthropic(compat_result):
api_key=api_key,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_5m/test_azure.py
View file

@ -23,7 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -32,6 +32,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -43,18 +45,20 @@ AZURE_MODELS = [
]
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_5m_azure(compat_result):
def test_prompt_caching_5m_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -80,7 +84,7 @@ def test_prompt_caching_5m_azure(compat_result):
api_key=api_key,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -36,18 +38,20 @@ BEDROCK_CONVERSE_MODELS = [
]
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_5m_bedrock_converse(compat_result):
def test_prompt_caching_5m_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -73,7 +77,7 @@ def test_prompt_caching_5m_bedrock_converse(compat_result):
api_key=api_key,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -36,18 +38,20 @@ BEDROCK_INVOKE_MODELS = [
]
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_5m_bedrock_invoke(compat_result):
def test_prompt_caching_5m_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -73,7 +77,7 @@ def test_prompt_caching_5m_bedrock_invoke(compat_result):
api_key=api_key,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

12
tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Optional
from typing import Mapping
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -36,18 +38,20 @@ VERTEX_AI_MODELS = [
]
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int:
if not isinstance(usage, Mapping):
return 0
creation = usage.get("cache_creation_input_tokens") or 0
read = usage.get("cache_read_input_tokens") or 0
if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)):
return 0
try:
return int(creation) + int(read)
except (TypeError, ValueError):
return 0
def test_prompt_caching_5m_vertex_ai(compat_result):
def test_prompt_caching_5m_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream usage block surfaces a non-zero cache token count."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -73,7 +77,7 @@ def test_prompt_caching_5m_vertex_ai(compat_result):
api_key=api_key,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

33
tests/e2e/claude_code/structured_outputs/test_anthropic.py
View file

@ -49,8 +49,7 @@ from __future__ import annotations
import json
import os
import re
from typing import Any, Mapping, Optional, Sequence, Tuple
from typing import Mapping, Optional, Sequence
import pytest
@ -59,6 +58,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -75,7 +76,7 @@ ANTHROPIC_MODELS = [
# integer schema gives every tier (including Haiku) enough headroom
# that schema satisfaction is essentially deterministic, isolating
# failures to the proxy / transport.
SCHEMA = {
SCHEMA: dict[str, JSONValue] = {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output."
def _extract_structured_output(
events: Sequence[Mapping[str, Any]],
) -> Optional[Mapping[str, Any]]:
events: Sequence[Mapping[str, JSONValue]],
) -> Optional[Mapping[str, JSONValue]]:
"""Return the `structured_output` payload from the last `result` event.
Claude Code emits its terminal stream-json line as
@ -113,7 +114,7 @@ def _extract_structured_output(
def _validate_against_schema(
payload: Mapping[str, Any], schema: Mapping[str, Any]
payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue]
) -> Optional[str]:
"""Tiny shape validator covering the subset we actually need.
@ -125,7 +126,7 @@ def _validate_against_schema(
not a LiteLLM-proxy bug, so a deeper check would only add false
failures on the wrong axis.
"""
type_map = {
type_map: dict[str, type[object] | tuple[type[object], ...]] = {
"integer": int,
"number": (int, float),
"string": str,
@ -133,13 +134,17 @@ def _validate_against_schema(
"array": list,
"object": Mapping,
}
required = schema.get("required") or []
properties = schema.get("properties") or {}
raw_required = schema.get("required")
required: list[JSONValue] = raw_required if isinstance(raw_required, list) else []
raw_properties = schema.get("properties")
properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {}
for key in required:
if key not in payload:
if not isinstance(key, str) or key not in payload:
return f"missing required key {key!r}"
expected = (properties.get(key) or {}).get("type")
if expected and expected in type_map:
prop = properties.get(key)
prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {}
expected = prop_schema.get("type")
if isinstance(expected, str) and expected in type_map:
if not isinstance(payload[key], type_map[expected]):
return (
f"key {key!r} has wrong type: "
@ -152,7 +157,7 @@ def _validate_against_schema(
return None
def test_structured_outputs_anthropic(compat_result):
def test_structured_outputs_anthropic(compat_result: CompatResult) -> None:
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming
`structured_output`."""
@ -181,7 +186,7 @@ def test_structured_outputs_anthropic(compat_result):
extra_args=["--json-schema", SCHEMA_JSON],
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

33
tests/e2e/claude_code/structured_outputs/test_azure.py
View file

@ -49,8 +49,7 @@ from __future__ import annotations
import json
import os
import re
from typing import Any, Mapping, Optional, Sequence, Tuple
from typing import Mapping, Optional, Sequence
import pytest
@ -59,6 +58,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -75,7 +76,7 @@ AZURE_MODELS = [
# integer schema gives every tier (including Haiku) enough headroom
# that schema satisfaction is essentially deterministic, isolating
# failures to the proxy / transport.
SCHEMA = {
SCHEMA: dict[str, JSONValue] = {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output."
def _extract_structured_output(
events: Sequence[Mapping[str, Any]],
) -> Optional[Mapping[str, Any]]:
events: Sequence[Mapping[str, JSONValue]],
) -> Optional[Mapping[str, JSONValue]]:
"""Return the `structured_output` payload from the last `result` event.
Claude Code emits its terminal stream-json line as
@ -113,7 +114,7 @@ def _extract_structured_output(
def _validate_against_schema(
payload: Mapping[str, Any], schema: Mapping[str, Any]
payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue]
) -> Optional[str]:
"""Tiny shape validator covering the subset we actually need.
@ -125,7 +126,7 @@ def _validate_against_schema(
not a LiteLLM-proxy bug, so a deeper check would only add false
failures on the wrong axis.
"""
type_map = {
type_map: dict[str, type[object] | tuple[type[object], ...]] = {
"integer": int,
"number": (int, float),
"string": str,
@ -133,13 +134,17 @@ def _validate_against_schema(
"array": list,
"object": Mapping,
}
required = schema.get("required") or []
properties = schema.get("properties") or {}
raw_required = schema.get("required")
required: list[JSONValue] = raw_required if isinstance(raw_required, list) else []
raw_properties = schema.get("properties")
properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {}
for key in required:
if key not in payload:
if not isinstance(key, str) or key not in payload:
return f"missing required key {key!r}"
expected = (properties.get(key) or {}).get("type")
if expected and expected in type_map:
prop = properties.get(key)
prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {}
expected = prop_schema.get("type")
if isinstance(expected, str) and expected in type_map:
if not isinstance(payload[key], type_map[expected]):
return (
f"key {key!r} has wrong type: "
@ -152,7 +157,7 @@ def _validate_against_schema(
return None
def test_structured_outputs_azure(compat_result):
def test_structured_outputs_azure(compat_result: CompatResult) -> None:
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming
`structured_output`."""
@ -181,7 +186,7 @@ def test_structured_outputs_azure(compat_result):
extra_args=["--json-schema", SCHEMA_JSON],
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

33
tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py
View file

@ -49,8 +49,7 @@ from __future__ import annotations
import json
import os
import re
from typing import Any, Mapping, Optional, Sequence, Tuple
from typing import Mapping, Optional, Sequence
import pytest
@ -59,6 +58,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -75,7 +76,7 @@ BEDROCK_CONVERSE_MODELS = [
# integer schema gives every tier (including Haiku) enough headroom
# that schema satisfaction is essentially deterministic, isolating
# failures to the proxy / transport.
SCHEMA = {
SCHEMA: dict[str, JSONValue] = {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output."
def _extract_structured_output(
events: Sequence[Mapping[str, Any]],
) -> Optional[Mapping[str, Any]]:
events: Sequence[Mapping[str, JSONValue]],
) -> Optional[Mapping[str, JSONValue]]:
"""Return the `structured_output` payload from the last `result` event.
Claude Code emits its terminal stream-json line as
@ -113,7 +114,7 @@ def _extract_structured_output(
def _validate_against_schema(
payload: Mapping[str, Any], schema: Mapping[str, Any]
payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue]
) -> Optional[str]:
"""Tiny shape validator covering the subset we actually need.
@ -125,7 +126,7 @@ def _validate_against_schema(
not a LiteLLM-proxy bug, so a deeper check would only add false
failures on the wrong axis.
"""
type_map = {
type_map: dict[str, type[object] | tuple[type[object], ...]] = {
"integer": int,
"number": (int, float),
"string": str,
@ -133,13 +134,17 @@ def _validate_against_schema(
"array": list,
"object": Mapping,
}
required = schema.get("required") or []
properties = schema.get("properties") or {}
raw_required = schema.get("required")
required: list[JSONValue] = raw_required if isinstance(raw_required, list) else []
raw_properties = schema.get("properties")
properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {}
for key in required:
if key not in payload:
if not isinstance(key, str) or key not in payload:
return f"missing required key {key!r}"
expected = (properties.get(key) or {}).get("type")
if expected and expected in type_map:
prop = properties.get(key)
prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {}
expected = prop_schema.get("type")
if isinstance(expected, str) and expected in type_map:
if not isinstance(payload[key], type_map[expected]):
return (
f"key {key!r} has wrong type: "
@ -152,7 +157,7 @@ def _validate_against_schema(
return None
def test_structured_outputs_bedrock_converse(compat_result):
def test_structured_outputs_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming
`structured_output`."""
@ -181,7 +186,7 @@ def test_structured_outputs_bedrock_converse(compat_result):
extra_args=["--json-schema", SCHEMA_JSON],
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

33
tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py
View file

@ -49,8 +49,7 @@ from __future__ import annotations
import json
import os
import re
from typing import Any, Mapping, Optional, Sequence, Tuple
from typing import Mapping, Optional, Sequence
import pytest
@ -59,6 +58,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -75,7 +76,7 @@ BEDROCK_INVOKE_MODELS = [
# integer schema gives every tier (including Haiku) enough headroom
# that schema satisfaction is essentially deterministic, isolating
# failures to the proxy / transport.
SCHEMA = {
SCHEMA: dict[str, JSONValue] = {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output."
def _extract_structured_output(
events: Sequence[Mapping[str, Any]],
) -> Optional[Mapping[str, Any]]:
events: Sequence[Mapping[str, JSONValue]],
) -> Optional[Mapping[str, JSONValue]]:
"""Return the `structured_output` payload from the last `result` event.
Claude Code emits its terminal stream-json line as
@ -113,7 +114,7 @@ def _extract_structured_output(
def _validate_against_schema(
payload: Mapping[str, Any], schema: Mapping[str, Any]
payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue]
) -> Optional[str]:
"""Tiny shape validator covering the subset we actually need.
@ -125,7 +126,7 @@ def _validate_against_schema(
not a LiteLLM-proxy bug, so a deeper check would only add false
failures on the wrong axis.
"""
type_map = {
type_map: dict[str, type[object] | tuple[type[object], ...]] = {
"integer": int,
"number": (int, float),
"string": str,
@ -133,13 +134,17 @@ def _validate_against_schema(
"array": list,
"object": Mapping,
}
required = schema.get("required") or []
properties = schema.get("properties") or {}
raw_required = schema.get("required")
required: list[JSONValue] = raw_required if isinstance(raw_required, list) else []
raw_properties = schema.get("properties")
properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {}
for key in required:
if key not in payload:
if not isinstance(key, str) or key not in payload:
return f"missing required key {key!r}"
expected = (properties.get(key) or {}).get("type")
if expected and expected in type_map:
prop = properties.get(key)
prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {}
expected = prop_schema.get("type")
if isinstance(expected, str) and expected in type_map:
if not isinstance(payload[key], type_map[expected]):
return (
f"key {key!r} has wrong type: "
@ -152,7 +157,7 @@ def _validate_against_schema(
return None
def test_structured_outputs_bedrock_invoke(compat_result):
def test_structured_outputs_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming
`structured_output`."""
@ -181,7 +186,7 @@ def test_structured_outputs_bedrock_invoke(compat_result):
extra_args=["--json-schema", SCHEMA_JSON],
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

33
tests/e2e/claude_code/structured_outputs/test_vertex_ai.py
View file

@ -49,8 +49,7 @@ from __future__ import annotations
import json
import os
import re
from typing import Any, Mapping, Optional, Sequence, Tuple
from typing import Mapping, Optional, Sequence
import pytest
@ -59,6 +58,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -75,7 +76,7 @@ VERTEX_AI_MODELS = [
# integer schema gives every tier (including Haiku) enough headroom
# that schema satisfaction is essentially deterministic, isolating
# failures to the proxy / transport.
SCHEMA = {
SCHEMA: dict[str, JSONValue] = {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output."
def _extract_structured_output(
events: Sequence[Mapping[str, Any]],
) -> Optional[Mapping[str, Any]]:
events: Sequence[Mapping[str, JSONValue]],
) -> Optional[Mapping[str, JSONValue]]:
"""Return the `structured_output` payload from the last `result` event.
Claude Code emits its terminal stream-json line as
@ -113,7 +114,7 @@ def _extract_structured_output(
def _validate_against_schema(
payload: Mapping[str, Any], schema: Mapping[str, Any]
payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue]
) -> Optional[str]:
"""Tiny shape validator covering the subset we actually need.
@ -125,7 +126,7 @@ def _validate_against_schema(
not a LiteLLM-proxy bug, so a deeper check would only add false
failures on the wrong axis.
"""
type_map = {
type_map: dict[str, type[object] | tuple[type[object], ...]] = {
"integer": int,
"number": (int, float),
"string": str,
@ -133,13 +134,17 @@ def _validate_against_schema(
"array": list,
"object": Mapping,
}
required = schema.get("required") or []
properties = schema.get("properties") or {}
raw_required = schema.get("required")
required: list[JSONValue] = raw_required if isinstance(raw_required, list) else []
raw_properties = schema.get("properties")
properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {}
for key in required:
if key not in payload:
if not isinstance(key, str) or key not in payload:
return f"missing required key {key!r}"
expected = (properties.get(key) or {}).get("type")
if expected and expected in type_map:
prop = properties.get(key)
prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {}
expected = prop_schema.get("type")
if isinstance(expected, str) and expected in type_map:
if not isinstance(payload[key], type_map[expected]):
return (
f"key {key!r} has wrong type: "
@ -152,7 +157,7 @@ def _validate_against_schema(
return None
def test_structured_outputs_vertex_ai(compat_result):
def test_structured_outputs_vertex_ai(compat_result: CompatResult) -> None:
"""Drive `claude --json-schema ...` against the LiteLLM proxy and
assert the trailing `result` event contains a schema-conforming
`structured_output`."""
@ -181,7 +186,7 @@ def test_structured_outputs_vertex_ai(compat_result):
extra_args=["--json-schema", SCHEMA_JSON],
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking/test_anthropic.py
View file

@ -21,7 +21,7 @@ still sees three rows for this (feature, provider).
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -30,6 +30,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -60,14 +62,14 @@ THINKING_PROMPT = (
)
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `thinking` content block."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -76,7 +78,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_thinking_anthropic(compat_result):
def test_thinking_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -103,7 +105,7 @@ def test_thinking_anthropic(compat_result):
extra_args=THINKING_ARGS,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking/test_azure.py
View file

@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -33,6 +33,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -50,12 +52,12 @@ THINKING_PROMPT = (
)
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -64,7 +66,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_thinking_azure(compat_result):
def test_thinking_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -91,7 +93,7 @@ def test_thinking_azure(compat_result):
extra_args=THINKING_ARGS,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking/test_bedrock_converse.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -42,12 +44,12 @@ THINKING_PROMPT = (
)
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -56,7 +58,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_thinking_bedrock_converse(compat_result):
def test_thinking_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -83,7 +85,7 @@ def test_thinking_bedrock_converse(compat_result):
extra_args=THINKING_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking/test_bedrock_invoke.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -42,12 +44,12 @@ THINKING_PROMPT = (
)
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -56,7 +58,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_thinking_bedrock_invoke(compat_result):
def test_thinking_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -83,7 +85,7 @@ def test_thinking_bedrock_invoke(compat_result):
extra_args=THINKING_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking/test_vertex_ai.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -42,12 +44,12 @@ THINKING_PROMPT = (
)
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -56,7 +58,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_thinking_vertex_ai(compat_result):
def test_thinking_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and assert a `thinking` content block was emitted."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -83,7 +85,7 @@ def test_thinking_vertex_ai(compat_result):
extra_args=THINKING_ARGS,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py
View file

@ -25,7 +25,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -34,6 +34,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -72,7 +74,7 @@ TOOL_USE_ARGS = [
def _has_block_type(
events: Sequence[Mapping[str, Any]],
events: Sequence[Mapping[str, JSONValue]],
block_type: str,
) -> bool:
"""Walk the stream-json events and return True if any assistant
@ -80,8 +82,8 @@ def _has_block_type(
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -90,7 +92,7 @@ def _has_block_type(
return False
def test_thinking_with_tool_use_anthropic(compat_result):
def test_thinking_with_tool_use_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
enabled and tool use, and assert both `thinking` and `tool_use`
content blocks landed in the same turn."""
@ -119,7 +121,7 @@ def test_thinking_with_tool_use_anthropic(compat_result):
extra_args=THINKING_ARGS + TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking_with_tool_use/test_azure.py
View file

@ -19,7 +19,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -28,6 +28,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -55,14 +57,14 @@ TOOL_USE_ARGS = [
def _has_block_type(
events: Sequence[Mapping[str, Any]],
events: Sequence[Mapping[str, JSONValue]],
block_type: str,
) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -71,7 +73,7 @@ def _has_block_type(
return False
def test_thinking_with_tool_use_azure(compat_result):
def test_thinking_with_tool_use_azure(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -97,7 +99,7 @@ def test_thinking_with_tool_use_azure(compat_result):
extra_args=THINKING_ARGS + TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py
View file

@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -33,6 +33,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -60,14 +62,14 @@ TOOL_USE_ARGS = [
def _has_block_type(
events: Sequence[Mapping[str, Any]],
events: Sequence[Mapping[str, JSONValue]],
block_type: str,
) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -76,7 +78,7 @@ def _has_block_type(
return False
def test_thinking_with_tool_use_bedrock_converse(compat_result):
def test_thinking_with_tool_use_bedrock_converse(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -102,7 +104,7 @@ def test_thinking_with_tool_use_bedrock_converse(compat_result):
extra_args=THINKING_ARGS + TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py
View file

@ -26,7 +26,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -35,6 +35,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -62,14 +64,14 @@ TOOL_USE_ARGS = [
def _has_block_type(
events: Sequence[Mapping[str, Any]],
events: Sequence[Mapping[str, JSONValue]],
block_type: str,
) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -78,7 +80,7 @@ def _has_block_type(
return False
def test_thinking_with_tool_use_bedrock_invoke(compat_result):
def test_thinking_with_tool_use_bedrock_invoke(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -104,7 +106,7 @@ def test_thinking_with_tool_use_bedrock_invoke(compat_result):
extra_args=THINKING_ARGS + TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py
View file

@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -33,6 +33,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -60,14 +62,14 @@ TOOL_USE_ARGS = [
def _has_block_type(
events: Sequence[Mapping[str, Any]],
events: Sequence[Mapping[str, JSONValue]],
block_type: str,
) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -76,7 +78,7 @@ def _has_block_type(
return False
def test_thinking_with_tool_use_vertex_ai(compat_result):
def test_thinking_with_tool_use_vertex_ai(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -102,7 +104,7 @@ def test_thinking_with_tool_use_vertex_ai(compat_result):
extra_args=THINKING_ARGS + TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

5
tests/e2e/claude_code/tool_search/test_anthropic.py
View file

@ -47,6 +47,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_tool_search_shape,
probe_tool_search,
@ -62,7 +63,7 @@ ANTHROPIC_MODELS = [
]
def test_tool_search_anthropic(compat_result):
def test_tool_search_anthropic(compat_result: CompatResult) -> None:
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Anthropic
tier."""
@ -83,7 +84,7 @@ def test_tool_search_anthropic(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
result = probe_tool_search(base_url=base_url, api_key=api_key, model=model)
shape_error = assert_tool_search_shape(result)

5
tests/e2e/claude_code/tool_search/test_azure.py
View file

@ -47,6 +47,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_tool_search_shape,
probe_tool_search,
@ -62,7 +63,7 @@ AZURE_MODELS = [
]
def test_tool_search_azure(compat_result):
def test_tool_search_azure(compat_result: CompatResult) -> None:
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry)
tier."""
@ -83,7 +84,7 @@ def test_tool_search_azure(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
result = probe_tool_search(base_url=base_url, api_key=api_key, model=model)
shape_error = assert_tool_search_shape(result)

5
tests/e2e/claude_code/tool_search/test_bedrock_converse.py
View file

@ -47,6 +47,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_tool_search_shape,
probe_tool_search,
@ -62,7 +63,7 @@ BEDROCK_CONVERSE_MODELS = [
]
def test_tool_search_bedrock_converse(compat_result):
def test_tool_search_bedrock_converse(compat_result: CompatResult) -> None:
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Bedrock (Converse)
tier."""
@ -83,7 +84,7 @@ def test_tool_search_bedrock_converse(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
result = probe_tool_search(base_url=base_url, api_key=api_key, model=model)
shape_error = assert_tool_search_shape(result)

5
tests/e2e/claude_code/tool_search/test_bedrock_invoke.py
View file

@ -47,6 +47,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_tool_search_shape,
probe_tool_search,
@ -62,7 +63,7 @@ BEDROCK_INVOKE_MODELS = [
]
def test_tool_search_bedrock_invoke(compat_result):
def test_tool_search_bedrock_invoke(compat_result: CompatResult) -> None:
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Bedrock (Invoke)
tier."""
@ -83,7 +84,7 @@ def test_tool_search_bedrock_invoke(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
result = probe_tool_search(base_url=base_url, api_key=api_key, model=model)
shape_error = assert_tool_search_shape(result)

5
tests/e2e/claude_code/tool_search/test_vertex_ai.py
View file

@ -47,6 +47,7 @@ import os
import pytest
from claude_code.conftest import CompatResult
from claude_code.http_probe import (
assert_tool_search_shape,
probe_tool_search,
@ -62,7 +63,7 @@ VERTEX_AI_MODELS = [
]
def test_tool_search_vertex_ai(compat_result):
def test_tool_search_vertex_ai(compat_result: CompatResult) -> None:
"""Probe `/v1/messages` with a `tool_search_tool_regex_20251119`
tool and assert the proxy + upstream accept it for every Vertex AI
tier."""
@ -83,7 +84,7 @@ def test_tool_search_vertex_ai(compat_result):
pytrace=False,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
result = probe_tool_search(base_url=base_url, api_key=api_key, model=model)
shape_error = assert_tool_search_shape(result)

14
tests/e2e/claude_code/tool_use/test_anthropic.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -57,14 +59,14 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` content block."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -73,7 +75,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_tool_use_anthropic(compat_result):
def test_tool_use_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -100,7 +102,7 @@ def test_tool_use_anthropic(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/tool_use/test_azure.py
View file

@ -20,7 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -29,6 +29,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -53,12 +55,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -67,7 +69,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_tool_use_azure(compat_result):
def test_tool_use_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -94,7 +96,7 @@ def test_tool_use_azure(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/tool_use/test_bedrock_converse.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -49,12 +51,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -63,7 +65,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_tool_use_bedrock_converse(compat_result):
def test_tool_use_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -90,7 +92,7 @@ def test_tool_use_bedrock_converse(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/tool_use/test_bedrock_invoke.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -49,12 +51,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -63,7 +65,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_tool_use_bedrock_invoke(compat_result):
def test_tool_use_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -90,7 +92,7 @@ def test_tool_use_bedrock_invoke(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/tool_use/test_vertex_ai.py
View file

@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -25,6 +25,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -49,12 +51,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -63,7 +65,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_tool_use_vertex_ai(compat_result):
def test_tool_use_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
tool call was emitted on the wire."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -90,7 +92,7 @@ def test_tool_use_vertex_ai(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

20
tests/e2e/claude_code/tool_use_streaming/test_anthropic.py
View file

@ -26,7 +26,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -35,6 +35,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,14 +67,14 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` content block."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -81,7 +83,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int:
"""Count `input_json_delta` records among the `stream_event`
entries. Zero means the proxy collapsed the streamed tool input
into a single complete block instead of forwarding the incremental
@ -94,12 +96,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
for inner in inner_events
if isinstance(inner, Mapping)
and inner.get("type") == "content_block_delta"
and isinstance(inner.get("delta"), Mapping)
and inner["delta"].get("type") == "input_json_delta"
and isinstance(delta := inner.get("delta"), Mapping)
and delta.get("type") == "input_json_delta"
)
def test_tool_use_streaming_anthropic(compat_result):
def test_tool_use_streaming_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
proxy preserves fine-grained tool streaming end-to-end."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -126,7 +128,7 @@ def test_tool_use_streaming_anthropic(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

20
tests/e2e/claude_code/tool_use_streaming/test_azure.py
View file

@ -18,7 +18,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -27,6 +27,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -52,12 +54,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -66,7 +68,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int:
"""Count `input_json_delta` records among the `stream_event`
entries. Zero means the proxy collapsed the streamed tool input
into a single complete block instead of forwarding the incremental
@ -79,12 +81,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
for inner in inner_events
if isinstance(inner, Mapping)
and inner.get("type") == "content_block_delta"
and isinstance(inner.get("delta"), Mapping)
and inner["delta"].get("type") == "input_json_delta"
and isinstance(delta := inner.get("delta"), Mapping)
and delta.get("type") == "input_json_delta"
)
def test_tool_use_streaming_azure(compat_result):
def test_tool_use_streaming_azure(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -109,7 +111,7 @@ def test_tool_use_streaming_azure(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

20
tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py
View file

@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -33,6 +33,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -58,12 +60,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -72,7 +74,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int:
"""Count `input_json_delta` records among the `stream_event`
entries. Zero means the proxy collapsed the streamed tool input
into a single complete block instead of forwarding the incremental
@ -85,12 +87,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
for inner in inner_events
if isinstance(inner, Mapping)
and inner.get("type") == "content_block_delta"
and isinstance(inner.get("delta"), Mapping)
and inner["delta"].get("type") == "input_json_delta"
and isinstance(delta := inner.get("delta"), Mapping)
and delta.get("type") == "input_json_delta"
)
def test_tool_use_streaming_bedrock_converse(compat_result):
def test_tool_use_streaming_bedrock_converse(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -115,7 +117,7 @@ def test_tool_use_streaming_bedrock_converse(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

20
tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py
View file

@ -22,7 +22,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -31,6 +31,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -56,12 +58,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -70,7 +72,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int:
"""Count `input_json_delta` records among the `stream_event`
entries. Zero means the proxy collapsed the streamed tool input
into a single complete block instead of forwarding the incremental
@ -83,12 +85,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
for inner in inner_events
if isinstance(inner, Mapping)
and inner.get("type") == "content_block_delta"
and isinstance(inner.get("delta"), Mapping)
and inner["delta"].get("type") == "input_json_delta"
and isinstance(delta := inner.get("delta"), Mapping)
and delta.get("type") == "input_json_delta"
)
def test_tool_use_streaming_bedrock_invoke(compat_result):
def test_tool_use_streaming_bedrock_invoke(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -113,7 +115,7 @@ def test_tool_use_streaming_bedrock_invoke(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

20
tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py
View file

@ -21,7 +21,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -30,6 +30,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -55,12 +57,12 @@ TOOL_USE_ARGS = [
]
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool:
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -69,7 +71,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int:
"""Count `input_json_delta` records among the `stream_event`
entries. Zero means the proxy collapsed the streamed tool input
into a single complete block instead of forwarding the incremental
@ -82,12 +84,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int:
for inner in inner_events
if isinstance(inner, Mapping)
and inner.get("type") == "content_block_delta"
and isinstance(inner.get("delta"), Mapping)
and inner["delta"].get("type") == "input_json_delta"
and isinstance(delta := inner.get("delta"), Mapping)
and delta.get("type") == "input_json_delta"
)
def test_tool_use_streaming_vertex_ai(compat_result):
def test_tool_use_streaming_vertex_ai(compat_result: CompatResult) -> None:
base_url = os.environ.get(PROXY_BASE_URL_ENV)
api_key = os.environ.get(PROXY_API_KEY_ENV)
if not base_url or not api_key:
@ -112,7 +114,7 @@ def test_tool_use_streaming_vertex_ai(compat_result):
extra_args=TOOL_USE_ARGS,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

5
tests/e2e/claude_code/vision/test_anthropic.py
View file

@ -34,6 +34,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -85,7 +86,7 @@ def _build_stdin_input() -> str:
return json.dumps(user_event) + "\n"
def test_vision_anthropic(compat_result):
def test_vision_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -115,7 +116,7 @@ def test_vision_anthropic(compat_result):
stdin_input=_build_stdin_input(),
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

5
tests/e2e/claude_code/vision/test_azure.py
View file

@ -34,6 +34,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -85,7 +86,7 @@ def _build_stdin_input() -> str:
return json.dumps(user_event) + "\n"
def test_vision_azure(compat_result):
def test_vision_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -115,7 +116,7 @@ def test_vision_azure(compat_result):
stdin_input=_build_stdin_input(),
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

5
tests/e2e/claude_code/vision/test_bedrock_converse.py
View file

@ -34,6 +34,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -85,7 +86,7 @@ def _build_stdin_input() -> str:
return json.dumps(user_event) + "\n"
def test_vision_bedrock_converse(compat_result):
def test_vision_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -115,7 +116,7 @@ def test_vision_bedrock_converse(compat_result):
stdin_input=_build_stdin_input(),
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

5
tests/e2e/claude_code/vision/test_bedrock_invoke.py
View file

@ -34,6 +34,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -85,7 +86,7 @@ def _build_stdin_input() -> str:
return json.dumps(user_event) + "\n"
def test_vision_bedrock_invoke(compat_result):
def test_vision_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -115,7 +116,7 @@ def test_vision_bedrock_invoke(compat_result):
stdin_input=_build_stdin_input(),
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

5
tests/e2e/claude_code/vision/test_vertex_ai.py
View file

@ -34,6 +34,7 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -85,7 +86,7 @@ def _build_stdin_input() -> str:
return json.dumps(user_event) + "\n"
def test_vision_vertex_ai(compat_result):
def test_vision_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy with an image
attached via stream-json input and assert a non-empty reply."""
base_url = os.environ.get(PROXY_BASE_URL_ENV)
@ -115,7 +116,7 @@ def test_vision_vertex_ai(compat_result):
stdin_input=_build_stdin_input(),
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/web_search/test_anthropic.py
View file

@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -37,6 +37,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
WEB_SEARCH_TOOL_NAME = "WebSearch"
def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` block calling `WebSearch`."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_web_search_anthropic(compat_result):
def test_web_search_anthropic(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving
the proxy preserved both the request-side tool definition and the
@ -115,7 +117,7 @@ def test_web_search_anthropic(compat_result):
extra_args=WEB_SEARCH_ARGS,
)
failures = []
failures: list[str] = []
for model in ANTHROPIC_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/web_search/test_azure.py
View file

@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -37,6 +37,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
WEB_SEARCH_TOOL_NAME = "WebSearch"
def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` block calling `WebSearch`."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_web_search_azure(compat_result):
def test_web_search_azure(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving
the proxy preserved both the request-side tool definition and the
@ -115,7 +117,7 @@ def test_web_search_azure(compat_result):
extra_args=WEB_SEARCH_ARGS,
)
failures = []
failures: list[str] = []
for model in AZURE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/web_search/test_bedrock_converse.py
View file

@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -37,6 +37,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
WEB_SEARCH_TOOL_NAME = "WebSearch"
def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` block calling `WebSearch`."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_web_search_bedrock_converse(compat_result):
def test_web_search_bedrock_converse(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving
the proxy preserved both the request-side tool definition and the
@ -115,7 +117,7 @@ def test_web_search_bedrock_converse(compat_result):
extra_args=WEB_SEARCH_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_CONVERSE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/web_search/test_bedrock_invoke.py
View file

@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -37,6 +37,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
WEB_SEARCH_TOOL_NAME = "WebSearch"
def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` block calling `WebSearch`."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_web_search_bedrock_invoke(compat_result):
def test_web_search_bedrock_invoke(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving
the proxy preserved both the request-side tool definition and the
@ -115,7 +117,7 @@ def test_web_search_bedrock_invoke(compat_result):
extra_args=WEB_SEARCH_ARGS,
)
failures = []
failures: list[str] = []
for model in BEDROCK_INVOKE_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):

14
tests/e2e/claude_code/web_search/test_vertex_ai.py
View file

@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by
from __future__ import annotations
import os
from typing import Any, Mapping, Sequence
from typing import Mapping, Sequence
import pytest
@ -37,6 +37,8 @@ from claude_code.cli_driver import (
failure_diagnostic,
run_claude_models_parallel,
)
from claude_code.conftest import CompatResult
from claude_code.json_types import JSONValue
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"]
WEB_SEARCH_TOOL_NAME = "WebSearch"
def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool:
"""Walk the stream-json events and return True if any assistant
message included a `tool_use` block calling `WebSearch`."""
for event in events:
if event.get("type") != "assistant":
continue
message = event.get("message") or {}
content = message.get("content")
message = event.get("message")
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for block in content:
@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool:
return False
def test_web_search_vertex_ai(compat_result):
def test_web_search_vertex_ai(compat_result: CompatResult) -> None:
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
upstream emitted a `tool_use` block calling `WebSearch`, proving
the proxy preserved both the request-side tool definition and the
@ -115,7 +117,7 @@ def test_web_search_vertex_ai(compat_result):
extra_args=WEB_SEARCH_ARGS,
)
failures = []
failures: list[str] = []
for model in VERTEX_AI_MODELS:
outcome = outcomes[model]
if isinstance(outcome, ClaudeCLIError):