diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py index 9ddbdd29846..aef7434290f 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -15,6 +15,7 @@ from pathlib import Path import pytest +from claude_code.json_types import JSON_OBJECT_ADAPTER, JSONValue from claude_code.matrix_builder import ( ManifestError, ResultsError, @@ -27,7 +28,22 @@ from claude_code.matrix_builder import ( FIXTURES = Path(__file__).parent / "fixtures" -def test_build_matrix_matches_golden_file(tmp_path): +def _as_dict(value: JSONValue) -> dict[str, JSONValue]: + assert isinstance(value, dict) + return value + + +def _as_list(value: JSONValue) -> list[JSONValue]: + assert isinstance(value, list) + return value + + +def _as_str(value: JSONValue) -> str: + assert isinstance(value, str) + return value + + +def test_build_matrix_matches_golden_file(tmp_path: Path) -> None: manifest = load_manifest(FIXTURES / "manifest.yaml") results = load_results(FIXTURES / "results.json") matrix = build_matrix( @@ -37,18 +53,18 @@ def test_build_matrix_matches_golden_file(tmp_path): claude_code_version="2.1.120", generated_at="2026-04-25T00:00:00Z", ) - expected = json.loads((FIXTURES / "expected_matrix.json").read_text()) + expected = JSON_OBJECT_ADAPTER.validate_python(json.loads((FIXTURES / "expected_matrix.json").read_text())) assert matrix == expected -def test_build_matrix_pass_requires_all_models_pass(): +def test_build_matrix_pass_requires_all_models_pass() -> None: """Multiple results in one cell must all be pass for the cell to be pass.""" - manifest = { + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, @@ -60,16 +76,16 @@ def test_build_matrix_pass_requires_all_models_pass(): claude_code_version="c", generated_at="t", ) - assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "pass"} -def test_build_matrix_any_fail_makes_cell_fail(): - manifest = { +def test_build_matrix_any_fail_makes_cell_fail() -> None: + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, { "feature_id": "f", @@ -85,22 +101,22 @@ def test_build_matrix_any_fail_makes_cell_fail(): claude_code_version="c", generated_at="t", ) - cell = matrix["features"][0]["providers"]["anthropic"] + cell = _as_dict(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"]) assert cell["status"] == "fail" assert cell["error"] == "[claude-opus-4-7] timeout" -def test_build_matrix_joins_all_failure_errors_in_one_cell(): +def test_build_matrix_joins_all_failure_errors_in_one_cell() -> None: """When multiple tiers fail for different reasons within the same cell, every failure's error must appear in the published cell so triage isn't reduced to a single tier's diagnostic. """ - manifest = { + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ { "feature_id": "f", "provider": "anthropic", @@ -120,25 +136,25 @@ def test_build_matrix_joins_all_failure_errors_in_one_cell(): claude_code_version="c", generated_at="t", ) - cell = matrix["features"][0]["providers"]["anthropic"] + cell = _as_dict(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"]) assert cell["status"] == "fail" - assert "[claude-haiku-4-5] 429" in cell["error"] - assert "[claude-opus-4-7] timeout" in cell["error"] + assert "[claude-haiku-4-5] 429" in _as_str(cell["error"]) + assert "[claude-opus-4-7] timeout" in _as_str(cell["error"]) -def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass(): +def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass() -> None: """A `not_tested` row mixed with `pass` rows must not silently demote the cell to `not_tested` — `not_tested` is "absent data", not a negative signal. Otherwise a partial crash mid-test, or a test that explicitly recorded "tier didn't run", would discard real passing results from the published cell. """ - manifest = { + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, { "feature_id": "f", @@ -154,20 +170,20 @@ def test_build_matrix_mixed_pass_and_not_tested_surfaces_pass(): claude_code_version="c", generated_at="t", ) - assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "pass"} -def test_build_matrix_all_not_tested_stays_not_tested(): +def test_build_matrix_all_not_tested_stays_not_tested() -> None: """A cell whose every row is `not_tested` (or empty) must remain `not_tested` — the absent-data rule only drops `not_tested` rows when there's other signal to surface. """ - manifest = { + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ { "feature_id": "f", "provider": "anthropic", @@ -186,22 +202,22 @@ def test_build_matrix_all_not_tested_stays_not_tested(): claude_code_version="c", generated_at="t", ) - assert matrix["features"][0]["providers"]["anthropic"] == {"status": "not_tested"} + assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "not_tested"} -def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass(): +def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass() -> None: """A `not_applicable` row mixed with `pass` rows must surface as `pass`, not `not_applicable`. The published cell answers "does this feature work on this provider?"; if any tier passes, the feature works there. Discarding passing tiers because one tier is NA would misrepresent the cell as unsupported. """ - manifest = { + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, { "feature_id": "f", @@ -220,20 +236,20 @@ def test_build_matrix_mixed_pass_and_not_applicable_surfaces_pass(): claude_code_version="c", generated_at="t", ) - assert matrix["features"][0]["providers"]["anthropic"] == {"status": "pass"} + assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == {"status": "pass"} -def test_build_matrix_all_not_applicable_stays_not_applicable(): +def test_build_matrix_all_not_applicable_stays_not_applicable() -> None: """When every observed row is `not_applicable`, the cell remains `not_applicable` and the first row's reason carries through to the published matrix. """ - manifest = { + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ { "feature_id": "f", "provider": "anthropic", @@ -255,19 +271,19 @@ def test_build_matrix_all_not_applicable_stays_not_applicable(): claude_code_version="c", generated_at="t", ) - assert matrix["features"][0]["providers"]["anthropic"] == { + assert _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["anthropic"] == { "status": "not_applicable", "reason": "feature unsupported on this provider", } -def test_build_matrix_fills_not_tested_for_missing_cells(): - manifest = { +def test_build_matrix_fills_not_tested_for_missing_cells() -> None: + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic", "azure"], "features": [{"id": "f", "name": "F"}], } - results = [ + results: list[JSONValue] = [ {"feature_id": "f", "provider": "anthropic", "result": {"status": "pass"}}, ] matrix = build_matrix( @@ -277,13 +293,13 @@ def test_build_matrix_fills_not_tested_for_missing_cells(): claude_code_version="c", generated_at="t", ) - cells = matrix["features"][0]["providers"] + cells = _as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"]) assert cells["anthropic"] == {"status": "pass"} assert cells["azure"] == {"status": "not_tested"} -def test_build_matrix_preserves_provider_and_feature_order(): - manifest = { +def test_build_matrix_preserves_provider_and_feature_order() -> None: + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["azure", "anthropic", "vertex_ai"], "features": [ @@ -299,16 +315,16 @@ def test_build_matrix_preserves_provider_and_feature_order(): generated_at="t", ) assert matrix["providers"] == ["azure", "anthropic", "vertex_ai"] - assert [f["id"] for f in matrix["features"]] == ["z", "a"] - assert list(matrix["features"][0]["providers"].keys()) == [ + assert [_as_dict(f)["id"] for f in _as_list(matrix["features"])] == ["z", "a"] + assert list(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"]).keys()) == [ "azure", "anthropic", "vertex_ai", ] -def test_build_matrix_emits_schema_version_one(): - manifest = { +def test_build_matrix_emits_schema_version_one() -> None: + manifest: dict[str, JSONValue] = { "schema_version": "1", "providers": ["anthropic"], "features": [{"id": "f", "name": "F"}], @@ -323,7 +339,7 @@ def test_build_matrix_emits_schema_version_one(): assert matrix["schema_version"] == "1" -def test_load_manifest_rejects_wrong_schema_version(tmp_path): +def test_load_manifest_rejects_wrong_schema_version(tmp_path: Path) -> None: bad = tmp_path / "manifest.yaml" bad.write_text( 'schema_version: "2"\nproviders: [anthropic]\nfeatures:\n - id: f\n name: F\n' @@ -332,21 +348,21 @@ def test_load_manifest_rejects_wrong_schema_version(tmp_path): load_manifest(bad) -def test_load_manifest_rejects_empty_features(tmp_path): +def test_load_manifest_rejects_empty_features(tmp_path: Path) -> None: bad = tmp_path / "manifest.yaml" bad.write_text('schema_version: "1"\nproviders: [anthropic]\nfeatures: []\n') with pytest.raises(ManifestError): load_manifest(bad) -def test_load_results_rejects_missing_results_key(tmp_path): +def test_load_results_rejects_missing_results_key(tmp_path: Path) -> None: bad = tmp_path / "results.json" bad.write_text(json.dumps({"schema_version": "1"})) with pytest.raises(ResultsError): load_results(bad) -def test_build_matrix_6x5_grid_matches_published_sample(): +def test_build_matrix_6x5_grid_matches_published_sample() -> None: """Slice 5 acceptance: feeding the per-model results the full v0 row set produces reproduces the hand-authored 6x5 sample that the docs page renders. @@ -386,16 +402,16 @@ def test_build_matrix_6x5_grid_matches_published_sample(): ] v0_features = [ feature - for feature in full_manifest["features"] - if feature["id"] in v0_feature_ids + for feature in _as_list(full_manifest["features"]) + if _as_dict(feature)["id"] in v0_feature_ids ] - manifest = {**full_manifest, "features": v0_features} + manifest: dict[str, JSONValue] = {**full_manifest, "features": v0_features} - feature_ids = [feature["id"] for feature in manifest["features"]] - providers = manifest["providers"] + feature_ids = [_as_str(_as_dict(feature)["id"]) for feature in _as_list(manifest["features"])] + providers = [_as_str(provider) for provider in _as_list(manifest["providers"])] models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"] - results = [] + results: list[JSONValue] = [] for feature_id in feature_ids: for provider in providers: for model in models: @@ -418,18 +434,20 @@ def test_build_matrix_6x5_grid_matches_published_sample(): claude_code_version="2.1.120", generated_at="2026-04-25T00:00:00Z", ) - expected = json.loads((repo_root / "sample_compatibility-matrix.json").read_text()) + expected = JSON_OBJECT_ADAPTER.validate_python( + json.loads((repo_root / "sample_compatibility-matrix.json").read_text()) + ) assert matrix == expected -def test_build_matrix_1x5_grid_one_failing_model_breaks_cell(): +def test_build_matrix_1x5_grid_one_failing_model_breaks_cell() -> None: """If even one of three models fails on a provider, that cell is fail and the error string carries the failing model id so the docs tooltip can name the outlier.""" repo_root = Path(__file__).resolve().parents[1] manifest = load_manifest(repo_root / "manifest.yaml") - results = [ + results: list[JSONValue] = [ { "feature_id": "basic_messaging_non_streaming", "provider": "bedrock_invoke", @@ -457,12 +475,12 @@ def test_build_matrix_1x5_grid_one_failing_model_breaks_cell(): claude_code_version="c", generated_at="t", ) - cell = matrix["features"][0]["providers"]["bedrock_invoke"] + cell = _as_dict(_as_dict(_as_dict(_as_list(matrix["features"])[0])["providers"])["bedrock_invoke"]) assert cell["status"] == "fail" - assert "claude-opus-4-7-bedrock-invoke" in cell["error"] + assert "claude-opus-4-7-bedrock-invoke" in _as_str(cell["error"]) -def test_build_from_paths_writes_output(tmp_path): +def test_build_from_paths_writes_output(tmp_path: Path) -> None: out = tmp_path / "compatibility-matrix.json" matrix = build_from_paths( manifest_path=FIXTURES / "manifest.yaml", @@ -473,7 +491,7 @@ def test_build_from_paths_writes_output(tmp_path): output_path=out, ) assert out.exists() - on_disk = json.loads(out.read_text()) + on_disk = JSON_OBJECT_ADAPTER.validate_python(json.loads(out.read_text())) assert on_disk == matrix - expected = json.loads((FIXTURES / "expected_matrix.json").read_text()) + expected = JSON_OBJECT_ADAPTER.validate_python(json.loads((FIXTURES / "expected_matrix.json").read_text())) assert on_disk == expected diff --git a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py index b1745008fac..83d0f2999a4 100644 --- a/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py +++ b/tests/e2e/claude_code/_builder_unit_tests/test_v0_layout.py @@ -16,9 +16,26 @@ from pathlib import Path import pytest import yaml +from claude_code.json_types import JSON_OBJECT_ADAPTER, JSONValue + REPO_ROOT = Path(__file__).resolve().parents[1] MANIFEST_PATH = REPO_ROOT / "manifest.yaml" + +def _as_dict(value: JSONValue) -> dict[str, JSONValue]: + assert isinstance(value, dict) + return value + + +def _as_list(value: JSONValue) -> list[JSONValue]: + assert isinstance(value, list) + return value + + +def _as_str(value: JSONValue) -> str: + assert isinstance(value, str) + return value + # The PRD's "Features in v0" section, in row order. EXPECTED_FEATURE_IDS = [ "basic_messaging_non_streaming", @@ -55,8 +72,10 @@ def _all_manifest_feature_ids() -> list[str]: constants above only validate the original six rows by design. """ return [ - feature["id"] - for feature in yaml.safe_load(MANIFEST_PATH.read_text())["features"] + _as_str(_as_dict(feature)["id"]) + for feature in _as_list( + JSON_OBJECT_ADAPTER.validate_python(yaml.safe_load(MANIFEST_PATH.read_text()))["features"] + ) ] @@ -64,45 +83,46 @@ ALL_FEATURE_IDS = _all_manifest_feature_ids() @pytest.fixture(scope="module") -def manifest() -> dict: - return yaml.safe_load(MANIFEST_PATH.read_text()) +def manifest() -> dict[str, JSONValue]: + return JSON_OBJECT_ADAPTER.validate_python(yaml.safe_load(MANIFEST_PATH.read_text())) -def test_manifest_lists_all_six_v0_features_in_order(manifest): +def test_manifest_lists_all_six_v0_features_in_order(manifest: dict[str, JSONValue]) -> None: """The PRD's v0 row set must appear at the top of the manifest in order. Features beyond v0 (extensions added after the matrix shipped) are allowed but must not reorder or displace the v0 rows — the docs page anchors row links by index, so v0 stays pinned at positions [0:6] for the lifetime of the schema. """ - ids = [feature["id"] for feature in manifest["features"]] + ids = [_as_dict(feature)["id"] for feature in _as_list(manifest["features"])] assert ids[: len(EXPECTED_FEATURE_IDS)] == EXPECTED_FEATURE_IDS -def test_manifest_lists_all_five_v0_providers_in_order(manifest): +def test_manifest_lists_all_five_v0_providers_in_order(manifest: dict[str, JSONValue]) -> None: assert manifest["providers"] == EXPECTED_PROVIDERS -def test_manifest_every_feature_has_human_readable_name(manifest): - for feature in manifest["features"]: - assert isinstance(feature["name"], str) and feature["name"].strip() +def test_manifest_every_feature_has_human_readable_name(manifest: dict[str, JSONValue]) -> None: + for feature in _as_list(manifest["features"]): + name = _as_dict(feature)["name"] + assert isinstance(name, str) and name.strip() @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) -def test_feature_directory_exists(feature_id): +def test_feature_directory_exists(feature_id: str) -> None: feature_dir = REPO_ROOT / feature_id assert feature_dir.is_dir(), f"missing feature directory: {feature_dir}" @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) @pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) -def test_per_provider_test_file_exists(feature_id, provider): +def test_per_provider_test_file_exists(feature_id: str, provider: str) -> None: test_file = REPO_ROOT / feature_id / f"test_{provider}.py" assert test_file.is_file(), f"missing per-provider test file: {test_file}" @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) -def test_feature_directory_has_init_file(feature_id): +def test_feature_directory_has_init_file(feature_id: str) -> None: """Each feature directory needs an __init__.py so pytest collects the per-provider test files as a package — matches the layout established by `basic_messaging_non_streaming/`.""" @@ -116,7 +136,7 @@ def test_feature_directory_has_init_file(feature_id): # extend the same structural guarantees to any row added afterward so # a broken post-v0 directory still fails CI. @pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS) -def test_every_manifest_feature_has_directory(feature_id): +def test_every_manifest_feature_has_directory(feature_id: str) -> None: feature_dir = REPO_ROOT / feature_id assert feature_dir.is_dir(), ( f"manifest declares {feature_id!r} but {feature_dir} is missing — " @@ -125,14 +145,14 @@ def test_every_manifest_feature_has_directory(feature_id): @pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS) -def test_every_manifest_feature_has_init_file(feature_id): +def test_every_manifest_feature_has_init_file(feature_id: str) -> None: init_file = REPO_ROOT / feature_id / "__init__.py" assert init_file.is_file(), f"missing __init__.py: {init_file}" @pytest.mark.parametrize("feature_id", ALL_FEATURE_IDS) @pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) -def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider): +def test_every_manifest_feature_has_per_provider_test_file(feature_id: str, provider: str) -> None: """Every (feature, provider) cell in the rendered matrix must be backed by a per-provider test file. Without this check, a missing file silently becomes a `not_tested` cell in the published matrix @@ -144,8 +164,8 @@ def test_every_manifest_feature_has_per_provider_test_file(feature_id, provider) @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) @pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) def test_per_provider_test_file_imports_and_parametrizes_three_models( - feature_id, provider -): + feature_id: str, provider: str +) -> None: """Every test file must reference the three Claude tiers required by the PRD: Haiku 4.5, Sonnet 4.6, Opus 4.7. Implementations may use plain aliases or per-provider-suffixed aliases (e.g. @@ -159,7 +179,7 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models( @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) -def test_azure_test_file_drives_the_proxy(feature_id): +def test_azure_test_file_drives_the_proxy(feature_id: str) -> None: """Azure (Microsoft Foundry) hosts Anthropic Claude as of 2025-11-18, so every Azure cell in the v0 matrix exercises a real route through the LiteLLM proxy — same shape as the other provider columns. Pin diff --git a/tests/e2e/claude_code/_driver_unit_tests/conftest.py b/tests/e2e/claude_code/_driver_unit_tests/conftest.py index bfeaa57c736..f9c5ad55da2 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/conftest.py +++ b/tests/e2e/claude_code/_driver_unit_tests/conftest.py @@ -14,6 +14,9 @@ limiter was added. from __future__ import annotations +from pathlib import Path +from typing import Generator + import pytest from claude_code.rate_limiter import ( @@ -25,7 +28,9 @@ from claude_code.rate_limiter import ( @pytest.fixture(autouse=True) -def _hermetic_rate_limiter(tmp_path): +def _hermetic_rate_limiter( # pyright: ignore[reportUnusedFunction] # autouse fixture, invoked by pytest + tmp_path: Path, +) -> Generator[None]: config = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS} limiter = RateLimiter(config=config, state_dir=tmp_path) with use_limiter(limiter): diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py index 018121a8e5c..aefe1865b41 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_basic_messaging.py @@ -9,21 +9,22 @@ upstream stream must turn the cell red, not green. from __future__ import annotations -import os -from typing import Any, Dict, List, Mapping, Optional, Sequence +from typing import Mapping, Sequence, TypedDict import pytest from claude_code import _basic_messaging from claude_code._basic_messaging import ( MIN_STREAM_DELTA_EVENTS, - _count_stream_event_deltas, + _count_stream_event_deltas, # pyright: ignore[reportPrivateUsage] # unit tests exercise the private helper run_basic_messaging_cell, ) -from claude_code.cli_driver import DriverResult +from claude_code.cli_driver import DriverResult, ModelResult +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONObject, JSONValue -class _FakeResult: +class _FakeResult(CompatResult): """Stand-in for the test's `compat_result` fixture. Records every `set` / `add` payload so assertions can inspect what @@ -32,24 +33,25 @@ class _FakeResult: """ def __init__(self) -> None: - self.rows: List[Dict[str, Any]] = [] - self.single: Optional[Dict[str, Any]] = None + super().__init__() + self.rows: list[dict[str, JSONValue]] = [] + self.single: dict[str, JSONValue] | None = None - def set(self, payload: Mapping[str, Any]) -> None: - self.single = dict(payload) + def set(self, result: Mapping[str, JSONValue]) -> None: + self.single = dict(result) - def add(self, payload: Mapping[str, Any]) -> None: - self.rows.append(dict(payload)) + def add(self, result: Mapping[str, JSONValue]) -> None: + self.rows.append(dict(result)) -def _streamed_events(n_deltas: int = 5) -> List[Dict[str, Any]]: +def _streamed_events(n_deltas: int = 5) -> list[JSONObject]: """Build a stream-json event list that *looks* streamed. Includes `n_deltas` `stream_event` records (matching what `--include-partial-messages` produces) plus the usual `system`/`assistant`/`result` boilerplate the CLI always emits. """ - events: List[Dict[str, Any]] = [{"type": "system", "subtype": "init"}] + events: list[JSONObject] = [{"type": "system", "subtype": "init"}] for i in range(n_deltas): events.append( { @@ -70,7 +72,7 @@ def _streamed_events(n_deltas: int = 5) -> List[Dict[str, Any]]: return events -def _buffered_events() -> List[Dict[str, Any]]: +def _buffered_events() -> list[JSONObject]: """Event list a buffering proxy would produce: zero `stream_event`s.""" return [ {"type": "system", "subtype": "init"}, @@ -82,16 +84,40 @@ def _buffered_events() -> List[Dict[str, Any]]: ] -def _install_fake_runner(monkeypatch, *, outcomes_by_model): +class _CapturedCall(TypedDict): + models: list[str] + prompt: str | None + base_url: str + api_key: str + extra_args: list[str] + + +def _install_fake_runner( + monkeypatch: pytest.MonkeyPatch, *, outcomes_by_model: Mapping[str, ModelResult] +) -> _CapturedCall: """Patch `run_claude_models_parallel` to return canned outcomes. Captures the kwargs the cell passed in so tests can assert on `extra_args` (which is how the streaming variant opts into `--include-partial-messages`). """ - captured: Dict[str, Any] = {} + captured: _CapturedCall = { + "models": [], + "prompt": None, + "base_url": "", + "api_key": "", + "extra_args": [], + } - def fake(*, models, prompt, base_url, api_key, extra_args=None, **_kwargs): + def fake( + *, + models: Sequence[str], + prompt: str | None, + base_url: str, + api_key: str, + extra_args: Sequence[str] | None = None, + **_kwargs: object, + ) -> dict[str, ModelResult]: captured["models"] = list(models) captured["prompt"] = prompt captured["base_url"] = base_url @@ -104,13 +130,15 @@ def _install_fake_runner(monkeypatch, *, outcomes_by_model): @pytest.fixture(autouse=True) -def _proxy_env(monkeypatch): +def _proxy_env( # pyright: ignore[reportUnusedFunction] # autouse fixture, invoked by pytest + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv("LITELLM_PROXY_BASE_URL", "http://localhost:4000") monkeypatch.setenv("LITELLM_PROXY_API_KEY", "sk-test") -def test_count_stream_event_deltas_only_counts_records_with_event_payload(): - events = [ +def test_count_stream_event_deltas_only_counts_records_with_event_payload() -> None: + events: list[JSONObject] = [ {"type": "system"}, {"type": "stream_event", "event": {"type": "message_start"}}, {"type": "stream_event", "event": {"type": "content_block_delta"}}, @@ -123,7 +151,7 @@ def test_count_stream_event_deltas_only_counts_records_with_event_payload(): assert _count_stream_event_deltas(events) == 2 -def test_verify_streaming_passes_when_proxy_streams(monkeypatch): +def test_verify_streaming_passes_when_proxy_streams(monkeypatch: pytest.MonkeyPatch) -> None: fake_result = _FakeResult() model = "claude-haiku-4-5" outcome = DriverResult(text="1\n2\n3", events=_streamed_events(n_deltas=5)) @@ -140,7 +168,7 @@ def test_verify_streaming_passes_when_proxy_streams(monkeypatch): assert fake_result.rows == [{"status": "pass"}] -def test_verify_streaming_fails_when_proxy_buffers(monkeypatch): +def test_verify_streaming_fails_when_proxy_buffers(monkeypatch: pytest.MonkeyPatch) -> None: fake_result = _FakeResult() model = "claude-haiku-4-5" outcome = DriverResult(text="1\n2\n3", events=_buffered_events()) @@ -157,11 +185,12 @@ def test_verify_streaming_fails_when_proxy_buffers(monkeypatch): assert len(fake_result.rows) == 1 row = fake_result.rows[0] assert row["status"] == "fail" + assert isinstance(row["error"], str) assert "stream_event" in row["error"] assert f"< {MIN_STREAM_DELTA_EVENTS}" in row["error"] -def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch): +def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch: pytest.MonkeyPatch) -> None: """Default `verify_streaming=False` keeps the non-streaming wire identical.""" fake_result = _FakeResult() model = "claude-haiku-4-5" @@ -178,11 +207,11 @@ def test_non_streaming_variant_omits_partial_messages_flag(monkeypatch): assert fake_result.rows == [{"status": "pass"}] -def test_verify_streaming_requires_all_models_to_stream(monkeypatch): +def test_verify_streaming_requires_all_models_to_stream(monkeypatch: pytest.MonkeyPatch) -> None: """If any one tier buffers, the cell fails — same all-must-pass shape as the non-streaming check.""" fake_result = _FakeResult() - outcomes = { + outcomes: dict[str, ModelResult] = { "claude-haiku-4-5": DriverResult(text="ok", events=_streamed_events(5)), "claude-sonnet-4-6": DriverResult(text="ok", events=_buffered_events()), "claude-opus-4-7": DriverResult(text="ok", events=_streamed_events(5)), diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py index 786f6029993..1ebf78b8577 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_cli_driver.py @@ -10,31 +10,47 @@ from __future__ import annotations import json import subprocess -from dataclasses import dataclass -from typing import List, Optional +from typing import Mapping, Sequence, TypedDict import pytest from claude_code.cli_driver import ( ClaudeCLIError, + CommandRunner, DriverResult, failure_diagnostic, run_claude, run_claude_models_parallel, ) +from claude_code.json_types import JSONObject -@dataclass -class _Completed: - returncode: int = 0 - stdout: str = "" - stderr: str = "" +class _Completed(subprocess.CompletedProcess[str]): + def __init__(self, returncode: int = 0, stdout: str = "", stderr: str = "") -> None: + super().__init__(args=(), returncode=returncode, stdout=stdout, stderr=stderr) -def _make_runner(*, stdout: str = "", returncode: int = 0, stderr: str = ""): - captured = {} +class _Captured(TypedDict): + cmd: Sequence[str] + env: Mapping[str, str] + timeout: float + input: str | None - def runner(cmd, env, capture_output, text, timeout, check, input=None): + +def _make_runner( + *, stdout: str = "", returncode: int = 0, stderr: str = "" +) -> tuple[CommandRunner, _Captured]: + captured: _Captured = {"cmd": (), "env": {}, "timeout": 0.0, "input": None} + + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: captured["cmd"] = cmd captured["env"] = env captured["timeout"] = timeout @@ -44,7 +60,7 @@ def _make_runner(*, stdout: str = "", returncode: int = 0, stderr: str = ""): return runner, captured -def test_run_claude_assembles_command_correctly(): +def test_run_claude_assembles_command_correctly() -> None: runner, captured = _make_runner( stdout='{"type":"assistant","message":{"content":[{"type":"text","text":"ok"}]}}\n' ) @@ -66,7 +82,7 @@ def test_run_claude_assembles_command_correctly(): assert cmd[-2:] == ["--", "hello"] -def test_run_claude_places_extra_args_before_prompt(): +def test_run_claude_places_extra_args_before_prompt() -> None: """`claude --print` expects the prompt as the final positional arg. Flags appearing after the prompt are ignored or eaten by the prompt @@ -93,7 +109,7 @@ def test_run_claude_places_extra_args_before_prompt(): assert cmd.index("--allowed-tools") < cmd.index("--") -def test_run_claude_overlays_proxy_env(): +def test_run_claude_overlays_proxy_env() -> None: runner, captured = _make_runner(stdout="") run_claude( prompt="hi", @@ -107,7 +123,7 @@ def test_run_claude_overlays_proxy_env(): assert env["ANTHROPIC_AUTH_TOKEN"] == "sk-abc" -def test_run_claude_extra_env_is_added_to_subprocess_env(): +def test_run_claude_extra_env_is_added_to_subprocess_env() -> None: """Caller-supplied extra_env entries land on the subprocess env.""" runner, captured = _make_runner(stdout="") run_claude( @@ -121,7 +137,7 @@ def test_run_claude_extra_env_is_added_to_subprocess_env(): assert captured["env"]["MAX_THINKING_TOKENS"] == "4096" -def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch): +def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch: pytest.MonkeyPatch) -> None: """Process-runtime vars (PATH) flow through; credentials don't. The `claude` CLI is a Node binary installed dynamically from npm in @@ -160,7 +176,7 @@ def test_run_claude_inherits_only_allowlisted_os_environ(monkeypatch): assert "GITHUB_TOKEN" not in env -def test_run_claude_uses_isolated_per_invocation_home(monkeypatch, tmp_path): +def test_run_claude_uses_isolated_per_invocation_home(monkeypatch: pytest.MonkeyPatch) -> None: """`claude` subprocess never sees the runtime user's real $HOME. The CLI needs *a* HOME (it caches per-session state under @@ -196,7 +212,7 @@ def test_run_claude_uses_isolated_per_invocation_home(monkeypatch, tmp_path): assert "claude-cli-home-" in env["HOME"] -def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch): +def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch: pytest.MonkeyPatch) -> None: """Two consecutive calls get two different isolated HOMEs. Reusing a single tmpdir across calls would defeat the isolation @@ -229,7 +245,7 @@ def test_run_claude_isolated_home_is_distinct_per_invocation(monkeypatch): assert home_a != home_b -def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch): +def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch: pytest.MonkeyPatch) -> None: """The per-invocation HOME tmpdir is rm-rf'd when run_claude returns. Without cleanup, a long matrix run would accumulate one tmpdir @@ -254,7 +270,9 @@ def test_run_claude_isolated_home_cleaned_up_after_run(monkeypatch): ), f"isolated HOME {isolated_home!r} should be removed after run_claude returns" -def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch): +def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: """Cleanup runs even when the CLI subprocess raises. If the CLI is missing or times out, `run_claude` raises @@ -266,9 +284,17 @@ def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch): monkeypatch.setenv("HOME", "/home/runner") - captured: dict = {} + captured: dict[str, Mapping[str, str]] = {} - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: captured["env"] = env raise subprocess.TimeoutExpired(cmd=cmd, timeout=timeout) @@ -287,7 +313,9 @@ def test_run_claude_isolated_home_cleaned_up_on_subprocess_failure(monkeypatch): ), f"isolated HOME {isolated_home!r} should be removed even on timeout" -def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(monkeypatch): +def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var( + monkeypatch: pytest.MonkeyPatch, +) -> None: """The allowlist applies to inherited os.environ; extra_env is the sanctioned way for a test to opt-in to passing something extra.""" monkeypatch.setenv("ANTHROPIC_API_KEY", "from-os") @@ -303,8 +331,8 @@ def test_run_claude_extra_env_can_pass_through_otherwise_blocked_var(monkeypatch assert captured["env"]["ANTHROPIC_API_KEY"] == "from-arg" -def test_run_claude_parses_stream_json_assistant_text(): - events = [ +def test_run_claude_parses_stream_json_assistant_text() -> None: + events: list[JSONObject] = [ {"type": "system", "session_id": "abc"}, { "type": "assistant", @@ -333,7 +361,7 @@ def test_run_claude_parses_stream_json_assistant_text(): assert result.exit_code == 0 -def test_run_claude_handles_string_message_content(): +def test_run_claude_handles_string_message_content() -> None: """Some CLI versions emit `message.content` as a plain string.""" stdout = ( json.dumps({"type": "assistant", "message": {"content": "bare text"}}) + "\n" @@ -349,7 +377,7 @@ def test_run_claude_handles_string_message_content(): assert result.text == "bare text" -def test_run_claude_skips_malformed_lines(): +def test_run_claude_skips_malformed_lines() -> None: stdout = ( "not-json\n" + json.dumps( @@ -373,7 +401,7 @@ def test_run_claude_skips_malformed_lines(): assert len(result.events) == 1 -def test_run_claude_propagates_nonzero_exit_code(): +def test_run_claude_propagates_nonzero_exit_code() -> None: runner, _ = _make_runner(stdout="", returncode=2, stderr="auth failed") result = run_claude( prompt="hi", @@ -387,8 +415,8 @@ def test_run_claude_propagates_nonzero_exit_code(): assert result.text == "" -def test_run_claude_raises_on_missing_cli(): - def runner(*args, **kwargs): +def test_run_claude_raises_on_missing_cli() -> None: + def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: raise FileNotFoundError(2, "no such file", "claude") with pytest.raises(ClaudeCLIError, match="claude CLI not found"): @@ -401,8 +429,8 @@ def test_run_claude_raises_on_missing_cli(): ) -def test_run_claude_raises_on_timeout(): - def runner(*args, **kwargs): +def test_run_claude_raises_on_timeout() -> None: + def runner(*_args: object, **_kwargs: object) -> subprocess.CompletedProcess[str]: raise subprocess.TimeoutExpired(cmd="claude", timeout=1) with pytest.raises(ClaudeCLIError, match="timed out"): @@ -416,7 +444,7 @@ def test_run_claude_raises_on_timeout(): ) -def test_run_claude_validates_required_params(): +def test_run_claude_validates_required_params() -> None: runner, _ = _make_runner() with pytest.raises(ValueError, match="prompt"): run_claude( @@ -473,7 +501,7 @@ def test_run_claude_validates_required_params(): # --------------------------------------------------------------------------- -def test_failure_diagnostic_surfaces_api_error_text_from_stdout(): +def test_failure_diagnostic_surfaces_api_error_text_from_stdout() -> None: """The CLI hides 4xx/5xx from the proxy in `assistant.message.content` text.""" api_error_text = ( 'API Error: 400 {"error":{"message":"litellm.BadRequestError: ' @@ -505,14 +533,14 @@ def test_failure_diagnostic_surfaces_api_error_text_from_stdout(): assert "There are no healthy deployments" in diag -def test_failure_diagnostic_falls_back_to_stderr_when_no_text(): +def test_failure_diagnostic_falls_back_to_stderr_when_no_text() -> None: result = DriverResult(text="", events=[], exit_code=2, stderr="boom\n") diag = failure_diagnostic(result) assert "exit=2" in diag assert "stderr=boom" in diag -def test_failure_diagnostic_handles_completely_empty_result(): +def test_failure_diagnostic_handles_completely_empty_result() -> None: """A run that produced literally nothing should still yield a useful string.""" result = DriverResult(text="", events=[], exit_code=137, stderr="") diag = failure_diagnostic(result) @@ -520,7 +548,7 @@ def test_failure_diagnostic_handles_completely_empty_result(): assert "no diagnostic output" in diag -def test_failure_diagnostic_truncates_long_text(): +def test_failure_diagnostic_truncates_long_text() -> None: """Don't let a 5MB HTML 502 page from a load balancer wreck the matrix JSON.""" huge = "x" * 5000 result = DriverResult(text=huge, events=[], exit_code=1, stderr="") @@ -530,7 +558,7 @@ def test_failure_diagnostic_truncates_long_text(): assert len(diag) < 300 -def test_failure_diagnostic_ignores_non_int_api_error_status(): +def test_failure_diagnostic_ignores_non_int_api_error_status() -> None: """The CLI sometimes emits api_error_status as a string; don't crash.""" result = DriverResult( text="oops", @@ -554,11 +582,19 @@ def test_failure_diagnostic_ignores_non_int_api_error_status(): # --------------------------------------------------------------------------- -def test_run_claude_models_parallel_returns_one_result_per_model(): +def test_run_claude_models_parallel_returns_one_result_per_model() -> None: """Each model gets its own DriverResult keyed under the helper's dict.""" - seen_models: List[str] = [] + seen_models: list[str] = [] - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: # The model id is two slots after `--model` in the assembled command. idx = cmd.index("--model") model = cmd[idx + 1] @@ -593,10 +629,18 @@ def test_run_claude_models_parallel_returns_one_result_per_model(): assert sorted(seen_models) == ["a", "b", "c"] -def test_run_claude_models_parallel_returns_errors_as_values(): +def test_run_claude_models_parallel_returns_errors_as_values() -> None: """A model whose CLI is missing surfaces as a ClaudeCLIError, not a raise.""" - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: idx = cmd.index("--model") model = cmd[idx + 1] if model == "boom": @@ -626,10 +670,18 @@ def test_run_claude_models_parallel_returns_errors_as_values(): assert "claude CLI not found" in str(outcomes["boom"]) -def test_run_claude_models_parallel_preserves_nonzero_exit_codes(): +def test_run_claude_models_parallel_preserves_nonzero_exit_codes() -> None: """Mixed success/failure on exit code should not collapse into one verdict.""" - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: idx = cmd.index("--model") model = cmd[idx + 1] if model == "fail": @@ -653,12 +705,14 @@ def test_run_claude_models_parallel_preserves_nonzero_exit_codes(): runner=runner, ) + assert isinstance(outcomes["ok-model"], DriverResult) + assert isinstance(outcomes["fail"], DriverResult) assert outcomes["ok-model"].exit_code == 0 assert outcomes["fail"].exit_code == 2 assert outcomes["fail"].stderr == "auth failed" -def test_run_claude_models_parallel_rejects_empty_models(): +def test_run_claude_models_parallel_rejects_empty_models() -> None: with pytest.raises(ValueError, match="non-empty"): run_claude_models_parallel( models=[], @@ -668,7 +722,7 @@ def test_run_claude_models_parallel_rejects_empty_models(): ) -def test_run_claude_models_parallel_stamps_duration_on_each_result(): +def test_run_claude_models_parallel_stamps_duration_on_each_result() -> None: """Each DriverResult carries the per-model wall time so callers can attribute slow cells without re-timing the work themselves. @@ -680,7 +734,15 @@ def test_run_claude_models_parallel_stamps_duration_on_each_result(): """ import time - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: idx = cmd.index("--model") model = cmd[idx + 1] time.sleep(0.05 if model == "fast" else 0.40) @@ -694,6 +756,8 @@ def test_run_claude_models_parallel_stamps_duration_on_each_result(): runner=runner, ) + assert isinstance(outcomes["fast"], DriverResult) + assert isinstance(outcomes["slow"], DriverResult) fast_ms = outcomes["fast"].duration_ms slow_ms = outcomes["slow"].duration_ms assert fast_ms is not None and slow_ms is not None @@ -705,11 +769,21 @@ def test_run_claude_models_parallel_stamps_duration_on_each_result(): assert slow_ms > fast_ms -def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys): +def test_run_claude_models_parallel_breakdown_logs_to_stderr( + capsys: pytest.CaptureFixture[str], +) -> None: """The breakdown helper must emit a per-model timing block so users can answer "why didn't parallel help?" without re-instrumenting.""" - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: return _Completed(returncode=0, stdout="") run_claude_models_parallel( @@ -728,11 +802,21 @@ def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys): assert "slowest=" in captured.err -def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys): +def test_run_claude_models_parallel_breakdown_marks_cli_errors( + capsys: pytest.CaptureFixture[str], +) -> None: """When a model raises ClaudeCLIError, the breakdown should still show its row tagged as `cli-error` rather than crashing or omitting it.""" - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: idx = cmd.index("--model") if cmd[idx + 1] == "boom": raise FileNotFoundError(2, "no such file", "claude") @@ -752,12 +836,20 @@ def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys): assert "cli-error" in captured.err -def test_run_claude_models_parallel_forwards_extra_args_and_env(): +def test_run_claude_models_parallel_forwards_extra_args_and_env() -> None: """Shared kwargs must reach every per-model invocation unchanged.""" - captured_envs: List[dict] = [] - captured_cmds: List[List[str]] = [] + captured_envs: list[Mapping[str, str]] = [] + captured_cmds: list[Sequence[str]] = [] - def runner(cmd, env, capture_output, text, timeout, check, input=None): + def runner( + cmd: Sequence[str], + env: Mapping[str, str], + capture_output: bool, + text: bool, + timeout: float, + check: bool, + input: str | None = None, + ) -> subprocess.CompletedProcess[str]: captured_envs.append(env) captured_cmds.append(cmd) return _Completed(returncode=0, stdout="") @@ -778,7 +870,7 @@ def test_run_claude_models_parallel_forwards_extra_args_and_env(): assert "Bash" in cmd -def test_failure_diagnostic_uses_last_result_event_status(): +def test_failure_diagnostic_uses_last_result_event_status() -> None: """If multiple `result` events appear, the most recent status wins.""" result = DriverResult( text="", diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py b/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py index b3a904946d3..63886811a84 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_compat_result.py @@ -13,60 +13,61 @@ import pytest from claude_code.conftest import CompatResult -def test_set_pass_is_accepted(): +def test_set_pass_is_accepted() -> None: r = CompatResult() r.set({"status": "pass"}) assert r.value == {"status": "pass"} -def test_set_fail_requires_error(): +def test_set_fail_requires_error() -> None: r = CompatResult() with pytest.raises(ValueError, match="requires 'error'"): r.set({"status": "fail"}) -def test_set_fail_with_error_is_accepted(): +def test_set_fail_with_error_is_accepted() -> None: r = CompatResult() r.set({"status": "fail", "error": "boom"}) assert r.value == {"status": "fail", "error": "boom"} -def test_set_not_applicable_requires_reason(): +def test_set_not_applicable_requires_reason() -> None: r = CompatResult() with pytest.raises(ValueError, match="requires 'reason'"): r.set({"status": "not_applicable"}) -def test_set_not_applicable_with_reason_is_accepted(): +def test_set_not_applicable_with_reason_is_accepted() -> None: r = CompatResult() r.set({"status": "not_applicable", "reason": "Bedrock has no /thinking"}) assert r.value == {"status": "not_applicable", "reason": "Bedrock has no /thinking"} -def test_set_not_tested_is_accepted(): +def test_set_not_tested_is_accepted() -> None: r = CompatResult() r.set({"status": "not_tested"}) assert r.value == {"status": "not_tested"} -def test_set_rejects_unknown_status(): +def test_set_rejects_unknown_status() -> None: r = CompatResult() with pytest.raises(ValueError, match="status must be one of"): r.set({"status": "maybe"}) -def test_set_rejects_non_dict(): +def test_set_rejects_non_dict() -> None: r = CompatResult() with pytest.raises(TypeError): - r.set("pass") # type: ignore[arg-type] + r.set("pass") # pyright: ignore[reportArgumentType] # deliberately wrong type to assert the TypeError -def test_set_copies_input(): +def test_set_copies_input() -> None: """Mutating the dict after set() must not change the stored value.""" r = CompatResult() payload = {"status": "fail", "error": "x"} r.set(payload) payload["error"] = "mutated" + assert r.value is not None assert r.value["error"] == "x" @@ -80,7 +81,7 @@ def test_set_copies_input(): # --------------------------------------------------------------------------- -def test_add_appends_each_call_to_values(): +def test_add_appends_each_call_to_values() -> None: r = CompatResult() r.add({"status": "pass"}) r.add({"status": "fail", "error": "bad"}) @@ -90,7 +91,7 @@ def test_add_appends_each_call_to_values(): ] -def test_add_validates_like_set(): +def test_add_validates_like_set() -> None: """The add() and set() validators are the same; both must reject bad payloads.""" r = CompatResult() with pytest.raises(ValueError, match="requires 'error'"): @@ -100,10 +101,10 @@ def test_add_validates_like_set(): with pytest.raises(ValueError, match="status must be one of"): r.add({"status": "maybe"}) with pytest.raises(TypeError): - r.add("pass") # type: ignore[arg-type] + r.add("pass") # pyright: ignore[reportArgumentType] # deliberately wrong type to assert the TypeError -def test_add_copies_input(): +def test_add_copies_input() -> None: """Same defensive copy contract as set().""" r = CompatResult() payload = {"status": "fail", "error": "x"} @@ -112,21 +113,21 @@ def test_add_copies_input(): assert r.values[0]["error"] == "x" -def test_collected_returns_values_when_added(): +def test_collected_returns_values_when_added() -> None: r = CompatResult() r.add({"status": "pass"}) r.add({"status": "pass"}) assert r.collected() == [{"status": "pass"}, {"status": "pass"}] -def test_collected_returns_single_value_when_only_set_called(): +def test_collected_returns_single_value_when_only_set_called() -> None: """Legacy single-result tests should still surface their one outcome.""" r = CompatResult() r.set({"status": "pass"}) assert r.collected() == [{"status": "pass"}] -def test_collected_prefers_added_values_over_set_value(): +def test_collected_prefers_added_values_over_set_value() -> None: """If both are populated, the per-tier list wins — that's the multi-model shape.""" r = CompatResult() r.set({"status": "pass"}) @@ -134,5 +135,5 @@ def test_collected_prefers_added_values_over_set_value(): assert r.collected() == [{"status": "fail", "error": "tier-2 broke"}] -def test_collected_returns_empty_when_nothing_reported(): +def test_collected_returns_empty_when_nothing_reported() -> None: assert CompatResult().collected() == [] diff --git a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py index 92907eda3c4..71797c2b7de 100644 --- a/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py +++ b/tests/e2e/claude_code/_driver_unit_tests/test_rate_limiter.py @@ -25,13 +25,11 @@ fixtures + xdist already do that for the integration suite. from __future__ import annotations -import json -import time from pathlib import Path -from typing import List import pytest +from claude_code.json_types import JSON_OBJECT_ADAPTER from claude_code.rate_limiter import ( ALL_PROVIDERS, BURST_ENV, @@ -69,22 +67,22 @@ from claude_code.rate_limiter import ( ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE), ], ) -def test_infer_provider_maps_alias_suffix_to_column(model, expected): +def test_infer_provider_maps_alias_suffix_to_column(model: str, expected: str) -> None: assert infer_provider(model) == expected -def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order(): +def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order() -> None: """Both bedrock suffixes contain `bedrock`; the more-specific suffix wins.""" assert infer_provider("claude-foo-bedrock-converse") == PROVIDER_BEDROCK_CONVERSE assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE -def test_infer_provider_rejects_empty_string(): +def test_infer_provider_rejects_empty_string() -> None: with pytest.raises(ValueError, match="non-empty"): infer_provider("") -def test_infer_provider_is_case_insensitive(): +def test_infer_provider_is_case_insensitive() -> None: """Aliases in the proxy config sometimes drift between cases; we should still route them to the right column.""" assert infer_provider("CLAUDE-OPUS-4-7-AZURE") == PROVIDER_AZURE @@ -95,14 +93,14 @@ def test_infer_provider_is_case_insensitive(): # --------------------------------------------------------------------------- -def test_load_config_uses_default_rate_when_env_absent(): +def test_load_config_uses_default_rate_when_env_absent() -> None: cfg = load_config(env={}) for provider in ALL_PROVIDERS: assert cfg[provider].rate_per_sec == DEFAULT_RATE assert cfg[provider].burst == DEFAULT_RATE -def test_load_config_reads_per_provider_rate(): +def test_load_config_reads_per_provider_rate() -> None: cfg = load_config( env={ "LITELLM_COMPAT_RATE_ANTHROPIC": "10", @@ -114,12 +112,12 @@ def test_load_config_reads_per_provider_rate(): assert cfg[PROVIDER_VERTEX_AI].rate_per_sec == DEFAULT_RATE -def test_load_config_zero_rate_disables_provider(): +def test_load_config_zero_rate_disables_provider() -> None: cfg = load_config(env={"LITELLM_COMPAT_RATE_BEDROCK_INVOKE": "0"}) assert cfg[PROVIDER_BEDROCK_INVOKE].enabled is False -def test_load_config_burst_override_applies_to_every_provider(): +def test_load_config_burst_override_applies_to_every_provider() -> None: cfg = load_config( env={ "LITELLM_COMPAT_RATE_ANTHROPIC": "5", @@ -130,12 +128,12 @@ def test_load_config_burst_override_applies_to_every_provider(): assert cfg[provider].burst == 20.0 -def test_load_config_falls_back_on_malformed_value(): +def test_load_config_falls_back_on_malformed_value() -> None: cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "not-a-number"}) assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == DEFAULT_RATE -def test_load_config_burst_floors_at_one_when_rate_is_low(): +def test_load_config_burst_floors_at_one_when_rate_is_low() -> None: """A 0.5/s rate with no burst override must still allow at least one immediate request — otherwise the very first call would block.""" cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "0.5"}) @@ -147,8 +145,21 @@ def test_load_config_burst_floors_at_one_when_rate_is_low(): # --------------------------------------------------------------------------- +class _Clock: + def __init__(self) -> None: + self.now: float = 1_000.0 + self.sleeps: list[float] = [] + + def __call__(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + @pytest.fixture -def fake_clock(): +def fake_clock() -> _Clock: """A controllable monotonic clock + sleep for the limiter under test. Tests advance `clock.now` to simulate elapsed wall time. `sleep` @@ -156,23 +167,12 @@ def fake_clock(): sleeping, so a "wait 200ms" code path runs in microseconds and is deterministic. """ - - class Clock: - def __init__(self): - self.now = 1_000.0 - self.sleeps: List[float] = [] - - def __call__(self): - return self.now - - def sleep(self, seconds: float) -> None: - self.sleeps.append(seconds) - self.now += seconds - - return Clock() + return _Clock() -def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None): +def _make_limiter( + tmp_path: Path, fake_clock: _Clock, *, rate: float = 10.0, burst: float | None = None +) -> RateLimiter: cfg = { p: ProviderConfig(rate_per_sec=rate, burst=burst if burst is not None else rate) for p in ALL_PROVIDERS @@ -185,7 +185,7 @@ def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None): ) -def test_acquire_first_call_does_not_wait(tmp_path, fake_clock): +def test_acquire_first_call_does_not_wait(tmp_path: Path, fake_clock: _Clock) -> None: """A freshly-initialized bucket starts full; the first acquire is free.""" limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=10.0) waited = limiter.acquire(PROVIDER_ANTHROPIC) @@ -193,7 +193,7 @@ def test_acquire_first_call_does_not_wait(tmp_path, fake_clock): assert fake_clock.sleeps == [] -def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock): +def test_acquire_disabled_provider_returns_immediately(tmp_path: Path, fake_clock: _Clock) -> None: """rate=0 ⇒ no throttling, even if every other provider is throttled.""" cfg = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS} limiter = RateLimiter( @@ -204,7 +204,7 @@ def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock): assert fake_clock.sleeps == [] -def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock): +def test_acquire_burns_through_burst_then_throttles(tmp_path: Path, fake_clock: _Clock) -> None: """`burst` immediate requests succeed; the next one waits 1/rate seconds.""" limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=3.0) @@ -213,10 +213,10 @@ def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock): # Bucket is empty; next call must sleep ~0.5s to earn one token at 2/s. waited = limiter.acquire(PROVIDER_ANTHROPIC) - assert waited == pytest.approx(0.5, abs=0.01) + assert waited == pytest.approx(0.5, abs=0.01) # pyright: ignore[reportUnknownMemberType] # untyped upstream -def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock): +def test_acquire_refills_with_elapsed_time(tmp_path: Path, fake_clock: _Clock) -> None: """Advancing the clock between calls credits tokens at the configured rate.""" limiter = _make_limiter(tmp_path, fake_clock, rate=4.0, burst=1.0) @@ -225,7 +225,7 @@ def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock): assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 -def test_acquire_caps_refill_at_burst(tmp_path, fake_clock): +def test_acquire_caps_refill_at_burst(tmp_path: Path, fake_clock: _Clock) -> None: """A long quiet period must not let the bucket grow past `burst`.""" limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=2.0) @@ -237,7 +237,7 @@ def test_acquire_caps_refill_at_burst(tmp_path, fake_clock): assert waited > 0 -def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock): +def test_acquire_independent_buckets_per_provider(tmp_path: Path, fake_clock: _Clock) -> None: """Anthropic exhaustion must not throttle Azure (each column has its own bucket).""" limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=1.0) @@ -246,7 +246,7 @@ def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock): assert limiter.acquire(PROVIDER_AZURE) == 0.0 -def test_acquire_persists_state_across_limiter_instances(tmp_path): +def test_acquire_persists_state_across_limiter_instances(tmp_path: Path) -> None: """A fresh RateLimiter must read the on-disk state, not start fresh. This is the property that makes the limiter cross-process: an @@ -254,26 +254,23 @@ def test_acquire_persists_state_across_limiter_instances(tmp_path): workers, instead of getting its own private bucket. """ cfg = {p: ProviderConfig(rate_per_sec=10.0, burst=2.0) for p in ALL_PROVIDERS} - state = {"now": 1_000.0, "sleeps": []} + shared_clock = _Clock() - def clock(): - return state["now"] - - def sleep(seconds): - state["sleeps"].append(seconds) - state["now"] += seconds - - first = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep) + first = RateLimiter( + config=cfg, state_dir=tmp_path, clock=shared_clock, sleep=shared_clock.sleep + ) first.acquire(PROVIDER_ANTHROPIC) first.acquire(PROVIDER_ANTHROPIC) # bucket is now empty - second = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep) + second = RateLimiter( + config=cfg, state_dir=tmp_path, clock=shared_clock, sleep=shared_clock.sleep + ) waited = second.acquire(PROVIDER_ANTHROPIC) assert waited > 0 # had to wait, didn't see a fresh full bucket -def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock): +def test_acquire_recovers_from_corrupt_state_file(tmp_path: Path, fake_clock: _Clock) -> None: """A truncated/garbage state file must not crash the test session.""" state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json" state_file.write_text("not-json {{") @@ -282,7 +279,7 @@ def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock): assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 -def test_acquire_handles_clock_going_backward(tmp_path, fake_clock): +def test_acquire_handles_clock_going_backward(tmp_path: Path, fake_clock: _Clock) -> None: """Across a host suspend/resume the monotonic clock can briefly go backward; we must not interpret that as removing tokens.""" limiter = _make_limiter(tmp_path, fake_clock, rate=1.0, burst=2.0) @@ -297,7 +294,7 @@ def test_acquire_handles_clock_going_backward(tmp_path, fake_clock): # --------------------------------------------------------------------------- -def test_use_limiter_swaps_default_for_block(tmp_path): +def test_use_limiter_swaps_default_for_block(tmp_path: Path) -> None: sentinel_cfg = { p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS } @@ -319,11 +316,11 @@ def test_use_limiter_swaps_default_for_block(tmp_path): # --------------------------------------------------------------------------- -def test_state_file_is_json_after_acquire(tmp_path, fake_clock): +def test_state_file_is_json_after_acquire(tmp_path: Path, fake_clock: _Clock) -> None: limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0) limiter.acquire(PROVIDER_ANTHROPIC) state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json" - payload = json.loads(state_file.read_text()) + payload = JSON_OBJECT_ADAPTER.validate_json(state_file.read_text()) assert "tokens" in payload assert "last_refill" in payload - assert payload["tokens"] == pytest.approx(4.0) + assert payload["tokens"] == pytest.approx(4.0) # pyright: ignore[reportUnknownMemberType] # untyped upstream diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py index d698131670a..b2738e52ab4 100644 --- a/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_bash_tool_restrictions.py @@ -49,6 +49,10 @@ BASH_FEATURE_DIRS = ( ) +def _cell_id(path: Path) -> str: + return str(path.relative_to(REPO_ROOT)) + + def _bash_cells() -> Iterable[Path]: for feature in BASH_FEATURE_DIRS: feature_dir = CLAUDE_CODE_DIR / feature @@ -73,9 +77,7 @@ def _has_bare_bash_token(text: str) -> bool: return '"Bash"' in text.replace('"Bash(echo pong)"', "") -@pytest.mark.parametrize( - "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) -) +@pytest.mark.parametrize("cell", list(_bash_cells()), ids=_cell_id) def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: """The cell must pass `Bash(echo pong)` as the allow rule, not the unrestricted `Bash` value that was originally flagged.""" @@ -101,7 +103,7 @@ def test_bash_allow_rule_is_pinned_to_exact_echo_pong(cell: Path) -> None: ) -def test_has_bare_bash_token_flags_unrestricted_value(): +def test_has_bare_bash_token_flags_unrestricted_value() -> None: """A file that allows the bare `"Bash"` token alongside the exact-match rule must be flagged. Without this guard the security pin reverts to the dead-code `or` it had originally, which let @@ -111,7 +113,7 @@ def test_has_bare_bash_token_flags_unrestricted_value(): assert _has_bare_bash_token(text) -def test_has_bare_bash_token_accepts_only_exact_match(): +def test_has_bare_bash_token_accepts_only_exact_match() -> None: """The standard pattern — only the exact-match allow rule, no bare `"Bash"` — must be accepted. This is the shape every Bash-using cell in the suite is required to take. @@ -120,7 +122,7 @@ def test_has_bare_bash_token_accepts_only_exact_match(): assert not _has_bare_bash_token(text) -def test_has_bare_bash_token_ignores_unrelated_substrings(): +def test_has_bare_bash_token_ignores_unrelated_substrings() -> None: """`Bash(echo pong)` is the only allowed shape; substrings like `BashTool` or `Bashing` are unrelated identifiers and must not be confused with the bare `"Bash"` token (i.e. the exact quoted @@ -129,9 +131,7 @@ def test_has_bare_bash_token_ignores_unrelated_substrings(): assert not _has_bare_bash_token(text) -@pytest.mark.parametrize( - "cell", list(_bash_cells()), ids=lambda p: str(p.relative_to(REPO_ROOT)) -) +@pytest.mark.parametrize("cell", list(_bash_cells()), ids=_cell_id) def test_bash_cell_uses_dontask_permission_mode(cell: Path) -> None: """The cell must pair the allow rule with `--permission-mode dontAsk` so tool calls that don't match the allow rule are auto-denied (as diff --git a/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py b/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py index 5c516da81c5..08e04381846 100644 --- a/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py +++ b/tests/e2e/claude_code/_pr_gate_unit_tests/test_pr_gate_version_resolver.py @@ -17,6 +17,7 @@ from datetime import datetime, timedelta, timezone import pytest +from claude_code.json_types import JSONValue from claude_code.pr_gate_version_resolver import ( NoEligibleVersionError, resolve_pr_gate_version, @@ -32,7 +33,7 @@ def _t(iso: str) -> str: NOW = datetime(2026, 4, 25, 12, 0, 0, tzinfo=timezone.utc) -def _metadata_with_times(times: dict) -> dict: +def _metadata_with_times(times: dict[str, JSONValue]) -> dict[str, JSONValue]: """Shape an npm `packument`-like dict with the `time` field populated. The npm registry response includes `time.created` / `time.modified` @@ -48,7 +49,7 @@ def _metadata_with_times(times: dict) -> dict: } -def test_picks_newest_version_at_least_three_days_old(): +def test_picks_newest_version_at_least_three_days_old() -> None: metadata = _metadata_with_times( { "2.1.118": _t("2026-04-15T10:00:00.000Z"), @@ -60,9 +61,9 @@ def test_picks_newest_version_at_least_three_days_old(): assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.119" -def test_skips_created_and_modified_meta_keys(): +def test_skips_created_and_modified_meta_keys() -> None: """`time` contains `created` / `modified` non-version entries — must be ignored.""" - metadata = { + metadata: dict[str, JSONValue] = { "name": "@anthropic-ai/claude-code", "time": { "created": _t("2024-01-01T00:00:00.000Z"), @@ -73,7 +74,7 @@ def test_skips_created_and_modified_meta_keys(): assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.0.0" -def test_min_age_boundary_is_inclusive(): +def test_min_age_boundary_is_inclusive() -> None: """A version published exactly 3 days ago is eligible (>= cutoff).""" three_days_ago = NOW - timedelta(days=3) metadata = _metadata_with_times( @@ -84,7 +85,7 @@ def test_min_age_boundary_is_inclusive(): assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.0" -def test_raises_when_every_version_is_too_new(): +def test_raises_when_every_version_is_too_new() -> None: metadata = _metadata_with_times( { "2.1.121": _t("2026-04-25T08:00:00.000Z"), # 4h old @@ -95,13 +96,13 @@ def test_raises_when_every_version_is_too_new(): resolve_pr_gate_version(metadata=metadata, as_of=NOW) -def test_raises_when_metadata_has_no_versions(): - metadata = {"name": "@anthropic-ai/claude-code", "time": {}} +def test_raises_when_metadata_has_no_versions() -> None: + metadata: dict[str, JSONValue] = {"name": "@anthropic-ai/claude-code", "time": {}} with pytest.raises(NoEligibleVersionError): resolve_pr_gate_version(metadata=metadata, as_of=NOW) -def test_picks_latest_publish_time_not_largest_semver(): +def test_picks_latest_publish_time_not_largest_semver() -> None: """If a patch is published to an old major after a newer release, "newest" is by publish time, not semver string ordering.""" metadata = _metadata_with_times( @@ -113,7 +114,7 @@ def test_picks_latest_publish_time_not_largest_semver(): assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "1.9.99" -def test_uses_custom_min_age(): +def test_uses_custom_min_age() -> None: metadata = _metadata_with_times( { "1.0.0": _t("2026-04-23T10:00:00.000Z"), # 2d 2h old @@ -127,7 +128,7 @@ def test_uses_custom_min_age(): assert out == "0.9.0" -def test_excludes_prerelease_versions(): +def test_excludes_prerelease_versions() -> None: """Pre-release tags (1.0.0-alpha.1, 2.0.0-rc.1, etc.) must never win, even if their publish timestamp is the newest eligible one.""" metadata = _metadata_with_times( @@ -141,7 +142,7 @@ def test_excludes_prerelease_versions(): assert resolve_pr_gate_version(metadata=metadata, as_of=NOW) == "2.1.119" -def test_raises_when_only_prereleases_are_eligible(): +def test_raises_when_only_prereleases_are_eligible() -> None: metadata = _metadata_with_times( { "2.2.0-alpha.1": _t("2026-04-22T10:00:00.000Z"), @@ -152,10 +153,10 @@ def test_raises_when_only_prereleases_are_eligible(): resolve_pr_gate_version(metadata=metadata, as_of=NOW) -def test_resolver_uses_fetcher_when_metadata_not_provided(): - captured = {} +def test_resolver_uses_fetcher_when_metadata_not_provided() -> None: + captured: dict[str, str] = {} - def fake_fetch(package_name: str) -> dict: + def fake_fetch(package_name: str) -> dict[str, JSONValue]: captured["package"] = package_name return _metadata_with_times({"3.0.0": _t("2026-04-10T10:00:00.000Z")}) diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py index c06fff28d2d..2e479e9deb1 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult # Per the PRD: each cell is exercised against three Claude tiers via the # Anthropic provider. Aliases are configured in the LiteLLM proxy's @@ -32,7 +33,7 @@ ANTHROPIC_MODELS = [ ] -def test_basic_messaging_non_streaming_anthropic(compat_result): +def test_basic_messaging_non_streaming_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. "Basic messaging" means: send a single user prompt, receive any diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py index 2a962b244a8..bb225c6dce7 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_azure.py @@ -26,6 +26,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult # Per-model aliases registered in the LiteLLM proxy's routing config to # point at Microsoft Foundry's Anthropic deployments. The driver only @@ -38,7 +39,7 @@ AZURE_MODELS = [ ] -def test_basic_messaging_non_streaming_azure(compat_result): +def test_basic_messaging_non_streaming_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. "Basic messaging" means: send a single user prompt, receive any diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index 2245ed7417a..cbd8e4c7df6 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult # Per-model aliases registered in the LiteLLM proxy's routing config to # point at Bedrock's Converse endpoint. The driver only sends the alias; @@ -33,7 +34,7 @@ BEDROCK_CONVERSE_MODELS = [ ] -def test_basic_messaging_non_streaming_bedrock_converse(compat_result): +def test_basic_messaging_non_streaming_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( compat_result=compat_result, diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index e0a6e77f3c1..59748b654bf 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult # Per-model aliases registered in the LiteLLM proxy's routing config to # point at Bedrock's legacy InvokeModel endpoint. The driver only sends @@ -33,7 +34,7 @@ BEDROCK_INVOKE_MODELS = [ ] -def test_basic_messaging_non_streaming_bedrock_invoke(compat_result): +def test_basic_messaging_non_streaming_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( compat_result=compat_result, diff --git a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py index e4e2a39e6cd..ed9bbf86046 100644 --- a/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -21,6 +21,7 @@ the matrix builder still sees three rows for this (feature, provider). from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult # Per-model aliases registered in the LiteLLM proxy's routing config to # point at Vertex AI's Anthropic model endpoints. The driver only sends @@ -33,7 +34,7 @@ VERTEX_AI_MODELS = [ ] -def test_basic_messaging_non_streaming_vertex_ai(compat_result): +def test_basic_messaging_non_streaming_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" run_basic_messaging_cell( compat_result=compat_result, diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py index 56e3fb6c181..04c9e3ddea6 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_anthropic.py @@ -26,6 +26,7 @@ sees three rows for this (feature, provider). from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult ANTHROPIC_MODELS = [ "claude-haiku-4-5", @@ -34,7 +35,7 @@ ANTHROPIC_MODELS = [ ] -def test_basic_messaging_streaming_anthropic(compat_result): +def test_basic_messaging_streaming_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). """ diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py index b6c002d0b27..fbea0d41f39 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_azure.py @@ -20,6 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult AZURE_MODELS = [ "claude-haiku-4-5-azure", @@ -28,7 +29,7 @@ AZURE_MODELS = [ ] -def test_basic_messaging_streaming_azure(compat_result): +def test_basic_messaging_streaming_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). """ diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py index 44ac54515f0..864979cf5db 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult BEDROCK_CONVERSE_MODELS = [ "claude-haiku-4-5-bedrock-converse", @@ -24,7 +25,7 @@ BEDROCK_CONVERSE_MODELS = [ ] -def test_basic_messaging_streaming_bedrock_converse(compat_result): +def test_basic_messaging_streaming_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). """ diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index 1d59d16cdc1..7a0667745e2 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult BEDROCK_INVOKE_MODELS = [ "claude-haiku-4-5-bedrock-invoke", @@ -24,7 +25,7 @@ BEDROCK_INVOKE_MODELS = [ ] -def test_basic_messaging_streaming_bedrock_invoke(compat_result): +def test_basic_messaging_streaming_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). """ diff --git a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py index 014a31160a8..734d4d9fe9c 100644 --- a/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations from claude_code._basic_messaging import run_basic_messaging_cell +from claude_code.conftest import CompatResult VERTEX_AI_MODELS = [ "claude-haiku-4-5-vertex", @@ -24,7 +25,7 @@ VERTEX_AI_MODELS = [ ] -def test_basic_messaging_streaming_vertex_ai(compat_result): +def test_basic_messaging_streaming_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (one row per Claude tier). """ diff --git a/tests/e2e/claude_code/count_tokens/test_anthropic.py b/tests/e2e/claude_code/count_tokens/test_anthropic.py index 3508063459c..bc7c8225343 100644 --- a/tests/e2e/claude_code/count_tokens/test_anthropic.py +++ b/tests/e2e/claude_code/count_tokens/test_anthropic.py @@ -41,6 +41,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, @@ -56,7 +57,7 @@ ANTHROPIC_MODELS = [ ] -def test_count_tokens_anthropic(compat_result): +def test_count_tokens_anthropic(compat_result: CompatResult) -> None: """Probe `/v1/messages/count_tokens` for each Anthropic tier and assert the response shape.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -76,7 +77,7 @@ def test_count_tokens_anthropic(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: result = probe_count_tokens( base_url=base_url, api_key=api_key, model=model diff --git a/tests/e2e/claude_code/count_tokens/test_azure.py b/tests/e2e/claude_code/count_tokens/test_azure.py index 2b8707b50b0..506e156cb54 100644 --- a/tests/e2e/claude_code/count_tokens/test_azure.py +++ b/tests/e2e/claude_code/count_tokens/test_azure.py @@ -41,6 +41,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, @@ -56,7 +57,7 @@ AZURE_MODELS = [ ] -def test_count_tokens_azure(compat_result): +def test_count_tokens_azure(compat_result: CompatResult) -> None: """Probe `/v1/messages/count_tokens` for each Azure (Microsoft Foundry) tier and assert the response shape.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -76,7 +77,7 @@ def test_count_tokens_azure(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: result = probe_count_tokens( base_url=base_url, api_key=api_key, model=model diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py index 4221773ead2..3d3dfb82d97 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_converse.py @@ -41,6 +41,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, @@ -56,7 +57,7 @@ BEDROCK_CONVERSE_MODELS = [ ] -def test_count_tokens_bedrock_converse(compat_result): +def test_count_tokens_bedrock_converse(compat_result: CompatResult) -> None: """Probe `/v1/messages/count_tokens` for each Bedrock (Converse) tier and assert the response shape.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -76,7 +77,7 @@ def test_count_tokens_bedrock_converse(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: result = probe_count_tokens( base_url=base_url, api_key=api_key, model=model diff --git a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py index cc70bf12392..3d7fa371b6c 100644 --- a/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/count_tokens/test_bedrock_invoke.py @@ -41,6 +41,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, @@ -56,7 +57,7 @@ BEDROCK_INVOKE_MODELS = [ ] -def test_count_tokens_bedrock_invoke(compat_result): +def test_count_tokens_bedrock_invoke(compat_result: CompatResult) -> None: """Probe `/v1/messages/count_tokens` for each Bedrock (Invoke) tier and assert the response shape.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -76,7 +77,7 @@ def test_count_tokens_bedrock_invoke(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: result = probe_count_tokens( base_url=base_url, api_key=api_key, model=model diff --git a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py index 8c2678f7010..852dbcd055e 100644 --- a/tests/e2e/claude_code/count_tokens/test_vertex_ai.py +++ b/tests/e2e/claude_code/count_tokens/test_vertex_ai.py @@ -41,6 +41,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_count_tokens_shape, probe_count_tokens, @@ -56,7 +57,7 @@ VERTEX_AI_MODELS = [ ] -def test_count_tokens_vertex_ai(compat_result): +def test_count_tokens_vertex_ai(compat_result: CompatResult) -> None: """Probe `/v1/messages/count_tokens` for each Vertex AI tier and assert the response shape.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -76,7 +77,7 @@ def test_count_tokens_vertex_ai(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: result = probe_count_tokens( base_url=base_url, api_key=api_key, model=model diff --git a/tests/e2e/claude_code/long_context_1m/test_anthropic.py b/tests/e2e/claude_code/long_context_1m/test_anthropic.py index fb74d5fd40d..3f55607760c 100644 --- a/tests/e2e/claude_code/long_context_1m/test_anthropic.py +++ b/tests/e2e/claude_code/long_context_1m/test_anthropic.py @@ -64,6 +64,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) - pad_lines = [] + pad_lines: list[str] = [] pad_len = 0 idx = 0 while pad_len < pad_target_chars: @@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing -def test_long_context_1m_anthropic(compat_result): +def test_long_context_1m_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" @@ -197,7 +198,7 @@ def test_long_context_1m_anthropic(compat_result): timeout=300.0, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/long_context_1m/test_azure.py b/tests/e2e/claude_code/long_context_1m/test_azure.py index 5800fdadbfc..adc1c0df4c1 100644 --- a/tests/e2e/claude_code/long_context_1m/test_azure.py +++ b/tests/e2e/claude_code/long_context_1m/test_azure.py @@ -64,6 +64,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) - pad_lines = [] + pad_lines: list[str] = [] pad_len = 0 idx = 0 while pad_len < pad_target_chars: @@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing -def test_long_context_1m_azure(compat_result): +def test_long_context_1m_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI (Azure (Microsoft Foundry)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" @@ -197,7 +198,7 @@ def test_long_context_1m_azure(compat_result): timeout=300.0, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py index 18587f7c2d6..0759c386fc4 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_converse.py @@ -64,6 +64,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) - pad_lines = [] + pad_lines: list[str] = [] pad_len = 0 idx = 0 while pad_len < pad_target_chars: @@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing -def test_long_context_1m_bedrock_converse(compat_result): +def test_long_context_1m_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI (Bedrock (Converse)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" @@ -197,7 +198,7 @@ def test_long_context_1m_bedrock_converse(compat_result): timeout=300.0, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py index 0270197ce2a..eb38ab5e56e 100644 --- a/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/long_context_1m/test_bedrock_invoke.py @@ -64,6 +64,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) - pad_lines = [] + pad_lines: list[str] = [] pad_len = 0 idx = 0 while pad_len < pad_target_chars: @@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing -def test_long_context_1m_bedrock_invoke(compat_result): +def test_long_context_1m_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI (Bedrock (Invoke)) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" @@ -197,7 +198,7 @@ def test_long_context_1m_bedrock_invoke(compat_result): timeout=300.0, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py index d2db4a1b4ee..34ecc5057fb 100644 --- a/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py +++ b/tests/e2e/claude_code/long_context_1m/test_vertex_ai.py @@ -64,6 +64,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -144,7 +145,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: closing = "\n\nEnd of excerpt. Please reply with the single word 'ok'." pad_target_chars = target_tokens * _CHARS_PER_TOKEN - len(preamble) - len(closing) - pad_lines = [] + pad_lines: list[str] = [] pad_len = 0 idx = 0 while pad_len < pad_target_chars: @@ -155,7 +156,7 @@ def _build_long_prompt(target_tokens: int = TARGET_INPUT_TOKENS) -> str: return preamble + "".join(pad_lines) + closing -def test_long_context_1m_vertex_ai(compat_result): +def test_long_context_1m_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI (Vertex AI) with a ~210k-token prompt and the `context-1m-2025-08-07` beta header; assert no 400 / 413 and a non-empty reply for Sonnet + Opus.""" @@ -197,7 +198,7 @@ def test_long_context_1m_vertex_ai(compat_result): timeout=300.0, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/pdf_input/test_anthropic.py b/tests/e2e/claude_code/pdf_input/test_anthropic.py index 36fb69a1db6..d01c7da9222 100644 --- a/tests/e2e/claude_code/pdf_input/test_anthropic.py +++ b/tests/e2e/claude_code/pdf_input/test_anthropic.py @@ -23,6 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os +from pathlib import Path import pytest @@ -31,6 +32,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -88,7 +90,7 @@ def _build_minimal_pdf(marker: str) -> bytes: ) out = bytearray(b"%PDF-1.4\n") - offsets = [] + offsets: list[int] = [] for i, obj in enumerate(objects, start=1): offsets.append(len(out)) out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" @@ -108,7 +110,7 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) -def test_pdf_input_anthropic(compat_result, tmp_path): +def test_pdf_input_anthropic(compat_result: CompatResult, tmp_path: Path) -> None: """Drive the `claude` CLI against the LiteLLM proxy with a PDF attached via the Read tool and assert the reply references it.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -141,7 +143,7 @@ def test_pdf_input_anthropic(compat_result, tmp_path): extra_args=["--allowed-tools", "Read"], ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/pdf_input/test_azure.py b/tests/e2e/claude_code/pdf_input/test_azure.py index 810c857e407..d3c78f57933 100644 --- a/tests/e2e/claude_code/pdf_input/test_azure.py +++ b/tests/e2e/claude_code/pdf_input/test_azure.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os +from pathlib import Path import pytest @@ -24,6 +25,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,7 +67,7 @@ def _build_minimal_pdf(marker: str) -> bytes: ) out = bytearray(b"%PDF-1.4\n") - offsets = [] + offsets: list[int] = [] for i, obj in enumerate(objects, start=1): offsets.append(len(out)) out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" @@ -85,7 +87,7 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) -def test_pdf_input_azure(compat_result, tmp_path): +def test_pdf_input_azure(compat_result: CompatResult, tmp_path: Path) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -116,7 +118,7 @@ def test_pdf_input_azure(compat_result, tmp_path): extra_args=["--allowed-tools", "Read"], ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py index 191a27c6d46..2f6e21f7e55 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_converse.py @@ -22,6 +22,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os +from pathlib import Path import pytest @@ -30,6 +31,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -71,7 +73,7 @@ def _build_minimal_pdf(marker: str) -> bytes: ) out = bytearray(b"%PDF-1.4\n") - offsets = [] + offsets: list[int] = [] for i, obj in enumerate(objects, start=1): offsets.append(len(out)) out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" @@ -91,7 +93,7 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) -def test_pdf_input_bedrock_converse(compat_result, tmp_path): +def test_pdf_input_bedrock_converse(compat_result: CompatResult, tmp_path: Path) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -122,7 +124,7 @@ def test_pdf_input_bedrock_converse(compat_result, tmp_path): extra_args=["--allowed-tools", "Read"], ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py index 163cabb45a0..3d258c5105b 100644 --- a/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/pdf_input/test_bedrock_invoke.py @@ -21,6 +21,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os +from pathlib import Path import pytest @@ -29,6 +30,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -70,7 +72,7 @@ def _build_minimal_pdf(marker: str) -> bytes: ) out = bytearray(b"%PDF-1.4\n") - offsets = [] + offsets: list[int] = [] for i, obj in enumerate(objects, start=1): offsets.append(len(out)) out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" @@ -90,7 +92,7 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) -def test_pdf_input_bedrock_invoke(compat_result, tmp_path): +def test_pdf_input_bedrock_invoke(compat_result: CompatResult, tmp_path: Path) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -121,7 +123,7 @@ def test_pdf_input_bedrock_invoke(compat_result, tmp_path): extra_args=["--allowed-tools", "Read"], ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py index 0d0573d05b3..bc3b5b22d54 100644 --- a/tests/e2e/claude_code/pdf_input/test_vertex_ai.py +++ b/tests/e2e/claude_code/pdf_input/test_vertex_ai.py @@ -16,6 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os +from pathlib import Path import pytest @@ -24,6 +25,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,7 +67,7 @@ def _build_minimal_pdf(marker: str) -> bytes: ) out = bytearray(b"%PDF-1.4\n") - offsets = [] + offsets: list[int] = [] for i, obj in enumerate(objects, start=1): offsets.append(len(out)) out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" @@ -85,7 +87,7 @@ def _build_minimal_pdf(marker: str) -> bytes: return bytes(out) -def test_pdf_input_vertex_ai(compat_result, tmp_path): +def test_pdf_input_vertex_ai(compat_result: CompatResult, tmp_path: Path) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -116,7 +118,7 @@ def test_pdf_input_vertex_ai(compat_result, tmp_path): extra_args=["--allowed-tools", "Read"], ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py index d81887231d8..4a24953261f 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_anthropic.py @@ -25,7 +25,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -34,6 +34,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -52,20 +54,22 @@ ANTHROPIC_MODELS = [ CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: """Return cache_creation_input_tokens + cache_read_input_tokens from the upstream usage block, or 0 if the keys are missing.""" if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_1h_anthropic(compat_result): +def test_prompt_caching_1h_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with the 1h TTL opt-in env var set, and assert the upstream usage block surfaces a non-zero cache token count.""" @@ -93,7 +97,7 @@ def test_prompt_caching_1h_anthropic(compat_result): extra_env=CACHE_1H_ENV, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py index 416757f8691..571240753c6 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_azure.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -38,18 +40,20 @@ AZURE_MODELS = [ CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_1h_azure(compat_result): +def test_prompt_caching_1h_azure(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -74,7 +78,7 @@ def test_prompt_caching_1h_azure(compat_result): extra_env=CACHE_1H_ENV, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py index 5bc632c6f1b..cb9f83c56b9 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -21,7 +21,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -30,6 +30,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -46,18 +48,20 @@ CACHE_1H_ENV = { } -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_1h_bedrock_converse(compat_result): +def test_prompt_caching_1h_bedrock_converse(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -82,7 +86,7 @@ def test_prompt_caching_1h_bedrock_converse(compat_result): extra_env=CACHE_1H_ENV, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py index 4501834956b..9786a4ca3bb 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -23,7 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -32,6 +32,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -50,18 +52,20 @@ CACHE_1H_ENV = { } -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_1h_bedrock_invoke(compat_result): +def test_prompt_caching_1h_bedrock_invoke(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -86,7 +90,7 @@ def test_prompt_caching_1h_bedrock_invoke(compat_result): extra_env=CACHE_1H_ENV, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py index 09ded634b45..c776dfc6374 100644 --- a/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_1h/test_vertex_ai.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -38,18 +40,20 @@ VERTEX_AI_MODELS = [ CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_1h_vertex_ai(compat_result): +def test_prompt_caching_1h_vertex_ai(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -74,7 +78,7 @@ def test_prompt_caching_1h_vertex_ai(compat_result): extra_env=CACHE_1H_ENV, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py index 4b20a65f31b..6b2c8d0582d 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_anthropic.py @@ -23,7 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -32,6 +32,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -43,20 +45,22 @@ ANTHROPIC_MODELS = [ ] -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: """Return cache_creation_input_tokens + cache_read_input_tokens from the upstream usage block, or 0 if the keys are missing.""" if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_5m_anthropic(compat_result): +def test_prompt_caching_5m_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -82,7 +86,7 @@ def test_prompt_caching_5m_anthropic(compat_result): api_key=api_key, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py index 22bd5aa7048..c96ff49ccac 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_azure.py @@ -23,7 +23,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -32,6 +32,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -43,18 +45,20 @@ AZURE_MODELS = [ ] -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_5m_azure(compat_result): +def test_prompt_caching_5m_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -80,7 +84,7 @@ def test_prompt_caching_5m_azure(compat_result): api_key=api_key, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py index 681a6ecce10..5730f8536c3 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -36,18 +38,20 @@ BEDROCK_CONVERSE_MODELS = [ ] -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_5m_bedrock_converse(compat_result): +def test_prompt_caching_5m_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -73,7 +77,7 @@ def test_prompt_caching_5m_bedrock_converse(compat_result): api_key=api_key, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py index f1a3109b3a1..b23bcf13d00 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -36,18 +38,20 @@ BEDROCK_INVOKE_MODELS = [ ] -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_5m_bedrock_invoke(compat_result): +def test_prompt_caching_5m_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -73,7 +77,7 @@ def test_prompt_caching_5m_bedrock_invoke(compat_result): api_key=api_key, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py index cc5d337dfbe..ce741b2f2d7 100644 --- a/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py +++ b/tests/e2e/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Optional +from typing import Mapping import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -36,18 +38,20 @@ VERTEX_AI_MODELS = [ ] -def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: +def _cache_tokens(usage: Mapping[str, JSONValue] | None) -> int: if not isinstance(usage, Mapping): return 0 creation = usage.get("cache_creation_input_tokens") or 0 read = usage.get("cache_read_input_tokens") or 0 + if not isinstance(creation, (str, int, float)) or not isinstance(read, (str, int, float)): + return 0 try: return int(creation) + int(read) except (TypeError, ValueError): return 0 -def test_prompt_caching_5m_vertex_ai(compat_result): +def test_prompt_caching_5m_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -73,7 +77,7 @@ def test_prompt_caching_5m_vertex_ai(compat_result): api_key=api_key, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/structured_outputs/test_anthropic.py b/tests/e2e/claude_code/structured_outputs/test_anthropic.py index 610d8433b72..4acad14e9da 100644 --- a/tests/e2e/claude_code/structured_outputs/test_anthropic.py +++ b/tests/e2e/claude_code/structured_outputs/test_anthropic.py @@ -49,8 +49,7 @@ from __future__ import annotations import json import os -import re -from typing import Any, Mapping, Optional, Sequence, Tuple +from typing import Mapping, Optional, Sequence import pytest @@ -59,6 +58,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,7 +76,7 @@ ANTHROPIC_MODELS = [ # integer schema gives every tier (including Haiku) enough headroom # that schema satisfaction is essentially deterministic, isolating # failures to the proxy / transport. -SCHEMA = { +SCHEMA: dict[str, JSONValue] = { "type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"], @@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output." def _extract_structured_output( - events: Sequence[Mapping[str, Any]], -) -> Optional[Mapping[str, Any]]: + events: Sequence[Mapping[str, JSONValue]], +) -> Optional[Mapping[str, JSONValue]]: """Return the `structured_output` payload from the last `result` event. Claude Code emits its terminal stream-json line as @@ -113,7 +114,7 @@ def _extract_structured_output( def _validate_against_schema( - payload: Mapping[str, Any], schema: Mapping[str, Any] + payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue] ) -> Optional[str]: """Tiny shape validator covering the subset we actually need. @@ -125,7 +126,7 @@ def _validate_against_schema( not a LiteLLM-proxy bug, so a deeper check would only add false failures on the wrong axis. """ - type_map = { + type_map: dict[str, type[object] | tuple[type[object], ...]] = { "integer": int, "number": (int, float), "string": str, @@ -133,13 +134,17 @@ def _validate_against_schema( "array": list, "object": Mapping, } - required = schema.get("required") or [] - properties = schema.get("properties") or {} + raw_required = schema.get("required") + required: list[JSONValue] = raw_required if isinstance(raw_required, list) else [] + raw_properties = schema.get("properties") + properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {} for key in required: - if key not in payload: + if not isinstance(key, str) or key not in payload: return f"missing required key {key!r}" - expected = (properties.get(key) or {}).get("type") - if expected and expected in type_map: + prop = properties.get(key) + prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {} + expected = prop_schema.get("type") + if isinstance(expected, str) and expected in type_map: if not isinstance(payload[key], type_map[expected]): return ( f"key {key!r} has wrong type: " @@ -152,7 +157,7 @@ def _validate_against_schema( return None -def test_structured_outputs_anthropic(compat_result): +def test_structured_outputs_anthropic(compat_result: CompatResult) -> None: """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" @@ -181,7 +186,7 @@ def test_structured_outputs_anthropic(compat_result): extra_args=["--json-schema", SCHEMA_JSON], ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/structured_outputs/test_azure.py b/tests/e2e/claude_code/structured_outputs/test_azure.py index 290f9156910..f0fd3016c55 100644 --- a/tests/e2e/claude_code/structured_outputs/test_azure.py +++ b/tests/e2e/claude_code/structured_outputs/test_azure.py @@ -49,8 +49,7 @@ from __future__ import annotations import json import os -import re -from typing import Any, Mapping, Optional, Sequence, Tuple +from typing import Mapping, Optional, Sequence import pytest @@ -59,6 +58,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,7 +76,7 @@ AZURE_MODELS = [ # integer schema gives every tier (including Haiku) enough headroom # that schema satisfaction is essentially deterministic, isolating # failures to the proxy / transport. -SCHEMA = { +SCHEMA: dict[str, JSONValue] = { "type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"], @@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output." def _extract_structured_output( - events: Sequence[Mapping[str, Any]], -) -> Optional[Mapping[str, Any]]: + events: Sequence[Mapping[str, JSONValue]], +) -> Optional[Mapping[str, JSONValue]]: """Return the `structured_output` payload from the last `result` event. Claude Code emits its terminal stream-json line as @@ -113,7 +114,7 @@ def _extract_structured_output( def _validate_against_schema( - payload: Mapping[str, Any], schema: Mapping[str, Any] + payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue] ) -> Optional[str]: """Tiny shape validator covering the subset we actually need. @@ -125,7 +126,7 @@ def _validate_against_schema( not a LiteLLM-proxy bug, so a deeper check would only add false failures on the wrong axis. """ - type_map = { + type_map: dict[str, type[object] | tuple[type[object], ...]] = { "integer": int, "number": (int, float), "string": str, @@ -133,13 +134,17 @@ def _validate_against_schema( "array": list, "object": Mapping, } - required = schema.get("required") or [] - properties = schema.get("properties") or {} + raw_required = schema.get("required") + required: list[JSONValue] = raw_required if isinstance(raw_required, list) else [] + raw_properties = schema.get("properties") + properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {} for key in required: - if key not in payload: + if not isinstance(key, str) or key not in payload: return f"missing required key {key!r}" - expected = (properties.get(key) or {}).get("type") - if expected and expected in type_map: + prop = properties.get(key) + prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {} + expected = prop_schema.get("type") + if isinstance(expected, str) and expected in type_map: if not isinstance(payload[key], type_map[expected]): return ( f"key {key!r} has wrong type: " @@ -152,7 +157,7 @@ def _validate_against_schema( return None -def test_structured_outputs_azure(compat_result): +def test_structured_outputs_azure(compat_result: CompatResult) -> None: """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" @@ -181,7 +186,7 @@ def test_structured_outputs_azure(compat_result): extra_args=["--json-schema", SCHEMA_JSON], ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py index 5179014773c..0fb09e44fd7 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_converse.py @@ -49,8 +49,7 @@ from __future__ import annotations import json import os -import re -from typing import Any, Mapping, Optional, Sequence, Tuple +from typing import Mapping, Optional, Sequence import pytest @@ -59,6 +58,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,7 +76,7 @@ BEDROCK_CONVERSE_MODELS = [ # integer schema gives every tier (including Haiku) enough headroom # that schema satisfaction is essentially deterministic, isolating # failures to the proxy / transport. -SCHEMA = { +SCHEMA: dict[str, JSONValue] = { "type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"], @@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output." def _extract_structured_output( - events: Sequence[Mapping[str, Any]], -) -> Optional[Mapping[str, Any]]: + events: Sequence[Mapping[str, JSONValue]], +) -> Optional[Mapping[str, JSONValue]]: """Return the `structured_output` payload from the last `result` event. Claude Code emits its terminal stream-json line as @@ -113,7 +114,7 @@ def _extract_structured_output( def _validate_against_schema( - payload: Mapping[str, Any], schema: Mapping[str, Any] + payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue] ) -> Optional[str]: """Tiny shape validator covering the subset we actually need. @@ -125,7 +126,7 @@ def _validate_against_schema( not a LiteLLM-proxy bug, so a deeper check would only add false failures on the wrong axis. """ - type_map = { + type_map: dict[str, type[object] | tuple[type[object], ...]] = { "integer": int, "number": (int, float), "string": str, @@ -133,13 +134,17 @@ def _validate_against_schema( "array": list, "object": Mapping, } - required = schema.get("required") or [] - properties = schema.get("properties") or {} + raw_required = schema.get("required") + required: list[JSONValue] = raw_required if isinstance(raw_required, list) else [] + raw_properties = schema.get("properties") + properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {} for key in required: - if key not in payload: + if not isinstance(key, str) or key not in payload: return f"missing required key {key!r}" - expected = (properties.get(key) or {}).get("type") - if expected and expected in type_map: + prop = properties.get(key) + prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {} + expected = prop_schema.get("type") + if isinstance(expected, str) and expected in type_map: if not isinstance(payload[key], type_map[expected]): return ( f"key {key!r} has wrong type: " @@ -152,7 +157,7 @@ def _validate_against_schema( return None -def test_structured_outputs_bedrock_converse(compat_result): +def test_structured_outputs_bedrock_converse(compat_result: CompatResult) -> None: """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" @@ -181,7 +186,7 @@ def test_structured_outputs_bedrock_converse(compat_result): extra_args=["--json-schema", SCHEMA_JSON], ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py index 313a714be34..d94cccefd3d 100644 --- a/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/structured_outputs/test_bedrock_invoke.py @@ -49,8 +49,7 @@ from __future__ import annotations import json import os -import re -from typing import Any, Mapping, Optional, Sequence, Tuple +from typing import Mapping, Optional, Sequence import pytest @@ -59,6 +58,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,7 +76,7 @@ BEDROCK_INVOKE_MODELS = [ # integer schema gives every tier (including Haiku) enough headroom # that schema satisfaction is essentially deterministic, isolating # failures to the proxy / transport. -SCHEMA = { +SCHEMA: dict[str, JSONValue] = { "type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"], @@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output." def _extract_structured_output( - events: Sequence[Mapping[str, Any]], -) -> Optional[Mapping[str, Any]]: + events: Sequence[Mapping[str, JSONValue]], +) -> Optional[Mapping[str, JSONValue]]: """Return the `structured_output` payload from the last `result` event. Claude Code emits its terminal stream-json line as @@ -113,7 +114,7 @@ def _extract_structured_output( def _validate_against_schema( - payload: Mapping[str, Any], schema: Mapping[str, Any] + payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue] ) -> Optional[str]: """Tiny shape validator covering the subset we actually need. @@ -125,7 +126,7 @@ def _validate_against_schema( not a LiteLLM-proxy bug, so a deeper check would only add false failures on the wrong axis. """ - type_map = { + type_map: dict[str, type[object] | tuple[type[object], ...]] = { "integer": int, "number": (int, float), "string": str, @@ -133,13 +134,17 @@ def _validate_against_schema( "array": list, "object": Mapping, } - required = schema.get("required") or [] - properties = schema.get("properties") or {} + raw_required = schema.get("required") + required: list[JSONValue] = raw_required if isinstance(raw_required, list) else [] + raw_properties = schema.get("properties") + properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {} for key in required: - if key not in payload: + if not isinstance(key, str) or key not in payload: return f"missing required key {key!r}" - expected = (properties.get(key) or {}).get("type") - if expected and expected in type_map: + prop = properties.get(key) + prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {} + expected = prop_schema.get("type") + if isinstance(expected, str) and expected in type_map: if not isinstance(payload[key], type_map[expected]): return ( f"key {key!r} has wrong type: " @@ -152,7 +157,7 @@ def _validate_against_schema( return None -def test_structured_outputs_bedrock_invoke(compat_result): +def test_structured_outputs_bedrock_invoke(compat_result: CompatResult) -> None: """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" @@ -181,7 +186,7 @@ def test_structured_outputs_bedrock_invoke(compat_result): extra_args=["--json-schema", SCHEMA_JSON], ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py index ec04c724193..09416cb4803 100644 --- a/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py +++ b/tests/e2e/claude_code/structured_outputs/test_vertex_ai.py @@ -49,8 +49,7 @@ from __future__ import annotations import json import os -import re -from typing import Any, Mapping, Optional, Sequence, Tuple +from typing import Mapping, Optional, Sequence import pytest @@ -59,6 +58,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,7 +76,7 @@ VERTEX_AI_MODELS = [ # integer schema gives every tier (including Haiku) enough headroom # that schema satisfaction is essentially deterministic, isolating # failures to the proxy / transport. -SCHEMA = { +SCHEMA: dict[str, JSONValue] = { "type": "object", "properties": {"answer": {"type": "integer"}}, "required": ["answer"], @@ -91,8 +92,8 @@ PROMPT = "What is 2 + 2? Reply only via the structured output." def _extract_structured_output( - events: Sequence[Mapping[str, Any]], -) -> Optional[Mapping[str, Any]]: + events: Sequence[Mapping[str, JSONValue]], +) -> Optional[Mapping[str, JSONValue]]: """Return the `structured_output` payload from the last `result` event. Claude Code emits its terminal stream-json line as @@ -113,7 +114,7 @@ def _extract_structured_output( def _validate_against_schema( - payload: Mapping[str, Any], schema: Mapping[str, Any] + payload: Mapping[str, JSONValue], schema: Mapping[str, JSONValue] ) -> Optional[str]: """Tiny shape validator covering the subset we actually need. @@ -125,7 +126,7 @@ def _validate_against_schema( not a LiteLLM-proxy bug, so a deeper check would only add false failures on the wrong axis. """ - type_map = { + type_map: dict[str, type[object] | tuple[type[object], ...]] = { "integer": int, "number": (int, float), "string": str, @@ -133,13 +134,17 @@ def _validate_against_schema( "array": list, "object": Mapping, } - required = schema.get("required") or [] - properties = schema.get("properties") or {} + raw_required = schema.get("required") + required: list[JSONValue] = raw_required if isinstance(raw_required, list) else [] + raw_properties = schema.get("properties") + properties: dict[str, JSONValue] = raw_properties if isinstance(raw_properties, dict) else {} for key in required: - if key not in payload: + if not isinstance(key, str) or key not in payload: return f"missing required key {key!r}" - expected = (properties.get(key) or {}).get("type") - if expected and expected in type_map: + prop = properties.get(key) + prop_schema: dict[str, JSONValue] = prop if isinstance(prop, dict) else {} + expected = prop_schema.get("type") + if isinstance(expected, str) and expected in type_map: if not isinstance(payload[key], type_map[expected]): return ( f"key {key!r} has wrong type: " @@ -152,7 +157,7 @@ def _validate_against_schema( return None -def test_structured_outputs_vertex_ai(compat_result): +def test_structured_outputs_vertex_ai(compat_result: CompatResult) -> None: """Drive `claude --json-schema ...` against the LiteLLM proxy and assert the trailing `result` event contains a schema-conforming `structured_output`.""" @@ -181,7 +186,7 @@ def test_structured_outputs_vertex_ai(compat_result): extra_args=["--json-schema", SCHEMA_JSON], ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking/test_anthropic.py b/tests/e2e/claude_code/thinking/test_anthropic.py index 1090d1b384e..7f6ca6df27a 100644 --- a/tests/e2e/claude_code/thinking/test_anthropic.py +++ b/tests/e2e/claude_code/thinking/test_anthropic.py @@ -21,7 +21,7 @@ still sees three rows for this (feature, provider). from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -30,6 +30,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -60,14 +62,14 @@ THINKING_PROMPT = ( ) -def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `thinking` content block.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -76,7 +78,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_thinking_anthropic(compat_result): +def test_thinking_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -103,7 +105,7 @@ def test_thinking_anthropic(compat_result): extra_args=THINKING_ARGS, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking/test_azure.py b/tests/e2e/claude_code/thinking/test_azure.py index 1fd5138d574..f9344d7ba96 100644 --- a/tests/e2e/claude_code/thinking/test_azure.py +++ b/tests/e2e/claude_code/thinking/test_azure.py @@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -33,6 +33,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -50,12 +52,12 @@ THINKING_PROMPT = ( ) -def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -64,7 +66,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_thinking_azure(compat_result): +def test_thinking_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -91,7 +93,7 @@ def test_thinking_azure(compat_result): extra_args=THINKING_ARGS, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking/test_bedrock_converse.py b/tests/e2e/claude_code/thinking/test_bedrock_converse.py index 793ce8542da..40f7f74e477 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_converse.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -42,12 +44,12 @@ THINKING_PROMPT = ( ) -def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -56,7 +58,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_thinking_bedrock_converse(compat_result): +def test_thinking_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -83,7 +85,7 @@ def test_thinking_bedrock_converse(compat_result): extra_args=THINKING_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py index e31b60eb004..84faf4cbf63 100644 --- a/tests/e2e/claude_code/thinking/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking/test_bedrock_invoke.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -42,12 +44,12 @@ THINKING_PROMPT = ( ) -def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -56,7 +58,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_thinking_bedrock_invoke(compat_result): +def test_thinking_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -83,7 +85,7 @@ def test_thinking_bedrock_invoke(compat_result): extra_args=THINKING_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking/test_vertex_ai.py b/tests/e2e/claude_code/thinking/test_vertex_ai.py index c5c7df1f9b8..cbaa87ba812 100644 --- a/tests/e2e/claude_code/thinking/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking/test_vertex_ai.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -42,12 +44,12 @@ THINKING_PROMPT = ( ) -def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_thinking_block(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -56,7 +58,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_thinking_vertex_ai(compat_result): +def test_thinking_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -83,7 +85,7 @@ def test_thinking_vertex_ai(compat_result): extra_args=THINKING_ARGS, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py index 2c573ea039e..0f5548fa4a1 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_anthropic.py @@ -25,7 +25,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -34,6 +34,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -72,7 +74,7 @@ TOOL_USE_ARGS = [ def _has_block_type( - events: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, JSONValue]], block_type: str, ) -> bool: """Walk the stream-json events and return True if any assistant @@ -80,8 +82,8 @@ def _has_block_type( for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -90,7 +92,7 @@ def _has_block_type( return False -def test_thinking_with_tool_use_anthropic(compat_result): +def test_thinking_with_tool_use_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and tool use, and assert both `thinking` and `tool_use` content blocks landed in the same turn.""" @@ -119,7 +121,7 @@ def test_thinking_with_tool_use_anthropic(compat_result): extra_args=THINKING_ARGS + TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py index 3d65e82cdec..187c89a441d 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_azure.py @@ -19,7 +19,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -28,6 +28,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -55,14 +57,14 @@ TOOL_USE_ARGS = [ def _has_block_type( - events: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, JSONValue]], block_type: str, ) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -71,7 +73,7 @@ def _has_block_type( return False -def test_thinking_with_tool_use_azure(compat_result): +def test_thinking_with_tool_use_azure(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -97,7 +99,7 @@ def test_thinking_with_tool_use_azure(compat_result): extra_args=THINKING_ARGS + TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py index eb916323546..f845be6cd6c 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -33,6 +33,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -60,14 +62,14 @@ TOOL_USE_ARGS = [ def _has_block_type( - events: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, JSONValue]], block_type: str, ) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -76,7 +78,7 @@ def _has_block_type( return False -def test_thinking_with_tool_use_bedrock_converse(compat_result): +def test_thinking_with_tool_use_bedrock_converse(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -102,7 +104,7 @@ def test_thinking_with_tool_use_bedrock_converse(compat_result): extra_args=THINKING_ARGS + TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py index d1a61a59772..0e21fdb4eb2 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -26,7 +26,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -35,6 +35,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -62,14 +64,14 @@ TOOL_USE_ARGS = [ def _has_block_type( - events: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, JSONValue]], block_type: str, ) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -78,7 +80,7 @@ def _has_block_type( return False -def test_thinking_with_tool_use_bedrock_invoke(compat_result): +def test_thinking_with_tool_use_bedrock_invoke(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -104,7 +106,7 @@ def test_thinking_with_tool_use_bedrock_invoke(compat_result): extra_args=THINKING_ARGS + TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py index 285419c67f7..a1f92bc07b2 100644 --- a/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -33,6 +33,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -60,14 +62,14 @@ TOOL_USE_ARGS = [ def _has_block_type( - events: Sequence[Mapping[str, Any]], + events: Sequence[Mapping[str, JSONValue]], block_type: str, ) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -76,7 +78,7 @@ def _has_block_type( return False -def test_thinking_with_tool_use_vertex_ai(compat_result): +def test_thinking_with_tool_use_vertex_ai(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -102,7 +104,7 @@ def test_thinking_with_tool_use_vertex_ai(compat_result): extra_args=THINKING_ARGS + TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_search/test_anthropic.py b/tests/e2e/claude_code/tool_search/test_anthropic.py index 3495c882e06..d9b0d091eb3 100644 --- a/tests/e2e/claude_code/tool_search/test_anthropic.py +++ b/tests/e2e/claude_code/tool_search/test_anthropic.py @@ -47,6 +47,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, @@ -62,7 +63,7 @@ ANTHROPIC_MODELS = [ ] -def test_tool_search_anthropic(compat_result): +def test_tool_search_anthropic(compat_result: CompatResult) -> None: """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Anthropic tier.""" @@ -83,7 +84,7 @@ def test_tool_search_anthropic(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) shape_error = assert_tool_search_shape(result) diff --git a/tests/e2e/claude_code/tool_search/test_azure.py b/tests/e2e/claude_code/tool_search/test_azure.py index 1d9cb5673c5..11f403989b2 100644 --- a/tests/e2e/claude_code/tool_search/test_azure.py +++ b/tests/e2e/claude_code/tool_search/test_azure.py @@ -47,6 +47,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, @@ -62,7 +63,7 @@ AZURE_MODELS = [ ] -def test_tool_search_azure(compat_result): +def test_tool_search_azure(compat_result: CompatResult) -> None: """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Azure (Microsoft Foundry) tier.""" @@ -83,7 +84,7 @@ def test_tool_search_azure(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) shape_error = assert_tool_search_shape(result) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py index 5ca0792529a..8520f0bea5a 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_converse.py @@ -47,6 +47,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, @@ -62,7 +63,7 @@ BEDROCK_CONVERSE_MODELS = [ ] -def test_tool_search_bedrock_converse(compat_result): +def test_tool_search_bedrock_converse(compat_result: CompatResult) -> None: """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Converse) tier.""" @@ -83,7 +84,7 @@ def test_tool_search_bedrock_converse(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) shape_error = assert_tool_search_shape(result) diff --git a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py index 21bb33e34bd..eadf37f54b1 100644 --- a/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_search/test_bedrock_invoke.py @@ -47,6 +47,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, @@ -62,7 +63,7 @@ BEDROCK_INVOKE_MODELS = [ ] -def test_tool_search_bedrock_invoke(compat_result): +def test_tool_search_bedrock_invoke(compat_result: CompatResult) -> None: """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Bedrock (Invoke) tier.""" @@ -83,7 +84,7 @@ def test_tool_search_bedrock_invoke(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) shape_error = assert_tool_search_shape(result) diff --git a/tests/e2e/claude_code/tool_search/test_vertex_ai.py b/tests/e2e/claude_code/tool_search/test_vertex_ai.py index f91400b1817..5e153825da9 100644 --- a/tests/e2e/claude_code/tool_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_search/test_vertex_ai.py @@ -47,6 +47,7 @@ import os import pytest +from claude_code.conftest import CompatResult from claude_code.http_probe import ( assert_tool_search_shape, probe_tool_search, @@ -62,7 +63,7 @@ VERTEX_AI_MODELS = [ ] -def test_tool_search_vertex_ai(compat_result): +def test_tool_search_vertex_ai(compat_result: CompatResult) -> None: """Probe `/v1/messages` with a `tool_search_tool_regex_20251119` tool and assert the proxy + upstream accept it for every Vertex AI tier.""" @@ -83,7 +84,7 @@ def test_tool_search_vertex_ai(compat_result): pytrace=False, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: result = probe_tool_search(base_url=base_url, api_key=api_key, model=model) shape_error = assert_tool_search_shape(result) diff --git a/tests/e2e/claude_code/tool_use/test_anthropic.py b/tests/e2e/claude_code/tool_use/test_anthropic.py index 7d2aa4be683..8f09ebec74e 100644 --- a/tests/e2e/claude_code/tool_use/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use/test_anthropic.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -57,14 +59,14 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` content block.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -73,7 +75,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_tool_use_anthropic(compat_result): +def test_tool_use_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -100,7 +102,7 @@ def test_tool_use_anthropic(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use/test_azure.py b/tests/e2e/claude_code/tool_use/test_azure.py index 484f50a5508..486ddfd2133 100644 --- a/tests/e2e/claude_code/tool_use/test_azure.py +++ b/tests/e2e/claude_code/tool_use/test_azure.py @@ -20,7 +20,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -29,6 +29,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -53,12 +55,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -67,7 +69,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_tool_use_azure(compat_result): +def test_tool_use_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -94,7 +96,7 @@ def test_tool_use_azure(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py index 7d1b58fce90..dc28a3bf9fe 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_converse.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -49,12 +51,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -63,7 +65,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_tool_use_bedrock_converse(compat_result): +def test_tool_use_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -90,7 +92,7 @@ def test_tool_use_bedrock_converse(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py index 7d2b72b951d..9cb9aa19737 100644 --- a/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use/test_bedrock_invoke.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -49,12 +51,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -63,7 +65,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_tool_use_bedrock_invoke(compat_result): +def test_tool_use_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -90,7 +92,7 @@ def test_tool_use_bedrock_invoke(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use/test_vertex_ai.py b/tests/e2e/claude_code/tool_use/test_vertex_ai.py index 0a8ecc9f7a7..aa4c0779c4f 100644 --- a/tests/e2e/claude_code/tool_use/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use/test_vertex_ai.py @@ -16,7 +16,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -25,6 +25,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -49,12 +51,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -63,7 +65,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_tool_use_vertex_ai(compat_result): +def test_tool_use_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -90,7 +92,7 @@ def test_tool_use_vertex_ai(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py index 9aa94c89241..24b8c24a2da 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_anthropic.py @@ -26,7 +26,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -35,6 +35,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,14 +67,14 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` content block.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -81,7 +83,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: +def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int: """Count `input_json_delta` records among the `stream_event` entries. Zero means the proxy collapsed the streamed tool input into a single complete block instead of forwarding the incremental @@ -94,12 +96,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: for inner in inner_events if isinstance(inner, Mapping) and inner.get("type") == "content_block_delta" - and isinstance(inner.get("delta"), Mapping) - and inner["delta"].get("type") == "input_json_delta" + and isinstance(delta := inner.get("delta"), Mapping) + and delta.get("type") == "input_json_delta" ) -def test_tool_use_streaming_anthropic(compat_result): +def test_tool_use_streaming_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the proxy preserves fine-grained tool streaming end-to-end.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -126,7 +128,7 @@ def test_tool_use_streaming_anthropic(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use_streaming/test_azure.py b/tests/e2e/claude_code/tool_use_streaming/test_azure.py index c73062b72cd..3965f29d4a8 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_azure.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_azure.py @@ -18,7 +18,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -27,6 +27,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -52,12 +54,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -66,7 +68,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: +def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int: """Count `input_json_delta` records among the `stream_event` entries. Zero means the proxy collapsed the streamed tool input into a single complete block instead of forwarding the incremental @@ -79,12 +81,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: for inner in inner_events if isinstance(inner, Mapping) and inner.get("type") == "content_block_delta" - and isinstance(inner.get("delta"), Mapping) - and inner["delta"].get("type") == "input_json_delta" + and isinstance(delta := inner.get("delta"), Mapping) + and delta.get("type") == "input_json_delta" ) -def test_tool_use_streaming_azure(compat_result): +def test_tool_use_streaming_azure(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -109,7 +111,7 @@ def test_tool_use_streaming_azure(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py index 3642551c7c3..fb6b97ff2e2 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -24,7 +24,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -33,6 +33,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -58,12 +60,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -72,7 +74,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: +def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int: """Count `input_json_delta` records among the `stream_event` entries. Zero means the proxy collapsed the streamed tool input into a single complete block instead of forwarding the incremental @@ -85,12 +87,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: for inner in inner_events if isinstance(inner, Mapping) and inner.get("type") == "content_block_delta" - and isinstance(inner.get("delta"), Mapping) - and inner["delta"].get("type") == "input_json_delta" + and isinstance(delta := inner.get("delta"), Mapping) + and delta.get("type") == "input_json_delta" ) -def test_tool_use_streaming_bedrock_converse(compat_result): +def test_tool_use_streaming_bedrock_converse(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -115,7 +117,7 @@ def test_tool_use_streaming_bedrock_converse(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py index af4689b2847..0a870ff43a2 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -22,7 +22,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -31,6 +31,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -56,12 +58,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -70,7 +72,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: +def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int: """Count `input_json_delta` records among the `stream_event` entries. Zero means the proxy collapsed the streamed tool input into a single complete block instead of forwarding the incremental @@ -83,12 +85,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: for inner in inner_events if isinstance(inner, Mapping) and inner.get("type") == "content_block_delta" - and isinstance(inner.get("delta"), Mapping) - and inner["delta"].get("type") == "input_json_delta" + and isinstance(delta := inner.get("delta"), Mapping) + and delta.get("type") == "input_json_delta" ) -def test_tool_use_streaming_bedrock_invoke(compat_result): +def test_tool_use_streaming_bedrock_invoke(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -113,7 +115,7 @@ def test_tool_use_streaming_bedrock_invoke(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py index 19ef9a4e90e..c360b740c56 100644 --- a/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py +++ b/tests/e2e/claude_code/tool_use_streaming/test_vertex_ai.py @@ -21,7 +21,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -30,6 +30,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -55,12 +57,12 @@ TOOL_USE_ARGS = [ ] -def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_tool_use_event(events: Sequence[Mapping[str, JSONValue]]) -> bool: for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -69,7 +71,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: +def _count_input_json_deltas(events: Sequence[Mapping[str, JSONValue]]) -> int: """Count `input_json_delta` records among the `stream_event` entries. Zero means the proxy collapsed the streamed tool input into a single complete block instead of forwarding the incremental @@ -82,12 +84,12 @@ def _count_input_json_deltas(events: Sequence[Mapping[str, Any]]) -> int: for inner in inner_events if isinstance(inner, Mapping) and inner.get("type") == "content_block_delta" - and isinstance(inner.get("delta"), Mapping) - and inner["delta"].get("type") == "input_json_delta" + and isinstance(delta := inner.get("delta"), Mapping) + and delta.get("type") == "input_json_delta" ) -def test_tool_use_streaming_vertex_ai(compat_result): +def test_tool_use_streaming_vertex_ai(compat_result: CompatResult) -> None: base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) if not base_url or not api_key: @@ -112,7 +114,7 @@ def test_tool_use_streaming_vertex_ai(compat_result): extra_args=TOOL_USE_ARGS, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/vision/test_anthropic.py b/tests/e2e/claude_code/vision/test_anthropic.py index 650940248ea..f980d432da8 100644 --- a/tests/e2e/claude_code/vision/test_anthropic.py +++ b/tests/e2e/claude_code/vision/test_anthropic.py @@ -34,6 +34,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -85,7 +86,7 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" -def test_vision_anthropic(compat_result): +def test_vision_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -115,7 +116,7 @@ def test_vision_anthropic(compat_result): stdin_input=_build_stdin_input(), ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/vision/test_azure.py b/tests/e2e/claude_code/vision/test_azure.py index 3b03c0f2b35..3db139648a1 100644 --- a/tests/e2e/claude_code/vision/test_azure.py +++ b/tests/e2e/claude_code/vision/test_azure.py @@ -34,6 +34,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -85,7 +86,7 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" -def test_vision_azure(compat_result): +def test_vision_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -115,7 +116,7 @@ def test_vision_azure(compat_result): stdin_input=_build_stdin_input(), ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/vision/test_bedrock_converse.py b/tests/e2e/claude_code/vision/test_bedrock_converse.py index 4201f9e64fc..b76f01f29d0 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_converse.py +++ b/tests/e2e/claude_code/vision/test_bedrock_converse.py @@ -34,6 +34,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -85,7 +86,7 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" -def test_vision_bedrock_converse(compat_result): +def test_vision_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -115,7 +116,7 @@ def test_vision_bedrock_converse(compat_result): stdin_input=_build_stdin_input(), ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/vision/test_bedrock_invoke.py b/tests/e2e/claude_code/vision/test_bedrock_invoke.py index d2e641f1462..5728ee3a4b5 100644 --- a/tests/e2e/claude_code/vision/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/vision/test_bedrock_invoke.py @@ -34,6 +34,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -85,7 +86,7 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" -def test_vision_bedrock_invoke(compat_result): +def test_vision_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -115,7 +116,7 @@ def test_vision_bedrock_invoke(compat_result): stdin_input=_build_stdin_input(), ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/vision/test_vertex_ai.py b/tests/e2e/claude_code/vision/test_vertex_ai.py index a39ef1a34b7..b63db249a4e 100644 --- a/tests/e2e/claude_code/vision/test_vertex_ai.py +++ b/tests/e2e/claude_code/vision/test_vertex_ai.py @@ -34,6 +34,7 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -85,7 +86,7 @@ def _build_stdin_input() -> str: return json.dumps(user_event) + "\n" -def test_vision_vertex_ai(compat_result): +def test_vision_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy with an image attached via stream-json input and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -115,7 +116,7 @@ def test_vision_vertex_ai(compat_result): stdin_input=_build_stdin_input(), ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/web_search/test_anthropic.py b/tests/e2e/claude_code/web_search/test_anthropic.py index b8fa806f923..691d345929e 100644 --- a/tests/e2e/claude_code/web_search/test_anthropic.py +++ b/tests/e2e/claude_code/web_search/test_anthropic.py @@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -37,6 +37,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] WEB_SEARCH_TOOL_NAME = "WebSearch" -def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` block calling `WebSearch`.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_web_search_anthropic(compat_result): +def test_web_search_anthropic(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the @@ -115,7 +117,7 @@ def test_web_search_anthropic(compat_result): extra_args=WEB_SEARCH_ARGS, ) - failures = [] + failures: list[str] = [] for model in ANTHROPIC_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/web_search/test_azure.py b/tests/e2e/claude_code/web_search/test_azure.py index e70dc848dcf..eb77e6987bb 100644 --- a/tests/e2e/claude_code/web_search/test_azure.py +++ b/tests/e2e/claude_code/web_search/test_azure.py @@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -37,6 +37,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] WEB_SEARCH_TOOL_NAME = "WebSearch" -def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` block calling `WebSearch`.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_web_search_azure(compat_result): +def test_web_search_azure(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the @@ -115,7 +117,7 @@ def test_web_search_azure(compat_result): extra_args=WEB_SEARCH_ARGS, ) - failures = [] + failures: list[str] = [] for model in AZURE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/web_search/test_bedrock_converse.py b/tests/e2e/claude_code/web_search/test_bedrock_converse.py index cbeea03df40..3cfb32dcf44 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_converse.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_converse.py @@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -37,6 +37,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] WEB_SEARCH_TOOL_NAME = "WebSearch" -def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` block calling `WebSearch`.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_web_search_bedrock_converse(compat_result): +def test_web_search_bedrock_converse(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the @@ -115,7 +117,7 @@ def test_web_search_bedrock_converse(compat_result): extra_args=WEB_SEARCH_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_CONVERSE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py index 86068e1e22b..9cee81a0472 100644 --- a/tests/e2e/claude_code/web_search/test_bedrock_invoke.py +++ b/tests/e2e/claude_code/web_search/test_bedrock_invoke.py @@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -37,6 +37,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] WEB_SEARCH_TOOL_NAME = "WebSearch" -def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` block calling `WebSearch`.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_web_search_bedrock_invoke(compat_result): +def test_web_search_bedrock_invoke(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the @@ -115,7 +117,7 @@ def test_web_search_bedrock_invoke(compat_result): extra_args=WEB_SEARCH_ARGS, ) - failures = [] + failures: list[str] = [] for model in BEDROCK_INVOKE_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError): diff --git a/tests/e2e/claude_code/web_search/test_vertex_ai.py b/tests/e2e/claude_code/web_search/test_vertex_ai.py index a33515771f3..69d7b7454c0 100644 --- a/tests/e2e/claude_code/web_search/test_vertex_ai.py +++ b/tests/e2e/claude_code/web_search/test_vertex_ai.py @@ -28,7 +28,7 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations import os -from typing import Any, Mapping, Sequence +from typing import Mapping, Sequence import pytest @@ -37,6 +37,8 @@ from claude_code.cli_driver import ( failure_diagnostic, run_claude_models_parallel, ) +from claude_code.conftest import CompatResult +from claude_code.json_types import JSONValue PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -65,14 +67,14 @@ WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] WEB_SEARCH_TOOL_NAME = "WebSearch" -def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: +def _has_web_search_tool_use(events: Sequence[Mapping[str, JSONValue]]) -> bool: """Walk the stream-json events and return True if any assistant message included a `tool_use` block calling `WebSearch`.""" for event in events: if event.get("type") != "assistant": continue - message = event.get("message") or {} - content = message.get("content") + message = event.get("message") + content = message.get("content") if isinstance(message, dict) else None if not isinstance(content, list): continue for block in content: @@ -86,7 +88,7 @@ def _has_web_search_tool_use(events: Sequence[Mapping[str, Any]]) -> bool: return False -def test_web_search_vertex_ai(compat_result): +def test_web_search_vertex_ai(compat_result: CompatResult) -> None: """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream emitted a `tool_use` block calling `WebSearch`, proving the proxy preserved both the request-side tool definition and the @@ -115,7 +117,7 @@ def test_web_search_vertex_ai(compat_result): extra_args=WEB_SEARCH_ARGS, ) - failures = [] + failures: list[str] = [] for model in VERTEX_AI_MODELS: outcome = outcomes[model] if isinstance(outcome, ClaudeCLIError):