diff --git a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py index 87988722b69..dbca0440c21 100644 --- a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -175,17 +175,19 @@ def test_load_results_rejects_missing_results_key(tmp_path): load_results(bad) -def test_build_matrix_1x5_grid_matches_published_sample(): - """Slice 2 acceptance: feeding the per-model results that the four - new provider tests produce reproduces the hand-authored 1x5 sample - that the docs page renders. +def test_build_matrix_6x5_grid_matches_published_sample(): + """Slice 5 acceptance: feeding the per-model results the full v0 + row set produces reproduces the hand-authored 6x5 sample that the + docs page renders. Inputs mirror the structure of `compat-results.json` after a real - run with the proxy configured for all five columns: + run with the proxy configured for all five columns and all six + feature directories: - anthropic, bedrock_invoke, bedrock_converse, vertex_ai: three - per-model `pass` results each (Haiku, Sonnet, Opus). - - azure: three `not_applicable` results — Azure does not host - Claude, so the column is gray on every row. + per-model `pass` results each (Haiku, Sonnet, Opus) for every + feature. + - azure: three `not_applicable` results per feature; Azure does + not host Claude, so the column is gray on every row. The aggregated matrix must equal the checked-in `sample_compatibility-matrix.json` byte-for-byte (after JSON load), @@ -194,6 +196,7 @@ def test_build_matrix_1x5_grid_matches_published_sample(): repo_root = Path(__file__).resolve().parents[1] manifest = load_manifest(repo_root / "manifest.yaml") + feature_ids = [feature["id"] for feature in manifest["features"]] pass_providers = ["anthropic", "bedrock_invoke", "bedrock_converse", "vertex_ai"] models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"] azure_reason = ( @@ -202,31 +205,32 @@ def test_build_matrix_1x5_grid_matches_published_sample(): ) results = [] - for provider in pass_providers: + for feature_id in feature_ids: + for provider in pass_providers: + for model in models: + results.append( + { + "feature_id": feature_id, + "provider": provider, + "nodeid": ( + f"tests/claude_code/{feature_id}/test_{provider}.py" + f"::test[{model}]" + ), + "result": {"status": "pass"}, + } + ) for model in models: results.append( { - "feature_id": "basic_messaging_non_streaming", - "provider": provider, + "feature_id": feature_id, + "provider": "azure", "nodeid": ( - f"tests/claude_code/basic_messaging_non_streaming/test_{provider}.py" + f"tests/claude_code/{feature_id}/test_azure.py" f"::test[{model}]" ), - "result": {"status": "pass"}, + "result": {"status": "not_applicable", "reason": azure_reason}, } ) - for model in models: - results.append( - { - "feature_id": "basic_messaging_non_streaming", - "provider": "azure", - "nodeid": ( - "tests/claude_code/basic_messaging_non_streaming/test_azure.py" - f"::test[{model}]" - ), - "result": {"status": "not_applicable", "reason": azure_reason}, - } - ) matrix = build_matrix( manifest=manifest, diff --git a/tests/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/claude_code/_builder_unit_tests/test_v0_layout.py new file mode 100644 index 00000000000..12cfeba8b60 --- /dev/null +++ b/tests/claude_code/_builder_unit_tests/test_v0_layout.py @@ -0,0 +1,108 @@ +"""Structural tests for the full v0 6x5 matrix layout. + +These tests don't run the `claude` CLI — they only verify that the +shape of the test suite on disk matches what the PRD declares: six +features in the prescribed order, and for each feature a directory +with one test file per provider column. + +Catching layout drift here means the daily-cron VM and the PR gate +both see the same row set the docs page declares. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +MANIFEST_PATH = REPO_ROOT / "manifest.yaml" + +# The PRD's "Features in v0" section, in row order. +EXPECTED_FEATURE_IDS = [ + "basic_messaging_non_streaming", + "basic_messaging_streaming", + "tool_use", + "prompt_caching_5m", + "vision", + "extended_thinking", +] + +# The PRD's column order. Every feature directory must have one +# `test_.py` for each of these. +EXPECTED_PROVIDERS = [ + "anthropic", + "bedrock_invoke", + "bedrock_converse", + "vertex_ai", + "azure", +] + + +@pytest.fixture(scope="module") +def manifest() -> dict: + return yaml.safe_load(MANIFEST_PATH.read_text()) + + +def test_manifest_lists_all_six_v0_features_in_order(manifest): + ids = [feature["id"] for feature in manifest["features"]] + assert ids == EXPECTED_FEATURE_IDS + + +def test_manifest_lists_all_five_v0_providers_in_order(manifest): + assert manifest["providers"] == EXPECTED_PROVIDERS + + +def test_manifest_every_feature_has_human_readable_name(manifest): + for feature in manifest["features"]: + assert isinstance(feature["name"], str) and feature["name"].strip() + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +def test_feature_directory_exists(feature_id): + feature_dir = REPO_ROOT / feature_id + assert feature_dir.is_dir(), f"missing feature directory: {feature_dir}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) +def test_per_provider_test_file_exists(feature_id, provider): + test_file = REPO_ROOT / feature_id / f"test_{provider}.py" + assert test_file.is_file(), f"missing per-provider test file: {test_file}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +def test_feature_directory_has_init_file(feature_id): + """Each feature directory needs an __init__.py so pytest collects + the per-provider test files as a package — matches the layout + established by `basic_messaging_non_streaming/`.""" + init_file = REPO_ROOT / feature_id / "__init__.py" + assert init_file.is_file(), f"missing __init__.py: {init_file}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +@pytest.mark.parametrize("provider", EXPECTED_PROVIDERS) +def test_per_provider_test_file_imports_and_parametrizes_three_models( + feature_id, provider +): + """Every test file must reference the three Claude tiers required + by the PRD: Haiku 4.5, Sonnet 4.6, Opus 4.7. Implementations may + use plain aliases or per-provider-suffixed aliases (e.g. + `claude-opus-4-7-bedrock-invoke`), so we check for the tier + substrings rather than exact alias names.""" + text = (REPO_ROOT / feature_id / f"test_{provider}.py").read_text() + for tier in ("haiku-4-5", "sonnet-4-6", "opus-4-7"): + assert ( + tier in text + ), f"{feature_id}/test_{provider}.py does not reference {tier}" + + +@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) +def test_azure_test_file_reports_not_applicable(feature_id): + """Azure OpenAI Service does not host Claude on any v0 feature, so + every Azure cell in the v0 matrix is `not_applicable`. Pin that + here so a future "let's just call the proxy and see what happens" + edit doesn't silently turn the gray cells red.""" + text = (REPO_ROOT / feature_id / "test_azure.py").read_text() + assert '"status": "not_applicable"' in text diff --git a/tests/claude_code/basic_messaging_streaming/__init__.py b/tests/claude_code/basic_messaging_streaming/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/claude_code/basic_messaging_streaming/test_anthropic.py new file mode 100644 index 00000000000..7a7ebfbe3db --- /dev/null +++ b/tests/claude_code/basic_messaging_streaming/test_anthropic.py @@ -0,0 +1,103 @@ +"""basic_messaging_streaming x Anthropic. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes to Anthropic, and +report the outcome via `compat_result`. + +The CLI is run with `--print --output-format stream-json`, which streams +incremental events as the upstream produces tokens. The cell goes green +only when every Claude tier returns a non-empty reply over a streamed +wire (i.e. at least one stream-json event is observed). This catches +regressions where the proxy buffers the full response before flushing, +silently degrading the streaming experience customers rely on. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/basic_messaging_streaming/test_anthropic.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +@pytest.mark.parametrize("model", ANTHROPIC_MODELS) +def test_basic_messaging_streaming_anthropic(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (at least one stream-json event observed). + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Count from 1 to 5, one number per line.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.events: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no stream-json events emitted; streaming wire silent", + } + ) + pytest.fail(f"no stream events for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/basic_messaging_streaming/test_azure.py b/tests/claude_code/basic_messaging_streaming/test_azure.py new file mode 100644 index 00000000000..16ab0dfa3b8 --- /dev/null +++ b/tests/claude_code/basic_messaging_streaming/test_azure.py @@ -0,0 +1,37 @@ +"""basic_messaging_streaming x Azure. + +Azure (Azure OpenAI Service) does not host Anthropic Claude models — +the platform's first-party catalog is OpenAI models, plus a smaller set +of Microsoft and partner models. There is no supported route for the +`claude` CLI to talk to Claude through Azure via LiteLLM, so every +(model, Azure) combination for `basic_messaging_streaming` reports +`not_applicable` rather than `fail`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/basic_messaging_streaming/test_azure.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import pytest + +AZURE_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +NOT_APPLICABLE_REASON = ( + "Azure OpenAI Service does not host Anthropic Claude models. " + "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +) + + +@pytest.mark.parametrize("model", AZURE_MODELS) +def test_basic_messaging_streaming_azure(compat_result, model): + """Report `not_applicable` for every (model, Azure) combination.""" + compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py new file mode 100644 index 00000000000..c3b1bcdf52c --- /dev/null +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -0,0 +1,97 @@ +"""basic_messaging_streaming x Bedrock (Converse). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to AWS +Bedrock via the unified `Converse` API path, and report the outcome via +`compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) +def test_basic_messaging_streaming_bedrock_converse(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (at least one stream-json event observed). + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Count from 1 to 5, one number per line.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.events: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no stream-json events emitted; streaming wire silent", + } + ) + pytest.fail(f"no stream events for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py new file mode 100644 index 00000000000..6f0aa0fec0b --- /dev/null +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -0,0 +1,97 @@ +"""basic_messaging_streaming x Bedrock (Invoke). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to AWS +Bedrock via the legacy `InvokeModel` API path, and report the outcome +via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) +def test_basic_messaging_streaming_bedrock_invoke(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (at least one stream-json event observed). + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Count from 1 to 5, one number per line.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.events: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no stream-json events emitted; streaming wire silent", + } + ) + pytest.fail(f"no stream events for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py new file mode 100644 index 00000000000..b2c914e2917 --- /dev/null +++ b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -0,0 +1,97 @@ +"""basic_messaging_streaming x Vertex AI. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +Anthropic's models on Google Cloud Vertex AI, and report the outcome +via `compat_result`. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/basic_messaging_streaming/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +@pytest.mark.parametrize("model", VERTEX_AI_MODELS) +def test_basic_messaging_streaming_vertex_ai(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (at least one stream-json event observed). + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Count from 1 to 5, one number per line.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.events: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no stream-json events emitted; streaming wire silent", + } + ) + pytest.fail(f"no stream events for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/extended_thinking/__init__.py b/tests/claude_code/extended_thinking/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/extended_thinking/test_anthropic.py b/tests/claude_code/extended_thinking/test_anthropic.py new file mode 100644 index 00000000000..39511f484e9 --- /dev/null +++ b/tests/claude_code/extended_thinking/test_anthropic.py @@ -0,0 +1,114 @@ +"""extended_thinking x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, enable extended thinking via `MAX_THINKING_TOKENS`, and +assert that the upstream returned a `thinking` content block. This +proves the proxy preserves Anthropic's `thinking` request parameter +and the upstream response's `thinking` content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/extended_thinking/test_anthropic.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# A small budget is enough to surface a non-empty thinking block on +# even a trivial reasoning prompt; the test cares about wire shape, not +# answer quality. +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_PROMPT = ( + "Think step by step: if I have three apples and eat two, how many remain? " + "Answer with the single digit only." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `thinking` content block.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +@pytest.mark.parametrize("model", ANTHROPIC_MODELS) +def test_extended_thinking_anthropic(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=THINKING_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_thinking_block(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no `thinking` content block observed in stream-json events", + } + ) + pytest.fail(f"no thinking block for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/extended_thinking/test_azure.py b/tests/claude_code/extended_thinking/test_azure.py new file mode 100644 index 00000000000..d7d28a4311a --- /dev/null +++ b/tests/claude_code/extended_thinking/test_azure.py @@ -0,0 +1,35 @@ +"""extended_thinking x Azure. + +Azure OpenAI Service does not host Anthropic Claude models, so the +`extended_thinking` × Azure cell is structurally `not_applicable` — +there is no route for the `claude` CLI to talk to Claude through Azure +via LiteLLM. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/extended_thinking/test_azure.py + ^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import pytest + +AZURE_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +NOT_APPLICABLE_REASON = ( + "Azure OpenAI Service does not host Anthropic Claude models. " + "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +) + + +@pytest.mark.parametrize("model", AZURE_MODELS) +def test_extended_thinking_azure(compat_result, model): + """Report `not_applicable` for every (model, Azure) combination.""" + compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) diff --git a/tests/claude_code/extended_thinking/test_bedrock_converse.py b/tests/claude_code/extended_thinking/test_bedrock_converse.py new file mode 100644 index 00000000000..a6941a90afe --- /dev/null +++ b/tests/claude_code/extended_thinking/test_bedrock_converse.py @@ -0,0 +1,108 @@ +"""extended_thinking x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, +enable extended thinking via `MAX_THINKING_TOKENS`, and assert that the +upstream returned a `thinking` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/extended_thinking/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_PROMPT = ( + "Think step by step: if I have three apples and eat two, how many remain? " + "Answer with the single digit only." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) +def test_extended_thinking_bedrock_converse(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=THINKING_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_thinking_block(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no `thinking` content block observed in stream-json events", + } + ) + pytest.fail(f"no thinking block for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/extended_thinking/test_bedrock_invoke.py b/tests/claude_code/extended_thinking/test_bedrock_invoke.py new file mode 100644 index 00000000000..7fd67aa8efd --- /dev/null +++ b/tests/claude_code/extended_thinking/test_bedrock_invoke.py @@ -0,0 +1,108 @@ +"""extended_thinking x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +enable extended thinking via `MAX_THINKING_TOKENS`, and assert that the +upstream returned a `thinking` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/extended_thinking/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_PROMPT = ( + "Think step by step: if I have three apples and eat two, how many remain? " + "Answer with the single digit only." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) +def test_extended_thinking_bedrock_invoke(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=THINKING_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_thinking_block(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no `thinking` content block observed in stream-json events", + } + ) + pytest.fail(f"no thinking block for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/extended_thinking/test_vertex_ai.py b/tests/claude_code/extended_thinking/test_vertex_ai.py new file mode 100644 index 00000000000..ee36470d4b0 --- /dev/null +++ b/tests/claude_code/extended_thinking/test_vertex_ai.py @@ -0,0 +1,108 @@ +"""extended_thinking x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, enable +extended thinking via `MAX_THINKING_TOKENS`, and assert that the +upstream returned a `thinking` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/extended_thinking/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_PROMPT = ( + "Think step by step: if I have three apples and eat two, how many remain? " + "Answer with the single digit only." +) + + +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + +@pytest.mark.parametrize("model", VERTEX_AI_MODELS) +def test_extended_thinking_vertex_ai(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=THINKING_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_thinking_block(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no `thinking` content block observed in stream-json events", + } + ) + pytest.fail(f"no thinking block for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/manifest.yaml b/tests/claude_code/manifest.yaml index a9ae1e0d9fe..fb70ce4623f 100644 --- a/tests/claude_code/manifest.yaml +++ b/tests/claude_code/manifest.yaml @@ -24,3 +24,13 @@ providers: features: - id: basic_messaging_non_streaming name: Basic messaging (non-streaming) + - id: basic_messaging_streaming + name: Basic messaging (streaming) + - id: tool_use + name: Tool use + - id: prompt_caching_5m + name: Prompt caching (5m TTL) + - id: vision + name: Vision + - id: extended_thinking + name: Extended thinking diff --git a/tests/claude_code/prompt_caching_5m/__init__.py b/tests/claude_code/prompt_caching_5m/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/prompt_caching_5m/test_anthropic.py b/tests/claude_code/prompt_caching_5m/test_anthropic.py new file mode 100644 index 00000000000..d5b5f75fe3f --- /dev/null +++ b/tests/claude_code/prompt_caching_5m/test_anthropic.py @@ -0,0 +1,111 @@ +"""prompt_caching_5m x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0 — i.e. +the proxy preserves Claude Code's `cache_control` annotations end-to-end +and the upstream actually honored them. This is the 5-minute (default) +cache TTL row. + +Claude Code itself sets `cache_control: { type: "ephemeral" }` on the +system prompt and the most recent user turn for every request, so a +single live invocation is enough to surface a cache-creation count on +the first call and a cache-read count on a warm follow-up call. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_5m/test_anthropic.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + """Return cache_creation_input_tokens + cache_read_input_tokens from + the upstream usage block, or 0 if the keys are missing.""" + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +@pytest.mark.parametrize("model", ANTHROPIC_MODELS) +def test_prompt_caching_5m_anthropic(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Reply with the single word 'pong' and nothing else.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if _cache_tokens(result.usage) <= 0: + compat_result.set( + { + "status": "fail", + "error": ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ), + } + ) + pytest.fail(f"no cache tokens for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/prompt_caching_5m/test_azure.py b/tests/claude_code/prompt_caching_5m/test_azure.py new file mode 100644 index 00000000000..2aa5f0483d4 --- /dev/null +++ b/tests/claude_code/prompt_caching_5m/test_azure.py @@ -0,0 +1,35 @@ +"""prompt_caching_5m x Azure. + +Azure OpenAI Service does not host Anthropic Claude models, so the +`prompt_caching_5m` × Azure cell is structurally `not_applicable` — +there is no route for the `claude` CLI to talk to Claude through Azure +via LiteLLM. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_5m/test_azure.py + ^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import pytest + +AZURE_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +NOT_APPLICABLE_REASON = ( + "Azure OpenAI Service does not host Anthropic Claude models. " + "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +) + + +@pytest.mark.parametrize("model", AZURE_MODELS) +def test_prompt_caching_5m_azure(compat_result, model): + """Report `not_applicable` for every (model, Azure) combination.""" + compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) diff --git a/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py new file mode 100644 index 00000000000..417ea9b10ab --- /dev/null +++ b/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -0,0 +1,102 @@ +"""prompt_caching_5m x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, and +assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_5m/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) +def test_prompt_caching_5m_bedrock_converse(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Reply with the single word 'pong' and nothing else.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if _cache_tokens(result.usage) <= 0: + compat_result.set( + { + "status": "fail", + "error": ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ), + } + ) + pytest.fail(f"no cache tokens for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py new file mode 100644 index 00000000000..c70d4ed3e32 --- /dev/null +++ b/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -0,0 +1,102 @@ +"""prompt_caching_5m x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) +def test_prompt_caching_5m_bedrock_invoke(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Reply with the single word 'pong' and nothing else.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if _cache_tokens(result.usage) <= 0: + compat_result.set( + { + "status": "fail", + "error": ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ), + } + ) + pytest.fail(f"no cache tokens for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/claude_code/prompt_caching_5m/test_vertex_ai.py new file mode 100644 index 00000000000..bf33d89fffe --- /dev/null +++ b/tests/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -0,0 +1,102 @@ +"""prompt_caching_5m x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, and +assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_5m/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +@pytest.mark.parametrize("model", VERTEX_AI_MODELS) +def test_prompt_caching_5m_vertex_ai(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Reply with the single word 'pong' and nothing else.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if _cache_tokens(result.usage) <= 0: + compat_result.set( + { + "status": "fail", + "error": ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ), + } + ) + pytest.fail(f"no cache tokens for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/sample_compatibility-matrix.json b/tests/claude_code/sample_compatibility-matrix.json index 7503bb3c42a..dc87f4c8939 100644 --- a/tests/claude_code/sample_compatibility-matrix.json +++ b/tests/claude_code/sample_compatibility-matrix.json @@ -32,6 +32,116 @@ "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." } } + }, + { + "id": "basic_messaging_streaming", + "name": "Basic messaging (streaming)", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "not_applicable", + "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + } + } + }, + { + "id": "tool_use", + "name": "Tool use", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "not_applicable", + "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + } + } + }, + { + "id": "prompt_caching_5m", + "name": "Prompt caching (5m TTL)", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "not_applicable", + "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + } + } + }, + { + "id": "vision", + "name": "Vision", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "not_applicable", + "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + } + } + }, + { + "id": "extended_thinking", + "name": "Extended thinking", + "providers": { + "anthropic": { + "status": "pass" + }, + "bedrock_invoke": { + "status": "pass" + }, + "bedrock_converse": { + "status": "pass" + }, + "vertex_ai": { + "status": "pass" + }, + "azure": { + "status": "not_applicable", + "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + } + } } ] } diff --git a/tests/claude_code/tool_use/__init__.py b/tests/claude_code/tool_use/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/tool_use/test_anthropic.py b/tests/claude_code/tool_use/test_anthropic.py new file mode 100644 index 00000000000..d2d40654a13 --- /dev/null +++ b/tests/claude_code/tool_use/test_anthropic.py @@ -0,0 +1,112 @@ +"""tool_use x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, ask Claude to invoke a built-in tool (`Bash`), and assert +that the upstream returned a `tool_use` content block. This proves the +proxy preserves Claude Code's tool-call wire shape end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use/test_anthropic.py + ^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Built-in tool-use prompt: ask Claude to use the `Bash` tool. The CLI +# allow-lists the tool via `--allowed-tools` so the run completes without +# an interactive permission prompt. +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` content block.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +@pytest.mark.parametrize("model", ANTHROPIC_MODELS) +def test_tool_use_anthropic(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=TOOL_USE_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_tool_use_event(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no tool_use content block observed in stream-json events", + } + ) + pytest.fail(f"no tool_use for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/tool_use/test_azure.py b/tests/claude_code/tool_use/test_azure.py new file mode 100644 index 00000000000..24f402f5910 --- /dev/null +++ b/tests/claude_code/tool_use/test_azure.py @@ -0,0 +1,34 @@ +"""tool_use x Azure. + +Azure OpenAI Service does not host Anthropic Claude models, so the +`tool_use` × Azure cell is structurally `not_applicable` — there is no +route for the `claude` CLI to talk to Claude through Azure via LiteLLM. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use/test_azure.py + ^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import pytest + +AZURE_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +NOT_APPLICABLE_REASON = ( + "Azure OpenAI Service does not host Anthropic Claude models. " + "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +) + + +@pytest.mark.parametrize("model", AZURE_MODELS) +def test_tool_use_azure(compat_result, model): + """Report `not_applicable` for every (model, Azure) combination.""" + compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) diff --git a/tests/claude_code/tool_use/test_bedrock_converse.py b/tests/claude_code/tool_use/test_bedrock_converse.py new file mode 100644 index 00000000000..9a99c877d59 --- /dev/null +++ b/tests/claude_code/tool_use/test_bedrock_converse.py @@ -0,0 +1,107 @@ +"""tool_use x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, ask +Claude to invoke a built-in tool (`Bash`), and assert that the upstream +returned a `tool_use` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use/test_bedrock_converse.py + ^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) +def test_tool_use_bedrock_converse(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=TOOL_USE_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_tool_use_event(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no tool_use content block observed in stream-json events", + } + ) + pytest.fail(f"no tool_use for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/tool_use/test_bedrock_invoke.py b/tests/claude_code/tool_use/test_bedrock_invoke.py new file mode 100644 index 00000000000..9bc9b66a81c --- /dev/null +++ b/tests/claude_code/tool_use/test_bedrock_invoke.py @@ -0,0 +1,107 @@ +"""tool_use x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +ask Claude to invoke a built-in tool (`Bash`), and assert that the +upstream returned a `tool_use` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use/test_bedrock_invoke.py + ^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) +def test_tool_use_bedrock_invoke(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=TOOL_USE_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_tool_use_event(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no tool_use content block observed in stream-json events", + } + ) + pytest.fail(f"no tool_use for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/tool_use/test_vertex_ai.py b/tests/claude_code/tool_use/test_vertex_ai.py new file mode 100644 index 00000000000..dcfe8d002e5 --- /dev/null +++ b/tests/claude_code/tool_use/test_vertex_ai.py @@ -0,0 +1,107 @@ +"""tool_use x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, ask +Claude to invoke a built-in tool (`Bash`), and assert that the upstream +returned a `tool_use` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use/test_vertex_ai.py + ^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +@pytest.mark.parametrize("model", VERTEX_AI_MODELS) +def test_tool_use_vertex_ai(compat_result, model): + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=TOOL_USE_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_tool_use_event(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no tool_use content block observed in stream-json events", + } + ) + pytest.fail(f"no tool_use for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/vision/__init__.py b/tests/claude_code/vision/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/vision/test_anthropic.py b/tests/claude_code/vision/test_anthropic.py new file mode 100644 index 00000000000..a381f9ed018 --- /dev/null +++ b/tests/claude_code/vision/test_anthropic.py @@ -0,0 +1,97 @@ +"""vision x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, attach a small image via the CLI's `--image` flag, and +assert that the upstream produces a non-empty reply that references the +attached image. This proves the proxy preserves Claude Code's +multimodal content blocks end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/vision/test_anthropic.py + ^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import base64 +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Minimal 1x1 red PNG, base64-encoded. Decoded at test time and written +# to `tmp_path` so the CLI has a real file to attach without requiring +# any image-generation library or a checked-in binary fixture. +RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + + +@pytest.mark.parametrize("model", ANTHROPIC_MODELS) +def test_vision_anthropic(compat_result, model, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + image_path = tmp_path / "red_pixel.png" + image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) + + try: + result = run_claude( + prompt="What single color do you see in the attached image? Answer in one word.", + model=model, + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text on a vision prompt", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/vision/test_azure.py b/tests/claude_code/vision/test_azure.py new file mode 100644 index 00000000000..b6e29b7d02f --- /dev/null +++ b/tests/claude_code/vision/test_azure.py @@ -0,0 +1,34 @@ +"""vision x Azure. + +Azure OpenAI Service does not host Anthropic Claude models, so the +`vision` × Azure cell is structurally `not_applicable` — there is no +route for the `claude` CLI to talk to Claude through Azure via LiteLLM. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/vision/test_azure.py + ^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import pytest + +AZURE_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +NOT_APPLICABLE_REASON = ( + "Azure OpenAI Service does not host Anthropic Claude models. " + "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +) + + +@pytest.mark.parametrize("model", AZURE_MODELS) +def test_vision_azure(compat_result, model): + """Report `not_applicable` for every (model, Azure) combination.""" + compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) diff --git a/tests/claude_code/vision/test_bedrock_converse.py b/tests/claude_code/vision/test_bedrock_converse.py new file mode 100644 index 00000000000..50067e9c594 --- /dev/null +++ b/tests/claude_code/vision/test_bedrock_converse.py @@ -0,0 +1,93 @@ +"""vision x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the unified `Converse` API path, +attach a small image via the CLI's `--image` flag, and assert that the +upstream produces a non-empty reply. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/vision/test_bedrock_converse.py + ^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import base64 +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + + +@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) +def test_vision_bedrock_converse(compat_result, model, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + image_path = tmp_path / "red_pixel.png" + image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) + + try: + result = run_claude( + prompt="What single color do you see in the attached image? Answer in one word.", + model=model, + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text on a vision prompt", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/vision/test_bedrock_invoke.py b/tests/claude_code/vision/test_bedrock_invoke.py new file mode 100644 index 00000000000..e0032e24145 --- /dev/null +++ b/tests/claude_code/vision/test_bedrock_invoke.py @@ -0,0 +1,93 @@ +"""vision x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +attach a small image via the CLI's `--image` flag, and assert that the +upstream produces a non-empty reply. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/vision/test_bedrock_invoke.py + ^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import base64 +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + + +@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) +def test_vision_bedrock_invoke(compat_result, model, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + image_path = tmp_path / "red_pixel.png" + image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) + + try: + result = run_claude( + prompt="What single color do you see in the attached image? Answer in one word.", + model=model, + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text on a vision prompt", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/vision/test_vertex_ai.py b/tests/claude_code/vision/test_vertex_ai.py new file mode 100644 index 00000000000..7e1101e12ce --- /dev/null +++ b/tests/claude_code/vision/test_vertex_ai.py @@ -0,0 +1,93 @@ +"""vision x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models on Google Cloud Vertex AI, attach +a small image via the CLI's `--image` flag, and assert that the +upstream produces a non-empty reply. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/vision/test_vertex_ai.py + ^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import base64 +import os + +import pytest + +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + + +@pytest.mark.parametrize("model", VERTEX_AI_MODELS) +def test_vision_vertex_ai(compat_result, model, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + image_path = tmp_path / "red_pixel.png" + image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) + + try: + result = run_claude( + prompt="What single color do you see in the attached image? Answer in one word.", + model=model, + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text on a vision prompt", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"})