diff --git a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py index dbca0440c21..9aa7a96609d 100644 --- a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -182,12 +182,10 @@ def test_build_matrix_6x5_grid_matches_published_sample(): Inputs mirror the structure of `compat-results.json` after a real run with the proxy configured for all five columns and all six - feature directories: - - anthropic, bedrock_invoke, bedrock_converse, vertex_ai: three - per-model `pass` results each (Haiku, Sonnet, Opus) for every - feature. - - azure: three `not_applicable` results per feature; Azure does - not host Claude, so the column is gray on every row. + feature directories: every (feature, provider, model) cell yields a + `pass`. Anthropic announced Claude in Microsoft Foundry on + 2025-11-18, so the Azure column is now exercised end-to-end like + the others rather than reporting `not_applicable`. The aggregated matrix must equal the checked-in `sample_compatibility-matrix.json` byte-for-byte (after JSON load), @@ -197,16 +195,12 @@ def test_build_matrix_6x5_grid_matches_published_sample(): manifest = load_manifest(repo_root / "manifest.yaml") feature_ids = [feature["id"] for feature in manifest["features"]] - pass_providers = ["anthropic", "bedrock_invoke", "bedrock_converse", "vertex_ai"] + providers = manifest["providers"] models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"] - azure_reason = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." - ) results = [] for feature_id in feature_ids: - for provider in pass_providers: + for provider in providers: for model in models: results.append( { @@ -219,18 +213,6 @@ def test_build_matrix_6x5_grid_matches_published_sample(): "result": {"status": "pass"}, } ) - for model in models: - results.append( - { - "feature_id": feature_id, - "provider": "azure", - "nodeid": ( - f"tests/claude_code/{feature_id}/test_azure.py" - f"::test[{model}]" - ), - "result": {"status": "not_applicable", "reason": azure_reason}, - } - ) matrix = build_matrix( manifest=manifest, diff --git a/tests/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/claude_code/_builder_unit_tests/test_v0_layout.py index 12cfeba8b60..1ae1e8f2a67 100644 --- a/tests/claude_code/_builder_unit_tests/test_v0_layout.py +++ b/tests/claude_code/_builder_unit_tests/test_v0_layout.py @@ -99,10 +99,18 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models( @pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS) -def test_azure_test_file_reports_not_applicable(feature_id): - """Azure OpenAI Service does not host Claude on any v0 feature, so - every Azure cell in the v0 matrix is `not_applicable`. Pin that - here so a future "let's just call the proxy and see what happens" - edit doesn't silently turn the gray cells red.""" +def test_azure_test_file_drives_the_proxy(feature_id): + """Azure (Microsoft Foundry) hosts Anthropic Claude as of 2025-11-18, + so every Azure cell in the v0 matrix exercises a real route through + the LiteLLM proxy — same shape as the other provider columns. Pin + that here so a future regression doesn't silently revert these + cells to the old `not_applicable` boilerplate.""" text = (REPO_ROOT / feature_id / "test_azure.py").read_text() - assert '"status": "not_applicable"' in text + assert "run_claude" in text, ( + f"{feature_id}/test_azure.py must drive the claude CLI via run_claude(); " + "the not_applicable stub was removed when Foundry started hosting Claude." + ) + assert '"status": "not_applicable"' not in text, ( + f"{feature_id}/test_azure.py still reports not_applicable; Microsoft Foundry " + "now hosts Claude (Haiku 4.5, Sonnet 4.6, Opus 4.7), so this row must run." + ) diff --git a/tests/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/claude_code/basic_messaging_non_streaming/test_azure.py index 5b3bd509bac..22b09d24bc5 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_azure.py @@ -1,14 +1,13 @@ -"""basic_messaging_non_streaming x Azure. +"""basic_messaging_non_streaming x Azure (Microsoft Foundry). -Azure (Azure OpenAI Service) does not host Anthropic Claude models — -the platform's first-party catalog is OpenAI models, plus a smaller set -of Microsoft and partner models. There is no supported route for the -`claude` CLI to talk to Claude through Azure via LiteLLM, so every -(model, Azure) combination for `basic_messaging_non_streaming` reports -`not_applicable` rather than `fail`. This is what the matrix's -`not_applicable` state is for: the renderer paints the cell gray with a -tooltip explaining the cell will never apply, distinct from a -genuine regression. +Drive the real `claude` CLI in headless mode against a running LiteLLM +proxy that routes Claude requests to Anthropic's models hosted in +Microsoft Foundry on Azure, and report the outcome via `compat_result`. + +Anthropic announced Claude Haiku 4.5, Sonnet 4.5/4.6, and Opus 4.1/4.6/4.7 +in Microsoft Foundry on 2025-11-18; LiteLLM exposes them via the +`azure_ai/claude-*` provider prefix, which talks to Foundry's +Anthropic-shape `/anthropic/v1/messages` endpoint. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -16,29 +15,91 @@ The (feature, provider) for this cell is inferred from the file path by tests/claude_code/basic_messaging_non_streaming/test_azure.py ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ feature_id provider + +Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus +4.7; the cell only goes green if all three pass. We parametrize over the +three models and the conftest aggregator produces one cell from the +three results, naming the failing model in the error string when any +model fails. """ from __future__ import annotations +import os + import pytest -# Same three Claude tiers the other provider columns parametrize over, -# so the test count per cell stays uniform across the matrix and any -# future "Azure adds Anthropic" announcement only requires flipping the -# body, not the parametrization. -AZURE_MODELS = [ - "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", -] +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude -NOT_APPLICABLE_REASON = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." -) +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +# Per-model aliases registered in the LiteLLM proxy's routing config to +# point at Microsoft Foundry's Anthropic deployments. The driver only +# sends the alias; the proxy is the one that knows the upstream Foundry +# resource URL and API key. +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] @pytest.mark.parametrize("model", AZURE_MODELS) def test_basic_messaging_non_streaming_azure(compat_result, model): - """Report `not_applicable` for every (model, Azure) combination.""" - compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) + """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. + + "Basic messaging" means: send a single user prompt, receive any + non-empty assistant text reply, no tools, no streaming, no thinking. + The whole point of this slice is to prove the path works at all — + so the assertion is intentionally lenient on the reply contents. + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Reply with the single word 'pong' and nothing else.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/basic_messaging_streaming/test_azure.py b/tests/claude_code/basic_messaging_streaming/test_azure.py index 16ab0dfa3b8..96c10439b54 100644 --- a/tests/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_streaming/test_azure.py @@ -1,11 +1,13 @@ -"""basic_messaging_streaming x Azure. +"""basic_messaging_streaming x Azure (Microsoft Foundry). -Azure (Azure OpenAI Service) does not host Anthropic Claude models — -the platform's first-party catalog is OpenAI models, plus a smaller set -of Microsoft and partner models. There is no supported route for the -`claude` CLI to talk to Claude through Azure via LiteLLM, so every -(model, Azure) combination for `basic_messaging_streaming` reports -`not_applicable` rather than `fail`. +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +Anthropic's models hosted in Microsoft Foundry on Azure, and report the +outcome via `compat_result`. + +Foundry exposes Claude on an Anthropic-shape `/anthropic/v1/messages` +endpoint with native SSE streaming; LiteLLM forwards stream events +through the `azure_ai/claude-*` provider unchanged. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -17,21 +19,83 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import os + import pytest -AZURE_MODELS = [ - "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", -] +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude -NOT_APPLICABLE_REASON = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." -) +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] @pytest.mark.parametrize("model", AZURE_MODELS) def test_basic_messaging_streaming_azure(compat_result, model): - """Report `not_applicable` for every (model, Azure) combination.""" - compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) + """Drive the `claude` CLI against the LiteLLM proxy and assert a + non-empty streamed reply (at least one stream-json event observed). + """ + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Count from 1 to 5, one number per line.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.events: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no stream-json events emitted; streaming wire silent", + } + ) + pytest.fail(f"no stream events for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/extended_thinking/test_azure.py b/tests/claude_code/extended_thinking/test_azure.py index d7d28a4311a..6d5e89bfcfa 100644 --- a/tests/claude_code/extended_thinking/test_azure.py +++ b/tests/claude_code/extended_thinking/test_azure.py @@ -1,9 +1,17 @@ -"""extended_thinking x Azure. +"""extended_thinking x Azure (Microsoft Foundry). -Azure OpenAI Service does not host Anthropic Claude models, so the -`extended_thinking` × Azure cell is structurally `not_applicable` — -there is no route for the `claude` CLI to talk to Claude through Azure -via LiteLLM. +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, enable extended thinking via `MAX_THINKING_TOKENS`, and assert +that the upstream returned a `thinking` content block. + +Foundry's Claude deployments advertise `supports_reasoning: true` in +LiteLLM's pricing metadata; the `thinking={"type": "enabled", ...}` +parameter passes through `azure_ai/claude-*` to Foundry's +`/anthropic/v1/messages` endpoint unchanged. Note that +`claude-opus-4-7-preview` documents thinking as not supported on +Foundry; if that lands, this row may flip to a partial pass and we'll +re-evaluate. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -15,21 +23,94 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import os +from typing import Any, Mapping, Sequence + import pytest +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + AZURE_MODELS = [ - "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", ] -NOT_APPLICABLE_REASON = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_PROMPT = ( + "Think step by step: if I have three apples and eat two, how many remain? " + "Answer with the single digit only." ) +def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "thinking": + return True + return False + + @pytest.mark.parametrize("model", AZURE_MODELS) def test_extended_thinking_azure(compat_result, model): - """Report `not_applicable` for every (model, Azure) combination.""" - compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and assert a `thinking` content block was emitted.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=THINKING_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_thinking_block(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no `thinking` content block observed in stream-json events", + } + ) + pytest.fail(f"no thinking block for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/prompt_caching_5m/test_azure.py b/tests/claude_code/prompt_caching_5m/test_azure.py index 2aa5f0483d4..f470d950514 100644 --- a/tests/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/claude_code/prompt_caching_5m/test_azure.py @@ -1,9 +1,16 @@ -"""prompt_caching_5m x Azure. +"""prompt_caching_5m x Azure (Microsoft Foundry). -Azure OpenAI Service does not host Anthropic Claude models, so the -`prompt_caching_5m` × Azure cell is structurally `not_applicable` — -there is no route for the `claude` CLI to talk to Claude through Azure -via LiteLLM. +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0. + +Foundry's Anthropic deployments honor the default 5-minute ephemeral +`cache_control` exactly like anthropic.com. The 1-hour `scope: "global"` +variant is *not* supported on Foundry — LiteLLM strips that field +before forwarding (see `_remove_scope_from_cache_control` in +`litellm/llms/azure_ai/anthropic/messages_transformation.py`) — but +this row exercises the 5-minute TTL only, so that quirk does not apply. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -15,21 +22,88 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import os +from typing import Any, Mapping, Optional + import pytest +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + AZURE_MODELS = [ - "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", ] -NOT_APPLICABLE_REASON = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." -) + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 @pytest.mark.parametrize("model", AZURE_MODELS) def test_prompt_caching_5m_azure(compat_result, model): - """Report `not_applicable` for every (model, Azure) combination.""" - compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream usage block surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt="Reply with the single word 'pong' and nothing else.", + model=model, + base_url=base_url, + api_key=api_key, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if _cache_tokens(result.usage) <= 0: + compat_result.set( + { + "status": "fail", + "error": ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ), + } + ) + pytest.fail(f"no cache tokens for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/sample_compatibility-matrix.json b/tests/claude_code/sample_compatibility-matrix.json index dc87f4c8939..04c790e0b44 100644 --- a/tests/claude_code/sample_compatibility-matrix.json +++ b/tests/claude_code/sample_compatibility-matrix.json @@ -28,8 +28,7 @@ "status": "pass" }, "azure": { - "status": "not_applicable", - "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + "status": "pass" } } }, @@ -50,8 +49,7 @@ "status": "pass" }, "azure": { - "status": "not_applicable", - "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + "status": "pass" } } }, @@ -72,8 +70,7 @@ "status": "pass" }, "azure": { - "status": "not_applicable", - "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + "status": "pass" } } }, @@ -94,8 +91,7 @@ "status": "pass" }, "azure": { - "status": "not_applicable", - "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + "status": "pass" } } }, @@ -116,8 +112,7 @@ "status": "pass" }, "azure": { - "status": "not_applicable", - "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + "status": "pass" } } }, @@ -138,8 +133,7 @@ "status": "pass" }, "azure": { - "status": "not_applicable", - "reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." + "status": "pass" } } } diff --git a/tests/claude_code/test_config.yaml b/tests/claude_code/test_config.yaml index 8c9be506ed4..05716fa41ca 100644 --- a/tests/claude_code/test_config.yaml +++ b/tests/claude_code/test_config.yaml @@ -13,6 +13,7 @@ # - claude-{tier}-bedrock-invoke → Bedrock InvokeModel API # - claude-{tier}-bedrock-converse → Bedrock Converse API # - claude-{tier}-vertex → GCP Vertex AI +# - claude-{tier}-azure → Microsoft Foundry (Anthropic deployments) model_list: # ---- Anthropic ---- @@ -74,6 +75,29 @@ model_list: vertex_ai_project: pathrise-convert-1606954137718 vertex_ai_location: us-east5 + # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- + # Anthropic announced Claude Haiku 4.5, Sonnet 4.5/4.6, and Opus 4.1/4.6/4.7 + # in Microsoft Foundry on 2025-11-18. Foundry exposes these models on an + # Anthropic-shape `/anthropic/v1/messages` endpoint (not Azure OpenAI's + # chat-completions route), and LiteLLM routes them via the `azure_ai/` + # provider prefix. Set AZURE_FOUNDRY_API_BASE to the host (or host + + # `/anthropic`); LiteLLM normalizes the URL to `…/anthropic/v1/messages`. + - model_name: claude-haiku-4-5-azure + litellm_params: + model: azure_ai/claude-haiku-4-5 + api_base: os.environ/AZURE_FOUNDRY_API_BASE + api_key: os.environ/AZURE_FOUNDRY_API_KEY + - model_name: claude-sonnet-4-6-azure + litellm_params: + model: azure_ai/claude-sonnet-4-6 + api_base: os.environ/AZURE_FOUNDRY_API_BASE + api_key: os.environ/AZURE_FOUNDRY_API_KEY + - model_name: claude-opus-4-7-azure + litellm_params: + model: azure_ai/claude-opus-4-7 + api_base: os.environ/AZURE_FOUNDRY_API_BASE + api_key: os.environ/AZURE_FOUNDRY_API_KEY + general_settings: # Claude Code sends provider-specific headers (e.g. anthropic-beta) we # want to forward verbatim to the upstream so the wire-shape under diff --git a/tests/claude_code/tool_use/test_azure.py b/tests/claude_code/tool_use/test_azure.py index 24f402f5910..1103adc610d 100644 --- a/tests/claude_code/tool_use/test_azure.py +++ b/tests/claude_code/tool_use/test_azure.py @@ -1,8 +1,13 @@ -"""tool_use x Azure. +"""tool_use x Azure (Microsoft Foundry). -Azure OpenAI Service does not host Anthropic Claude models, so the -`tool_use` × Azure cell is structurally `not_applicable` — there is no -route for the `claude` CLI to talk to Claude through Azure via LiteLLM. +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, ask Claude to invoke a built-in tool (`Bash`), and assert that +the upstream returned a `tool_use` content block. + +Foundry's Anthropic deployments support function/tool calling +identically to anthropic.com; LiteLLM's `azure_ai/claude-*` route +inherits the full Anthropic tool-use transformation. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -14,21 +19,93 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import os +from typing import Any, Mapping, Sequence + import pytest +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + AZURE_MODELS = [ - "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", ] -NOT_APPLICABLE_REASON = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." ) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False @pytest.mark.parametrize("model", AZURE_MODELS) def test_tool_use_azure(compat_result, model): - """Report `not_applicable` for every (model, Azure) combination.""" - compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) + """Drive the `claude` CLI against the LiteLLM proxy and assert a + tool call was emitted on the wire.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + try: + result = run_claude( + prompt=TOOL_USE_PROMPT, + model=model, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not _has_tool_use_event(result.events): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] no tool_use content block observed in stream-json events", + } + ) + pytest.fail(f"no tool_use for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"}) diff --git a/tests/claude_code/vision/test_azure.py b/tests/claude_code/vision/test_azure.py index b6e29b7d02f..c24c41c05ac 100644 --- a/tests/claude_code/vision/test_azure.py +++ b/tests/claude_code/vision/test_azure.py @@ -1,8 +1,14 @@ -"""vision x Azure. +"""vision x Azure (Microsoft Foundry). -Azure OpenAI Service does not host Anthropic Claude models, so the -`vision` × Azure cell is structurally `not_applicable` — there is no -route for the `claude` CLI to talk to Claude through Azure via LiteLLM. +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Anthropic's models hosted in Microsoft Foundry on +Azure, attach a small image via the CLI's `--image` flag, and assert +that the upstream produces a non-empty reply. + +Foundry's Anthropic deployments accept image content blocks identically +to anthropic.com (text + image input on Haiku 4.5, Sonnet 4.6, and Opus +4.7); LiteLLM passes them through unchanged on the `azure_ai/claude-*` +route. The (feature, provider) for this cell is inferred from the file path by `tests/claude_code/conftest.py`: @@ -14,21 +20,79 @@ The (feature, provider) for this cell is inferred from the file path by from __future__ import annotations +import base64 +import os + import pytest +from tests.claude_code.cli_driver import ClaudeCLIError, run_claude + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + AZURE_MODELS = [ - "claude-haiku-4-5", - "claude-sonnet-4-6", - "claude-opus-4-7", + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", ] -NOT_APPLICABLE_REASON = ( - "Azure OpenAI Service does not host Anthropic Claude models. " - "Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI." -) +RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" @pytest.mark.parametrize("model", AZURE_MODELS) -def test_vision_azure(compat_result, model): - """Report `not_applicable` for every (model, Azure) combination.""" - compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON}) +def test_vision_azure(compat_result, model, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with an image + attached and assert a non-empty reply.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + image_path = tmp_path / "red_pixel.png" + image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) + + try: + result = run_claude( + prompt="What single color do you see in the attached image? Answer in one word.", + model=model, + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) + except ClaudeCLIError as exc: + compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) + pytest.fail(str(exc), pytrace=False) + return + + if result.exit_code != 0: + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + } + ) + pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + return + + if not result.text.strip(): + compat_result.set( + { + "status": "fail", + "error": f"[{model}] claude returned empty assistant text on a vision prompt", + } + ) + pytest.fail(f"empty reply for {model}", pytrace=False) + return + + compat_result.set({"status": "pass"})