mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-17 23:51:30 +00:00
RALPH: fix compat matrix - Azure now hosts Claude via Microsoft Foundry
Anthropic and Microsoft announced Claude Haiku 4.5, Sonnet 4.5/4.6, and
Opus 4.1/4.6/4.7 in Microsoft Foundry on 2025-11-18, so the matrix's
Azure column should exercise a real route through the LiteLLM proxy
rather than report not_applicable.
Foundry serves Claude on an Anthropic-shape /anthropic/v1/messages
endpoint (not the Azure OpenAI chat-completions route), and LiteLLM
already supports it via the azure_ai/claude-* provider prefix
(litellm/llms/azure_ai/anthropic/{handler,transformation,messages_transformation}.py).
- test_config.yaml: add 3 azure aliases pointing at azure_ai/claude-*
with AZURE_FOUNDRY_API_BASE / AZURE_FOUNDRY_API_KEY env
- 6x test_azure.py: replace not_applicable stubs with real run_claude
drivers, mirroring the existing test_vertex_ai.py shape exactly
- sample_compatibility-matrix.json: Azure cells flip to pass
- _builder_unit_tests: pin the new invariant (run_claude is used,
not_applicable is gone) and feed pass results across all 5 providers
in the 6x5 golden test
This commit is contained in:
mateo-berri
parent
28cdbb4485
commit
c1955a15ca
10 changed files with 567 additions and 138 deletions
|
|
@ -182,12 +182,10 @@ def test_build_matrix_6x5_grid_matches_published_sample():
|
|||
|
||||
Inputs mirror the structure of `compat-results.json` after a real
|
||||
run with the proxy configured for all five columns and all six
|
||||
feature directories:
|
||||
- anthropic, bedrock_invoke, bedrock_converse, vertex_ai: three
|
||||
per-model `pass` results each (Haiku, Sonnet, Opus) for every
|
||||
feature.
|
||||
- azure: three `not_applicable` results per feature; Azure does
|
||||
not host Claude, so the column is gray on every row.
|
||||
feature directories: every (feature, provider, model) cell yields a
|
||||
`pass`. Anthropic announced Claude in Microsoft Foundry on
|
||||
2025-11-18, so the Azure column is now exercised end-to-end like
|
||||
the others rather than reporting `not_applicable`.
|
||||
|
||||
The aggregated matrix must equal the checked-in
|
||||
`sample_compatibility-matrix.json` byte-for-byte (after JSON load),
|
||||
|
|
@ -197,16 +195,12 @@ def test_build_matrix_6x5_grid_matches_published_sample():
|
|||
manifest = load_manifest(repo_root / "manifest.yaml")
|
||||
|
||||
feature_ids = [feature["id"] for feature in manifest["features"]]
|
||||
pass_providers = ["anthropic", "bedrock_invoke", "bedrock_converse", "vertex_ai"]
|
||||
providers = manifest["providers"]
|
||||
models = ["claude-haiku-4-5", "claude-sonnet-4-6", "claude-opus-4-7"]
|
||||
azure_reason = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
)
|
||||
|
||||
results = []
|
||||
for feature_id in feature_ids:
|
||||
for provider in pass_providers:
|
||||
for provider in providers:
|
||||
for model in models:
|
||||
results.append(
|
||||
{
|
||||
|
|
@ -219,18 +213,6 @@ def test_build_matrix_6x5_grid_matches_published_sample():
|
|||
"result": {"status": "pass"},
|
||||
}
|
||||
)
|
||||
for model in models:
|
||||
results.append(
|
||||
{
|
||||
"feature_id": feature_id,
|
||||
"provider": "azure",
|
||||
"nodeid": (
|
||||
f"tests/claude_code/{feature_id}/test_azure.py"
|
||||
f"::test[{model}]"
|
||||
),
|
||||
"result": {"status": "not_applicable", "reason": azure_reason},
|
||||
}
|
||||
)
|
||||
|
||||
matrix = build_matrix(
|
||||
manifest=manifest,
|
||||
|
|
|
|||
|
|
@ -99,10 +99,18 @@ def test_per_provider_test_file_imports_and_parametrizes_three_models(
|
|||
|
||||
|
||||
@pytest.mark.parametrize("feature_id", EXPECTED_FEATURE_IDS)
|
||||
def test_azure_test_file_reports_not_applicable(feature_id):
|
||||
"""Azure OpenAI Service does not host Claude on any v0 feature, so
|
||||
every Azure cell in the v0 matrix is `not_applicable`. Pin that
|
||||
here so a future "let's just call the proxy and see what happens"
|
||||
edit doesn't silently turn the gray cells red."""
|
||||
def test_azure_test_file_drives_the_proxy(feature_id):
|
||||
"""Azure (Microsoft Foundry) hosts Anthropic Claude as of 2025-11-18,
|
||||
so every Azure cell in the v0 matrix exercises a real route through
|
||||
the LiteLLM proxy — same shape as the other provider columns. Pin
|
||||
that here so a future regression doesn't silently revert these
|
||||
cells to the old `not_applicable` boilerplate."""
|
||||
text = (REPO_ROOT / feature_id / "test_azure.py").read_text()
|
||||
assert '"status": "not_applicable"' in text
|
||||
assert "run_claude" in text, (
|
||||
f"{feature_id}/test_azure.py must drive the claude CLI via run_claude(); "
|
||||
"the not_applicable stub was removed when Foundry started hosting Claude."
|
||||
)
|
||||
assert '"status": "not_applicable"' not in text, (
|
||||
f"{feature_id}/test_azure.py still reports not_applicable; Microsoft Foundry "
|
||||
"now hosts Claude (Haiku 4.5, Sonnet 4.6, Opus 4.7), so this row must run."
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,13 @@
|
|||
"""basic_messaging_non_streaming x Azure.
|
||||
"""basic_messaging_non_streaming x Azure (Microsoft Foundry).
|
||||
|
||||
Azure (Azure OpenAI Service) does not host Anthropic Claude models —
|
||||
the platform's first-party catalog is OpenAI models, plus a smaller set
|
||||
of Microsoft and partner models. There is no supported route for the
|
||||
`claude` CLI to talk to Claude through Azure via LiteLLM, so every
|
||||
(model, Azure) combination for `basic_messaging_non_streaming` reports
|
||||
`not_applicable` rather than `fail`. This is what the matrix's
|
||||
`not_applicable` state is for: the renderer paints the cell gray with a
|
||||
tooltip explaining the cell will never apply, distinct from a
|
||||
genuine regression.
|
||||
Drive the real `claude` CLI in headless mode against a running LiteLLM
|
||||
proxy that routes Claude requests to Anthropic's models hosted in
|
||||
Microsoft Foundry on Azure, and report the outcome via `compat_result`.
|
||||
|
||||
Anthropic announced Claude Haiku 4.5, Sonnet 4.5/4.6, and Opus 4.1/4.6/4.7
|
||||
in Microsoft Foundry on 2025-11-18; LiteLLM exposes them via the
|
||||
`azure_ai/claude-*` provider prefix, which talks to Foundry's
|
||||
Anthropic-shape `/anthropic/v1/messages` endpoint.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/claude_code/conftest.py`:
|
||||
|
|
@ -16,29 +15,91 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
tests/claude_code/basic_messaging_non_streaming/test_azure.py
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^
|
||||
feature_id provider
|
||||
|
||||
Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus
|
||||
4.7; the cell only goes green if all three pass. We parametrize over the
|
||||
three models and the conftest aggregator produces one cell from the
|
||||
three results, naming the failing model in the error string when any
|
||||
model fails.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
# Same three Claude tiers the other provider columns parametrize over,
|
||||
# so the test count per cell stays uniform across the matrix and any
|
||||
# future "Azure adds Anthropic" announcement only requires flipping the
|
||||
# body, not the parametrization.
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
]
|
||||
from tests.claude_code.cli_driver import ClaudeCLIError, run_claude
|
||||
|
||||
NOT_APPLICABLE_REASON = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
)
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
# Per-model aliases registered in the LiteLLM proxy's routing config to
|
||||
# point at Microsoft Foundry's Anthropic deployments. The driver only
|
||||
# sends the alias; the proxy is the one that knows the upstream Foundry
|
||||
# resource URL and API key.
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5-azure",
|
||||
"claude-sonnet-4-6-azure",
|
||||
"claude-opus-4-7-azure",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", AZURE_MODELS)
|
||||
def test_basic_messaging_non_streaming_azure(compat_result, model):
|
||||
"""Report `not_applicable` for every (model, Azure) combination."""
|
||||
compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON})
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a reply.
|
||||
|
||||
"Basic messaging" means: send a single user prompt, receive any
|
||||
non-empty assistant text reply, no tools, no streaming, no thinking.
|
||||
The whole point of this slice is to prove the path works at all —
|
||||
so the assertion is intentionally lenient on the reply contents.
|
||||
"""
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
|
||||
try:
|
||||
result = run_claude(
|
||||
prompt="Reply with the single word 'pong' and nothing else.",
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
)
|
||||
except ClaudeCLIError as exc:
|
||||
compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
|
||||
pytest.fail(str(exc), pytrace=False)
|
||||
return
|
||||
|
||||
if result.exit_code != 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if not result.text.strip():
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude returned empty assistant text",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"empty reply for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
compat_result.set({"status": "pass"})
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
"""basic_messaging_streaming x Azure.
|
||||
"""basic_messaging_streaming x Azure (Microsoft Foundry).
|
||||
|
||||
Azure (Azure OpenAI Service) does not host Anthropic Claude models —
|
||||
the platform's first-party catalog is OpenAI models, plus a smaller set
|
||||
of Microsoft and partner models. There is no supported route for the
|
||||
`claude` CLI to talk to Claude through Azure via LiteLLM, so every
|
||||
(model, Azure) combination for `basic_messaging_streaming` reports
|
||||
`not_applicable` rather than `fail`.
|
||||
Drive the real `claude` CLI in headless `--output-format stream-json`
|
||||
mode against a running LiteLLM proxy that routes Claude requests to
|
||||
Anthropic's models hosted in Microsoft Foundry on Azure, and report the
|
||||
outcome via `compat_result`.
|
||||
|
||||
Foundry exposes Claude on an Anthropic-shape `/anthropic/v1/messages`
|
||||
endpoint with native SSE streaming; LiteLLM forwards stream events
|
||||
through the `azure_ai/claude-*` provider unchanged.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/claude_code/conftest.py`:
|
||||
|
|
@ -17,21 +19,83 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
]
|
||||
from tests.claude_code.cli_driver import ClaudeCLIError, run_claude
|
||||
|
||||
NOT_APPLICABLE_REASON = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
)
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5-azure",
|
||||
"claude-sonnet-4-6-azure",
|
||||
"claude-opus-4-7-azure",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", AZURE_MODELS)
|
||||
def test_basic_messaging_streaming_azure(compat_result, model):
|
||||
"""Report `not_applicable` for every (model, Azure) combination."""
|
||||
compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON})
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
non-empty streamed reply (at least one stream-json event observed).
|
||||
"""
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
|
||||
try:
|
||||
result = run_claude(
|
||||
prompt="Count from 1 to 5, one number per line.",
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
)
|
||||
except ClaudeCLIError as exc:
|
||||
compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
|
||||
pytest.fail(str(exc), pytrace=False)
|
||||
return
|
||||
|
||||
if result.exit_code != 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if not result.events:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] no stream-json events emitted; streaming wire silent",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"no stream events for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if not result.text.strip():
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude returned empty assistant text",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"empty reply for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
compat_result.set({"status": "pass"})
|
||||
|
|
|
|||
|
|
@ -1,9 +1,17 @@
|
|||
"""extended_thinking x Azure.
|
||||
"""extended_thinking x Azure (Microsoft Foundry).
|
||||
|
||||
Azure OpenAI Service does not host Anthropic Claude models, so the
|
||||
`extended_thinking` × Azure cell is structurally `not_applicable` —
|
||||
there is no route for the `claude` CLI to talk to Claude through Azure
|
||||
via LiteLLM.
|
||||
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
||||
Claude requests to Anthropic's models hosted in Microsoft Foundry on
|
||||
Azure, enable extended thinking via `MAX_THINKING_TOKENS`, and assert
|
||||
that the upstream returned a `thinking` content block.
|
||||
|
||||
Foundry's Claude deployments advertise `supports_reasoning: true` in
|
||||
LiteLLM's pricing metadata; the `thinking={"type": "enabled", ...}`
|
||||
parameter passes through `azure_ai/claude-*` to Foundry's
|
||||
`/anthropic/v1/messages` endpoint unchanged. Note that
|
||||
`claude-opus-4-7-preview` documents thinking as not supported on
|
||||
Foundry; if that lands, this row may flip to a partial pass and we'll
|
||||
re-evaluate.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/claude_code/conftest.py`:
|
||||
|
|
@ -15,21 +23,94 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.claude_code.cli_driver import ClaudeCLIError, run_claude
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
"claude-haiku-4-5-azure",
|
||||
"claude-sonnet-4-6-azure",
|
||||
"claude-opus-4-7-azure",
|
||||
]
|
||||
|
||||
NOT_APPLICABLE_REASON = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"}
|
||||
THINKING_PROMPT = (
|
||||
"Think step by step: if I have three apples and eat two, how many remain? "
|
||||
"Answer with the single digit only."
|
||||
)
|
||||
|
||||
|
||||
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
||||
for event in events:
|
||||
if event.get("type") != "assistant":
|
||||
continue
|
||||
message = event.get("message") or {}
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "thinking":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", AZURE_MODELS)
|
||||
def test_extended_thinking_azure(compat_result, model):
|
||||
"""Report `not_applicable` for every (model, Azure) combination."""
|
||||
compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON})
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
||||
enabled and assert a `thinking` content block was emitted."""
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
|
||||
try:
|
||||
result = run_claude(
|
||||
prompt=THINKING_PROMPT,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
extra_env=THINKING_ENV,
|
||||
)
|
||||
except ClaudeCLIError as exc:
|
||||
compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
|
||||
pytest.fail(str(exc), pytrace=False)
|
||||
return
|
||||
|
||||
if result.exit_code != 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if not _has_thinking_block(result.events):
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] no `thinking` content block observed in stream-json events",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"no thinking block for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
compat_result.set({"status": "pass"})
|
||||
|
|
|
|||
|
|
@ -1,9 +1,16 @@
|
|||
"""prompt_caching_5m x Azure.
|
||||
"""prompt_caching_5m x Azure (Microsoft Foundry).
|
||||
|
||||
Azure OpenAI Service does not host Anthropic Claude models, so the
|
||||
`prompt_caching_5m` × Azure cell is structurally `not_applicable` —
|
||||
there is no route for the `claude` CLI to talk to Claude through Azure
|
||||
via LiteLLM.
|
||||
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
||||
Claude requests to Anthropic's models hosted in Microsoft Foundry on
|
||||
Azure, and assert that the upstream's usage block reports either
|
||||
`cache_creation_input_tokens` or `cache_read_input_tokens` > 0.
|
||||
|
||||
Foundry's Anthropic deployments honor the default 5-minute ephemeral
|
||||
`cache_control` exactly like anthropic.com. The 1-hour `scope: "global"`
|
||||
variant is *not* supported on Foundry — LiteLLM strips that field
|
||||
before forwarding (see `_remove_scope_from_cache_control` in
|
||||
`litellm/llms/azure_ai/anthropic/messages_transformation.py`) — but
|
||||
this row exercises the 5-minute TTL only, so that quirk does not apply.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/claude_code/conftest.py`:
|
||||
|
|
@ -15,21 +22,88 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.claude_code.cli_driver import ClaudeCLIError, run_claude
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
"claude-haiku-4-5-azure",
|
||||
"claude-sonnet-4-6-azure",
|
||||
"claude-opus-4-7-azure",
|
||||
]
|
||||
|
||||
NOT_APPLICABLE_REASON = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
)
|
||||
|
||||
def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int:
|
||||
if not isinstance(usage, Mapping):
|
||||
return 0
|
||||
creation = usage.get("cache_creation_input_tokens") or 0
|
||||
read = usage.get("cache_read_input_tokens") or 0
|
||||
try:
|
||||
return int(creation) + int(read)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", AZURE_MODELS)
|
||||
def test_prompt_caching_5m_azure(compat_result, model):
|
||||
"""Report `not_applicable` for every (model, Azure) combination."""
|
||||
compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON})
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert the
|
||||
upstream usage block surfaces a non-zero cache token count."""
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
|
||||
try:
|
||||
result = run_claude(
|
||||
prompt="Reply with the single word 'pong' and nothing else.",
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
)
|
||||
except ClaudeCLIError as exc:
|
||||
compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
|
||||
pytest.fail(str(exc), pytrace=False)
|
||||
return
|
||||
|
||||
if result.exit_code != 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if _cache_tokens(result.usage) <= 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"[{model}] usage block reported zero cache tokens; "
|
||||
"expected cache_control on the system prompt to produce a non-zero "
|
||||
"cache_creation_input_tokens or cache_read_input_tokens"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(f"no cache tokens for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
compat_result.set({"status": "pass"})
|
||||
|
|
|
|||
|
|
@ -28,8 +28,7 @@
|
|||
"status": "pass"
|
||||
},
|
||||
"azure": {
|
||||
"status": "not_applicable",
|
||||
"reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
"status": "pass"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -50,8 +49,7 @@
|
|||
"status": "pass"
|
||||
},
|
||||
"azure": {
|
||||
"status": "not_applicable",
|
||||
"reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
"status": "pass"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -72,8 +70,7 @@
|
|||
"status": "pass"
|
||||
},
|
||||
"azure": {
|
||||
"status": "not_applicable",
|
||||
"reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
"status": "pass"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -94,8 +91,7 @@
|
|||
"status": "pass"
|
||||
},
|
||||
"azure": {
|
||||
"status": "not_applicable",
|
||||
"reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
"status": "pass"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -116,8 +112,7 @@
|
|||
"status": "pass"
|
||||
},
|
||||
"azure": {
|
||||
"status": "not_applicable",
|
||||
"reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
"status": "pass"
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -138,8 +133,7 @@
|
|||
"status": "pass"
|
||||
},
|
||||
"azure": {
|
||||
"status": "not_applicable",
|
||||
"reason": "Azure OpenAI Service does not host Anthropic Claude models. Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
"status": "pass"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@
|
|||
# - claude-{tier}-bedrock-invoke → Bedrock InvokeModel API
|
||||
# - claude-{tier}-bedrock-converse → Bedrock Converse API
|
||||
# - claude-{tier}-vertex → GCP Vertex AI
|
||||
# - claude-{tier}-azure → Microsoft Foundry (Anthropic deployments)
|
||||
|
||||
model_list:
|
||||
# ---- Anthropic ----
|
||||
|
|
@ -74,6 +75,29 @@ model_list:
|
|||
vertex_ai_project: pathrise-convert-1606954137718
|
||||
vertex_ai_location: us-east5
|
||||
|
||||
# ---- Microsoft Foundry (Anthropic deployments on Azure) ----
|
||||
# Anthropic announced Claude Haiku 4.5, Sonnet 4.5/4.6, and Opus 4.1/4.6/4.7
|
||||
# in Microsoft Foundry on 2025-11-18. Foundry exposes these models on an
|
||||
# Anthropic-shape `/anthropic/v1/messages` endpoint (not Azure OpenAI's
|
||||
# chat-completions route), and LiteLLM routes them via the `azure_ai/`
|
||||
# provider prefix. Set AZURE_FOUNDRY_API_BASE to the host (or host +
|
||||
# `/anthropic`); LiteLLM normalizes the URL to `…/anthropic/v1/messages`.
|
||||
- model_name: claude-haiku-4-5-azure
|
||||
litellm_params:
|
||||
model: azure_ai/claude-haiku-4-5
|
||||
api_base: os.environ/AZURE_FOUNDRY_API_BASE
|
||||
api_key: os.environ/AZURE_FOUNDRY_API_KEY
|
||||
- model_name: claude-sonnet-4-6-azure
|
||||
litellm_params:
|
||||
model: azure_ai/claude-sonnet-4-6
|
||||
api_base: os.environ/AZURE_FOUNDRY_API_BASE
|
||||
api_key: os.environ/AZURE_FOUNDRY_API_KEY
|
||||
- model_name: claude-opus-4-7-azure
|
||||
litellm_params:
|
||||
model: azure_ai/claude-opus-4-7
|
||||
api_base: os.environ/AZURE_FOUNDRY_API_BASE
|
||||
api_key: os.environ/AZURE_FOUNDRY_API_KEY
|
||||
|
||||
general_settings:
|
||||
# Claude Code sends provider-specific headers (e.g. anthropic-beta) we
|
||||
# want to forward verbatim to the upstream so the wire-shape under
|
||||
|
|
|
|||
|
|
@ -1,8 +1,13 @@
|
|||
"""tool_use x Azure.
|
||||
"""tool_use x Azure (Microsoft Foundry).
|
||||
|
||||
Azure OpenAI Service does not host Anthropic Claude models, so the
|
||||
`tool_use` × Azure cell is structurally `not_applicable` — there is no
|
||||
route for the `claude` CLI to talk to Claude through Azure via LiteLLM.
|
||||
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
||||
Claude requests to Anthropic's models hosted in Microsoft Foundry on
|
||||
Azure, ask Claude to invoke a built-in tool (`Bash`), and assert that
|
||||
the upstream returned a `tool_use` content block.
|
||||
|
||||
Foundry's Anthropic deployments support function/tool calling
|
||||
identically to anthropic.com; LiteLLM's `azure_ai/claude-*` route
|
||||
inherits the full Anthropic tool-use transformation.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/claude_code/conftest.py`:
|
||||
|
|
@ -14,21 +19,93 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Mapping, Sequence
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.claude_code.cli_driver import ClaudeCLIError, run_claude
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
"claude-haiku-4-5-azure",
|
||||
"claude-sonnet-4-6-azure",
|
||||
"claude-opus-4-7-azure",
|
||||
]
|
||||
|
||||
NOT_APPLICABLE_REASON = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
TOOL_USE_PROMPT = (
|
||||
"Use the Bash tool to run the command `echo pong` and report what it printed."
|
||||
)
|
||||
TOOL_USE_ARGS = ["--allowed-tools", "Bash"]
|
||||
|
||||
|
||||
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
||||
for event in events:
|
||||
if event.get("type") != "assistant":
|
||||
continue
|
||||
message = event.get("message") or {}
|
||||
content = message.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if isinstance(block, dict) and block.get("type") == "tool_use":
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", AZURE_MODELS)
|
||||
def test_tool_use_azure(compat_result, model):
|
||||
"""Report `not_applicable` for every (model, Azure) combination."""
|
||||
compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON})
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
||||
tool call was emitted on the wire."""
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
|
||||
try:
|
||||
result = run_claude(
|
||||
prompt=TOOL_USE_PROMPT,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
extra_args=TOOL_USE_ARGS,
|
||||
)
|
||||
except ClaudeCLIError as exc:
|
||||
compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
|
||||
pytest.fail(str(exc), pytrace=False)
|
||||
return
|
||||
|
||||
if result.exit_code != 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if not _has_tool_use_event(result.events):
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] no tool_use content block observed in stream-json events",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"no tool_use for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
compat_result.set({"status": "pass"})
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
"""vision x Azure.
|
||||
"""vision x Azure (Microsoft Foundry).
|
||||
|
||||
Azure OpenAI Service does not host Anthropic Claude models, so the
|
||||
`vision` × Azure cell is structurally `not_applicable` — there is no
|
||||
route for the `claude` CLI to talk to Claude through Azure via LiteLLM.
|
||||
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
||||
Claude requests to Anthropic's models hosted in Microsoft Foundry on
|
||||
Azure, attach a small image via the CLI's `--image` flag, and assert
|
||||
that the upstream produces a non-empty reply.
|
||||
|
||||
Foundry's Anthropic deployments accept image content blocks identically
|
||||
to anthropic.com (text + image input on Haiku 4.5, Sonnet 4.6, and Opus
|
||||
4.7); LiteLLM passes them through unchanged on the `azure_ai/claude-*`
|
||||
route.
|
||||
|
||||
The (feature, provider) for this cell is inferred from the file path by
|
||||
`tests/claude_code/conftest.py`:
|
||||
|
|
@ -14,21 +20,79 @@ The (feature, provider) for this cell is inferred from the file path by
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.claude_code.cli_driver import ClaudeCLIError, run_claude
|
||||
|
||||
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
||||
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
||||
|
||||
AZURE_MODELS = [
|
||||
"claude-haiku-4-5",
|
||||
"claude-sonnet-4-6",
|
||||
"claude-opus-4-7",
|
||||
"claude-haiku-4-5-azure",
|
||||
"claude-sonnet-4-6-azure",
|
||||
"claude-opus-4-7-azure",
|
||||
]
|
||||
|
||||
NOT_APPLICABLE_REASON = (
|
||||
"Azure OpenAI Service does not host Anthropic Claude models. "
|
||||
"Route Claude requests through Anthropic, AWS Bedrock, or GCP Vertex AI."
|
||||
)
|
||||
RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", AZURE_MODELS)
|
||||
def test_vision_azure(compat_result, model):
|
||||
"""Report `not_applicable` for every (model, Azure) combination."""
|
||||
compat_result.set({"status": "not_applicable", "reason": NOT_APPLICABLE_REASON})
|
||||
def test_vision_azure(compat_result, model, tmp_path):
|
||||
"""Drive the `claude` CLI against the LiteLLM proxy with an image
|
||||
attached and assert a non-empty reply."""
|
||||
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
||||
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
||||
if not base_url or not api_key:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": (
|
||||
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
||||
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
||||
),
|
||||
}
|
||||
)
|
||||
pytest.fail(
|
||||
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
||||
)
|
||||
|
||||
image_path = tmp_path / "red_pixel.png"
|
||||
image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64))
|
||||
|
||||
try:
|
||||
result = run_claude(
|
||||
prompt="What single color do you see in the attached image? Answer in one word.",
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
api_key=api_key,
|
||||
extra_args=["--image", str(image_path)],
|
||||
)
|
||||
except ClaudeCLIError as exc:
|
||||
compat_result.set({"status": "fail", "error": f"[{model}] {exc}"})
|
||||
pytest.fail(str(exc), pytrace=False)
|
||||
return
|
||||
|
||||
if result.exit_code != 0:
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
if not result.text.strip():
|
||||
compat_result.set(
|
||||
{
|
||||
"status": "fail",
|
||||
"error": f"[{model}] claude returned empty assistant text on a vision prompt",
|
||||
}
|
||||
)
|
||||
pytest.fail(f"empty reply for {model}", pytrace=False)
|
||||
return
|
||||
|
||||
compat_result.set({"status": "pass"})
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue