mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
0ae55bdf7a
* test(e2e): add enum values, auto-discovering label gates and secret hiding for e2e metadata * test(e2e): tag claude_code tests with Subject metadata and record CLI driver steps * docs(e2e): name every markerless harness test file that carries no Subject * test(e2e): keep the step discovery comprehensions to one for clause * test(e2e): decorate run_claude directly so the label gate discovers its step
119 lines
3.6 KiB
Python
119 lines
3.6 KiB
Python
"""tool_use x Azure (Microsoft Foundry).
|
|
|
|
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
|
Claude requests to Anthropic's models hosted in Microsoft Foundry on
|
|
Azure, ask Claude to invoke a built-in tool (`Bash`), and assert that
|
|
the upstream returned a `tool_use` content block.
|
|
|
|
Foundry's Anthropic deployments support function/tool calling
|
|
identically to anthropic.com; LiteLLM's `azure_ai/claude-*` route
|
|
inherits the full Anthropic tool-use transformation.
|
|
|
|
The (feature, provider) for this cell is inferred from the file path by
|
|
`tests/e2e/claude_code/conftest.py`:
|
|
|
|
tests/e2e/claude_code/tool_use/test_azure.py
|
|
^^^^^^^^ ^^^^^
|
|
feature_id provider
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
import pytest
|
|
|
|
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
|
|
from claude_code._env import require_proxy
|
|
from claude_code.cli_driver import (
|
|
ClaudeCLIError,
|
|
failure_diagnostic,
|
|
run_claude_models_parallel,
|
|
)
|
|
|
|
|
|
AZURE_MODELS = [
|
|
"claude-haiku-4-5-azure",
|
|
"claude-sonnet-4-5-azure",
|
|
"claude-opus-4-7-azure",
|
|
]
|
|
|
|
TOOL_USE_PROMPT = (
|
|
"Use the Bash tool to run the command `echo pong` and report what it printed."
|
|
)
|
|
# Bash is restricted to the exact command `echo pong` + `dontAsk`
|
|
# permission mode; see `tool_use/test_anthropic.py` for the security
|
|
# rationale.
|
|
TOOL_USE_ARGS = [
|
|
"--allowed-tools",
|
|
"Bash(echo pong)",
|
|
"--permission-mode",
|
|
"dontAsk",
|
|
]
|
|
|
|
|
|
def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool:
|
|
for event in events:
|
|
if event.get("type") != "assistant":
|
|
continue
|
|
message = event.get("message") or {}
|
|
content = message.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
for block in content:
|
|
if isinstance(block, dict) and block.get("type") == "tool_use":
|
|
return True
|
|
return False
|
|
|
|
|
|
@pytest.mark.covers("llm.messages.azure_foundry.tool_use.nonstream.works")
|
|
@meta(
|
|
Subject(
|
|
domain=Domain.LLM_TRANSLATION,
|
|
route=Route.MESSAGES,
|
|
providers=(Provider.AZURE_AI,),
|
|
models=tuple(AZURE_MODELS),
|
|
capabilities=(Capability.FUNCTION_CALLING,),
|
|
mode=Mode.NONSTREAM,
|
|
)
|
|
)
|
|
def test_tool_use_azure(compat_result):
|
|
"""Drive the `claude` CLI against the LiteLLM proxy and assert a
|
|
tool call was emitted on the wire."""
|
|
base_url, api_key = require_proxy(compat_result)
|
|
|
|
outcomes = run_claude_models_parallel(
|
|
models=AZURE_MODELS,
|
|
prompt=TOOL_USE_PROMPT,
|
|
base_url=base_url,
|
|
api_key=api_key,
|
|
extra_args=TOOL_USE_ARGS,
|
|
)
|
|
|
|
failures = []
|
|
for model in AZURE_MODELS:
|
|
outcome = outcomes[model]
|
|
if isinstance(outcome, ClaudeCLIError):
|
|
error = f"[{model}] {outcome}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
if outcome.exit_code != 0:
|
|
error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
if not _has_tool_use_event(outcome.events):
|
|
error = (
|
|
f"[{model}] no tool_use content block observed in stream-json events"
|
|
)
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
compat_result.add({"status": "pass"})
|
|
|
|
if failures:
|
|
pytest.fail("; ".join(failures), pytrace=False)
|