mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-22 00:31:44 +00:00
be65b4e23b
Matrix grows from 11 to 15 feature rows. All new tests collected + 180 unit tests still pass; smoke runs hit real LiteLLM bug surfaces on bedrock_invoke, bedrock_converse, and vertex_ai (cells correctly red in PR #142). Rename ------ `extended_thinking` -> `thinking` (directory, manifest id+name, 5 test fn names, 5 docstrings, builder unit-test fixtures, sample JSON, run_compat.sh). Existing test logic already covers both manual (`thinking.type=enabled`, Haiku 4.5) and adaptive (`thinking.type=adaptive`, Opus 4.7) shapes because Claude Code picks the shape per model from `--effort max`; the name change just stops the column from looking like a Claude 3.7 reference. New rows -------- - structured_outputs (5 files, CLI `--json-schema`). Claude Code synthesizes a single `StructuredOutput` tool from the schema and surfaces the tool_use input as `structured_output` on the trailing `result` event. Test ships its own `_validate_against_schema` so we don't take a jsonschema dep just for matrix surface. - count_tokens (5 files, HTTP probe). POSTs the proxy's `/v1/messages/count_tokens` directly and asserts the response is `{input_tokens: positive int}`. No CLI hook exists for this endpoint; the test goes through the new http_probe helper instead. - tool_search (5 files, HTTP probe). Sends `tools: [{type: tool_search_tool_regex_20251119, name: tool_search_tool_regex}]` and asserts the proxy doesn't 400. MCP fan-out via `--mcp-config` would also exercise the tool-search beta header path, but it's flaky w.r.t. Claude Code's internal tool-deferral threshold; the HTTP probe hits the actual bug surface (per-provider beta-header translation `advanced-tool-use-2025-11-20` vs `tool-search-tool-2025-10-19`). - long_context_1m (5 files, CLI `--betas context-1m-2025-08-07 --max-budget-usd 6`). A ~210k-token padded prompt over stdin exercises the 1M-context beta. Sonnet 4.6 + Opus 4.7 only -- Haiku 4.5's window is 200k, so it's excluded from MODELS (not marked not_applicable) to keep the per-cell aggregator semantics intact. Prompt uses a document-style preamble + 8 cycling pangrams rather than repeating identical chunks; without that, Opus 4.7 trips the safety filter mid-response with a Usage Policy refusal. `--max-budget-usd 6` is a runaway-loop guard, ~2x worst-case Opus per-cell spend. New helper ---------- `tests/claude_code/http_probe.py`: shared `ProbeResult` dataclass plus per-endpoint `probe_*` + `assert_*_shape` pairs for the HTTP-probe rows. Uses httpx with `anthropic-version: 2023-06-01` and a 30s timeout.
112 lines
3.5 KiB
Python
112 lines
3.5 KiB
Python
"""thinking x Bedrock (Invoke).
|
|
|
|
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
|
Claude requests to AWS Bedrock via the legacy `InvokeModel` API path,
|
|
enable extended thinking via `--effort high`, and assert that the
|
|
upstream returned a `thinking` content block.
|
|
|
|
The (feature, provider) for this cell is inferred from the file path by
|
|
`tests/claude_code/conftest.py`:
|
|
|
|
tests/claude_code/thinking/test_bedrock_invoke.py
|
|
^^^^^^^^ ^^^^^^^^^^^^^^
|
|
feature_id provider
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
import pytest
|
|
|
|
from tests.claude_code.cli_driver import (
|
|
ClaudeCLIError,
|
|
failure_diagnostic,
|
|
run_claude_models_parallel,
|
|
)
|
|
|
|
PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL"
|
|
PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY"
|
|
|
|
BEDROCK_INVOKE_MODELS = [
|
|
"claude-haiku-4-5-bedrock-invoke",
|
|
"claude-sonnet-4-6-bedrock-invoke",
|
|
"claude-opus-4-7-bedrock-invoke",
|
|
]
|
|
|
|
THINKING_ARGS = ["--effort", "max"]
|
|
THINKING_PROMPT = (
|
|
"I have a 3-gallon jug and a 5-gallon jug. How can I measure "
|
|
"exactly 4 gallons of water? Think through the steps carefully."
|
|
)
|
|
|
|
|
|
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|
for event in events:
|
|
if event.get("type") != "assistant":
|
|
continue
|
|
message = event.get("message") or {}
|
|
content = message.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
for block in content:
|
|
if isinstance(block, dict) and block.get("type") == "thinking":
|
|
return True
|
|
return False
|
|
|
|
|
|
def test_thinking_bedrock_invoke(compat_result):
|
|
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
|
enabled and assert a `thinking` content block was emitted."""
|
|
base_url = os.environ.get(PROXY_BASE_URL_ENV)
|
|
api_key = os.environ.get(PROXY_API_KEY_ENV)
|
|
if not base_url or not api_key:
|
|
compat_result.set(
|
|
{
|
|
"status": "fail",
|
|
"error": (
|
|
f"missing required env: set {PROXY_BASE_URL_ENV} and "
|
|
f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy"
|
|
),
|
|
}
|
|
)
|
|
pytest.fail(
|
|
f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False
|
|
)
|
|
|
|
outcomes = run_claude_models_parallel(
|
|
models=BEDROCK_INVOKE_MODELS,
|
|
prompt=THINKING_PROMPT,
|
|
base_url=base_url,
|
|
api_key=api_key,
|
|
extra_args=THINKING_ARGS,
|
|
)
|
|
|
|
failures = []
|
|
for model in BEDROCK_INVOKE_MODELS:
|
|
outcome = outcomes[model]
|
|
if isinstance(outcome, ClaudeCLIError):
|
|
error = f"[{model}] {outcome}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
if outcome.exit_code != 0:
|
|
error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
if not _has_thinking_block(outcome.events):
|
|
error = (
|
|
f"[{model}] no `thinking` content block observed in stream-json events"
|
|
)
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
compat_result.add({"status": "pass"})
|
|
|
|
if failures:
|
|
pytest.fail("; ".join(failures), pytrace=False)
|