mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
10462eddaf
* test(e2e): harness fixes for long_context, complexity router, UI, and unit coverage Point long_context_1m at 1M-capable models, harden complexity-smart-router registration and spend-log assertions, fix key models dropdown selectors, and add gateway/lifecycle/transport and claude_code unit tests * test(e2e): harden remaining stage failures in harness Register complexity-smart-router via create_model + callable probe, fix create-key UI navigation race, retry management writes and budget ALB 502s, mark Vertex count_tokens N/A when unsupported, and tighten tool_search model lists for Azure/Bedrock capability gaps * test(e2e): drop claude_code and harness unit tests from this PR Keep management, router, budget, and shared conftest harness fixes only * test(e2e): restore E2E_RESULT pytest_runtest_makereport hook Accidentally dropped in an earlier harness commit; Grafana status history depends on these structured log lines * test(e2e): drop management control-plane write retries Transient 500 retries do not fix the underlying control plane failures * test(e2e): skip stage-red claude_code cells; fix multi-window budget latency Mark the twelve failing claude_code matrix cells skip until product/config lands. Multi-window budget polls gpt-5.5 with max_tokens=1 instead of Claude so the reset wait stays under ALB target idle timeout rather than masking awselb 502s * test(e2e): require exactly one LLM-tier spend row for complexity router Keep alias membership for compose vs stage model names, but assert len(served) == 1 so a leaked classifier sub-call cannot pass. Also pin LIT-4521 skip and align LIT-4522/23/24 skip reasons * test(e2e): harden router callable probe and multi-window budget exhaustion _router_is_callable treated any non-success chat whose body lacked "Invalid model name" as callable, so an unpropagated probe key (401), a generic 502, or a connection reset let the session proceed and hit real "Invalid model name" failures inside the tests. Require a Success outcome instead; the reload-race 400 and every infra/auth error now correctly read as not-callable. The multi-window budget test capped the tight window at 3e-6, which gpt-5.5 exhausts on the first call but a cheaper CHEAP_OPENAI_MODEL might not within the 20-call loop, turning a reset test into a spurious "window never enforced" failure. Drop the tight cap to 1e-9 so the first billed call exhausts it regardless of model price; the roomy 1m window stays at 1.0 and never blocks. * test(e2e): use a tradeoff-decision prompt for the complexity router classifier "Is P equal to NP?" reads to the LLM classifier as a short yes/no question, so gpt-5.5 classified it SIMPLE and the request routed to the openai backend, which made the test fail even though the classifier was running. The tier definitions key on what the request demands, not how hard the answer is, and a short direct question maps to SIMPLE regardless of subject. Swap in "Should I pay off my mortgage early or invest the extra money instead?". It carries none of the heuristic scorer's reasoning/technical/code keywords and stays short, so heuristic scoring still lands SIMPLE (openai), but the LLM reads it as a decision that has to weigh tradeoffs and lands it above SIMPLE, which the config routes to anthropic. Any non-SIMPLE tier serves anthropic, so the classifier only has to avoid SIMPLE for the test to distinguish a real classifier run from the heuristic fallback.
101 lines
3.2 KiB
Python
101 lines
3.2 KiB
Python
"""thinking x Bedrock (Converse).
|
|
|
|
Drive the real `claude` CLI against a running LiteLLM proxy that routes
|
|
Claude requests to AWS Bedrock via the unified `Converse` API path,
|
|
enable extended thinking via `--effort high`, and assert that the
|
|
upstream returned a `thinking` content block.
|
|
|
|
The (feature, provider) for this cell is inferred from the file path by
|
|
`tests/e2e/claude_code/conftest.py`:
|
|
|
|
tests/e2e/claude_code/thinking/test_bedrock_converse.py
|
|
^^^^^^^^ ^^^^^^^^^^^^^^^^
|
|
feature_id provider
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any, Mapping, Sequence
|
|
|
|
import pytest
|
|
|
|
from claude_code._env import require_proxy
|
|
from claude_code.cli_driver import (
|
|
ClaudeCLIError,
|
|
failure_diagnostic,
|
|
run_claude_models_parallel,
|
|
)
|
|
|
|
|
|
BEDROCK_CONVERSE_MODELS = [
|
|
"claude-haiku-4-5-bedrock-converse",
|
|
"claude-sonnet-4-5-bedrock-converse",
|
|
"claude-opus-4-7-bedrock-converse",
|
|
]
|
|
|
|
THINKING_ARGS = ["--effort", "max"]
|
|
THINKING_PROMPT = (
|
|
"I have a 3-gallon jug and a 5-gallon jug. How can I measure "
|
|
"exactly 4 gallons of water? Think through the steps carefully."
|
|
)
|
|
|
|
|
|
def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool:
|
|
for event in events:
|
|
if event.get("type") != "assistant":
|
|
continue
|
|
message = event.get("message") or {}
|
|
content = message.get("content")
|
|
if not isinstance(content, list):
|
|
continue
|
|
for block in content:
|
|
if isinstance(block, dict) and block.get("type") == "thinking":
|
|
return True
|
|
return False
|
|
|
|
|
|
@pytest.mark.skip(
|
|
reason="product bug LIT-4524: Bedrock Converse streaming Content block is not a text block; "
|
|
"re-enable when empty/mismatched content_block_delta is fixed"
|
|
)
|
|
@pytest.mark.covers("llm.messages.bedrock_converse.thinking.nonstream.works")
|
|
def test_thinking_bedrock_converse(compat_result):
|
|
"""Drive the `claude` CLI against the LiteLLM proxy with thinking
|
|
enabled and assert a `thinking` content block was emitted."""
|
|
base_url, api_key = require_proxy(compat_result)
|
|
|
|
outcomes = run_claude_models_parallel(
|
|
models=BEDROCK_CONVERSE_MODELS,
|
|
prompt=THINKING_PROMPT,
|
|
base_url=base_url,
|
|
api_key=api_key,
|
|
extra_args=THINKING_ARGS,
|
|
)
|
|
|
|
failures = []
|
|
for model in BEDROCK_CONVERSE_MODELS:
|
|
outcome = outcomes[model]
|
|
if isinstance(outcome, ClaudeCLIError):
|
|
error = f"[{model}] {outcome}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
if outcome.exit_code != 0:
|
|
error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}"
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
if not _has_thinking_block(outcome.events):
|
|
error = (
|
|
f"[{model}] no `thinking` content block observed in stream-json events"
|
|
)
|
|
compat_result.add({"status": "fail", "error": error})
|
|
failures.append(error)
|
|
continue
|
|
|
|
compat_result.add({"status": "pass"})
|
|
|
|
if failures:
|
|
pytest.fail("; ".join(failures), pytrace=False)
|