diff --git a/tests/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/claude_code/_driver_unit_tests/test_cli_driver.py index fbdea1a0aa4..4766111c08a 100644 --- a/tests/claude_code/_driver_unit_tests/test_cli_driver.py +++ b/tests/claude_code/_driver_unit_tests/test_cli_driver.py @@ -18,6 +18,7 @@ import pytest from tests.claude_code.cli_driver import ( ClaudeCLIError, DriverResult, + failure_diagnostic, run_claude, ) @@ -237,3 +238,102 @@ def test_run_claude_validates_required_params(): api_key="", runner=runner, ) + + +# --------------------------------------------------------------------------- +# failure_diagnostic +# +# Regression coverage for the bring-up incident where the proxy was started +# with the wrong config and tests reported only `claude CLI exited 1` while +# the actual 400 from LiteLLM was sitting in stdout. The helper must surface +# api_status, the assistant text (where API errors land), stderr, and the +# exit code together — and gracefully degrade when individual pieces are +# missing. +# --------------------------------------------------------------------------- + + +def test_failure_diagnostic_surfaces_api_error_text_from_stdout(): + """The CLI hides 4xx/5xx from the proxy in `assistant.message.content` text.""" + api_error_text = ( + 'API Error: 400 {"error":{"message":"litellm.BadRequestError: ' + "You passed in model=claude-haiku-4-5. There are no healthy " + 'deployments..."}}' + ) + result = DriverResult( + text=api_error_text, + events=[ + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": api_error_text}]}, + }, + { + "type": "result", + "is_error": True, + "api_error_status": 400, + "result": api_error_text, + }, + ], + exit_code=1, + stderr="", + ) + + diag = failure_diagnostic(result) + + assert "exit=1" in diag + assert "api_status=400" in diag + assert "There are no healthy deployments" in diag + + +def test_failure_diagnostic_falls_back_to_stderr_when_no_text(): + result = DriverResult(text="", events=[], exit_code=2, stderr="boom\n") + diag = failure_diagnostic(result) + assert "exit=2" in diag + assert "stderr=boom" in diag + + +def test_failure_diagnostic_handles_completely_empty_result(): + """A run that produced literally nothing should still yield a useful string.""" + result = DriverResult(text="", events=[], exit_code=137, stderr="") + diag = failure_diagnostic(result) + assert "exit=137" in diag + assert "no diagnostic output" in diag + + +def test_failure_diagnostic_truncates_long_text(): + """Don't let a 5MB HTML 502 page from a load balancer wreck the matrix JSON.""" + huge = "x" * 5000 + result = DriverResult(text=huge, events=[], exit_code=1, stderr="") + diag = failure_diagnostic(result, max_len=100) + assert "truncated" in diag + # Allow some slack for the prefix/suffix/separator characters. + assert len(diag) < 300 + + +def test_failure_diagnostic_ignores_non_int_api_error_status(): + """The CLI sometimes emits api_error_status as a string; don't crash.""" + result = DriverResult( + text="oops", + events=[{"type": "result", "api_error_status": "n/a"}], + exit_code=1, + stderr="", + ) + diag = failure_diagnostic(result) + assert "api_status" not in diag + assert "text=oops" in diag + + +def test_failure_diagnostic_uses_last_result_event_status(): + """If multiple `result` events appear, the most recent status wins.""" + result = DriverResult( + text="", + events=[ + {"type": "result", "api_error_status": 500}, + {"type": "assistant", "message": {"content": []}}, + {"type": "result", "api_error_status": 429}, + ], + exit_code=1, + stderr="", + ) + diag = failure_diagnostic(result) + assert "api_status=429" in diag + assert "500" not in diag diff --git a/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py index 7e0a1f16583..ee6314d5ded 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -23,7 +23,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -63,6 +63,9 @@ def test_basic_messaging_non_streaming_anthropic(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) + print(f"base_url: {base_url}") + print(f"api_key: {api_key}") + try: result = run_claude( prompt="Reply with the single word 'pong' and nothing else.", @@ -79,10 +82,12 @@ def test_basic_messaging_non_streaming_anthropic(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/claude_code/basic_messaging_non_streaming/test_azure.py index 22b09d24bc5..43e10f65d66 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_azure.py @@ -29,7 +29,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -86,10 +86,12 @@ def test_basic_messaging_non_streaming_azure(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index dc604f65acb..dca60237a28 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -24,7 +24,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,10 +75,12 @@ def test_basic_messaging_non_streaming_bedrock_converse(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index 8dc4ef9fff0..e6bd347c167 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -24,7 +24,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,10 +75,12 @@ def test_basic_messaging_non_streaming_bedrock_invoke(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py index a32d8d720ff..df6d79c710d 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -24,7 +24,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -75,10 +75,12 @@ def test_basic_messaging_non_streaming_vertex_ai(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/claude_code/basic_messaging_streaming/test_anthropic.py index 7a7ebfbe3db..ec52a24b5ae 100644 --- a/tests/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/claude_code/basic_messaging_streaming/test_anthropic.py @@ -25,7 +25,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -74,10 +74,13 @@ def test_basic_messaging_streaming_anthropic(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.events: diff --git a/tests/claude_code/basic_messaging_streaming/test_azure.py b/tests/claude_code/basic_messaging_streaming/test_azure.py index 96c10439b54..3b9d78b4912 100644 --- a/tests/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_streaming/test_azure.py @@ -23,7 +23,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -72,10 +72,12 @@ def test_basic_messaging_streaming_azure(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.events: diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py index c3b1bcdf52c..26008ed482b 100644 --- a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -19,7 +19,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -68,10 +68,12 @@ def test_basic_messaging_streaming_bedrock_converse(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.events: diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index 6f0aa0fec0b..0d0d1ee9a33 100644 --- a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -19,7 +19,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -68,10 +68,12 @@ def test_basic_messaging_streaming_bedrock_invoke(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.events: diff --git a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py index b2c914e2917..d909547673b 100644 --- a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -19,7 +19,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -68,10 +68,12 @@ def test_basic_messaging_streaming_vertex_ai(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.events: diff --git a/tests/claude_code/cli_driver.py b/tests/claude_code/cli_driver.py index cd8732491d3..c15b38bcb72 100644 --- a/tests/claude_code/cli_driver.py +++ b/tests/claude_code/cli_driver.py @@ -173,6 +173,73 @@ def _extract_assistant_text(events: Sequence[Mapping[str, Any]]) -> str: return "".join(chunks) +def failure_diagnostic(result: "DriverResult", *, max_len: int = 800) -> str: + """Build a human-readable error string from a non-zero `claude` CLI run. + + The CLI is annoying to debug because the most useful failure signal + rarely lands on stderr. When the proxy returns an HTTP error, the CLI + swallows it into an `assistant`/`result` event on **stdout** with + `is_error: true` and a JSON-shaped `text` block — and exits non-zero. + Tests that only print `stderr.strip()` see an empty string, which is + exactly the situation that masked a misconfigured proxy in early + bring-up of the compat matrix. + + This helper concatenates the most useful diagnostic we can find, in + priority order: + + 1. `result.text` (the assistant's user-visible reply, which is where + API errors land in stream-json mode), trimmed + 2. `api_error_status` from any `result` event, if present + 3. `result.stderr`, trimmed + 4. `` as a last resort + + The output is truncated to `max_len` characters so a giant HTML 502 + page from a misbehaving load balancer doesn't blow up the matrix + JSON. + """ + pieces: List[str] = [f"exit={result.exit_code}"] + + # api_error_status only appears on the final `result` event when the + # CLI received an HTTP error from the upstream API. Surfacing it + # explicitly makes "is this a proxy/auth problem or a CLI problem?" + # answerable without re-reading the events list. + api_status = _extract_api_error_status(result.events) + if api_status is not None: + pieces.append(f"api_status={api_status}") + + text = (result.text or "").strip() + if text: + pieces.append(f"text={_truncate(text, max_len)}") + + stderr = (result.stderr or "").strip() + if stderr: + pieces.append(f"stderr={_truncate(stderr, max_len)}") + + if len(pieces) == 1: + pieces.append("(no diagnostic output)") + + return "; ".join(pieces) + + +def _extract_api_error_status( + events: Sequence[Mapping[str, Any]], +) -> Optional[int]: + """Return the `api_error_status` from the last `result` event, if any.""" + for event in reversed(list(events)): + if event.get("type") != "result": + continue + status = event.get("api_error_status") + if isinstance(status, int): + return status + return None + + +def _truncate(s: str, max_len: int) -> str: + if len(s) <= max_len: + return s + return s[:max_len] + "...(truncated)" + + def _extract_usage(events: Sequence[Mapping[str, Any]]) -> Optional[Dict[str, Any]]: """Return the most recent `usage` block seen on any event, if any. diff --git a/tests/claude_code/extended_thinking/test_anthropic.py b/tests/claude_code/extended_thinking/test_anthropic.py index 39511f484e9..b9d67f887a0 100644 --- a/tests/claude_code/extended_thinking/test_anthropic.py +++ b/tests/claude_code/extended_thinking/test_anthropic.py @@ -21,7 +21,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -95,10 +95,12 @@ def test_extended_thinking_anthropic(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_thinking_block(result.events): diff --git a/tests/claude_code/extended_thinking/test_azure.py b/tests/claude_code/extended_thinking/test_azure.py index 6d5e89bfcfa..a03fdcf4b76 100644 --- a/tests/claude_code/extended_thinking/test_azure.py +++ b/tests/claude_code/extended_thinking/test_azure.py @@ -28,7 +28,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -97,10 +97,12 @@ def test_extended_thinking_azure(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_thinking_block(result.events): diff --git a/tests/claude_code/extended_thinking/test_bedrock_converse.py b/tests/claude_code/extended_thinking/test_bedrock_converse.py index a6941a90afe..de33f343a8b 100644 --- a/tests/claude_code/extended_thinking/test_bedrock_converse.py +++ b/tests/claude_code/extended_thinking/test_bedrock_converse.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -89,10 +89,12 @@ def test_extended_thinking_bedrock_converse(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_thinking_block(result.events): diff --git a/tests/claude_code/extended_thinking/test_bedrock_invoke.py b/tests/claude_code/extended_thinking/test_bedrock_invoke.py index 7fd67aa8efd..27b502207f4 100644 --- a/tests/claude_code/extended_thinking/test_bedrock_invoke.py +++ b/tests/claude_code/extended_thinking/test_bedrock_invoke.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -89,10 +89,12 @@ def test_extended_thinking_bedrock_invoke(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_thinking_block(result.events): diff --git a/tests/claude_code/extended_thinking/test_vertex_ai.py b/tests/claude_code/extended_thinking/test_vertex_ai.py index ee36470d4b0..55dae72ccde 100644 --- a/tests/claude_code/extended_thinking/test_vertex_ai.py +++ b/tests/claude_code/extended_thinking/test_vertex_ai.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -89,10 +89,12 @@ def test_extended_thinking_vertex_ai(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_thinking_block(result.events): diff --git a/tests/claude_code/prompt_caching_5m/test_anthropic.py b/tests/claude_code/prompt_caching_5m/test_anthropic.py index d5b5f75fe3f..c8e450198b3 100644 --- a/tests/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/claude_code/prompt_caching_5m/test_anthropic.py @@ -27,7 +27,7 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -88,10 +88,12 @@ def test_prompt_caching_5m_anthropic(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if _cache_tokens(result.usage) <= 0: diff --git a/tests/claude_code/prompt_caching_5m/test_azure.py b/tests/claude_code/prompt_caching_5m/test_azure.py index f470d950514..0fc12771997 100644 --- a/tests/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/claude_code/prompt_caching_5m/test_azure.py @@ -27,7 +27,7 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -86,10 +86,12 @@ def test_prompt_caching_5m_azure(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if _cache_tokens(result.usage) <= 0: diff --git a/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py index 417ea9b10ab..782a96c2ace 100644 --- a/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -79,10 +79,12 @@ def test_prompt_caching_5m_bedrock_converse(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if _cache_tokens(result.usage) <= 0: diff --git a/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py index c70d4ed3e32..5fe6679e7ca 100644 --- a/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -79,10 +79,12 @@ def test_prompt_caching_5m_bedrock_invoke(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if _cache_tokens(result.usage) <= 0: diff --git a/tests/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/claude_code/prompt_caching_5m/test_vertex_ai.py index bf33d89fffe..f2665979bf1 100644 --- a/tests/claude_code/prompt_caching_5m/test_vertex_ai.py +++ b/tests/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -79,10 +79,12 @@ def test_prompt_caching_5m_vertex_ai(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if _cache_tokens(result.usage) <= 0: diff --git a/tests/claude_code/test_config.yaml b/tests/claude_code/test_config.yaml index 05716fa41ca..2943f301fab 100644 --- a/tests/claude_code/test_config.yaml +++ b/tests/claude_code/test_config.yaml @@ -76,12 +76,6 @@ model_list: vertex_ai_location: us-east5 # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- - # Anthropic announced Claude Haiku 4.5, Sonnet 4.5/4.6, and Opus 4.1/4.6/4.7 - # in Microsoft Foundry on 2025-11-18. Foundry exposes these models on an - # Anthropic-shape `/anthropic/v1/messages` endpoint (not Azure OpenAI's - # chat-completions route), and LiteLLM routes them via the `azure_ai/` - # provider prefix. Set AZURE_FOUNDRY_API_BASE to the host (or host + - # `/anthropic`); LiteLLM normalizes the URL to `…/anthropic/v1/messages`. - model_name: claude-haiku-4-5-azure litellm_params: model: azure_ai/claude-haiku-4-5 diff --git a/tests/claude_code/tool_use/test_anthropic.py b/tests/claude_code/tool_use/test_anthropic.py index d2d40654a13..6a0627f0233 100644 --- a/tests/claude_code/tool_use/test_anthropic.py +++ b/tests/claude_code/tool_use/test_anthropic.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -93,10 +93,12 @@ def test_tool_use_anthropic(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_tool_use_event(result.events): diff --git a/tests/claude_code/tool_use/test_azure.py b/tests/claude_code/tool_use/test_azure.py index 1103adc610d..133e9d2b48d 100644 --- a/tests/claude_code/tool_use/test_azure.py +++ b/tests/claude_code/tool_use/test_azure.py @@ -24,7 +24,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -92,10 +92,12 @@ def test_tool_use_azure(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_tool_use_event(result.events): diff --git a/tests/claude_code/tool_use/test_bedrock_converse.py b/tests/claude_code/tool_use/test_bedrock_converse.py index 9a99c877d59..81b25f76668 100644 --- a/tests/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/claude_code/tool_use/test_bedrock_converse.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -88,10 +88,12 @@ def test_tool_use_bedrock_converse(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_tool_use_event(result.events): diff --git a/tests/claude_code/tool_use/test_bedrock_invoke.py b/tests/claude_code/tool_use/test_bedrock_invoke.py index 9bc9b66a81c..d54776b9bab 100644 --- a/tests/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/claude_code/tool_use/test_bedrock_invoke.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -88,10 +88,12 @@ def test_tool_use_bedrock_invoke(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_tool_use_event(result.events): diff --git a/tests/claude_code/tool_use/test_vertex_ai.py b/tests/claude_code/tool_use/test_vertex_ai.py index dcfe8d002e5..20c93e54fed 100644 --- a/tests/claude_code/tool_use/test_vertex_ai.py +++ b/tests/claude_code/tool_use/test_vertex_ai.py @@ -20,7 +20,7 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -88,10 +88,12 @@ def test_tool_use_vertex_ai(compat_result, model): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not _has_tool_use_event(result.events): diff --git a/tests/claude_code/vision/test_anthropic.py b/tests/claude_code/vision/test_anthropic.py index a381f9ed018..e0d178fb0ce 100644 --- a/tests/claude_code/vision/test_anthropic.py +++ b/tests/claude_code/vision/test_anthropic.py @@ -21,7 +21,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -78,10 +78,12 @@ def test_vision_anthropic(compat_result, model, tmp_path): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/vision/test_azure.py b/tests/claude_code/vision/test_azure.py index c24c41c05ac..f5e5c249036 100644 --- a/tests/claude_code/vision/test_azure.py +++ b/tests/claude_code/vision/test_azure.py @@ -25,7 +25,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -79,10 +79,12 @@ def test_vision_azure(compat_result, model, tmp_path): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/vision/test_bedrock_converse.py b/tests/claude_code/vision/test_bedrock_converse.py index 50067e9c594..2bdfa1196fd 100644 --- a/tests/claude_code/vision/test_bedrock_converse.py +++ b/tests/claude_code/vision/test_bedrock_converse.py @@ -20,7 +20,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -74,10 +74,12 @@ def test_vision_bedrock_converse(compat_result, model, tmp_path): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/vision/test_bedrock_invoke.py b/tests/claude_code/vision/test_bedrock_invoke.py index e0032e24145..ec9a4287cc3 100644 --- a/tests/claude_code/vision/test_bedrock_invoke.py +++ b/tests/claude_code/vision/test_bedrock_invoke.py @@ -20,7 +20,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -74,10 +74,12 @@ def test_vision_bedrock_invoke(compat_result, model, tmp_path): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip(): diff --git a/tests/claude_code/vision/test_vertex_ai.py b/tests/claude_code/vision/test_vertex_ai.py index 7e1101e12ce..2739206dd08 100644 --- a/tests/claude_code/vision/test_vertex_ai.py +++ b/tests/claude_code/vision/test_vertex_ai.py @@ -20,7 +20,7 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, run_claude +from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -74,10 +74,12 @@ def test_vision_vertex_ai(compat_result, model, tmp_path): compat_result.set( { "status": "fail", - "error": f"[{model}] claude CLI exited {result.exit_code}: {result.stderr.strip()}", + "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", } ) - pytest.fail(f"claude CLI exited {result.exit_code} for {model}", pytrace=False) + pytest.fail( + f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False + ) return if not result.text.strip():