From d05e45893e017e096897da167d2df28feed8348e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 6 May 2026 23:31:19 +0000 Subject: [PATCH] compat-matrix: parallel-fanout refactor + 5 new feature dirs + rate limiter - Switch all per-cell tests from @pytest.mark.parametrize("model", ...) (3 sequential invocations) to a single test that fans out to all 3 Claude tiers via run_claude_models_parallel. Per-cell wall time is now bounded by the slowest model rather than the sum. - Add 5 new v0 feature dirs (5 providers each, 25 new test files): web_search, pdf_input, prompt_caching_1h, tool_use_streaming, thinking_with_tool_use Manifest expanded to match. - Add cross-process token-bucket rate limiter (rate_limiter.py + tests) so xdist workers stay under per-provider req/s limits during full-grid runs. New env knobs: LITELLM_COMPAT_RATE_{ANTHROPIC,AZURE,VERTEX_AI, BEDROCK_CONVERSE,BEDROCK_INVOKE}. - conftest.py: write per-worker shards under .shards/, merge in the controller; preserve the "don't write empty artifact" guard so unit-test runs don't clobber a real compat-results.json. - Vertex test_config.yaml: route project/location through env so the cron VM can target a different GCP project than the upstream default. - Add run_compat.sh wrapper for binary-searching ideal req/s per provider against compat-rate-limit-summary.json output. --- .gitignore | 2 + .../test_matrix_builder.py | 23 +- .../_builder_unit_tests/test_v0_layout.py | 8 +- .../_driver_unit_tests/conftest.py | 32 ++ .../_driver_unit_tests/test_cli_driver.py | 236 ++++++++++ .../_driver_unit_tests/test_compat_result.py | 68 +++ .../_driver_unit_tests/test_rate_limiter.py | 329 ++++++++++++++ .../test_anthropic.py | 78 ++-- .../test_azure.py | 78 ++-- .../test_bedrock_converse.py | 78 ++-- .../test_bedrock_invoke.py | 78 ++-- .../test_vertex_ai.py | 78 ++-- .../test_anthropic.py | 89 ++-- .../basic_messaging_streaming/test_azure.py | 84 ++-- .../test_bedrock_converse.py | 84 ++-- .../test_bedrock_invoke.py | 84 ++-- .../test_vertex_ai.py | 84 ++-- tests/claude_code/cli_driver.py | 164 ++++++- tests/claude_code/conftest.py | 408 +++++++++++++++--- .../extended_thinking/test_anthropic.py | 78 ++-- .../extended_thinking/test_azure.py | 74 ++-- .../test_bedrock_converse.py | 74 ++-- .../extended_thinking/test_bedrock_invoke.py | 74 ++-- .../extended_thinking/test_vertex_ai.py | 74 ++-- tests/claude_code/manifest.yaml | 10 + tests/claude_code/pdf_input/__init__.py | 0 tests/claude_code/pdf_input/test_anthropic.py | 176 ++++++++ tests/claude_code/pdf_input/test_azure.py | 146 +++++++ .../pdf_input/test_bedrock_converse.py | 152 +++++++ .../pdf_input/test_bedrock_invoke.py | 151 +++++++ tests/claude_code/pdf_input/test_vertex_ai.py | 146 +++++++ .../claude_code/prompt_caching_1h/__init__.py | 0 .../prompt_caching_1h/test_anthropic.py | 124 ++++++ .../prompt_caching_1h/test_azure.py | 104 +++++ .../test_bedrock_converse.py | 112 +++++ .../prompt_caching_1h/test_bedrock_invoke.py | 117 +++++ .../prompt_caching_1h/test_vertex_ai.py | 104 +++++ .../prompt_caching_5m/test_anthropic.py | 78 ++-- .../prompt_caching_5m/test_azure.py | 78 ++-- .../test_bedrock_converse.py | 78 ++-- .../prompt_caching_5m/test_bedrock_invoke.py | 78 ++-- .../prompt_caching_5m/test_vertex_ai.py | 78 ++-- tests/claude_code/rate_limiter.py | 358 +++++++++++++++ tests/claude_code/run_compat.sh | 106 +++++ tests/claude_code/test_config.yaml | 12 +- .../thinking_with_tool_use/__init__.py | 0 .../thinking_with_tool_use/test_anthropic.py | 148 +++++++ .../thinking_with_tool_use/test_azure.py | 123 ++++++ .../test_bedrock_converse.py | 128 ++++++ .../test_bedrock_invoke.py | 130 ++++++ .../thinking_with_tool_use/test_vertex_ai.py | 128 ++++++ tests/claude_code/tool_use/test_anthropic.py | 74 ++-- tests/claude_code/tool_use/test_azure.py | 74 ++-- .../tool_use/test_bedrock_converse.py | 74 ++-- .../tool_use/test_bedrock_invoke.py | 74 ++-- tests/claude_code/tool_use/test_vertex_ai.py | 74 ++-- .../tool_use_streaming/__init__.py | 0 .../tool_use_streaming/test_anthropic.py | 147 +++++++ .../tool_use_streaming/test_azure.py | 122 ++++++ .../test_bedrock_converse.py | 128 ++++++ .../tool_use_streaming/test_bedrock_invoke.py | 126 ++++++ .../tool_use_streaming/test_vertex_ai.py | 125 ++++++ tests/claude_code/vision/test_anthropic.py | 72 ++-- tests/claude_code/vision/test_azure.py | 72 ++-- .../vision/test_bedrock_converse.py | 72 ++-- .../claude_code/vision/test_bedrock_invoke.py | 72 ++-- tests/claude_code/vision/test_vertex_ai.py | 72 ++-- tests/claude_code/web_search/__init__.py | 0 .../claude_code/web_search/test_anthropic.py | 142 ++++++ tests/claude_code/web_search/test_azure.py | 115 +++++ .../web_search/test_bedrock_converse.py | 115 +++++ .../web_search/test_bedrock_invoke.py | 122 ++++++ .../claude_code/web_search/test_vertex_ai.py | 120 ++++++ 73 files changed, 6100 insertions(+), 1216 deletions(-) create mode 100644 tests/claude_code/_driver_unit_tests/conftest.py create mode 100644 tests/claude_code/_driver_unit_tests/test_rate_limiter.py create mode 100644 tests/claude_code/pdf_input/__init__.py create mode 100644 tests/claude_code/pdf_input/test_anthropic.py create mode 100644 tests/claude_code/pdf_input/test_azure.py create mode 100644 tests/claude_code/pdf_input/test_bedrock_converse.py create mode 100644 tests/claude_code/pdf_input/test_bedrock_invoke.py create mode 100644 tests/claude_code/pdf_input/test_vertex_ai.py create mode 100644 tests/claude_code/prompt_caching_1h/__init__.py create mode 100644 tests/claude_code/prompt_caching_1h/test_anthropic.py create mode 100644 tests/claude_code/prompt_caching_1h/test_azure.py create mode 100644 tests/claude_code/prompt_caching_1h/test_bedrock_converse.py create mode 100644 tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py create mode 100644 tests/claude_code/prompt_caching_1h/test_vertex_ai.py create mode 100644 tests/claude_code/rate_limiter.py create mode 100755 tests/claude_code/run_compat.sh create mode 100644 tests/claude_code/thinking_with_tool_use/__init__.py create mode 100644 tests/claude_code/thinking_with_tool_use/test_anthropic.py create mode 100644 tests/claude_code/thinking_with_tool_use/test_azure.py create mode 100644 tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py create mode 100644 tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py create mode 100644 tests/claude_code/thinking_with_tool_use/test_vertex_ai.py create mode 100644 tests/claude_code/tool_use_streaming/__init__.py create mode 100644 tests/claude_code/tool_use_streaming/test_anthropic.py create mode 100644 tests/claude_code/tool_use_streaming/test_azure.py create mode 100644 tests/claude_code/tool_use_streaming/test_bedrock_converse.py create mode 100644 tests/claude_code/tool_use_streaming/test_bedrock_invoke.py create mode 100644 tests/claude_code/tool_use_streaming/test_vertex_ai.py create mode 100644 tests/claude_code/web_search/__init__.py create mode 100644 tests/claude_code/web_search/test_anthropic.py create mode 100644 tests/claude_code/web_search/test_azure.py create mode 100644 tests/claude_code/web_search/test_bedrock_converse.py create mode 100644 tests/claude_code/web_search/test_bedrock_invoke.py create mode 100644 tests/claude_code/web_search/test_vertex_ai.py diff --git a/.gitignore b/.gitignore index 458bcba0dca..fe6d506f870 100644 --- a/.gitignore +++ b/.gitignore @@ -104,5 +104,7 @@ test-config # Claude Code compatibility-matrix pytest artifact (CI-only output). compat-results.json +compat-results.json.shards/ +compat-rate-limit-summary.json # Matrix JSON produced by the daily-cron publisher (pushed to litellm-docs). compatibility-matrix.json diff --git a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py index 9aa7a96609d..6cf34a05e3f 100644 --- a/tests/claude_code/_builder_unit_tests/test_matrix_builder.py +++ b/tests/claude_code/_builder_unit_tests/test_matrix_builder.py @@ -192,7 +192,28 @@ def test_build_matrix_6x5_grid_matches_published_sample(): so any future schema drift surfaces here in review. """ repo_root = Path(__file__).resolve().parents[1] - manifest = load_manifest(repo_root / "manifest.yaml") + full_manifest = load_manifest(repo_root / "manifest.yaml") + + # The v0 sample matrix is a frozen baseline: it covers exactly the + # six features the PRD shipped with, in their canonical order. The + # live manifest may carry additional rows (extensions added after + # v0 shipped), but the sample is derived only from the v0 slice so + # this test stays a meaningful regression gate for the v0 cell + # shape rather than chasing every new row added downstream. + v0_feature_ids = [ + "basic_messaging_non_streaming", + "basic_messaging_streaming", + "tool_use", + "prompt_caching_5m", + "vision", + "extended_thinking", + ] + v0_features = [ + feature + for feature in full_manifest["features"] + if feature["id"] in v0_feature_ids + ] + manifest = {**full_manifest, "features": v0_features} feature_ids = [feature["id"] for feature in manifest["features"]] providers = manifest["providers"] diff --git a/tests/claude_code/_builder_unit_tests/test_v0_layout.py b/tests/claude_code/_builder_unit_tests/test_v0_layout.py index 1ae1e8f2a67..19064c15df1 100644 --- a/tests/claude_code/_builder_unit_tests/test_v0_layout.py +++ b/tests/claude_code/_builder_unit_tests/test_v0_layout.py @@ -46,8 +46,14 @@ def manifest() -> dict: def test_manifest_lists_all_six_v0_features_in_order(manifest): + """The PRD's v0 row set must appear at the top of the manifest in + order. Features beyond v0 (extensions added after the matrix + shipped) are allowed but must not reorder or displace the v0 + rows — the docs page anchors row links by index, so v0 stays + pinned at positions [0:6] for the lifetime of the schema. + """ ids = [feature["id"] for feature in manifest["features"]] - assert ids == EXPECTED_FEATURE_IDS + assert ids[: len(EXPECTED_FEATURE_IDS)] == EXPECTED_FEATURE_IDS def test_manifest_lists_all_five_v0_providers_in_order(manifest): diff --git a/tests/claude_code/_driver_unit_tests/conftest.py b/tests/claude_code/_driver_unit_tests/conftest.py new file mode 100644 index 00000000000..f32a841f677 --- /dev/null +++ b/tests/claude_code/_driver_unit_tests/conftest.py @@ -0,0 +1,32 @@ +"""Local conftest for the driver unit tests. + +Installs a hermetic, no-op rate limiter for every test in this +subdirectory. Without this, importing `cli_driver` and calling +`run_claude(..., runner=fake)` would silently consume tokens from the +shared default limiter (which writes to `$TMPDIR/...`), polluting the +on-disk state another test run might rely on and adding flakiness if +the env vars say "rate=0.1/s". + +A no-op limiter (rate=0 for every provider) returns immediately from +`acquire(...)`, so unit tests behave exactly as they did before the +limiter was added. +""" + +from __future__ import annotations + +import pytest + +from tests.claude_code.rate_limiter import ( + ALL_PROVIDERS, + ProviderConfig, + RateLimiter, + use_limiter, +) + + +@pytest.fixture(autouse=True) +def _hermetic_rate_limiter(tmp_path): + config = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS} + limiter = RateLimiter(config=config, state_dir=tmp_path) + with use_limiter(limiter): + yield diff --git a/tests/claude_code/_driver_unit_tests/test_cli_driver.py b/tests/claude_code/_driver_unit_tests/test_cli_driver.py index 073b1559504..e7725b1116c 100644 --- a/tests/claude_code/_driver_unit_tests/test_cli_driver.py +++ b/tests/claude_code/_driver_unit_tests/test_cli_driver.py @@ -20,6 +20,7 @@ from tests.claude_code.cli_driver import ( DriverResult, failure_diagnostic, run_claude, + run_claude_models_parallel, ) @@ -345,6 +346,241 @@ def test_failure_diagnostic_ignores_non_int_api_error_status(): assert "text=oops" in diag +# --------------------------------------------------------------------------- +# run_claude_models_parallel +# +# The matrix runs three Claude tiers per cell, so the parallel helper has to +# (a) invoke `run_claude` once per model, (b) preserve each model's outcome +# separately, and (c) return errors as values rather than raising — callers +# need both the failed and the succeeded model results to report per-cell +# rows accurately. +# --------------------------------------------------------------------------- + + +def test_run_claude_models_parallel_returns_one_result_per_model(): + """Each model gets its own DriverResult keyed under the helper's dict.""" + seen_models: List[str] = [] + + def runner(cmd, env, capture_output, text, timeout, check): + # The model id is two slots after `--model` in the assembled command. + idx = cmd.index("--model") + model = cmd[idx + 1] + seen_models.append(model) + return _Completed( + returncode=0, + stdout=json.dumps( + { + "type": "assistant", + "message": { + "content": [{"type": "text", "text": f"reply-{model}"}] + }, + } + ) + + "\n", + ) + + outcomes = run_claude_models_parallel( + models=["a", "b", "c"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + assert set(outcomes.keys()) == {"a", "b", "c"} + for model in ("a", "b", "c"): + result = outcomes[model] + assert isinstance(result, DriverResult) + assert result.text == f"reply-{model}" + assert result.exit_code == 0 + assert sorted(seen_models) == ["a", "b", "c"] + + +def test_run_claude_models_parallel_returns_errors_as_values(): + """A model whose CLI is missing surfaces as a ClaudeCLIError, not a raise.""" + + def runner(cmd, env, capture_output, text, timeout, check): + idx = cmd.index("--model") + model = cmd[idx + 1] + if model == "boom": + raise FileNotFoundError(2, "no such file", "claude") + return _Completed( + returncode=0, + stdout=json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "ok"}]}, + } + ) + + "\n", + ) + + outcomes = run_claude_models_parallel( + models=["ok-model", "boom"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + assert isinstance(outcomes["ok-model"], DriverResult) + assert outcomes["ok-model"].text == "ok" + assert isinstance(outcomes["boom"], ClaudeCLIError) + assert "claude CLI not found" in str(outcomes["boom"]) + + +def test_run_claude_models_parallel_preserves_nonzero_exit_codes(): + """Mixed success/failure on exit code should not collapse into one verdict.""" + + def runner(cmd, env, capture_output, text, timeout, check): + idx = cmd.index("--model") + model = cmd[idx + 1] + if model == "fail": + return _Completed(returncode=2, stdout="", stderr="auth failed") + return _Completed( + returncode=0, + stdout=json.dumps( + { + "type": "assistant", + "message": {"content": [{"type": "text", "text": "ok"}]}, + } + ) + + "\n", + ) + + outcomes = run_claude_models_parallel( + models=["ok-model", "fail"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + assert outcomes["ok-model"].exit_code == 0 + assert outcomes["fail"].exit_code == 2 + assert outcomes["fail"].stderr == "auth failed" + + +def test_run_claude_models_parallel_rejects_empty_models(): + with pytest.raises(ValueError, match="non-empty"): + run_claude_models_parallel( + models=[], + prompt="hi", + base_url="http://x", + api_key="k", + ) + + +def test_run_claude_models_parallel_stamps_duration_on_each_result(): + """Each DriverResult carries the per-model wall time so callers can + attribute slow cells without re-timing the work themselves. + + The fake runner sleeps for very different durations per model so + we can prove each result is timing its own work (not the batch + wall time). We use generous absolute bounds because thread-pool + scheduling on a loaded CI box adds noise on the order of tens of + milliseconds. + """ + import time + + def runner(cmd, env, capture_output, text, timeout, check): + idx = cmd.index("--model") + model = cmd[idx + 1] + time.sleep(0.05 if model == "fast" else 0.40) + return _Completed(returncode=0, stdout="") + + outcomes = run_claude_models_parallel( + models=["fast", "slow"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + fast_ms = outcomes["fast"].duration_ms + slow_ms = outcomes["slow"].duration_ms + assert fast_ms is not None and slow_ms is not None + # 50ms sleep ⇒ ~50–250ms after scheduling overhead; 400ms sleep ⇒ + # 400–700ms. We just need the two distributions to be non-overlapping + # so we know each row's duration is its own work, not the batch's. + assert fast_ms < 300, fast_ms + assert slow_ms >= 350, slow_ms + assert slow_ms > fast_ms + + +def test_run_claude_models_parallel_breakdown_logs_to_stderr(capsys): + """The breakdown helper must emit a per-model timing block so users + can answer "why didn't parallel help?" without re-instrumenting.""" + + def runner(cmd, env, capture_output, text, timeout, check): + return _Completed(returncode=0, stdout="") + + run_claude_models_parallel( + models=["model-x", "model-y"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + captured = capsys.readouterr() + assert "[parallel] per-model wall time:" in captured.err + assert "model-x" in captured.err + assert "model-y" in captured.err + assert "speedup=" in captured.err + assert "slowest=" in captured.err + + +def test_run_claude_models_parallel_breakdown_marks_cli_errors(capsys): + """When a model raises ClaudeCLIError, the breakdown should still + show its row tagged as `cli-error` rather than crashing or omitting it.""" + + def runner(cmd, env, capture_output, text, timeout, check): + idx = cmd.index("--model") + if cmd[idx + 1] == "boom": + raise FileNotFoundError(2, "no such file", "claude") + return _Completed(returncode=0, stdout="") + + run_claude_models_parallel( + models=["ok-model", "boom"], + prompt="hi", + base_url="http://x", + api_key="k", + runner=runner, + ) + + captured = capsys.readouterr() + assert "ok-model" in captured.err + assert "boom" in captured.err + assert "cli-error" in captured.err + + +def test_run_claude_models_parallel_forwards_extra_args_and_env(): + """Shared kwargs must reach every per-model invocation unchanged.""" + captured_envs: List[dict] = [] + captured_cmds: List[List[str]] = [] + + def runner(cmd, env, capture_output, text, timeout, check): + captured_envs.append(env) + captured_cmds.append(cmd) + return _Completed(returncode=0, stdout="") + + run_claude_models_parallel( + models=["a", "b"], + prompt="hi", + base_url="http://x", + api_key="k", + extra_env={"MAX_THINKING_TOKENS": "4096"}, + extra_args=["--allowed-tools", "Bash"], + runner=runner, + ) + + assert all(env["MAX_THINKING_TOKENS"] == "4096" for env in captured_envs) + for cmd in captured_cmds: + assert "--allowed-tools" in cmd + assert "Bash" in cmd + + def test_failure_diagnostic_uses_last_result_event_status(): """If multiple `result` events appear, the most recent status wins.""" result = DriverResult( diff --git a/tests/claude_code/_driver_unit_tests/test_compat_result.py b/tests/claude_code/_driver_unit_tests/test_compat_result.py index 04e19bdc482..7e7c007d5fd 100644 --- a/tests/claude_code/_driver_unit_tests/test_compat_result.py +++ b/tests/claude_code/_driver_unit_tests/test_compat_result.py @@ -68,3 +68,71 @@ def test_set_copies_input(): r.set(payload) payload["error"] = "mutated" assert r.value["error"] == "x" + + +# --------------------------------------------------------------------------- +# add() / collected() +# +# When a single test exercises three Claude tiers in parallel, each tier +# needs its own row in the results artifact so the matrix builder can +# apply its "all three must pass" aggregation. `add()` is the per-tier +# recorder; `collected()` is what the conftest hook reads. +# --------------------------------------------------------------------------- + + +def test_add_appends_each_call_to_values(): + r = CompatResult() + r.add({"status": "pass"}) + r.add({"status": "fail", "error": "bad"}) + assert r.values == [ + {"status": "pass"}, + {"status": "fail", "error": "bad"}, + ] + + +def test_add_validates_like_set(): + """The add() and set() validators are the same; both must reject bad payloads.""" + r = CompatResult() + with pytest.raises(ValueError, match="requires 'error'"): + r.add({"status": "fail"}) + with pytest.raises(ValueError, match="requires 'reason'"): + r.add({"status": "not_applicable"}) + with pytest.raises(ValueError, match="status must be one of"): + r.add({"status": "maybe"}) + with pytest.raises(TypeError): + r.add("pass") # type: ignore[arg-type] + + +def test_add_copies_input(): + """Same defensive copy contract as set().""" + r = CompatResult() + payload = {"status": "fail", "error": "x"} + r.add(payload) + payload["error"] = "mutated" + assert r.values[0]["error"] == "x" + + +def test_collected_returns_values_when_added(): + r = CompatResult() + r.add({"status": "pass"}) + r.add({"status": "pass"}) + assert r.collected() == [{"status": "pass"}, {"status": "pass"}] + + +def test_collected_returns_single_value_when_only_set_called(): + """Legacy single-result tests should still surface their one outcome.""" + r = CompatResult() + r.set({"status": "pass"}) + assert r.collected() == [{"status": "pass"}] + + +def test_collected_prefers_added_values_over_set_value(): + """If both are populated, the per-tier list wins — that's the multi-model shape.""" + r = CompatResult() + r.set({"status": "pass"}) + r.add({"status": "fail", "error": "tier-2 broke"}) + assert r.collected() == [{"status": "fail", "error": "tier-2 broke"}] + + +def test_collected_returns_empty_when_nothing_reported(): + assert CompatResult().collected() == [] diff --git a/tests/claude_code/_driver_unit_tests/test_rate_limiter.py b/tests/claude_code/_driver_unit_tests/test_rate_limiter.py new file mode 100644 index 00000000000..2c6a46c4cb2 --- /dev/null +++ b/tests/claude_code/_driver_unit_tests/test_rate_limiter.py @@ -0,0 +1,329 @@ +"""Unit tests for the cross-process token-bucket rate limiter. + +The tests cover three layers: + +1. Provider inference from model alias — the matrix-column mapping the + live tests rely on (`-bedrock-converse` vs `-bedrock-invoke` vs + `-azure` vs `-vertex` vs bare = anthropic). + +2. Config parsing — env-var precedence, fallback to default, malformed + input handling, burst override semantics. These run against + `os.environ`-shaped dicts so we don't have to monkeypatch globals. + +3. Token-bucket behavior — enforcing rate, accumulating burst, never + over-spending across a fake clock. Filesystem state is exercised + with a real `tmp_path` because the persistence is the whole point; + the only injected seam is `clock` (and `sleep`, so tests don't + actually wait on wall time). + +The cross-process flock semantics are exercised indirectly: every +test creates a fresh `RateLimiter` rooted at `tmp_path`, so the same +file lock that protects production is exercised here too. We don't +fork to test multi-process behavior in this file because pytest +fixtures + xdist already do that for the integration suite. +""" + +from __future__ import annotations + +import json +import time +from pathlib import Path +from typing import List + +import pytest + +from tests.claude_code.rate_limiter import ( + ALL_PROVIDERS, + BURST_ENV, + DEFAULT_RATE, + PROVIDER_ANTHROPIC, + PROVIDER_AZURE, + PROVIDER_BEDROCK_CONVERSE, + PROVIDER_BEDROCK_INVOKE, + PROVIDER_VERTEX_AI, + ProviderConfig, + RateLimiter, + get_default_limiter, + infer_provider, + load_config, + reset_default_limiter, + use_limiter, +) + + +# --------------------------------------------------------------------------- +# Provider inference +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "model, expected", + [ + ("claude-haiku-4-5", PROVIDER_ANTHROPIC), + ("claude-sonnet-4-6", PROVIDER_ANTHROPIC), + ("claude-opus-4-7", PROVIDER_ANTHROPIC), + ("claude-haiku-4-5-azure", PROVIDER_AZURE), + ("claude-sonnet-4-6-azure", PROVIDER_AZURE), + ("claude-opus-4-7-vertex", PROVIDER_VERTEX_AI), + ("claude-haiku-4-5-bedrock-converse", PROVIDER_BEDROCK_CONVERSE), + ("claude-haiku-4-5-bedrock-invoke", PROVIDER_BEDROCK_INVOKE), + ], +) +def test_infer_provider_maps_alias_suffix_to_column(model, expected): + assert infer_provider(model) == expected + + +def test_infer_provider_bedrock_converse_beats_bedrock_invoke_lookup_order(): + """Both bedrock suffixes contain `bedrock`; the more-specific suffix wins.""" + assert infer_provider("claude-foo-bedrock-converse") == PROVIDER_BEDROCK_CONVERSE + assert infer_provider("claude-foo-bedrock-invoke") == PROVIDER_BEDROCK_INVOKE + + +def test_infer_provider_rejects_empty_string(): + with pytest.raises(ValueError, match="non-empty"): + infer_provider("") + + +def test_infer_provider_is_case_insensitive(): + """Aliases in the proxy config sometimes drift between cases; we + should still route them to the right column.""" + assert infer_provider("CLAUDE-OPUS-4-7-AZURE") == PROVIDER_AZURE + + +# --------------------------------------------------------------------------- +# Config parsing +# --------------------------------------------------------------------------- + + +def test_load_config_uses_default_rate_when_env_absent(): + cfg = load_config(env={}) + for provider in ALL_PROVIDERS: + assert cfg[provider].rate_per_sec == DEFAULT_RATE + assert cfg[provider].burst == DEFAULT_RATE + + +def test_load_config_reads_per_provider_rate(): + cfg = load_config( + env={ + "LITELLM_COMPAT_RATE_ANTHROPIC": "10", + "LITELLM_COMPAT_RATE_AZURE": "0.5", + } + ) + assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == 10.0 + assert cfg[PROVIDER_AZURE].rate_per_sec == 0.5 + assert cfg[PROVIDER_VERTEX_AI].rate_per_sec == DEFAULT_RATE + + +def test_load_config_zero_rate_disables_provider(): + cfg = load_config(env={"LITELLM_COMPAT_RATE_BEDROCK_INVOKE": "0"}) + assert cfg[PROVIDER_BEDROCK_INVOKE].enabled is False + + +def test_load_config_burst_override_applies_to_every_provider(): + cfg = load_config( + env={ + "LITELLM_COMPAT_RATE_ANTHROPIC": "5", + BURST_ENV: "20", + } + ) + for provider in ALL_PROVIDERS: + assert cfg[provider].burst == 20.0 + + +def test_load_config_falls_back_on_malformed_value(): + cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "not-a-number"}) + assert cfg[PROVIDER_ANTHROPIC].rate_per_sec == DEFAULT_RATE + + +def test_load_config_burst_floors_at_one_when_rate_is_low(): + """A 0.5/s rate with no burst override must still allow at least + one immediate request — otherwise the very first call would block.""" + cfg = load_config(env={"LITELLM_COMPAT_RATE_ANTHROPIC": "0.5"}) + assert cfg[PROVIDER_ANTHROPIC].burst == 1.0 + + +# --------------------------------------------------------------------------- +# Token bucket +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_clock(): + """A controllable monotonic clock + sleep for the limiter under test. + + Tests advance `clock.now` to simulate elapsed wall time. `sleep` + adds the requested duration to `clock.now` instead of actually + sleeping, so a "wait 200ms" code path runs in microseconds and + is deterministic. + """ + + class Clock: + def __init__(self): + self.now = 1_000.0 + self.sleeps: List[float] = [] + + def __call__(self): + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + return Clock() + + +def _make_limiter(tmp_path: Path, fake_clock, *, rate=10.0, burst=None): + cfg = { + p: ProviderConfig(rate_per_sec=rate, burst=burst if burst is not None else rate) + for p in ALL_PROVIDERS + } + return RateLimiter( + config=cfg, + state_dir=tmp_path, + clock=fake_clock, + sleep=fake_clock.sleep, + ) + + +def test_acquire_first_call_does_not_wait(tmp_path, fake_clock): + """A freshly-initialized bucket starts full; the first acquire is free.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=10.0) + waited = limiter.acquire(PROVIDER_ANTHROPIC) + assert waited == 0.0 + assert fake_clock.sleeps == [] + + +def test_acquire_disabled_provider_returns_immediately(tmp_path, fake_clock): + """rate=0 ⇒ no throttling, even if every other provider is throttled.""" + cfg = {p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS} + limiter = RateLimiter( + config=cfg, state_dir=tmp_path, clock=fake_clock, sleep=fake_clock.sleep + ) + for _ in range(100): + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + assert fake_clock.sleeps == [] + + +def test_acquire_burns_through_burst_then_throttles(tmp_path, fake_clock): + """`burst` immediate requests succeed; the next one waits 1/rate seconds.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=3.0) + + for _ in range(3): + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + # Bucket is empty; next call must sleep ~0.5s to earn one token at 2/s. + waited = limiter.acquire(PROVIDER_ANTHROPIC) + assert waited == pytest.approx(0.5, abs=0.01) + + +def test_acquire_refills_with_elapsed_time(tmp_path, fake_clock): + """Advancing the clock between calls credits tokens at the configured rate.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=4.0, burst=1.0) + + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 # consumes the 1-token burst + fake_clock.now += 0.25 # 0.25s × 4/s = 1 token earned + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + +def test_acquire_caps_refill_at_burst(tmp_path, fake_clock): + """A long quiet period must not let the bucket grow past `burst`.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=10.0, burst=2.0) + + fake_clock.now += 1_000 # would earn 10_000 tokens uncapped + # Only `burst` (=2) immediate calls should succeed before throttling. + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + waited = limiter.acquire(PROVIDER_ANTHROPIC) + assert waited > 0 + + +def test_acquire_independent_buckets_per_provider(tmp_path, fake_clock): + """Anthropic exhaustion must not throttle Azure (each column has its own bucket).""" + limiter = _make_limiter(tmp_path, fake_clock, rate=2.0, burst=1.0) + + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + # Anthropic bucket is now empty; Azure is untouched. + assert limiter.acquire(PROVIDER_AZURE) == 0.0 + + +def test_acquire_persists_state_across_limiter_instances(tmp_path): + """A fresh RateLimiter must read the on-disk state, not start fresh. + + This is the property that makes the limiter cross-process: an + xdist worker created mid-run sees the credit consumed by other + workers, instead of getting its own private bucket. + """ + cfg = {p: ProviderConfig(rate_per_sec=10.0, burst=2.0) for p in ALL_PROVIDERS} + state = {"now": 1_000.0, "sleeps": []} + + def clock(): + return state["now"] + + def sleep(seconds): + state["sleeps"].append(seconds) + state["now"] += seconds + + first = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep) + first.acquire(PROVIDER_ANTHROPIC) + first.acquire(PROVIDER_ANTHROPIC) + # bucket is now empty + + second = RateLimiter(config=cfg, state_dir=tmp_path, clock=clock, sleep=sleep) + waited = second.acquire(PROVIDER_ANTHROPIC) + assert waited > 0 # had to wait, didn't see a fresh full bucket + + +def test_acquire_recovers_from_corrupt_state_file(tmp_path, fake_clock): + """A truncated/garbage state file must not crash the test session.""" + state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json" + state_file.write_text("not-json {{") + + limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0) + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + +def test_acquire_handles_clock_going_backward(tmp_path, fake_clock): + """Across a host suspend/resume the monotonic clock can briefly + go backward; we must not interpret that as removing tokens.""" + limiter = _make_limiter(tmp_path, fake_clock, rate=1.0, burst=2.0) + limiter.acquire(PROVIDER_ANTHROPIC) + fake_clock.now -= 10 # clock moved backward + # Bucket should still have ~1 token left from the burst, not -9. + assert limiter.acquire(PROVIDER_ANTHROPIC) == 0.0 + + +# --------------------------------------------------------------------------- +# Process-default singleton +# --------------------------------------------------------------------------- + + +def test_use_limiter_swaps_default_for_block(tmp_path): + sentinel_cfg = { + p: ProviderConfig(rate_per_sec=0.0, burst=0.0) for p in ALL_PROVIDERS + } + sentinel = RateLimiter(config=sentinel_cfg, state_dir=tmp_path) + reset_default_limiter() + try: + with use_limiter(sentinel): + assert get_default_limiter() is sentinel + # After the context exits, the default goes back to whatever it + # was — in this test that's "rebuilt on next access" because we + # called reset_default_limiter() above. + assert get_default_limiter() is not sentinel + finally: + reset_default_limiter() + + +# --------------------------------------------------------------------------- +# Persistence shape +# --------------------------------------------------------------------------- + + +def test_state_file_is_json_after_acquire(tmp_path, fake_clock): + limiter = _make_limiter(tmp_path, fake_clock, rate=5.0, burst=5.0) + limiter.acquire(PROVIDER_ANTHROPIC) + state_file = tmp_path / f"{PROVIDER_ANTHROPIC}.json" + payload = json.loads(state_file.read_text()) + assert "tokens" in payload + assert "last_refill" in payload + assert payload["tokens"] == pytest.approx(4.0) diff --git a/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py b/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py index 4929e8eeccf..7366ecd6fdd 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_anthropic.py @@ -12,9 +12,11 @@ The (feature, provider) for this cell is inferred from the file path by feature_id provider Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. We parametrize over the -three models and the conftest aggregator produces one cell from the three -results. +4.7; the cell only goes green if all three pass. We fan the three model +runs out in parallel inside this single test so the per-cell wall time is +bounded by the slowest model rather than the sum, and report one +`compat_result.add(...)` entry per model so the matrix builder still sees +three rows for this (feature, provider). """ from __future__ import annotations @@ -23,7 +25,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -38,8 +44,7 @@ ANTHROPIC_MODELS = [ ] -@pytest.mark.parametrize("model", ANTHROPIC_MODELS) -def test_basic_messaging_non_streaming_anthropic(compat_result, model): +def test_basic_messaging_non_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. "Basic messaging" means: send a single user prompt, receive any @@ -63,38 +68,35 @@ def test_basic_messaging_non_streaming_anthropic(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_non_streaming/test_azure.py b/tests/claude_code/basic_messaging_non_streaming/test_azure.py index 43e10f65d66..98d4ebce861 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_azure.py @@ -17,10 +17,10 @@ The (feature, provider) for this cell is inferred from the file path by feature_id provider Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. We parametrize over the -three models and the conftest aggregator produces one cell from the -three results, naming the failing model in the error string when any -model fails. +4.7; the cell only goes green if all three pass. We fan the three model +runs out in parallel inside this single test and report one +`compat_result.add(...)` entry per model so the matrix builder still sees +three rows for this (feature, provider). """ from __future__ import annotations @@ -29,7 +29,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -45,8 +49,7 @@ AZURE_MODELS = [ ] -@pytest.mark.parametrize("model", AZURE_MODELS) -def test_basic_messaging_non_streaming_azure(compat_result, model): +def test_basic_messaging_non_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply. "Basic messaging" means: send a single user prompt, receive any @@ -70,38 +73,35 @@ def test_basic_messaging_non_streaming_azure(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py index dca60237a28..aace83c425f 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_converse.py @@ -12,10 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by feature_id provider Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. We parametrize over the -three models and the conftest aggregator produces one cell from the -three results, naming the failing model in the error string when any -model fails. +4.7; the cell only goes green if all three pass. We fan the three model +runs out in parallel inside this single test and report one +`compat_result.add(...)` entry per model so the matrix builder still sees +three rows for this (feature, provider). """ from __future__ import annotations @@ -24,7 +24,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -40,8 +44,7 @@ BEDROCK_CONVERSE_MODELS = [ ] -@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) -def test_basic_messaging_non_streaming_bedrock_converse(compat_result, model): +def test_basic_messaging_non_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) @@ -59,38 +62,35 @@ def test_basic_messaging_non_streaming_bedrock_converse(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py index e6bd347c167..db97d6aa85d 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_bedrock_invoke.py @@ -12,10 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by feature_id provider Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. We parametrize over the -three models and the conftest aggregator produces one cell from the -three results, naming the failing model in the error string when any -model fails. +4.7; the cell only goes green if all three pass. We fan the three model +runs out in parallel inside this single test and report one +`compat_result.add(...)` entry per model so the matrix builder still sees +three rows for this (feature, provider). """ from __future__ import annotations @@ -24,7 +24,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -40,8 +44,7 @@ BEDROCK_INVOKE_MODELS = [ ] -@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) -def test_basic_messaging_non_streaming_bedrock_invoke(compat_result, model): +def test_basic_messaging_non_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) @@ -59,38 +62,35 @@ def test_basic_messaging_non_streaming_bedrock_invoke(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py b/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py index df6d79c710d..c7a7906af84 100644 --- a/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py +++ b/tests/claude_code/basic_messaging_non_streaming/test_vertex_ai.py @@ -12,10 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by feature_id provider Per the PRD, every cell exercises Claude Haiku 4.5, Sonnet 4.6, and Opus -4.7; the cell only goes green if all three pass. We parametrize over the -three models and the conftest aggregator produces one cell from the -three results, naming the failing model in the error string when any -model fails. +4.7; the cell only goes green if all three pass. We fan the three model +runs out in parallel inside this single test and report one +`compat_result.add(...)` entry per model so the matrix builder still sees +three rows for this (feature, provider). """ from __future__ import annotations @@ -24,7 +24,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -40,8 +44,7 @@ VERTEX_AI_MODELS = [ ] -@pytest.mark.parametrize("model", VERTEX_AI_MODELS) -def test_basic_messaging_non_streaming_vertex_ai(compat_result, model): +def test_basic_messaging_non_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) api_key = os.environ.get(PROXY_API_KEY_ENV) @@ -59,38 +62,35 @@ def test_basic_messaging_non_streaming_vertex_ai(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_streaming/test_anthropic.py b/tests/claude_code/basic_messaging_streaming/test_anthropic.py index ec52a24b5ae..e49bf6f0151 100644 --- a/tests/claude_code/basic_messaging_streaming/test_anthropic.py +++ b/tests/claude_code/basic_messaging_streaming/test_anthropic.py @@ -17,6 +17,10 @@ The (feature, provider) for this cell is inferred from the file path by tests/claude_code/basic_messaging_streaming/test_anthropic.py ^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ feature_id provider + +The three Claude tiers run in parallel inside this single test, with +one `compat_result.add(...)` entry per model so the matrix builder +still sees three rows for this (feature, provider). """ from __future__ import annotations @@ -25,7 +29,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -37,8 +45,7 @@ ANTHROPIC_MODELS = [ ] -@pytest.mark.parametrize("model", ANTHROPIC_MODELS) -def test_basic_messaging_streaming_anthropic(compat_result, model): +def test_basic_messaging_streaming_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (at least one stream-json event observed). """ @@ -58,49 +65,41 @@ def test_basic_messaging_streaming_anthropic(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Count from 1 to 5, one number per line.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="Count from 1 to 5, one number per line.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.events: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no stream-json events emitted; streaming wire silent", - } - ) - pytest.fail(f"no stream events for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if not outcome.events: + error = f"[{model}] no stream-json events emitted; streaming wire silent" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_streaming/test_azure.py b/tests/claude_code/basic_messaging_streaming/test_azure.py index 3b9d78b4912..377cf9e3f05 100644 --- a/tests/claude_code/basic_messaging_streaming/test_azure.py +++ b/tests/claude_code/basic_messaging_streaming/test_azure.py @@ -23,7 +23,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -35,8 +39,7 @@ AZURE_MODELS = [ ] -@pytest.mark.parametrize("model", AZURE_MODELS) -def test_basic_messaging_streaming_azure(compat_result, model): +def test_basic_messaging_streaming_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (at least one stream-json event observed). """ @@ -56,48 +59,41 @@ def test_basic_messaging_streaming_azure(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Count from 1 to 5, one number per line.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="Count from 1 to 5, one number per line.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.events: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no stream-json events emitted; streaming wire silent", - } - ) - pytest.fail(f"no stream events for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if not outcome.events: + error = f"[{model}] no stream-json events emitted; streaming wire silent" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py index 26008ed482b..4d95edab495 100644 --- a/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_converse.py @@ -19,7 +19,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -31,8 +35,7 @@ BEDROCK_CONVERSE_MODELS = [ ] -@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) -def test_basic_messaging_streaming_bedrock_converse(compat_result, model): +def test_basic_messaging_streaming_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (at least one stream-json event observed). """ @@ -52,48 +55,41 @@ def test_basic_messaging_streaming_bedrock_converse(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Count from 1 to 5, one number per line.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="Count from 1 to 5, one number per line.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.events: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no stream-json events emitted; streaming wire silent", - } - ) - pytest.fail(f"no stream events for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if not outcome.events: + error = f"[{model}] no stream-json events emitted; streaming wire silent" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py index 0d0d1ee9a33..68e96003828 100644 --- a/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py +++ b/tests/claude_code/basic_messaging_streaming/test_bedrock_invoke.py @@ -19,7 +19,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -31,8 +35,7 @@ BEDROCK_INVOKE_MODELS = [ ] -@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) -def test_basic_messaging_streaming_bedrock_invoke(compat_result, model): +def test_basic_messaging_streaming_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (at least one stream-json event observed). """ @@ -52,48 +55,41 @@ def test_basic_messaging_streaming_bedrock_invoke(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Count from 1 to 5, one number per line.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="Count from 1 to 5, one number per line.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.events: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no stream-json events emitted; streaming wire silent", - } - ) - pytest.fail(f"no stream events for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if not outcome.events: + error = f"[{model}] no stream-json events emitted; streaming wire silent" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py index d909547673b..0da7cf4403c 100644 --- a/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py +++ b/tests/claude_code/basic_messaging_streaming/test_vertex_ai.py @@ -19,7 +19,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -31,8 +35,7 @@ VERTEX_AI_MODELS = [ ] -@pytest.mark.parametrize("model", VERTEX_AI_MODELS) -def test_basic_messaging_streaming_vertex_ai(compat_result, model): +def test_basic_messaging_streaming_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a non-empty streamed reply (at least one stream-json event observed). """ @@ -52,48 +55,41 @@ def test_basic_messaging_streaming_vertex_ai(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Count from 1 to 5, one number per line.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="Count from 1 to 5, one number per line.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.events: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no stream-json events emitted; streaming wire silent", - } - ) - pytest.fail(f"no stream events for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if not outcome.events: + error = f"[{model}] no stream-json events emitted; streaming wire silent" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/cli_driver.py b/tests/claude_code/cli_driver.py index d4ec7667bbb..823ae73f86d 100644 --- a/tests/claude_code/cli_driver.py +++ b/tests/claude_code/cli_driver.py @@ -16,11 +16,27 @@ from __future__ import annotations import json import os import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass, field -from typing import Any, Dict, List, Mapping, Optional, Sequence +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union + +from tests.claude_code.rate_limiter import ( + RateLimiter, + get_default_limiter, + infer_provider, +) CLAUDE_CLI_DEFAULT = "claude" -DEFAULT_TIMEOUT_SECONDS = 120 +# 120s is fine for a single isolated CLI call against an unloaded +# upstream, but the matrix run launches up to 75 concurrent calls and +# upstreams can take several minutes to respond under that contention. +# We expose the timeout as an env var so binary-search runs can grow +# it without touching the test code. +DEFAULT_TIMEOUT_SECONDS = float( + os.environ.get("LITELLM_COMPAT_CLI_TIMEOUT_SECONDS") or 120 +) class ClaudeCLIError(RuntimeError): @@ -57,6 +73,7 @@ def run_claude( cli_path: str = CLAUDE_CLI_DEFAULT, timeout: float = DEFAULT_TIMEOUT_SECONDS, runner: Optional[Any] = None, + rate_limiter: Optional[RateLimiter] = None, ) -> DriverResult: """Invoke `claude` once in headless stream-JSON mode and return the result. @@ -68,6 +85,12 @@ def run_claude( `runner` is an injection seam used by the unit tests: by default we call `subprocess.run`, but the test suite swaps in a fake that yields canned stream-JSON. Production callers should never set it. + + `rate_limiter` is the second injection seam: the cross-process + token-bucket limiter throttles outbound calls per provider so a + fully-parallel matrix run doesn't trip 429s. Defaults to the + process-wide singleton; unit tests pass a no-op limiter or one + backed by a tmp dir to keep tests hermetic. """ if not prompt: raise ValueError("prompt must be a non-empty string") @@ -100,6 +123,15 @@ def run_claude( env["ANTHROPIC_BASE_URL"] = base_url env["ANTHROPIC_AUTH_TOKEN"] = api_key + # Throttle by provider *before* launching the CLI. Doing this here + # (rather than per-test) means every code path that lands on + # `run_claude` is rate-limited automatically — including + # `run_claude_models_parallel`, which is the hot path during the + # full matrix run. + limiter = rate_limiter if rate_limiter is not None else get_default_limiter() + provider = infer_provider(model) + limiter.acquire(provider) + run_fn = runner or subprocess.run try: completed = run_fn( @@ -130,6 +162,134 @@ def run_claude( ) +ModelResult = Union[DriverResult, ClaudeCLIError] + + +def run_claude_models_parallel( + *, + models: Sequence[str], + prompt: str, + base_url: str, + api_key: str, + extra_env: Optional[Mapping[str, str]] = None, + extra_args: Optional[Sequence[str]] = None, + cli_path: str = CLAUDE_CLI_DEFAULT, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + runner: Optional[Callable[..., Any]] = None, +) -> Dict[str, ModelResult]: + """Invoke `run_claude` for every `models[i]` concurrently and collect outcomes. + + Each `claude` CLI invocation is a long-lived subprocess that spends + almost all of its time waiting on the upstream API; running the + three Claude tiers in parallel cuts the per-cell wall time roughly + threefold without changing what each invocation does. + + Threads (rather than asyncio) are the right primitive here because + `subprocess.run` releases the GIL while it waits, and we want to + keep the synchronous CLI driver unchanged so unit tests can keep + injecting a fake `runner`. + + Returns a dict keyed by model id. Each value is either the + `DriverResult` produced by `run_claude` or the `ClaudeCLIError` + that aborted that model's run — callers decide how to map either + into a `compat_result` entry. The shared kwargs (prompt, env, args, + timeout, runner) are forwarded verbatim so the per-model wire is + identical to what the sequential path produces. + """ + if not models: + raise ValueError("models must be a non-empty sequence") + + def _one(model: str) -> Tuple[str, ModelResult, float]: + # Per-model wall clock: this is what the matrix run actually pays for. + # We record it whether the run succeeded or raised so the breakdown + # log below covers both code paths and surfaces "which model is the + # long pole?" without requiring per-test instrumentation. + started = time.monotonic() + try: + result = run_claude( + prompt=prompt, + model=model, + base_url=base_url, + api_key=api_key, + extra_env=extra_env, + extra_args=extra_args, + cli_path=cli_path, + timeout=timeout, + runner=runner, + ) + elapsed = time.monotonic() - started + # Stamp the duration onto the DriverResult so callers (tests, + # diagnostics) can attribute slow cells without re-timing. + result.duration_ms = int(elapsed * 1000) + return model, result, elapsed + except ClaudeCLIError as exc: + elapsed = time.monotonic() - started + return model, exc, elapsed + + outcomes: Dict[str, ModelResult] = {} + durations: Dict[str, float] = {} + overall_started = time.monotonic() + with ThreadPoolExecutor(max_workers=len(models)) as pool: + futures = [pool.submit(_one, model) for model in models] + for future in as_completed(futures): + model, outcome, elapsed = future.result() + outcomes[model] = outcome + durations[model] = elapsed + overall_elapsed = time.monotonic() - overall_started + + _log_parallel_breakdown(models, durations, outcomes, overall_elapsed) + return outcomes + + +def _log_parallel_breakdown( + models: Sequence[str], + durations: Mapping[str, float], + outcomes: Mapping[str, ModelResult], + overall_elapsed: float, +) -> None: + """Emit a one-block timing breakdown to stderr. + + Pytest only shows captured output for failing tests by default, but + `-s` surfaces it for passing tests too — which is exactly when you + care about "did parallelization actually help?". The block reports: + + - per-model wall time and outcome (ok / cli-error / non-zero exit) + - the slowest model (the parallel run's wall-time floor) + - the sum of sequential model times (what the old serial path + would have paid) + - the overall parallel wall time and the speedup ratio + + If one model dominates, `slowest ≈ overall ≈ sequential / 1`, and + the speedup will be near 1× — exactly the diagnostic that explains + "why didn't this get faster?". + """ + sequential_total = sum(durations.values()) + slowest_model = max(durations, key=durations.get) if durations else None + slowest = durations[slowest_model] if slowest_model else 0.0 + speedup = sequential_total / overall_elapsed if overall_elapsed > 0 else 0.0 + + lines: List[str] = [] + lines.append("[parallel] per-model wall time:") + for model in models: + elapsed = durations.get(model, 0.0) + outcome = outcomes.get(model) + if isinstance(outcome, ClaudeCLIError): + status = "cli-error" + elif isinstance(outcome, DriverResult): + status = f"exit={outcome.exit_code}" + else: + status = "missing" + lines.append(f" {model:<40s} {elapsed:6.2f}s ({status})") + if slowest_model is not None: + lines.append( + f"[parallel] slowest={slowest_model} ({slowest:.2f}s); " + f"sequential_sum={sequential_total:.2f}s; " + f"parallel_wall={overall_elapsed:.2f}s; " + f"speedup={speedup:.2f}x" + ) + print("\n".join(lines), file=sys.stderr, flush=True) + + def _parse_stream_json(stdout: str) -> List[Dict[str, Any]]: """Parse newline-delimited JSON emitted by `claude --output-format stream-json`. diff --git a/tests/claude_code/conftest.py b/tests/claude_code/conftest.py index 9baef4e2a30..fe827c2bb22 100644 --- a/tests/claude_code/conftest.py +++ b/tests/claude_code/conftest.py @@ -1,16 +1,26 @@ """Pytest plumbing for the Claude Code compatibility matrix. -Two responsibilities live here: +Three responsibilities live here: 1. The `compat_result` fixture — the only API a test author needs to learn. Tests call `compat_result.set({"status": "pass"})` (or fail / not_applicable) - to report their outcome as a tagged union. The fixture is per-test and - stores the last value reported. + to report their outcome as a tagged union. Multi-model tests call + `.add(...)` once per Claude tier so each tier lands as its own row in + the results artifact. 2. The `pytest_runtest_makereport` hook — captures each test's reported result, - infers (feature, provider) from the file path, and writes a single - `compat-results.json` artifact next to JUnit XML. The Matrix JSON Builder - consumes this artifact to produce the published `compatibility-matrix.json`. + infers (feature, provider) from the file path, and accumulates rows into + a per-process collector. At session end we serialize them to + `compat-results.json` (or a per-worker file under xdist) so the Matrix + JSON Builder can consume them. + +3. xdist coordination — when `pytest -n auto` is used, every worker writes + its own results shard and the controller merges them into the canonical + `compat-results.json` in `pytest_sessionfinish`. Without this, the + workers race on the same path and the artifact only reflects whichever + worker finished last. The same merge step also emits a rate-limit + summary that the binary-search helper consumes to decide whether the + current X/Y/Z values were too aggressive. The (feature, provider) inference comes from the test file path: the parent directory name is the feature_id (matching `manifest.yaml`), and the file @@ -22,43 +32,105 @@ from __future__ import annotations import json import os +import re +import sys +from collections import Counter, defaultdict from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import pytest VALID_STATUSES = {"pass", "fail", "not_applicable", "not_tested"} RESULTS_ARTIFACT_ENV = "COMPAT_RESULTS_PATH" DEFAULT_ARTIFACT_PATH = "compat-results.json" +RATE_LIMIT_SUMMARY_ENV = "COMPAT_RATE_LIMIT_SUMMARY_PATH" +DEFAULT_RATE_LIMIT_SUMMARY_PATH = "compat-rate-limit-summary.json" + +# Heuristic: detect 429s and rate-limit-shaped errors anywhere in the +# error string. The CLI buries upstream errors in `assistant.message.content` +# text on stdout (see `failure_diagnostic`), so we don't get a structured +# status code in every code path — a regex over the joined error text is +# the most reliable signal we have. +# +# We also treat a CLI timeout (`claude CLI timed out after Ns`) as a +# rate-limit-shaped failure for binary-search purposes: in practice the +# only reason every model in a cell stalls past the timeout is the +# upstream collapsing under concurrency, which is exactly the situation +# the rate limiter is supposed to back off from. False positives on a +# genuinely slow upstream are tolerable here because the worst case is +# the binary search runs at a slightly lower rate than necessary. +_RATE_LIMIT_RE = re.compile( + r"(?:\b429\b|rate[\s_-]?limit|too\s+many\s+requests|throttl(?:ed|ing)|" + r"claude\s+CLI\s+timed\s+out)", + re.IGNORECASE, +) @dataclass class CompatResult: """Per-test recorder for compatibility outcomes. - Tests interact only via `.set(...)`. `.value` is read by the + Tests interact via `.set(...)` (single result) or `.add(...)` (one + result per Claude tier when the test fans the three models out in + parallel). `.value` and `.values` are read by the `pytest_runtest_makereport` hook after the test body finishes. + + Multi-result usage exists because every cell in the compat matrix is + backed by three model invocations (Haiku/Sonnet/Opus) per (feature, + provider). When a test runs them concurrently in a single pytest + node, each model needs its own entry in the results artifact so the + matrix builder's per-cell aggregator can apply its "all three must + pass" rule. """ value: Optional[Dict[str, Any]] = None + values: List[Dict[str, Any]] = field(default_factory=list) def set(self, result: Dict[str, Any]) -> None: + validated = self._validate(result) + self.value = validated + + def add(self, result: Dict[str, Any]) -> None: + """Append one model's outcome to the per-test results list. + + Use this when a single test exercises multiple Claude tiers + concurrently and needs to report one outcome per tier. The + conftest hook will emit one entry per appended result. + """ + validated = self._validate(result) + self.values.append(validated) + + @staticmethod + def _validate(result: Dict[str, Any]) -> Dict[str, Any]: if not isinstance(result, dict): - raise TypeError("compat_result.set() requires a dict") + raise TypeError("compat_result requires a dict") status = result.get("status") if status not in VALID_STATUSES: raise ValueError( - f"compat_result.set() status must be one of {sorted(VALID_STATUSES)}, " + f"compat_result status must be one of {sorted(VALID_STATUSES)}, " f"got {status!r}" ) if status == "fail" and not result.get("error"): - raise ValueError("compat_result.set({'status': 'fail'}) requires 'error'") + raise ValueError("compat_result {'status': 'fail'} requires 'error'") if status == "not_applicable" and not result.get("reason"): raise ValueError( - "compat_result.set({'status': 'not_applicable'}) requires 'reason'" + "compat_result {'status': 'not_applicable'} requires 'reason'" ) - self.value = dict(result) + return dict(result) + + def collected(self) -> List[Dict[str, Any]]: + """Return every result reported during the test, preserving order. + + Multi-model tests use `.add(...)` per model; legacy tests use + `.set(...)` once. We surface both shapes in a single list so + the makereport hook only has to think about a list of results. + """ + if self.values: + return list(self.values) + if self.value is not None: + return [dict(self.value)] + return [] @dataclass @@ -109,7 +181,14 @@ def _infer_feature_and_provider(node_path: Path) -> Optional[tuple]: @pytest.hookimpl(hookwrapper=True) def pytest_runtest_makereport(item, call): - """Capture compat_result.value at end-of-test and remember it for the artifact.""" + """Capture compat_result reports at end-of-test and remember them for the artifact. + + A single test may report multiple results (one per Claude tier when + the three are run in parallel inside one node). We emit one + `_CollectedResult` per reported entry so the matrix builder's + per-cell aggregator sees the same shape it would have seen if the + test were parametrized — every model lands in the artifact. + """ outcome = yield report = outcome.get_result() if report.when != "call": @@ -121,30 +200,200 @@ def pytest_runtest_makereport(item, call): feature_id, provider = inferred fixture = item.funcargs.get("compat_result") if hasattr(item, "funcargs") else None - reported: Optional[Dict[str, Any]] = getattr(fixture, "value", None) - - if reported is None: - if report.passed: - reported = { - "status": "fail", - "error": "test passed without calling compat_result.set(); " - "every compat test must report a status.", - } - else: - reported = { - "status": "fail", - "error": (str(report.longrepr) if report.longrepr else "test failed"), - } - - _COLLECTOR.items.append( - _CollectedResult( - feature_id=feature_id, - provider=provider, - nodeid=report.nodeid, - result=reported, - ) + collected: List[Dict[str, Any]] = ( + fixture.collected() if isinstance(fixture, CompatResult) else [] ) + if not collected: + if report.passed: + collected = [ + { + "status": "fail", + "error": "test passed without reporting via compat_result; " + "every compat test must report a status.", + } + ] + else: + collected = [ + { + "status": "fail", + "error": ( + str(report.longrepr) if report.longrepr else "test failed" + ), + } + ] + + for reported in collected: + _COLLECTOR.items.append( + _CollectedResult( + feature_id=feature_id, + provider=provider, + nodeid=report.nodeid, + result=reported, + ) + ) + + +def _is_xdist_worker(session) -> bool: + """Return True iff the current pytest session is an xdist worker. + + The standard idiom is to look up `workerinput` on the config; the + controller process doesn't have it, the workers do. We deliberately + don't `import xdist` because the suite must keep running when xdist + isn't installed at all. + """ + return hasattr(session.config, "workerinput") + + +def _xdist_worker_id(session) -> Optional[str]: + info = getattr(session.config, "workerinput", None) + if not info: + return None + return info.get("workerid") + + +def _shard_dir(artifact_path: Path) -> Path: + """Workers write their shards next to the canonical results path. + + Putting shards in a sibling directory (rather than inline JSON + files in the same dir) keeps the controller's merge step simple + — it just lists `*.json` in `.shards/` — and avoids + accidental shard/canonical filename collisions. + """ + return artifact_path.with_name(artifact_path.name + ".shards") + + +def _serialize_items(items: List["_CollectedResult"]) -> List[Dict[str, Any]]: + return [ + { + "feature_id": item.feature_id, + "provider": item.provider, + "nodeid": item.nodeid, + "result": item.result, + } + for item in items + ] + + +def _build_rate_limit_summary( + rows: List[Dict[str, Any]], +) -> Dict[str, Any]: + """Aggregate per-provider rate-limit signals from the result rows. + + We classify any failure whose error string matches `_RATE_LIMIT_RE` + as a "rate-limited" failure. The binary-search helper reads this + summary to decide whether the current X/Y/Z values were too + aggressive: if any provider has `rate_limited > 0`, the harness + should back off that provider's rate and retry. + + Returns a dict shaped: + + { + "totals": {"pass": N, "fail": N, "rate_limited": N, ...}, + "per_provider": { + "anthropic": {"pass": ..., "fail": ..., "rate_limited": ...}, + ... + }, + "rate_limited_examples": [ + {"feature_id": ..., "provider": ..., "error": "..."}, ... + ], + } + """ + totals: Counter = Counter() + per_provider: Dict[str, Counter] = defaultdict(Counter) + rate_limited_examples: List[Dict[str, Any]] = [] + + for row in rows: + result = row.get("result") or {} + status = result.get("status") or "unknown" + provider = row.get("provider") or "unknown" + totals[status] += 1 + per_provider[provider][status] += 1 + + if status == "fail": + error = str(result.get("error") or "") + if _RATE_LIMIT_RE.search(error): + totals["rate_limited"] += 1 + per_provider[provider]["rate_limited"] += 1 + # Cap examples so a stuck-throttled run doesn't write + # a multi-MB summary file the helper has to slurp. + if len(rate_limited_examples) < 25: + rate_limited_examples.append( + { + "feature_id": row.get("feature_id"), + "provider": provider, + "error": error[:500], + } + ) + + return { + "totals": dict(totals), + "per_provider": {p: dict(c) for p, c in per_provider.items()}, + "rate_limited_examples": rate_limited_examples, + } + + +def _print_rate_limit_summary(summary: Dict[str, Any]) -> None: + """Emit a human-readable per-provider table to stderr. + + Pytest only captures stderr when `-s` isn't set; we deliberately + write here anyway because the binary-search workflow runs pytest + with `-q` and grep-checks the structured JSON artifact, while a + human running locally with `-s` sees the same numbers inline. + """ + totals = summary.get("totals", {}) + per_provider = summary.get("per_provider", {}) + lines: List[str] = [] + lines.append("[compat] session totals:") + for status in ("pass", "fail", "rate_limited", "not_applicable", "not_tested"): + if status in totals: + lines.append(f" {status:<16s} {totals[status]}") + if per_provider: + lines.append("[compat] per-provider breakdown:") + for provider in sorted(per_provider): + counts = per_provider[provider] + parts = " ".join( + f"{k}={v}" + for k, v in sorted(counts.items()) + if k != "not_tested" or v > 0 + ) + lines.append(f" {provider:<20s} {parts}") + if totals.get("rate_limited", 0): + lines.append( + "[compat] WARNING: at least one cell hit a rate-limit-shaped error; " + "lower the corresponding LITELLM_COMPAT_RATE_ and retry" + ) + print("\n".join(lines), file=sys.stderr, flush=True) + + +def pytest_sessionstart(session): + """Clear stale per-worker shards from any prior session. + + Without this, a previous run's shard directory leaks into the next + `pytest_sessionfinish` merge — yielding a `compat-results.json` + that includes results from runs that aren't part of the current + session, and a misleading rate-limit summary that re-flags + failures the user already saw and addressed. + + Only the controller (non-xdist-worker) clears; workers must not + race the controller while it's wiping the directory. + """ + if _is_xdist_worker(session): + return + artifact_path = Path(os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH) + shard_dir = _shard_dir(artifact_path) + if not shard_dir.exists(): + return + for stale in shard_dir.glob("*.json"): + try: + stale.unlink() + except OSError: + # If we can't remove a stale shard (permissions, race with + # an unrelated process), keep going — the merge step is + # robust to malformed shards, and a stale row landing in + # the artifact is recoverable; aborting the session isn't. + continue + def pytest_sessionstart(session): """Clear collected results at the start of each session. @@ -158,27 +407,74 @@ def pytest_sessionstart(session): def pytest_sessionfinish(session, exitstatus): - """Write the structured results artifact at end of session. + """Write the per-process results shard, then merge if we're the controller. - Skip writing when no compat results were collected — this conftest is - loaded by pytest for every test under `tests/claude_code/`, including - sibling unit-test trees (e.g. `_driver_unit_tests/`). Writing an empty - artifact in those runs would silently overwrite a real artifact from a - prior compat-test run on the same checkout. + Worker processes (xdist `gw0`, `gw1`, ...) only write their shard + under `.shards/.json`. The controller writes + its own shard if it ran any tests itself, then walks the shards + directory and produces the canonical `compat-results.json` plus + the rate-limit summary. Single-process runs (no xdist) take the + same code path with a single shard, so behavior is consistent. + + Skip when no compat results were collected — this conftest is + loaded for every test under `tests/claude_code/`, including sibling + unit-test trees (e.g. `_driver_unit_tests/`). Writing an empty + artifact would silently overwrite a real artifact from a prior + compat-test run on the same checkout. """ - if not _COLLECTOR.items: + if not _COLLECTOR.items and not _is_xdist_worker(session): return - artifact_path = os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH - payload = { - "schema_version": "1", - "results": [ + artifact_path = Path(os.environ.get(RESULTS_ARTIFACT_ENV) or DEFAULT_ARTIFACT_PATH) + shard_dir = _shard_dir(artifact_path) + shard_dir.mkdir(parents=True, exist_ok=True) + + worker_id = _xdist_worker_id(session) or "main" + shard_path = shard_dir / f"{worker_id}.json" + shard_path.write_text( + json.dumps( { - "feature_id": item.feature_id, - "provider": item.provider, - "nodeid": item.nodeid, - "result": item.result, - } - for item in _COLLECTOR.items - ], - } - Path(artifact_path).write_text(json.dumps(payload, indent=2, sort_keys=True)) + "schema_version": "1", + "worker_id": worker_id, + "results": _serialize_items(_COLLECTOR.items), + }, + indent=2, + sort_keys=True, + ) + ) + + # Workers stop here. The controller merges; if we're not running + # under xdist, we are effectively the controller. + if _is_xdist_worker(session): + return + + merged_rows: List[Dict[str, Any]] = [] + for shard_file in sorted(shard_dir.glob("*.json")): + try: + shard = json.loads(shard_file.read_text()) + except (OSError, ValueError): + continue + rows = shard.get("results") + if isinstance(rows, list): + merged_rows.extend(rows) + + # Skip writing artifact + summary entirely for unit-test-only runs + # (no per-feature compat rows). Otherwise every `pytest tests/...` + # run — including local unit-test invocations — would silently + # overwrite a real artifact from a prior compat-test run. + if not merged_rows: + return + + artifact_path.write_text( + json.dumps( + {"schema_version": "1", "results": merged_rows}, + indent=2, + sort_keys=True, + ) + ) + + summary = _build_rate_limit_summary(merged_rows) + summary_path = Path( + os.environ.get(RATE_LIMIT_SUMMARY_ENV) or DEFAULT_RATE_LIMIT_SUMMARY_PATH + ) + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True)) + _print_rate_limit_summary(summary) diff --git a/tests/claude_code/extended_thinking/test_anthropic.py b/tests/claude_code/extended_thinking/test_anthropic.py index b9d67f887a0..880083c52da 100644 --- a/tests/claude_code/extended_thinking/test_anthropic.py +++ b/tests/claude_code/extended_thinking/test_anthropic.py @@ -12,6 +12,10 @@ The (feature, provider) for this cell is inferred from the file path by tests/claude_code/extended_thinking/test_anthropic.py ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ feature_id provider + +The three Claude tiers run in parallel inside this single test, with +one `compat_result.add(...)` entry per model so the matrix builder +still sees three rows for this (feature, provider). """ from __future__ import annotations @@ -21,7 +25,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -58,8 +66,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", ANTHROPIC_MODELS) -def test_extended_thinking_anthropic(compat_result, model): +def test_extended_thinking_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -78,39 +85,38 @@ def test_extended_thinking_anthropic(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=THINKING_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_env=THINKING_ENV, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_thinking_block(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no `thinking` content block observed in stream-json events", - } - ) - pytest.fail(f"no thinking block for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/extended_thinking/test_azure.py b/tests/claude_code/extended_thinking/test_azure.py index a03fdcf4b76..fb777cca8c9 100644 --- a/tests/claude_code/extended_thinking/test_azure.py +++ b/tests/claude_code/extended_thinking/test_azure.py @@ -28,7 +28,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -60,8 +64,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", AZURE_MODELS) -def test_extended_thinking_azure(compat_result, model): +def test_extended_thinking_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -80,39 +83,38 @@ def test_extended_thinking_azure(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=THINKING_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_env=THINKING_ENV, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_thinking_block(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no `thinking` content block observed in stream-json events", - } - ) - pytest.fail(f"no thinking block for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/extended_thinking/test_bedrock_converse.py b/tests/claude_code/extended_thinking/test_bedrock_converse.py index de33f343a8b..472a80e4aa6 100644 --- a/tests/claude_code/extended_thinking/test_bedrock_converse.py +++ b/tests/claude_code/extended_thinking/test_bedrock_converse.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -52,8 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) -def test_extended_thinking_bedrock_converse(compat_result, model): +def test_extended_thinking_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -72,39 +75,38 @@ def test_extended_thinking_bedrock_converse(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=THINKING_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_env=THINKING_ENV, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_thinking_block(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no `thinking` content block observed in stream-json events", - } - ) - pytest.fail(f"no thinking block for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/extended_thinking/test_bedrock_invoke.py b/tests/claude_code/extended_thinking/test_bedrock_invoke.py index 27b502207f4..f42031ba44c 100644 --- a/tests/claude_code/extended_thinking/test_bedrock_invoke.py +++ b/tests/claude_code/extended_thinking/test_bedrock_invoke.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -52,8 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) -def test_extended_thinking_bedrock_invoke(compat_result, model): +def test_extended_thinking_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -72,39 +75,38 @@ def test_extended_thinking_bedrock_invoke(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=THINKING_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_env=THINKING_ENV, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_thinking_block(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no `thinking` content block observed in stream-json events", - } - ) - pytest.fail(f"no thinking block for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/extended_thinking/test_vertex_ai.py b/tests/claude_code/extended_thinking/test_vertex_ai.py index 55dae72ccde..1f3a15ec16c 100644 --- a/tests/claude_code/extended_thinking/test_vertex_ai.py +++ b/tests/claude_code/extended_thinking/test_vertex_ai.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -52,8 +56,7 @@ def _has_thinking_block(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", VERTEX_AI_MODELS) -def test_extended_thinking_vertex_ai(compat_result, model): +def test_extended_thinking_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy with thinking enabled and assert a `thinking` content block was emitted.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -72,39 +75,38 @@ def test_extended_thinking_vertex_ai(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=THINKING_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_env=THINKING_ENV, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=THINKING_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_thinking_block(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no `thinking` content block observed in stream-json events", - } - ) - pytest.fail(f"no thinking block for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_thinking_block(outcome.events): + error = ( + f"[{model}] no `thinking` content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/manifest.yaml b/tests/claude_code/manifest.yaml index fb70ce4623f..ccafe10a904 100644 --- a/tests/claude_code/manifest.yaml +++ b/tests/claude_code/manifest.yaml @@ -34,3 +34,13 @@ features: name: Vision - id: extended_thinking name: Extended thinking + - id: tool_use_streaming + name: Tool use (streaming / fine-grained) + - id: thinking_with_tool_use + name: Extended thinking + tool use + - id: pdf_input + name: PDF document input + - id: prompt_caching_1h + name: Prompt caching (1h TTL) + - id: web_search + name: Web search (server tool) diff --git a/tests/claude_code/pdf_input/__init__.py b/tests/claude_code/pdf_input/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/pdf_input/test_anthropic.py b/tests/claude_code/pdf_input/test_anthropic.py new file mode 100644 index 00000000000..647cc6a4dc7 --- /dev/null +++ b/tests/claude_code/pdf_input/test_anthropic.py @@ -0,0 +1,176 @@ +"""pdf_input x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, write a tiny valid PDF to disk, allow the built-in `Read` +tool, and ask Claude to read the PDF and report what it contains. + +The Read tool inlines the PDF bytes as `document` content blocks on the +next assistant turn, which is exactly the gateway path we want to +exercise: it's distinct from image content blocks (which are tested in +`vision/`) and uses a different transformation in LiteLLM's Anthropic +provider. We assert the upstream produces a non-empty reply that +references the contents of the PDF — proving the proxy preserved the +document content block end-to-end. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/pdf_input/test_anthropic.py + ^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Smallest valid PDF that renders a single visible word ("PONG"). Built +# inline rather than checked in as a binary fixture so the test stays +# self-contained and the marker word is easy to grep for in CI logs. +# The structure is a hand-crafted single-page PDF with one Helvetica +# text show; offsets are computed at write time so the xref table +# stays consistent regardless of platform line endings. +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`. + + We construct the PDF imperatively because `pypdf`/`reportlab` are + not in the test deps and we want the cell to work in a clean + environment. The xref offsets are recomputed for each `marker` + length so the file stays well-formed. + """ + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + # Page content stream: position the text and show the marker. + # `BT ... ET` is a text object; `Tf` selects font, `Td` moves + # the cursor, `Tj` paints a string. + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + # Fix up the /Length on the content stream to match its body. + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_anthropic(compat_result, tmp_path): + """Drive the `claude` CLI against the LiteLLM proxy with a PDF + attached via the Read tool and assert the reply references it.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + # The strongest gateway-level signal we can assert without + # parsing every event type: the model's final user-visible + # reply names the marker word that only the PDF carries. + # If the proxy dropped the `document` content block, the + # model has no way to produce this token. + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/pdf_input/test_azure.py b/tests/claude_code/pdf_input/test_azure.py new file mode 100644 index 00000000000..3fcca4d6c60 --- /dev/null +++ b/tests/claude_code/pdf_input/test_azure.py @@ -0,0 +1,146 @@ +"""pdf_input x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +write a tiny valid PDF to disk, allow the built-in `Read` tool, and +ask Claude to read the PDF and report what it contains. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/pdf_input/test_azure.py + ^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_azure(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/pdf_input/test_bedrock_converse.py b/tests/claude_code/pdf_input/test_bedrock_converse.py new file mode 100644 index 00000000000..a94588c3b6b --- /dev/null +++ b/tests/claude_code/pdf_input/test_bedrock_converse.py @@ -0,0 +1,152 @@ +"""pdf_input x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, write a tiny +valid PDF to disk, allow the built-in `Read` tool, and ask Claude to +read the PDF and report what it contains. + +Bedrock Converse expresses documents via its own +`document = { format, name, source: { bytes } }` shape; this cell +catches gateway regressions where the proxy fails to translate +Anthropic's `document` content block to Converse's document format +(or vice versa on the response). + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/pdf_input/test_bedrock_converse.py + ^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_bedrock_converse(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/pdf_input/test_bedrock_invoke.py b/tests/claude_code/pdf_input/test_bedrock_invoke.py new file mode 100644 index 00000000000..487e851e169 --- /dev/null +++ b/tests/claude_code/pdf_input/test_bedrock_invoke.py @@ -0,0 +1,151 @@ +"""pdf_input x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +write a tiny valid PDF to disk, allow the built-in `Read` tool, and +ask Claude to read the PDF and report what it contains. + +Bedrock InvokeModel for Anthropic models accepts the native +`document` content block shape; this cell catches gateway regressions +where the proxy drops or mis-encodes the document content block on +the way through (e.g. base64-only encoding, missing media_type, etc.). + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/pdf_input/test_bedrock_invoke.py + ^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_bedrock_invoke(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/pdf_input/test_vertex_ai.py b/tests/claude_code/pdf_input/test_vertex_ai.py new file mode 100644 index 00000000000..0378038df5a --- /dev/null +++ b/tests/claude_code/pdf_input/test_vertex_ai.py @@ -0,0 +1,146 @@ +"""pdf_input x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, write a tiny valid PDF to disk, allow +the built-in `Read` tool, and ask Claude to read the PDF and report +what it contains. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/pdf_input/test_vertex_ai.py + ^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +PDF_MARKER = "PONG" + + +def _build_minimal_pdf(marker: str) -> bytes: + """Return a single-page PDF whose only visible text is `marker`.""" + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + ( + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>" + ), + ( + b"<< /Length %d >>\nstream\nBT /F1 24 Tf 50 100 Td (" + + marker.encode("ascii") + + b") Tj ET\nendstream" + ), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + body_open = objects[3].index(b"\nstream\n") + len(b"\nstream\n") + body_close = objects[3].index(b"\nendstream") + body_len = body_close - body_open + objects[3] = ( + b"<< /Length " + + str(body_len).encode("ascii") + + b" >>\nstream\n" + + objects[3][body_open:body_close] + + b"\nendstream" + ) + + out = bytearray(b"%PDF-1.4\n") + offsets = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode("ascii") + obj + b"\nendobj\n" + + xref_offset = len(out) + out += b"xref\n0 %d\n" % (len(objects) + 1) + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode("ascii") + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode("ascii") + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_offset).encode("ascii") + + b"\n%%EOF\n" + ) + return bytes(out) + + +def test_pdf_input_vertex_ai(compat_result, tmp_path): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + pdf_path = tmp_path / "marker.pdf" + pdf_path.write_bytes(_build_minimal_pdf(PDF_MARKER)) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=( + f"Use the Read tool to read the file at {pdf_path}. " + "Report the single word that appears in the document." + ), + base_url=base_url, + api_key=api_key, + extra_args=["--allowed-tools", "Read"], + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if PDF_MARKER not in outcome.text.upper(): + error = ( + f"[{model}] reply did not reference the PDF marker {PDF_MARKER!r}; " + f"got: {outcome.text.strip()!r}" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_1h/__init__.py b/tests/claude_code/prompt_caching_1h/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/prompt_caching_1h/test_anthropic.py b/tests/claude_code/prompt_caching_1h/test_anthropic.py new file mode 100644 index 00000000000..616373aea44 --- /dev/null +++ b/tests/claude_code/prompt_caching_1h/test_anthropic.py @@ -0,0 +1,124 @@ +"""prompt_caching_1h x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, +and assert that the upstream's usage block reports either +`cache_creation_input_tokens` or `cache_read_input_tokens` > 0 — i.e. +the proxy preserved Claude Code's `cache_control: { ttl: "1h" }` +annotations end-to-end and the upstream actually honored them. + +This is the 1-hour-TTL companion to `prompt_caching_5m/`. It exists as +its own cell because the 1h TTL travels through the proxy with a +distinct `cache_control` shape (and a distinct beta-header gate on +some providers); a regression that strips or downgrades the TTL on the +way through is invisible to the 5m cell, which would still see cache +hits with a default-TTL annotation. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_1h/test_anthropic.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Per the changelog (2.1.108): `ENABLE_PROMPT_CACHING_1H` flips Claude +# Code from the default 5-minute cache TTL to a 1-hour TTL on the +# `cache_control` annotations it adds to the system prompt and the +# most recent user turn. Setting it here is what we are validating +# the proxy faithfully forwards to the upstream. +CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + """Return cache_creation_input_tokens + cache_read_input_tokens from + the upstream usage block, or 0 if the keys are missing.""" + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with the 1h + TTL opt-in env var set, and assert the upstream usage block + surfaces a non-zero cache token count.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "ENABLE_PROMPT_CACHING_1H=1; the proxy likely stripped the " + "1h TTL beta header or rejected the cache_control shape" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_1h/test_azure.py b/tests/claude_code/prompt_caching_1h/test_azure.py new file mode 100644 index 00000000000..667c72099db --- /dev/null +++ b/tests/claude_code/prompt_caching_1h/test_azure.py @@ -0,0 +1,104 @@ +"""prompt_caching_1h x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and +assert the upstream's usage block reports a non-zero cache token count. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_1h/test_azure.py + ^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "ENABLE_PROMPT_CACHING_1H=1" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_1h/test_bedrock_converse.py b/tests/claude_code/prompt_caching_1h/test_bedrock_converse.py new file mode 100644 index 00000000000..6b2cab1ec8d --- /dev/null +++ b/tests/claude_code/prompt_caching_1h/test_bedrock_converse.py @@ -0,0 +1,112 @@ +"""prompt_caching_1h x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, opt into the +1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and assert the +upstream's usage block reports a non-zero cache token count. + +Bedrock Converse expresses prompt caching via `cachePoint` markers in +the message list, with TTL controlled out-of-band; this cell catches +proxy regressions where the 1h opt-in fails to translate into the +correct Converse cache configuration. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_1h/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +CACHE_1H_ENV = { + "ENABLE_PROMPT_CACHING_1H": "1", + "ENABLE_PROMPT_CACHING_1H_BEDROCK": "1", +} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "1h-TTL opt-in env vars set" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py b/tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py new file mode 100644 index 00000000000..667db6598bf --- /dev/null +++ b/tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py @@ -0,0 +1,117 @@ +"""prompt_caching_1h x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +opt into the 1-hour cache TTL via `ENABLE_PROMPT_CACHING_1H`, and +assert the upstream's usage block reports a non-zero cache token count. + +Bedrock historically gated 1h prompt caching behind a separate +`ENABLE_PROMPT_CACHING_1H_BEDROCK` env var (see 2.1.108: deprecated but +still honored). The proxy must accept either env var and forward an +appropriate `cache_control` shape to the Bedrock InvokeModel endpoint; +this cell catches regressions where the TTL is silently downgraded to +5 minutes on the way through. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_1h/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +# Set both the modern and the deprecated-but-honored Bedrock var so we +# match whichever code path the proxy is following. +CACHE_1H_ENV = { + "ENABLE_PROMPT_CACHING_1H": "1", + "ENABLE_PROMPT_CACHING_1H_BEDROCK": "1", +} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "1h-TTL opt-in env vars set; the proxy likely stripped or " + "downgraded the cache_control TTL" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_1h/test_vertex_ai.py b/tests/claude_code/prompt_caching_1h/test_vertex_ai.py new file mode 100644 index 00000000000..e7224bfa49b --- /dev/null +++ b/tests/claude_code/prompt_caching_1h/test_vertex_ai.py @@ -0,0 +1,104 @@ +"""prompt_caching_1h x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, opt into the 1-hour cache TTL via +`ENABLE_PROMPT_CACHING_1H`, and assert the upstream's usage block +reports a non-zero cache token count. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/prompt_caching_1h/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Optional + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +CACHE_1H_ENV = {"ENABLE_PROMPT_CACHING_1H": "1"} + + +def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: + if not isinstance(usage, Mapping): + return 0 + creation = usage.get("cache_creation_input_tokens") or 0 + read = usage.get("cache_read_input_tokens") or 0 + try: + return int(creation) + int(read) + except (TypeError, ValueError): + return 0 + + +def test_prompt_caching_1h_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + extra_env=CACHE_1H_ENV, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens with " + "ENABLE_PROMPT_CACHING_1H=1" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_5m/test_anthropic.py b/tests/claude_code/prompt_caching_5m/test_anthropic.py index c8e450198b3..230256e6e01 100644 --- a/tests/claude_code/prompt_caching_5m/test_anthropic.py +++ b/tests/claude_code/prompt_caching_5m/test_anthropic.py @@ -27,7 +27,11 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -52,8 +56,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 -@pytest.mark.parametrize("model", ANTHROPIC_MODELS) -def test_prompt_caching_5m_anthropic(compat_result, model): +def test_prompt_caching_5m_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -72,42 +75,39 @@ def test_prompt_caching_5m_anthropic(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if _cache_tokens(result.usage) <= 0: - compat_result.set( - { - "status": "fail", - "error": ( - f"[{model}] usage block reported zero cache tokens; " - "expected cache_control on the system prompt to produce a non-zero " - "cache_creation_input_tokens or cache_read_input_tokens" - ), - } - ) - pytest.fail(f"no cache tokens for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_5m/test_azure.py b/tests/claude_code/prompt_caching_5m/test_azure.py index 0fc12771997..99ed8e36b08 100644 --- a/tests/claude_code/prompt_caching_5m/test_azure.py +++ b/tests/claude_code/prompt_caching_5m/test_azure.py @@ -27,7 +27,11 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -50,8 +54,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 -@pytest.mark.parametrize("model", AZURE_MODELS) -def test_prompt_caching_5m_azure(compat_result, model): +def test_prompt_caching_5m_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -70,42 +73,39 @@ def test_prompt_caching_5m_azure(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if _cache_tokens(result.usage) <= 0: - compat_result.set( - { - "status": "fail", - "error": ( - f"[{model}] usage block reported zero cache tokens; " - "expected cache_control on the system prompt to produce a non-zero " - "cache_creation_input_tokens or cache_read_input_tokens" - ), - } - ) - pytest.fail(f"no cache tokens for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py b/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py index 782a96c2ace..22035be4c19 100644 --- a/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py +++ b/tests/claude_code/prompt_caching_5m/test_bedrock_converse.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -43,8 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 -@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) -def test_prompt_caching_5m_bedrock_converse(compat_result, model): +def test_prompt_caching_5m_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -63,42 +66,39 @@ def test_prompt_caching_5m_bedrock_converse(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if _cache_tokens(result.usage) <= 0: - compat_result.set( - { - "status": "fail", - "error": ( - f"[{model}] usage block reported zero cache tokens; " - "expected cache_control on the system prompt to produce a non-zero " - "cache_creation_input_tokens or cache_read_input_tokens" - ), - } - ) - pytest.fail(f"no cache tokens for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py b/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py index 5fe6679e7ca..115c3069e81 100644 --- a/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py +++ b/tests/claude_code/prompt_caching_5m/test_bedrock_invoke.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -43,8 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 -@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) -def test_prompt_caching_5m_bedrock_invoke(compat_result, model): +def test_prompt_caching_5m_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -63,42 +66,39 @@ def test_prompt_caching_5m_bedrock_invoke(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if _cache_tokens(result.usage) <= 0: - compat_result.set( - { - "status": "fail", - "error": ( - f"[{model}] usage block reported zero cache tokens; " - "expected cache_control on the system prompt to produce a non-zero " - "cache_creation_input_tokens or cache_read_input_tokens" - ), - } - ) - pytest.fail(f"no cache tokens for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/prompt_caching_5m/test_vertex_ai.py b/tests/claude_code/prompt_caching_5m/test_vertex_ai.py index f2665979bf1..aa62d95764d 100644 --- a/tests/claude_code/prompt_caching_5m/test_vertex_ai.py +++ b/tests/claude_code/prompt_caching_5m/test_vertex_ai.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Optional import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -43,8 +47,7 @@ def _cache_tokens(usage: Optional[Mapping[str, Any]]) -> int: return 0 -@pytest.mark.parametrize("model", VERTEX_AI_MODELS) -def test_prompt_caching_5m_vertex_ai(compat_result, model): +def test_prompt_caching_5m_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert the upstream usage block surfaces a non-zero cache token count.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -63,42 +66,39 @@ def test_prompt_caching_5m_vertex_ai(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt="Reply with the single word 'pong' and nothing else.", - model=model, - base_url=base_url, - api_key=api_key, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="Reply with the single word 'pong' and nothing else.", + base_url=base_url, + api_key=api_key, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if _cache_tokens(result.usage) <= 0: - compat_result.set( - { - "status": "fail", - "error": ( - f"[{model}] usage block reported zero cache tokens; " - "expected cache_control on the system prompt to produce a non-zero " - "cache_creation_input_tokens or cache_read_input_tokens" - ), - } - ) - pytest.fail(f"no cache tokens for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if _cache_tokens(outcome.usage) <= 0: + error = ( + f"[{model}] usage block reported zero cache tokens; " + "expected cache_control on the system prompt to produce a non-zero " + "cache_creation_input_tokens or cache_read_input_tokens" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/rate_limiter.py b/tests/claude_code/rate_limiter.py new file mode 100644 index 00000000000..06d21b83832 --- /dev/null +++ b/tests/claude_code/rate_limiter.py @@ -0,0 +1,358 @@ +"""Cross-process token-bucket rate limiter for the Claude Code compat suite. + +The compat matrix runs 75 live `claude` CLI invocations (25 cells × 3 +Claude tiers per cell). When pytest-xdist fans these out across worker +processes, each worker would maintain its own in-memory rate limiter +and the *aggregate* request rate hitting any one upstream provider +would be `workers × per-worker rate` — exactly the situation that +trips Anthropic / Azure / Bedrock / Vertex 429s in the middle of a +matrix run and silently flips green cells red. + +The fix is a **shared, cross-process** token bucket per provider, +backed by a small JSON state file and an OS-level `flock`. Each +`run_claude` invocation acquires one token (sleeping if the bucket is +empty) before launching the CLI; refills happen lazily based on wall +time, so workers can be killed and restarted without losing or +double-spending budget. + +Configuration is driven entirely by environment variables so a +binary-search workflow can shift per-provider rates without code +edits: + + LITELLM_COMPAT_RATE_ANTHROPIC (req/s, default 5.0) + LITELLM_COMPAT_RATE_AZURE (req/s, default 5.0) + LITELLM_COMPAT_RATE_VERTEX_AI (req/s, default 5.0) + LITELLM_COMPAT_RATE_BEDROCK_CONVERSE (req/s, default 5.0) + LITELLM_COMPAT_RATE_BEDROCK_INVOKE (req/s, default 5.0) + LITELLM_COMPAT_RATE_BURST (per-bucket burst override; + default = rate) + LITELLM_COMPAT_RATE_STATE_DIR (state file directory; + default = $TMPDIR/litellm-claude-compat-ratelimit) + +A rate of 0 (or any non-positive value) disables throttling for that +provider — useful when you trust the upstream to handle the burst or +when running unit-test-shaped workloads that never actually hit the +network. + +The provider id is inferred from the model id by `infer_provider`, +mirroring the matrix's column layout (`anthropic`, `azure`, +`vertex_ai`, `bedrock_converse`, `bedrock_invoke`). +""" + +from __future__ import annotations + +import contextlib +import json +import math +import os +import tempfile +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterator, Mapping, Optional + +# `fcntl` is POSIX-only; the suite is Linux/macOS only, so we don't +# attempt a Windows fallback. Importing at module load fails fast on +# the (currently non-existent) Windows runner so we don't silently +# degrade to no-locking behavior. +import fcntl + +PROVIDER_ANTHROPIC = "anthropic" +PROVIDER_AZURE = "azure" +PROVIDER_VERTEX_AI = "vertex_ai" +PROVIDER_BEDROCK_CONVERSE = "bedrock_converse" +PROVIDER_BEDROCK_INVOKE = "bedrock_invoke" + +ALL_PROVIDERS = ( + PROVIDER_ANTHROPIC, + PROVIDER_AZURE, + PROVIDER_VERTEX_AI, + PROVIDER_BEDROCK_CONVERSE, + PROVIDER_BEDROCK_INVOKE, +) + +DEFAULT_RATE = 5.0 # req/s per provider, conservative starting point +RATE_ENV_PREFIX = "LITELLM_COMPAT_RATE_" +BURST_ENV = "LITELLM_COMPAT_RATE_BURST" +STATE_DIR_ENV = "LITELLM_COMPAT_RATE_STATE_DIR" +DEFAULT_STATE_DIR_NAME = "litellm-claude-compat-ratelimit" + + +def infer_provider(model: str) -> str: + """Map a model alias to its compat-matrix provider id. + + The matrix column layout is fixed; aliases registered in the proxy + encode the provider via a suffix (`-bedrock-converse`, + `-bedrock-invoke`, `-azure`, `-vertex`) or its absence (Anthropic). + Order matters: the bedrock suffixes both contain `bedrock`, so we + test the more-specific ones first. + """ + if not model: + raise ValueError("model must be a non-empty string") + lower = model.lower() + if lower.endswith("-bedrock-converse"): + return PROVIDER_BEDROCK_CONVERSE + if lower.endswith("-bedrock-invoke"): + return PROVIDER_BEDROCK_INVOKE + if lower.endswith("-azure"): + return PROVIDER_AZURE + if lower.endswith("-vertex"): + return PROVIDER_VERTEX_AI + return PROVIDER_ANTHROPIC + + +@dataclass(frozen=True) +class ProviderConfig: + """Static config snapshot for one provider's bucket. + + Captured up front (rather than re-read per acquire) so the limiter's + behavior in a single process is stable even if env vars are mutated + mid-run. A fresh `RateLimiter` picks up env changes on construction. + """ + + rate_per_sec: float + burst: float + + @property + def enabled(self) -> bool: + return self.rate_per_sec > 0 and self.burst > 0 + + +def load_config( + env: Optional[Mapping[str, str]] = None, +) -> Dict[str, ProviderConfig]: + """Build a {provider: ProviderConfig} from env, applying defaults. + + Parsing failures fall back to the default rate rather than + crashing the test session — a typo in `LITELLM_COMPAT_RATE_AZURE` + should not silently disable throttling, but it also shouldn't + abort 75 live tests with a `ValueError` ten minutes in. + """ + src = env if env is not None else os.environ + + def _as_float(value: Optional[str], default: float) -> float: + if value is None or value == "": + return default + try: + return float(value) + except (TypeError, ValueError): + return default + + burst_override = _as_float(src.get(BURST_ENV), -1.0) + + out: Dict[str, ProviderConfig] = {} + for provider in ALL_PROVIDERS: + env_key = RATE_ENV_PREFIX + provider.upper() + rate = _as_float(src.get(env_key), DEFAULT_RATE) + burst = burst_override if burst_override > 0 else max(rate, 1.0) + out[provider] = ProviderConfig(rate_per_sec=rate, burst=burst) + return out + + +def _state_dir(env: Optional[Mapping[str, str]] = None) -> Path: + """Resolve the directory holding per-provider state files. + + A user-supplied `LITELLM_COMPAT_RATE_STATE_DIR` wins for tests and + container environments where `$TMPDIR` may be ephemeral or shared + in surprising ways. The directory is created lazily with + `parents=True, exist_ok=True` so first-run setup needs no + fixture wiring. + """ + src = env if env is not None else os.environ + explicit = src.get(STATE_DIR_ENV) + if explicit: + return Path(explicit) + return Path(tempfile.gettempdir()) / DEFAULT_STATE_DIR_NAME + + +class RateLimiter: + """Cross-process token bucket per provider. + + Each `acquire(provider)` call: + 1. Opens (creating if needed) `/.json`. + 2. Holds an exclusive `flock` while reading + updating the + {tokens, last_refill} state. + 3. Refills tokens based on `now - last_refill`, capped at burst. + 4. If tokens >= 1, subtracts one and returns immediately. + 5. Otherwise, computes the wall-time delay needed to earn a + single token at the configured rate, releases the lock, and + sleeps. After sleeping it retries — staying under the lock + while sleeping would serialize all workers behind whichever + one held it longest. + + This bounds the *aggregate* req/s seen by the upstream, regardless + of how many xdist workers, threads, or processes are concurrently + running tests against the same provider. + + A `_clock` / `_sleep` injection seam keeps the unit tests fast and + deterministic; production callers should never override either. + """ + + def __init__( + self, + config: Optional[Mapping[str, ProviderConfig]] = None, + state_dir: Optional[Path] = None, + clock: Optional[callable] = None, + sleep: Optional[callable] = None, + ) -> None: + self._config = dict(config) if config is not None else load_config() + self._state_dir = Path(state_dir) if state_dir is not None else _state_dir() + self._clock = clock or time.monotonic + self._sleep = sleep or time.sleep + # `_state_dir.mkdir` once on construction is fine; concurrent + # workers all racing to create the same directory is benign. + self._state_dir.mkdir(parents=True, exist_ok=True) + + def acquire(self, provider: str) -> float: + """Block until one token is available for `provider`. + + Returns the cumulative wall-time spent waiting (0.0 when the + bucket had budget and we returned immediately). Callers can + log this to attribute slow cells to throttling vs. upstream + latency — same role `DriverResult.duration_ms` plays for the + actual CLI invocation. + """ + cfg = self._config.get(provider) + if cfg is None or not cfg.enabled: + return 0.0 + + path = self._state_path(provider) + total_waited = 0.0 + while True: + now = self._clock() + sleep_for = self._try_consume(path, cfg, now) + if sleep_for <= 0: + return total_waited + self._sleep(sleep_for) + total_waited += sleep_for + + def _state_path(self, provider: str) -> Path: + return self._state_dir / f"{provider}.json" + + def _try_consume(self, path: Path, cfg: ProviderConfig, now: float) -> float: + """Atomically refill and try to take one token. + + Returns 0.0 if a token was consumed, or a positive sleep + duration (seconds) if the caller must wait before retrying. + + We hold an exclusive `flock` only across the read-modify-write + of the JSON state — never across a `sleep` — so workers don't + serialize while one of them is parked. + """ + # `os.open` + `os.O_CREAT | os.O_RDWR` gives us a fd we can + # both lock and read/write through. Opening with `"a+"` then + # seeking is equivalent but uglier; this version is closer to + # the canonical flock recipe. + fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + try: + tokens, last_refill = self._read_state(fd, cfg, now) + tokens, last_refill = self._refill(tokens, last_refill, now, cfg) + if tokens >= 1.0: + tokens -= 1.0 + self._write_state(fd, tokens, last_refill) + return 0.0 + # Not enough budget. Persist the refilled state so a + # subsequent caller doesn't have to redo the math, then + # release the lock and tell the caller how long to + # sleep before retrying. + self._write_state(fd, tokens, last_refill) + deficit = 1.0 - tokens + # `deficit / rate` seconds will earn exactly enough + # for one token. Add a tiny safety margin so we don't + # wake up nanoseconds early and spin. + return (deficit / cfg.rate_per_sec) + 1e-3 + finally: + fcntl.flock(fd, fcntl.LOCK_UN) + finally: + os.close(fd) + + @staticmethod + def _read_state(fd: int, cfg: ProviderConfig, now: float) -> tuple: + """Read {tokens, last_refill} from `fd`, defaulting to a full + bucket on a missing/empty/corrupt file. + + New files start full so the very first request never waits; + corrupt files are treated like new files because the + alternative — refusing to run — is worse than briefly + over-spending one bucket's worth of budget. + """ + os.lseek(fd, 0, os.SEEK_SET) + raw = os.read(fd, 4096).decode("utf-8") + if not raw.strip(): + return cfg.burst, now + try: + obj = json.loads(raw) + tokens = float(obj.get("tokens", cfg.burst)) + last_refill = float(obj.get("last_refill", now)) + return tokens, last_refill + except (ValueError, TypeError): + return cfg.burst, now + + @staticmethod + def _refill( + tokens: float, last_refill: float, now: float, cfg: ProviderConfig + ) -> tuple: + """Apply elapsed time to the bucket, capped at burst. + + Negative elapsed (clock went backward, e.g. across host + sleep/resume or a manually-tweaked monotonic mock) is clamped + to zero so we never *remove* tokens. + """ + elapsed = max(0.0, now - last_refill) + tokens = min(cfg.burst, tokens + elapsed * cfg.rate_per_sec) + return tokens, now + + @staticmethod + def _write_state(fd: int, tokens: float, last_refill: float) -> None: + payload = json.dumps({"tokens": tokens, "last_refill": last_refill}).encode( + "utf-8" + ) + os.lseek(fd, 0, os.SEEK_SET) + os.ftruncate(fd, 0) + os.write(fd, payload) + + +# A single process-wide instance is all we need: provider-keyed state +# is stored in files, so multiple `RateLimiter` instances would just +# duplicate the in-process bookkeeping. We expose a getter rather than +# the instance directly so unit tests can install a custom limiter +# scoped to a tmp directory without monkeypatching globals. +_default: Optional[RateLimiter] = None + + +def get_default_limiter() -> RateLimiter: + global _default + if _default is None: + _default = RateLimiter() + return _default + + +def reset_default_limiter() -> None: + """Drop the cached default limiter; the next `get_default_limiter` + call rebuilds it from the current environment. + + Useful between unit tests that patch env vars: without this they'd + keep reading the stale config snapshot from the first call. + """ + global _default + _default = None + + +@contextlib.contextmanager +def use_limiter(limiter: RateLimiter) -> Iterator[RateLimiter]: + """Temporarily install `limiter` as the process default. + + The driver's `run_claude` calls `get_default_limiter()`; tests that + want a controlled tmp-dir-backed limiter use this contextmanager + to swap one in without touching env vars or the on-disk default + state. + """ + global _default + previous = _default + _default = limiter + try: + yield limiter + finally: + _default = previous diff --git a/tests/claude_code/run_compat.sh b/tests/claude_code/run_compat.sh new file mode 100755 index 00000000000..2e62dc65fb5 --- /dev/null +++ b/tests/claude_code/run_compat.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Run the Claude Code compat matrix end-to-end against a live LiteLLM +# proxy, with per-provider rate limits applied via the cross-process +# token bucket in `tests/claude_code/rate_limiter.py`. +# +# Designed for binary-searching the ideal X / Y / Z req/s per provider: +# 1. Pick an initial rate (e.g. 5/s for everyone). +# 2. Run this script. +# 3. Read `compat-rate-limit-summary.json` to see whether any provider +# hit a 429-shaped error during the run. +# 4. If a provider has `rate_limited > 0`, halve its rate; else, double it. +# 5. Repeat until the highest no-429 rate is found. +# +# Required env (proxy connection): +# LITELLM_PROXY_BASE_URL e.g. http://localhost:4000 +# LITELLM_PROXY_API_KEY e.g. sk-1234 +# +# Optional env (rate limits, all default to 5 req/s; 0 disables a column): +# LITELLM_COMPAT_RATE_ANTHROPIC +# LITELLM_COMPAT_RATE_AZURE +# LITELLM_COMPAT_RATE_VERTEX_AI +# LITELLM_COMPAT_RATE_BEDROCK_CONVERSE +# LITELLM_COMPAT_RATE_BEDROCK_INVOKE +# LITELLM_COMPAT_RATE_BURST override per-bucket burst +# +# Optional env (parallelism): +# COMPAT_XDIST_WORKERS passed to `pytest -n` (default: auto) +# +# Optional env (artifacts): +# COMPAT_RESULTS_PATH default: compat-results.json +# COMPAT_RATE_LIMIT_SUMMARY_PATH default: compat-rate-limit-summary.json + +set -euo pipefail + +if [[ -z "${LITELLM_PROXY_BASE_URL:-}" || -z "${LITELLM_PROXY_API_KEY:-}" ]]; then + echo "error: LITELLM_PROXY_BASE_URL and LITELLM_PROXY_API_KEY must be set" >&2 + exit 64 +fi + +# Reset the cross-process rate-limiter state from any prior run. Stale +# token-bucket files would let a previous run's accumulated budget bleed +# into the new one, which subtly biases the binary search. +state_dir="${LITELLM_COMPAT_RATE_STATE_DIR:-${TMPDIR:-/tmp}/litellm-claude-compat-ratelimit}" +if [[ -d "$state_dir" ]]; then + rm -rf "$state_dir" +fi + +# Worker count. `auto` picks one worker per CPU; the rate limiter +# enforces aggregate provider rates regardless of worker count, so +# this is a "go as fast as the limiter allows" knob, not a tuning knob. +workers="${COMPAT_XDIST_WORKERS:-auto}" + +# Where the artifacts land. We resolve them now so the summary file is +# always at a known path the caller can grep, even if they didn't set +# the env explicitly. +results_path="${COMPAT_RESULTS_PATH:-compat-results.json}" +summary_path="${COMPAT_RATE_LIMIT_SUMMARY_PATH:-compat-rate-limit-summary.json}" + +echo "[run_compat] rates:" +for provider in ANTHROPIC AZURE VERTEX_AI BEDROCK_CONVERSE BEDROCK_INVOKE; do + var="LITELLM_COMPAT_RATE_${provider}" + echo " ${provider}=${!var:-default(5/s)}" +done +echo " BURST=${LITELLM_COMPAT_RATE_BURST:-default(=rate)}" +echo "[run_compat] xdist workers: ${workers}" +echo "[run_compat] results: ${results_path}" +echo "[run_compat] summary: ${summary_path}" + +# Run only the per-feature live tests; skip the unit-test directories +# (they're under directories starting with `_`). The dist=loadfile +# scheduler keeps each test file pinned to a single worker, which is +# what we want — every test in a file shares a single ThreadPoolExecutor +# fanout, and we don't gain anything by splitting it across workers. +start=$(date +%s) +COMPAT_RESULTS_PATH="${results_path}" \ +COMPAT_RATE_LIMIT_SUMMARY_PATH="${summary_path}" \ +PATH="$HOME/.local/bin:$PATH" \ +uv run pytest \ + tests/claude_code/basic_messaging_non_streaming \ + tests/claude_code/basic_messaging_streaming \ + tests/claude_code/extended_thinking \ + tests/claude_code/tool_use \ + tests/claude_code/vision \ + tests/claude_code/prompt_caching_5m \ + -n "${workers}" \ + --dist=loadfile \ + -q \ + "$@" + +exit_code=$? +end=$(date +%s) +echo "[run_compat] wall time: $((end - start))s" + +# Surface the rate-limit summary inline so a human reader doesn't have +# to `cat` the JSON file. The full file is still on disk for the binary +# search loop. +if [[ -f "${summary_path}" ]]; then + echo "[run_compat] summary: ${summary_path}" + if command -v jq >/dev/null 2>&1; then + jq '.totals, .per_provider' "${summary_path}" + else + cat "${summary_path}" + fi +fi + +exit "${exit_code}" diff --git a/tests/claude_code/test_config.yaml b/tests/claude_code/test_config.yaml index 2943f301fab..90216a00a35 100644 --- a/tests/claude_code/test_config.yaml +++ b/tests/claude_code/test_config.yaml @@ -62,18 +62,18 @@ model_list: - model_name: claude-haiku-4-5-vertex litellm_params: model: vertex_ai/claude-haiku-4-5 - vertex_ai_project: pathrise-convert-1606954137718 - vertex_ai_location: us-east5 + vertex_ai_project: os.environ/VERTEXAI_PROJECT + vertex_ai_location: os.environ/VERTEXAI_LOCATION - model_name: claude-sonnet-4-6-vertex litellm_params: model: vertex_ai/claude-sonnet-4-6 - vertex_ai_project: pathrise-convert-1606954137718 - vertex_ai_location: us-east5 + vertex_ai_project: os.environ/VERTEXAI_PROJECT + vertex_ai_location: os.environ/VERTEXAI_LOCATION - model_name: claude-opus-4-7-vertex litellm_params: model: vertex_ai/claude-opus-4-7 - vertex_ai_project: pathrise-convert-1606954137718 - vertex_ai_location: us-east5 + vertex_ai_project: os.environ/VERTEXAI_PROJECT + vertex_ai_location: os.environ/VERTEXAI_LOCATION # ---- Microsoft Foundry (Anthropic deployments on Azure) ---- - model_name: claude-haiku-4-5-azure diff --git a/tests/claude_code/thinking_with_tool_use/__init__.py b/tests/claude_code/thinking_with_tool_use/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/thinking_with_tool_use/test_anthropic.py b/tests/claude_code/thinking_with_tool_use/test_anthropic.py new file mode 100644 index 00000000000..093a017a93c --- /dev/null +++ b/tests/claude_code/thinking_with_tool_use/test_anthropic.py @@ -0,0 +1,148 @@ +"""thinking_with_tool_use x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, enable extended thinking via `MAX_THINKING_TOKENS`, allow +the built-in `Bash` tool, and ask Claude to plan-and-execute a task +that requires both reasoning and a tool call. Assert the upstream +returned both a `thinking` content block and a `tool_use` content +block in the same turn — proving the proxy preserves the wire shape +where extended thinking and tool use coexist. + +This is the cell that historically catches the most provider bugs: +"thinking blocks cannot be modified" 400s, the recurring Bedrock +"thinking.type.enabled is not supported" error, and the +`fine-grained-tool-streaming` + `interleaved-thinking` beta-header +interactions. A regression in any of those collapses this cell. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/thinking_with_tool_use/test_anthropic.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Extended thinking on, with a small budget — enough to surface a +# non-empty thinking block on a trivial reasoning prompt without +# blowing up wall time. +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} + +# Prompt designed to force both blocks: the model has to *reason* about +# what command to run before *invoking* the Bash tool. Using a fixed +# expected output keeps the assertion focused on the wire shape rather +# than on answer quality. +THINKING_TOOL_PROMPT = ( + "Think step by step about which shell command would print just the word " + "'pong'. Then use the Bash tool to run that exact command and report what " + "it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + """Walk the stream-json events and return True if any assistant + message included a content block of the given type.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy with thinking + enabled and tool use, and assert both `thinking` and `tool_use` + content blocks landed in the same turn.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = ( + f"[{model}] no `thinking` content block observed; thinking " + f"either disabled by the proxy or stripped by the upstream" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = ( + f"[{model}] no `tool_use` content block observed alongside " + f"thinking; the proxy may have dropped tools when thinking is on" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/thinking_with_tool_use/test_azure.py b/tests/claude_code/thinking_with_tool_use/test_azure.py new file mode 100644 index 00000000000..eec9bfd1790 --- /dev/null +++ b/tests/claude_code/thinking_with_tool_use/test_azure.py @@ -0,0 +1,123 @@ +"""thinking_with_tool_use x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +enable extended thinking via `MAX_THINKING_TOKENS`, allow the built-in +`Bash` tool, and ask Claude to plan-and-execute a task that requires +both reasoning and a tool call. Assert the upstream returned both a +`thinking` content block and a `tool_use` content block in the same +turn. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/thinking_with_tool_use/test_azure.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_TOOL_PROMPT = ( + "Think step by step about which shell command would print just the word " + "'pong'. Then use the Bash tool to run that exact command and report what " + "it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py b/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py new file mode 100644 index 00000000000..f9c9828e5c2 --- /dev/null +++ b/tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py @@ -0,0 +1,128 @@ +"""thinking_with_tool_use x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, enable extended +thinking via `MAX_THINKING_TOKENS`, allow the built-in `Bash` tool, and +ask Claude to plan-and-execute a task that requires both reasoning and +a tool call. Assert the upstream returned both a `thinking` content +block and a `tool_use` content block in the same turn. + +The Converse API has its own `additionalModelRequestFields.thinking` +shape and its own tool-use envelope; this cell catches gateway +regressions where the proxy fails to translate between Anthropic's +`thinking` parameter and Converse's reasoning configuration when tools +are also present. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/thinking_with_tool_use/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_TOOL_PROMPT = ( + "Think step by step about which shell command would print just the word " + "'pong'. Then use the Bash tool to run that exact command and report what " + "it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py b/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py new file mode 100644 index 00000000000..a0bd0753076 --- /dev/null +++ b/tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py @@ -0,0 +1,130 @@ +"""thinking_with_tool_use x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +enable extended thinking via `MAX_THINKING_TOKENS`, allow the built-in +`Bash` tool, and ask Claude to plan-and-execute a task that requires +both reasoning and a tool call. Assert the upstream returned both a +`thinking` content block and a `tool_use` content block in the same +turn. + +This is the cell most likely to flush out Bedrock-specific bugs in +LiteLLM's Anthropic <-> Bedrock translation: the recurring +"thinking.type.enabled is not supported" 400 error has reappeared on +several Bedrock model routes (notably application inference profile +ARNs), and the only reliable signal that the fix is wired through the +proxy is a successful round-trip on this cell. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/thinking_with_tool_use/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_TOOL_PROMPT = ( + "Think step by step about which shell command would print just the word " + "'pong'. Then use the Bash tool to run that exact command and report what " + "it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py b/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py new file mode 100644 index 00000000000..d784ddf640d --- /dev/null +++ b/tests/claude_code/thinking_with_tool_use/test_vertex_ai.py @@ -0,0 +1,128 @@ +"""thinking_with_tool_use x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, enable extended thinking via +`MAX_THINKING_TOKENS`, allow the built-in `Bash` tool, and ask Claude +to plan-and-execute a task that requires both reasoning and a tool +call. Assert the upstream returned both a `thinking` content block and +a `tool_use` content block in the same turn. + +Vertex AI exposes Anthropic models via `:rawPredict` / +`:streamRawPredict` and has its own beta-header allowlist. This cell +catches gateway regressions where the proxy strips +`anthropic-beta: interleaved-thinking-2025-05-14` (or the equivalent +header set the upstream needs) on the way to Vertex. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/thinking_with_tool_use/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +THINKING_ENV = {"MAX_THINKING_TOKENS": "4096"} +THINKING_TOOL_PROMPT = ( + "Think step by step about which shell command would print just the word " + "'pong'. Then use the Bash tool to run that exact command and report what " + "it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + + +def _has_block_type( + events: Sequence[Mapping[str, Any]], + block_type: str, +) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == block_type: + return True + return False + + +def test_thinking_with_tool_use_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=THINKING_TOOL_PROMPT, + base_url=base_url, + api_key=api_key, + extra_env=THINKING_ENV, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "thinking"): + error = f"[{model}] no `thinking` content block observed" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_block_type(outcome.events, "tool_use"): + error = f"[{model}] no `tool_use` content block observed alongside thinking" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use/test_anthropic.py b/tests/claude_code/tool_use/test_anthropic.py index 6a0627f0233..9ac9ae1da84 100644 --- a/tests/claude_code/tool_use/test_anthropic.py +++ b/tests/claude_code/tool_use/test_anthropic.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -56,8 +60,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", ANTHROPIC_MODELS) -def test_tool_use_anthropic(compat_result, model): +def test_tool_use_anthropic(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -76,39 +79,38 @@ def test_tool_use_anthropic(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=TOOL_USE_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_args=TOOL_USE_ARGS, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_tool_use_event(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no tool_use content block observed in stream-json events", - } - ) - pytest.fail(f"no tool_use for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use/test_azure.py b/tests/claude_code/tool_use/test_azure.py index 133e9d2b48d..ab5595c55e2 100644 --- a/tests/claude_code/tool_use/test_azure.py +++ b/tests/claude_code/tool_use/test_azure.py @@ -24,7 +24,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -55,8 +59,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", AZURE_MODELS) -def test_tool_use_azure(compat_result, model): +def test_tool_use_azure(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -75,39 +78,38 @@ def test_tool_use_azure(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=TOOL_USE_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_args=TOOL_USE_ARGS, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_tool_use_event(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no tool_use content block observed in stream-json events", - } - ) - pytest.fail(f"no tool_use for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use/test_bedrock_converse.py b/tests/claude_code/tool_use/test_bedrock_converse.py index 81b25f76668..1c9af3ee39b 100644 --- a/tests/claude_code/tool_use/test_bedrock_converse.py +++ b/tests/claude_code/tool_use/test_bedrock_converse.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -51,8 +55,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) -def test_tool_use_bedrock_converse(compat_result, model): +def test_tool_use_bedrock_converse(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -71,39 +74,38 @@ def test_tool_use_bedrock_converse(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=TOOL_USE_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_args=TOOL_USE_ARGS, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_tool_use_event(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no tool_use content block observed in stream-json events", - } - ) - pytest.fail(f"no tool_use for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use/test_bedrock_invoke.py b/tests/claude_code/tool_use/test_bedrock_invoke.py index d54776b9bab..ab9aa27edcd 100644 --- a/tests/claude_code/tool_use/test_bedrock_invoke.py +++ b/tests/claude_code/tool_use/test_bedrock_invoke.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -51,8 +55,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) -def test_tool_use_bedrock_invoke(compat_result, model): +def test_tool_use_bedrock_invoke(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -71,39 +74,38 @@ def test_tool_use_bedrock_invoke(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=TOOL_USE_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_args=TOOL_USE_ARGS, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_tool_use_event(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no tool_use content block observed in stream-json events", - } - ) - pytest.fail(f"no tool_use for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use/test_vertex_ai.py b/tests/claude_code/tool_use/test_vertex_ai.py index 20c93e54fed..e118bf39dd2 100644 --- a/tests/claude_code/tool_use/test_vertex_ai.py +++ b/tests/claude_code/tool_use/test_vertex_ai.py @@ -20,7 +20,11 @@ from typing import Any, Mapping, Sequence import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -51,8 +55,7 @@ def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: return False -@pytest.mark.parametrize("model", VERTEX_AI_MODELS) -def test_tool_use_vertex_ai(compat_result, model): +def test_tool_use_vertex_ai(compat_result): """Drive the `claude` CLI against the LiteLLM proxy and assert a tool call was emitted on the wire.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -71,39 +74,38 @@ def test_tool_use_vertex_ai(compat_result, model): f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False ) - try: - result = run_claude( - prompt=TOOL_USE_PROMPT, - model=model, - base_url=base_url, - api_key=api_key, - extra_args=TOOL_USE_ARGS, - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not _has_tool_use_event(result.events): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] no tool_use content block observed in stream-json events", - } - ) - pytest.fail(f"no tool_use for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use_streaming/__init__.py b/tests/claude_code/tool_use_streaming/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/tool_use_streaming/test_anthropic.py b/tests/claude_code/tool_use_streaming/test_anthropic.py new file mode 100644 index 00000000000..a782d74a7df --- /dev/null +++ b/tests/claude_code/tool_use_streaming/test_anthropic.py @@ -0,0 +1,147 @@ +"""tool_use_streaming x Anthropic. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes to Anthropic, ask +Claude to invoke a built-in tool (`Bash`), and assert that the upstream +(a) emitted a `tool_use` content block and (b) actually streamed the +events incrementally — i.e. more than one stream-json record was +observed before the final `result`. + +This is the "fine-grained tool streaming" path. Historically gateways +break it in two ways: they either buffer the entire response before +flushing (in which case `len(events)` collapses to ~1 final record) or +they strip the `fine-grained-tool-streaming-2025-05-14` beta header and +the upstream falls back to non-streaming tool_use. Both regressions are +caught by the assertions below. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use_streaming/test_anthropic.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# Same shape as the non-streaming `tool_use` cell: ask Claude to call +# the built-in `Bash` tool. The CLI is already in stream-json mode by +# default in `run_claude`, so we don't need to toggle anything to +# exercise the streaming wire — what we want to assert is that the +# stream-json transport actually carried more than one record, which +# is the wire-level signal that the proxy didn't buffer the upstream. +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + +# Floor on the number of stream-json records we expect to see for a +# tool-use turn. A buffered (non-streamed) wire collapses to one +# `system` init record + one `assistant` final + one `result`, total 3. +# Real fine-grained streaming produces many more (incremental +# input_json_delta events, intermediate assistant deltas, etc.). We +# pick a floor above the buffered case so the assertion catches the +# regression without being flaky on short responses. +MIN_STREAM_EVENTS = 4 + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a `tool_use` content block.""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_streaming_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + proxy preserves fine-grained tool streaming end-to-end.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if len(outcome.events) < MIN_STREAM_EVENTS: + error = ( + f"[{model}] only {len(outcome.events)} stream-json events observed " + f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response or " + f"stripped fine-grained tool streaming" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use_streaming/test_azure.py b/tests/claude_code/tool_use_streaming/test_azure.py new file mode 100644 index 00000000000..624a41c8710 --- /dev/null +++ b/tests/claude_code/tool_use_streaming/test_azure.py @@ -0,0 +1,122 @@ +"""tool_use_streaming x Microsoft Foundry (Azure). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +Microsoft Foundry's Anthropic deployments on Azure, ask Claude to +invoke a built-in tool (`Bash`), and assert that the upstream (a) +emitted a `tool_use` content block and (b) actually streamed events +incrementally. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use_streaming/test_azure.py + ^^^^^^^^^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + +MIN_STREAM_EVENTS = 4 + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_streaming_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if len(outcome.events) < MIN_STREAM_EVENTS: + error = ( + f"[{model}] only {len(outcome.events)} stream-json events observed " + f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use_streaming/test_bedrock_converse.py b/tests/claude_code/tool_use_streaming/test_bedrock_converse.py new file mode 100644 index 00000000000..af41f63031e --- /dev/null +++ b/tests/claude_code/tool_use_streaming/test_bedrock_converse.py @@ -0,0 +1,128 @@ +"""tool_use_streaming x Bedrock (Converse). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +AWS Bedrock via the `Converse` API (`ConverseStream`), ask Claude to +invoke a built-in tool (`Bash`), and assert that the upstream (a) +emitted a `tool_use` content block and (b) actually streamed events +incrementally. + +Bedrock Converse has its own tool-streaming envelope (`toolUse` blocks +with `delta` chunks); this cell catches gateway regressions where the +proxy buffers the response or fails to translate Converse's streaming +envelope back to the Anthropic `message_*` event shape Claude Code +expects. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use_streaming/test_bedrock_converse.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + +MIN_STREAM_EVENTS = 4 + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_streaming_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if len(outcome.events) < MIN_STREAM_EVENTS: + error = ( + f"[{model}] only {len(outcome.events)} stream-json events observed " + f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py b/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py new file mode 100644 index 00000000000..1721ee781b7 --- /dev/null +++ b/tests/claude_code/tool_use_streaming/test_bedrock_invoke.py @@ -0,0 +1,126 @@ +"""tool_use_streaming x Bedrock (Invoke). + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +AWS Bedrock via the legacy `InvokeModel` API path, ask Claude to invoke +a built-in tool (`Bash`), and assert that the upstream (a) emitted a +`tool_use` content block and (b) actually streamed events incrementally. + +Bedrock InvokeModel surfaces tool-streaming via `InvokeModelWithResponseStream`; +this cell catches gateway regressions where the proxy buffers the +response or fails to translate the streaming envelope to Anthropic +`message_*` event shape. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use_streaming/test_bedrock_invoke.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + +MIN_STREAM_EVENTS = 4 + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_streaming_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if len(outcome.events) < MIN_STREAM_EVENTS: + error = ( + f"[{model}] only {len(outcome.events)} stream-json events observed " + f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/tool_use_streaming/test_vertex_ai.py b/tests/claude_code/tool_use_streaming/test_vertex_ai.py new file mode 100644 index 00000000000..ddaad111b18 --- /dev/null +++ b/tests/claude_code/tool_use_streaming/test_vertex_ai.py @@ -0,0 +1,125 @@ +"""tool_use_streaming x Vertex AI. + +Drive the real `claude` CLI in headless `--output-format stream-json` +mode against a running LiteLLM proxy that routes Claude requests to +GCP Vertex AI, ask Claude to invoke a built-in tool (`Bash`), and +assert that the upstream (a) emitted a `tool_use` content block and +(b) actually streamed events incrementally. + +Vertex AI exposes Anthropic models via `:streamRawPredict`; this cell +catches gateway regressions where the proxy buffers the response or +strips the streaming beta header on the way to Vertex. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/tool_use_streaming/test_vertex_ai.py + ^^^^^^^^^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +TOOL_USE_PROMPT = ( + "Use the Bash tool to run the command `echo pong` and report what it printed." +) +TOOL_USE_ARGS = ["--allowed-tools", "Bash"] + +MIN_STREAM_EVENTS = 4 + + +def _has_tool_use_event(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_use": + return True + return False + + +def test_tool_use_streaming_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=TOOL_USE_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=TOOL_USE_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_tool_use_event(outcome.events): + error = ( + f"[{model}] no tool_use content block observed in stream-json events" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if len(outcome.events) < MIN_STREAM_EVENTS: + error = ( + f"[{model}] only {len(outcome.events)} stream-json events observed " + f"(< {MIN_STREAM_EVENTS}); proxy likely buffered the response" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/vision/test_anthropic.py b/tests/claude_code/vision/test_anthropic.py index e0d178fb0ce..e7017b18000 100644 --- a/tests/claude_code/vision/test_anthropic.py +++ b/tests/claude_code/vision/test_anthropic.py @@ -21,7 +21,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -38,8 +42,7 @@ ANTHROPIC_MODELS = [ RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" -@pytest.mark.parametrize("model", ANTHROPIC_MODELS) -def test_vision_anthropic(compat_result, model, tmp_path): +def test_vision_anthropic(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with an image attached and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -61,39 +64,36 @@ def test_vision_anthropic(compat_result, model, tmp_path): image_path = tmp_path / "red_pixel.png" image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) - try: - result = run_claude( - prompt="What single color do you see in the attached image? Answer in one word.", - model=model, - base_url=base_url, - api_key=api_key, - extra_args=["--image", str(image_path)], - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt="What single color do you see in the attached image? Answer in one word.", + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text on a vision prompt", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/vision/test_azure.py b/tests/claude_code/vision/test_azure.py index f5e5c249036..832170bf7f5 100644 --- a/tests/claude_code/vision/test_azure.py +++ b/tests/claude_code/vision/test_azure.py @@ -25,7 +25,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -39,8 +43,7 @@ AZURE_MODELS = [ RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" -@pytest.mark.parametrize("model", AZURE_MODELS) -def test_vision_azure(compat_result, model, tmp_path): +def test_vision_azure(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with an image attached and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -62,39 +65,36 @@ def test_vision_azure(compat_result, model, tmp_path): image_path = tmp_path / "red_pixel.png" image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) - try: - result = run_claude( - prompt="What single color do you see in the attached image? Answer in one word.", - model=model, - base_url=base_url, - api_key=api_key, - extra_args=["--image", str(image_path)], - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt="What single color do you see in the attached image? Answer in one word.", + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text on a vision prompt", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/vision/test_bedrock_converse.py b/tests/claude_code/vision/test_bedrock_converse.py index 2bdfa1196fd..b002f49068a 100644 --- a/tests/claude_code/vision/test_bedrock_converse.py +++ b/tests/claude_code/vision/test_bedrock_converse.py @@ -20,7 +20,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -34,8 +38,7 @@ BEDROCK_CONVERSE_MODELS = [ RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" -@pytest.mark.parametrize("model", BEDROCK_CONVERSE_MODELS) -def test_vision_bedrock_converse(compat_result, model, tmp_path): +def test_vision_bedrock_converse(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with an image attached and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -57,39 +60,36 @@ def test_vision_bedrock_converse(compat_result, model, tmp_path): image_path = tmp_path / "red_pixel.png" image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) - try: - result = run_claude( - prompt="What single color do you see in the attached image? Answer in one word.", - model=model, - base_url=base_url, - api_key=api_key, - extra_args=["--image", str(image_path)], - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt="What single color do you see in the attached image? Answer in one word.", + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text on a vision prompt", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/vision/test_bedrock_invoke.py b/tests/claude_code/vision/test_bedrock_invoke.py index ec9a4287cc3..6000c53d2ab 100644 --- a/tests/claude_code/vision/test_bedrock_invoke.py +++ b/tests/claude_code/vision/test_bedrock_invoke.py @@ -20,7 +20,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -34,8 +38,7 @@ BEDROCK_INVOKE_MODELS = [ RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" -@pytest.mark.parametrize("model", BEDROCK_INVOKE_MODELS) -def test_vision_bedrock_invoke(compat_result, model, tmp_path): +def test_vision_bedrock_invoke(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with an image attached and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -57,39 +60,36 @@ def test_vision_bedrock_invoke(compat_result, model, tmp_path): image_path = tmp_path / "red_pixel.png" image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) - try: - result = run_claude( - prompt="What single color do you see in the attached image? Answer in one word.", - model=model, - base_url=base_url, - api_key=api_key, - extra_args=["--image", str(image_path)], - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt="What single color do you see in the attached image? Answer in one word.", + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text on a vision prompt", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/vision/test_vertex_ai.py b/tests/claude_code/vision/test_vertex_ai.py index 2739206dd08..8a24ca3213c 100644 --- a/tests/claude_code/vision/test_vertex_ai.py +++ b/tests/claude_code/vision/test_vertex_ai.py @@ -20,7 +20,11 @@ import os import pytest -from tests.claude_code.cli_driver import ClaudeCLIError, failure_diagnostic, run_claude +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" @@ -34,8 +38,7 @@ VERTEX_AI_MODELS = [ RED_PIXEL_PNG_B64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" -@pytest.mark.parametrize("model", VERTEX_AI_MODELS) -def test_vision_vertex_ai(compat_result, model, tmp_path): +def test_vision_vertex_ai(compat_result, tmp_path): """Drive the `claude` CLI against the LiteLLM proxy with an image attached and assert a non-empty reply.""" base_url = os.environ.get(PROXY_BASE_URL_ENV) @@ -57,39 +60,36 @@ def test_vision_vertex_ai(compat_result, model, tmp_path): image_path = tmp_path / "red_pixel.png" image_path.write_bytes(base64.b64decode(RED_PIXEL_PNG_B64)) - try: - result = run_claude( - prompt="What single color do you see in the attached image? Answer in one word.", - model=model, - base_url=base_url, - api_key=api_key, - extra_args=["--image", str(image_path)], - ) - except ClaudeCLIError as exc: - compat_result.set({"status": "fail", "error": f"[{model}] {exc}"}) - pytest.fail(str(exc), pytrace=False) - return + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt="What single color do you see in the attached image? Answer in one word.", + base_url=base_url, + api_key=api_key, + extra_args=["--image", str(image_path)], + ) - if result.exit_code != 0: - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude CLI failed: {failure_diagnostic(result)}", - } - ) - pytest.fail( - f"[{model}] claude CLI failed: {failure_diagnostic(result)}", pytrace=False - ) - return + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - if not result.text.strip(): - compat_result.set( - { - "status": "fail", - "error": f"[{model}] claude returned empty assistant text on a vision prompt", - } - ) - pytest.fail(f"empty reply for {model}", pytrace=False) - return + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue - compat_result.set({"status": "pass"}) + if not outcome.text.strip(): + error = f"[{model}] claude returned empty assistant text on a vision prompt" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/web_search/__init__.py b/tests/claude_code/web_search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/claude_code/web_search/test_anthropic.py b/tests/claude_code/web_search/test_anthropic.py new file mode 100644 index 00000000000..767fba88061 --- /dev/null +++ b/tests/claude_code/web_search/test_anthropic.py @@ -0,0 +1,142 @@ +"""web_search x Anthropic. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +to Anthropic, allow the built-in `WebSearch` tool, ask a question that +requires fresh web data, and assert that the upstream emitted a +`server_tool_use` or `web_search_tool_result` content block — proving +the proxy preserves Anthropic's server-side web search end-to-end. + +Web search is a *server tool*: unlike `Bash`/`Read`/etc., the upstream +executes the search itself and embeds the results inline in the +response. The wire shape is distinctive: + + - `server_tool_use` block with `name: "web_search"` + - `web_search_tool_result` block carrying the encrypted result content + +A regression where the proxy strips the `web_search_20250305` tool +from the request, drops the result block from the response, or fails +to forward the required beta header collapses both signals. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/web_search/test_anthropic.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +ANTHROPIC_MODELS = [ + "claude-haiku-4-5", + "claude-sonnet-4-6", + "claude-opus-4-7", +] + +# A prompt the model cannot answer from training data alone — it forces +# the model to actually hit the web_search server tool rather than +# replying from memory. We pick "this week" as the freshness anchor +# because it's stable across long-running test schedules without +# pinning to a specific date that would go stale. +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +# Allow only WebSearch so the model has no fallback path: if the proxy +# strips the server tool, the run will fail loudly rather than silently +# answering from training data via a different tool. +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] + +# Block types that prove the server tool actually executed end-to-end. +SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"} + + +def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool: + """Walk the stream-json events and return True if any assistant + message included a server-tool block (server_tool_use or + web_search_tool_result).""" + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in SERVER_TOOL_BLOCK_TYPES: + return True + return False + + +def test_web_search_anthropic(compat_result): + """Drive the `claude` CLI against the LiteLLM proxy and assert the + upstream emitted a server-tool block proving web search ran.""" + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=ANTHROPIC_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in ANTHROPIC_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_server_tool_block(outcome.events): + error = ( + f"[{model}] no server_tool_use / web_search_tool_result block " + "observed; the proxy may have stripped the WebSearch server tool " + "or its result block" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/web_search/test_azure.py b/tests/claude_code/web_search/test_azure.py new file mode 100644 index 00000000000..9992f9bda46 --- /dev/null +++ b/tests/claude_code/web_search/test_azure.py @@ -0,0 +1,115 @@ +"""web_search x Microsoft Foundry (Azure). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to Microsoft Foundry's Anthropic deployments on Azure, +allow the built-in `WebSearch` tool, ask a question that requires +fresh web data, and assert that the upstream emitted a +`server_tool_use` or `web_search_tool_result` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/web_search/test_azure.py + ^^^^^^^^^^ ^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +AZURE_MODELS = [ + "claude-haiku-4-5-azure", + "claude-sonnet-4-6-azure", + "claude-opus-4-7-azure", +] + +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] +SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"} + + +def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in SERVER_TOOL_BLOCK_TYPES: + return True + return False + + +def test_web_search_azure(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=AZURE_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in AZURE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_server_tool_block(outcome.events): + error = ( + f"[{model}] no server_tool_use / web_search_tool_result block " + "observed" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/web_search/test_bedrock_converse.py b/tests/claude_code/web_search/test_bedrock_converse.py new file mode 100644 index 00000000000..5a314099e4b --- /dev/null +++ b/tests/claude_code/web_search/test_bedrock_converse.py @@ -0,0 +1,115 @@ +"""web_search x Bedrock (Converse). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the `Converse` API, allow the +built-in `WebSearch` tool, ask a question that requires fresh web +data, and assert that the upstream emitted a `server_tool_use` or +`web_search_tool_result` content block. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/web_search/test_bedrock_converse.py + ^^^^^^^^^^ ^^^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_CONVERSE_MODELS = [ + "claude-haiku-4-5-bedrock-converse", + "claude-sonnet-4-6-bedrock-converse", + "claude-opus-4-7-bedrock-converse", +] + +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] +SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"} + + +def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in SERVER_TOOL_BLOCK_TYPES: + return True + return False + + +def test_web_search_bedrock_converse(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_CONVERSE_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in BEDROCK_CONVERSE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_server_tool_block(outcome.events): + error = ( + f"[{model}] no server_tool_use / web_search_tool_result block " + "observed" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/web_search/test_bedrock_invoke.py b/tests/claude_code/web_search/test_bedrock_invoke.py new file mode 100644 index 00000000000..f6910057959 --- /dev/null +++ b/tests/claude_code/web_search/test_bedrock_invoke.py @@ -0,0 +1,122 @@ +"""web_search x Bedrock (Invoke). + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to AWS Bedrock via the legacy `InvokeModel` API path, +allow the built-in `WebSearch` tool, ask a question that requires +fresh web data, and assert that the upstream emitted a +`server_tool_use` or `web_search_tool_result` content block. + +Bedrock support for the Anthropic-hosted web_search server tool has +been historically uneven — when it works, the wire shape is identical +to Anthropic's native API; when it doesn't, the upstream returns a +400 ("server tools not supported") that the proxy must surface +faithfully rather than silently dropping the tool from the request. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/web_search/test_bedrock_invoke.py + ^^^^^^^^^^ ^^^^^^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +BEDROCK_INVOKE_MODELS = [ + "claude-haiku-4-5-bedrock-invoke", + "claude-sonnet-4-6-bedrock-invoke", + "claude-opus-4-7-bedrock-invoke", +] + +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] +SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"} + + +def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in SERVER_TOOL_BLOCK_TYPES: + return True + return False + + +def test_web_search_bedrock_invoke(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=BEDROCK_INVOKE_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in BEDROCK_INVOKE_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_server_tool_block(outcome.events): + error = ( + f"[{model}] no server_tool_use / web_search_tool_result block " + "observed; Bedrock may not support the web_search server tool " + "for this model, or the proxy stripped it" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False) diff --git a/tests/claude_code/web_search/test_vertex_ai.py b/tests/claude_code/web_search/test_vertex_ai.py new file mode 100644 index 00000000000..1230b0c32dd --- /dev/null +++ b/tests/claude_code/web_search/test_vertex_ai.py @@ -0,0 +1,120 @@ +"""web_search x Vertex AI. + +Drive the real `claude` CLI against a running LiteLLM proxy that routes +Claude requests to GCP Vertex AI, allow the built-in `WebSearch` tool, +ask a question that requires fresh web data, and assert that the +upstream emitted a `server_tool_use` or `web_search_tool_result` +content block. + +Per the changelog (1.0.110, 2.1.79+), Vertex has had inconsistent +support for the Anthropic web_search server tool — the cell will fail +loudly when the upstream rejects the tool, which is the diagnostic we +want for a compat matrix. + +The (feature, provider) for this cell is inferred from the file path by +`tests/claude_code/conftest.py`: + + tests/claude_code/web_search/test_vertex_ai.py + ^^^^^^^^^^ ^^^^^^^^^ + feature_id provider +""" + +from __future__ import annotations + +import os +from typing import Any, Mapping, Sequence + +import pytest + +from tests.claude_code.cli_driver import ( + ClaudeCLIError, + failure_diagnostic, + run_claude_models_parallel, +) + +PROXY_BASE_URL_ENV = "LITELLM_PROXY_BASE_URL" +PROXY_API_KEY_ENV = "LITELLM_PROXY_API_KEY" + +VERTEX_AI_MODELS = [ + "claude-haiku-4-5-vertex", + "claude-sonnet-4-6-vertex", + "claude-opus-4-7-vertex", +] + +WEB_SEARCH_PROMPT = ( + "Use web search to find a news headline published this week about " + "Anthropic. Reply with one sentence summarizing what you found." +) +WEB_SEARCH_ARGS = ["--allowed-tools", "WebSearch"] +SERVER_TOOL_BLOCK_TYPES = {"server_tool_use", "web_search_tool_result"} + + +def _has_server_tool_block(events: Sequence[Mapping[str, Any]]) -> bool: + for event in events: + if event.get("type") != "assistant": + continue + message = event.get("message") or {} + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if not isinstance(block, dict): + continue + if block.get("type") in SERVER_TOOL_BLOCK_TYPES: + return True + return False + + +def test_web_search_vertex_ai(compat_result): + base_url = os.environ.get(PROXY_BASE_URL_ENV) + api_key = os.environ.get(PROXY_API_KEY_ENV) + if not base_url or not api_key: + compat_result.set( + { + "status": "fail", + "error": ( + f"missing required env: set {PROXY_BASE_URL_ENV} and " + f"{PROXY_API_KEY_ENV} to point at a running LiteLLM proxy" + ), + } + ) + pytest.fail( + f"{PROXY_BASE_URL_ENV} / {PROXY_API_KEY_ENV} not configured", pytrace=False + ) + + outcomes = run_claude_models_parallel( + models=VERTEX_AI_MODELS, + prompt=WEB_SEARCH_PROMPT, + base_url=base_url, + api_key=api_key, + extra_args=WEB_SEARCH_ARGS, + ) + + failures = [] + for model in VERTEX_AI_MODELS: + outcome = outcomes[model] + if isinstance(outcome, ClaudeCLIError): + error = f"[{model}] {outcome}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if outcome.exit_code != 0: + error = f"[{model}] claude CLI failed: {failure_diagnostic(outcome)}" + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + if not _has_server_tool_block(outcome.events): + error = ( + f"[{model}] no server_tool_use / web_search_tool_result block " + "observed" + ) + compat_result.add({"status": "fail", "error": error}) + failures.append(error) + continue + + compat_result.add({"status": "pass"}) + + if failures: + pytest.fail("; ".join(failures), pytrace=False)